This commit is contained in:
barnett 2018-11-14 23:09:20 +08:00
commit 41aff85b99
51 changed files with 4155 additions and 86 deletions

View File

@ -21,7 +21,6 @@ package option
import (
"fmt"
"github.com/Sirupsen/logrus"
"github.com/goodrain/rainbond/gateway/controller/openresty/model"
"github.com/spf13/pflag"
)
@ -36,10 +35,17 @@ func NewGWServer() *GWServer {
//Config contains all configuration
type Config struct {
Nginx model.Nginx
Http model.Http
K8SConfPath string
Namespace string
ListenPorts ListenPorts
}
// ListenPorts describe the ports required to run the gateway controller
type ListenPorts struct {
HTTP int
HTTPS int
Status int
AuxiliaryPort int
}
// AddFlags adds flags
@ -48,6 +54,7 @@ func (g *GWServer) AddFlags(fs *pflag.FlagSet) {
// TODO change kube-conf
fs.StringVar(&g.K8SConfPath, "kube-conf", "/Users/abe/Documents/admin.kubeconfig", "absolute path to the kubeconfig file")
fs.StringVar(&g.Namespace, "namespace", "gateway", "namespace")
fs.IntVar(&g.ListenPorts.AuxiliaryPort, "auxiliary-port", 10253, "port of auxiliary server")
}
// SetLog sets log

View File

@ -20,18 +20,38 @@ package server
import (
"fmt"
"github.com/Sirupsen/logrus"
"github.com/goodrain/rainbond/cmd/gateway/option"
"github.com/goodrain/rainbond/gateway/controller"
"os"
"os/signal"
"syscall"
)
//Run start run
func Run(s *option.GWServer) error {
gwc := controller.NewGWController()
errCh := make(chan error)
gwc := controller.NewGWController(
&s.Config,
errCh)
if gwc == nil {
return fmt.Errorf("fail to new GWController")
}
gwc.Start()
if err := gwc.Start(); err != nil {
return err
}
defer gwc.Stop()
term := make(chan os.Signal)
signal.Notify(term, os.Interrupt, syscall.SIGTERM)
select {
case <-term:
logrus.Warn("Received SIGTERM, exiting gracefully...")
case err := <-errCh:
logrus.Errorf("Received a error %s, exiting gracefully...", err.Error())
}
logrus.Info("See you next time!")
return nil
}

View File

@ -1,6 +1,7 @@
package controller
import (
"fmt"
"github.com/Sirupsen/logrus"
"github.com/eapache/channels"
"github.com/golang/glog"
@ -8,27 +9,36 @@ import (
"github.com/goodrain/rainbond/gateway/controller/openresty"
"github.com/goodrain/rainbond/gateway/store"
"github.com/goodrain/rainbond/gateway/v1"
"k8s.io/client-go/util/flowcontrol"
"k8s.io/ingress-nginx/task"
"sync"
"time"
)
const (
TRY_TIMES = 2
TryTimes = 2
)
type GWController struct {
GWS GWServicer
store store.Storer // TODO 为什么不能是*store.Storer
store store.Storer
syncQueue *task.Queue
syncRateLimiter flowcontrol.RateLimiter
isShuttingDown bool
// stopLock is used to enforce that only a single call to Stop send at
// a given time. We allow stopping through an HTTP endpoint and
// allowing concurrent stoppers leads to stack traces.
stopLock *sync.Mutex
optionConfig option.Config
RunningConfig *v1.Config
RunningHttpPools []*v1.Pool
stopCh chan struct{}
updateCh *channels.RingChannel
errCh chan error // errCh is used to detect errors with the NGINX processes
errCh chan error // TODO: never used
}
func (gwc *GWController) syncGateway(key interface{}) error {
@ -55,44 +65,43 @@ func (gwc *GWController) syncGateway(key interface{}) error {
gwc.RunningConfig = currentConfig
err := gwc.GWS.PersistConfig(gwc.RunningConfig)
// TODO: check if the nginx is ready.
if err != nil {
logrus.Errorf("Fail to persist Nginx config: %v\n", err)
} else {
// refresh http pools dynamically
gwc.refreshPools(httpPools)
gwc.RunningHttpPools = httpPools
if err != nil {
logrus.Errorf("Fail to persist Nginx config: %v\n", err)
}
return nil
}
func (gwc *GWController) Start() {
func (gwc *GWController) Start() error {
gwc.store.Run(gwc.stopCh)
gws := &openresty.OpenrestyService{}
err := gws.Start()
err := gwc.GWS.Start()
if err != nil {
logrus.Fatalf("Can not start gateway plugin: %v", err)
return
return fmt.Errorf("Can not start gateway plugin: %v", err)
}
go gwc.syncQueue.Run(1*time.Second, gwc.stopCh)
// force initial sync
gwc.syncQueue.EnqueueTask(task.GetDummyObject("initial-sync"))
go gwc.handleEvent()
return nil
}
func (gwc *GWController) handleEvent() {
for {
select {
case event := <-gwc.updateCh.Out(): // 将ringChannel的output通道接收到event放到task.Queue中
case event := <-gwc.updateCh.Out():
if gwc.isShuttingDown {
break
}
if evt, ok := event.(store.Event); ok {
logrus.Infof("Event %v received - object %v", evt.Type, evt.Obj)
if evt.Type == store.ConfigurationEvent {
// TODO: is this necessary? Consider removing this special case
gwc.syncQueue.EnqueueTask(task.GetDummyObject("configmap-change"))
continue
}
gwc.syncQueue.EnqueueSkippableTask(evt.Obj)
} else {
glog.Warningf("Unexpected event type received %T", event)
@ -103,16 +112,35 @@ func (gwc *GWController) Start() {
}
}
func (gwc *GWController) Stop() error {
gwc.isShuttingDown = true
gwc.stopLock.Lock()
defer gwc.stopLock.Unlock()
if gwc.syncQueue.IsShuttingDown() {
return fmt.Errorf("shutdown already in progress")
}
logrus.Infof("Shutting down controller queues")
close(gwc.stopCh) // stop the loop in *GWController#Start()
go gwc.syncQueue.Shutdown()
return gwc.GWS.Stop()
}
// refreshPools refresh pools dynamically.
func (gwc *GWController) refreshPools(pools []*v1.Pool) {
gwc.GWS.WaitPluginReady()
delPools, updPools := gwc.getDelUpdPools(pools)
for i := 0; i < TRY_TIMES; i++ {
for i := 0; i < TryTimes; i++ {
err := gwc.GWS.UpdatePools(updPools)
if err == nil {
break
}
}
for i := 0; i < TRY_TIMES; i++ {
for i := 0; i < TryTimes; i++ {
err := gwc.GWS.DeletePools(delPools)
if err == nil {
break
@ -142,23 +170,29 @@ func (gwc *GWController) getDelUpdPools(updPools []*v1.Pool) ([]*v1.Pool, []*v1.
return delPools, updPools
}
func NewGWController() *GWController {
func NewGWController(config *option.Config, errCh chan error) *GWController {
logrus.Debug("NewGWController...")
gwc := &GWController{
updateCh: channels.NewRingChannel(1024),
errCh: make(chan error),
errCh: errCh,
stopLock: &sync.Mutex{},
stopCh: make(chan struct{}),
}
gws := &openresty.OpenrestyService{}
gws := &openresty.OpenrestyService{
AuxiliaryPort: config.ListenPorts.AuxiliaryPort,
IsShuttingDown: &gwc.isShuttingDown,
}
gwc.GWS = gws
clientSet, err := NewClientSet("/Users/abe/Documents/admin.kubeconfig")
clientSet, err := NewClientSet(config.K8SConfPath)
if err != nil {
// TODO
logrus.Error("can't create kubernetes's client.")
}
gwc.store = store.New(clientSet,
"gateway",
gwc.store = store.New(
clientSet,
config.Namespace,
gwc.updateCh)
gwc.syncQueue = task.NewTaskQueue(gwc.syncGateway)

View File

@ -9,11 +9,23 @@ import (
"github.com/goodrain/rainbond/gateway/controller/openresty/template"
"github.com/goodrain/rainbond/gateway/v1"
"io/ioutil"
"k8s.io/ingress-nginx/ingress/controller/process"
"net/http"
"os"
"strings"
"sync"
"time"
)
type OpenrestyService struct{}
type OpenrestyService struct{
AuxiliaryPort int
IsShuttingDown *bool
// stopLock is used to enforce that only a single call to Stop send at
// a given time. We allow stopping through an HTTP endpoint and
// allowing concurrent stoppers leads to stack traces.
stopLock *sync.Mutex
}
type Upstream struct {
Name string
@ -26,10 +38,35 @@ type Server struct {
}
func (osvc *OpenrestyService) Start() error {
//o, err := nginxExecCommand().CombinedOutput()
//if err != nil {
// return fmt.Errorf("%v\n%v", err, string(o))
//}
o, err := nginxExecCommand().CombinedOutput()
if err != nil {
return fmt.Errorf("%v\n%v", err, string(o))
}
return nil
}
// Stop gracefully stops the NGINX master process.
func (osvc *OpenrestyService) Stop() error {
// send stop signal to NGINX
logrus.Info("Stopping NGINX process")
cmd := nginxExecCommand("-s", "quit")
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err := cmd.Run()
if err != nil {
return err
}
// wait for the NGINX process to terminate
timer := time.NewTicker(time.Second * 1)
for range timer.C {
if !process.IsNginxRunning() {
logrus.Info("NGINX process has stopped")
timer.Stop()
break
}
}
return nil
}
@ -100,9 +137,9 @@ func (osvc *OpenrestyService) PersistConfig(conf *v1.Config) error {
logrus.Debug("Nginx configuration is ok.")
// reload nginx
//if out, err := nginxExecCommand("-s", "reload").CombinedOutput(); err != nil {
// return fmt.Errorf("%v\n%v", err, string(out))
//}
if out, err := nginxExecCommand("-s", "reload").CombinedOutput(); err != nil {
return fmt.Errorf("%v\n%v", err, string(out))
}
logrus.Debug("Nginx reloads successfully.")
return nil
@ -171,7 +208,9 @@ func getNgxServer(conf *v1.Config) (l7srv []*model.Server, l4srv []*model.Server
// UpdatePools updates http upstreams dynamically.
func (osvc *OpenrestyService) UpdatePools(pools []*v1.Pool) error {
logrus.Debug("update http upstreams dynamically.")
if len(pools) == 0 {
return nil
}
var upstreams []*Upstream
for _, pool := range pools {
upstream := &Upstream{}
@ -185,16 +224,16 @@ func (osvc *OpenrestyService) UpdatePools(pools []*v1.Pool) error {
}
upstreams = append(upstreams, upstream)
}
return updateUpstreams(upstreams)
return osvc.updateUpstreams(upstreams)
}
// updateUpstreams updates the upstreams in ngx.shared.dict by post
func updateUpstreams(upstream []*Upstream) error {
url := "http://localhost:33333/update-upstreams" // TODO
json, _ := json.Marshal(upstream)
logrus.Debugf("request contest of update-upstreams is %v", string(json))
func (osvc *OpenrestyService) updateUpstreams(upstream []*Upstream) error {
url := fmt.Sprintf("http://127.0.0.1:%v/update-upstreams", osvc.AuxiliaryPort)
data, _ := json.Marshal(upstream)
logrus.Debugf("request contest of update-upstreams is %v", string(data))
resp, err := http.Post(url, "application/json", bytes.NewBuffer(json))
resp, err := http.Post(url, "application/json", bytes.NewBuffer(data))
if err != nil {
logrus.Errorf("fail to update upstreams: %v", err)
return err
@ -213,18 +252,21 @@ func updateUpstreams(upstream []*Upstream) error {
}
func (osvc *OpenrestyService) DeletePools(pools []*v1.Pool) error {
if len(pools) == 0 {
return nil
}
var data []string
for _, pool := range pools {
data = append(data, pool.Name)
}
return deletePools(data)
return osvc.deletePools(data)
}
func deletePools(data []string) error {
url := "http://localhost:33333/delete-upstreams" // TODO
json, _ := json.Marshal(data)
logrus.Errorf("request content of delete-upstreams is %v", string(json))
func (osvc *OpenrestyService) deletePools(names []string) error {
url := fmt.Sprintf("http://127.0.0.1:%v/delete-upstreams", osvc.AuxiliaryPort)
data, _ := json.Marshal(names)
logrus.Errorf("request content of delete-upstreams is %v", string(data))
resp, err := http.Post(url, "application/json", bytes.NewBuffer(json))
resp, err := http.Post(url, "application/json", bytes.NewBuffer(data))
if err != nil {
logrus.Errorf("fail to delete upstreams: %v", err)
return err
@ -234,3 +276,16 @@ func deletePools(data []string) error {
logrus.Debugf("the status of dynamically deleting upstreams is %v.", resp.Status)
return nil
}
// WaitPluginReady waits for nginx to be ready.
func (osvc *OpenrestyService) WaitPluginReady() {
url := fmt.Sprintf("http://127.0.0.1:%v/healthz", osvc.AuxiliaryPort)
for {
resp, err := http.Get(url)
if err == nil && resp.StatusCode == 200 {
logrus.Info("Nginx is ready")
break
}
time.Sleep(200 * time.Microsecond)
}
}

View File

@ -50,7 +50,7 @@ func NewTemplate(fileName string) (*Template, error) {
}
// TODO change the template name
tmpl, err := text_template.New("").Funcs(funcMap).Parse(string(tmplFile))
tmpl, err := text_template.New("gateway").Funcs(funcMap).Parse(string(tmplFile))
if err != nil {
return nil, err
}

View File

@ -9,4 +9,6 @@ type GWServicer interface {
PersistConfig(conf *v1.Config) error
UpdatePools(pools []*v1.Pool) error
DeletePools(pools []*v1.Pool) error
WaitPluginReady()
Stop() error
}

View File

@ -50,8 +50,31 @@ http {
lua_shared_dict upstreams_dict 16m;
upstream def_upstream {
server localhost:7777 max_fails=2 fail_timeout=30s;
server localhost:8888 max_fails=2 fail_timeout=30s;
}
server {
listen 33333;
listen 10254;
location /healthz {
return 200 "ok";
}
location /list-upstreams {
content_by_lua_block {
local balancer = require "ngx.balancer"
local cjson = require("cjson")
local keys = ngx.shared.upstreams_dict:get_keys()
for _, name in pairs(keys) do
local servers = ngx.shared.upstreams_dict:get(name)
ngx.print(name..": ")
ngx.print(cjson.encode(servers))
ngx.print("\n")
end
}
}
location /update-upstreams {
content_by_lua_block {
@ -76,16 +99,14 @@ http {
}
}
location /list-upstreams {
location /delete-upstreams {
content_by_lua_block {
local balancer = require "ngx.balancer"
local cjson = require("cjson")
local keys = ngx.shared.upstreams_dict:get_keys()
for _, name in pairs(keys) do
local servers = ngx.shared.upstreams_dict:get(name)
ngx.print(name..": ")
ngx.print(cjson.encode(servers))
ngx.print("\n")
ngx.req.read_body()
local data = ngx.req.get_body_data()
local tbl = cjson.decode(data)
for _, name in pairs(tbl) do
ngx.shared.upstreams_dict:delete(name)
end
}
}

View File

@ -300,7 +300,6 @@ func (s *rbdStore) extractAnnotations(ing *extensions.Ingress) {
}
}
// TODO test
func (s *rbdStore) ListPool() ([]*v1.Pool, []*v1.Pool) {
var httpPools []*v1.Pool
var tcpPools []*v1.Pool
@ -308,12 +307,12 @@ func (s *rbdStore) ListPool() ([]*v1.Pool, []*v1.Pool) {
endpoint := item.(*corev1.Endpoints)
pool := &v1.Pool{
Nodes: []v1.Node{},
Nodes: []*v1.Node{},
}
pool.Name = endpoint.ObjectMeta.Name
for _, ss := range endpoint.Subsets { // TODO 这个SubSets为什么是slice?
for _, ss := range endpoint.Subsets {
for _, address := range ss.Addresses {
pool.Nodes = append(pool.Nodes, v1.Node{ // TODO 需不需要用指针?
pool.Nodes = append(pool.Nodes, &v1.Node{
Host: address.IP,
Port: ss.Ports[0].Port,
})

View File

@ -23,14 +23,14 @@ type Node struct {
Meta
Host string `json:"host"`
Port int32 `json:"port"`
Protocol string `json:"protocol"` //TODO: 应该新建几个类型???
Protocol string `json:"protocol"`
State string `json:"state"` //Active Draining Disabled
PoolName string `json:"pool_name"` //Belong to the pool TODO: PoolName中能有空格吗???
PoolName string `json:"pool_name"` //Belong to the pool
Ready bool `json:"ready"` //Whether ready
Weight int `json:"weight"`
}
func (n *Node) Equals(c *Node) bool { // TODO 这个Equals方法可以抽象出去吗???
func (n *Node) Equals(c *Node) bool { //
if n == c {
return true
}

View File

@ -32,7 +32,7 @@ type Pool struct {
NodeNumber int `json:"node_number"`
LoadBalancingType LoadBalancingType `json:"load_balancing_type"`
Monitors []Monitor `json:"monitors"`
Nodes []Node
Nodes []*Node
}
func (p *Pool) Equals(c *Pool) bool {
@ -86,7 +86,7 @@ func (p *Pool) Equals(c *Pool) bool {
for _, a := range p.Nodes {
flag := false
for _, b := range c.Nodes {
if a.Equals(&b) {
if a.Equals(b) {
flag = true
}
}

View File

@ -26,9 +26,9 @@ func TestPool_Equals(t *testing.T) {
node2 := newFakeNode()
node2.Name = "node-b"
p := NewFakePoolWithoutNodes()
p.Nodes = []Node{
*node1,
*node2,
p.Nodes = []*Node{
node1,
node2,
}
node3 := newFakeNode()
@ -36,9 +36,9 @@ func TestPool_Equals(t *testing.T) {
node4 := newFakeNode()
node4.Name = "node-b"
c := NewFakePoolWithoutNodes()
c.Nodes = []Node {
*node3,
*node4,
c.Nodes = []*Node {
node3,
node4,
}
if !p.Equals(c) {

View File

@ -77,7 +77,7 @@ func (v *VirtualService) Equals(c *VirtualService) bool {
return false
}
// TODO
// TODO: this snippet needs improvement
if len(v.Listening) != len(c.Listening) {
return false
}
@ -101,7 +101,7 @@ func (v *VirtualService) Equals(c *VirtualService) bool {
return false
}
// TODO
// TODO: this snippet needs improvement
if len(v.RuleNames) != len(c.RuleNames) {
return false
}
@ -163,5 +163,9 @@ func (v *VirtualService) Equals(c *VirtualService) bool {
}
}
if !v.SSLCert.Equals(c.SSLCert) {
return false
}
return true
}

View File

@ -28,6 +28,7 @@ func TestVirtualService_Equals(t *testing.T) {
vlocB.PoolName = "pool-bbb"
v.Locations = append(v.Locations, vlocA)
v.Locations = append(v.Locations, vlocB)
v.SSLCert = newFakeSSLCert()
c := newFakeVirtualService()
clocA:= newFakeLocation()
@ -36,6 +37,8 @@ func TestVirtualService_Equals(t *testing.T) {
clocB.PoolName = "pool-bbb"
c.Locations = append(c.Locations, clocA)
c.Locations = append(c.Locations, clocB)
c.SSLCert = newFakeSSLCert()
if !v.Equals(c) {
t.Errorf("v should equal c")
@ -55,7 +58,6 @@ func newFakeVirtualService() *VirtualService {
RuleNames: []string{"a", "b", "c"},
SSLdecrypt: true,
DefaultCertificateName: "default-certificate-name",
CertificateMapping: map[string]string{"a": "aaa", "b": "bbb", "c": "ccc"},
RequestLogEnable: true,
RequestLogFileName: "/var/log/gateway/request.log",
RequestLogFormat: "request-log-format",
@ -63,5 +65,6 @@ func newFakeVirtualService() *VirtualService {
Timeout: 70,
ServerName:"foo-server_name",
PoolName: "foo-pool-name",
ForceSSLRedirect: true,
}
}

View File

@ -0,0 +1,95 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package process
import (
"fmt"
"net"
"os"
"os/exec"
"syscall"
"time"
"github.com/golang/glog"
ps "github.com/mitchellh/go-ps"
"github.com/ncabatoff/process-exporter/proc"
)
// IsRespawnIfRequired checks if error type is exec.ExitError or not
func IsRespawnIfRequired(err error) bool {
exitError, ok := err.(*exec.ExitError)
if !ok {
return false
}
waitStatus := exitError.Sys().(syscall.WaitStatus)
glog.Warningf(`
-------------------------------------------------------------------------------
NGINX master process died (%v): %v
-------------------------------------------------------------------------------
`, waitStatus.ExitStatus(), err)
return true
}
// WaitUntilPortIsAvailable waits until there is no NGINX master or worker
// process/es listening in a particular port.
func WaitUntilPortIsAvailable(port int) {
// we wait until the workers are killed
for {
conn, err := net.DialTimeout("tcp", fmt.Sprintf("0.0.0.0:%v", port), 1*time.Second)
if err != nil {
break
}
conn.Close()
// kill nginx worker processes
fs, err := proc.NewFS("/proc")
if err != nil {
glog.Errorf("unexpected error reading /proc information: %v", err)
continue
}
procs, _ := fs.FS.AllProcs()
for _, p := range procs {
pn, err := p.Comm()
if err != nil {
glog.Errorf("unexpected error obtaining process information: %v", err)
continue
}
if pn == "nginx" {
osp, err := os.FindProcess(p.PID)
if err != nil {
glog.Errorf("unexpected error obtaining process information: %v", err)
continue
}
osp.Signal(syscall.SIGQUIT)
}
}
time.Sleep(100 * time.Millisecond)
}
}
// IsNginxRunning returns true if a process with the name 'nginx' is found
func IsNginxRunning() bool {
processes, _ := ps.Processes()
for _, p := range processes {
if p.Executable() == "nginx" {
return true
}
}
return false
}

View File

@ -0,0 +1 @@
.vagrant/

View File

@ -0,0 +1,4 @@
language: go
go:
- 1.2.1

View File

@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2014 Mitchell Hashimoto
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

View File

@ -0,0 +1,34 @@
# Process List Library for Go
go-ps is a library for Go that implements OS-specific APIs to list and
manipulate processes in a platform-safe way. The library can find and
list processes on Linux, Mac OS X, Solaris, and Windows.
If you're new to Go, this library has a good amount of advanced Go educational
value as well. It uses some advanced features of Go: build tags, accessing
DLL methods for Windows, cgo for Darwin, etc.
How it works:
* **Darwin** uses the `sysctl` syscall to retrieve the process table.
* **Unix** uses the procfs at `/proc` to inspect the process tree.
* **Windows** uses the Windows API, and methods such as
`CreateToolhelp32Snapshot` to get a point-in-time snapshot of
the process table.
## Installation
Install using standard `go get`:
```
$ go get github.com/mitchellh/go-ps
...
```
## TODO
Want to contribute? Here is a short TODO list of things that aren't
implemented for this library that would be nice:
* FreeBSD support
* Plan9 support

View File

@ -0,0 +1,43 @@
# -*- mode: ruby -*-
# vi: set ft=ruby :
# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
VAGRANTFILE_API_VERSION = "2"
Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
config.vm.box = "chef/ubuntu-12.04"
config.vm.provision "shell", inline: $script
["vmware_fusion", "vmware_workstation"].each do |p|
config.vm.provider "p" do |v|
v.vmx["memsize"] = "1024"
v.vmx["numvcpus"] = "2"
v.vmx["cpuid.coresPerSocket"] = "1"
end
end
end
$script = <<SCRIPT
SRCROOT="/opt/go"
# Install Go
sudo apt-get update
sudo apt-get install -y build-essential mercurial
sudo hg clone -u release https://code.google.com/p/go ${SRCROOT}
cd ${SRCROOT}/src
sudo ./all.bash
# Setup the GOPATH
sudo mkdir -p /opt/gopath
cat <<EOF >/tmp/gopath.sh
export GOPATH="/opt/gopath"
export PATH="/opt/go/bin:\$GOPATH/bin:\$PATH"
EOF
sudo mv /tmp/gopath.sh /etc/profile.d/gopath.sh
sudo chmod 0755 /etc/profile.d/gopath.sh
# Make sure the gopath is usable by bamboo
sudo chown -R vagrant:vagrant $SRCROOT
sudo chown -R vagrant:vagrant /opt/gopath
SCRIPT

View File

@ -0,0 +1,40 @@
// ps provides an API for finding and listing processes in a platform-agnostic
// way.
//
// NOTE: If you're reading these docs online via GoDocs or some other system,
// you might only see the Unix docs. This project makes heavy use of
// platform-specific implementations. We recommend reading the source if you
// are interested.
package ps
// Process is the generic interface that is implemented on every platform
// and provides common operations for processes.
type Process interface {
// Pid is the process ID for this process.
Pid() int
// PPid is the parent process ID for this process.
PPid() int
// Executable name running this process. This is not a path to the
// executable.
Executable() string
}
// Processes returns all processes.
//
// This of course will be a point-in-time snapshot of when this method was
// called. Some operating systems don't provide snapshot capability of the
// process table, in which case the process table returned might contain
// ephemeral entities that happened to be running when this was called.
func Processes() ([]Process, error) {
return processes()
}
// FindProcess looks up a single process by pid.
//
// Process will be nil and error will be nil if a matching process is
// not found.
func FindProcess(pid int) (Process, error) {
return findProcess(pid)
}

View File

@ -0,0 +1,138 @@
// +build darwin
package ps
import (
"bytes"
"encoding/binary"
"syscall"
"unsafe"
)
type DarwinProcess struct {
pid int
ppid int
binary string
}
func (p *DarwinProcess) Pid() int {
return p.pid
}
func (p *DarwinProcess) PPid() int {
return p.ppid
}
func (p *DarwinProcess) Executable() string {
return p.binary
}
func findProcess(pid int) (Process, error) {
ps, err := processes()
if err != nil {
return nil, err
}
for _, p := range ps {
if p.Pid() == pid {
return p, nil
}
}
return nil, nil
}
func processes() ([]Process, error) {
buf, err := darwinSyscall()
if err != nil {
return nil, err
}
procs := make([]*kinfoProc, 0, 50)
k := 0
for i := _KINFO_STRUCT_SIZE; i < buf.Len(); i += _KINFO_STRUCT_SIZE {
proc := &kinfoProc{}
err = binary.Read(bytes.NewBuffer(buf.Bytes()[k:i]), binary.LittleEndian, proc)
if err != nil {
return nil, err
}
k = i
procs = append(procs, proc)
}
darwinProcs := make([]Process, len(procs))
for i, p := range procs {
darwinProcs[i] = &DarwinProcess{
pid: int(p.Pid),
ppid: int(p.PPid),
binary: darwinCstring(p.Comm),
}
}
return darwinProcs, nil
}
func darwinCstring(s [16]byte) string {
i := 0
for _, b := range s {
if b != 0 {
i++
} else {
break
}
}
return string(s[:i])
}
func darwinSyscall() (*bytes.Buffer, error) {
mib := [4]int32{_CTRL_KERN, _KERN_PROC, _KERN_PROC_ALL, 0}
size := uintptr(0)
_, _, errno := syscall.Syscall6(
syscall.SYS___SYSCTL,
uintptr(unsafe.Pointer(&mib[0])),
4,
0,
uintptr(unsafe.Pointer(&size)),
0,
0)
if errno != 0 {
return nil, errno
}
bs := make([]byte, size)
_, _, errno = syscall.Syscall6(
syscall.SYS___SYSCTL,
uintptr(unsafe.Pointer(&mib[0])),
4,
uintptr(unsafe.Pointer(&bs[0])),
uintptr(unsafe.Pointer(&size)),
0,
0)
if errno != 0 {
return nil, errno
}
return bytes.NewBuffer(bs[0:size]), nil
}
const (
_CTRL_KERN = 1
_KERN_PROC = 14
_KERN_PROC_ALL = 0
_KINFO_STRUCT_SIZE = 648
)
type kinfoProc struct {
_ [40]byte
Pid int32
_ [199]byte
Comm [16]byte
_ [301]byte
PPid int32
_ [84]byte
}

View File

@ -0,0 +1,260 @@
// +build freebsd,amd64
package ps
import (
"bytes"
"encoding/binary"
"syscall"
"unsafe"
)
// copied from sys/sysctl.h
const (
CTL_KERN = 1 // "high kernel": proc, limits
KERN_PROC = 14 // struct: process entries
KERN_PROC_PID = 1 // by process id
KERN_PROC_PROC = 8 // only return procs
KERN_PROC_PATHNAME = 12 // path to executable
)
// copied from sys/user.h
type Kinfo_proc struct {
Ki_structsize int32
Ki_layout int32
Ki_args int64
Ki_paddr int64
Ki_addr int64
Ki_tracep int64
Ki_textvp int64
Ki_fd int64
Ki_vmspace int64
Ki_wchan int64
Ki_pid int32
Ki_ppid int32
Ki_pgid int32
Ki_tpgid int32
Ki_sid int32
Ki_tsid int32
Ki_jobc [2]byte
Ki_spare_short1 [2]byte
Ki_tdev int32
Ki_siglist [16]byte
Ki_sigmask [16]byte
Ki_sigignore [16]byte
Ki_sigcatch [16]byte
Ki_uid int32
Ki_ruid int32
Ki_svuid int32
Ki_rgid int32
Ki_svgid int32
Ki_ngroups [2]byte
Ki_spare_short2 [2]byte
Ki_groups [64]byte
Ki_size int64
Ki_rssize int64
Ki_swrss int64
Ki_tsize int64
Ki_dsize int64
Ki_ssize int64
Ki_xstat [2]byte
Ki_acflag [2]byte
Ki_pctcpu int32
Ki_estcpu int32
Ki_slptime int32
Ki_swtime int32
Ki_cow int32
Ki_runtime int64
Ki_start [16]byte
Ki_childtime [16]byte
Ki_flag int64
Ki_kiflag int64
Ki_traceflag int32
Ki_stat [1]byte
Ki_nice [1]byte
Ki_lock [1]byte
Ki_rqindex [1]byte
Ki_oncpu [1]byte
Ki_lastcpu [1]byte
Ki_ocomm [17]byte
Ki_wmesg [9]byte
Ki_login [18]byte
Ki_lockname [9]byte
Ki_comm [20]byte
Ki_emul [17]byte
Ki_sparestrings [68]byte
Ki_spareints [36]byte
Ki_cr_flags int32
Ki_jid int32
Ki_numthreads int32
Ki_tid int32
Ki_pri int32
Ki_rusage [144]byte
Ki_rusage_ch [144]byte
Ki_pcb int64
Ki_kstack int64
Ki_udata int64
Ki_tdaddr int64
Ki_spareptrs [48]byte
Ki_spareint64s [96]byte
Ki_sflag int64
Ki_tdflags int64
}
// UnixProcess is an implementation of Process that contains Unix-specific
// fields and information.
type UnixProcess struct {
pid int
ppid int
state rune
pgrp int
sid int
binary string
}
func (p *UnixProcess) Pid() int {
return p.pid
}
func (p *UnixProcess) PPid() int {
return p.ppid
}
func (p *UnixProcess) Executable() string {
return p.binary
}
// Refresh reloads all the data associated with this process.
func (p *UnixProcess) Refresh() error {
mib := []int32{CTL_KERN, KERN_PROC, KERN_PROC_PID, int32(p.pid)}
buf, length, err := call_syscall(mib)
if err != nil {
return err
}
proc_k := Kinfo_proc{}
if length != uint64(unsafe.Sizeof(proc_k)) {
return err
}
k, err := parse_kinfo_proc(buf)
if err != nil {
return err
}
p.ppid, p.pgrp, p.sid, p.binary = copy_params(&k)
return nil
}
func copy_params(k *Kinfo_proc) (int, int, int, string) {
n := -1
for i, b := range k.Ki_comm {
if b == 0 {
break
}
n = i + 1
}
comm := string(k.Ki_comm[:n])
return int(k.Ki_ppid), int(k.Ki_pgid), int(k.Ki_sid), comm
}
func findProcess(pid int) (Process, error) {
mib := []int32{CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, int32(pid)}
_, _, err := call_syscall(mib)
if err != nil {
return nil, err
}
return newUnixProcess(pid)
}
func processes() ([]Process, error) {
results := make([]Process, 0, 50)
mib := []int32{CTL_KERN, KERN_PROC, KERN_PROC_PROC, 0}
buf, length, err := call_syscall(mib)
if err != nil {
return results, err
}
// get kinfo_proc size
k := Kinfo_proc{}
procinfo_len := int(unsafe.Sizeof(k))
count := int(length / uint64(procinfo_len))
// parse buf to procs
for i := 0; i < count; i++ {
b := buf[i*procinfo_len : i*procinfo_len+procinfo_len]
k, err := parse_kinfo_proc(b)
if err != nil {
continue
}
p, err := newUnixProcess(int(k.Ki_pid))
if err != nil {
continue
}
p.ppid, p.pgrp, p.sid, p.binary = copy_params(&k)
results = append(results, p)
}
return results, nil
}
func parse_kinfo_proc(buf []byte) (Kinfo_proc, error) {
var k Kinfo_proc
br := bytes.NewReader(buf)
err := binary.Read(br, binary.LittleEndian, &k)
if err != nil {
return k, err
}
return k, nil
}
func call_syscall(mib []int32) ([]byte, uint64, error) {
miblen := uint64(len(mib))
// get required buffer size
length := uint64(0)
_, _, err := syscall.RawSyscall6(
syscall.SYS___SYSCTL,
uintptr(unsafe.Pointer(&mib[0])),
uintptr(miblen),
0,
uintptr(unsafe.Pointer(&length)),
0,
0)
if err != 0 {
b := make([]byte, 0)
return b, length, err
}
if length == 0 {
b := make([]byte, 0)
return b, length, err
}
// get proc info itself
buf := make([]byte, length)
_, _, err = syscall.RawSyscall6(
syscall.SYS___SYSCTL,
uintptr(unsafe.Pointer(&mib[0])),
uintptr(miblen),
uintptr(unsafe.Pointer(&buf[0])),
uintptr(unsafe.Pointer(&length)),
0,
0)
if err != 0 {
return buf, length, err
}
return buf, length, nil
}
func newUnixProcess(pid int) (*UnixProcess, error) {
p := &UnixProcess{pid: pid}
return p, p.Refresh()
}

View File

@ -0,0 +1,35 @@
// +build linux
package ps
import (
"fmt"
"io/ioutil"
"strings"
)
// Refresh reloads all the data associated with this process.
func (p *UnixProcess) Refresh() error {
statPath := fmt.Sprintf("/proc/%d/stat", p.pid)
dataBytes, err := ioutil.ReadFile(statPath)
if err != nil {
return err
}
// First, parse out the image name
data := string(dataBytes)
binStart := strings.IndexRune(data, '(') + 1
binEnd := strings.IndexRune(data[binStart:], ')')
p.binary = data[binStart : binStart+binEnd]
// Move past the image name and start parsing the rest
data = data[binStart+binEnd+2:]
_, err = fmt.Sscanf(data,
"%c %d %d %d",
&p.state,
&p.ppid,
&p.pgrp,
&p.sid)
return err
}

View File

@ -0,0 +1,96 @@
// +build solaris
package ps
import (
"encoding/binary"
"fmt"
"os"
)
type ushort_t uint16
type id_t int32
type pid_t int32
type uid_t int32
type gid_t int32
type dev_t uint64
type size_t uint64
type uintptr_t uint64
type timestruc_t [16]byte
// This is copy from /usr/include/sys/procfs.h
type psinfo_t struct {
Pr_flag int32 /* process flags (DEPRECATED; do not use) */
Pr_nlwp int32 /* number of active lwps in the process */
Pr_pid pid_t /* unique process id */
Pr_ppid pid_t /* process id of parent */
Pr_pgid pid_t /* pid of process group leader */
Pr_sid pid_t /* session id */
Pr_uid uid_t /* real user id */
Pr_euid uid_t /* effective user id */
Pr_gid gid_t /* real group id */
Pr_egid gid_t /* effective group id */
Pr_addr uintptr_t /* address of process */
Pr_size size_t /* size of process image in Kbytes */
Pr_rssize size_t /* resident set size in Kbytes */
Pr_pad1 size_t
Pr_ttydev dev_t /* controlling tty device (or PRNODEV) */
// Guess this following 2 ushort_t values require a padding to properly
// align to the 64bit mark.
Pr_pctcpu ushort_t /* % of recent cpu time used by all lwps */
Pr_pctmem ushort_t /* % of system memory used by process */
Pr_pad64bit [4]byte
Pr_start timestruc_t /* process start time, from the epoch */
Pr_time timestruc_t /* usr+sys cpu time for this process */
Pr_ctime timestruc_t /* usr+sys cpu time for reaped children */
Pr_fname [16]byte /* name of execed file */
Pr_psargs [80]byte /* initial characters of arg list */
Pr_wstat int32 /* if zombie, the wait() status */
Pr_argc int32 /* initial argument count */
Pr_argv uintptr_t /* address of initial argument vector */
Pr_envp uintptr_t /* address of initial environment vector */
Pr_dmodel [1]byte /* data model of the process */
Pr_pad2 [3]byte
Pr_taskid id_t /* task id */
Pr_projid id_t /* project id */
Pr_nzomb int32 /* number of zombie lwps in the process */
Pr_poolid id_t /* pool id */
Pr_zoneid id_t /* zone id */
Pr_contract id_t /* process contract */
Pr_filler int32 /* reserved for future use */
Pr_lwp [128]byte /* information for representative lwp */
}
func (p *UnixProcess) Refresh() error {
var psinfo psinfo_t
path := fmt.Sprintf("/proc/%d/psinfo", p.pid)
fh, err := os.Open(path)
if err != nil {
return err
}
defer fh.Close()
err = binary.Read(fh, binary.LittleEndian, &psinfo)
if err != nil {
return err
}
p.ppid = int(psinfo.Pr_ppid)
p.binary = toString(psinfo.Pr_fname[:], 16)
return nil
}
func toString(array []byte, len int) string {
for i := 0; i < len; i++ {
if array[i] == 0 {
return string(array[:i])
}
}
return string(array[:])
}

View File

@ -0,0 +1,101 @@
// +build linux solaris
package ps
import (
"fmt"
"io"
"os"
"strconv"
)
// UnixProcess is an implementation of Process that contains Unix-specific
// fields and information.
type UnixProcess struct {
pid int
ppid int
state rune
pgrp int
sid int
binary string
}
func (p *UnixProcess) Pid() int {
return p.pid
}
func (p *UnixProcess) PPid() int {
return p.ppid
}
func (p *UnixProcess) Executable() string {
return p.binary
}
func findProcess(pid int) (Process, error) {
dir := fmt.Sprintf("/proc/%d", pid)
_, err := os.Stat(dir)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
return newUnixProcess(pid)
}
func processes() ([]Process, error) {
d, err := os.Open("/proc")
if err != nil {
return nil, err
}
defer d.Close()
results := make([]Process, 0, 50)
for {
fis, err := d.Readdir(10)
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
for _, fi := range fis {
// We only care about directories, since all pids are dirs
if !fi.IsDir() {
continue
}
// We only care if the name starts with a numeric
name := fi.Name()
if name[0] < '0' || name[0] > '9' {
continue
}
// From this point forward, any errors we just ignore, because
// it might simply be that the process doesn't exist anymore.
pid, err := strconv.ParseInt(name, 10, 0)
if err != nil {
continue
}
p, err := newUnixProcess(int(pid))
if err != nil {
continue
}
results = append(results, p)
}
}
return results, nil
}
func newUnixProcess(pid int) (*UnixProcess, error) {
p := &UnixProcess{pid: pid}
return p, p.Refresh()
}

View File

@ -0,0 +1,119 @@
// +build windows
package ps
import (
"fmt"
"syscall"
"unsafe"
)
// Windows API functions
var (
modKernel32 = syscall.NewLazyDLL("kernel32.dll")
procCloseHandle = modKernel32.NewProc("CloseHandle")
procCreateToolhelp32Snapshot = modKernel32.NewProc("CreateToolhelp32Snapshot")
procProcess32First = modKernel32.NewProc("Process32FirstW")
procProcess32Next = modKernel32.NewProc("Process32NextW")
)
// Some constants from the Windows API
const (
ERROR_NO_MORE_FILES = 0x12
MAX_PATH = 260
)
// PROCESSENTRY32 is the Windows API structure that contains a process's
// information.
type PROCESSENTRY32 struct {
Size uint32
CntUsage uint32
ProcessID uint32
DefaultHeapID uintptr
ModuleID uint32
CntThreads uint32
ParentProcessID uint32
PriorityClassBase int32
Flags uint32
ExeFile [MAX_PATH]uint16
}
// WindowsProcess is an implementation of Process for Windows.
type WindowsProcess struct {
pid int
ppid int
exe string
}
func (p *WindowsProcess) Pid() int {
return p.pid
}
func (p *WindowsProcess) PPid() int {
return p.ppid
}
func (p *WindowsProcess) Executable() string {
return p.exe
}
func newWindowsProcess(e *PROCESSENTRY32) *WindowsProcess {
// Find when the string ends for decoding
end := 0
for {
if e.ExeFile[end] == 0 {
break
}
end++
}
return &WindowsProcess{
pid: int(e.ProcessID),
ppid: int(e.ParentProcessID),
exe: syscall.UTF16ToString(e.ExeFile[:end]),
}
}
func findProcess(pid int) (Process, error) {
ps, err := processes()
if err != nil {
return nil, err
}
for _, p := range ps {
if p.Pid() == pid {
return p, nil
}
}
return nil, nil
}
func processes() ([]Process, error) {
handle, _, _ := procCreateToolhelp32Snapshot.Call(
0x00000002,
0)
if handle < 0 {
return nil, syscall.GetLastError()
}
defer procCloseHandle.Call(handle)
var entry PROCESSENTRY32
entry.Size = uint32(unsafe.Sizeof(entry))
ret, _, _ := procProcess32First.Call(handle, uintptr(unsafe.Pointer(&entry)))
if ret == 0 {
return nil, fmt.Errorf("Error retrieving process info.")
}
results := make([]Process, 0, 50)
for {
results = append(results, newWindowsProcess(&entry))
ret, _, _ := procProcess32Next.Call(handle, uintptr(unsafe.Pointer(&entry)))
if ret == 0 {
break
}
}
return results, nil
}

View File

@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2016 Mitchell Hashimoto
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

View File

@ -0,0 +1,65 @@
# hashstructure [![GoDoc](https://godoc.org/github.com/mitchellh/hashstructure?status.svg)](https://godoc.org/github.com/mitchellh/hashstructure)
hashstructure is a Go library for creating a unique hash value
for arbitrary values in Go.
This can be used to key values in a hash (for use in a map, set, etc.)
that are complex. The most common use case is comparing two values without
sending data across the network, caching values locally (de-dup), and so on.
## Features
* Hash any arbitrary Go value, including complex types.
* Tag a struct field to ignore it and not affect the hash value.
* Tag a slice type struct field to treat it as a set where ordering
doesn't affect the hash code but the field itself is still taken into
account to create the hash value.
* Optionally specify a custom hash function to optimize for speed, collision
avoidance for your data set, etc.
* Optionally hash the output of `.String()` on structs that implement fmt.Stringer,
allowing effective hashing of time.Time
## Installation
Standard `go get`:
```
$ go get github.com/mitchellh/hashstructure
```
## Usage & Example
For usage and examples see the [Godoc](http://godoc.org/github.com/mitchellh/hashstructure).
A quick code example is shown below:
```go
type ComplexStruct struct {
Name string
Age uint
Metadata map[string]interface{}
}
v := ComplexStruct{
Name: "mitchellh",
Age: 64,
Metadata: map[string]interface{}{
"car": true,
"location": "California",
"siblings": []string{"Bob", "John"},
},
}
hash, err := hashstructure.Hash(v, nil)
if err != nil {
panic(err)
}
fmt.Printf("%d", hash)
// Output:
// 2307517237273902113
```

View File

@ -0,0 +1 @@
module github.com/mitchellh/hashstructure

View File

@ -0,0 +1,358 @@
package hashstructure
import (
"encoding/binary"
"fmt"
"hash"
"hash/fnv"
"reflect"
)
// ErrNotStringer is returned when there's an error with hash:"string"
type ErrNotStringer struct {
Field string
}
// Error implements error for ErrNotStringer
func (ens *ErrNotStringer) Error() string {
return fmt.Sprintf("hashstructure: %s has hash:\"string\" set, but does not implement fmt.Stringer", ens.Field)
}
// HashOptions are options that are available for hashing.
type HashOptions struct {
// Hasher is the hash function to use. If this isn't set, it will
// default to FNV.
Hasher hash.Hash64
// TagName is the struct tag to look at when hashing the structure.
// By default this is "hash".
TagName string
// ZeroNil is flag determining if nil pointer should be treated equal
// to a zero value of pointed type. By default this is false.
ZeroNil bool
}
// Hash returns the hash value of an arbitrary value.
//
// If opts is nil, then default options will be used. See HashOptions
// for the default values. The same *HashOptions value cannot be used
// concurrently. None of the values within a *HashOptions struct are
// safe to read/write while hashing is being done.
//
// Notes on the value:
//
// * Unexported fields on structs are ignored and do not affect the
// hash value.
//
// * Adding an exported field to a struct with the zero value will change
// the hash value.
//
// For structs, the hashing can be controlled using tags. For example:
//
// struct {
// Name string
// UUID string `hash:"ignore"`
// }
//
// The available tag values are:
//
// * "ignore" or "-" - The field will be ignored and not affect the hash code.
//
// * "set" - The field will be treated as a set, where ordering doesn't
// affect the hash code. This only works for slices.
//
// * "string" - The field will be hashed as a string, only works when the
// field implements fmt.Stringer
//
func Hash(v interface{}, opts *HashOptions) (uint64, error) {
// Create default options
if opts == nil {
opts = &HashOptions{}
}
if opts.Hasher == nil {
opts.Hasher = fnv.New64()
}
if opts.TagName == "" {
opts.TagName = "hash"
}
// Reset the hash
opts.Hasher.Reset()
// Create our walker and walk the structure
w := &walker{
h: opts.Hasher,
tag: opts.TagName,
zeronil: opts.ZeroNil,
}
return w.visit(reflect.ValueOf(v), nil)
}
type walker struct {
h hash.Hash64
tag string
zeronil bool
}
type visitOpts struct {
// Flags are a bitmask of flags to affect behavior of this visit
Flags visitFlag
// Information about the struct containing this field
Struct interface{}
StructField string
}
func (w *walker) visit(v reflect.Value, opts *visitOpts) (uint64, error) {
t := reflect.TypeOf(0)
// Loop since these can be wrapped in multiple layers of pointers
// and interfaces.
for {
// If we have an interface, dereference it. We have to do this up
// here because it might be a nil in there and the check below must
// catch that.
if v.Kind() == reflect.Interface {
v = v.Elem()
continue
}
if v.Kind() == reflect.Ptr {
if w.zeronil {
t = v.Type().Elem()
}
v = reflect.Indirect(v)
continue
}
break
}
// If it is nil, treat it like a zero.
if !v.IsValid() {
v = reflect.Zero(t)
}
// Binary writing can use raw ints, we have to convert to
// a sized-int, we'll choose the largest...
switch v.Kind() {
case reflect.Int:
v = reflect.ValueOf(int64(v.Int()))
case reflect.Uint:
v = reflect.ValueOf(uint64(v.Uint()))
case reflect.Bool:
var tmp int8
if v.Bool() {
tmp = 1
}
v = reflect.ValueOf(tmp)
}
k := v.Kind()
// We can shortcut numeric values by directly binary writing them
if k >= reflect.Int && k <= reflect.Complex64 {
// A direct hash calculation
w.h.Reset()
err := binary.Write(w.h, binary.LittleEndian, v.Interface())
return w.h.Sum64(), err
}
switch k {
case reflect.Array:
var h uint64
l := v.Len()
for i := 0; i < l; i++ {
current, err := w.visit(v.Index(i), nil)
if err != nil {
return 0, err
}
h = hashUpdateOrdered(w.h, h, current)
}
return h, nil
case reflect.Map:
var includeMap IncludableMap
if opts != nil && opts.Struct != nil {
if v, ok := opts.Struct.(IncludableMap); ok {
includeMap = v
}
}
// Build the hash for the map. We do this by XOR-ing all the key
// and value hashes. This makes it deterministic despite ordering.
var h uint64
for _, k := range v.MapKeys() {
v := v.MapIndex(k)
if includeMap != nil {
incl, err := includeMap.HashIncludeMap(
opts.StructField, k.Interface(), v.Interface())
if err != nil {
return 0, err
}
if !incl {
continue
}
}
kh, err := w.visit(k, nil)
if err != nil {
return 0, err
}
vh, err := w.visit(v, nil)
if err != nil {
return 0, err
}
fieldHash := hashUpdateOrdered(w.h, kh, vh)
h = hashUpdateUnordered(h, fieldHash)
}
return h, nil
case reflect.Struct:
parent := v.Interface()
var include Includable
if impl, ok := parent.(Includable); ok {
include = impl
}
t := v.Type()
h, err := w.visit(reflect.ValueOf(t.Name()), nil)
if err != nil {
return 0, err
}
l := v.NumField()
for i := 0; i < l; i++ {
if innerV := v.Field(i); v.CanSet() || t.Field(i).Name != "_" {
var f visitFlag
fieldType := t.Field(i)
if fieldType.PkgPath != "" {
// Unexported
continue
}
tag := fieldType.Tag.Get(w.tag)
if tag == "ignore" || tag == "-" {
// Ignore this field
continue
}
// if string is set, use the string value
if tag == "string" {
if impl, ok := innerV.Interface().(fmt.Stringer); ok {
innerV = reflect.ValueOf(impl.String())
} else {
return 0, &ErrNotStringer{
Field: v.Type().Field(i).Name,
}
}
}
// Check if we implement includable and check it
if include != nil {
incl, err := include.HashInclude(fieldType.Name, innerV)
if err != nil {
return 0, err
}
if !incl {
continue
}
}
switch tag {
case "set":
f |= visitFlagSet
}
kh, err := w.visit(reflect.ValueOf(fieldType.Name), nil)
if err != nil {
return 0, err
}
vh, err := w.visit(innerV, &visitOpts{
Flags: f,
Struct: parent,
StructField: fieldType.Name,
})
if err != nil {
return 0, err
}
fieldHash := hashUpdateOrdered(w.h, kh, vh)
h = hashUpdateUnordered(h, fieldHash)
}
}
return h, nil
case reflect.Slice:
// We have two behaviors here. If it isn't a set, then we just
// visit all the elements. If it is a set, then we do a deterministic
// hash code.
var h uint64
var set bool
if opts != nil {
set = (opts.Flags & visitFlagSet) != 0
}
l := v.Len()
for i := 0; i < l; i++ {
current, err := w.visit(v.Index(i), nil)
if err != nil {
return 0, err
}
if set {
h = hashUpdateUnordered(h, current)
} else {
h = hashUpdateOrdered(w.h, h, current)
}
}
return h, nil
case reflect.String:
// Directly hash
w.h.Reset()
_, err := w.h.Write([]byte(v.String()))
return w.h.Sum64(), err
default:
return 0, fmt.Errorf("unknown kind to hash: %s", k)
}
}
func hashUpdateOrdered(h hash.Hash64, a, b uint64) uint64 {
// For ordered updates, use a real hash function
h.Reset()
// We just panic if the binary writes fail because we are writing
// an int64 which should never be fail-able.
e1 := binary.Write(h, binary.LittleEndian, a)
e2 := binary.Write(h, binary.LittleEndian, b)
if e1 != nil {
panic(e1)
}
if e2 != nil {
panic(e2)
}
return h.Sum64()
}
func hashUpdateUnordered(a, b uint64) uint64 {
return a ^ b
}
// visitFlag is used as a bitmask for affecting visit behavior
type visitFlag uint
const (
visitFlagInvalid visitFlag = iota
visitFlagSet = iota << 1
)

View File

@ -0,0 +1,15 @@
package hashstructure
// Includable is an interface that can optionally be implemented by
// a struct. It will be called for each field in the struct to check whether
// it should be included in the hash.
type Includable interface {
HashInclude(field string, v interface{}) (bool, error)
}
// IncludableMap is an interface that can optionally be implemented by
// a struct. It will be called when a map-type field is found to ask the
// struct if the map item should be included in the hash.
type IncludableMap interface {
HashIncludeMap(field string, k, v interface{}) (bool, error)
}

View File

@ -0,0 +1,8 @@
language: go
go:
- 1.9.x
- tip
script:
- go test

View File

@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2013 Mitchell Hashimoto
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

View File

@ -0,0 +1,46 @@
# mapstructure [![Godoc](https://godoc.org/github.com/mitchellh/mapstructure?status.svg)](https://godoc.org/github.com/mitchellh/mapstructure)
mapstructure is a Go library for decoding generic map values to structures
and vice versa, while providing helpful error handling.
This library is most useful when decoding values from some data stream (JSON,
Gob, etc.) where you don't _quite_ know the structure of the underlying data
until you read a part of it. You can therefore read a `map[string]interface{}`
and use this library to decode it into the proper underlying native Go
structure.
## Installation
Standard `go get`:
```
$ go get github.com/mitchellh/mapstructure
```
## Usage & Example
For usage and examples see the [Godoc](http://godoc.org/github.com/mitchellh/mapstructure).
The `Decode` function has examples associated with it there.
## But Why?!
Go offers fantastic standard libraries for decoding formats such as JSON.
The standard method is to have a struct pre-created, and populate that struct
from the bytes of the encoded format. This is great, but the problem is if
you have configuration or an encoding that changes slightly depending on
specific fields. For example, consider this JSON:
```json
{
"type": "person",
"name": "Mitchell"
}
```
Perhaps we can't populate a specific structure without first reading
the "type" field from the JSON. We could always do two passes over the
decoding of the JSON (reading the "type" first, and the rest later).
However, it is much simpler to just decode this into a `map[string]interface{}`
structure, read the "type" key, then use something like this library
to decode it into the proper structure.

View File

@ -0,0 +1,171 @@
package mapstructure
import (
"errors"
"reflect"
"strconv"
"strings"
"time"
)
// typedDecodeHook takes a raw DecodeHookFunc (an interface{}) and turns
// it into the proper DecodeHookFunc type, such as DecodeHookFuncType.
func typedDecodeHook(h DecodeHookFunc) DecodeHookFunc {
// Create variables here so we can reference them with the reflect pkg
var f1 DecodeHookFuncType
var f2 DecodeHookFuncKind
// Fill in the variables into this interface and the rest is done
// automatically using the reflect package.
potential := []interface{}{f1, f2}
v := reflect.ValueOf(h)
vt := v.Type()
for _, raw := range potential {
pt := reflect.ValueOf(raw).Type()
if vt.ConvertibleTo(pt) {
return v.Convert(pt).Interface()
}
}
return nil
}
// DecodeHookExec executes the given decode hook. This should be used
// since it'll naturally degrade to the older backwards compatible DecodeHookFunc
// that took reflect.Kind instead of reflect.Type.
func DecodeHookExec(
raw DecodeHookFunc,
from reflect.Type, to reflect.Type,
data interface{}) (interface{}, error) {
switch f := typedDecodeHook(raw).(type) {
case DecodeHookFuncType:
return f(from, to, data)
case DecodeHookFuncKind:
return f(from.Kind(), to.Kind(), data)
default:
return nil, errors.New("invalid decode hook signature")
}
}
// ComposeDecodeHookFunc creates a single DecodeHookFunc that
// automatically composes multiple DecodeHookFuncs.
//
// The composed funcs are called in order, with the result of the
// previous transformation.
func ComposeDecodeHookFunc(fs ...DecodeHookFunc) DecodeHookFunc {
return func(
f reflect.Type,
t reflect.Type,
data interface{}) (interface{}, error) {
var err error
for _, f1 := range fs {
data, err = DecodeHookExec(f1, f, t, data)
if err != nil {
return nil, err
}
// Modify the from kind to be correct with the new data
f = nil
if val := reflect.ValueOf(data); val.IsValid() {
f = val.Type()
}
}
return data, nil
}
}
// StringToSliceHookFunc returns a DecodeHookFunc that converts
// string to []string by splitting on the given sep.
func StringToSliceHookFunc(sep string) DecodeHookFunc {
return func(
f reflect.Kind,
t reflect.Kind,
data interface{}) (interface{}, error) {
if f != reflect.String || t != reflect.Slice {
return data, nil
}
raw := data.(string)
if raw == "" {
return []string{}, nil
}
return strings.Split(raw, sep), nil
}
}
// StringToTimeDurationHookFunc returns a DecodeHookFunc that converts
// strings to time.Duration.
func StringToTimeDurationHookFunc() DecodeHookFunc {
return func(
f reflect.Type,
t reflect.Type,
data interface{}) (interface{}, error) {
if f.Kind() != reflect.String {
return data, nil
}
if t != reflect.TypeOf(time.Duration(5)) {
return data, nil
}
// Convert it by parsing
return time.ParseDuration(data.(string))
}
}
// StringToTimeHookFunc returns a DecodeHookFunc that converts
// strings to time.Time.
func StringToTimeHookFunc(layout string) DecodeHookFunc {
return func(
f reflect.Type,
t reflect.Type,
data interface{}) (interface{}, error) {
if f.Kind() != reflect.String {
return data, nil
}
if t != reflect.TypeOf(time.Time{}) {
return data, nil
}
// Convert it by parsing
return time.Parse(layout, data.(string))
}
}
// WeaklyTypedHook is a DecodeHookFunc which adds support for weak typing to
// the decoder.
//
// Note that this is significantly different from the WeaklyTypedInput option
// of the DecoderConfig.
func WeaklyTypedHook(
f reflect.Kind,
t reflect.Kind,
data interface{}) (interface{}, error) {
dataVal := reflect.ValueOf(data)
switch t {
case reflect.String:
switch f {
case reflect.Bool:
if dataVal.Bool() {
return "1", nil
}
return "0", nil
case reflect.Float32:
return strconv.FormatFloat(dataVal.Float(), 'f', -1, 64), nil
case reflect.Int:
return strconv.FormatInt(dataVal.Int(), 10), nil
case reflect.Slice:
dataType := dataVal.Type()
elemKind := dataType.Elem().Kind()
if elemKind == reflect.Uint8 {
return string(dataVal.Interface().([]uint8)), nil
}
case reflect.Uint:
return strconv.FormatUint(dataVal.Uint(), 10), nil
}
}
return data, nil
}

View File

@ -0,0 +1,50 @@
package mapstructure
import (
"errors"
"fmt"
"sort"
"strings"
)
// Error implements the error interface and can represents multiple
// errors that occur in the course of a single decode.
type Error struct {
Errors []string
}
func (e *Error) Error() string {
points := make([]string, len(e.Errors))
for i, err := range e.Errors {
points[i] = fmt.Sprintf("* %s", err)
}
sort.Strings(points)
return fmt.Sprintf(
"%d error(s) decoding:\n\n%s",
len(e.Errors), strings.Join(points, "\n"))
}
// WrappedErrors implements the errwrap.Wrapper interface to make this
// return value more useful with the errwrap and go-multierror libraries.
func (e *Error) WrappedErrors() []error {
if e == nil {
return nil
}
result := make([]error, len(e.Errors))
for i, e := range e.Errors {
result[i] = errors.New(e)
}
return result
}
func appendErrors(errors []string, err error) []string {
switch e := err.(type) {
case *Error:
return append(errors, e.Errors...)
default:
return append(errors, e.Error())
}
}

View File

@ -0,0 +1 @@
module github.com/mitchellh/mapstructure

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,4 @@
.*.sw?
process-exporter
.tarballs
process-exporter-*.tar.gz

View File

@ -0,0 +1,35 @@
repository:
path: github.com/ncabatoff/process-exporter
build:
binaries:
- name: process-exporter
path: ./cmd/process-exporter
flags: -a -tags netgo
tarball:
files:
- LICENSE
crossbuild:
platforms:
- linux/amd64
- linux/386
- darwin/amd64
- darwin/386
- freebsd/amd64
- freebsd/386
- openbsd/amd64
- openbsd/386
- netbsd/amd64
- netbsd/386
- dragonfly/amd64
- linux/arm
- linux/arm64
- freebsd/arm
# Temporarily deactivated as golang.org/x/sys does not have syscalls
# implemented for that os/platform combination.
#- openbsd/arm
#- linux/mips64
#- linux/mips64le
- netbsd/arm
- linux/ppc64
- linux/ppc64le

View File

@ -0,0 +1,17 @@
# Start from a Debian image with the latest version of Go installed
# and a workspace (GOPATH) configured at /go.
FROM golang
# Copy the local package files to the container's workspace.
ADD . /go/src/github.com/ncabatoff/process-exporter
# Build the process-exporter command inside the container.
RUN make -C /go/src/github.com/ncabatoff/process-exporter
USER root
# Run the process-exporter command by default when the container starts.
ENTRYPOINT ["/go/src/github.com/ncabatoff/process-exporter/process-exporter"]
# Document that the service listens on port 9256.
EXPOSE 9256

View File

@ -0,0 +1,75 @@
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[[projects]]
branch = "master"
name = "github.com/beorn7/perks"
packages = ["quantile"]
revision = "4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9"
[[projects]]
branch = "master"
name = "github.com/golang/protobuf"
packages = ["proto"]
revision = "17ce1425424ab154092bbb43af630bd647f3bb0d"
[[projects]]
branch = "master"
name = "github.com/kylelemons/godebug"
packages = ["diff","pretty"]
revision = "d65d576e9348f5982d7f6d83682b694e731a45c6"
[[projects]]
name = "github.com/matttproud/golang_protobuf_extensions"
packages = ["pbutil"]
revision = "3247c84500bff8d9fb6d579d800f20b3e091582c"
version = "v1.0.0"
[[projects]]
branch = "master"
name = "github.com/ncabatoff/fakescraper"
packages = ["."]
revision = "15938421d91a82d197de7fc59aebcac65c43407d"
[[projects]]
name = "github.com/prometheus/client_golang"
packages = ["prometheus"]
revision = "c5b7fccd204277076155f10851dad72b76a49317"
version = "v0.8.0"
[[projects]]
branch = "master"
name = "github.com/prometheus/client_model"
packages = ["go"]
revision = "6f3806018612930941127f2a7c6c453ba2c527d2"
[[projects]]
branch = "master"
name = "github.com/prometheus/common"
packages = ["expfmt","internal/bitbucket.org/ww/goautoneg","model"]
revision = "2f17f4a9d485bf34b4bfaccc273805040e4f86c8"
[[projects]]
branch = "master"
name = "github.com/prometheus/procfs"
packages = [".","xfs"]
revision = "e645f4e5aaa8506fc71d6edbc5c4ff02c04c46f2"
[[projects]]
branch = "v1"
name = "gopkg.in/check.v1"
packages = ["."]
revision = "20d25e2804050c1cd24a7eea1e7a6447dd0e74ec"
[[projects]]
branch = "v2"
name = "gopkg.in/yaml.v2"
packages = ["."]
revision = "eb3733d160e74a9c7e442f435eb3bea458e1d19f"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "abd920f891c3e5fe2ee27ce40acbdde66e0799704d160b01f22530df003adfe1"
solver-name = "gps-cdcl"
solver-version = 1

View File

@ -0,0 +1,46 @@
# Gopkg.toml example
#
# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
# for detailed Gopkg.toml documentation.
#
# required = ["github.com/user/thing/cmd/thing"]
# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
#
# [[constraint]]
# name = "github.com/user/project"
# version = "1.0.0"
#
# [[constraint]]
# name = "github.com/user/project2"
# branch = "dev"
# source = "github.com/myfork/project2"
#
# [[override]]
# name = "github.com/x/y"
# version = "2.4.0"
[[constraint]]
branch = "master"
name = "github.com/kylelemons/godebug"
[[constraint]]
branch = "master"
name = "github.com/ncabatoff/fakescraper"
[[constraint]]
name = "github.com/prometheus/client_golang"
version = "0.8.0"
[[constraint]]
branch = "master"
name = "github.com/prometheus/procfs"
[[constraint]]
branch = "v1"
name = "gopkg.in/check.v1"
[[constraint]]
branch = "v2"
name = "gopkg.in/yaml.v2"

View File

@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2016 ncabatoff
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View File

@ -0,0 +1,71 @@
# Copyright 2015 The Prometheus Authors
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
GO := GO15VENDOREXPERIMENT=1 go
FIRST_GOPATH := $(firstword $(subst :, ,$(shell $(GO) env GOPATH)))
PROMU := $(FIRST_GOPATH)/bin/promu
pkgs = $(shell $(GO) list ./... | grep -v /vendor/)
PREFIX ?= $(shell pwd)
BIN_DIR ?= $(shell pwd)
DOCKER_IMAGE_NAME ?= process-exporter
DOCKER_IMAGE_TAG ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD))
ifdef DEBUG
bindata_flags = -debug
endif
all: format vet build test
style:
@echo ">> checking code style"
@! gofmt -d $(shell find . -path ./vendor -prune -o -name '*.go' -print) | grep '^'
test:
@echo ">> running short tests"
@$(GO) test -short $(pkgs)
format:
@echo ">> formatting code"
@$(GO) fmt $(pkgs)
vet:
@echo ">> vetting code"
@$(GO) vet $(pkgs)
build: promu
@echo ">> building binaries"
@$(PROMU) build --prefix $(PREFIX)
tarball: promu
@echo ">> building release tarball"
@$(PROMU) tarball --prefix $(PREFIX) $(BIN_DIR)
crossbuild: promu
@echo ">> cross-building"
@$(PROMU) crossbuild
@$(PROMU) crossbuild tarballs
docker:
@echo ">> building docker image"
@docker build -t "$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)" .
promu:
@echo ">> fetching promu"
@GOOS=$(shell uname -s | tr A-Z a-z) \
GOARCH=$(subst x86_64,amd64,$(patsubst i%86,386,$(patsubst arm%,arm,$(shell uname -m)))) \
$(GO) get -u github.com/prometheus/promu
.PHONY: all style format build test vet tarball crossbuild docker promu

View File

@ -0,0 +1,160 @@
# process-exporter
Prometheus exporter that mines /proc to report on selected processes.
The premise for this exporter is that sometimes you have apps that are
impractical to instrument directly, either because you don't control the code
or they're written in a language that isn't easy to instrument with Prometheus.
A fair bit of information can be gleaned from /proc, especially for
long-running programs.
For most systems it won't be beneficial to create metrics for every process by
name: there are just too many of them and most don't do enough to merit it.
Various command-line options are provided to control how processes are grouped
and the groups are named. Run "process-exporter -man" to see a help page
giving details.
Metrics available currently include CPU usage, bytes written and read, and
number of processes in each group.
Bytes read and written come from /proc/[pid]/io in recent enough kernels.
These correspond to the fields `read_bytes` and `write_bytes` respectively.
These IO stats come with plenty of caveats, see either the Linux kernel
documentation or man 5 proc.
CPU usage comes from /proc/[pid]/stat fields utime (user time) and stime (system
time.) It has been translated into fractional seconds of CPU consumed. Since
it is a counter, using rate() will tell you how many fractional cores were running
code from this process during the interval given.
An example Grafana dashboard to view the metrics is available at https://grafana.net/dashboards/249
## Instrumentation cost
process-exporter will consume CPU in proportion to the number of processes in
the system and the rate at which new ones are created. The most expensive
parts - applying regexps and executing templates - are only applied once per
process seen. If you have mostly long-running processes process-exporter
should be lightweight: each time a scrape occurs, parsing of /proc/$pid/stat
and /proc/$pid/cmdline for every process being monitored and adding a few
numbers.
## Config
To select and group the processes to monitor, either provide command-line
arguments or use a YAML configuration file.
To avoid confusion with the cmdline YAML element, we'll refer to the
null-delimited contents of `/proc/<pid>/cmdline` as the array `argv[]`.
Each item in `process_names` gives a recipe for identifying and naming
processes. The optional `name` tag defines a template to use to name
matching processes; if not specified, `name` defaults to `{{.ExeBase}}`.
Template variables available:
- `{{.ExeBase}}` contains the basename of the executable
- `{{.ExeFull}}` contains the fully qualified path of the executable
- `{{.Matches}}` map contains all the matches resulting from applying cmdline regexps
Each item in `process_names` must contain one or more selectors (`comm`, `exe`
or `cmdline`); if more than one selector is present, they must all match. Each
selector is a list of strings to match against a process's `comm`, `argv[0]`,
or in the case of `cmdline`, a regexp to apply to the command line.
For `comm` and `exe`, the list of strings is an OR, meaning any process
matching any of the strings will be added to the item's group.
For `cmdline`, the list of regexes is an AND, meaning they all must match. Any
capturing groups in a regexp must use the `?P<name>` option to assign a name to
the capture, which is used to populate `.Matches`.
A process may only belong to one group: even if multiple items would match, the
first one listed in the file wins.
Other performance tips: give an exe or comm clause in addition to any cmdline
clause, so you avoid executing the regexp when the executable name doesn't
match.
```
process_names:
# comm is the second field of /proc/<pid>/stat minus parens.
# It is the base executable name, truncated at 15 chars.
# It cannot be modified by the program, unlike exe.
- comm:
- bash
# exe is argv[0]. If no slashes, only basename of argv[0] need match.
# If exe contains slashes, argv[0] must match exactly.
- exe:
- postgres
- /usr/local/bin/prometheus
# cmdline is a list of regexps applied to argv.
# Each must match, and any captures are added to the .Matches map.
- name: "{{.ExeFull}}:{{.Matches.Cfgfile}}"
exe:
- /usr/local/bin/process-exporter
cmdline:
- -config.path\\s+(?P<Cfgfile>\\S+)
```
Here's the config I use on my home machine:
```
process_names:
- comm:
- chromium-browse
- bash
- prometheus
- gvim
- exe:
- /sbin/upstart
cmdline:
- --user
name: upstart:-user
```
## Docker
A docker image can be created with
```
make docker
```
Then run the docker, e.g.
```
docker run --privileged --name pexporter -d -v /proc:/host/proc -p 127.0.0.1:9256:9256 process-exporter:master -procfs /host/proc -procnames chromium-browse,bash,prometheus,gvim,upstart:-user -namemapping "upstart,(-user)"
```
This will expose metrics on http://localhost:9256/metrics. Leave off the
`127.0.0.1:` to publish on all interfaces. Leave off the --priviliged and
add the --user docker run argument if you only need to monitor processes
belonging to a single user.
## History
An earlier version of this exporter had options to enable auto-discovery of
which processes were consuming resources. This functionality has been removed.
These options were based on a percentage of resource usage, e.g. if an
untracked process consumed X% of CPU during a scrape, start tracking processes
with that name. However during any given scrape it's likely that most
processes are idle, so we could add a process that consumes minimal resources
but which happened to be active during the interval preceding the current
scrape. Over time this means that a great many processes wind up being
scraped, which becomes unmanageable to visualize. This could be mitigated by
looking at resource usage over longer intervals, but ultimately I didn't feel
this feature was important enough to invest more time in at this point. It may
re-appear at some point in the future, but no promises.
Another lost feature: the "other" group was used to count usage by non-tracked
procs. This was useful to get an idea of what wasn't being monitored. But it
comes at a high cost: if you know what processes you care about, you're wasting
a lot of CPU to compute the usage of everything else that you don't care about.
The new approach is to minimize resources expended on non-tracked processes and
to require the user to whitelist the processes to track.

View File

@ -0,0 +1 @@
0.1.0

View File

@ -0,0 +1,14 @@
package common
type (
NameAndCmdline struct {
Name string
Cmdline []string
}
MatchNamer interface {
// MatchAndName returns false if the match failed, otherwise
// true and the resulting name.
MatchAndName(NameAndCmdline) (bool, string)
}
)

View File

@ -0,0 +1,173 @@
package proc
import (
common "github.com/ncabatoff/process-exporter"
"time"
)
type (
Grouper struct {
namer common.MatchNamer
trackChildren bool
// track how much was seen last time so we can report the delta
GroupStats map[string]Counts
tracker *Tracker
}
GroupCountMap map[string]GroupCounts
GroupCounts struct {
Counts
Procs int
Memresident uint64
Memvirtual uint64
OldestStartTime time.Time
OpenFDs uint64
WorstFDratio float64
}
)
func NewGrouper(trackChildren bool, namer common.MatchNamer) *Grouper {
g := Grouper{
trackChildren: trackChildren,
namer: namer,
GroupStats: make(map[string]Counts),
tracker: NewTracker(),
}
return &g
}
func (g *Grouper) checkAncestry(idinfo ProcIdInfo, newprocs map[ProcId]ProcIdInfo) string {
ppid := idinfo.ParentPid
pProcId := g.tracker.ProcIds[ppid]
if pProcId.Pid < 1 {
// Reached root of process tree without finding a tracked parent.
g.tracker.Ignore(idinfo.ProcId)
return ""
}
// Is the parent already known to the tracker?
if ptproc, ok := g.tracker.Tracked[pProcId]; ok {
if ptproc != nil {
// We've found a tracked parent.
g.tracker.Track(ptproc.GroupName, idinfo)
return ptproc.GroupName
} else {
// We've found an untracked parent.
g.tracker.Ignore(idinfo.ProcId)
return ""
}
}
// Is the parent another new process?
if pinfoid, ok := newprocs[pProcId]; ok {
if name := g.checkAncestry(pinfoid, newprocs); name != "" {
// We've found a tracked parent, which implies this entire lineage should be tracked.
g.tracker.Track(name, idinfo)
return name
}
}
// Parent is dead, i.e. we never saw it, or there's no tracked proc in our ancestry.
g.tracker.Ignore(idinfo.ProcId)
return ""
}
// Update tracks any new procs that should be according to policy, and updates
// the metrics for already tracked procs. Permission errors are returned as a
// count, and will not affect the error return value.
func (g *Grouper) Update(iter ProcIter) (int, error) {
newProcs, permErrs, err := g.tracker.Update(iter)
if err != nil {
return permErrs, err
}
// Step 1: track any new proc that should be tracked based on its name and cmdline.
untracked := make(map[ProcId]ProcIdInfo)
for _, idinfo := range newProcs {
wanted, gname := g.namer.MatchAndName(common.NameAndCmdline{Name: idinfo.Name, Cmdline: idinfo.Cmdline})
if !wanted {
untracked[idinfo.ProcId] = idinfo
continue
}
g.tracker.Track(gname, idinfo)
}
// Step 2: track any untracked new proc that should be tracked because its parent is tracked.
if !g.trackChildren {
return permErrs, nil
}
for _, idinfo := range untracked {
if _, ok := g.tracker.Tracked[idinfo.ProcId]; ok {
// Already tracked or ignored
continue
}
g.checkAncestry(idinfo, untracked)
}
return permErrs, nil
}
// groups returns the aggregate metrics for all groups tracked. This reflects
// solely what's currently running.
func (g *Grouper) groups() GroupCountMap {
gcounts := make(GroupCountMap)
var zeroTime time.Time
for _, tinfo := range g.tracker.Tracked {
if tinfo == nil {
continue
}
cur := gcounts[tinfo.GroupName]
cur.Procs++
tstats := tinfo.GetStats()
cur.Memresident += tstats.Memory.Resident
cur.Memvirtual += tstats.Memory.Virtual
cur.OpenFDs += tstats.Filedesc.Open
openratio := float64(tstats.Filedesc.Open) / float64(tstats.Filedesc.Limit)
if cur.WorstFDratio < openratio {
cur.WorstFDratio = openratio
}
cur.Counts.Cpu += tstats.latest.Cpu
cur.Counts.ReadBytes += tstats.latest.ReadBytes
cur.Counts.WriteBytes += tstats.latest.WriteBytes
if cur.OldestStartTime == zeroTime || tstats.start.Before(cur.OldestStartTime) {
cur.OldestStartTime = tstats.start
}
gcounts[tinfo.GroupName] = cur
}
return gcounts
}
// Groups returns GroupCounts with Counts that never decrease in value from one
// call to the next. Even if processes exit, their CPU and IO contributions up
// to that point are included in the results. Even if no processes remain
// in a group it will still be included in the results.
func (g *Grouper) Groups() GroupCountMap {
groups := g.groups()
// First add any accumulated counts to what was just observed,
// and update the accumulators.
for gname, group := range groups {
if oldcounts, ok := g.GroupStats[gname]; ok {
group.Counts.Cpu += oldcounts.Cpu
group.Counts.ReadBytes += oldcounts.ReadBytes
group.Counts.WriteBytes += oldcounts.WriteBytes
}
g.GroupStats[gname] = group.Counts
groups[gname] = group
}
// Now add any groups that were observed in the past but aren't running now.
for gname, gcounts := range g.GroupStats {
if _, ok := groups[gname]; !ok {
groups[gname] = GroupCounts{Counts: gcounts}
}
}
return groups
}

View File

@ -0,0 +1,319 @@
package proc
import (
"fmt"
"time"
"github.com/prometheus/procfs"
)
func newProcIdStatic(pid, ppid int, startTime uint64, name string, cmdline []string) ProcIdStatic {
return ProcIdStatic{ProcId{pid, startTime}, ProcStatic{name, cmdline, ppid, time.Time{}}}
}
type (
// ProcId uniquely identifies a process.
ProcId struct {
// UNIX process id
Pid int
// The time the process started after system boot, the value is expressed
// in clock ticks.
StartTimeRel uint64
}
// ProcStatic contains data read from /proc/pid/*
ProcStatic struct {
Name string
Cmdline []string
ParentPid int
StartTime time.Time
}
// ProcMetrics contains data read from /proc/pid/*
ProcMetrics struct {
CpuTime float64
ReadBytes uint64
WriteBytes uint64
ResidentBytes uint64
VirtualBytes uint64
OpenFDs uint64
MaxFDs uint64
}
ProcIdStatic struct {
ProcId
ProcStatic
}
ProcInfo struct {
ProcStatic
ProcMetrics
}
ProcIdInfo struct {
ProcId
ProcStatic
ProcMetrics
}
// Proc wraps the details of the underlying procfs-reading library.
Proc interface {
// GetPid() returns the POSIX PID (process id). They may be reused over time.
GetPid() int
// GetProcId() returns (pid,starttime), which can be considered a unique process id.
// It may fail if the caller doesn't have permission to read /proc/<pid>/stat, or if
// the process has disapeared.
GetProcId() (ProcId, error)
// GetStatic() returns various details read from files under /proc/<pid>/. Technically
// name may not be static, but we'll pretend it is.
// It may fail if the caller doesn't have permission to read those files, or if
// the process has disapeared.
GetStatic() (ProcStatic, error)
// GetMetrics() returns various metrics read from files under /proc/<pid>/.
// It may fail if the caller doesn't have permission to read those files, or if
// the process has disapeared.
GetMetrics() (ProcMetrics, error)
}
// proc is a wrapper for procfs.Proc that caches results of some reads and implements Proc.
proc struct {
procfs.Proc
procid *ProcId
stat *procfs.ProcStat
cmdline []string
io *procfs.ProcIO
bootTime uint64
}
procs interface {
get(int) Proc
length() int
}
procfsprocs struct {
Procs []procfs.Proc
bootTime uint64
}
// ProcIter is an iterator over a sequence of procs.
ProcIter interface {
// Next returns true if the iterator is not exhausted.
Next() bool
// Close releases any resources the iterator uses.
Close() error
// The iterator satisfies the Proc interface.
Proc
}
// procIterator implements the ProcIter interface using procfs.
procIterator struct {
// procs is the list of Proc we're iterating over.
procs
// idx is the current iteration, i.e. it's an index into procs.
idx int
// err is set with an error when Next() fails. It is not affected by failures accessing
// the current iteration variable, e.g. with GetProcId.
err error
// Proc is the current iteration variable, or nil if Next() has never been called or the
// iterator is exhausted.
Proc
}
procIdInfos []ProcIdInfo
)
func procInfoIter(ps ...ProcIdInfo) ProcIter {
return &procIterator{procs: procIdInfos(ps), idx: -1}
}
func Info(p Proc) (ProcIdInfo, error) {
id, err := p.GetProcId()
if err != nil {
return ProcIdInfo{}, err
}
static, err := p.GetStatic()
if err != nil {
return ProcIdInfo{}, err
}
metrics, err := p.GetMetrics()
if err != nil {
return ProcIdInfo{}, err
}
return ProcIdInfo{id, static, metrics}, nil
}
func (p procIdInfos) get(i int) Proc {
return &p[i]
}
func (p procIdInfos) length() int {
return len(p)
}
func (p ProcIdInfo) GetPid() int {
return p.ProcId.Pid
}
func (p ProcIdInfo) GetProcId() (ProcId, error) {
return p.ProcId, nil
}
func (p ProcIdInfo) GetStatic() (ProcStatic, error) {
return p.ProcStatic, nil
}
func (p ProcIdInfo) GetMetrics() (ProcMetrics, error) {
return p.ProcMetrics, nil
}
func (p procfsprocs) get(i int) Proc {
return &proc{Proc: p.Procs[i], bootTime: p.bootTime}
}
func (p procfsprocs) length() int {
return len(p.Procs)
}
func (p *proc) GetPid() int {
return p.Proc.PID
}
func (p *proc) GetStat() (procfs.ProcStat, error) {
if p.stat == nil {
stat, err := p.Proc.NewStat()
if err != nil {
return procfs.ProcStat{}, err
}
p.stat = &stat
}
return *p.stat, nil
}
func (p *proc) GetProcId() (ProcId, error) {
if p.procid == nil {
stat, err := p.GetStat()
if err != nil {
return ProcId{}, err
}
p.procid = &ProcId{Pid: p.GetPid(), StartTimeRel: stat.Starttime}
}
return *p.procid, nil
}
func (p *proc) GetCmdLine() ([]string, error) {
if p.cmdline == nil {
cmdline, err := p.Proc.CmdLine()
if err != nil {
return nil, err
}
p.cmdline = cmdline
}
return p.cmdline, nil
}
func (p *proc) GetIo() (procfs.ProcIO, error) {
if p.io == nil {
io, err := p.Proc.NewIO()
if err != nil {
return procfs.ProcIO{}, err
}
p.io = &io
}
return *p.io, nil
}
func (p proc) GetStatic() (ProcStatic, error) {
cmdline, err := p.GetCmdLine()
if err != nil {
return ProcStatic{}, err
}
stat, err := p.GetStat()
if err != nil {
return ProcStatic{}, err
}
startTime := time.Unix(int64(p.bootTime), 0)
startTime = startTime.Add(time.Second / userHZ * time.Duration(stat.Starttime))
return ProcStatic{
Name: stat.Comm,
Cmdline: cmdline,
ParentPid: stat.PPID,
StartTime: startTime,
}, nil
}
func (p proc) GetMetrics() (ProcMetrics, error) {
io, err := p.GetIo()
if err != nil {
return ProcMetrics{}, err
}
stat, err := p.GetStat()
if err != nil {
return ProcMetrics{}, err
}
numfds, err := p.Proc.FileDescriptorsLen()
if err != nil {
return ProcMetrics{}, err
}
limits, err := p.NewLimits()
if err != nil {
return ProcMetrics{}, err
}
return ProcMetrics{
CpuTime: stat.CPUTime(),
ReadBytes: io.ReadBytes,
WriteBytes: io.WriteBytes,
ResidentBytes: uint64(stat.ResidentMemory()),
VirtualBytes: uint64(stat.VirtualMemory()),
OpenFDs: uint64(numfds),
MaxFDs: uint64(limits.OpenFiles),
}, nil
}
type FS struct {
procfs.FS
BootTime uint64
}
// See https://github.com/prometheus/procfs/blob/master/proc_stat.go for details on userHZ.
const userHZ = 100
// NewFS returns a new FS mounted under the given mountPoint. It will error
// if the mount point can't be read.
func NewFS(mountPoint string) (*FS, error) {
fs, err := procfs.NewFS(mountPoint)
if err != nil {
return nil, err
}
stat, err := fs.NewStat()
if err != nil {
return nil, err
}
return &FS{fs, stat.BootTime}, nil
}
func (fs *FS) AllProcs() ProcIter {
procs, err := fs.FS.AllProcs()
if err != nil {
err = fmt.Errorf("Error reading procs: %v", err)
}
return &procIterator{procs: procfsprocs{procs, fs.BootTime}, err: err, idx: -1}
}
func (pi *procIterator) Next() bool {
pi.idx++
if pi.idx < pi.procs.length() {
pi.Proc = pi.procs.get(pi.idx)
} else {
pi.Proc = nil
}
return pi.idx < pi.procs.length()
}
func (pi *procIterator) Close() error {
pi.Next()
pi.procs = nil
pi.Proc = nil
return pi.err
}

View File

@ -0,0 +1,180 @@
package proc
import (
"fmt"
"os"
"time"
)
type (
Counts struct {
Cpu float64
ReadBytes uint64
WriteBytes uint64
}
Memory struct {
Resident uint64
Virtual uint64
}
Filedesc struct {
Open uint64
Limit uint64
}
// Tracker tracks processes and records metrics.
Tracker struct {
// Tracked holds the processes are being monitored. Processes
// may be blacklisted such that they no longer get tracked by
// setting their value in the Tracked map to nil.
Tracked map[ProcId]*TrackedProc
// ProcIds is a map from pid to ProcId. This is a convenience
// to allow finding the Tracked entry of a parent process.
ProcIds map[int]ProcId
}
// TrackedProc accumulates metrics for a process, as well as
// remembering an optional GroupName tag associated with it.
TrackedProc struct {
// lastUpdate is used internally during the update cycle to find which procs have exited
lastUpdate time.Time
// info is the most recently obtained info for this proc
info ProcInfo
// accum is the total CPU and IO accrued since we started tracking this proc
accum Counts
// lastaccum is the CPU and IO accrued in the last Update()
lastaccum Counts
// GroupName is an optional tag for this proc.
GroupName string
}
trackedStats struct {
aggregate, latest Counts
Memory
Filedesc
start time.Time
}
)
func (tp *TrackedProc) GetName() string {
return tp.info.Name
}
func (tp *TrackedProc) GetCmdLine() []string {
return tp.info.Cmdline
}
func (tp *TrackedProc) GetStats() trackedStats {
mem := Memory{Resident: tp.info.ResidentBytes, Virtual: tp.info.VirtualBytes}
fd := Filedesc{Open: tp.info.OpenFDs, Limit: tp.info.MaxFDs}
return trackedStats{
aggregate: tp.accum,
latest: tp.lastaccum,
Memory: mem,
Filedesc: fd,
start: tp.info.StartTime,
}
}
func NewTracker() *Tracker {
return &Tracker{Tracked: make(map[ProcId]*TrackedProc), ProcIds: make(map[int]ProcId)}
}
func (t *Tracker) Track(groupName string, idinfo ProcIdInfo) {
info := ProcInfo{idinfo.ProcStatic, idinfo.ProcMetrics}
t.Tracked[idinfo.ProcId] = &TrackedProc{GroupName: groupName, info: info}
}
func (t *Tracker) Ignore(id ProcId) {
t.Tracked[id] = nil
}
// Scan procs and update metrics for those which are tracked. Processes that have gone
// away get removed from the Tracked map. New processes are returned, along with the count
// of permission errors.
func (t *Tracker) Update(procs ProcIter) ([]ProcIdInfo, int, error) {
now := time.Now()
var newProcs []ProcIdInfo
var permissionErrors int
for procs.Next() {
procId, err := procs.GetProcId()
if err != nil {
continue
}
last, known := t.Tracked[procId]
// Are we ignoring this proc?
if known && last == nil {
continue
}
// TODO if just the io file is unreadable, should we still return the other metrics?
metrics, err := procs.GetMetrics()
if err != nil {
if os.IsPermission(err) {
permissionErrors++
t.Ignore(procId)
}
continue
}
if known {
var newaccum, lastaccum Counts
dcpu := metrics.CpuTime - last.info.CpuTime
drbytes := metrics.ReadBytes - last.info.ReadBytes
dwbytes := metrics.WriteBytes - last.info.WriteBytes
lastaccum = Counts{Cpu: dcpu, ReadBytes: drbytes, WriteBytes: dwbytes}
newaccum = Counts{
Cpu: last.accum.Cpu + lastaccum.Cpu,
ReadBytes: last.accum.ReadBytes + lastaccum.ReadBytes,
WriteBytes: last.accum.WriteBytes + lastaccum.WriteBytes,
}
last.info.ProcMetrics = metrics
last.lastUpdate = now
last.accum = newaccum
last.lastaccum = lastaccum
} else {
static, err := procs.GetStatic()
if err != nil {
continue
}
newProcs = append(newProcs, ProcIdInfo{procId, static, metrics})
// Is this a new process with the same pid as one we already know?
if oldProcId, ok := t.ProcIds[procId.Pid]; ok {
// Delete it from known, otherwise the cleanup below will remove the
// ProcIds entry we're about to create
delete(t.Tracked, oldProcId)
}
t.ProcIds[procId.Pid] = procId
}
}
err := procs.Close()
if err != nil {
return nil, permissionErrors, fmt.Errorf("Error reading procs: %v", err)
}
// Rather than allocating a new map each time to detect procs that have
// disappeared, we bump the last update time on those that are still
// present. Then as a second pass we traverse the map looking for
// stale procs and removing them.
for procId, pinfo := range t.Tracked {
if pinfo == nil {
// TODO is this a bug? we're not tracking the proc so we don't see it go away so ProcIds
// and Tracked are leaking?
continue
}
if pinfo.lastUpdate != now {
delete(t.Tracked, procId)
delete(t.ProcIds, procId.Pid)
}
}
return newProcs, permissionErrors, nil
}