test/Rainbond
1
0
Fork 0
mirror of https://gitee.com/rainbond/Rainbond.git synced 2024-12-02 03:37:46 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[ADD] merge master branch

Browse Source
This commit is contained in:
goodrain 2018-02-05 13:40:08 +08:00
commit 807f9b86dc
114 changed files with 6289 additions and 559 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Makefile
View File

@ -94,9 +94,8 @@ run-mq:build-mq
run-worker:build-worker
CUR_NET=midonet EX_DOMAIN=test-ali.goodrain.net:10080 ${BIN_PATH}/${BASE_NAME}-worker \
--log-level=debug \
--db-type=cockroachdb \
--mysql="postgresql://root@localhost:26257/region" \
--kube-config=./admin.kubeconfig
--mysql="root:admin@tcp(127.0.0.1:3306)/region" \
--kube-config=./test/admin.kubeconfig
run-chaos:build-chaos
${BIN_PATH}/${BASE_NAME}-chaos
run-eventlog:build-eventlog
@ -112,6 +111,7 @@ run-node:build-node
--run-mode=master --kube-conf=`pwd`/test/admin.kubeconfig \
--nodeid-file=`pwd`/test/host_id.conf \
--static-task-path=`pwd`/test/tasks \
--statsd.mapping-config=`pwd`/test/mapper.yml \
--log-level=debug
doc: ## build the docs

19
cmd/README.md
View File

@ -0,0 +1,19 @@
# Component Description
## API
## BUILDER
## ENTRANCE
## EVENTLOG
## GRCTL
## MQ
## NODE
## WEBCLI
## WORKER

16
cmd/api/main.go
View File

@ -1,34 +1,38 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"github.com/goodrain/rainbond/cmd/api/option"
"github.com/goodrain/rainbond/cmd/api/server"
"fmt"
"os"
"github.com/goodrain/rainbond/cmd"
"github.com/goodrain/rainbond/cmd/api/option"
"github.com/goodrain/rainbond/cmd/api/server"
"github.com/spf13/pflag"
)
func main() {
if len(os.Args) > 1 && os.Args[1] == "version" {
cmd.ShowVersion("api")
}
s := option.NewAPIServer()
s.AddFlags(pflag.CommandLine)
pflag.Parse()

11
cmd/builder/builder.go
View File

@ -1,19 +1,18 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
@ -23,6 +22,7 @@ import (
"fmt"
"os"
"github.com/goodrain/rainbond/cmd"
"github.com/goodrain/rainbond/cmd/builder/option"
"github.com/goodrain/rainbond/cmd/builder/server"
@ -30,6 +30,9 @@ import (
)
func main() {
if len(os.Args) > 1 && os.Args[1] == "version" {
cmd.ShowVersion("chaos")
}
s := option.NewBuilder()
s.AddFlags(pflag.CommandLine)
pflag.Parse()

11
cmd/entrance/entrance.go
View File

@ -1,19 +1,18 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
@ -23,6 +22,7 @@ import (
"fmt"
"os"
"github.com/goodrain/rainbond/cmd"
"github.com/goodrain/rainbond/cmd/entrance/option"
"github.com/goodrain/rainbond/cmd/entrance/server"
@ -30,6 +30,9 @@ import (
)
func main() {
if len(os.Args) > 1 && os.Args[1] == "version" {
cmd.ShowVersion("entrance")
}
s := option.NewACPLBServer()
s.AddFlags(pflag.CommandLine)
pflag.Parse()

11
cmd/eventlog/eventlog.go
View File

@ -1,19 +1,18 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
@ -22,12 +21,16 @@ package main
import (
"os"
"github.com/goodrain/rainbond/cmd"
"github.com/goodrain/rainbond/cmd/eventlog/server"
"github.com/spf13/pflag"
)
func main() {
if len(os.Args) > 1 && os.Args[1] == "version" {
cmd.ShowVersion("eventlog")
}
s := server.NewLogServer()
s.AddFlags(pflag.CommandLine)
pflag.Parse()

12
cmd/eventlog/server/server.go
View File

@ -107,6 +107,8 @@ func (s *LogServer) AddFlags(fs *pflag.FlagSet) {
fs.StringVar(&s.Conf.EventStore.DB.HomePath, "docker.log.homepath", "/grdata/logs/", "container log persistent home path")
fs.StringVar(&s.Conf.WebHook.ConsoleURL, "webhook.console.url", "http://console.goodrain.me", "console web api url")
fs.StringVar(&s.Conf.WebHook.ConsoleToken, "webhook.console.token", "", "console web api token")
fs.StringVar(&s.Conf.Entry.NewMonitorMessageServerConf.ListenerHost, "monitor.udp.host", "0.0.0.0", "receive new monitor udp server host")
fs.IntVar(&s.Conf.Entry.NewMonitorMessageServerConf.ListenerPort, "monitor.udp.port", 6166, "receive new monitor udp server port")
}
//InitLog 初始化log
@ -227,6 +229,16 @@ func (s *LogServer) Run() error {
}
defer grpckeepalive.Stop()
udpkeepalive, err := discover.CreateKeepAlive(s.Conf.Cluster.Discover.EtcdAddr, "event_log_event_udp",
s.Conf.Cluster.Discover.InstanceIP, s.Conf.Cluster.Discover.InstanceIP, s.Conf.Entry.NewMonitorMessageServerConf.ListenerPort)
if err != nil {
return err
}
if err := udpkeepalive.Start(); err != nil {
return err
}
defer udpkeepalive.Stop()
httpkeepalive, err := discover.CreateKeepAlive(s.Conf.Cluster.Discover.EtcdAddr, "event_log_event_http",
s.Conf.Cluster.Discover.InstanceIP, s.Conf.Cluster.Discover.InstanceIP, s.Conf.WebSocket.BindPort)
if err != nil {

5
cmd/grctl/grctl.go
View File

@ -21,10 +21,15 @@ package main
import (
"fmt"
"os"
"github.com/goodrain/rainbond/cmd"
"github.com/goodrain/rainbond/cmd/grctl/server"
)
func main() {
if len(os.Args) > 1 && os.Args[1] == "version" {
cmd.ShowVersion("grctl")
}
if err := server.Run(); err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err)
os.Exit(1)

11
cmd/mq/mq.go
View File

@ -1,19 +1,18 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
@ -23,6 +22,7 @@ import (
"fmt"
"os"
"github.com/goodrain/rainbond/cmd"
"github.com/goodrain/rainbond/cmd/mq/option"
"github.com/goodrain/rainbond/cmd/mq/server"
@ -30,6 +30,9 @@ import (
)
func main() {
if len(os.Args) > 1 && os.Args[1] == "version" {
cmd.ShowVersion("mq")
}
s := option.NewMQServer()
s.AddFlags(pflag.CommandLine)
pflag.Parse()

4
cmd/node/node.go
View File

@ -22,11 +22,15 @@ import (
"fmt"
"os"
"github.com/goodrain/rainbond/cmd"
"github.com/goodrain/rainbond/cmd/node/option"
"github.com/goodrain/rainbond/cmd/node/server"
)
func main() {
if len(os.Args) > 1 && os.Args[1] == "version" {
cmd.ShowVersion("node")
}
option.Init()
if err := server.Run(option.Config); err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err)

24
cmd/node/option/conf.go
View File

@ -114,7 +114,24 @@ type Conf struct {
// 默认 300
LockTTL int64
Etcd client.Config
Etcd client.Config
StatsdConfig StatsdConfig
UDPMonitorConfig UDPMonitorConfig
}
//StatsdConfig StatsdConfig
type StatsdConfig struct {
StatsdListenAddress string
StatsdListenUDP string
StatsdListenTCP string
MappingConfig string
ReadBuffer int
}
//UDPMonitorConfig UDPMonitorConfig
type UDPMonitorConfig struct {
ListenHost string
ListenPort string
}
//AddFlags AddFlags
@ -153,6 +170,11 @@ func (a *Conf) AddFlags(fs *pflag.FlagSet) {
//fs.StringSliceVar(&a.EventServerAddress, "event-servers", []string{"http://127.0.0.1:6363"}, "event message server address.")
fs.StringVar(&a.DBType, "db-type", "mysql", "db type mysql or etcd")
fs.StringVar(&a.DBConnectionInfo, "mysql", "admin:admin@tcp(127.0.0.1:3306)/region", "mysql db connection info")
fs.StringVar(&a.StatsdConfig.StatsdListenAddress, "statsd.listen-address", "", "The UDP address on which to receive statsd metric lines. DEPRECATED, use statsd.listen-udp instead.")
fs.StringVar(&a.StatsdConfig.StatsdListenUDP, "statsd.listen-udp", ":9125", "The UDP address on which to receive statsd metric lines. \"\" disables it.")
fs.StringVar(&a.StatsdConfig.StatsdListenTCP, "statsd.listen-tcp", ":9125", "The TCP address on which to receive statsd metric lines. \"\" disables it.")
fs.StringVar(&a.StatsdConfig.MappingConfig, "statsd.mapping-config", "", "Metric mapping configuration file name.")
fs.IntVar(&a.StatsdConfig.ReadBuffer, "statsd.read-buffer", 0, "Size (in bytes) of the operating system's transmit read buffer associated with the UDP connection. Please make sure the kernel parameters net.core.rmem_max is set to a value greater than the value specified.")
}
//SetLog 设置log

93
cmd/node/server/server.go
View File

@ -20,28 +20,33 @@ package server
import (
"fmt"
"k8s.io/client-go/pkg/api/v1"
"github.com/goodrain/rainbond/cmd/node/option"
"github.com/goodrain/rainbond/pkg/node/api/controller"
"github.com/goodrain/rainbond/pkg/node/core/job"
"github.com/goodrain/rainbond/pkg/node/core/k8s"
"github.com/goodrain/rainbond/pkg/node/core/store"
"github.com/goodrain/rainbond/pkg/node/masterserver"
"github.com/goodrain/rainbond/pkg/node/monitormessage"
"github.com/goodrain/rainbond/pkg/node/nodeserver"
"github.com/goodrain/rainbond/pkg/node/statsd"
"github.com/prometheus/client_golang/prometheus"
"k8s.io/client-go/pkg/api/v1"
"github.com/Sirupsen/logrus"
eventLog "github.com/goodrain/rainbond/pkg/event"
"github.com/goodrain/rainbond/pkg/node/api"
"github.com/goodrain/rainbond/pkg/node/event"
"bytes"
"os/exec"
"encoding/json"
"io/ioutil"
"net/http"
"os/exec"
"strconv"
"strings"
"net/http"
"io/ioutil"
"github.com/goodrain/rainbond/pkg/node/api"
"github.com/goodrain/rainbond/pkg/node/event"
)
//Run start run
@ -53,6 +58,7 @@ func Run(c *option.Conf) error {
})
if err != nil {
logrus.Errorf("error creating eventlog manager")
return nil
}
defer eventLog.CloseManager()
@ -87,7 +93,7 @@ func Run(c *option.Conf) error {
logrus.Errorf(err.Error())
return err
}
if !s.HostNode.Role.HasRule("compute"){
if !s.HostNode.Role.HasRule("compute") {
getInfoForMaster(s)
}
ms.Cluster.UpdateNode(s.HostNode)
@ -97,8 +103,19 @@ func Run(c *option.Conf) error {
}
event.On(event.EXIT, ms.Stop)
}
//statsd exporter
registry := prometheus.NewRegistry()
exporter := statsd.CreateExporter(c.StatsdConfig, registry)
if err := exporter.Start(); err != nil {
logrus.Errorf("start statsd exporter server error,%s", err.Error())
return err
}
meserver := monitormessage.CreateUDPServer("0.0.0.0", 6666)
if err := meserver.Start(); err != nil {
return err
}
//启动API服务
apiManager := api.NewManager(*s.Conf, s.HostNode, ms)
apiManager := api.NewManager(*s.Conf, s.HostNode, ms, exporter)
if err := apiManager.Start(errChan); err != nil {
return err
}
@ -127,53 +144,53 @@ func getInfoForMaster(s *nodeserver.NodeServer) {
logrus.Errorf("error get response from sysinfo script,details %s", err.Error())
return
}
cmd := exec.Command("bash","-c", string(b))
cmd := exec.Command("bash", "-c", string(b))
//cmd := exec.Command("bash", "/usr/share/gr-rainbond-node/gaops/jobs/install/manage/tasks/ex_domain.sh")
outbuf:=bytes.NewBuffer(nil)
cmd.Stderr=outbuf
err=cmd.Run()
outbuf := bytes.NewBuffer(nil)
cmd.Stderr = outbuf
err = cmd.Run()
if err != nil {
logrus.Infof("err run command ,details %s",err.Error())
logrus.Infof("err run command ,details %s", err.Error())
return
}
result:=make(map[string]string)
result := make(map[string]string)
out:=outbuf.Bytes()
logrus.Infof("get system info is %s ",string(out))
err=json.Unmarshal(out,&result)
out := outbuf.Bytes()
logrus.Infof("get system info is %s ", string(out))
err = json.Unmarshal(out, &result)
if err != nil {
logrus.Infof("err unmarshal shell output ,details %s",err.Error())
logrus.Infof("err unmarshal shell output ,details %s", err.Error())
return
}
s.HostNode.NodeStatus=&v1.NodeStatus{
NodeInfo:v1.NodeSystemInfo{
KernelVersion:result["KERNEL"],
Architecture:result["PLATFORM"],
OperatingSystem:result["OS"],
KubeletVersion:"N/A",
s.HostNode.NodeStatus = &v1.NodeStatus{
NodeInfo: v1.NodeSystemInfo{
KernelVersion: result["KERNEL"],
Architecture: result["PLATFORM"],
OperatingSystem: result["OS"],
KubeletVersion: "N/A",
},
}
if cpuStr,ok:=result["LOGIC_CORES"];ok{
if cpu,err:=strconv.Atoi(cpuStr);err==nil{
logrus.Infof("server cpu is %v",cpu)
s.HostNode.AvailableCPU=int64(cpu)
if cpuStr, ok := result["LOGIC_CORES"]; ok {
if cpu, err := strconv.Atoi(cpuStr); err == nil {
logrus.Infof("server cpu is %v", cpu)
s.HostNode.AvailableCPU = int64(cpu)
s.HostNode.NodeStatus.Allocatable.Cpu().Set(int64(cpu))
}
}
if memStr,ok:=result["MEMORY"];ok{
memStr=strings.Replace(memStr," ","",-1)
memStr=strings.Replace(memStr,"G","",-1)
memStr=strings.Replace(memStr,"B","",-1)
if mem,err:=strconv.ParseFloat(memStr,64);err==nil{
s.HostNode.AvailableMemory=int64(mem*1024*1024*1024)
s.HostNode.NodeStatus.Allocatable.Memory().SetScaled(int64(mem*1024*1024*1024),0)
}else {
logrus.Warnf("get master memory info failed ,details %s",err.Error())
if memStr, ok := result["MEMORY"]; ok {
memStr = strings.Replace(memStr, " ", "", -1)
memStr = strings.Replace(memStr, "G", "", -1)
memStr = strings.Replace(memStr, "B", "", -1)
if mem, err := strconv.ParseFloat(memStr, 64); err == nil {
s.HostNode.AvailableMemory = int64(mem * 1024 * 1024 * 1024)
s.HostNode.NodeStatus.Allocatable.Memory().SetScaled(int64(mem*1024*1024*1024), 0)
} else {
logrus.Warnf("get master memory info failed ,details %s", err.Error())
}
}
logrus.Infof("memory is %v",s.HostNode.AvailableMemory)
logrus.Infof("memory is %v", s.HostNode.AvailableMemory)
s.Update()
}

33
cmd/version.go Normal file
View File

@ -0,0 +1,33 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"fmt"
"os"
)
//CodeVersion 代码版本
const CodeVersion = "0.0.0"
//ShowVersion 显示版本
func ShowVersion(module string) {
fmt.Printf("Rainbond %s %s\n", module, CodeVersion)
os.Exit(0)
}

4
cmd/webcli/webcli.go
View File

@ -22,6 +22,7 @@ import (
"fmt"
"os"
"github.com/goodrain/rainbond/cmd"
"github.com/goodrain/rainbond/cmd/webcli/option"
"github.com/goodrain/rainbond/cmd/webcli/server"
@ -29,6 +30,9 @@ import (
)
func main() {
if len(os.Args) > 1 && os.Args[1] == "version" {
cmd.ShowVersion("webcli")
}
s := option.NewWebCliServer()
s.AddFlags(pflag.CommandLine)
pflag.Parse()

2
cmd/worker/option/option.go
View File

@ -38,6 +38,7 @@ type Config struct {
MQAPI string
NodeName string
NodeAPI string
Listen string
}
//Worker worker server
@ -59,6 +60,7 @@ func (a *Worker) AddFlags(fs *pflag.FlagSet) {
fs.IntVar(&a.EtcdTimeout, "etcd-timeout", 5, "etcd http timeout seconds")
fs.StringVar(&a.EtcdPrefix, "etcd-prefix", "/store", "the etcd data save key prefix ")
fs.StringVar(&a.PrometheusMetricPath, "metric", "/metrics", "prometheus metrics path")
fs.StringVar(&a.Listen, "listen", ":6369", "prometheus listen host and port")
fs.StringVar(&a.DBType, "db-type", "mysql", "db type mysql or etcd")
fs.StringVar(&a.MysqlConnectionInfo, "mysql", "root:admin@tcp(127.0.0.1:3306)/region", "mysql db connection info")
fs.StringSliceVar(&a.EventLogServers, "event-servers", []string{"127.0.0.1:6367"}, "event log server address. simple lb")

22
cmd/worker/server/server.go
View File

@ -1,25 +1,28 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package server
import (
"os"
"os/signal"
"syscall"
"github.com/goodrain/rainbond/cmd/worker/option"
"github.com/goodrain/rainbond/pkg/db"
"github.com/goodrain/rainbond/pkg/db/config"
@ -28,9 +31,7 @@ import (
"github.com/goodrain/rainbond/pkg/worker/appm"
"github.com/goodrain/rainbond/pkg/worker/discover"
"github.com/goodrain/rainbond/pkg/worker/executor"
"os"
"os/signal"
"syscall"
"github.com/goodrain/rainbond/pkg/worker/monitor"
"github.com/Sirupsen/logrus"
)
@ -86,6 +87,13 @@ func Run(s *option.Worker) error {
}
defer taskManager.Stop()
//step 5 :create application use resource exporter.
exporterManager := monitor.NewManager(s.Config, statusManager)
if err := exporterManager.Start(); err != nil {
return err
}
defer exporterManager.Stop()
logrus.Info("worker begin running...")
//step finally: listen Signal
term := make(chan os.Signal)

11
cmd/worker/worker.go
View File

@ -1,19 +1,18 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
@ -23,6 +22,7 @@ import (
"fmt"
"os"
"github.com/goodrain/rainbond/cmd"
"github.com/goodrain/rainbond/cmd/worker/option"
"github.com/goodrain/rainbond/cmd/worker/server"
@ -30,6 +30,9 @@ import (
)
func main() {
if len(os.Args) > 1 && os.Args[1] == "version" {
cmd.ShowVersion("worker")
}
s := option.NewWorker()
s.AddFlags(pflag.CommandLine)
pflag.Parse()

44
hack/contrib/docker/api/html/swagger.json
View File

@ -1884,9 +1884,15 @@
"plugin_cpu",
"plugin_memory",
"build_version",
"info"
"info",
"tenant_id"
],
"properties": {
"build_image": {
"description": "镜像地址\nin: body",
"type": "string",
"x-go-name": "BuildImage"
},
"build_version": {
"description": "部署的版本号\nin: body",
"type": "string",
@ -1918,6 +1924,11 @@
"format": "int64",
"x-go-name": "PluginCPU"
},
"plugin_from": {
"description": "安装来源\nin: body",
"type": "string",
"x-go-name": "PluginFrom"
},
"plugin_memory": {
"description": "插件最大内存, 默认50\nin: body",
"type": "integer",
@ -6481,6 +6492,37 @@
}
}
}
},
"/v2/tenants/{tenant_name}/transplugins": {
"post": {
"description": "trans plugins",
"produces": [
"application/json",
"application/xml"
],
"tags": [
"v2"
],
"summary": "安装云帮默认plugins",
"operationId": "transPlugins",
"parameters": [
{
"type": "string",
"description": "tenant name",
"name": "tenant_name",
"in": "path",
"required": true
}
],
"responses": {
"default": {
"description": "统一返回格式",
"schema": {
"$ref": "#/responses/commandResponse"
}
}
}
}
}
},
"definitions": {

2
hack/contrib/docker/chaos/plugins/build_work.py
View File

@ -330,7 +330,7 @@ class RepoBuilder():
h = self.user_cs_client
try:
h.update_service(self.service_id, json.dumps(update_items))
self.region_client.update_service_region(self.service_id,json.dumps(update_items))
self.region_client.update_service_region(self.service_id, json.dumps(update_items))
except h.CallApiError, e:
self.log.error(
"网络异常，更新应用镜像名称失败. {}".format(e.message),

1
pkg/api/apiFunc/api_interface.go
View File

@ -35,6 +35,7 @@ type TenantInterface interface {
SumTenants(w http.ResponseWriter, r *http.Request)
SingleTenantResources(w http.ResponseWriter, r *http.Request)
GetSupportProtocols(w http.ResponseWriter, r *http.Request)
TransPlugins(w http.ResponseWriter, r *http.Request)
}
//ServiceInterface ServiceInterface

30
pkg/api/apiRouters/version1/v1Func.go
View File

@ -121,7 +121,9 @@ func (s *ServiceStruct) StartService(w http.ResponseWriter, r *http.Request) {
w.Write(controller.RestInfo(500, `"msg":"start falied"`))
return
}
_, err := s.MQClient.Enqueue(context.Background(), eq)
ctx, cancel := context.WithCancel(context.Background())
_, err := s.MQClient.Enqueue(ctx, eq)
cancel()
if err != nil {
logrus.Errorf("equque mq error, %v", err)
w.WriteHeader(500)
@ -178,7 +180,9 @@ func (s *ServiceStruct) StopService(w http.ResponseWriter, r *http.Request) {
w.Write(controller.RestInfo(500))
return
}
_, err = s.MQClient.Enqueue(context.Background(), eq)
ctx, cancel := context.WithCancel(context.Background())
_, err = s.MQClient.Enqueue(ctx, eq)
cancel()
if err != nil {
logrus.Errorf("equque mq error, %v", err)
w.WriteHeader(500)
@ -238,7 +242,9 @@ func (s *ServiceStruct) RestartService(w http.ResponseWriter, r *http.Request) {
w.Write(controller.RestInfo(500))
return
}
_, err = s.MQClient.Enqueue(context.Background(), eq)
ctx, cancel := context.WithCancel(context.Background())
_, err = s.MQClient.Enqueue(ctx, eq)
cancel()
if err != nil {
logrus.Errorf("equque mq error, %v", err)
w.WriteHeader(500)
@ -296,7 +302,9 @@ func (s *ServiceStruct) VerticalService(w http.ResponseWriter, r *http.Request)
w.Write(controller.RestInfo(500))
return
}
_, err = s.MQClient.Enqueue(context.Background(), eq)
ctx, cancel := context.WithCancel(context.Background())
_, err = s.MQClient.Enqueue(ctx, eq)
cancel()
if err != nil {
logrus.Errorf("equque mq error, %v", err)
w.WriteHeader(500)
@ -352,7 +360,9 @@ func (s *ServiceStruct) HorizontalService(w http.ResponseWriter, r *http.Request
w.Write(controller.RestInfo(500))
return
}
_, err = s.MQClient.Enqueue(context.Background(), eq)
ctx, cancel := context.WithCancel(context.Background())
_, err = s.MQClient.Enqueue(ctx, eq)
cancel()
if err != nil {
logrus.Errorf("equque mq error, %v", err)
w.WriteHeader(500)
@ -486,7 +496,9 @@ func (s *ServiceStruct) DeployService(w http.ResponseWriter, r *http.Request) {
w.Write(controller.RestInfo(500))
return
}
_, err = s.MQClient.Enqueue(context.Background(), eq)
ctx, cancel := context.WithCancel(context.Background())
_, err = s.MQClient.Enqueue(ctx, eq)
cancel()
if err != nil {
logrus.Errorf("equque mq error, %v", err)
w.WriteHeader(500)
@ -523,7 +535,7 @@ func (s *ServiceStruct) UpgradeService(w http.ResponseWriter, r *http.Request) {
w.Write(controller.RestInfo(500))
return
}
//两个deploy version
//两<EFBFBD><EFBFBD>deploy version
upgradeTask := model.RollingUpgradeTaskBody{
TenantID: services.TenantID,
ServiceID: services.ServiceID,
@ -543,7 +555,9 @@ func (s *ServiceStruct) UpgradeService(w http.ResponseWriter, r *http.Request) {
w.Write(controller.RestInfo(500))
return
}
_, err = s.MQClient.Enqueue(context.Background(), eq)
ctx, cancel := context.WithCancel(context.Background())
_, err = s.MQClient.Enqueue(ctx, eq)
cancel()
if err != nil {
logrus.Errorf("equque mq error, %v", err)
w.WriteHeader(500)

2
pkg/api/apiRouters/version2/v2Routers.go
View File

@ -61,6 +61,8 @@ func (v2 *V2) tenantNameRouter() chi.Router {
//租户中的日志
r.Post("/event-log", controller.GetManager().TenantLogByAction)
r.Get("/protocols", controller.GetManager().GetSupportProtocols)
//插件预安装
r.Post("/transplugins", controller.GetManager().TransPlugins)
//代码检测
r.Post("/code-check", controller.GetManager().CheckCode)
r.Post("/servicecheck", controller.Check)

8
pkg/api/apiRouters/websocket/websocket.go
View File

@ -1,19 +1,18 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
@ -31,6 +30,7 @@ func Routes() chi.Router {
r.Get("/docker_console", controller.GetDockerConsole().Get)
r.Get("/docker_log", controller.GetDockerLog().Get)
r.Get("/monitor_message", controller.GetMonitorMessage().Get)
r.Get("/new_monitor_message", controller.GetMonitorMessage().Get)
r.Get("/event_log", controller.GetEventLog().Get)
return r
}

6
pkg/api/controller/plugin.go
View File

@ -105,11 +105,12 @@ func (t *TenantStruct) UpdatePlugin(w http.ResponseWriter, r *http.Request) {
// description: 统一返回格式
pluginID := r.Context().Value(middleware.ContextKey("plugin_id")).(string)
tenantID := r.Context().Value(middleware.ContextKey("tenant_id")).(string)
var ups api_model.UpdatePluginStruct
if ok := httputil.ValidatorRequestStructAndErrorResponse(r, w, &ups.Body, nil); !ok {
return
}
if err := handler.GetPluginManager().UpdatePluginAct(pluginID, &ups); err != nil {
if err := handler.GetPluginManager().UpdatePluginAct(pluginID,tenantID, &ups); err != nil {
err.Handle(r, w)
return
}
@ -139,7 +140,8 @@ func (t *TenantStruct) DeletePlugin(w http.ResponseWriter, r *http.Request) {
// "$ref": "#/responses/commandResponse"
// description: 统一返回格式
pluginID := r.Context().Value(middleware.ContextKey("plugin_id")).(string)
if err := handler.GetPluginManager().DeletePluginAct(pluginID); err != nil {
tenantID := r.Context().Value(middleware.ContextKey("tenant_id")).(string)
if err := handler.GetPluginManager().DeletePluginAct(pluginID, tenantID); err != nil {
err.Handle(r, w)
return
}

42
pkg/api/controller/resources.go
View File

@ -1716,3 +1716,45 @@ func (t *TenantStruct) GetSupportProtocols(w http.ResponseWriter, r *http.Reques
httputil.ReturnSuccess(r, w, rps)
return
}
//TransPlugins transPlugins
// swagger:operation POST /v2/tenants/{tenant_name}/transplugins v2 transPlugins
//
// 安装云帮默认plugins
//
// trans plugins
//
// ---
// produces:
// - application/json
// - application/xml
// parameters:
// - name: tenant_name
// in: path
// description: tenant name
// required: true
// type: string
//
// responses:
// default:
// schema:
// "$ref": "#/responses/commandResponse"
// description: 统一返回格式
func (t *TenantStruct) TransPlugins(w http.ResponseWriter, r *http.Request) {
var tps api_model.TransPlugins
ok := httputil.ValidatorRequestStructAndErrorResponse(r, w, &tps.Body, nil)
if !ok {
return
}
tenantID := r.Context().Value(middleware.ContextKey("tenant_id")).(string)
tenantName := r.Context().Value(middleware.ContextKey("tenant_name")).(string)
rc := make(map[string]string)
err := handler.GetTenantManager().TransPlugins(tenantID, tenantName, tps.Body.FromTenantName, tps.Body.PluginsID)
if err != nil {
err.Handle(r, w)
return
}
rc["result"] = "success"
httputil.ReturnSuccess(r, w, rc)
return
}

4
pkg/api/handler/PluginHandler.go
View File

@ -28,8 +28,8 @@ import (
//PluginHandler plugin handler
type PluginHandler interface {
CreatePluginAct(cps *api_model.CreatePluginStruct) *util.APIHandleError
UpdatePluginAct(pluginID string, cps *api_model.UpdatePluginStruct) *util.APIHandleError
DeletePluginAct(pluginID string) *util.APIHandleError
UpdatePluginAct(pluginID,tenantID string, cps *api_model.UpdatePluginStruct) *util.APIHandleError
DeletePluginAct(pluginID, tenantID string) *util.APIHandleError
GetPlugins(tenantID string) ([]*dbmodel.TenantPlugin, *util.APIHandleError)
AddDefaultEnv(est *api_model.ENVStruct) *util.APIHandleError
UpdateDefaultEnv(est *api_model.ENVStruct) *util.APIHandleError

3
pkg/api/handler/TenantHandler.go
View File

@ -36,9 +36,10 @@ type TenantHandler interface {
TotalMemCPU(services []*dbmodel.TenantServices) (*api_model.StatsInfo, error)
//QueryTsdb(md *api_model.MontiorData) (*tsdbClient.QueryResponse, error)
HTTPTsdb(md *api_model.MontiorData) ([]byte, error)
GetTenantsResources(tr *api_model.TenantResources) ([]*map[string]interface{}, error)
GetTenantsResources(tr *api_model.TenantResources) ([]map[string]interface{}, error)
TenantsSum() (int, error)
GetProtocols() ([]*dbmodel.RegionProcotols, *util.APIHandleError)
TransPlugins(tenantID, tenantName, fromTenant string, pluginList []string) *util.APIHandleError
}
var defaultTenantHandler TenantHandler

227
pkg/api/handler/plugin.go
View File

@ -26,6 +26,8 @@ import (
"strings"
"time"
"github.com/jinzhu/gorm"
"github.com/goodrain/rainbond/cmd/api/option"
api_db "github.com/goodrain/rainbond/pkg/api/db"
api_model "github.com/goodrain/rainbond/pkg/api/model"
@ -83,8 +85,8 @@ func (p *PluginAction) CreatePluginAct(cps *api_model.CreatePluginStruct) *util.
}
//UpdatePluginAct UpdatePluginAct
func (p *PluginAction) UpdatePluginAct(pluginID string, cps *api_model.UpdatePluginStruct) *util.APIHandleError {
tp, err := db.GetManager().TenantPluginDao().GetPluginByID(pluginID)
func (p *PluginAction) UpdatePluginAct(pluginID, tenantID string, cps *api_model.UpdatePluginStruct) *util.APIHandleError {
tp, err := db.GetManager().TenantPluginDao().GetPluginByID(pluginID, tenantID)
if err != nil {
return util.CreateAPIHandleErrorFromDBError("get old plugin info", err)
}
@ -104,10 +106,10 @@ func (p *PluginAction) UpdatePluginAct(pluginID string, cps *api_model.UpdatePlu
}
//DeletePluginAct DeletePluginAct
func (p *PluginAction) DeletePluginAct(pluginID string) *util.APIHandleError {
func (p *PluginAction) DeletePluginAct(pluginID, tenantID string) *util.APIHandleError {
//TODO: 事务
tx := db.GetManager().Begin()
err := db.GetManager().TenantPluginDaoTransactions(tx).DeletePluginByID(pluginID)
err := db.GetManager().TenantPluginDaoTransactions(tx).DeletePluginByID(pluginID, tenantID)
if err != nil {
tx.Rollback()
return util.CreateAPIHandleErrorFromDBError("delete plugin", err)
@ -212,10 +214,24 @@ func (p *PluginAction) BuildPluginManual(bps *api_model.BuildPluginStruct) (*dbm
eventID := bps.Body.EventID
logger := event.GetManager().GetLogger(eventID)
defer event.CloseManager()
plugin, err := db.GetManager().TenantPluginDao().GetPluginByID(bps.PluginID)
plugin, err := db.GetManager().TenantPluginDao().GetPluginByID(bps.PluginID, bps.Body.TenantID)
if err != nil {
return nil, util.CreateAPIHandleErrorFromDBError(fmt.Sprintf("get plugin by %v", bps.PluginID), err)
}
if bps.Body.PluginFrom != "" {
switch bps.Body.PluginFrom {
case "yb":
pbv, err := p.InstallPluginFromYB(bps, plugin)
if err != nil {
logrus.Debugf("install plugin from yb error %s", err.Error())
return nil, util.CreateAPIHandleError(500, fmt.Errorf("install plugin from yb error"))
}
return pbv, nil
case "ys":
default:
return nil, util.CreateAPIHandleError(400, fmt.Errorf("unexpect plugin from"))
}
}
switch plugin.BuildModel {
case "image":
pbv, err := p.ImageBuildPlugin(bps, plugin)
@ -243,6 +259,34 @@ func createVersionID(s []byte) string {
return hex.EncodeToString(h.Sum(nil))
}
//InstallPluginFromYB InstallPluginFromYB
func (p *PluginAction) InstallPluginFromYB(b *api_model.BuildPluginStruct, plugin *dbmodel.TenantPlugin) (
*dbmodel.TenantPluginBuildVersion, error) {
if b.Body.Operator == "" {
b.Body.Operator = "define"
}
pbv := &dbmodel.TenantPluginBuildVersion{
VersionID: b.Body.BuildVersion,
PluginID: b.PluginID,
Kind: plugin.BuildModel,
BaseImage: plugin.ImageURL,
BuildLocalImage: b.Body.BuildImage,
ContainerCPU: b.Body.PluginCPU,
ContainerMemory: b.Body.PluginMemory,
ContainerCMD: b.Body.PluginCMD,
BuildTime: time.Now().Format(time.RFC3339),
Info: b.Body.Info,
Status: "complete",
}
if err := db.GetManager().TenantPluginBuildVersionDao().AddModel(pbv); err != nil {
if !strings.Contains(err.Error(), "exist") {
logrus.Errorf("build plugin error: %s", err.Error())
return nil, err
}
}
return pbv, nil
}
//ImageBuildPlugin ImageBuildPlugin
func (p *PluginAction) ImageBuildPlugin(b *api_model.BuildPluginStruct, plugin *dbmodel.TenantPlugin) (
*dbmodel.TenantPluginBuildVersion, error) {
@ -255,31 +299,63 @@ func (p *PluginAction) ImageBuildPlugin(b *api_model.BuildPluginStruct, plugin *
//TODO: build_version create in console
//diffStr := fmt.Sprintf("%s%s%s%s", b.TenantName, plugin.ImageURL, b.PluginID, time.Now().Format(time.RFC3339))
//buildVersion := createVersionID([]byte(diffStr))
pbv := &dbmodel.TenantPluginBuildVersion{
VersionID: b.Body.BuildVersion,
PluginID: b.PluginID,
Kind: plugin.BuildModel,
BaseImage: plugin.ImageURL,
ContainerCPU: b.Body.PluginCPU,
ContainerMemory: b.Body.PluginMemory,
ContainerCMD: b.Body.PluginCMD,
BuildTime: time.Now().Format(time.RFC3339),
Info: b.Body.Info,
Status: "building",
}
if b.Body.PluginCPU == 0 {
pbv.ContainerCPU = 125
}
if b.Body.PluginMemory == 0 {
pbv.ContainerMemory = 50
}
tx := db.GetManager().Begin()
if err := db.GetManager().TenantPluginBuildVersionDaoTransactions(tx).AddModel(pbv); err != nil {
if !strings.Contains(err.Error(), "exist") {
tx.Rollback()
logrus.Errorf("build plugin error: %s", err.Error())
rebuild := false
tpbv, err := db.GetManager().TenantPluginBuildVersionDao().GetBuildVersionByVersionID(
b.PluginID, b.Body.BuildVersion)
if err != nil {
if err.Error() == gorm.ErrRecordNotFound.Error() {
rebuild = false
} else {
return nil, err
}
} else {
rebuild = true
}
tx := db.GetManager().Begin()
if rebuild {
tpbv.Info = b.Body.Info
tpbv.Status = "building"
tpbv.BuildTime = time.Now().Format(time.RFC3339)
if b.Body.PluginCPU == 0 {
tpbv.ContainerCPU = 125
}
if b.Body.PluginMemory == 0 {
tpbv.ContainerMemory = 50
}
if err := db.GetManager().TenantPluginBuildVersionDaoTransactions(tx).UpdateModel(tpbv); err != nil {
if err != nil {
tx.Rollback()
logrus.Errorf("build plugin error: %s", err.Error())
return nil, err
}
}
} else {
pbv := &dbmodel.TenantPluginBuildVersion{
VersionID: b.Body.BuildVersion,
PluginID: b.PluginID,
Kind: plugin.BuildModel,
BaseImage: plugin.ImageURL,
ContainerCPU: b.Body.PluginCPU,
ContainerMemory: b.Body.PluginMemory,
ContainerCMD: b.Body.PluginCMD,
BuildTime: time.Now().Format(time.RFC3339),
Info: b.Body.Info,
Status: "building",
}
if b.Body.PluginCPU == 0 {
pbv.ContainerCPU = 125
}
if b.Body.PluginMemory == 0 {
pbv.ContainerMemory = 50
}
if err := db.GetManager().TenantPluginBuildVersionDaoTransactions(tx).AddModel(pbv); err != nil {
if !strings.Contains(err.Error(), "exist") {
tx.Rollback()
logrus.Errorf("build plugin error: %s", err.Error())
return nil, err
}
}
tpbv = pbv
}
taskBody := &builder_model.BuildPluginTaskBody{
TenantID: b.Body.TenantID,
@ -310,7 +386,10 @@ func (p *PluginAction) ImageBuildPlugin(b *api_model.BuildPluginStruct, plugin *
tx.Rollback()
return nil, err
}
if _, err := p.MQClient.Enqueue(context.Background(), eq); err != nil {
ctx, cancel := context.WithCancel(context.Background())
_, err = p.MQClient.Enqueue(ctx, eq)
cancel()
if err != nil {
logrus.Errorf("equque mq error, %v", err)
tx.Rollback()
return nil, err
@ -321,7 +400,7 @@ func (p *PluginAction) ImageBuildPlugin(b *api_model.BuildPluginStruct, plugin *
return nil, nil
}
logrus.Debugf("equeue mq build plugin from image success")
return pbv, nil
return tpbv, nil
}
//DockerfileBuildPlugin DockerfileBuildPlugin
@ -339,32 +418,64 @@ func (p *PluginAction) DockerfileBuildPlugin(b *api_model.BuildPluginStruct, plu
// TODO: build_version create in console
// diffStr := fmt.Sprintf("%s%s%s%s", b.TenantName, b.Body.RepoURL, b.PluginID, time.Now().Format(time.RFC3339))
// buildVersion := createVersionID([]byte(diffStr))
pbv := &dbmodel.TenantPluginBuildVersion{
VersionID: b.Body.BuildVersion,
PluginID: b.PluginID,
Kind: plugin.BuildModel,
Repo: b.Body.RepoURL,
GitURL: plugin.GitURL,
Info: b.Body.Info,
ContainerCPU: b.Body.PluginCPU,
ContainerMemory: b.Body.PluginMemory,
ContainerCMD: b.Body.PluginCMD,
BuildTime: time.Now().Format(time.RFC3339),
Status: "building",
}
if b.Body.PluginCPU == 0 {
pbv.ContainerCPU = 125
}
if b.Body.PluginMemory == 0 {
pbv.ContainerMemory = 50
}
tx := db.GetManager().Begin()
if err := db.GetManager().TenantPluginBuildVersionDaoTransactions(tx).AddModel(pbv); err != nil {
if !strings.Contains(err.Error(), "exist") {
tx.Rollback()
logrus.Errorf("build plugin error: %s", err.Error())
rebuild := false
tpbv, err := db.GetManager().TenantPluginBuildVersionDao().GetBuildVersionByVersionID(
b.PluginID, b.Body.BuildVersion)
if err != nil {
if err.Error() == gorm.ErrRecordNotFound.Error() {
rebuild = false
} else {
return nil, err
}
} else {
rebuild = true
}
tx := db.GetManager().Begin()
if rebuild {
tpbv.Info = b.Body.Info
tpbv.Status = "building"
tpbv.BuildTime = time.Now().Format(time.RFC3339)
if b.Body.PluginCPU == 0 {
tpbv.ContainerCPU = 125
}
if b.Body.PluginMemory == 0 {
tpbv.ContainerMemory = 50
}
if err := db.GetManager().TenantPluginBuildVersionDaoTransactions(tx).UpdateModel(tpbv); err != nil {
if err != nil {
tx.Rollback()
logrus.Errorf("build plugin error: %s", err.Error())
return nil, err
}
}
} else {
pbv := &dbmodel.TenantPluginBuildVersion{
VersionID: b.Body.BuildVersion,
PluginID: b.PluginID,
Kind: plugin.BuildModel,
Repo: b.Body.RepoURL,
GitURL: plugin.GitURL,
Info: b.Body.Info,
ContainerCPU: b.Body.PluginCPU,
ContainerMemory: b.Body.PluginMemory,
ContainerCMD: b.Body.PluginCMD,
BuildTime: time.Now().Format(time.RFC3339),
Status: "building",
}
if b.Body.PluginCPU == 0 {
pbv.ContainerCPU = 125
}
if b.Body.PluginMemory == 0 {
pbv.ContainerMemory = 50
}
if err := db.GetManager().TenantPluginBuildVersionDaoTransactions(tx).AddModel(pbv); err != nil {
if !strings.Contains(err.Error(), "exist") {
tx.Rollback()
logrus.Errorf("build plugin error: %s", err.Error())
return nil, err
}
}
tpbv = pbv
}
taskBody := &builder_model.BuildPluginTaskBody{
TenantID: b.Body.TenantID,
@ -396,7 +507,10 @@ func (p *PluginAction) DockerfileBuildPlugin(b *api_model.BuildPluginStruct, plu
tx.Rollback()
return nil, err
}
if _, err := p.MQClient.Enqueue(context.Background(), eq); err != nil {
ctx, cancel := context.WithCancel(context.Background())
_, errE := p.MQClient.Enqueue(ctx, eq)
cancel()
if errE != nil {
logrus.Errorf("equque mq error, %v", err)
tx.Rollback()
return nil, err
@ -407,7 +521,7 @@ func (p *PluginAction) DockerfileBuildPlugin(b *api_model.BuildPluginStruct, plu
return nil, nil
}
logrus.Debugf("equeue mq build plugin from dockerfile success")
return pbv, nil
return tpbv, nil
}
//GetAllPluginBuildVersions GetAllPluginBuildVersions
@ -434,6 +548,7 @@ func (p *PluginAction) DeletePluginBuildVersion(pluginID, versionID string) *uti
tx := db.GetManager().Begin()
err := db.GetManager().TenantPluginBuildVersionDaoTransactions(tx).DeleteBuildVersionByVersionID(versionID)
if err != nil {
tx.Rollback()
return util.CreateAPIHandleErrorFromDBError(fmt.Sprintf("delete plugin build version by id %v", versionID), err)
}
if err := tx.Commit().Error; err != nil {

106
pkg/api/handler/service.go
View File

@ -375,7 +375,9 @@ func (s *ServiceAction) sendTask(body map[string]interface{}, taskType string) e
logrus.Errorf("build equeue stop request error, %v", errEq)
return errEq
}
_, err = s.MQClient.Enqueue(context.Background(), eq)
ctx, cancel := context.WithCancel(context.Background())
_, err = s.MQClient.Enqueue(ctx, eq)
cancel()
if err != nil {
logrus.Errorf("equque mq error, %v", err)
return err
@ -455,7 +457,9 @@ func (s *ServiceAction) StartStopService(sss *api_model.StartStopStruct) error {
logrus.Errorf("build equeue startstop request error, %v", errEq)
return errEq
}
_, err = s.MQClient.Enqueue(context.Background(), eq)
ctx, cancel := context.WithCancel(context.Background())
_, err = s.MQClient.Enqueue(ctx, eq)
cancel()
if err != nil {
logrus.Errorf("equque mq error, %v", err)
return err
@ -476,7 +480,9 @@ func (s *ServiceAction) ServiceVertical(vs *model.VerticalScalingTaskBody) error
logrus.Errorf("build equeue vertical request error, %v", errEq)
return errEq
}
_, err := s.MQClient.Enqueue(context.Background(), eq)
ctx, cancel := context.WithCancel(context.Background())
_, err := s.MQClient.Enqueue(ctx, eq)
cancel()
if err != nil {
logrus.Errorf("equque mq error, %v", err)
return err
@ -497,7 +503,9 @@ func (s *ServiceAction) ServiceHorizontal(hs *model.HorizontalScalingTaskBody) e
logrus.Errorf("build equeue horizontal request error, %v", errEq)
return errEq
}
_, err := s.MQClient.Enqueue(context.Background(), eq)
ctx, cancel := context.WithCancel(context.Background())
_, err := s.MQClient.Enqueue(ctx, eq)
cancel()
if err != nil {
logrus.Errorf("equque mq error, %v", err)
return err
@ -524,7 +532,10 @@ func (s *ServiceAction) ServiceUpgrade(ru *model.RollingUpgradeTaskBody) error {
logrus.Errorf("build equeue upgrade request error, %v", errEq)
return errEq
}
if _, err := s.MQClient.Enqueue(context.Background(), eq); err != nil {
ctx, cancel := context.WithCancel(context.Background())
_, err = s.MQClient.Enqueue(ctx, eq)
cancel()
if err != nil {
logrus.Errorf("equque mq error, %v", err)
return err
}
@ -605,16 +616,18 @@ func (s *ServiceAction) ServiceCreate(sc *api_model.ServiceStruct) error {
if volumn.HostPath == "" {
//step 1 设置主机目录
switch volumn.VolumeType {
//共享文件存储
//共享文件<EFBFBD><EFBFBD>储
case dbmodel.ShareFileVolumeType.String():
volumn.HostPath = fmt.Sprintf("%s/tenant/%s/service/%s%s", sharePath, sc.TenantID, volumn.ServiceID, volumn.VolumePath)
//本地文件存<E4BBB6><E5AD98><EFBFBD><EFBFBD><EFBFBD>
case dbmodel.LocalVolumeType.String():
serviceType, err := db.GetManager().TenantServiceLabelDao().GetTenantServiceTypeLabel(volumn.ServiceID)
if err != nil {
tx.Rollback()
return util.CreateAPIHandleErrorFromDBError("service type", err)
}
if serviceType.LabelValue != core_util.StatefulServiceType {
tx.Rollback()
return util.CreateAPIHandleError(400, fmt.Errorf("应用类型不为有状态应用.不支持本地存储"))
}
volumn.HostPath = fmt.Sprintf("%s/tenant/%s/service/%s%s", localPath, sc.TenantID, volumn.ServiceID, volumn.VolumePath)
@ -660,7 +673,10 @@ func (s *ServiceAction) ServiceCreate(sc *api_model.ServiceStruct) error {
tx.Rollback()
return err
}
tx.Commit()
if err := tx.Commit().Error; err != nil {
tx.Rollback()
return err
}
return nil
}
@ -773,7 +789,9 @@ func (s *ServiceAction) CodeCheck(c *api_model.CheckCodeStruct) error {
logrus.Errorf("build equeue code check error, %v", errEq)
return errEq
}
_, err = s.MQClient.Enqueue(context.Background(), eq)
ctx, cancel := context.WithCancel(context.Background())
_, err = s.MQClient.Enqueue(ctx, eq)
cancel()
if err != nil {
logrus.Errorf("equque mq error, %v", err)
return err
@ -816,7 +834,9 @@ func (s *ServiceAction) ShareCloud(c *api_model.CloudShareStruct) error {
logrus.Errorf("build equeue share cloud error, %v", errEq)
return errEq
}
_, err := s.MQClient.Enqueue(context.Background(), eq)
ctx, cancel := context.WithCancel(context.Background())
_, err := s.MQClient.Enqueue(ctx, eq)
cancel()
if err != nil {
logrus.Errorf("equque mq error, %v", err)
return err
@ -948,6 +968,7 @@ func (s *ServiceAction) PortVar(action, tenantID, serviceID string, vps *api_mod
}
vpD, err := db.GetManager().TenantServicesPortDao().GetPort(serviceID, oldPort)
if err != nil {
tx.Rollback()
return err
}
vpD.ServiceID = serviceID
@ -1024,7 +1045,7 @@ func (s *ServiceAction) PortOuter(tenantName, serviceID, operation string, port
p.IsOuterService = false
tx := db.GetManager().Begin()
if err = db.GetManager().TenantServicesPortDaoTransactions(tx).UpdateModel(p); err != nil {
tx.Callback()
tx.Rollback()
return nil, "", err
}
service, err := db.GetManager().K8sServiceDao().GetK8sService(serviceID, port, true)
@ -1034,12 +1055,12 @@ func (s *ServiceAction) PortOuter(tenantName, serviceID, operation string, port
if service != nil {
err := s.KubeClient.Core().Services(p.TenantID).Delete(service.K8sServiceID, &metav1.DeleteOptions{})
if err != nil {
tx.Callback()
tx.Rollback()
return nil, "", fmt.Errorf("delete deploy k8s service info from kube-api error.%s", err.Error())
}
err = db.GetManager().K8sServiceDaoTransactions(tx).DeleteK8sServiceByName(service.K8sServiceID)
if err != nil {
tx.Callback()
tx.Rollback()
return nil, "", fmt.Errorf("delete deploy k8s service info from db error")
}
}
@ -1054,7 +1075,7 @@ func (s *ServiceAction) PortOuter(tenantName, serviceID, operation string, port
logrus.Debugf("outer, plugin port (%d) is not exist, do not need delete", port)
goto OUTERCLOSEPASS
}
tx.Callback()
tx.Rollback()
return nil, "", fmt.Errorf("outer, get plugin mapping port error:(%s)", err)
}
if p.IsInnerService {
@ -1067,7 +1088,7 @@ func (s *ServiceAction) PortOuter(tenantName, serviceID, operation string, port
dbmodel.UpNetPlugin,
port,
); err != nil {
tx.Callback()
tx.Rollback()
return nil, "", fmt.Errorf("outer, delete plugin mapping port %d error:(%s)", port, err)
}
logrus.Debugf(fmt.Sprintf("outer, delete plugin port %d->%d", port, pluginPort.PluginPort))
@ -1098,13 +1119,13 @@ func (s *ServiceAction) PortOuter(tenantName, serviceID, operation string, port
p.IsOuterService = true
tx := db.GetManager().Begin()
if err = db.GetManager().TenantServicesPortDaoTransactions(tx).UpdateModel(p); err != nil {
tx.Callback()
tx.Rollback()
return nil, "", err
}
if p.Protocol != "http" && p.Protocol != "https" {
vsPort, err = s.createVSPort(serviceID, p.ContainerPort)
if vsPort == nil {
tx.Callback()
tx.Rollback()
return nil, "", fmt.Errorf("create or get vs map port for service error,%s", err.Error())
}
}
@ -1112,7 +1133,7 @@ func (s *ServiceAction) PortOuter(tenantName, serviceID, operation string, port
if deploy != nil {
k8sService, err = s.createOuterK8sService(tenantName, vsPort, service, p, deploy)
if err != nil && !strings.HasSuffix(err.Error(), "is exist") {
tx.Callback()
tx.Rollback()
return nil, "", fmt.Errorf("create k8s service error,%s", err.Error())
}
}
@ -1132,14 +1153,14 @@ func (s *ServiceAction) PortOuter(tenantName, serviceID, operation string, port
port,
)
if err != nil {
tx.Callback()
tx.Rollback()
logrus.Errorf("outer, set plugin mapping port error:(%s)", err)
return nil, "", fmt.Errorf("outer, set plugin mapping port error:(%s)", err)
}
pPort = ppPort
goto OUTEROPENPASS
}
tx.Callback()
tx.Rollback()
return nil, "", fmt.Errorf("outer, in setting plugin mapping port, get plugin mapping port error:(%s)", err)
}
logrus.Debugf("outer, plugin mapping port is already exist, %d->%d", pluginPort.ContainerPort, pluginPort.PluginPort)
@ -1178,7 +1199,8 @@ func (s *ServiceAction) createOuterK8sService(tenantName string, mapPort *dbmode
"key": "",
"event_id": tenantservice.EventID,
}
if port.Protocol == "stream" && mapPort != nil { //stream 协议获取映射端口
//TODO: "stream" to ! http
if port.Protocol != "http" && mapPort != nil { //stream 协议获取映射端口
service.Labels["lbmap_port"] = fmt.Sprintf("%d", mapPort.Port)
}
var servicePort v1.ServicePort
@ -1290,7 +1312,7 @@ func (s *ServiceAction) PortInner(tenantName, serviceID, operation string, port
if p.IsInnerService { //如果端口已经开了对内
p.IsInnerService = false
if err = db.GetManager().TenantServicesPortDaoTransactions(tx).UpdateModel(p); err != nil {
tx.Callback()
tx.Rollback()
return fmt.Errorf("update service port error: %s", err.Error())
}
service, err := db.GetManager().K8sServiceDao().GetK8sService(serviceID, port, false)
@ -1300,12 +1322,12 @@ func (s *ServiceAction) PortInner(tenantName, serviceID, operation string, port
if service != nil {
err := s.KubeClient.Core().Services(p.TenantID).Delete(service.K8sServiceID, &metav1.DeleteOptions{})
if err != nil && !strings.HasSuffix(err.Error(), "not found") {
tx.Callback()
tx.Rollback()
return fmt.Errorf("delete deploy k8s service info from kube-api error")
}
err = db.GetManager().K8sServiceDao().DeleteK8sServiceByName(service.K8sServiceID)
if err != nil {
tx.Callback()
tx.Rollback()
return fmt.Errorf("delete deploy k8s service info from db error")
}
}
@ -1320,7 +1342,7 @@ func (s *ServiceAction) PortInner(tenantName, serviceID, operation string, port
logrus.Debugf("inner, plugin port (%d) is not exist, do not need delete", port)
goto INNERCLOSEPASS
}
tx.Callback()
tx.Rollback()
return fmt.Errorf("inner, get plugin mapping port error:(%s)", err)
}
if p.IsOuterService {
@ -1332,31 +1354,31 @@ func (s *ServiceAction) PortInner(tenantName, serviceID, operation string, port
dbmodel.UpNetPlugin,
port,
); err != nil {
tx.Callback()
tx.Rollback()
return fmt.Errorf("inner, delete plugin mapping port %d error:(%s)", port, err)
}
logrus.Debugf(fmt.Sprintf("inner, delete plugin port %d->%d", port, pluginPort.PluginPort))
INNERCLOSEPASS:
}
} else {
tx.Callback()
tx.Rollback()
return fmt.Errorf("already close")
}
case "open":
if p.IsInnerService {
tx.Callback()
tx.Rollback()
return fmt.Errorf("already open")
}
p.IsInnerService = true
if err = db.GetManager().TenantServicesPortDaoTransactions(tx).UpdateModel(p); err != nil {
tx.Callback()
tx.Rollback()
return err
}
deploy, _ := db.GetManager().K8sDeployReplicationDao().GetK8sCurrentDeployReplicationByService(serviceID)
if deploy != nil {
k8sService, err = s.createInnerService(service, p, deploy)
if err != nil {
tx.Callback()
tx.Rollback()
return fmt.Errorf("create k8s service error,%s", err.Error())
}
@ -1377,14 +1399,14 @@ func (s *ServiceAction) PortInner(tenantName, serviceID, operation string, port
port,
)
if err != nil {
tx.Callback()
tx.Rollback()
logrus.Errorf("inner, set plugin mapping port error:(%s)", err)
return fmt.Errorf("inner, set plugin mapping port error:(%s)", err)
}
pPort = ppPort
goto INNEROPENPASS
}
tx.Callback()
tx.Rollback()
return fmt.Errorf("inner, in setting plugin mapping port, get plugin mapping port error:(%s)", err)
}
logrus.Debugf("inner, plugin mapping port is already exist, %d->%d", pluginPort.ContainerPort, pluginPort.PluginPort)
@ -1545,13 +1567,16 @@ func (s *ServiceAction) RollBack(rs *api_model.RollbackStruct) error {
tx := db.GetManager().Begin()
service, err := db.GetManager().TenantServiceDaoTransactions(tx).GetServiceByID(rs.ServiceID)
if err != nil {
tx.Rollback()
return err
}
if service.DeployVersion == rs.DeployVersion {
tx.Rollback()
return fmt.Errorf("current version is %v, don't need rollback", rs.DeployVersion)
}
service.DeployVersion = rs.DeployVersion
if err := db.GetManager().TenantServiceDaoTransactions(tx).UpdateModel(service); err != nil {
tx.Rollback()
return err
}
//发送重启消息到MQ
@ -1634,15 +1659,10 @@ func (s *ServiceAction) CreateTenant(t *dbmodel.Tenants) error {
}
tx := db.GetManager().Begin()
if err := db.GetManager().TenantDaoTransactions(tx).AddModel(t); err != nil {
if strings.HasSuffix(err.Error(), "is exist") {
_, err := s.KubeClient.Core().Namespaces().Get(t.UUID, metav1.GetOptions{})
if err == nil {
tx.Commit()
return nil
}
if !strings.HasSuffix(err.Error(), "is exist") {
tx.Rollback()
return err
}
tx.Callback()
return err
}
ns := &v1.Namespace{
ObjectMeta: metav1.ObjectMeta{
@ -1656,7 +1676,11 @@ func (s *ServiceAction) CreateTenant(t *dbmodel.Tenants) error {
return err
}
}
return tx.Commit().Error
if err := tx.Commit().Error; err != nil {
tx.Rollback()
return err
}
return nil
}
//CreateTenandIDAndName create tenant_id and tenant_name
@ -1832,7 +1856,7 @@ func (s *ServiceAction) TenantServiceDeletePluginRelation(serviceID, pluginID st
//SetTenantServicePluginRelation SetTenantServicePluginRelation
func (s *ServiceAction) SetTenantServicePluginRelation(tenantID, serviceID string, pss *api_model.PluginSetStruct) *util.APIHandleError {
tx := db.GetManager().Begin()
plugin, err := db.GetManager().TenantPluginDao().GetPluginByID(pss.Body.PluginID)
plugin, err := db.GetManager().TenantPluginDao().GetPluginByID(pss.Body.PluginID, tenantID)
if err != nil {
tx.Rollback()
return util.CreateAPIHandleErrorFromDBError("get plugin by plugin id", err)

35
pkg/api/handler/tenant.go
View File

@ -108,7 +108,6 @@ func (t *TenantAction) TotalMemCPU(services []*dbmodel.TenantServices) (*api_mod
cpus := 0
mem := 0
for _, service := range services {
logrus.Debugf("service is %s, cpus is %v, mem is %v", service.ID, service.ContainerCPU, service.ContainerMemory)
cpus += service.ContainerCPU
mem += service.ContainerMemory
@ -225,8 +224,9 @@ func (t *TenantAction) HTTPTsdb(md *api_model.MontiorData) ([]byte, error) {
}
//GetTenantsResources GetTenantsResources
func (t *TenantAction) GetTenantsResources(tr *api_model.TenantResources) ([]*map[string]interface{}, error) {
func (t *TenantAction) GetTenantsResources(tr *api_model.TenantResources) ([]map[string]interface{}, error) {
//返回全部资源
//TODO: 应用关闭，硬盘存储资源仍会占用
return db.GetManager().TenantServiceDao().GetCPUAndMEM(tr.Body.TenantName)
}
@ -247,3 +247,34 @@ func (t *TenantAction) GetProtocols() ([]*dbmodel.RegionProcotols, *util.APIHand
}
return rps, nil
}
//TransPlugins TransPlugins
func (t *TenantAction) TransPlugins(tenantID, tenantName, fromTenant string, pluginList []string) *util.APIHandleError {
tenantInfo, err := db.GetManager().TenantDao().GetTenantIDByName(fromTenant)
if err != nil {
return util.CreateAPIHandleErrorFromDBError("get tenant infos", err)
}
goodrainID := tenantInfo.UUID
tx := db.GetManager().Begin()
for _, p := range pluginList {
pluginInfo, err := db.GetManager().TenantPluginDao().GetPluginByID(p, goodrainID)
if err != nil {
tx.Rollback()
return util.CreateAPIHandleErrorFromDBError("get plugin infos", err)
}
pluginInfo.TenantID = tenantID
pluginInfo.Domain = tenantName
pluginInfo.ID = 0
err = db.GetManager().TenantPluginDaoTransactions(tx).AddModel(pluginInfo)
if err != nil {
if !strings.Contains(err.Error(), "is exist") {
tx.Rollback()
return util.CreateAPIHandleErrorFromDBError("add plugin Info", err)
}
}
}
if err := tx.Commit().Error; err != nil {
return util.CreateAPIHandleErrorFromDBError("trans plugins infos", err)
}
return nil
}

3
pkg/api/handler/tokenIden.go
View File

@ -23,7 +23,6 @@ import (
"strings"
"time"
"github.com/Sirupsen/logrus"
"github.com/goodrain/rainbond/cmd/api/option"
api_model "github.com/goodrain/rainbond/pkg/api/model"
"github.com/goodrain/rainbond/pkg/api/util"
@ -109,7 +108,7 @@ func (t *TokenIdenAction) DeleteAPIManager(am *api_model.APIManager) *util.APIHa
//CheckToken CheckToken
func (t *TokenIdenAction) CheckToken(token, uri string) bool {
m := GetDefaultTokenMap()
logrus.Debugf("default token map is %v", m)
//logrus.Debugf("default token map is %v", m)
regionInfo, ok := m[token]
if !ok {
return false

3
pkg/api/middleware/mideware.go
View File

@ -93,11 +93,12 @@ func InitService(next http.Handler) http.Handler {
func InitPlugin(next http.Handler) http.Handler {
fn := func(w http.ResponseWriter, r *http.Request) {
pluginID := chi.URLParam(r, "plugin_id")
tenantID := r.Context().Value(ContextKey("tenant_id")).(string)
if pluginID == "" {
httputil.ReturnError(r, w, 404, "need plugin id")
return
}
_, err := db.GetManager().TenantPluginDao().GetPluginByID(pluginID)
_, err := db.GetManager().TenantPluginDao().GetPluginByID(pluginID,tenantID)
if err != nil {
if err.Error() == gorm.ErrRecordNotFound.Error() {
httputil.ReturnError(r, w, 404, "cant find plugin")

2
pkg/api/middleware/proxy.go
View File

@ -44,7 +44,7 @@ func InitProxy(conf option.Config) {
func GetNodeProxy() proxy.Proxy {
return nodeProxy
}
//GetNodeProxy GetNodeProxy
//GetBuilderProxy GetNodeProxy
func GetBuilderProxy() proxy.Proxy {
return builderProxy
}

2
pkg/api/middleware/token.go
View File

@ -132,7 +132,7 @@ func FullToken(next http.Handler) http.Handler {
w.WriteHeader(http.StatusUnauthorized)
return
}
logrus.Debugf("request uri is %s", r.RequestURI)
//logrus.Debugf("request uri is %s", r.RequestURI)
t := r.Header.Get("Authorization")
if tt := strings.Split(t, " "); len(tt) == 2 {
if handler.GetTokenIdenHandler().CheckToken(tt[1], r.RequestURI) {

28
pkg/api/model/pluginModel.go
View File

@ -224,8 +224,16 @@ type BuildPluginStruct struct {
Operator string `json:"operator" validate:"operator"`
//租户id
// in: body
// required: false
// required: true
TenantID string `json:"tenant_id" validate:"tenant_id"`
//安装来源
// in: body
// required: false
PluginFrom string `json:"plugin_from" validate:"plugin_from"`
// 镜像地址
// in: body
// required: false
BuildImage string `json:"build_image" validate:"build_image"`
}
}
@ -387,3 +395,21 @@ type VersionEnv struct {
//required: true
EnvValue string `json:"env_value" validate:"env_value"`
}
//TransPlugins TransPlugins
type TransPlugins struct {
// in: path
// required: true
TenantName string `json:"tenant_name"`
//in: body
Body struct {
// 从该租户安装
// in: body
// required: true
FromTenantName string `json:"from_tenant_name" validate:"from_tenant_name"`
// 插件id
// in: body
// required: true
PluginsID []string `json:"plugins_id" validate:"plugins_id"`
}
}

29
pkg/api/server/api.go
View File

@ -29,6 +29,7 @@ import (
"github.com/goodrain/rainbond/pkg/api/apiRouters/doc"
"github.com/goodrain/rainbond/pkg/api/apiRouters/license"
"github.com/goodrain/rainbond/pkg/api/proxy"
"github.com/goodrain/rainbond/pkg/api/apiRouters/cloud"
"github.com/goodrain/rainbond/pkg/api/apiRouters/version2"
@ -43,11 +44,12 @@ import (
//Manager apiserver
type Manager struct {
ctx context.Context
cancel context.CancelFunc
conf option.Config
stopChan chan struct{}
r *chi.Mux
ctx context.Context
cancel context.CancelFunc
conf option.Config
stopChan chan struct{}
r *chi.Mux
prometheusProxy proxy.Proxy
}
//NewManager newManager
@ -73,6 +75,11 @@ func NewManager(c option.Config) *Manager {
//simple api version
r.Use(apimiddleware.APIVersion)
r.Use(apimiddleware.Proxy)
r.Get("/monitor", func(res http.ResponseWriter, req *http.Request) {
res.Write([]byte("ok"))
})
return &Manager{
ctx: ctx,
cancel: cancel,
@ -118,6 +125,10 @@ func (m *Manager) Run() {
m.r.Mount("/license", license.Routes())
//兼容老版docker
m.r.Get("/v1/etcd/event-log/instances", m.EventLogInstance)
//prometheus单节点代理
m.r.Get("/api/v1/query", m.PrometheusAPI)
m.r.Get("/api/v1/query_range", m.PrometheusAPI)
//开启对浏览器的websocket服务和文件服务
go func() {
websocketRouter := chi.NewRouter()
@ -172,3 +183,11 @@ func (m *Manager) EventLogInstance(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(404)
return
}
//PrometheusAPI prometheus api 代理
func (m *Manager) PrometheusAPI(w http.ResponseWriter, r *http.Request) {
if m.prometheusProxy == nil {
m.prometheusProxy = proxy.CreateProxy("prometheus", "http", []string{"127.0.0.1:9999"})
}
m.prometheusProxy.Proxy(w, r)
}

11
pkg/builder/discover/discover.go
View File

@ -1,19 +1,18 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
@ -81,7 +80,9 @@ func (t *TaskManager) Do() {
case <-t.ctx.Done():
return
default:
data, err := t.client.Dequeue(t.ctx, &pb.DequeueRequest{Topic: WTOPIC, ClientHost: hostName + "-builder"})
ctx, cancel := context.WithCancel(t.ctx)
data, err := t.client.Dequeue(ctx, &pb.DequeueRequest{Topic: WTOPIC, ClientHost: hostName + "-builder"})
cancel()
if err != nil {
if grpc1.ErrorDesc(err) == context.DeadlineExceeded.Error() {
logrus.Warn(err.Error())

30
pkg/builder/exector/dockerExec.go
View File

@ -73,17 +73,25 @@ func ShowExec(command string, params []string, logger ...event.Logger) error {
logger[0].Debug(fmt.Sprintf("builder:%v", line), map[string]string{"step": "build-exector"})
}
}()
errReader := bufio.NewReader(stderr)
go func() {
for {
errLine, errL := errReader.ReadString('\n')
if errL != nil || io.EOF == errL {
break
errW := cmd.Wait()
if errW != nil {
//bytesErr, errR := ioutil.ReadAll(stderr)
//logrus.Debugf(fmt.Sprintf("builder error: %v", errR))
//logrus.Debugf(fmt.Sprintf("builder error: %v", string(bytesErr)))
//logger[0].Error(fmt.Sprintf("build Error: %v", string(bytesErr)), map[string]string{"step": "builder-exector"})
//return errR
go func() {
readerr := bufio.NewReader(stderr)
for {
line, errL := readerr.ReadString('\n')
if errL != nil || io.EOF == errL {
break
}
logrus.Errorf(fmt.Sprintf("builder err: %v", line))
logger[0].Error(fmt.Sprintf("builder err:%v", line), map[string]string{"step": "build-exector"})
}
logrus.Debugf(fmt.Sprintf("builder error: %v", errLine))
logger[0].Error(fmt.Sprintf("build Error: %v", errLine), map[string]string{"step": "builder-exector"})
}
}()
cmd.Wait()
}()
return errW
}
return nil
}

88
pkg/builder/exector/plugin_dockerfile.go
View File

@ -19,8 +19,11 @@
package exector
import (
"bufio"
"fmt"
"io"
"os"
"os/exec"
"strings"
"time"
@ -73,24 +76,21 @@ func (e *exectorManager) pluginDockerfileBuild(in []byte) {
for retry := 0; retry < 3; retry++ {
err := e.runD(&tb, config, logger)
if err != nil {
logrus.Errorf("exec plugin build from image error:%s", err.Error())
if retry < 3 {
logger.Info("dockerfile构建插件任务执行失败，开始重试", map[string]string{"step": "builder-exector", "status": "failure"})
} else {
version, err := db.GetManager().TenantPluginBuildVersionDao().GetBuildVersionByVersionID(tb.PluginID, tb.VersionID)
if err != nil {
logrus.Errorf("get version error, %v", err)
}
version.Status = "failure"
if err := db.GetManager().TenantPluginBuildVersionDao().UpdateModel(version); err != nil {
logrus.Errorf("update version error, %v", err)
}
logger.Error("dockerfile构建插件任务执行失败", map[string]string{"step": "callback", "status": "failure"})
}
logrus.Errorf("exec plugin build from dockerfile error:%s", err.Error())
logger.Info("dockerfile构建插件任务执行失败，开始重试", map[string]string{"step": "builder-exector", "status": "failure"})
} else {
break
return
}
}
version, err := db.GetManager().TenantPluginBuildVersionDao().GetBuildVersionByVersionID(tb.PluginID, tb.VersionID)
if err != nil {
logrus.Errorf("get version error, %v", err)
}
version.Status = "failure"
if err := db.GetManager().TenantPluginBuildVersionDao().UpdateModel(version); err != nil {
logrus.Errorf("update version error, %v", err)
}
logger.Error("dockerfile构建插件任务执行失败", map[string]string{"step": "callback", "status": "failure"})
}()
}
@ -163,16 +163,66 @@ func clone(gitURL string, sourceDir string, logger event.Logger, repo string) er
}
} else {
logrus.Debugf("pull: %s", fmt.Sprintf("sudo -P git -C %s pull", sourceDir))
mm := []string{"-P", "git", "-C", sourceDir, "pull"}
if err := ShowExec("sudo", mm, logger); err != nil {
mm := []string{"-C", sourceDir, "pull"}
if err := ShowExec("git", mm, logger); err != nil {
return err
}
}
return nil
}
func checkGitDir(sourceDir string, logger event.Logger) {
func gitclone(gitURL string, sourceDir string, logger event.Logger, repo string) error {
path := fmt.Sprintf("%s/.git/config", sourceDir)
_, err := os.Stat(path)
if err != nil {
if os.IsNotExist(err) {
logrus.Debugf("clone: %s", fmt.Sprintf("git clone -b %s %s %s", repo, gitURL, sourceDir))
mm := []string{"-P", "git", "clone", "-b", repo, gitURL, sourceDir}
cmd := exec.Command("sudo", mm...)
stdout, err := cmd.StdoutPipe()
if err != nil {
logrus.Errorf(fmt.Sprintf("builder err: %v", err))
return err
}
errC := cmd.Start()
if errC != nil {
logrus.Debugf(fmt.Sprintf("builder: %v", errC))
logger.Error(fmt.Sprintf("builder:%v", errC), map[string]string{"step": "build-exector"})
return errC
}
reader := bufio.NewReader(stdout)
go func() {
for {
line, errL := reader.ReadString('\n')
if errL != nil || io.EOF == errL {
break
}
//fmt.Print(line)
logrus.Debugf(fmt.Sprintf("builder: %v", line))
logger.Debug(fmt.Sprintf("builder:%v", line), map[string]string{"step": "build-exector"})
}
}()
errW := cmd.Wait()
logrus.Debugf("errw is %v", errW)
if errW != nil {
cierr := strings.Split(errW.Error(), "\n")
if strings.Contains(errW.Error(), "Cloning into") && len(cierr) < 3 {
logrus.Errorf(fmt.Sprintf("builder:%v", errW))
logger.Error(fmt.Sprintf("builder:%v", errW), map[string]string{"step": "build-exector"})
return errW
}
}
return nil
}
logrus.Debugf("file check error: %v", err)
return err
}
logrus.Debugf("pull: %s", fmt.Sprintf("sudo -P git -C %s pull", sourceDir))
mm := []string{"-P", "git", "-C", sourceDir, "pull"}
if err := ShowExec("sudo", mm, logger); err != nil {
return err
}
return nil
}
func checkDockerfile(sourceDir string) bool {

25
pkg/builder/exector/plugin_image.go
View File

@ -92,23 +92,20 @@ func (e *exectorManager) pluginImageBuild(in []byte) {
err := e.run(&tb, config, logger)
if err != nil {
logrus.Errorf("exec plugin build from image error:%s", err.Error())
if retry < 3 {
logger.Info("镜像构建插件任务执行失败，开始重试", map[string]string{"step": "builder-exector", "status": "failure"})
} else {
version, err := db.GetManager().TenantPluginBuildVersionDao().GetBuildVersionByVersionID(tb.PluginID, tb.VersionID)
if err != nil {
logrus.Errorf("get version error, %v", err)
}
version.Status = "failure"
if err := db.GetManager().TenantPluginBuildVersionDao().UpdateModel(version); err != nil {
logrus.Errorf("update version error, %v", err)
}
logger.Info("镜像构建插件任务执行失败", map[string]string{"step": "callback", "status": "failure"})
}
logger.Info("镜像构建插件任务执行失败，开始重试", map[string]string{"step": "builder-exector", "status": "failure"})
} else {
break
return
}
}
version, err := db.GetManager().TenantPluginBuildVersionDao().GetBuildVersionByVersionID(tb.PluginID, tb.VersionID)
if err != nil {
logrus.Errorf("get version error, %v", err)
}
version.Status = "failure"
if err := db.GetManager().TenantPluginBuildVersionDao().UpdateModel(version); err != nil {
logrus.Errorf("update version error, %v", err)
}
logger.Info("镜像构建插件任务执行失败", map[string]string{"step": "callback", "status": "failure"})
}()
}

8
pkg/db/dao/dao.go
View File

@ -64,9 +64,10 @@ type TenantServiceDao interface {
GetServicesByTenantID(tenantID string) ([]*model.TenantServices, error)
GetServicesAllInfoByTenantID(tenantID string) ([]*model.TenantServices, error)
DeleteServiceByServiceID(serviceID string) error
GetCPUAndMEM(tenantName []string) ([]*map[string]interface{}, error)
GetCPUAndMEM(tenantName []string) ([]map[string]interface{}, error)
GetPagedTenantService(offset, len int) ([]map[string]interface{}, error)
GetTenantServiceRes(uuid string) (map[string]interface{}, error)
GetAllServices() ([]*model.TenantServices, error)
}
//TenantServiceDeleteDao TenantServiceDeleteDao
@ -88,8 +89,8 @@ type TenantServicesPortDao interface {
//TenantPluginDao TenantPluginDao
type TenantPluginDao interface {
Dao
GetPluginByID(pluginID string) (*model.TenantPlugin, error)
DeletePluginByID(pluginID string) error
GetPluginByID(pluginID, tenantID string) (*model.TenantPlugin, error)
DeletePluginByID(pluginID, tenantID string) error
GetPluginsByTenantID(tenantID string) ([]*model.TenantPlugin, error)
}
@ -199,6 +200,7 @@ type TenantServiceVolumeDao interface {
DeleteTenantServiceVolumesByServiceID(serviceID string) error
DeleteByServiceIDAndVolumePath(serviceID string, volumePath string) error
GetVolumeByServiceIDAndName(serviceID, name string) (*model.TenantServiceVolume, error)
GetAllVolumes() ([]*model.TenantServiceVolume, error)
}
//TenantServiceLBMappingPortDao vs lb mapping port dao

14
pkg/db/db_test.go
View File

@ -1,30 +1,30 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package db
import (
"testing"
"time"
dbconfig "github.com/goodrain/rainbond/pkg/db/config"
"github.com/goodrain/rainbond/pkg/db/model"
"github.com/goodrain/rainbond/pkg/util"
"testing"
"time"
)
func TestTenantDao(t *testing.T) {
@ -134,7 +134,7 @@ func TestCreateTenantServiceLBMappingPortTran(t *testing.T) {
tx := GetManager().Begin()
mapPort, err := GetManager().TenantServiceLBMappingPortDaoTransactions(tx).CreateTenantServiceLBMappingPort("889bb1f028f655bebd545f24aa184a0b", 8082)
if err != nil {
tx.Callback()
tx.Rollback()
t.Fatal(err)
return
}

12
pkg/db/mysql/dao/plugin.go
View File

@ -34,12 +34,12 @@ type PluginDaoImpl struct {
func (t *PluginDaoImpl) AddModel(mo model.Interface) error {
plugin := mo.(*model.TenantPlugin)
var oldPlugin model.TenantPlugin
if ok := t.DB.Where("plugin_id = ?", plugin.PluginID).Find(&oldPlugin).RecordNotFound(); ok {
if ok := t.DB.Where("plugin_id = ? and tenant_id = ?", plugin.PluginID, plugin.TenantID).Find(&oldPlugin).RecordNotFound(); ok {
if err := t.DB.Create(plugin).Error; err != nil {
return err
}
} else {
return fmt.Errorf("plugin %s is exist", plugin.PluginName)
return fmt.Errorf("plugin %s in tenant %s is exist", plugin.PluginName, plugin.TenantID)
}
return nil
}
@ -54,20 +54,20 @@ func (t *PluginDaoImpl) UpdateModel(mo model.Interface) error {
}
//GetPluginByID GetPluginByID
func (t *PluginDaoImpl) GetPluginByID(id string) (*model.TenantPlugin, error) {
func (t *PluginDaoImpl) GetPluginByID(id, tenantID string) (*model.TenantPlugin, error) {
var plugin model.TenantPlugin
if err := t.DB.Where("plugin_id = ? ", id).Find(&plugin).Error; err != nil {
if err := t.DB.Where("plugin_id = ? and tenant_id = ?", id, tenantID).Find(&plugin).Error; err != nil {
return nil, err
}
return &plugin, nil
}
//DeletePluginByID DeletePluginByID
func (t *PluginDaoImpl) DeletePluginByID(id string) error {
func (t *PluginDaoImpl) DeletePluginByID(id, tenantID string) error {
relation := &model.TenantPlugin{
PluginID: id,
}
if err := t.DB.Where("plugin_id=?", id).Delete(relation).Error; err != nil {
if err := t.DB.Where("plugin_id=? and tenant_id=?", id, tenantID).Delete(relation).Error; err != nil {
return err
}
return nil

74
pkg/db/mysql/dao/tenants.go
View File

@ -21,7 +21,9 @@ package dao
import (
"fmt"
"os"
"os/exec"
"strconv"
"strings"
"time"
"github.com/goodrain/rainbond/pkg/db/model"
@ -87,7 +89,7 @@ func (t *TenantDaoImpl) GetALLTenants() ([]*model.Tenants, error) {
}
//GetALLTenants GetALLTenants
func (t *TenantDaoImpl) GetPagedTenants(offset,len int) ([]*model.Tenants, error) {
func (t *TenantDaoImpl) GetPagedTenants(offset, len int) ([]*model.Tenants, error) {
var tenants []*model.Tenants
if err := t.DB.Find(&tenants).Group("").Error; err != nil {
@ -101,6 +103,18 @@ type TenantServicesDaoImpl struct {
DB *gorm.DB
}
//GetAllServices 获取全部应用信息的资源相关信息
func (t *TenantServicesDaoImpl) GetAllServices() ([]*model.TenantServices, error) {
var services []*model.TenantServices
if err := t.DB.Select("tenant_id,service_id,service_alias,host_path,replicas,container_memory").Find(&services).Error; err != nil {
if err == gorm.ErrRecordNotFound {
return services, nil
}
return nil, err
}
return services, nil
}
//AddModel 添加租户应用
func (t *TenantServicesDaoImpl) AddModel(mo model.Interface) error {
service := mo.(*model.TenantServices)
@ -133,9 +147,8 @@ func (t *TenantServicesDaoImpl) GetServiceByID(serviceID string) (*model.TenantS
return &service, nil
}
//GetCPUAndMEM GetCPUAndMEM
func (t *TenantServicesDaoImpl) GetCPUAndMEM(tenantName []string) ([]*map[string]interface{}, error) {
func (t *TenantServicesDaoImpl) GetCPUAndMEM(tenantName []string) ([]map[string]interface{}, error) {
if len(tenantName) == 0 {
rows, err := t.DB.Raw("select sum(container_cpu) as cpu,sum(container_memory * replicas) as memory from tenant_services where service_id in (select service_id from tenant_service_status where status != 'closed' && status != 'undeploy')").Rows()
if err != nil {
@ -147,14 +160,14 @@ func (t *TenantServicesDaoImpl) GetCPUAndMEM(tenantName []string) ([]*map[string
for rows.Next() {
rows.Scan(&cpu, &mem)
}
var rc []*map[string]interface{}
var rc []map[string]interface{}
res := make(map[string]interface{})
res["cpu"] = cpu
res["memory"] = mem
rc = append(rc, &res)
rc = append(rc, res)
return rc, nil
}
var rc []*map[string]interface{}
var rc []map[string]interface{}
for _, tenant := range tenantName {
rows, err := t.DB.Raw("select tenant_id, sum(container_cpu) as cpu, sum(container_memory * replicas) as memory from tenant_services where service_id in (select service_id from tenant_service_status where (status != 'closed' && status != 'undeploy') && service_id in (select service_id from tenant_services where domain = (?))) group by tenant_id", tenant).Rows()
if err != nil {
@ -170,16 +183,29 @@ func (t *TenantServicesDaoImpl) GetCPUAndMEM(tenantName []string) ([]*map[string
res["cpu"] = cpu
res["memory"] = mem
res["tenant_id"] = id
logrus.Infof("res is $v", res)
rc = append(rc, &res)
res["tenant_name"] = tenant
dirPath := fmt.Sprintf("/grdata/tenant/%s", id)
cmd := []string{"-sh", "-m", dirPath}
f, err := exec.Command("du", cmd...).Output()
if err != nil {
f = []byte("1 xxx")
}
st := strings.Split(string(f), "\t")[0]
//TODO: disk默认单位为MB
intSt, err := strconv.Atoi(st)
if err != nil {
return nil, err
}
res["disk"] = intSt
rc = append(rc, res)
}
}
return rc, nil
}
//GetPagedTenantResource GetPagedTenantResource
func (t *TenantServicesDaoImpl) GetPagedTenantService(offset,len int) ([]map[string]interface{}, error) {
rows, err := t.DB.Raw("select tenant_id,sum(if (cur_status != 'closed' && cur_status != 'undeploy',container_cpu * replicas,0)) as use_cpu,sum(container_cpu*replicas) as cap_cpu,sum(if (cur_status != 'closed' && cur_status != 'undeploy',container_memory * replicas,0)) as use_memory,sum(container_memory*replicas) as cap_memory from tenant_services group by tenant_id order by use_memory desc limit ?,?",offset,len).Rows()
func (t *TenantServicesDaoImpl) GetPagedTenantService(offset, len int) ([]map[string]interface{}, error) {
rows, err := t.DB.Raw("select tenant_id,sum(if (cur_status != 'closed' && cur_status != 'undeploy',container_cpu * replicas,0)) as use_cpu,sum(container_cpu*replicas) as cap_cpu,sum(if (cur_status != 'closed' && cur_status != 'undeploy',container_memory * replicas,0)) as use_memory,sum(container_memory*replicas) as cap_memory from tenant_services group by tenant_id order by use_memory desc limit ?,?", offset, len).Rows()
if err != nil {
return nil, err
}
@ -191,37 +217,37 @@ func (t *TenantServicesDaoImpl) GetPagedTenantService(offset,len int) ([]map[str
var useCpu int
var capMem int
var useMem int
rows.Scan(&tenantID, &useCpu,&capCpu,&useMem,&capMem)
rows.Scan(&tenantID, &useCpu, &capCpu, &useMem, &capMem)
res := make(map[string]interface{})
res["capcpu"] = capCpu
res["usecpu"] = useCpu
res["capmem"] = capMem
res["usemem"] = useMem
res["tenant"] =tenantID
res["tenant"] = tenantID
rc = append(rc, res)
}
return rc, nil
}
//GetTenantServiceRes GetTenantServiceRes
func (t *TenantServicesDaoImpl) GetTenantServiceRes(uuid string) (map[string]interface{}, error) {
row := t.DB.Raw("select sum(if (cur_status != 'closed' && cur_status != 'undeploy',container_cpu * replicas,0)) as use_cpu,sum(container_cpu*replicas) as cap_cpu,sum(if (cur_status != 'closed' && cur_status != 'undeploy',container_memory * replicas,0)) as use_memory,sum(container_memory*replicas) as cap_memory from tenant_services where tenant_id =? order by use_memory desc",uuid).Row()
row := t.DB.Raw("select sum(if (cur_status != 'closed' && cur_status != 'undeploy',container_cpu * replicas,0)) as use_cpu,sum(container_cpu*replicas) as cap_cpu,sum(if (cur_status != 'closed' && cur_status != 'undeploy',container_memory * replicas,0)) as use_memory,sum(container_memory*replicas) as cap_memory from tenant_services where tenant_id =? order by use_memory desc", uuid).Row()
var capCpu int
var useCpu int
var capMem int
var useMem int
row.Scan( &useCpu,&capCpu,&useMem,&capMem)
row.Scan(&useCpu, &capCpu, &useMem, &capMem)
res := make(map[string]interface{})
res["capcpu"] = capCpu
res["usecpu"] = useCpu
res["capmem"] = capMem
res["usemem"] = useMem
res["tenant"] =uuid
logrus.Infof("get tenant %s service resource :%v",res)
res["tenant"] = uuid
logrus.Infof("get tenant %s service resource :%v", res)
return res, nil
}
//GetServiceAliasByIDs 获取应用别名
func (t *TenantServicesDaoImpl) GetServiceAliasByIDs(uids []string) ([]*model.TenantServices, error) {
var services []*model.TenantServices
@ -702,6 +728,18 @@ type TenantServiceVolumeDaoImpl struct {
DB *gorm.DB
}
//GetAllVolumes 获取全部存储信息
func (t *TenantServiceVolumeDaoImpl) GetAllVolumes() ([]*model.TenantServiceVolume, error) {
var volumes []*model.TenantServiceVolume
if err := t.DB.Find(&volumes).Error; err != nil {
if err == gorm.ErrRecordNotFound {
return volumes, nil
}
return nil, err
}
return volumes, nil
}
//AddModel 添加应用挂载
func (t *TenantServiceVolumeDaoImpl) AddModel(mo model.Interface) error {
volume := mo.(*model.TenantServiceVolume)
@ -716,7 +754,7 @@ func (t *TenantServiceVolumeDaoImpl) AddModel(mo model.Interface) error {
return nil
}
//UpdateModel 更新应用挂载
//UpdateModel 更<EFBFBD><EFBFBD>应用挂载
func (t *TenantServiceVolumeDaoImpl) UpdateModel(mo model.Interface) error {
volume := mo.(*model.TenantServiceVolume)
if volume.ID == 0 {

1
pkg/db/mysql/mysql.go
View File

@ -176,6 +176,7 @@ func (m *Manager) patchTable() {
m.db.Exec("insert into region_api_class VALUES ('','','node_manager','/v2/taskgroups','','','')")
m.db.Exec("insert into region_api_class VALUES ('','','node_manager','/v2/tasktemps','','','')")
m.db.Exec("insert into region_api_class VALUES ('','','node_manager','/v2/configs','','','')")
m.db.Exec("insert into region_api_class VALUES ('','','server_source','/v2/builder','','','')")
}
}

16
pkg/entrance/plugin/nginx/nginx_plugin.go
View File

@ -121,7 +121,8 @@ func (n *nginxAPI) AddNode(nodes ...*object.NodeObject) error {
var nodeList []string
nodeList = append(nodeList, fmt.Sprintf("%s:%d", node.Host, node.Port))
sns.NodeList = nodeList
if node.Protocol == "stream" {
//TODO: "stream" to !http
if node.Protocol != "http" {
logrus.Debugf("node protocol --stream %s", node.PoolName)
if !n.addStreamNode(&sns) {
errs = append(errs, errors.New("addPoolNode strem error"))
@ -162,7 +163,8 @@ func (n *nginxAPI) DeleteNode(nodes ...*object.NodeObject) error {
nodeList = append(nodeList, fmt.Sprintf("%s:%d", node.Host, node.Port))
logrus.Debugf("in DeleteNode nodelist is %v", nodeList)
sns.NodeList = nodeList
if node.Protocol == "stream" {
//TODO: "stream" to !http
if node.Protocol != "http" {
if !n.deleteStreamNode(&sns) {
errs = append(errs, errors.New("addPoolNode error"))
}
@ -462,14 +464,14 @@ func (n *nginxAPI) addDomain(ads *AddDomainS) bool {
if !bytes.HasPrefix([]byte(ads.Domain), []byte(fmt.Sprintf("%s.%s", p.Port, p.Servicename))) {
if ads.HTTPS && ads.CertificateName != "" {
httpsInfo := bytes.NewBuffer(nil)
httpsInfo.WriteString(`https=true`)
httpsInfo.WriteString(`https=https`)
httpsInfo.WriteString(fmt.Sprintf(`&cert_name=%s&`, ads.CertificateName))
httpsInfo.WriteString(string(upstream))
logrus.Debugf("https info is %v", string(httpsInfo.Bytes()))
pha.UpStream = httpsInfo.Bytes()
} else if ads.TransferHTTP && ads.CertificateName != "" {
httpsInfo := bytes.NewBuffer(nil)
httpsInfo.WriteString(`tran_https=true`)
httpsInfo.WriteString(`https=tran_https`)
httpsInfo.WriteString(fmt.Sprintf(`&cert_name=%s&`, ads.CertificateName))
httpsInfo.WriteString(string(upstream))
logrus.Debugf("trans https info is %v", string(httpsInfo.Bytes()))
@ -799,8 +801,10 @@ func (n *nginxAPI) pHTTPSCert(ssl *SSLCert, errs []error) []error {
certInfo := bytes.NewBuffer(nil)
certInfo.WriteString(fmt.Sprintf(`cert_name=%s`, ssl.CertName))
if ssl.HTTPMethod == MethodPOST {
certInfo.WriteString(fmt.Sprintf(`&ca=%s`, ssl.CA))
certInfo.WriteString(fmt.Sprintf(`&key=%s`, ssl.Key))
transCA := strings.Replace(ssl.CA, "+", "%2B", -1)
transKey := strings.Replace(ssl.Key, "+", "%2B", -1)
certInfo.WriteString(fmt.Sprintf(`&ca=%s`, transCA))
certInfo.WriteString(fmt.Sprintf(`&key=%s`, transKey))
}
logrus.Debugf("cert info is %v", string(certInfo.Bytes()))
resp, err := n.urlPPAction(ssl.HTTPMethod, url, certInfo.Bytes())

39
pkg/entrance/plugin/zeus/model.go
View File

@ -1,29 +1,29 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package zeus
import (
"github.com/goodrain/rainbond/pkg/entrance/core/object"
"bytes"
"encoding/json"
"fmt"
"github.com/goodrain/rainbond/pkg/entrance/core/object"
)
//Source zeus资源模型
@ -53,7 +53,8 @@ type Properties interface {
}
type PoolProperties struct {
Basic PoolBasic `json:"basic"`
Basic PoolBasic `json:"basic"`
Connection PoolConnection `json:"connection"`
}
type VSProperties struct {
Basic VSBasic `json:"basic"`
@ -87,17 +88,23 @@ type PoolBasic struct {
Note string `json:"note"`
}
type PoolConnection struct {
MaxReplyTime int `json:"max_reply_time"`
}
type VSBasic struct {
ListenONAny bool `json:"listen_on_any"`
ListenONHosts []string `json:"listen_on_hosts,omitempty"`
Note string `json:"note"`
DefaultPoolName string `json:"pool"`
Port int32 `json:"port"`
Enabled bool `json:"enabled"`
Protocol string `json:"protocol"` //stream http https 等等
RequestRules []string `json:"request_rules,omitempty"`
ResponseRules []string `json:"response_rules,omitempty"`
SSLDecrypt bool `json:"ssl_decrypt"`
AddXForwardedFor bool `json:"add_x_forwarded_for"`
ListenONAny bool `json:"listen_on_any"`
ListenONHosts []string `json:"listen_on_hosts,omitempty"`
Note string `json:"note"`
DefaultPoolName string `json:"pool"`
Port int32 `json:"port"`
Enabled bool `json:"enabled"`
Protocol string `json:"protocol"` //stream http https 等等
RequestRules []string `json:"request_rules,omitempty"`
ResponseRules []string `json:"response_rules,omitempty"`
SSLDecrypt bool `json:"ssl_decrypt"`
ConnectTimeout int `json:"connect_timeout"`
}
type ZeusNode struct {

37
pkg/entrance/plugin/zeus/zeus_plugin.go
View File

@ -1,19 +1,18 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
@ -26,8 +25,6 @@ package zeus
//4.需要缓存数据，可以从ctx.Store中获取
import (
"github.com/goodrain/rainbond/pkg/entrance/core/object"
"github.com/goodrain/rainbond/pkg/entrance/plugin"
"bytes"
"crypto/tls"
"errors"
@ -38,6 +35,9 @@ import (
"strings"
"time"
"github.com/goodrain/rainbond/pkg/entrance/core/object"
"github.com/goodrain/rainbond/pkg/entrance/plugin"
"github.com/Sirupsen/logrus"
"golang.org/x/net/context"
)
@ -300,6 +300,9 @@ func (z *zeus) AddPool(pools ...*object.PoolObject) error {
zeusSource := Source{
Properties: PoolProperties{
Basic: poolBasic,
Connection: PoolConnection{
MaxReplyTime: 100,
},
},
}
body, err := zeusSource.GetJSON()
@ -361,6 +364,9 @@ func (z *zeus) UpdatePool(pools ...*object.PoolObject) error {
zeusSource := Source{
Properties: PoolProperties{
Basic: poolBasic,
Connection: PoolConnection{
MaxReplyTime: 100,
},
},
}
body, err := zeusSource.GetJSON()
@ -557,19 +563,24 @@ func (z *zeus) closeSSl() VSssl {
func (z *zeus) UpdateVirtualService(services ...*object.VirtualServiceObject) error {
for _, vs := range services {
basic := VSBasic{
Note: vs.Note,
Port: vs.Port,
DefaultPoolName: vs.DefaultPoolName,
Enabled: true,
Note: vs.Note,
Port: vs.Port,
DefaultPoolName: vs.DefaultPoolName,
Enabled: true,
AddXForwardedFor: true,
ConnectTimeout: 300,
}
if vs.Name == "HTTPS.VS" {
basic.ConnectTimeout = 10
}
if vs.Listening == nil || len(vs.Listening) == 0 {
basic.ListenONAny = true
} else {
basic.ListenONHosts = vs.Listening
}
if vs.Protocol != "" {
basic.Protocol = vs.Protocol
} else {
if vs.Protocol == "udp" {
basic.Protocol = "udp"
} else if vs.Protocol != "http" {
basic.Protocol = "stream"
}
vsPro := VSProperties{

3
pkg/entrance/source/handle.go
View File

@ -466,7 +466,8 @@ func (m *Manager) serviceSource(services *v1.Service, method core.EventMethod) {
s.Domain = m.replaceDomain(s.Domain, s)
m.RcDomain(s)
logrus.Debugf("Fprotocol is %s", services.Labels["protocol"])
if services.Labels["protocol"] == "stream" {
//TODO: "stream" to !http
if services.Labels["protocol"] != "http" && services.Labels["protocol"] != "https" {
// event vs
m.RcVS(s)
} else {

10
pkg/event/manager.go
View File

@ -101,7 +101,9 @@ func GetManager() Manager {
//CloseManager 关闭日志服务
func CloseManager() {
defaultManager.Close()
if defaultManager != nil {
defaultManager.Close()
}
}
func (m *manager) Start() error {
@ -285,13 +287,15 @@ func (m *manager) RemoveHandle(server string) {
func (m *handle) HandleLog() error {
defer m.manager.RemoveHandle(m.server)
return util.Exec(m.ctx, func() error {
client, err := eventclient.NewEventClient(m.ctx, m.server)
ctx, cancel := context.WithCancel(m.ctx)
defer cancel()
client, err := eventclient.NewEventClient(ctx, m.server)
if err != nil {
logrus.Error("create event client error.", err.Error())
return err
}
logrus.Infof("start a event log handle core. connect server %s", m.server)
logClient, err := client.Log(m.ctx)
logClient, err := client.Log(ctx)
if err != nil {
logrus.Error("create event log client error.", err.Error())
//切换使用此chan的logger到其他chan

9
pkg/eventlog/cluster/connect/sub.go
View File

@ -1,19 +1,18 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
@ -170,7 +169,7 @@ func (s *Sub) listen(ins *dis.Instance) {
return err
}
if len(msgs) == 2 {
if string(msgs[0]) == string(db.EventMessage) || string(msgs[0]) == string(db.ServiceMonitorMessage) {
if string(msgs[0]) == string(db.EventMessage) || string(msgs[0]) == string(db.ServiceMonitorMessage) || string(msgs[0]) == string(db.ServiceNewMonitorMessage) {
s.subMessageChan <- msgs
} else if string(msgs[0]) == string(db.MonitorMessage) {
//s.log.Debug("Receive a monitor message ", string(msgs[1]))

19
pkg/eventlog/conf/conf.go
View File

@ -1,19 +1,18 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
@ -63,9 +62,10 @@ type LogConf struct {
}
type EntryConf struct {
EventLogServer EventLogServerConf
DockerLogServer DockerLogServerConf
MonitorMessageServer MonitorMessageServerConf
EventLogServer EventLogServerConf
DockerLogServer DockerLogServerConf
MonitorMessageServer MonitorMessageServerConf
NewMonitorMessageServerConf NewMonitorMessageServerConf
}
type EventLogServerConf struct {
@ -126,3 +126,8 @@ type MonitorMessageServerConf struct {
SubSubscribe string
CacheMessageSize int
}
type NewMonitorMessageServerConf struct {
ListenerHost string
ListenerPort int
}

9
pkg/eventlog/db/model.go
View File

@ -1,19 +1,18 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
@ -44,6 +43,8 @@ const (
EventMessage ClusterMessageType = "event_log"
//ServiceMonitorMessage 业务监控数据消息
ServiceMonitorMessage ClusterMessageType = "monitor_message"
//ServiceNewMonitorMessage 新业务监控数据消息
ServiceNewMonitorMessage ClusterMessageType = "new_monitor_message"
//MonitorMessage 节点监控数据
MonitorMessage ClusterMessageType = "monitor"
)

105
pkg/eventlog/entry/new_monitor_message_server.go Normal file
View File

@ -0,0 +1,105 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package entry
import (
"errors"
"fmt"
"net"
"time"
"github.com/goodrain/rainbond/pkg/eventlog/conf"
"github.com/goodrain/rainbond/pkg/eventlog/store"
"golang.org/x/net/context"
"sync"
"github.com/Sirupsen/logrus"
)
//NMonitorMessageServer 新性能分析实时数据接受服务
type NMonitorMessageServer struct {
conf conf.NewMonitorMessageServerConf
log *logrus.Entry
cancel func()
context context.Context
storemanager store.Manager
messageChan chan []byte
listenErr chan error
serverLock sync.Mutex
stopReceiveMessage bool
listener *net.UDPConn
}
//NewNMonitorMessageServer 创建UDP服务端
func NewNMonitorMessageServer(conf conf.NewMonitorMessageServerConf, log *logrus.Entry, storeManager store.Manager) (*NMonitorMessageServer, error) {
ctx, cancel := context.WithCancel(context.Background())
s := &NMonitorMessageServer{
conf: conf,
log: log,
cancel: cancel,
context: ctx,
storemanager: storeManager,
listenErr: make(chan error),
}
listener, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP(conf.ListenerHost), Port: conf.ListenerPort})
if err != nil {
fmt.Println(err)
return nil, err
}
log.Infof("UDP Server Listener: %s", listener.LocalAddr().String())
s.listener = listener
s.messageChan = s.storemanager.NewMonitorMessageChan()
if s.messageChan == nil {
return nil, errors.New("receive monitor message server can not get store message chan ")
}
return s, nil
}
//Serve 执行
func (s *NMonitorMessageServer) Serve() {
s.handleMessage()
}
//Stop 停止
func (s *NMonitorMessageServer) Stop() {
s.cancel()
s.log.Info("receive new monitor message server stop")
}
func (s *NMonitorMessageServer) handleMessage() {
buf := make([]byte, 65535)
defer s.listener.Close()
s.log.Infoln("start receive monitor message by udp")
for {
n, _, err := s.listener.ReadFromUDP(buf)
if err != nil {
logrus.Errorf("read new monitor message from udp error,%s", err.Error())
time.Sleep(time.Second * 2)
continue
}
s.messageChan <- buf[0:n]
}
}
//ListenError listen error chan
func (s *NMonitorMessageServer) ListenError() chan error {
return s.listenErr
}

13
pkg/eventlog/entry/server.go
View File

@ -1,19 +1,18 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
@ -63,9 +62,15 @@ func (e *Entry) Start() error {
if err != nil {
return err
}
newmonitorServer, err := NewNMonitorMessageServer(e.conf.NewMonitorMessageServerConf, e.log.WithField("server", "NewMonitorMessage"), e.storeManager)
if err != nil {
return err
}
supervisor.Add(eventServer)
supervisor.Add(dockerServer)
supervisor.Add(monitorServer)
supervisor.Add(newmonitorServer)
supervisor.ServeBackground()
e.supervisor = supervisor
return nil

100
pkg/eventlog/exit/web/manager.go
View File

@ -1,19 +1,18 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
@ -21,14 +20,15 @@ package web
import (
"encoding/json"
"io/ioutil"
"net/http"
"time"
"github.com/goodrain/rainbond/pkg/eventlog/cluster"
"github.com/goodrain/rainbond/pkg/eventlog/cluster/discover"
"github.com/goodrain/rainbond/pkg/eventlog/conf"
"github.com/goodrain/rainbond/pkg/eventlog/exit/monitor"
"github.com/goodrain/rainbond/pkg/eventlog/store"
"io/ioutil"
"net/http"
"time"
"golang.org/x/net/context"
@ -210,6 +210,7 @@ func (s *SocketServer) pushDockerLog(w http.ResponseWriter, r *http.Request) {
s.log.Debug("Push docker log message request closed")
s.storemanager.RealseWebSocketMessageChan("docker", ServiceID, SubID)
}()
conn.WriteMessage(websocket.TextMessage, []byte("ok"))
stop := make(chan struct{})
go s.reader(conn, stop)
pingTicker := time.NewTicker(s.timeout * 8 / 10)
@ -292,6 +293,7 @@ func (s *SocketServer) pushMonitorMessage(w http.ResponseWriter, r *http.Request
s.log.Debug("Push docker log message request closed")
s.storemanager.RealseWebSocketMessageChan("monitor", ServiceID, SubID)
}()
conn.WriteMessage(websocket.TextMessage, []byte("ok"))
stop := make(chan struct{})
go s.reader(conn, stop)
pingTicker := time.NewTicker(s.timeout * 8 / 10)
@ -323,6 +325,89 @@ func (s *SocketServer) pushMonitorMessage(w http.ResponseWriter, r *http.Request
}
}
}
func (s *SocketServer) pushNewMonitorMessage(w http.ResponseWriter, r *http.Request) {
// if r.FormValue("host") == "" || r.FormValue("host") != s.cluster.GetInstanceID() {
// w.WriteHeader(404)
// return
// }
upgrader := websocket.Upgrader{
ReadBufferSize: s.conf.ReadBufferSize,
WriteBufferSize: s.conf.WriteBufferSize,
EnableCompression: s.conf.EnableCompression,
Error: func(w http.ResponseWriter, r *http.Request, status int, reason error) {
},
CheckOrigin: func(r *http.Request) bool {
return true
},
}
conn, err := upgrader.Upgrade(w, r, nil)
if err != nil {
s.log.Error("Create web socket conn error.", err.Error())
return
}
defer conn.Close()
_, me, err := conn.ReadMessage()
if err != nil {
s.log.Error("Read tag key from first message error.", err.Error())
return
}
info := strings.Split(string(me), "=")
if len(info) != 2 {
s.log.Error("Read tag key from first message error. The data format is not correct")
return
}
ServiceID := info[1]
if ServiceID == "" {
s.log.Error("tag key can not be empty when get socket message")
return
}
s.log.Infof("Begin push monitor message of service (%s)", ServiceID)
SubID := uuid.NewV4().String()
ch := s.storemanager.WebSocketMessageChan("newmonitor", ServiceID, SubID)
if ch == nil {
// w.Write([]byte("Real-time message does not exist."))
// w.Header().Set("Status Code", "200")
s.log.Error("get web socket message chan from storemanager error.")
return
}
defer func() {
s.log.Debug("Push new monitor message request closed")
s.storemanager.RealseWebSocketMessageChan("newmonitor", ServiceID, SubID)
}()
conn.WriteMessage(websocket.TextMessage, []byte("ok"))
stop := make(chan struct{})
go s.reader(conn, stop)
pingTicker := time.NewTicker(s.timeout * 8 / 10)
defer pingTicker.Stop()
for {
select {
case message, ok := <-ch:
if !ok {
return
}
if message != nil {
s.log.Debugf("websocket push a new monitor message")
conn.SetWriteDeadline(time.Now().Add(10 * time.Second))
err = conn.WriteMessage(websocket.TextMessage, message.MonitorData)
if err != nil {
s.log.Warn("Push message to client error.", err.Error())
return
}
}
case <-stop:
return
case <-s.context.Done():
return
case <-pingTicker.C:
conn.SetWriteDeadline(time.Now().Add(10 * time.Second))
if err := conn.WriteMessage(websocket.PingMessage, []byte{}); err != nil {
return
}
}
}
}
func (s *SocketServer) reader(ws *websocket.Conn, ch chan struct{}) {
defer ws.Close()
@ -350,6 +435,7 @@ func (s *SocketServer) listen() {
http.HandleFunc("/event_log", s.pushEventMessage)
http.HandleFunc("/docker_log", s.pushDockerLog)
http.HandleFunc("/monitor_message", s.pushMonitorMessage)
http.HandleFunc("/new_monitor_message", s.pushNewMonitorMessage)
http.HandleFunc("/monitor", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(200)
w.Write([]byte("ok"))

37
pkg/eventlog/store/barrel.go
View File

@ -1,19 +1,18 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
@ -128,7 +127,10 @@ func (r *readEventBarrel) insertMessage(message *db.EventLogMessage) {
r.subLock.Lock()
defer r.subLock.Unlock()
for _, v := range r.subSocketChan { //向订阅的通道发送消息
v <- message
select {
case v <- message:
default:
}
}
}
@ -136,7 +138,10 @@ func (r *readEventBarrel) pushCashMessage(ch chan *db.EventLogMessage, subID str
r.subLock.Lock()
defer r.subLock.Unlock()
for _, m := range r.barrel {
ch <- m
select {
case ch <- m:
default:
}
}
r.subSocketChan[subID] = ch
}
@ -199,7 +204,10 @@ func (r *dockerLogEventBarrel) insertMessage(message *db.EventLogMessage) {
r.barrel = append(r.barrel, message)
r.updateTime = time.Now()
for _, v := range r.subSocketChan { //向订阅的通道发送消息
v <- message
select {
case v <- message:
default:
}
}
r.size++
if int64(len(r.barrel)) >= r.cacheSize {
@ -211,7 +219,10 @@ func (r *dockerLogEventBarrel) pushCashMessage(ch chan *db.EventLogMessage, subI
r.subLock.Lock()
defer r.subLock.Unlock()
for _, m := range r.barrel {
ch <- m
select {
case ch <- m:
default:
}
}
r.subSocketChan[subID] = ch
}
@ -285,7 +296,10 @@ func (r *monitorMessageBarrel) insertMessage(message *db.EventLogMessage) {
r.subLock.Lock()
defer r.subLock.Unlock()
for _, v := range r.subSocketChan { //向订阅的通道发送消息
v <- message
select {
case v <- message:
default:
}
}
}
@ -293,7 +307,10 @@ func (r *monitorMessageBarrel) pushCashMessage(ch chan *db.EventLogMessage, subI
r.subLock.Lock()
defer r.subLock.Unlock()
for _, m := range r.barrel {
ch <- m
select {
case ch <- m:
default:
}
}
r.subSocketChan[subID] = ch
}

115
pkg/eventlog/store/manager.go
View File

@ -1,19 +1,18 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
@ -52,6 +51,7 @@ type Manager interface {
DockerLogMessageChan() chan []byte
MonitorMessageChan() chan [][]byte
WebSocketMessageChan(mode, eventID, subID string) chan *db.EventLogMessage
NewMonitorMessageChan() chan []byte
RealseWebSocketMessageChan(mode, EventID, subID string)
Run() error
Stop()
@ -74,48 +74,53 @@ func NewManager(conf conf.EventStoreConf, log *logrus.Entry) (Manager, error) {
return nil, err
}
storeManager := &storeManager{
cancel: cancel,
context: ctx,
conf: conf,
log: log,
receiveChan: make(chan []byte, 300),
subChan: make(chan [][]byte, 300),
pubChan: make(chan [][]byte, 300),
dockerLogChan: make(chan []byte, 2048),
monitorMessageChan: make(chan [][]byte, 100),
chanCacheSize: 100,
dbPlugin: dbPlugin,
filePlugin: filePlugin,
errChan: make(chan error),
cancel: cancel,
context: ctx,
conf: conf,
log: log,
receiveChan: make(chan []byte, 300),
subChan: make(chan [][]byte, 300),
pubChan: make(chan [][]byte, 300),
dockerLogChan: make(chan []byte, 2048),
monitorMessageChan: make(chan [][]byte, 100),
newmonitorMessageChan: make(chan []byte, 2048),
chanCacheSize: 100,
dbPlugin: dbPlugin,
filePlugin: filePlugin,
errChan: make(chan error),
}
handle := NewStore("handle", storeManager)
read := NewStore("read", storeManager)
docker := NewStore("docker_log", storeManager)
monitor := NewStore("monitor", storeManager)
newmonitor := NewStore("newmonitor", storeManager)
storeManager.handleMessageStore = handle
storeManager.readMessageStore = read
storeManager.dockerLogStore = docker
storeManager.monitorMessageStore = monitor
storeManager.newmonitorMessageStore = newmonitor
return storeManager, nil
}
type storeManager struct {
cancel func()
context context.Context
handleMessageStore MessageStore
readMessageStore MessageStore
dockerLogStore MessageStore
monitorMessageStore MessageStore
receiveChan chan []byte
pubChan, subChan chan [][]byte
dockerLogChan chan []byte
monitorMessageChan chan [][]byte
chanCacheSize int
conf conf.EventStoreConf
log *logrus.Entry
dbPlugin db.Manager
filePlugin db.Manager
errChan chan error
cancel func()
context context.Context
handleMessageStore MessageStore
readMessageStore MessageStore
dockerLogStore MessageStore
monitorMessageStore MessageStore
newmonitorMessageStore MessageStore
receiveChan chan []byte
pubChan, subChan chan [][]byte
dockerLogChan chan []byte
monitorMessageChan chan [][]byte
newmonitorMessageChan chan []byte
chanCacheSize int
conf conf.EventStoreConf
log *logrus.Entry
dbPlugin db.Manager
filePlugin db.Manager
errChan chan error
}
//Scrape prometheue monitor metrics
@ -188,6 +193,12 @@ func (s *storeManager) MonitorMessageChan() chan [][]byte {
}
return s.monitorMessageChan
}
func (s *storeManager) NewMonitorMessageChan() chan []byte {
if s.newmonitorMessageChan == nil {
s.newmonitorMessageChan = make(chan []byte, 2048)
}
return s.newmonitorMessageChan
}
func (s *storeManager) WebSocketMessageChan(mode, eventID, subID string) chan *db.EventLogMessage {
if mode == "event" {
@ -202,6 +213,10 @@ func (s *storeManager) WebSocketMessageChan(mode, eventID, subID string) chan *d
ch := s.monitorMessageStore.SubChan(eventID, subID)
return ch
}
if mode == "newmonitor" {
ch := s.newmonitorMessageStore.SubChan(eventID, subID)
return ch
}
return nil
}
@ -211,6 +226,7 @@ func (s *storeManager) Run() error {
s.readMessageStore.Run()
s.dockerLogStore.Run()
s.monitorMessageStore.Run()
s.newmonitorMessageStore.Run()
for i := 0; i < s.conf.HandleMessageCoreNumber; i++ {
go s.handleReceiveMessage()
}
@ -223,6 +239,7 @@ func (s *storeManager) Run() error {
for i := 0; i < s.conf.HandleMessageCoreNumber; i++ {
go s.handleMonitorMessage()
}
go s.handleNewMonitorMessage()
return nil
}
@ -249,6 +266,33 @@ func (s *storeManager) parsingMessage(msg []byte, messageType string) (*db.Event
}
return nil, errors.New("Unable to process configuration of message format type.")
}
//handleNewMonitorMessage 处理新监控数据
func (s *storeManager) handleNewMonitorMessage() {
loop:
for {
select {
case <-s.context.Done():
return
case msg, ok := <-s.newmonitorMessageChan:
if !ok {
s.log.Error("handle new monitor message core stop.monitor message log chan closed")
break loop
}
if msg == nil {
continue
}
//s.log.Debugf("receive message %s", string(message.Content))
if s.conf.ClusterMode {
//消息直接集群共享
s.pubChan <- [][]byte{[]byte(db.ServiceNewMonitorMessage), msg}
}
s.newmonitorMessageStore.InsertMessage(&db.EventLogMessage{MonitorData: msg})
}
}
s.errChan <- fmt.Errorf("handle monitor log core exist")
}
func (s *storeManager) handleReceiveMessage() {
s.log.Debug("event message store manager start handle receive message")
loop:
@ -301,6 +345,10 @@ func (s *storeManager) handleSubMessage() {
continue
}
if len(msg) == 2 {
if string(msg[0]) == string(db.ServiceNewMonitorMessage) {
s.newmonitorMessageStore.InsertMessage(&db.EventLogMessage{MonitorData: msg[1]})
continue
}
//s.log.Debugf("receive sub message %s", string(msg))
message, err := s.parsingMessage(msg[1], s.conf.MessageType)
if err != nil {
@ -313,6 +361,7 @@ func (s *storeManager) handleSubMessage() {
if string(msg[0]) == string(db.ServiceMonitorMessage) {
s.monitorMessageStore.InsertMessage(message)
}
}
}
}

385
pkg/eventlog/store/new_monitor_message_store.go Normal file
View File

@ -0,0 +1,385 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package store
import (
"context"
"math"
"sort"
"sync"
"time"
"github.com/Sirupsen/logrus"
"github.com/pquerna/ffjson/ffjson"
"github.com/prometheus/client_golang/prometheus"
"github.com/goodrain/rainbond/pkg/eventlog/conf"
"github.com/goodrain/rainbond/pkg/eventlog/db"
)
type newMonitorMessageStore struct {
conf conf.EventStoreConf
log *logrus.Entry
barrels map[string]*CacheMonitorMessageList
lock sync.RWMutex
cancel func()
ctx context.Context
size int64
allLogCount float64
}
func (h *newMonitorMessageStore) Scrape(ch chan<- prometheus.Metric, namespace, exporter, from string) error {
chanDesc := prometheus.NewDesc(
prometheus.BuildFQName(namespace, exporter, "new_monitor_store_barrel_count"),
"the handle container log count size.",
[]string{"from"}, nil,
)
ch <- prometheus.MustNewConstMetric(chanDesc, prometheus.GaugeValue, float64(len(h.barrels)), from)
logDesc := prometheus.NewDesc(
prometheus.BuildFQName(namespace, exporter, "new_monitor_store_log_count"),
"the handle monitor log count size.",
[]string{"from"}, nil,
)
ch <- prometheus.MustNewConstMetric(logDesc, prometheus.GaugeValue, h.allLogCount, from)
return nil
}
func (h *newMonitorMessageStore) insertMessage(message *db.EventLogMessage) ([]MonitorMessage, bool) {
h.lock.RLock()
defer h.lock.RUnlock()
mm := fromByte(message.MonitorData)
if len(mm) < 1 {
return mm, true
}
if mm[0].ServiceID == "" {
return mm, true
}
if ba, ok := h.barrels[mm[0].ServiceID]; ok {
ba.Insert(mm...)
return mm, true
}
return mm, false
}
func (h *newMonitorMessageStore) InsertMessage(message *db.EventLogMessage) {
if message == nil {
return
}
//h.log.Debug("Receive a monitor message:" + string(message.Content))
h.size++
h.allLogCount++
mm, ok := h.insertMessage(message)
if ok {
return
}
h.lock.Lock()
defer h.lock.Unlock()
ba := CreateCacheMonitorMessageList(mm[0].ServiceID)
ba.Insert(mm...)
h.barrels[mm[0].ServiceID] = ba
}
func (h *newMonitorMessageStore) GetMonitorData() *db.MonitorData {
data := &db.MonitorData{
ServiceSize: len(h.barrels),
LogSizePeerM: h.size,
}
return data
}
func (h *newMonitorMessageStore) SubChan(eventID, subID string) chan *db.EventLogMessage {
h.lock.Lock()
defer h.lock.Unlock()
if ba, ok := h.barrels[eventID]; ok {
return ba.addSubChan(subID)
}
ba := CreateCacheMonitorMessageList(eventID)
h.barrels[eventID] = ba
return ba.addSubChan(subID)
}
func (h *newMonitorMessageStore) RealseSubChan(eventID, subID string) {
h.lock.RLock()
defer h.lock.RUnlock()
if ba, ok := h.barrels[eventID]; ok {
ba.delSubChan(subID)
}
}
func (h *newMonitorMessageStore) Run() {
go h.Gc()
}
func (h *newMonitorMessageStore) Gc() {
tiker := time.NewTicker(time.Second * 30)
for {
select {
case <-tiker.C:
case <-h.ctx.Done():
h.log.Debug("read message store gc stop.")
tiker.Stop()
return
}
h.size = 0
if len(h.barrels) == 0 {
continue
}
var gcEvent []string
for k, v := range h.barrels {
if v.UpdateTime.Add(time.Minute * 3).Before(time.Now()) { // barrel 超时未收到消息
gcEvent = append(gcEvent, k)
}
}
if gcEvent != nil && len(gcEvent) > 0 {
for _, id := range gcEvent {
barrel := h.barrels[id]
barrel.empty()
delete(h.barrels, id)
}
}
}
}
func (h *newMonitorMessageStore) stop() {
h.cancel()
}
func (h *newMonitorMessageStore) InsertGarbageMessage(message ...*db.EventLogMessage) {}
//MonitorMessage 性能监控消息系统模型
type MonitorMessage struct {
ServiceID string
Port string
HostName string
MessageType string //mysql，http ...
Key string
//总时间
CumulativeTime float64
AverageTime float64
MaxTime float64
Count uint64
//异常请求次数
AbnormalCount uint64
}
//cacheMonitorMessage 每个实例的数据缓存
type cacheMonitorMessage struct {
updateTime time.Time
hostName string
mms MonitorMessageList
}
//CacheMonitorMessageList 某个应用性能分析数据
type CacheMonitorMessageList struct {
list []*cacheMonitorMessage
subSocketChan map[string]chan *db.EventLogMessage
subLock sync.Mutex
message db.EventLogMessage
UpdateTime time.Time
}
//CreateCacheMonitorMessageList 创建应用监控信息缓存器
func CreateCacheMonitorMessageList(eventID string) *CacheMonitorMessageList {
return &CacheMonitorMessageList{
subSocketChan: make(map[string]chan *db.EventLogMessage),
message: db.EventLogMessage{
EventID: eventID,
},
}
}
//Insert 认为mms的hostname一致
//每次收到消息进行gc
func (c *CacheMonitorMessageList) Insert(mms ...MonitorMessage) {
if mms == nil || len(mms) < 1 {
return
}
c.UpdateTime = time.Now()
hostname := mms[0].HostName
if len(c.list) == 0 {
c.list = []*cacheMonitorMessage{
&cacheMonitorMessage{
updateTime: time.Now(),
hostName: hostname,
mms: mms,
}}
}
var update bool
for i := range c.list {
cm := c.list[i]
if cm.hostName == hostname {
cm.updateTime = time.Now()
cm.mms = mms
update = true
break
}
}
if !update {
c.list = append(c.list, &cacheMonitorMessage{
updateTime: time.Now(),
hostName: hostname,
mms: mms,
})
}
c.Gc()
c.pushMessage()
}
//Gc 清理数据
func (c *CacheMonitorMessageList) Gc() {
var list []*cacheMonitorMessage
for i := range c.list {
cmm := c.list[i]
if !cmm.updateTime.Add(time.Second * 30).Before(time.Now()) {
list = append(list, cmm)
}
}
c.list = list
}
func (c *CacheMonitorMessageList) pushMessage() {
if len(c.list) == 0 {
return
}
var mdata []byte
if len(c.list) == 1 {
mdata = getByte(c.list[0].mms)
}
source := c.list[0].mms
for i := 1; i < len(c.list); i++ {
addSource := c.list[i].mms
source = merge(source, addSource)
}
//降序排序
sort.Sort(sort.Reverse(&source))
mdata = getByte(*source.Pop(20))
c.message.MonitorData = mdata
for _, ch := range c.subSocketChan {
select {
case ch <- &c.message:
default:
}
}
}
// 增加socket订阅
func (c *CacheMonitorMessageList) addSubChan(subID string) chan *db.EventLogMessage {
c.subLock.Lock()
defer c.subLock.Unlock()
if sub, ok := c.subSocketChan[subID]; ok {
return sub
}
ch := make(chan *db.EventLogMessage, 10)
c.subSocketChan[subID] = ch
c.pushMessage()
return ch
}
//删除socket订阅
func (c *CacheMonitorMessageList) delSubChan(subID string) {
c.subLock.Lock()
defer c.subLock.Unlock()
if _, ok := c.subSocketChan[subID]; ok {
delete(c.subSocketChan, subID)
}
}
func (c *CacheMonitorMessageList) empty() {
c.subLock.Lock()
defer c.subLock.Unlock()
for _, v := range c.subSocketChan {
close(v)
}
}
func getByte(source []MonitorMessage) []byte {
b, _ := ffjson.Marshal(source)
return b
}
func fromByte(source []byte) []MonitorMessage {
var mm []MonitorMessage
ffjson.Unmarshal(source, &mm)
return mm
}
func merge(source, addsource MonitorMessageList) (result MonitorMessageList) {
var cache = make(map[string]MonitorMessage)
for _, mm := range source {
cache[mm.Key] = mm
}
for _, mm := range addsource {
if oldmm, ok := cache[mm.Key]; ok {
oldmm.Count += mm.Count
oldmm.AbnormalCount += mm.AbnormalCount
//平均时间
oldmm.AverageTime = Round((oldmm.AverageTime+mm.AverageTime)/2, 2)
//<2F><><EFBFBD><EFBFBD><EFBFBD>积<EFBFBD><E7A7AF><EFBFBD>间
oldmm.CumulativeTime = Round(oldmm.CumulativeTime+mm.CumulativeTime, 2)
//最大时间
if mm.MaxTime > oldmm.MaxTime {
oldmm.MaxTime = mm.MaxTime
}
cache[mm.Key] = oldmm
continue
}
cache[mm.Key] = mm
}
for _, c := range cache {
result.Add(&c)
}
return
}
//Round Round
func Round(f float64, n int) float64 {
pow10n := math.Pow10(n)
return math.Trunc((f+0.5/pow10n)*pow10n) / pow10n
}
//MonitorMessageList 消息列表
type MonitorMessageList []MonitorMessage
//Add 添加
func (m *MonitorMessageList) Add(mm *MonitorMessage) {
*m = append(*m, *mm)
}
// Len 为集合内元素的总数
func (m *MonitorMessageList) Len() int {
return len(*m)
}
//Less 如果index为i的元素小于index为j的元素，则返回true，否则返回false
func (m *MonitorMessageList) Less(i, j int) bool {
return (*m)[i].CumulativeTime < (*m)[j].CumulativeTime
}
// Swap 交换索引为 i 和 j 的元素
func (m *MonitorMessageList) Swap(i, j int) {
tmp := (*m)[i]
(*m)[i] = (*m)[j]
(*m)[j] = tmp
}
//Pop Pop
func (m *MonitorMessageList) Pop(i int) *MonitorMessageList {
if len(*m) <= i {
return m
}
cache := (*m)[:i]
return &cache
}
//String json string
func (m *MonitorMessageList) String() string {
body, _ := ffjson.Marshal(m)
return string(body)
}

38
pkg/eventlog/store/new_monitor_message_store_test.go Normal file
View File

@ -0,0 +1,38 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package store
import "testing"
func TestMerge(t *testing.T) {
s1 := MonitorMessageList{
MonitorMessage{
Key: "/asdadasd",
},
}
s2 := MonitorMessageList{
MonitorMessage{
Key: "/asdadasd",
},
MonitorMessage{
Key: "/asda12dasd",
},
}
re := merge(s1, s2)
t.Log(re)
}

18
pkg/eventlog/store/store.go
View File

@ -1,19 +1,18 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
@ -141,5 +140,16 @@ func NewStore(storeType string, manager *storeManager) MessageStore {
}
return docker
}
if storeType == "newmonitor" {
monitor := &newMonitorMessageStore{
barrels: make(map[string]*CacheMonitorMessageList, 100),
conf: manager.conf,
log: manager.log.WithField("module", "NewMonitorMessageStore"),
ctx: ctx,
cancel: cancel,
}
return monitor
}
return nil
}

28
pkg/eventlog/test/monitorserver/monitorserver_test.go
View File

@ -1,19 +1,18 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
@ -29,6 +28,27 @@ import (
var urlData = `
2017-05-19 11:33:34 APPS SumTimeByUrl [{"tenant":"o2o","service":"zzcplus","url":"/active/js/wx_share.js","avgtime":"1.453","sumtime":"1.453","counts":"1"}]
`
var newMonitorMessage = `
[{"ServiceID":"test",
"Port":"5000",
"MessageType":"http",
"Key":"/test",
"CumulativeTime":0.1,
"AverageTime":0.1,
"MaxTime":0.1,
"Count":1,
"AbnormalCount":0}
,{"ServiceID":"test",
"Port":"5000",
"MessageType":"http",
"Key":"/test2",
"CumulativeTime":0.36,
"AverageTime":0.18,
"MaxTime":0.2,
"Count":2,
"AbnormalCount":2}
]
`
func BenchmarkMonitorServer(t *testing.B) {
client, _ := zmq4.NewSocket(zmq4.PUB)

50
pkg/node/api/manager.go
View File

@ -25,6 +25,9 @@ import (
"github.com/goodrain/rainbond/pkg/discover"
"github.com/goodrain/rainbond/pkg/node/masterserver"
"github.com/goodrain/rainbond/pkg/node/statsd"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/goodrain/rainbond/pkg/node/api/controller"
"github.com/goodrain/rainbond/pkg/node/api/model"
@ -52,21 +55,52 @@ type Manager struct {
lID client.LeaseID // lease id
ms *masterserver.MasterServer
keepalive *discover.KeepAlive
exporter *statsd.Exporter
}
//NewManager api manager
func NewManager(c option.Conf, node *model.HostNode, ms *masterserver.MasterServer) *Manager {
func NewManager(c option.Conf, node *model.HostNode, ms *masterserver.MasterServer, exporter *statsd.Exporter) *Manager {
r := router.Routers(c.RunMode)
ctx, cancel := context.WithCancel(context.Background())
controller.Init(&c, ms)
return &Manager{
ctx: ctx,
cancel: cancel,
conf: c,
router: r,
node: node,
ms: ms,
m := &Manager{
ctx: ctx,
cancel: cancel,
conf: c,
router: r,
node: node,
ms: ms,
exporter: exporter,
}
m.router.Get("/app/metrics", m.HandleStatsd)
m.router.Get("/-/statsdreload", m.ReloadStatsdMappConfig)
return m
}
//ReloadStatsdMappConfig ReloadStatsdMappConfig
func (m *Manager) ReloadStatsdMappConfig(w http.ResponseWriter, r *http.Request) {
if err := m.exporter.ReloadConfig(); err != nil {
w.Write([]byte(err.Error()))
w.WriteHeader(500)
} else {
w.Write([]byte("Success reload"))
w.WriteHeader(200)
}
}
//HandleStatsd statsd handle
func (m *Manager) HandleStatsd(w http.ResponseWriter, r *http.Request) {
gatherers := prometheus.Gatherers{
prometheus.DefaultGatherer,
m.exporter.GetRegister(),
}
// Delegate http serving to Prometheus client library, which will call collector.Collect.
h := promhttp.HandlerFor(gatherers,
promhttp.HandlerOpts{
ErrorLog: logrus.StandardLogger(),
ErrorHandling: promhttp.ContinueOnError,
})
h.ServeHTTP(w, r)
}
//Start 启动

17
pkg/node/api/model/discover.go
View File

@ -127,7 +127,13 @@ type RouteConfig struct {
type PieceHTTPVirtualHost struct {
Name string `json:"name"`
Domains []string `json:"domains"`
Routes []*PieceHTTPRoutes `json:"routes"`
//Routes []*PieceHTTPRoutes `json:"routes"`
Routes interface{} `json:"routes"`
}
//WeightedClusters WeightedClusters
type WeightedClusters struct {
Clusters []map[string]interface{} `json:"clusters"`
}
//PieceHTTPRoutes PieceHTTPRoutes
@ -135,7 +141,8 @@ type PieceHTTPRoutes struct {
TimeoutMS int `json:"timeout_ms"`
Prefix string `json:"prefix"`
Cluster string `json:"cluster"`
//Headers []*PieceHeader `json:"headers"`
//Headers []PieceHeader `json:"headers"`
Headers interface{} `json:"headers"`
}
//PieceHeader PieceHeader
@ -170,4 +177,10 @@ const (
UPSTREAM string = "upStream"
//DOWNSTREAM downStream
DOWNSTREAM string = "downStream"
//WEIGHT WEIGHT
WEIGHT string = "WEIGHT"
//MODELWEIGHT MODEL_WEIGHT
MODELWEIGHT string = "weight_model"
//MODELPREFIX MODEL_PREFIX
MODELPREFIX string = "prefix_model"
)

4
pkg/node/api/router/router.go
View File

@ -76,7 +76,6 @@ func Routers(mode string) *chi.Mux {
r.Get("/{node_ip}/init", controller.InitStatus)
r.Post("/{node_id}/install", controller.Install)
r.Get("/{node_id}/prometheus/cpu", controller.GetCpu)
r.Get("/{node_id}/prometheus/mem", controller.GetMem)
r.Get("/{node_id}/prometheus/disk", controller.GetDisk)
@ -88,7 +87,6 @@ func Routers(mode string) *chi.Mux {
r.Put("/prometheus/expr", controller.GetExpr)
r.Put("/prometheus/start/{start}/end/{end}/step/{step}/expr", controller.GetExpr)
})
//TODO:
@ -121,6 +119,6 @@ func Routers(mode string) *chi.Mux {
}
})
//节点监控
r.Get("/metrics", controller.NodeExporter)
r.Get("/node/metrics", controller.NodeExporter)
return r
}

40
pkg/node/core/config/datacenter.go
View File

@ -140,12 +140,48 @@ func (d *DataCenterConfig) GetConfig(name string) *model.ConfigUnit {
return d.config.Get(name)
}
//CacheConfig 更新配置缓存
func (d *DataCenterConfig) CacheConfig(c *model.ConfigUnit) error {
if c.Name == "" {
return fmt.Errorf("config name can not be empty")
}
logrus.Debugf("add config %v", c)
//将值类型由[]interface{} 转 []string
if c.ValueType == "array" {
switch c.Value.(type) {
case []interface{}:
var data []string
for _, v := range c.Value.([]interface{}) {
data = append(data, v.(string))
}
c.Value = data
}
oldC := d.config.Get(c.Name)
if oldC != nil {
switch oldC.Value.(type) {
case string:
value := append(c.Value.([]string), oldC.Value.(string))
util.Deweight(&value)
c.Value = value
case []string:
value := append(c.Value.([]string), oldC.Value.([]string)...)
util.Deweight(&value)
c.Value = value
default:
}
}
}
d.config.Add(*c)
return nil
}
//PutConfig 增加or更新配置
func (d *DataCenterConfig) PutConfig(c *model.ConfigUnit) error {
if c.Name == "" {
return fmt.Errorf("config name can not be empty")
}
logrus.Debugf("add config %v",c)
logrus.Debugf("add config %v", c)
//将值类型由[]interface{} 转 []string
if c.ValueType == "array" {
switch c.Value.(type) {
@ -186,7 +222,7 @@ func (d *DataCenterConfig) PutConfig(c *model.ConfigUnit) error {
func (d *DataCenterConfig) PutConfigKV(kv *mvccpb.KeyValue) {
var cn model.ConfigUnit
if err := ffjson.Unmarshal(kv.Value, &cn); err == nil {
d.PutConfig(&cn)
d.CacheConfig(&cn)
} else {
logrus.Errorf("parse config error,%s", err.Error())
}

192
pkg/node/core/service/discover_server.go
View File

@ -22,6 +22,7 @@ import (
"fmt"
"strconv"
"strings"
"reflect"
"github.com/Sirupsen/logrus"
"github.com/goodrain/rainbond/cmd/node/option"
@ -145,6 +146,7 @@ func (d *DiscoverAction) DiscoverListeners(
return nil, util.CreateAPIHandleError(500, fmt.Errorf(
"get env %s error: %v", namespace+serviceAlias+pluginID, err))
}
logrus.Debugf("process go on")
//TODO: console控制尽量不把小于1000的端口给用户使用
var vhL []*node_model.PieceHTTPVirtualHost
@ -195,6 +197,11 @@ func (d *DiscoverAction) DiscoverListeners(
}
if ok {
logrus.Debugf("port protocol is %s", portProtocol)
//TODO: resource protocol support
if portProtocol != "http" && portProtocol != "https" {
portProtocol = "stream"
}
logrus.Debugf("port protocol is %s after trans", portProtocol)
switch portProtocol {
case "stream":
ptr := &node_model.PieceTCPRoute{
@ -279,17 +286,25 @@ func (d *DiscoverAction) DiscoverListeners(
}
}
}
prs := &node_model.PieceHTTPRoutes{
TimeoutMS: 0,
Prefix: d.ToolsGetRouterItem(destServiceAlias, node_model.PREFIX, options).(string),
Cluster: fmt.Sprintf("%s_%s_%s_%d", namespace, serviceAlias, destServiceAlias, port),
//Headers: d.ToolsGetRouterItem(destServiceAlias,
// node_model.HEADERS, &sr).([]*node_model.PieceHeader),
headers := d.ToolsGetRouterItem(
destServiceAlias,
node_model.HEADERS, options)
prs := make(map[string]interface{})
prs["timeout_ms"] = 0
prs["prefix"] = d.ToolsGetRouterItem(destServiceAlias, node_model.PREFIX, options).(string)
c := make(map[string]interface{})
c["name"] = fmt.Sprintf("%s_%s_%s_%d", namespace, serviceAlias, destServiceAlias, port)
c["weight"] = d.ToolsGetRouterItem(destServiceAlias, node_model.WEIGHT, options).(int)
var wc node_model.WeightedClusters
wc.Clusters = []map[string]interface{}{c}
prs["weighted_clusters"] = wc
if len(headers.([]node_model.PieceHeader)) != 0 {
prs["headers"] = headers
}
pvh := &node_model.PieceHTTPVirtualHost{
Name: fmt.Sprintf("%s_%s_%s_%d", namespace, serviceAlias, destServiceAlias, port),
Domains: d.ToolsGetRouterItem(destServiceAlias, node_model.DOMAINS, options).([]string),
Routes: []*node_model.PieceHTTPRoutes{prs},
Routes: []map[string]interface{}{prs},
}
vhL = append(vhL, pvh)
}
@ -308,8 +323,62 @@ func (d *DiscoverAction) DiscoverListeners(
Name: "router",
Config: make(map[string]string),
}
var newVHL []*node_model.PieceHTTPVirtualHost
if len(vhL) > 1 {
domainL := d.CheckSameDomainAndPrefix(resources)
logrus.Debugf("domainL is %v", domainL)
if len(domainL) > 0 {
//存在相同的domain设置
for d := range domainL {
var c []map[string]interface{}
var r []interface{}
var pvh node_model.PieceHTTPVirtualHost
prs := make(map[string]interface{})
prs["timeout_ms"] = 0
for _, v := range vhL {
if pvh.Name == "" {
pvh.Name = v.Name
pvh.Domains = v.Domains
pvh.Routes = []map[string]interface{}{prs}
}
if v.Domains[0] == d {
switch domainL[d]{
case node_model.MODELWEIGHT:
prs["prefix"] = v.Routes.([]map[string]interface{})[0]["prefix"].(string)
if hasHeader, ok := v.Routes.([]map[string]interface{})[0]["headers"].([]node_model.PieceHeader); ok {
prs["headers"] = hasHeader
}
//pieceCluster := v.Routes.([]map[string]interface{})[0]["weighted_clusters"].(node_model.WeightedClusters).Clusters[0]
c = append(c, v.Routes.([]map[string]interface{})[0]["weighted_clusters"].(node_model.WeightedClusters).Clusters[0])
case node_model.MODELPREFIX:
r = append(r, v.Routes.([]map[string]interface{})[0])
}
}else {
newVHL = append(newVHL, v)
}
}
if len(r) != 0 {
pvh.Routes = r
newVHL = append(newVHL, &pvh)
}
if len(c) != 0 {
var wc node_model.WeightedClusters
wc.Clusters = c
prs["weighted_clusters"] = wc
logrus.Debugf("prs is %v", prs)
pvh.Routes = []map[string]interface{}{prs}
newVHL = append(newVHL, &pvh)
}
}
}else {
newVHL = vhL
}
}else {
newVHL = vhL
}
logrus.Debugf("newVHL is %v", newVHL)
rcg := &node_model.RouteConfig{
VirtualHosts: vhL,
VirtualHosts: newVHL,
}
lhc := &node_model.LDSHTTPConfig{
CodecType: "auto",
@ -335,6 +404,64 @@ func (d *DiscoverAction) DiscoverListeners(
return lds, nil
}
//Duplicate Duplicate
func Duplicate(a interface{}) (ret []interface{}) {
va := reflect.ValueOf(a)
for i := 0; i < va.Len(); i++ {
if i > 0 && reflect.DeepEqual(va.Index(i-1).Interface(), va.Index(i).Interface()) {
continue
}
ret = append(ret, va.Index(i).Interface())
}
return ret
}
//CheckSameDomainAndPrefix 检查是否存在相同domain以及prefix
func (d *DiscoverAction)CheckSameDomainAndPrefix(resources *api_model.ResourceSpec) (map[string]string){
baseServices := resources.BaseServices
domainL := make(map[string]string)
if len(baseServices) == 0 {
logrus.Debugf("has no base services resources")
return domainL
}
filterL := make(map[string]int)
for _, bs := range baseServices {
l := len(filterL)
domainName, _:= bs.Options[node_model.DOMAINS].(string)
filterL[domainName] = 0
if len(filterL) == l {
domainL[domainName] = "use"
}
}
for d := range domainL {
prefixM := make(map[string]int)
for _, bs := range baseServices {
domainName, _ := bs.Options[node_model.DOMAINS].(string)
if domainName == d {
prefix, _ := bs.Options[node_model.PREFIX].(string)
prefixM[prefix] = 0
}
// if strings.Contains(domainName, ","){
// mm := strings.Split(domainName, ",")
// for _, n := range mm {
// if n == d {
// prefix, _ := bs.Options[node_model.PREFIX].(string)
// prefixM[prefix] = 0
// continue
// }
// }
// }
}
logrus.Debugf("prefixM is %v", prefixM)
if len(prefixM) == 1 {
domainL[d] = node_model.MODELWEIGHT
}else{
domainL[d] = node_model.MODELPREFIX
}
}
return domainL
}
//DiscoverClusters cds
func (d *DiscoverAction) DiscoverClusters(
tenantService,
@ -495,8 +622,7 @@ func (d *DiscoverAction) ToolsBuildPieceLDS() {}
//ToolsGetRouterItem ToolsGetRouterItem
func (d *DiscoverAction) ToolsGetRouterItem(
destAlias, kind string,
sr map[string]interface{}) interface{} {
destAlias, kind string, sr map[string]interface{}) interface{} {
switch kind {
case node_model.PREFIX:
if prefix, ok := sr[node_model.PREFIX]; ok {
@ -549,26 +675,52 @@ func (d *DiscoverAction) ToolsGetRouterItem(
logrus.Errorf("strcon max retry error")
return 3
}
if mxr > 0 && mxr < 10 {
return mxr
}
if mxr == 11 {
return 0
}
return mxr
}
return 3
case node_model.HEADERS:
return ""
if headers, ok := sr[node_model.HEADERS]; ok {
var np []node_model.PieceHeader
parents := strings.Split(headers.(string), ";")
for _, h := range parents {
headers := strings.Split(h, ":")
//has_header:no 默认
if len(headers) == 2 {
if headers[0] == "has_header" && headers[1] == "no" {
continue
}
ph := node_model.PieceHeader{
Name: headers[0],
Value: headers[1],
}
np = append(np, ph)
}
}
return np
}
var rc []node_model.PieceHeader
return rc
case node_model.DOMAINS:
if domain, ok := sr[node_model.DOMAINS]; ok {
if destAlias != "" {
return []string{destAlias, domain.(string)}
if strings.Contains(domain.(string), ","){
mm := strings.Split(domain.(string), ",")
return mm
}
return []string{domain.(string)}
}
return []string{destAlias}
case node_model.WEIGHT:
if weight, ok := sr[node_model.WEIGHT]; ok {
w, err := strconv.Atoi(weight.(string))
if err != nil {
return 100
}
return w
}
return 100
default:
return nil
}
return ""
}
//ToolsGetRainbondResources 获取rainbond自定义resources

31
pkg/node/masterserver/task/engine.go
View File

@ -87,9 +87,15 @@ func (t *TaskEngine) Start() error {
t.LoadStaticTask()
//以下工作器只能由一个节点完成
//step1 获取调度权限，如果master节点多点部署，只能有一个节点具有调度权
var timer = time.NewTimer(time.Second * 4)
go func() {
defer close(t.down)
for {
select {
case <-t.ctx.Done():
return
case <-timer.C:
}
if ok, err := t.haveMaster(); ok {
logrus.Infof("Current node(%s) have task scheduler authority", t.currentNode.HostName)
keepchan := make(chan struct{}, 1)
@ -111,12 +117,7 @@ func (t *TaskEngine) Start() error {
} else if err != nil {
logrus.Error("check current node Whether has a scheduling authority error,", err.Error())
}
select {
case <-t.ctx.Done():
return
default:
}
time.Sleep(time.Second * 3)
timer.Reset(time.Second * 4)
}
}()
return nil
@ -142,18 +143,26 @@ func (t *TaskEngine) haveMaster() (bool, error) {
return false, err
}
if !resp.Succeeded {
ctx, cancel := context.WithTimeout(t.ctx, time.Second*3)
ch := store.DefalutClient.WatchByCtx(ctx, "/rainbond/task/scheduler/authority")
//判断是否为自己注册
res, err := store.DefalutClient.Get(key)
if err == nil && res != nil && len(res.Kvs) == 1 {
if string(res.Kvs[0].Value) == t.currentNode.HostName {
return true, nil
}
}
//监控变化
ctx, cancel := context.WithCancel(t.ctx)
defer cancel()
ch := store.DefalutClient.WatchByCtx(ctx, key)
for {
select {
case <-t.ctx.Done():
cancel()
return false, nil
case events := <-ch:
for _, event := range events.Events {
//watch 到删除操作，返回去获取权限
if event.Type == client.EventTypeDelete {
cancel()
logrus.Infof("get event that master offline,will strive for master node permissions.")
return false, nil
}
}
@ -165,7 +174,7 @@ func (t *TaskEngine) haveMaster() (bool, error) {
}
func (t *TaskEngine) keepMaster(errchan chan struct{}) {
duration := time.Second * 5
duration := time.Second * 3
timer := time.NewTimer(duration)
keep:
for {

132
pkg/node/monitormessage/udpserver.go Normal file
View File

@ -0,0 +1,132 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package monitormessage
import (
"fmt"
"net"
"strconv"
"strings"
"time"
"github.com/Sirupsen/logrus"
"github.com/goodrain/rainbond/pkg/discover"
"github.com/goodrain/rainbond/pkg/discover/config"
"github.com/prometheus/common/log"
)
//UDPServer udp server
type UDPServer struct {
ListenerHost string
ListenerPort int
eventServerEndpoint []string
client net.Conn
}
//CreateUDPServer create udpserver
func CreateUDPServer(lisHost string, lisPort int) *UDPServer {
return &UDPServer{
ListenerHost: lisHost,
ListenerPort: lisPort,
}
}
//Start start
func (u *UDPServer) Start() error {
dis, err := discover.GetDiscover(config.DiscoverConfig{})
if err != nil {
return err
}
dis.AddProject("event_log_event_udp", u)
if err := u.server(); err != nil {
return err
}
return nil
}
//UpdateEndpoints update event server address
func (u *UDPServer) UpdateEndpoints(endpoints ...*config.Endpoint) {
var eventServerEndpoint []string
for _, e := range endpoints {
eventServerEndpoint = append(eventServerEndpoint, e.URL)
u.eventServerEndpoint = eventServerEndpoint
}
if len(u.eventServerEndpoint) > 0 {
for i := range u.eventServerEndpoint {
info := strings.Split(u.eventServerEndpoint[i], ":")
if len(info) == 2 {
dip := net.ParseIP(info[0])
port, err := strconv.Atoi(info[1])
if err != nil {
continue
}
srcAddr := &net.UDPAddr{IP: net.IPv4zero, Port: 0}
dstAddr := &net.UDPAddr{IP: dip, Port: port}
conn, err := net.DialUDP("udp", srcAddr, dstAddr)
if err != nil {
logrus.Error(err)
continue
}
logrus.Infof("Update event server address is %s", u.eventServerEndpoint[i])
u.client = conn
break
}
}
}
}
//Error
func (u *UDPServer) Error(err error) {
}
//Server 服务
func (u *UDPServer) server() error {
listener, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP(u.ListenerHost), Port: u.ListenerPort})
if err != nil {
fmt.Println(err)
return err
}
log.Infof("UDP Server Listener: %s", listener.LocalAddr().String())
buf := make([]byte, 65535)
go func() {
defer listener.Close()
for {
n, _, err := listener.ReadFromUDP(buf)
if err != nil {
logrus.Errorf("read message from udp error,%s", err.Error())
time.Sleep(time.Second * 2)
continue
}
u.handlePacket(buf[0:n])
}
}()
return nil
}
func (u *UDPServer) handlePacket(packet []byte) {
lines := strings.Split(string(packet), "\n")
for _, line := range lines {
if line != "" && u.client != nil {
u.client.Write([]byte(line))
}
}
}

201
pkg/node/statsd/exporter/LICENSE Normal file
View File

@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

60
pkg/node/statsd/exporter/Makefile Normal file
View File

@ -0,0 +1,60 @@
# Copyright 2013 The Prometheus Authors
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
GO := GO15VENDOREXPERIMENT=1 go
PROMU := $(GOPATH)/bin/promu
pkgs = $(shell $(GO) list ./... | grep -v /vendor/)
PREFIX ?= $(shell pwd)
BIN_DIR ?= $(shell pwd)
DOCKER_IMAGE_NAME ?= statsd-exporter
DOCKER_IMAGE_TAG ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD))
all: format build test
style:
@echo ">> checking code style"
@! gofmt -d $(shell find . -path ./vendor -prune -o -name '*.go' -print) | grep '^'
test:
@echo ">> running tests"
@$(GO) test -short $(pkgs)
format:
@echo ">> formatting code"
@$(GO) fmt $(pkgs)
vet:
@echo ">> vetting code"
@$(GO) vet $(pkgs)
build: promu
@echo ">> building binaries"
@$(PROMU) build --prefix $(PREFIX)
tarball: promu
@echo ">> building release tarball"
@$(PROMU) tarball --prefix $(PREFIX) $(BIN_DIR)
docker:
@echo ">> building docker image"
@docker build -t "$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)" .
promu:
@GOOS=$(shell uname -s | tr A-Z a-z) \
GOARCH=$(subst x86_64,amd64,$(patsubst i%86,386,$(shell uname -m))) \
$(GO) get -u github.com/prometheus/promu
.PHONY: all style format build test vet tarball docker promu

595
pkg/node/statsd/exporter/exporter.go Normal file
View File

@ -0,0 +1,595 @@
// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package exporter
import (
"bufio"
"bytes"
"encoding/binary"
"fmt"
"hash/fnv"
"io"
"net"
"regexp"
"strconv"
"strings"
"unicode/utf8"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/log"
"github.com/prometheus/common/model"
)
const (
defaultHelp = "Metric autogenerated by statsd_exporter."
regErrF = "A change of configuration created inconsistent metrics for " +
"%q. You have to restart the statsd_exporter, and you should " +
"consider the effects on your monitoring setup. Error: %s"
)
var (
illegalCharsRE = regexp.MustCompile(`[^a-zA-Z0-9_]`)
hash = fnv.New64a()
strBuf bytes.Buffer // Used for hashing.
intBuf = make([]byte, 8)
)
// hashNameAndLabels returns a hash value of the provided name string and all
// the label names and values in the provided labels map.
//
// Not safe for concurrent use! (Uses a shared buffer and hasher to save on
// allocations.)
func hashNameAndLabels(name string, labels prometheus.Labels) uint64 {
hash.Reset()
strBuf.Reset()
strBuf.WriteString(name)
hash.Write(strBuf.Bytes())
binary.BigEndian.PutUint64(intBuf, model.LabelsToSignature(labels))
hash.Write(intBuf)
return hash.Sum64()
}
type CounterContainer struct {
Elements map[uint64]prometheus.Counter
Register prometheus.Registerer
}
func NewCounterContainer(Register prometheus.Registerer) *CounterContainer {
return &CounterContainer{
Elements: make(map[uint64]prometheus.Counter),
Register: Register,
}
}
func (c *CounterContainer) Get(metricName string, labels prometheus.Labels, help string) (prometheus.Counter, error) {
hash := hashNameAndLabels(metricName, labels)
counter, ok := c.Elements[hash]
if !ok {
counter = prometheus.NewCounter(prometheus.CounterOpts{
Name: metricName,
Help: help,
ConstLabels: labels,
})
if err := c.Register.Register(counter); err != nil {
return nil, err
}
c.Elements[hash] = counter
}
return counter, nil
}
type GaugeContainer struct {
Elements map[uint64]prometheus.Gauge
Register prometheus.Registerer
}
func NewGaugeContainer(Register prometheus.Registerer) *GaugeContainer {
return &GaugeContainer{
Elements: make(map[uint64]prometheus.Gauge),
Register: Register,
}
}
func (c *GaugeContainer) Get(metricName string, labels prometheus.Labels, help string) (prometheus.Gauge, error) {
hash := hashNameAndLabels(metricName, labels)
gauge, ok := c.Elements[hash]
if !ok {
gauge = prometheus.NewGauge(prometheus.GaugeOpts{
Name: metricName,
Help: help,
ConstLabels: labels,
})
if err := c.Register.Register(gauge); err != nil {
return nil, err
}
c.Elements[hash] = gauge
}
return gauge, nil
}
type SummaryContainer struct {
Elements map[uint64]prometheus.Summary
Register prometheus.Registerer
}
func NewSummaryContainer(Register prometheus.Registerer) *SummaryContainer {
return &SummaryContainer{
Elements: make(map[uint64]prometheus.Summary),
Register: Register,
}
}
func (c *SummaryContainer) Get(metricName string, labels prometheus.Labels, help string) (prometheus.Summary, error) {
hash := hashNameAndLabels(metricName, labels)
summary, ok := c.Elements[hash]
if !ok {
summary = prometheus.NewSummary(
prometheus.SummaryOpts{
Name: metricName,
Help: help,
ConstLabels: labels,
})
if err := c.Register.Register(summary); err != nil {
return nil, err
}
c.Elements[hash] = summary
}
return summary, nil
}
type HistogramContainer struct {
Elements map[uint64]prometheus.Histogram
mapper *MetricMapper
Register prometheus.Registerer
}
func NewHistogramContainer(mapper *MetricMapper, Register prometheus.Registerer) *HistogramContainer {
return &HistogramContainer{
Elements: make(map[uint64]prometheus.Histogram),
mapper: mapper,
Register: Register,
}
}
func (c *HistogramContainer) Get(metricName string, labels prometheus.Labels, help string, mapping *metricMapping) (prometheus.Histogram, error) {
hash := hashNameAndLabels(metricName, labels)
histogram, ok := c.Elements[hash]
if !ok {
buckets := c.mapper.Defaults.Buckets
if mapping != nil && mapping.Buckets != nil && len(mapping.Buckets) > 0 {
buckets = mapping.Buckets
}
histogram = prometheus.NewHistogram(
prometheus.HistogramOpts{
Name: metricName,
Help: help,
ConstLabels: labels,
Buckets: buckets,
})
c.Elements[hash] = histogram
if err := c.Register.Register(histogram); err != nil {
return nil, err
}
}
return histogram, nil
}
type Event interface {
MetricName() string
Value() float64
Labels() map[string]string
}
type CounterEvent struct {
metricName string
value float64
labels map[string]string
}
func (c *CounterEvent) MetricName() string { return c.metricName }
func (c *CounterEvent) Value() float64 { return c.value }
func (c *CounterEvent) Labels() map[string]string { return c.labels }
type GaugeEvent struct {
metricName string
value float64
relative bool
labels map[string]string
}
func (g *GaugeEvent) MetricName() string { return g.metricName }
func (g *GaugeEvent) Value() float64 { return g.value }
func (c *GaugeEvent) Labels() map[string]string { return c.labels }
type TimerEvent struct {
metricName string
value float64
labels map[string]string
}
func (t *TimerEvent) MetricName() string { return t.metricName }
func (t *TimerEvent) Value() float64 { return t.value }
func (c *TimerEvent) Labels() map[string]string { return c.labels }
type Events []Event
type Exporter struct {
Counters *CounterContainer
Gauges *GaugeContainer
Summaries *SummaryContainer
Histograms *HistogramContainer
mapper *MetricMapper
}
func escapeMetricName(metricName string) string {
// If a metric starts with a digit, prepend an underscore.
if metricName[0] >= '0' && metricName[0] <= '9' {
metricName = "_" + metricName
}
// Replace all illegal metric chars with underscores.
metricName = illegalCharsRE.ReplaceAllString(metricName, "_")
return metricName
}
func (b *Exporter) Listen(e <-chan Events) {
for {
events, ok := <-e
if !ok {
log.Debug("Channel is closed. Break out of Exporter.Listener.")
return
}
for _, event := range events {
var help string
metricName := ""
prometheusLabels := event.Labels()
mapping, labels, present := b.mapper.getMapping(event.MetricName())
if mapping == nil {
mapping = &metricMapping{}
}
if mapping.HelpText == "" {
help = defaultHelp
} else {
help = mapping.HelpText
}
if present {
metricName = mapping.Name
for label, value := range labels {
prometheusLabels[label] = value
}
} else {
eventsUnmapped.Inc()
metricName = escapeMetricName(event.MetricName())
}
switch ev := event.(type) {
case *CounterEvent:
// We don't accept negative values for counters. Incrementing the counter with a negative number
// will cause the exporter to panic. Instead we will warn and continue to the next event.
if event.Value() < 0.0 {
log.Debugf("Counter %q is: '%f' (counter must be non-negative value)", metricName, event.Value())
eventStats.WithLabelValues("illegal_negative_counter").Inc()
continue
}
counter, err := b.Counters.Get(
metricName,
prometheusLabels,
help,
)
if err == nil {
counter.Add(event.Value())
eventStats.WithLabelValues("counter").Inc()
} else {
log.Debugf(regErrF, metricName, err)
conflictingEventStats.WithLabelValues("counter").Inc()
}
case *GaugeEvent:
gauge, err := b.Gauges.Get(
metricName,
prometheusLabels,
help,
)
if err == nil {
if ev.relative {
gauge.Add(event.Value())
} else {
gauge.Set(event.Value())
}
eventStats.WithLabelValues("gauge").Inc()
} else {
log.Debugf(regErrF, metricName, err)
conflictingEventStats.WithLabelValues("gauge").Inc()
}
case *TimerEvent:
t := timerTypeDefault
if mapping != nil {
t = mapping.TimerType
}
if t == timerTypeDefault {
t = b.mapper.Defaults.TimerType
}
switch t {
case timerTypeHistogram:
histogram, err := b.Histograms.Get(
metricName,
prometheusLabels,
help,
mapping,
)
if err == nil {
histogram.Observe(event.Value() / 1000) // prometheus presumes seconds, statsd millisecond
eventStats.WithLabelValues("timer").Inc()
} else {
log.Debugf(regErrF, metricName, err)
conflictingEventStats.WithLabelValues("timer").Inc()
}
case timerTypeDefault, timerTypeSummary:
summary, err := b.Summaries.Get(
metricName,
prometheusLabels,
help,
)
if err == nil {
summary.Observe(event.Value())
eventStats.WithLabelValues("timer").Inc()
} else {
log.Debugf(regErrF, metricName, err)
conflictingEventStats.WithLabelValues("timer").Inc()
}
default:
panic(fmt.Sprintf("unknown timer type '%s'", t))
}
default:
log.Debugln("Unsupported event type")
eventStats.WithLabelValues("illegal").Inc()
}
}
}
}
//NewExporter new exporter
func NewExporter(mapper *MetricMapper, Register prometheus.Registerer) *Exporter {
return &Exporter{
Counters: NewCounterContainer(Register),
Gauges: NewGaugeContainer(Register),
Summaries: NewSummaryContainer(Register),
Histograms: NewHistogramContainer(mapper, Register),
mapper: mapper,
}
}
func buildEvent(statType, metric string, value float64, relative bool, labels map[string]string) (Event, error) {
switch statType {
case "c":
return &CounterEvent{
metricName: metric,
value: float64(value),
labels: labels,
}, nil
case "g":
return &GaugeEvent{
metricName: metric,
value: float64(value),
relative: relative,
labels: labels,
}, nil
case "ms", "h":
return &TimerEvent{
metricName: metric,
value: float64(value),
labels: labels,
}, nil
case "s":
return nil, fmt.Errorf("No support for StatsD sets")
default:
return nil, fmt.Errorf("Bad stat type %s", statType)
}
}
func parseDogStatsDTagsToLabels(component string) map[string]string {
labels := map[string]string{}
tagsReceived.Inc()
tags := strings.Split(component, ",")
for _, t := range tags {
t = strings.TrimPrefix(t, "#")
kv := strings.SplitN(t, ":", 2)
if len(kv) < 2 || len(kv[1]) == 0 {
tagErrors.Inc()
log.Debugf("Malformed or empty DogStatsD tag %s in component %s", t, component)
continue
}
labels[escapeMetricName(kv[0])] = kv[1]
}
return labels
}
func lineToEvents(line string) Events {
events := Events{}
if line == "" {
return events
}
elements := strings.SplitN(line, ":", 2)
if len(elements) < 2 || len(elements[0]) == 0 || !utf8.ValidString(line) {
sampleErrors.WithLabelValues("malformed_line").Inc()
log.Debugln("Bad line from StatsD:", line)
return events
}
metric := elements[0]
var samples []string
if strings.Contains(elements[1], "|#") {
// using datadog extensions, disable multi-metrics
samples = elements[1:]
} else {
samples = strings.Split(elements[1], ":")
}
samples:
for _, sample := range samples {
samplesReceived.Inc()
components := strings.Split(sample, "|")
samplingFactor := 1.0
if len(components) < 2 || len(components) > 4 {
sampleErrors.WithLabelValues("malformed_component").Inc()
log.Debugln("Bad component on line:", line)
continue
}
valueStr, statType := components[0], components[1]
var relative = false
if strings.Index(valueStr, "+") == 0 || strings.Index(valueStr, "-") == 0 {
relative = true
}
value, err := strconv.ParseFloat(valueStr, 64)
if err != nil {
log.Debugf("Bad value %s on line: %s", valueStr, line)
sampleErrors.WithLabelValues("malformed_value").Inc()
continue
}
multiplyEvents := 1
labels := map[string]string{}
if len(components) >= 3 {
for _, component := range components[2:] {
if len(component) == 0 {
log.Debugln("Empty component on line: ", line)
sampleErrors.WithLabelValues("malformed_component").Inc()
continue samples
}
}
for _, component := range components[2:] {
switch component[0] {
case '@':
if statType != "c" && statType != "ms" {
log.Debugln("Illegal sampling factor for non-counter metric on line", line)
sampleErrors.WithLabelValues("illegal_sample_factor").Inc()
continue
}
samplingFactor, err = strconv.ParseFloat(component[1:], 64)
if err != nil {
log.Debugf("Invalid sampling factor %s on line %s", component[1:], line)
sampleErrors.WithLabelValues("invalid_sample_factor").Inc()
}
if samplingFactor == 0 {
samplingFactor = 1
}
if statType == "c" {
value /= samplingFactor
} else if statType == "ms" {
multiplyEvents = int(1 / samplingFactor)
}
case '#':
labels = parseDogStatsDTagsToLabels(component)
default:
log.Debugf("Invalid sampling factor or tag section %s on line %s", components[2], line)
sampleErrors.WithLabelValues("invalid_sample_factor").Inc()
continue
}
}
}
for i := 0; i < multiplyEvents; i++ {
event, err := buildEvent(statType, metric, value, relative, labels)
if err != nil {
log.Debugf("Error building event on line %s: %s", line, err)
sampleErrors.WithLabelValues("illegal_event").Inc()
continue
}
events = append(events, event)
}
}
return events
}
type StatsDUDPListener struct {
Conn *net.UDPConn
}
func (l *StatsDUDPListener) Listen(e chan<- Events) {
buf := make([]byte, 65535)
for {
n, _, err := l.Conn.ReadFromUDP(buf)
if err != nil {
log.Fatal(err)
}
l.handlePacket(buf[0:n], e)
}
}
func (l *StatsDUDPListener) handlePacket(packet []byte, e chan<- Events) {
udpPackets.Inc()
lines := strings.Split(string(packet), "\n")
events := Events{}
for _, line := range lines {
linesReceived.Inc()
events = append(events, lineToEvents(line)...)
}
e <- events
}
type StatsDTCPListener struct {
Conn *net.TCPListener
}
func (l *StatsDTCPListener) Listen(e chan<- Events) {
defer l.Conn.Close()
for {
c, err := l.Conn.AcceptTCP()
if err != nil {
log.Fatalf("AcceptTCP failed: %v", err)
}
go l.handleConn(c, e)
}
}
func (l *StatsDTCPListener) handleConn(c *net.TCPConn, e chan<- Events) {
defer c.Close()
tcpConnections.Inc()
r := bufio.NewReader(c)
for {
line, isPrefix, err := r.ReadLine()
if err != nil {
if err != io.EOF {
tcpErrors.Inc()
log.Debugf("Read %s failed: %v", c.RemoteAddr(), err)
}
break
}
if isPrefix {
tcpLineTooLong.Inc()
log.Debugf("Read %s failed: line too long", c.RemoteAddr())
break
}
linesReceived.Inc()
e <- lineToEvents(string(line))
}
}

158
pkg/node/statsd/exporter/mapper.go Normal file
View File

@ -0,0 +1,158 @@
// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package exporter
import (
"fmt"
"io/ioutil"
"regexp"
"strings"
"sync"
"github.com/prometheus/client_golang/prometheus"
yaml "gopkg.in/yaml.v2"
)
var (
identifierRE = `[a-zA-Z_][a-zA-Z0-9_]+`
statsdMetricRE = `[a-zA-Z_](-?[a-zA-Z0-9_])+`
metricLineRE = regexp.MustCompile(`^(\*\.|` + statsdMetricRE + `\.)+(\*|` + statsdMetricRE + `)$`)
labelLineRE = regexp.MustCompile(`^(` + identifierRE + `)\s*=\s*"(.*)"$`)
metricNameRE = regexp.MustCompile(`^` + identifierRE + `$`)
)
type mapperConfigDefaults struct {
TimerType timerType `yaml:"timer_type"`
Buckets []float64 `yaml:"buckets"`
MatchType matchType `yaml:"match_type"`
}
//MetricMapper MetricMapper
type MetricMapper struct {
Defaults mapperConfigDefaults `yaml:"defaults"`
Mappings []metricMapping `yaml:"mappings"`
mutex sync.Mutex
}
type metricMapping struct {
Match string `yaml:"match"`
Name string `yaml:"name"`
regex *regexp.Regexp
Labels prometheus.Labels `yaml:"labels"`
TimerType timerType `yaml:"timer_type"`
Buckets []float64 `yaml:"buckets"`
MatchType matchType `yaml:"match_type"`
HelpText string `yaml:"help"`
}
func (m *MetricMapper) InitFromYAMLString(fileContents string) error {
var n MetricMapper
if err := yaml.Unmarshal([]byte(fileContents), &n); err != nil {
return err
}
if n.Defaults.Buckets == nil || len(n.Defaults.Buckets) == 0 {
n.Defaults.Buckets = prometheus.DefBuckets
}
if n.Defaults.MatchType == matchTypeDefault {
n.Defaults.MatchType = matchTypeGlob
}
for i := range n.Mappings {
currentMapping := &n.Mappings[i]
// check that label is correct
for k := range currentMapping.Labels {
if !metricNameRE.MatchString(k) {
return fmt.Errorf("invalid label key: %s", k)
}
}
if currentMapping.Name == "" {
return fmt.Errorf("line %d: metric mapping didn't set a metric name", i)
}
if !metricNameRE.MatchString(currentMapping.Name) {
return fmt.Errorf("metric name '%s' doesn't match regex '%s'", currentMapping.Name, metricNameRE)
}
if currentMapping.MatchType == "" {
currentMapping.MatchType = n.Defaults.MatchType
}
if currentMapping.MatchType == matchTypeGlob {
if !metricLineRE.MatchString(currentMapping.Match) {
return fmt.Errorf("invalid match: %s", currentMapping.Match)
}
// Translate the glob-style metric match line into a proper regex that we
// can use to match metrics later on.
metricRe := strings.Replace(currentMapping.Match, ".", "\\.", -1)
metricRe = strings.Replace(metricRe, "*", "([^.]*)", -1)
currentMapping.regex = regexp.MustCompile("^" + metricRe + "$")
} else {
currentMapping.regex = regexp.MustCompile(currentMapping.Match)
}
if currentMapping.TimerType == "" {
currentMapping.TimerType = n.Defaults.TimerType
}
if currentMapping.Buckets == nil || len(currentMapping.Buckets) == 0 {
currentMapping.Buckets = n.Defaults.Buckets
}
}
m.mutex.Lock()
defer m.mutex.Unlock()
m.Defaults = n.Defaults
m.Mappings = n.Mappings
mappingsCount.Set(float64(len(n.Mappings)))
return nil
}
func (m *MetricMapper) InitFromFile(fileName string) error {
mappingStr, err := ioutil.ReadFile(fileName)
if err != nil {
return err
}
return m.InitFromYAMLString(string(mappingStr))
}
func (m *MetricMapper) getMapping(statsdMetric string) (*metricMapping, prometheus.Labels, bool) {
m.mutex.Lock()
defer m.mutex.Unlock()
for _, mapping := range m.Mappings {
matches := mapping.regex.FindStringSubmatchIndex(statsdMetric)
if len(matches) == 0 {
continue
}
labels := prometheus.Labels{}
for label, valueExpr := range mapping.Labels {
value := mapping.regex.ExpandString([]byte{}, valueExpr, statsdMetric, matches)
labels[label] = string(value)
}
return &mapping, labels, true
}
return nil, nil, false
}

41
pkg/node/statsd/exporter/match.go Normal file
View File

@ -0,0 +1,41 @@
// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package exporter
import "fmt"
type matchType string
const (
matchTypeGlob matchType = "glob"
matchTypeRegex matchType = "regex"
matchTypeDefault matchType = ""
)
func (t *matchType) UnmarshalYAML(unmarshal func(interface{}) error) error {
var v string
if err := unmarshal(&v); err != nil {
return err
}
switch matchType(v) {
case matchTypeRegex:
*t = matchTypeRegex
case matchTypeGlob, matchTypeDefault:
*t = matchTypeGlob
default:
return fmt.Errorf("invalid match type %q", v)
}
return nil
}

122
pkg/node/statsd/exporter/telemetry.go Normal file
View File

@ -0,0 +1,122 @@
// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package exporter
import (
"github.com/prometheus/client_golang/prometheus"
)
var (
eventStats = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "statsd_exporter_events_total",
Help: "The total number of StatsD events seen.",
},
[]string{"type"},
)
eventsUnmapped = prometheus.NewCounter(prometheus.CounterOpts{
Name: "statsd_exporter_events_unmapped_total",
Help: "The total number of StatsD events no mapping was found for.",
})
udpPackets = prometheus.NewCounter(
prometheus.CounterOpts{
Name: "statsd_exporter_udp_packets_total",
Help: "The total number of StatsD packets received over UDP.",
},
)
tcpConnections = prometheus.NewCounter(
prometheus.CounterOpts{
Name: "statsd_exporter_tcp_connections_total",
Help: "The total number of TCP connections handled.",
},
)
tcpErrors = prometheus.NewCounter(
prometheus.CounterOpts{
Name: "statsd_exporter_tcp_connection_errors_total",
Help: "The number of errors encountered reading from TCP.",
},
)
tcpLineTooLong = prometheus.NewCounter(
prometheus.CounterOpts{
Name: "statsd_exporter_tcp_too_long_lines_total",
Help: "The number of lines discarded due to being too long.",
},
)
linesReceived = prometheus.NewCounter(
prometheus.CounterOpts{
Name: "statsd_exporter_lines_total",
Help: "The total number of StatsD lines received.",
},
)
samplesReceived = prometheus.NewCounter(
prometheus.CounterOpts{
Name: "statsd_exporter_samples_total",
Help: "The total number of StatsD samples received.",
},
)
sampleErrors = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "statsd_exporter_sample_errors_total",
Help: "The total number of errors parsing StatsD samples.",
},
[]string{"reason"},
)
tagsReceived = prometheus.NewCounter(
prometheus.CounterOpts{
Name: "statsd_exporter_tags_total",
Help: "The total number of DogStatsD tags processed.",
},
)
tagErrors = prometheus.NewCounter(
prometheus.CounterOpts{
Name: "statsd_exporter_tag_errors_total",
Help: "The number of errors parsign DogStatsD tags.",
},
)
ConfigLoads = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "statsd_exporter_config_reloads_total",
Help: "The number of configuration reloads.",
},
[]string{"outcome"},
)
mappingsCount = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "statsd_exporter_loaded_mappings",
Help: "The current number of configured metric mappings.",
})
conflictingEventStats = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "statsd_exporter_events_conflict_total",
Help: "The total number of StatsD events with conflicting names.",
},
[]string{"type"},
)
)
//MetryInit init
func MetryInit(registry *prometheus.Registry) {
registry.MustRegister(eventStats)
registry.MustRegister(udpPackets)
registry.MustRegister(tcpConnections)
registry.MustRegister(tcpErrors)
registry.MustRegister(tcpLineTooLong)
registry.MustRegister(linesReceived)
registry.MustRegister(samplesReceived)
registry.MustRegister(sampleErrors)
registry.MustRegister(tagsReceived)
registry.MustRegister(tagErrors)
registry.MustRegister(ConfigLoads)
registry.MustRegister(mappingsCount)
registry.MustRegister(conflictingEventStats)
}

41
pkg/node/statsd/exporter/timer.go Normal file
View File

@ -0,0 +1,41 @@
// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package exporter
import "fmt"
type timerType string
const (
timerTypeHistogram timerType = "histogram"
timerTypeSummary timerType = "summary"
timerTypeDefault timerType = ""
)
func (t *timerType) UnmarshalYAML(unmarshal func(interface{}) error) error {
var v string
if err := unmarshal(&v); err != nil {
return err
}
switch timerType(v) {
case timerTypeHistogram:
*t = timerTypeHistogram
case timerTypeSummary, timerTypeDefault:
*t = timerTypeSummary
default:
return fmt.Errorf("invalid timer type '%s'", v)
}
return nil
}

222
pkg/node/statsd/statsd_export.go Normal file
View File

@ -0,0 +1,222 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package statsd
import (
"fmt"
"net"
"strconv"
"github.com/Sirupsen/logrus"
"github.com/howeyc/fsnotify"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/version"
"github.com/goodrain/rainbond/cmd/node/option"
"github.com/goodrain/rainbond/pkg/node/statsd/exporter"
)
//Exporter receive statsd metric and export prometheus metric
type Exporter struct {
statsdListenAddress string
statsdListenUDP string
statsdListenTCP string
mappingConfig string
readBuffer int
exporter *exporter.Exporter
register *prometheus.Registry
mapper *exporter.MetricMapper
}
//CreateExporter create a exporter
func CreateExporter(sc option.StatsdConfig, register *prometheus.Registry) *Exporter {
exp := &Exporter{
statsdListenAddress: sc.StatsdListenAddress,
statsdListenTCP: sc.StatsdListenTCP,
statsdListenUDP: sc.StatsdListenUDP,
readBuffer: sc.ReadBuffer,
mappingConfig: sc.MappingConfig,
register: register,
}
exporter.MetryInit(register)
return exp
}
//Start Start
func (e *Exporter) Start() error {
e.register.Register(version.NewCollector("statsd_exporter"))
if e.statsdListenAddress != "" {
logrus.Warnln("Warning: statsd.listen-address is DEPRECATED, please use statsd.listen-udp instead.")
e.statsdListenUDP = e.statsdListenAddress
}
if e.statsdListenUDP == "" && e.statsdListenTCP == "" {
logrus.Fatalln("At least one of UDP/TCP listeners must be specified.")
return fmt.Errorf("At least one of UDP/TCP listeners must be specified")
}
logrus.Infoln("Starting StatsD -> Prometheus Exporter", version.Info())
logrus.Infoln("Build context", version.BuildContext())
logrus.Infof("Accepting StatsD Traffic: UDP %v, TCP %v", e.statsdListenUDP, e.statsdListenTCP)
events := make(chan exporter.Events, 1024)
if e.statsdListenUDP != "" {
udpListenAddr := udpAddrFromString(e.statsdListenUDP)
uconn, err := net.ListenUDP("udp", udpListenAddr)
if err != nil {
return err
}
if e.readBuffer != 0 {
err = uconn.SetReadBuffer(e.readBuffer)
if err != nil {
return err
}
}
ul := &exporter.StatsDUDPListener{Conn: uconn}
go ul.Listen(events)
}
if e.statsdListenTCP != "" {
tcpListenAddr := tcpAddrFromString(e.statsdListenTCP)
tconn, err := net.ListenTCP("tcp", tcpListenAddr)
if err != nil {
return err
}
tl := &exporter.StatsDTCPListener{Conn: tconn}
go tl.Listen(events)
}
mapper := &exporter.MetricMapper{}
if e.mappingConfig != "" {
err := mapper.InitFromFile(e.mappingConfig)
if err != nil {
logrus.Fatal("Error loading config:", err)
return err
}
//观察文件变化进行重新reload是有风险的,采用API重新加载
//go watchConfig(e.mappingConfig, mapper)
}
exporter := exporter.NewExporter(mapper, e.register)
e.exporter = exporter
e.mapper = mapper
go exporter.Listen(events)
return nil
}
// Describe implements the prometheus.Collector interface.
func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
}
// Collect implements the prometheus.Collector interface.
func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
}
//GetRegister GetRegister
func (e *Exporter) GetRegister() *prometheus.Registry {
return e.register
}
func ipPortFromString(addr string) (*net.IPAddr, int) {
host, portStr, err := net.SplitHostPort(addr)
if err != nil {
logrus.Fatal("Bad StatsD listening address", addr)
}
if host == "" {
host = "0.0.0.0"
}
ip, err := net.ResolveIPAddr("ip", host)
if err != nil {
logrus.Fatalf("Unable to resolve %s: %s", host, err)
}
port, err := strconv.Atoi(portStr)
if err != nil || port < 0 || port > 65535 {
logrus.Fatalf("Bad port %s: %s", portStr, err)
}
return ip, port
}
func udpAddrFromString(addr string) *net.UDPAddr {
ip, port := ipPortFromString(addr)
return &net.UDPAddr{
IP: ip.IP,
Port: port,
Zone: ip.Zone,
}
}
func tcpAddrFromString(addr string) *net.TCPAddr {
ip, port := ipPortFromString(addr)
return &net.TCPAddr{
IP: ip.IP,
Port: port,
Zone: ip.Zone,
}
}
func watchConfig(fileName string, mapper *exporter.MetricMapper) {
watcher, err := fsnotify.NewWatcher()
if err != nil {
logrus.Fatal(err)
}
err = watcher.WatchFlags(fileName, fsnotify.FSN_MODIFY)
if err != nil {
logrus.Fatal(err)
}
for {
select {
case ev := <-watcher.Event:
logrus.Infof("Config file changed (%s), attempting reload", ev)
err = mapper.InitFromFile(fileName)
if err != nil {
logrus.Errorln("Error reloading config:", err)
exporter.ConfigLoads.WithLabelValues("failure").Inc()
} else {
logrus.Infoln("Config reloaded successfully")
exporter.ConfigLoads.WithLabelValues("success").Inc()
}
// Re-add the file watcher since it can get lost on some changes. E.g.
// saving a file with vim results in a RENAME-MODIFY-DELETE event
// sequence, after which the newly written file is no longer watched.
err = watcher.WatchFlags(fileName, fsnotify.FSN_MODIFY)
case err := <-watcher.Error:
logrus.Errorln("Error watching config:", err)
}
}
}
//ReloadConfig reload mapper config file
func (e *Exporter) ReloadConfig() (err error) {
logrus.Infof("Config file changed, attempting reload")
err = e.mapper.InitFromFile(e.mappingConfig)
if err != nil {
logrus.Errorln("Error reloading config:", err)
exporter.ConfigLoads.WithLabelValues("failure").Inc()
} else {
logrus.Infoln("Config reloaded successfully")
exporter.ConfigLoads.WithLabelValues("success").Inc()
}
return
}

33
pkg/status/service_status.go
View File

@ -1,33 +1,33 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package status
import (
"github.com/goodrain/rainbond/cmd/worker/option"
"github.com/goodrain/rainbond/pkg/db"
"github.com/goodrain/rainbond/pkg/db/model"
"context"
"strings"
"sync"
"time"
"github.com/goodrain/rainbond/cmd/worker/option"
"github.com/goodrain/rainbond/pkg/db"
"github.com/goodrain/rainbond/pkg/db/model"
"github.com/jinzhu/gorm"
"k8s.io/client-go/kubernetes"
@ -49,6 +49,8 @@ const (
//升级中
UPGRADE = "upgrade"
UNDEPLOY = "undeploy"
//构建中
DEPLOYING = "deploying"
)
//ServiceStatusManager 应用运行状态控制器
@ -75,6 +77,7 @@ type statusManager struct {
checkChan chan string
ignoreDelete map[string]string
ignoreLock sync.Mutex
status map[string]string
}
//NewManager 创建一个应用运行状态控制器
@ -101,6 +104,7 @@ func NewManager(conf option.Config) ServiceStatusManager {
StatefulSetUpdateChan: make(chan StatefulSetUpdate, 10),
checkChan: make(chan string, 20),
ignoreDelete: make(map[string]string),
status: make(map[string]string),
}
}
@ -113,10 +117,17 @@ func (s *statusManager) SetStatus(serviceID, status string) error {
logrus.Error("set application service status error.", err.Error())
return err
}
//本地缓存
//s.status[serviceID] = status
return nil
}
func (s *statusManager) GetStatus(serviceID string) (string, error) {
// 本地缓存应用状态
// if status, ok := s.status[serviceID]; ok {
// return status, nil
// }
status, err := db.GetManager().TenantServiceStatusDao().GetTenantServiceStatus(serviceID)
if err != nil {
return "", err
@ -196,7 +207,7 @@ func (s *statusManager) checkStatus() {
continue
}
} else {
if d.Status.ReadyReplicas >= d.Status.Replicas {
if d.Status.ReadyReplicas >= d.Status.Replicas && d.Status.Replicas != 0 {
s.SetStatus(serviceID, RUNNING)
break
} else {
@ -219,7 +230,7 @@ func (s *statusManager) checkStatus() {
continue
}
} else {
if d.Status.ReadyReplicas >= d.Status.Replicas {
if d.Status.ReadyReplicas >= d.Status.Replicas && d.Status.Replicas != 0 {
s.SetStatus(serviceID, RUNNING)
break
} else {
@ -243,7 +254,7 @@ func (s *statusManager) checkStatus() {
} else {
readycount := s.getReadyCount(d.Namespace,
d.Labels["name"], d.Labels["version"])
if readycount >= d.Status.Replicas {
if readycount >= d.Status.Replicas && d.Status.Replicas != 0 {
s.SetStatus(serviceID, RUNNING)
break
} else {
@ -265,7 +276,7 @@ func (s *statusManager) checkStatus() {
continue
}
} else {
if d.Status.ReadyReplicas >= d.Status.Replicas {
if d.Status.ReadyReplicas >= d.Status.Replicas && d.Status.Replicas != 0 {
s.SetStatus(serviceID, RUNNING)
break
}

67
pkg/util/comman.go
View File

@ -25,7 +25,9 @@ import (
"os"
"os/exec"
"path"
"path/filepath"
"strings"
"sync"
"time"
"github.com/Sirupsen/logrus"
@ -251,3 +253,68 @@ func Deweight(data *[]string) {
}
*data = result
}
//GetDirSize kb为单位
func GetDirSize(path string) float64 {
if ok, err := FileExists(path); err != nil || !ok {
return 0
}
fileSizes := make(chan int64)
concurrent := make(chan int, 10)
var wg sync.WaitGroup
wg.Add(1)
go walkDir(path, &wg, fileSizes, concurrent)
go func() {
wg.Wait() //等待goroutine结束
close(fileSizes)
}()
var nfiles, nbytes int64
loop:
for {
select {
case size, ok := <-fileSizes:
if !ok {
break loop
}
nfiles++
nbytes += size
}
}
return float64(nbytes / 1024)
}
//获取目录dir下的文件大小
func walkDir(dir string, wg *sync.WaitGroup, fileSizes chan<- int64, concurrent chan int) {
defer wg.Done()
concurrent <- 1
defer func() {
<-concurrent
}()
for _, entry := range dirents(dir) {
if entry.IsDir() { //目录
wg.Add(1)
subDir := filepath.Join(dir, entry.Name())
go walkDir(subDir, wg, fileSizes, concurrent)
} else {
fileSizes <- entry.Size()
}
}
}
//sema is a counting semaphore for limiting concurrency in dirents
var sema = make(chan struct{}, 20)
//读取目录dir下的文件信息
func dirents(dir string) []os.FileInfo {
sema <- struct{}{}
defer func() { <-sema }()
entries, err := ioutil.ReadDir(dir)
if err != nil {
logrus.Errorf("get file sizt: %v\n", err)
return nil
}
return entries
}

4
pkg/util/comman_test.go
View File

@ -32,3 +32,7 @@ func TestDeweight(t *testing.T) {
Deweight(&data)
t.Log(data)
}
func TestGetDirSize(t *testing.T) {
t.Log(GetDirSize("/Users/qingguo/gopath"))
}

2
pkg/worker/appm/manager_rc.go
View File

@ -90,6 +90,8 @@ func (m *manager) StartReplicationController(serviceID string, logger event.Logg
}
return result, nil
}
//CheckVersionInfo CheckVersionInfo
func CheckVersionInfo(version *model.VersionInfo) bool {
if !strings.Contains(strings.ToLower(version.FinalStatus),"success") {
return false

16
pkg/worker/appm/manager_service.go
View File

@ -105,12 +105,12 @@ func (m *manager) updateService(serviceID, tenantID string, service *v1.Service,
ReplicationType: ReplicationType,
K8sServiceID: service.Name,
}
if len(service.Spec.Ports) > 0 {
if len(service.Spec.Ports) == 1 {
k8sService.ContainerPort = int(service.Spec.Ports[0].Port)
} else { //有状态服务 用于服务发现的service，不存port
}
if len(service.Spec.Ports) == 1 {
k8sService.ContainerPort = int(service.Spec.Ports[0].Port)
}
//有状态service不存储port,避免存储失败
if service.Labels["service_type"] == "stateful" {
k8sService.ContainerPort = 0
}
if strings.HasSuffix(service.Name, "out") {
k8sService.IsOut = true
@ -154,6 +154,10 @@ func (m *manager) createService(serviceID, tenantID string, service *v1.Service,
if len(service.Spec.Ports) > 0 {
k8sService.ContainerPort = int(service.Spec.Ports[0].TargetPort.IntVal)
}
//有状态service不存储port,避免存储失败
if service.Labels["service_type"] == "stateful" {
k8sService.ContainerPort = 0
}
err = m.dbmanager.K8sServiceDao().AddModel(k8sService)
if err != nil {
logger.Error(fmt.Sprintf("存储K8sService(%s)信息到数据库失败", service.Name), map[string]string{"step": "worker-appm", "status": "failure"})

33
pkg/worker/appm/pod.go
View File

@ -431,6 +431,31 @@ func (p *PodTemplateSpecBuild) createAdapterResources(memory int, cpu int) v1.Re
}
}
//createPluginResources
//memory Mb
//cpu (core*1000)
//TODO:插件的资源限制，CPU暂时不限制
func (p *PodTemplateSpecBuild) createPluginResources(memory int, cpu int) v1.ResourceRequirements {
limits := v1.ResourceList{}
// limits[v1.ResourceCPU] = *resource.NewMilliQuantity(
// int64(cpu*3),
// resource.DecimalSI)
limits[v1.ResourceMemory] = *resource.NewQuantity(
int64(memory*1024*1024),
resource.BinarySI)
request := v1.ResourceList{}
// request[v1.ResourceCPU] = *resource.NewMilliQuantity(
// int64(cpu*2),
// resource.DecimalSI)
request[v1.ResourceMemory] = *resource.NewQuantity(
int64(memory*1024*1024),
resource.BinarySI)
return v1.ResourceRequirements{
Limits: limits,
Requests: request,
}
}
func (p *PodTemplateSpecBuild) createResources() v1.ResourceRequirements {
var cpuRequest, cpuLimit int64
memory := p.service.ContainerMemory
@ -765,10 +790,10 @@ func (p *PodTemplateSpecBuild) createPluginsContainer(mainEnvs *[]v1.EnvVar) ([]
return nil, nil, err
}
pc := v1.Container{
Name: pluginR.PluginID,
Name: "plugin-" + pluginR.PluginID,
Image: versionInfo.BuildLocalImage,
Env: *envs,
Resources: p.createAdapterResources(versionInfo.ContainerMemory, versionInfo.ContainerCPU),
Resources: p.createPluginResources(versionInfo.ContainerMemory, versionInfo.ContainerCPU),
TerminationMessagePath: "",
Args: args,
}
@ -805,7 +830,7 @@ func (p *PodTemplateSpecBuild) createPluginsContainer(mainEnvs *[]v1.EnvVar) ([]
}
func (p *PodTemplateSpecBuild) getPluginModel(pluginID string) (string, error) {
plugin, err := p.dbmanager.TenantPluginDao().GetPluginByID(pluginID)
plugin, err := p.dbmanager.TenantPluginDao().GetPluginByID(pluginID, p.tenant.UUID)
if err != nil {
return "", err
}
@ -855,7 +880,7 @@ func (p *PodTemplateSpecBuild) sortPlugins() ([]string, error) {
var mid []int
//TODO: 目前同种插件只能出现一个
for _, plugin := range p.pluginsRelation {
pi, err := p.dbmanager.TenantPluginDao().GetPluginByID(plugin.PluginID)
pi, err := p.dbmanager.TenantPluginDao().GetPluginByID(plugin.PluginID, p.tenant.UUID)
if err != nil {
return nil, err
}

4
pkg/worker/appm/service.go
View File

@ -218,7 +218,8 @@ func (k *K8sServiceBuild) createOuterService(port *model.TenantServicesPort) *v1
"key": "",
"event_id": k.eventID,
}
if port.Protocol == "stream" { //stream 协议获取映射端口
//if port.Protocol == "stream" { //stream 协议获取映射端口
if port.Protocol != "http" { //stream 协议获取映射端口
mapPort, err := k.dbmanager.TenantServiceLBMappingPortDao().GetTenantServiceLBMappingPortByService(k.serviceID)
if err != nil {
logrus.Error("get tenant service lb map port error", err.Error())
@ -228,6 +229,7 @@ func (k *K8sServiceBuild) createOuterService(port *model.TenantServicesPort) *v1
}
}
var servicePort v1.ServicePort
//TODO: udp, tcp
if port.Protocol == "udp" {
servicePort.Protocol = "UDP"
} else {

15
pkg/worker/discover/manager.go
View File

@ -1,19 +1,18 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
@ -86,7 +85,9 @@ func (t *TaskManager) Do() {
case <-t.ctx.Done():
return
default:
data, err := t.client.Dequeue(t.ctx, &pb.DequeueRequest{Topic: WTOPIC, ClientHost: hostname + "-worker"})
ctx, cancel := context.WithCancel(t.ctx)
data, err := t.client.Dequeue(ctx, &pb.DequeueRequest{Topic: WTOPIC, ClientHost: hostname + "-worker"})
cancel()
if err != nil {
if grpc1.ErrorDesc(err) == context.DeadlineExceeded.Error() {
continue
@ -111,10 +112,12 @@ func (t *TaskManager) Do() {
rc := t.handleManager.AnalystToExec(transData)
if rc == 9 {
logrus.Debugf("rc is 9, enqueue task to mq")
reply, err := t.client.Enqueue(t.ctx, &pb.EnqueueRequest{
ctx, cancel := context.WithCancel(t.ctx)
reply, err := t.client.Enqueue(ctx, &pb.EnqueueRequest{
Topic: WTOPIC,
Message: data,
})
cancel()
logrus.Debugf("retry send task to mq ,reply is %v", reply)
if err != nil {
logrus.Errorf("enqueue task %v to mq topic %v Error", data, WTOPIC)

49
pkg/worker/executor/task/rolling_upgrade_task.go
View File

@ -1,36 +1,37 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package task
import (
"fmt"
"github.com/goodrain/rainbond/pkg/db"
dbmodel "github.com/goodrain/rainbond/pkg/db/model"
"github.com/goodrain/rainbond/pkg/event"
"github.com/goodrain/rainbond/pkg/util"
"github.com/goodrain/rainbond/pkg/worker/appm"
"github.com/goodrain/rainbond/pkg/worker/discover/model"
"fmt"
"github.com/goodrain/rainbond/pkg/status"
"github.com/Sirupsen/logrus"
"k8s.io/client-go/pkg/api/v1"
)
type rollingUpgradeTask struct {
@ -57,14 +58,14 @@ func (s *rollingUpgradeTask) RunError(e error) {
//是否关闭应用？
//暂时不自动关闭
s.logger.Error("滚动更新超时，应用关闭或启动缓慢，状态将由后台处理", map[string]string{"step": "callback", "status": "failure"})
return
}
//TODO:
//是否还原到原版本
if e.Error() == "应用容器重启" {
s.logger.Error("滚动升级失败，应用启动失败，请查询应用日志", map[string]string{"step": "callback", "status": "failure"})
} else if e.Error() != "dont't support" {
s.logger.Error("滚动升级失败，请重试", map[string]string{"step": "callback", "status": "failure"})
} else {
//TODO:
//是否还原到原版本
if e.Error() == "应用容器重启" {
s.logger.Error("滚动升级失败，应用启动失败，请查询应用日志", map[string]string{"step": "callback", "status": "failure"})
} else if e.Error() != "dont't support" {
s.logger.Error("滚动升级失败，请重试", map[string]string{"step": "callback", "status": "failure"})
}
}
s.taskManager.statusManager.CheckStatus(s.modelTask.ServiceID)
}
@ -100,26 +101,28 @@ func (s *rollingUpgradeTask) RollBack() {
func (s *rollingUpgradeTask) Run() error {
s.logger.Info("应用滚动升级任务开始执行", map[string]string{"step": "worker-executor", "status": "starting"})
var upgradeError error
switch s.serviceType {
case dbmodel.TypeStatefulSet:
_, err := s.taskManager.appm.RollingUpgradeStatefulSet(s.modelTask.ServiceID, s.logger)
if err != nil && err.Error() != appm.ErrTimeOut.Error() {
logrus.Error(err.Error())
_, upgradeError = s.taskManager.appm.RollingUpgradeStatefulSet(s.modelTask.ServiceID, s.logger)
if upgradeError != nil && upgradeError.Error() != appm.ErrTimeOut.Error() {
logrus.Error(upgradeError.Error())
s.logger.Info("应用升级发生错误", map[string]string{"step": "worker-executor", "status": "failure"})
return err
return upgradeError
}
break
case dbmodel.TypeDeployment:
s.logger.Error("版本构建成功，当前类型不支持滚动升级，请手动重启", map[string]string{"step": "callback", "status": "failure"})
return fmt.Errorf("dont't support")
case dbmodel.TypeReplicationController:
rc, err := s.taskManager.appm.RollingUpgradeReplicationController(s.modelTask.ServiceID, s.stopChan, s.logger)
if err != nil && err.Error() != appm.ErrTimeOut.Error() {
logrus.Error(err.Error())
var rc *v1.ReplicationController
rc, upgradeError = s.taskManager.appm.RollingUpgradeReplicationController(s.modelTask.ServiceID, s.stopChan, s.logger)
if upgradeError != nil && upgradeError.Error() != appm.ErrTimeOut.Error() {
logrus.Error(upgradeError.Error())
s.logger.Info("应用滚动升级发生错误", map[string]string{"step": "worker-executor", "status": "failure"})
return err
return upgradeError
}
err = s.taskManager.appm.UpdateService(s.modelTask.ServiceID, s.logger, rc.Name, dbmodel.TypeReplicationController)
err := s.taskManager.appm.UpdateService(s.modelTask.ServiceID, s.logger, rc.Name, dbmodel.TypeReplicationController)
if err != nil {
logrus.Error(err.Error())
s.logger.Info("应用Service升级发生错误", map[string]string{"step": "worker-executor", "status": "failure"})
@ -127,7 +130,7 @@ func (s *rollingUpgradeTask) Run() error {
}
break
}
return nil
return upgradeError
}
func (s *rollingUpgradeTask) TaskID() string {
return s.taskID

10
pkg/worker/executor/task/start_task.go
View File

@ -1,25 +1,26 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package task
import (
"fmt"
"github.com/goodrain/rainbond/pkg/db"
dbmodel "github.com/goodrain/rainbond/pkg/db/model"
"github.com/goodrain/rainbond/pkg/event"
@ -27,7 +28,6 @@ import (
"github.com/goodrain/rainbond/pkg/util"
"github.com/goodrain/rainbond/pkg/worker/appm"
"github.com/goodrain/rainbond/pkg/worker/discover/model"
"fmt"
"github.com/jinzhu/gorm"

119
pkg/worker/monitor/cache/cache.go vendored Normal file
View File

@ -0,0 +1,119 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cache
import (
"context"
"fmt"
"os"
"time"
"github.com/Sirupsen/logrus"
"github.com/goodrain/rainbond/pkg/db"
"github.com/goodrain/rainbond/pkg/db/model"
"github.com/goodrain/rainbond/pkg/status"
"github.com/goodrain/rainbond/pkg/util"
)
//DiskCache 磁盘异步统计
type DiskCache struct {
cache map[string]float64
dbmanager db.Manager
statusManager status.ServiceStatusManager
ctx context.Context
}
//CreatDiskCache 创建
func CreatDiskCache(ctx context.Context, statusManager status.ServiceStatusManager) *DiskCache {
return &DiskCache{
dbmanager: db.GetManager(),
statusManager: statusManager,
ctx: ctx,
}
}
//Start 开始启动统计
func (d *DiskCache) Start() {
d.setcache()
timer := time.NewTimer(time.Minute * 5)
defer timer.Stop()
for {
select {
case <-d.ctx.Done():
return
case <-timer.C:
d.setcache()
timer.Reset(time.Minute * 5)
}
}
}
func (d *DiskCache) setcache() {
logrus.Info("start get all service disk size")
start := time.Now()
d.cache = nil
d.cache = make(map[string]float64)
services, err := d.dbmanager.TenantServiceDao().GetAllServices()
if err != nil {
logrus.Errorln("Error get tenant service when select db :", err)
}
volumes, err := d.dbmanager.TenantServiceVolumeDao().GetAllVolumes()
if err != nil {
logrus.Errorln("Error get tenant service volume when select db :", err)
}
localPath := os.Getenv("LOCAL_DATA_PATH")
sharePath := os.Getenv("SHARE_DATA_PATH")
if localPath == "" {
localPath = "/grlocaldata"
}
if sharePath == "" {
sharePath = "/grdata"
}
var cache = make(map[string]*model.TenantServices)
for _, service := range services {
//默认目录
size := util.GetDirSize(fmt.Sprintf("%s/tenant/%s/service/%s", sharePath, service.TenantID, service.ServiceID))
if size != 0 {
d.cache[service.ServiceID+"_"+service.TenantID] = size
}
cache[service.ServiceID] = service
}
gettenantID := func(serviceID string) string {
if service, ok := cache[serviceID]; ok {
return service.TenantID
}
return ""
}
for _, v := range volumes {
if v.VolumeType == string(model.LocalVolumeType) {
//默认目录
size := util.GetDirSize(fmt.Sprintf("%s/tenant/%s/service/%s", localPath, gettenantID(v.ServiceID), v.ServiceID))
if size != 0 {
d.cache[v.ServiceID+"_"+gettenantID(v.ServiceID)] += size
}
}
}
logrus.Infof("end get all service disk size,time consum %d s", time.Now().Sub(start).Seconds())
}
//Get 获取磁盘统计结果
func (d *DiskCache) Get() map[string]float64 {
return d.cache
}

164
pkg/worker/monitor/collector/collector.go Normal file
View File

@ -0,0 +1,164 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package collector
import (
"os"
"strings"
"time"
"github.com/goodrain/rainbond/pkg/db/model"
"github.com/goodrain/rainbond/pkg/worker/monitor/cache"
"github.com/Sirupsen/logrus"
"github.com/goodrain/rainbond/pkg/db"
"github.com/goodrain/rainbond/pkg/status"
"github.com/prometheus/client_golang/prometheus"
)
//Exporter 收集器
type Exporter struct {
dsn string
error prometheus.Gauge
totalScrapes prometheus.Counter
scrapeErrors *prometheus.CounterVec
memoryUse *prometheus.GaugeVec
fsUse *prometheus.GaugeVec
workerUp prometheus.Gauge
dbmanager db.Manager
statusManager status.ServiceStatusManager
cache *cache.DiskCache
}
var scrapeDurationDesc = prometheus.NewDesc(
prometheus.BuildFQName(namespace, "exporter", "collector_duration_seconds"),
"Collector time duration.",
[]string{"collector"}, nil,
)
//Describe Describe
func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
metricCh := make(chan prometheus.Metric)
doneCh := make(chan struct{})
go func() {
for m := range metricCh {
ch <- m.Desc()
}
close(doneCh)
}()
e.Collect(metricCh)
close(metricCh)
<-doneCh
}
// Collect implements prometheus.Collector.
func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
e.scrape(ch)
ch <- e.totalScrapes
ch <- e.error
e.fsUse.Collect(ch)
e.memoryUse.Collect(ch)
e.scrapeErrors.Collect(ch)
ch <- e.workerUp
}
func (e *Exporter) scrape(ch chan<- prometheus.Metric) {
e.totalScrapes.Inc()
var err error
scrapeTime := time.Now()
services, err := e.dbmanager.TenantServiceDao().GetAllServices()
if err != nil {
logrus.Errorln("Error scraping for tenant service when select db :", err)
e.scrapeErrors.WithLabelValues("db.getservices").Inc()
e.error.Set(1)
}
localPath := os.Getenv("LOCAL_DATA_PATH")
sharePath := os.Getenv("SHARE_DATA_PATH")
if localPath == "" {
localPath = "/grlocaldata"
}
if sharePath == "" {
sharePath = "/grdata"
}
//获取内存使用情况
for _, service := range services {
if appstatus, err := e.statusManager.GetStatus(service.ServiceID); err == nil {
if appstatus != status.CLOSED && appstatus != status.UNDEPLOY && appstatus != status.DEPLOYING {
e.memoryUse.WithLabelValues(service.TenantID, service.ServiceID, appstatus).Set(float64(service.ContainerMemory * service.Replicas))
}
}
}
ch <- prometheus.MustNewConstMetric(scrapeDurationDesc, prometheus.GaugeValue, time.Since(scrapeTime).Seconds(), "collect.memory")
scrapeTime = time.Now()
diskcache := e.cache.Get()
for k, v := range diskcache {
key := strings.Split(k, "_")
if len(key) == 2 {
e.fsUse.WithLabelValues(key[1], key[0], string(model.ShareFileVolumeType)).Set(v)
}
}
ch <- prometheus.MustNewConstMetric(scrapeDurationDesc, prometheus.GaugeValue, time.Since(scrapeTime).Seconds(), "collect.fs")
}
var namespace = "app_resource"
//New 创建一个收集器
func New(statusManager status.ServiceStatusManager, cache *cache.DiskCache) *Exporter {
return &Exporter{
totalScrapes: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: "exporter",
Name: "scrapes_total",
Help: "Total number of times Worker was scraped for metrics.",
}),
scrapeErrors: prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: "exporter",
Name: "scrape_errors_total",
Help: "Total number of times an error occurred scraping a Worker.",
}, []string{"collector"}),
error: prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: "exporter",
Name: "last_scrape_error",
Help: "Whether the last scrape of metrics from Worker resulted in an error (1 for error, 0 for success).",
}),
workerUp: prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: namespace,
Name: "up",
Help: "Whether the Worker server is up.",
}),
memoryUse: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "appmemory",
Help: "tenant service memory used.",
}, []string{"tenant_id", "service_id", "service_status"}),
fsUse: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Name: "appfs",
Help: "tenant service fs used.",
}, []string{"tenant_id", "service_id", "volume_type"}),
dbmanager: db.GetManager(),
statusManager: statusManager,
cache: cache,
}
}

96
pkg/worker/monitor/exporter.go Normal file
View File

@ -0,0 +1,96 @@
// RAINBOND, Application Management Platform
// Copyright (C) 2014-2017 Goodrain Co., Ltd.
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package monitor
import (
"context"
"net/http"
"github.com/Sirupsen/logrus"
"github.com/goodrain/rainbond/cmd/worker/option"
"github.com/goodrain/rainbond/pkg/status"
"github.com/goodrain/rainbond/pkg/worker/monitor/cache"
"github.com/goodrain/rainbond/pkg/worker/monitor/collector"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/prometheus/common/log"
)
//ExporterManager app resource exporter
type ExporterManager struct {
ctx context.Context
cancel context.CancelFunc
config option.Config
stopChan chan struct{}
statusManager status.ServiceStatusManager
cache *cache.DiskCache
}
//NewManager return *NewManager
func NewManager(c option.Config, statusManager status.ServiceStatusManager) *ExporterManager {
ctx, cancel := context.WithCancel(context.Background())
cache := cache.CreatDiskCache(ctx, statusManager)
return &ExporterManager{
ctx: ctx,
cancel: cancel,
config: c,
stopChan: make(chan struct{}),
statusManager: statusManager,
cache: cache,
}
}
func (t *ExporterManager) handler(w http.ResponseWriter, r *http.Request) {
registry := prometheus.NewRegistry()
registry.MustRegister(collector.New(t.statusManager, t.cache))
gatherers := prometheus.Gatherers{
prometheus.DefaultGatherer,
registry,
}
// Delegate http serving to Prometheus client library, which will call collector.Collect.
h := promhttp.HandlerFor(gatherers, promhttp.HandlerOpts{})
h.ServeHTTP(w, r)
}
//Start 启动
func (t *ExporterManager) Start() error {
http.HandleFunc(t.config.PrometheusMetricPath, prometheus.InstrumentHandlerFunc("metrics", t.handler))
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(`<html>
<head><title>Worker exporter</title></head>
<body>
<h1>Worker exporter</h1>
<p><a href='` + t.config.PrometheusMetricPath + `'>Metrics</a></p>
</body>
</html>
`))
})
go t.cache.Start()
log.Infoln("Listening on", t.config.Listen)
go func() {
log.Fatal(http.ListenAndServe(t.config.Listen, nil))
}()
logrus.Info("start app resource exporter success.")
return nil
}
//Stop 停止
func (t *ExporterManager) Stop() {
t.cancel()
}

8
release.sh
View File

@ -25,7 +25,7 @@ else
buildRelease=0.$git_commit
fi
if [ -z "$VERSION" ];then
VERSION=3.4.1
VERSION=cloud
fi
release_desc=${branch_info}-${VERSION}-${buildRelease}
@ -46,13 +46,13 @@ function prepare() {
function build() {
echo "---> Build Binary For ACP"
echo "acp plugins version:$release_desc"
sed -i "s/0.0.0/$release_desc/g" ./cmd/version.go
echo "build rainbond-node"
docker run --rm -v `pwd`:${WORK_DIR} -w ${WORK_DIR} -it golang:1.8.3 go build -ldflags '-w -s' -o $releasedir/dist/usr/local/bin/${BASE_NAME}-node ./cmd/node
echo "grctl version:$release_desc"
sed -i "s/0.0.0/$release_desc/g" ./cmd/grctl/option/version.go
echo "build rainbond-grctl"
docker run --rm -v `pwd`:${WORK_DIR} -w ${WORK_DIR} -it golang:1.8.3 go build -ldflags '-w -s' -o $releasedir/dist/usr/local/bin/${BASE_NAME}-grctl ./cmd/grctl
sed -i "s/$release_desc/0.0.0/g" ./cmd/grctl/option/version.go
sed -i "s/$release_desc/0.0.0/g" ./cmd/version.go
}
function build::rpm() {

31
test/mapper.yml Normal file
View File

@ -0,0 +1,31 @@
#
mappings:
- match: "*.*.*.request.*"
name: "app_request"
labels:
service_id: "$1"
port: "$2"
protocol: $3
method: "$4"
- match: "*.*.*.request.unusual.*"
name: "app_request_unusual"
labels:
service_id: "$1"
port: "$2"
protocol: $3
code: "$4"
- match: "*.*.*.requesttime.*"
name: "app_requesttime"
labels:
service_id: "$1"
port: "$2"
protocol: $3
mode: "$4"
- match: "*.*.*.requestclient"
name: "app_requestclient"
labels:
service_id: "$1"
port: "$2"
protocol: $3

28
vendor/github.com/howeyc/fsnotify/AUTHORS generated vendored Normal file
View File

@ -0,0 +1,28 @@
# Names should be added to this file as
# Name or Organization <email address>
# The email address is not required for organizations.
# You can update this list using the following command:
#
# $ git shortlog -se | awk '{print $2 " " $3 " " $4}'
# Please keep the list sorted.
Adrien Bustany <adrien@bustany.org>
Caleb Spare <cespare@gmail.com>
Case Nelson <case@teammating.com>
Chris Howey <howeyc@gmail.com> <chris@howey.me>
Christoffer Buchholz <christoffer.buchholz@gmail.com>
Dave Cheney <dave@cheney.net>
Francisco Souza <f@souza.cc>
John C Barstow
Kelvin Fo <vmirage@gmail.com>
Nathan Youngman <git@nathany.com>
Paul Hammond <paul@paulhammond.org>
Pursuit92 <JoshChase@techpursuit.net>
Rob Figueiredo <robfig@gmail.com>
Travis Cline <travis.cline@gmail.com>
Tudor Golubenco <tudor.g@gmail.com>
bronze1man <bronze1man@gmail.com>
debrando <denis.brandolini@gmail.com>
henrikedwards <henrik.edwards@gmail.com>

160
vendor/github.com/howeyc/fsnotify/CHANGELOG.md generated vendored Normal file
View File

@ -0,0 +1,160 @@
# Changelog
## v0.9.0 / 2014-01-17
* IsAttrib() for events that only concern a file's metadata [#79][] (thanks @abustany)
* [Fix] kqueue: fix deadlock [#77][] (thanks @cespare)
* [NOTICE] Development has moved to `code.google.com/p/go.exp/fsnotify` in preparation for inclusion in the Go standard library.
## v0.8.12 / 2013-11-13
* [API] Remove FD_SET and friends from Linux adapter
## v0.8.11 / 2013-11-02
* [Doc] Add Changelog [#72][] (thanks @nathany)
* [Doc] Spotlight and double modify events on OS X [#62][] (reported by @paulhammond)
## v0.8.10 / 2013-10-19
* [Fix] kqueue: remove file watches when parent directory is removed [#71][] (reported by @mdwhatcott)
* [Fix] kqueue: race between Close and readEvents [#70][] (reported by @bernerdschaefer)
* [Doc] specify OS-specific limits in README (thanks @debrando)
## v0.8.9 / 2013-09-08
* [Doc] Contributing (thanks @nathany)
* [Doc] update package path in example code [#63][] (thanks @paulhammond)
* [Doc] GoCI badge in README (Linux only) [#60][]
* [Doc] Cross-platform testing with Vagrant [#59][] (thanks @nathany)
## v0.8.8 / 2013-06-17
* [Fix] Windows: handle `ERROR_MORE_DATA` on Windows [#49][] (thanks @jbowtie)
## v0.8.7 / 2013-06-03
* [API] Make syscall flags internal
* [Fix] inotify: ignore event changes
* [Fix] race in symlink test [#45][] (reported by @srid)
* [Fix] tests on Windows
* lower case error messages
## v0.8.6 / 2013-05-23
* kqueue: Use EVT_ONLY flag on Darwin
* [Doc] Update README with full example
## v0.8.5 / 2013-05-09
* [Fix] inotify: allow monitoring of "broken" symlinks (thanks @tsg)
## v0.8.4 / 2013-04-07
* [Fix] kqueue: watch all file events [#40][] (thanks @ChrisBuchholz)
## v0.8.3 / 2013-03-13
* [Fix] inoitfy/kqueue memory leak [#36][] (reported by @nbkolchin)
* [Fix] kqueue: use fsnFlags for watching a directory [#33][] (reported by @nbkolchin)
## v0.8.2 / 2013-02-07
* [Doc] add Authors
* [Fix] fix data races for map access [#29][] (thanks @fsouza)
## v0.8.1 / 2013-01-09
* [Fix] Windows path separators
* [Doc] BSD License
## v0.8.0 / 2012-11-09
* kqueue: directory watching improvements (thanks @vmirage)
* inotify: add `IN_MOVED_TO` [#25][] (requested by @cpisto)
* [Fix] kqueue: deleting watched directory [#24][] (reported by @jakerr)
## v0.7.4 / 2012-10-09
* [Fix] inotify: fixes from https://codereview.appspot.com/5418045/ (ugorji)
* [Fix] kqueue: preserve watch flags when watching for delete [#21][] (reported by @robfig)
* [Fix] kqueue: watch the directory even if it isn't a new watch (thanks @robfig)
* [Fix] kqueue: modify after recreation of file
## v0.7.3 / 2012-09-27
* [Fix] kqueue: watch with an existing folder inside the watched folder (thanks @vmirage)
* [Fix] kqueue: no longer get duplicate CREATE events
## v0.7.2 / 2012-09-01
* kqueue: events for created directories
## v0.7.1 / 2012-07-14
* [Fix] for renaming files
## v0.7.0 / 2012-07-02
* [Feature] FSNotify flags
* [Fix] inotify: Added file name back to event path
## v0.6.0 / 2012-06-06
* kqueue: watch files after directory created (thanks @tmc)
## v0.5.1 / 2012-05-22
* [Fix] inotify: remove all watches before Close()
## v0.5.0 / 2012-05-03
* [API] kqueue: return errors during watch instead of sending over channel
* kqueue: match symlink behavior on Linux
* inotify: add `DELETE_SELF` (requested by @taralx)
* [Fix] kqueue: handle EINTR (reported by @robfig)
* [Doc] Godoc example [#1][] (thanks @davecheney)
## v0.4.0 / 2012-03-30
* Go 1 released: build with go tool
* [Feature] Windows support using winfsnotify
* Windows does not have attribute change notifications
* Roll attribute notifications into IsModify
## v0.3.0 / 2012-02-19
* kqueue: add files when watch directory
## v0.2.0 / 2011-12-30
* update to latest Go weekly code
## v0.1.0 / 2011-10-19
* kqueue: add watch on file creation to match inotify
* kqueue: create file event
* inotify: ignore `IN_IGNORED` events
* event String()
* linux: common FileEvent functions
* initial commit
[#79]: https://github.com/howeyc/fsnotify/pull/79
[#77]: https://github.com/howeyc/fsnotify/pull/77
[#72]: https://github.com/howeyc/fsnotify/issues/72
[#71]: https://github.com/howeyc/fsnotify/issues/71
[#70]: https://github.com/howeyc/fsnotify/issues/70
[#63]: https://github.com/howeyc/fsnotify/issues/63
[#62]: https://github.com/howeyc/fsnotify/issues/62
[#60]: https://github.com/howeyc/fsnotify/issues/60
[#59]: https://github.com/howeyc/fsnotify/issues/59
[#49]: https://github.com/howeyc/fsnotify/issues/49
[#45]: https://github.com/howeyc/fsnotify/issues/45
[#40]: https://github.com/howeyc/fsnotify/issues/40
[#36]: https://github.com/howeyc/fsnotify/issues/36
[#33]: https://github.com/howeyc/fsnotify/issues/33
[#29]: https://github.com/howeyc/fsnotify/issues/29
[#25]: https://github.com/howeyc/fsnotify/issues/25
[#24]: https://github.com/howeyc/fsnotify/issues/24
[#21]: https://github.com/howeyc/fsnotify/issues/21
[#1]: https://github.com/howeyc/fsnotify/issues/1

7
vendor/github.com/howeyc/fsnotify/CONTRIBUTING.md generated vendored Normal file
View File

@ -0,0 +1,7 @@
# Contributing
## Moving Notice
There is a fork being actively developed with a new API in preparation for the Go Standard Library:
[github.com/go-fsnotify/fsnotify](https://github.com/go-fsnotify/fsnotify)

28
vendor/github.com/howeyc/fsnotify/LICENSE generated vendored Normal file
View File

@ -0,0 +1,28 @@
Copyright (c) 2012 The Go Authors. All rights reserved.
Copyright (c) 2012 fsnotify Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

93
vendor/github.com/howeyc/fsnotify/README.md generated vendored Normal file
View File

@ -0,0 +1,93 @@
# File system notifications for Go
[![GoDoc](https://godoc.org/github.com/howeyc/fsnotify?status.png)](http://godoc.org/github.com/howeyc/fsnotify)
Cross platform: Windows, Linux, BSD and OS X.
## Moving Notice
There is a fork being actively developed with a new API in preparation for the Go Standard Library:
[github.com/go-fsnotify/fsnotify](https://github.com/go-fsnotify/fsnotify)
## Example:
```go
package main
import (
"log"
"github.com/howeyc/fsnotify"
)
func main() {
watcher, err := fsnotify.NewWatcher()
if err != nil {
log.Fatal(err)
}
done := make(chan bool)
// Process events
go func() {
for {
select {
case ev := <-watcher.Event:
log.Println("event:", ev)
case err := <-watcher.Error:
log.Println("error:", err)
}
}
}()
err = watcher.Watch("testDir")
if err != nil {
log.Fatal(err)
}
// Hang so program doesn't exit
<-done
/* ... do stuff ... */
watcher.Close()
}
```
For each event:
* Name
* IsCreate()
* IsDelete()
* IsModify()
* IsRename()
## FAQ
**When a file is moved to another directory is it still being watched?**
No (it shouldn't be, unless you are watching where it was moved to).
**When I watch a directory, are all subdirectories watched as well?**
No, you must add watches for any directory you want to watch (a recursive watcher is in the works [#56][]).
**Do I have to watch the Error and Event channels in a separate goroutine?**
As of now, yes. Looking into making this single-thread friendly (see [#7][])
**Why am I receiving multiple events for the same file on OS X?**
Spotlight indexing on OS X can result in multiple events (see [#62][]). A temporary workaround is to add your folder(s) to the *Spotlight Privacy settings* until we have a native FSEvents implementation (see [#54][]).
**How many files can be watched at once?**
There are OS-specific limits as to how many watches can be created:
* Linux: /proc/sys/fs/inotify/max_user_watches contains the limit,
reaching this limit results in a "no space left on device" error.
* BSD / OSX: sysctl variables "kern.maxfiles" and "kern.maxfilesperproc", reaching these limits results in a "too many open files" error.
[#62]: https://github.com/howeyc/fsnotify/issues/62
[#56]: https://github.com/howeyc/fsnotify/issues/56
[#54]: https://github.com/howeyc/fsnotify/issues/54
[#7]: https://github.com/howeyc/fsnotify/issues/7

111
vendor/github.com/howeyc/fsnotify/fsnotify.go generated vendored Normal file
View File

@ -0,0 +1,111 @@
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package fsnotify implements file system notification.
package fsnotify
import "fmt"
const (
FSN_CREATE = 1
FSN_MODIFY = 2
FSN_DELETE = 4
FSN_RENAME = 8
FSN_ALL = FSN_MODIFY | FSN_DELETE | FSN_RENAME | FSN_CREATE
)
// Purge events from interal chan to external chan if passes filter
func (w *Watcher) purgeEvents() {
for ev := range w.internalEvent {
sendEvent := false
w.fsnmut.Lock()
fsnFlags := w.fsnFlags[ev.Name]
w.fsnmut.Unlock()
if (fsnFlags&FSN_CREATE == FSN_CREATE) && ev.IsCreate() {
sendEvent = true
}
if (fsnFlags&FSN_MODIFY == FSN_MODIFY) && ev.IsModify() {
sendEvent = true
}
if (fsnFlags&FSN_DELETE == FSN_DELETE) && ev.IsDelete() {
sendEvent = true
}
if (fsnFlags&FSN_RENAME == FSN_RENAME) && ev.IsRename() {
sendEvent = true
}
if sendEvent {
w.Event <- ev
}
// If there's no file, then no more events for user
// BSD must keep watch for internal use (watches DELETEs to keep track
// what files exist for create events)
if ev.IsDelete() {
w.fsnmut.Lock()
delete(w.fsnFlags, ev.Name)
w.fsnmut.Unlock()
}
}
close(w.Event)
}
// Watch a given file path
func (w *Watcher) Watch(path string) error {
return w.WatchFlags(path, FSN_ALL)
}
// Watch a given file path for a particular set of notifications (FSN_MODIFY etc.)
func (w *Watcher) WatchFlags(path string, flags uint32) error {
w.fsnmut.Lock()
w.fsnFlags[path] = flags
w.fsnmut.Unlock()
return w.watch(path)
}
// Remove a watch on a file
func (w *Watcher) RemoveWatch(path string) error {
w.fsnmut.Lock()
delete(w.fsnFlags, path)
w.fsnmut.Unlock()
return w.removeWatch(path)
}
// String formats the event e in the form
// "filename: DELETE|MODIFY|..."
func (e *FileEvent) String() string {
var events string = ""
if e.IsCreate() {
events += "|" + "CREATE"
}
if e.IsDelete() {
events += "|" + "DELETE"
}
if e.IsModify() {
events += "|" + "MODIFY"
}
if e.IsRename() {
events += "|" + "RENAME"
}
if e.IsAttrib() {
events += "|" + "ATTRIB"
}
if len(events) > 0 {
events = events[1:]
}
return fmt.Sprintf("%q: %s", e.Name, events)
}

496
vendor/github.com/howeyc/fsnotify/fsnotify_bsd.go generated vendored Normal file
View File

@ -0,0 +1,496 @@
// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build freebsd openbsd netbsd dragonfly darwin
package fsnotify
import (
"errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"sync"
"syscall"
)
const (
// Flags (from <sys/event.h>)
sys_NOTE_DELETE = 0x0001 /* vnode was removed */
sys_NOTE_WRITE = 0x0002 /* data contents changed */
sys_NOTE_EXTEND = 0x0004 /* size increased */
sys_NOTE_ATTRIB = 0x0008 /* attributes changed */
sys_NOTE_LINK = 0x0010 /* link count changed */
sys_NOTE_RENAME = 0x0020 /* vnode was renamed */
sys_NOTE_REVOKE = 0x0040 /* vnode access was revoked */
// Watch all events
sys_NOTE_ALLEVENTS = sys_NOTE_DELETE | sys_NOTE_WRITE | sys_NOTE_ATTRIB | sys_NOTE_RENAME
// Block for 100 ms on each call to kevent
keventWaitTime = 100e6
)
type FileEvent struct {
mask uint32 // Mask of events
Name string // File name (optional)
create bool // set by fsnotify package if found new file
}
// IsCreate reports whether the FileEvent was triggered by a creation
func (e *FileEvent) IsCreate() bool { return e.create }
// IsDelete reports whether the FileEvent was triggered by a delete
func (e *FileEvent) IsDelete() bool { return (e.mask & sys_NOTE_DELETE) == sys_NOTE_DELETE }
// IsModify reports whether the FileEvent was triggered by a file modification
func (e *FileEvent) IsModify() bool {
return ((e.mask&sys_NOTE_WRITE) == sys_NOTE_WRITE || (e.mask&sys_NOTE_ATTRIB) == sys_NOTE_ATTRIB)
}
// IsRename reports whether the FileEvent was triggered by a change name
func (e *FileEvent) IsRename() bool { return (e.mask & sys_NOTE_RENAME) == sys_NOTE_RENAME }
// IsAttrib reports whether the FileEvent was triggered by a change in the file metadata.
func (e *FileEvent) IsAttrib() bool {
return (e.mask & sys_NOTE_ATTRIB) == sys_NOTE_ATTRIB
}
type Watcher struct {
mu sync.Mutex // Mutex for the Watcher itself.
kq int // File descriptor (as returned by the kqueue() syscall)
watches map[string]int // Map of watched file descriptors (key: path)
wmut sync.Mutex // Protects access to watches.
fsnFlags map[string]uint32 // Map of watched files to flags used for filter
fsnmut sync.Mutex // Protects access to fsnFlags.
enFlags map[string]uint32 // Map of watched files to evfilt note flags used in kqueue
enmut sync.Mutex // Protects access to enFlags.
paths map[int]string // Map of watched paths (key: watch descriptor)
finfo map[int]os.FileInfo // Map of file information (isDir, isReg; key: watch descriptor)
pmut sync.Mutex // Protects access to paths and finfo.
fileExists map[string]bool // Keep track of if we know this file exists (to stop duplicate create events)
femut sync.Mutex // Protects access to fileExists.
externalWatches map[string]bool // Map of watches added by user of the library.
ewmut sync.Mutex // Protects access to externalWatches.
Error chan error // Errors are sent on this channel
internalEvent chan *FileEvent // Events are queued on this channel
Event chan *FileEvent // Events are returned on this channel
done chan bool // Channel for sending a "quit message" to the reader goroutine
isClosed bool // Set to true when Close() is first called
}
// NewWatcher creates and returns a new kevent instance using kqueue(2)
func NewWatcher() (*Watcher, error) {
fd, errno := syscall.Kqueue()
if fd == -1 {
return nil, os.NewSyscallError("kqueue", errno)
}
w := &Watcher{
kq: fd,
watches: make(map[string]int),
fsnFlags: make(map[string]uint32),
enFlags: make(map[string]uint32),
paths: make(map[int]string),
finfo: make(map[int]os.FileInfo),
fileExists: make(map[string]bool),
externalWatches: make(map[string]bool),
internalEvent: make(chan *FileEvent),
Event: make(chan *FileEvent),
Error: make(chan error),
done: make(chan bool, 1),
}
go w.readEvents()
go w.purgeEvents()
return w, nil
}
// Close closes a kevent watcher instance
// It sends a message to the reader goroutine to quit and removes all watches
// associated with the kevent instance
func (w *Watcher) Close() error {
w.mu.Lock()
if w.isClosed {
w.mu.Unlock()
return nil
}
w.isClosed = true
w.mu.Unlock()
// Send "quit" message to the reader goroutine
w.done <- true
w.wmut.Lock()
ws := w.watches
w.wmut.Unlock()
for path := range ws {
w.removeWatch(path)
}
return nil
}
// AddWatch adds path to the watched file set.
// The flags are interpreted as described in kevent(2).
func (w *Watcher) addWatch(path string, flags uint32) error {
w.mu.Lock()
if w.isClosed {
w.mu.Unlock()
return errors.New("kevent instance already closed")
}
w.mu.Unlock()
watchDir := false
w.wmut.Lock()
watchfd, found := w.watches[path]
w.wmut.Unlock()
if !found {
fi, errstat := os.Lstat(path)
if errstat != nil {
return errstat
}
// don't watch socket
if fi.Mode()&os.ModeSocket == os.ModeSocket {
return nil
}
// Follow Symlinks
// Unfortunately, Linux can add bogus symlinks to watch list without
// issue, and Windows can't do symlinks period (AFAIK). To maintain
// consistency, we will act like everything is fine. There will simply
// be no file events for broken symlinks.
// Hence the returns of nil on errors.
if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
path, err := filepath.EvalSymlinks(path)
if err != nil {
return nil
}
fi, errstat = os.Lstat(path)
if errstat != nil {
return nil
}
}
fd, errno := syscall.Open(path, open_FLAGS, 0700)
if fd == -1 {
return errno
}
watchfd = fd
w.wmut.Lock()
w.watches[path] = watchfd
w.wmut.Unlock()
w.pmut.Lock()
w.paths[watchfd] = path
w.finfo[watchfd] = fi
w.pmut.Unlock()
}
// Watch the directory if it has not been watched before.
w.pmut.Lock()
w.enmut.Lock()
if w.finfo[watchfd].IsDir() &&
(flags&sys_NOTE_WRITE) == sys_NOTE_WRITE &&
(!found || (w.enFlags[path]&sys_NOTE_WRITE) != sys_NOTE_WRITE) {
watchDir = true
}
w.enmut.Unlock()
w.pmut.Unlock()
w.enmut.Lock()
w.enFlags[path] = flags
w.enmut.Unlock()
var kbuf [1]syscall.Kevent_t
watchEntry := &kbuf[0]
watchEntry.Fflags = flags
syscall.SetKevent(watchEntry, watchfd, syscall.EVFILT_VNODE, syscall.EV_ADD|syscall.EV_CLEAR)
entryFlags := watchEntry.Flags
success, errno := syscall.Kevent(w.kq, kbuf[:], nil, nil)
if success == -1 {
return errno
} else if (entryFlags & syscall.EV_ERROR) == syscall.EV_ERROR {
return errors.New("kevent add error")
}
if watchDir {
errdir := w.watchDirectoryFiles(path)
if errdir != nil {
return errdir
}
}
return nil
}
// Watch adds path to the watched file set, watching all events.
func (w *Watcher) watch(path string) error {
w.ewmut.Lock()
w.externalWatches[path] = true
w.ewmut.Unlock()
return w.addWatch(path, sys_NOTE_ALLEVENTS)
}
// RemoveWatch removes path from the watched file set.
func (w *Watcher) removeWatch(path string) error {
w.wmut.Lock()
watchfd, ok := w.watches[path]
w.wmut.Unlock()
if !ok {
return errors.New(fmt.Sprintf("can't remove non-existent kevent watch for: %s", path))
}
var kbuf [1]syscall.Kevent_t
watchEntry := &kbuf[0]
syscall.SetKevent(watchEntry, watchfd, syscall.EVFILT_VNODE, syscall.EV_DELETE)
entryFlags := watchEntry.Flags
success, errno := syscall.Kevent(w.kq, kbuf[:], nil, nil)
if success == -1 {
return os.NewSyscallError("kevent_rm_watch", errno)
} else if (entryFlags & syscall.EV_ERROR) == syscall.EV_ERROR {
return errors.New("kevent rm error")
}
syscall.Close(watchfd)
w.wmut.Lock()
delete(w.watches, path)
w.wmut.Unlock()
w.enmut.Lock()
delete(w.enFlags, path)
w.enmut.Unlock()
w.pmut.Lock()
delete(w.paths, watchfd)
fInfo := w.finfo[watchfd]
delete(w.finfo, watchfd)
w.pmut.Unlock()
// Find all watched paths that are in this directory that are not external.
if fInfo.IsDir() {
var pathsToRemove []string
w.pmut.Lock()
for _, wpath := range w.paths {
wdir, _ := filepath.Split(wpath)
if filepath.Clean(wdir) == filepath.Clean(path) {
w.ewmut.Lock()
if !w.externalWatches[wpath] {
pathsToRemove = append(pathsToRemove, wpath)
}
w.ewmut.Unlock()
}
}
w.pmut.Unlock()
for _, p := range pathsToRemove {
// Since these are internal, not much sense in propagating error
// to the user, as that will just confuse them with an error about
// a path they did not explicitly watch themselves.
w.removeWatch(p)
}
}
return nil
}
// readEvents reads from the kqueue file descriptor, converts the
// received events into Event objects and sends them via the Event channel
func (w *Watcher) readEvents() {
var (
eventbuf [10]syscall.Kevent_t // Event buffer
events []syscall.Kevent_t // Received events
twait *syscall.Timespec // Time to block waiting for events
n int // Number of events returned from kevent
errno error // Syscall errno
)
events = eventbuf[0:0]
twait = new(syscall.Timespec)
*twait = syscall.NsecToTimespec(keventWaitTime)
for {
// See if there is a message on the "done" channel
var done bool
select {
case done = <-w.done:
default:
}
// If "done" message is received
if done {
errno := syscall.Close(w.kq)
if errno != nil {
w.Error <- os.NewSyscallError("close", errno)
}
close(w.internalEvent)
close(w.Error)
return
}
// Get new events
if len(events) == 0 {
n, errno = syscall.Kevent(w.kq, nil, eventbuf[:], twait)
// EINTR is okay, basically the syscall was interrupted before
// timeout expired.
if errno != nil && errno != syscall.EINTR {
w.Error <- os.NewSyscallError("kevent", errno)
continue
}
// Received some events
if n > 0 {
events = eventbuf[0:n]
}
}
// Flush the events we received to the events channel
for len(events) > 0 {
fileEvent := new(FileEvent)
watchEvent := &events[0]
fileEvent.mask = uint32(watchEvent.Fflags)
w.pmut.Lock()
fileEvent.Name = w.paths[int(watchEvent.Ident)]
fileInfo := w.finfo[int(watchEvent.Ident)]
w.pmut.Unlock()
if fileInfo != nil && fileInfo.IsDir() && !fileEvent.IsDelete() {
// Double check to make sure the directory exist. This can happen when
// we do a rm -fr on a recursively watched folders and we receive a
// modification event first but the folder has been deleted and later
// receive the delete event
if _, err := os.Lstat(fileEvent.Name); os.IsNotExist(err) {
// mark is as delete event
fileEvent.mask |= sys_NOTE_DELETE
}
}
if fileInfo != nil && fileInfo.IsDir() && fileEvent.IsModify() && !fileEvent.IsDelete() {
w.sendDirectoryChangeEvents(fileEvent.Name)
} else {
// Send the event on the events channel
w.internalEvent <- fileEvent
}
// Move to next event
events = events[1:]
if fileEvent.IsRename() {
w.removeWatch(fileEvent.Name)
w.femut.Lock()
delete(w.fileExists, fileEvent.Name)
w.femut.Unlock()
}
if fileEvent.IsDelete() {
w.removeWatch(fileEvent.Name)
w.femut.Lock()
delete(w.fileExists, fileEvent.Name)
w.femut.Unlock()
// Look for a file that may have overwritten this
// (ie mv f1 f2 will delete f2 then create f2)
fileDir, _ := filepath.Split(fileEvent.Name)
fileDir = filepath.Clean(fileDir)
w.wmut.Lock()
_, found := w.watches[fileDir]
w.wmut.Unlock()
if found {
// make sure the directory exist before we watch for changes. When we
// do a recursive watch and perform rm -fr, the parent directory might
// have gone missing, ignore the missing directory and let the
// upcoming delete event remove the watch form the parent folder
if _, err := os.Lstat(fileDir); !os.IsNotExist(err) {
w.sendDirectoryChangeEvents(fileDir)
}
}
}
}
}
}
func (w *Watcher) watchDirectoryFiles(dirPath string) error {
// Get all files
files, err := ioutil.ReadDir(dirPath)
if err != nil {
return err
}
// Search for new files
for _, fileInfo := range files {
filePath := filepath.Join(dirPath, fileInfo.Name())
// Inherit fsnFlags from parent directory
w.fsnmut.Lock()
if flags, found := w.fsnFlags[dirPath]; found {
w.fsnFlags[filePath] = flags
} else {
w.fsnFlags[filePath] = FSN_ALL
}
w.fsnmut.Unlock()
if fileInfo.IsDir() == false {
// Watch file to mimic linux fsnotify
e := w.addWatch(filePath, sys_NOTE_ALLEVENTS)
if e != nil {
return e
}
} else {
// If the user is currently watching directory
// we want to preserve the flags used
w.enmut.Lock()
currFlags, found := w.enFlags[filePath]
w.enmut.Unlock()
var newFlags uint32 = sys_NOTE_DELETE
if found {
newFlags |= currFlags
}
// Linux gives deletes if not explicitly watching
e := w.addWatch(filePath, newFlags)
if e != nil {
return e
}
}
w.femut.Lock()
w.fileExists[filePath] = true
w.femut.Unlock()
}
return nil
}
// sendDirectoryEvents searches the directory for newly created files
// and sends them over the event channel. This functionality is to have
// the BSD version of fsnotify match linux fsnotify which provides a
// create event for files created in a watched directory.
func (w *Watcher) sendDirectoryChangeEvents(dirPath string) {
// Get all files
files, err := ioutil.ReadDir(dirPath)
if err != nil {
w.Error <- err
}
// Search for new files
for _, fileInfo := range files {
filePath := filepath.Join(dirPath, fileInfo.Name())
w.femut.Lock()
_, doesExist := w.fileExists[filePath]
w.femut.Unlock()
if !doesExist {
// Inherit fsnFlags from parent directory
w.fsnmut.Lock()
if flags, found := w.fsnFlags[dirPath]; found {
w.fsnFlags[filePath] = flags
} else {
w.fsnFlags[filePath] = FSN_ALL
}
w.fsnmut.Unlock()
// Send create event
fileEvent := new(FileEvent)
fileEvent.Name = filePath
fileEvent.create = true
w.internalEvent <- fileEvent
}
w.femut.Lock()
w.fileExists[filePath] = true
w.femut.Unlock()
}
w.watchDirectoryFiles(dirPath)
}

304
vendor/github.com/howeyc/fsnotify/fsnotify_linux.go generated vendored Normal file
View File

@ -0,0 +1,304 @@
// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build linux
package fsnotify
import (
"errors"
"fmt"
"os"
"strings"
"sync"
"syscall"
"unsafe"
)
const (
// Options for inotify_init() are not exported
// sys_IN_CLOEXEC uint32 = syscall.IN_CLOEXEC
// sys_IN_NONBLOCK uint32 = syscall.IN_NONBLOCK
// Options for AddWatch
sys_IN_DONT_FOLLOW uint32 = syscall.IN_DONT_FOLLOW
sys_IN_ONESHOT uint32 = syscall.IN_ONESHOT
sys_IN_ONLYDIR uint32 = syscall.IN_ONLYDIR
// The "sys_IN_MASK_ADD" option is not exported, as AddWatch
// adds it automatically, if there is already a watch for the given path
// sys_IN_MASK_ADD uint32 = syscall.IN_MASK_ADD
// Events
sys_IN_ACCESS uint32 = syscall.IN_ACCESS
sys_IN_ALL_EVENTS uint32 = syscall.IN_ALL_EVENTS
sys_IN_ATTRIB uint32 = syscall.IN_ATTRIB
sys_IN_CLOSE uint32 = syscall.IN_CLOSE
sys_IN_CLOSE_NOWRITE uint32 = syscall.IN_CLOSE_NOWRITE
sys_IN_CLOSE_WRITE uint32 = syscall.IN_CLOSE_WRITE
sys_IN_CREATE uint32 = syscall.IN_CREATE
sys_IN_DELETE uint32 = syscall.IN_DELETE
sys_IN_DELETE_SELF uint32 = syscall.IN_DELETE_SELF
sys_IN_MODIFY uint32 = syscall.IN_MODIFY
sys_IN_MOVE uint32 = syscall.IN_MOVE
sys_IN_MOVED_FROM uint32 = syscall.IN_MOVED_FROM
sys_IN_MOVED_TO uint32 = syscall.IN_MOVED_TO
sys_IN_MOVE_SELF uint32 = syscall.IN_MOVE_SELF
sys_IN_OPEN uint32 = syscall.IN_OPEN
sys_AGNOSTIC_EVENTS = sys_IN_MOVED_TO | sys_IN_MOVED_FROM | sys_IN_CREATE | sys_IN_ATTRIB | sys_IN_MODIFY | sys_IN_MOVE_SELF | sys_IN_DELETE | sys_IN_DELETE_SELF
// Special events
sys_IN_ISDIR uint32 = syscall.IN_ISDIR
sys_IN_IGNORED uint32 = syscall.IN_IGNORED
sys_IN_Q_OVERFLOW uint32 = syscall.IN_Q_OVERFLOW
sys_IN_UNMOUNT uint32 = syscall.IN_UNMOUNT
)
type FileEvent struct {
mask uint32 // Mask of events
cookie uint32 // Unique cookie associating related events (for rename(2))
Name string // File name (optional)
}
// IsCreate reports whether the FileEvent was triggered by a creation
func (e *FileEvent) IsCreate() bool {
return (e.mask&sys_IN_CREATE) == sys_IN_CREATE || (e.mask&sys_IN_MOVED_TO) == sys_IN_MOVED_TO
}
// IsDelete reports whether the FileEvent was triggered by a delete
func (e *FileEvent) IsDelete() bool {
return (e.mask&sys_IN_DELETE_SELF) == sys_IN_DELETE_SELF || (e.mask&sys_IN_DELETE) == sys_IN_DELETE
}
// IsModify reports whether the FileEvent was triggered by a file modification or attribute change
func (e *FileEvent) IsModify() bool {
return ((e.mask&sys_IN_MODIFY) == sys_IN_MODIFY || (e.mask&sys_IN_ATTRIB) == sys_IN_ATTRIB)
}
// IsRename reports whether the FileEvent was triggered by a change name
func (e *FileEvent) IsRename() bool {
return ((e.mask&sys_IN_MOVE_SELF) == sys_IN_MOVE_SELF || (e.mask&sys_IN_MOVED_FROM) == sys_IN_MOVED_FROM)
}
// IsAttrib reports whether the FileEvent was triggered by a change in the file metadata.
func (e *FileEvent) IsAttrib() bool {
return (e.mask & sys_IN_ATTRIB) == sys_IN_ATTRIB
}
type watch struct {
wd uint32 // Watch descriptor (as returned by the inotify_add_watch() syscall)
flags uint32 // inotify flags of this watch (see inotify(7) for the list of valid flags)
}
type Watcher struct {
mu sync.Mutex // Map access
fd int // File descriptor (as returned by the inotify_init() syscall)
watches map[string]*watch // Map of inotify watches (key: path)
fsnFlags map[string]uint32 // Map of watched files to flags used for filter
fsnmut sync.Mutex // Protects access to fsnFlags.
paths map[int]string // Map of watched paths (key: watch descriptor)
Error chan error // Errors are sent on this channel
internalEvent chan *FileEvent // Events are queued on this channel
Event chan *FileEvent // Events are returned on this channel
done chan bool // Channel for sending a "quit message" to the reader goroutine
isClosed bool // Set to true when Close() is first called
}
// NewWatcher creates and returns a new inotify instance using inotify_init(2)
func NewWatcher() (*Watcher, error) {
fd, errno := syscall.InotifyInit()
if fd == -1 {
return nil, os.NewSyscallError("inotify_init", errno)
}
w := &Watcher{
fd: fd,
watches: make(map[string]*watch),
fsnFlags: make(map[string]uint32),
paths: make(map[int]string),
internalEvent: make(chan *FileEvent),
Event: make(chan *FileEvent),
Error: make(chan error),
done: make(chan bool, 1),
}
go w.readEvents()
go w.purgeEvents()
return w, nil
}
// Close closes an inotify watcher instance
// It sends a message to the reader goroutine to quit and removes all watches
// associated with the inotify instance
func (w *Watcher) Close() error {
if w.isClosed {
return nil
}
w.isClosed = true
// Remove all watches
for path := range w.watches {
w.RemoveWatch(path)
}
// Send "quit" message to the reader goroutine
w.done <- true
return nil
}
// AddWatch adds path to the watched file set.
// The flags are interpreted as described in inotify_add_watch(2).
func (w *Watcher) addWatch(path string, flags uint32) error {
if w.isClosed {
return errors.New("inotify instance already closed")
}
w.mu.Lock()
watchEntry, found := w.watches[path]
w.mu.Unlock()
if found {
watchEntry.flags |= flags
flags |= syscall.IN_MASK_ADD
}
wd, errno := syscall.InotifyAddWatch(w.fd, path, flags)
if wd == -1 {
return errno
}
w.mu.Lock()
w.watches[path] = &watch{wd: uint32(wd), flags: flags}
w.paths[wd] = path
w.mu.Unlock()
return nil
}
// Watch adds path to the watched file set, watching all events.
func (w *Watcher) watch(path string) error {
return w.addWatch(path, sys_AGNOSTIC_EVENTS)
}
// RemoveWatch removes path from the watched file set.
func (w *Watcher) removeWatch(path string) error {
w.mu.Lock()
defer w.mu.Unlock()
watch, ok := w.watches[path]
if !ok {
return errors.New(fmt.Sprintf("can't remove non-existent inotify watch for: %s", path))
}
success, errno := syscall.InotifyRmWatch(w.fd, watch.wd)
if success == -1 {
return os.NewSyscallError("inotify_rm_watch", errno)
}
delete(w.watches, path)
return nil
}
// readEvents reads from the inotify file descriptor, converts the
// received events into Event objects and sends them via the Event channel
func (w *Watcher) readEvents() {
var (
buf [syscall.SizeofInotifyEvent * 4096]byte // Buffer for a maximum of 4096 raw events
n int // Number of bytes read with read()
errno error // Syscall errno
)
for {
// See if there is a message on the "done" channel
select {
case <-w.done:
syscall.Close(w.fd)
close(w.internalEvent)
close(w.Error)
return
default:
}
n, errno = syscall.Read(w.fd, buf[:])
// If EOF is received
if n == 0 {
syscall.Close(w.fd)
close(w.internalEvent)
close(w.Error)
return
}
if n < 0 {
w.Error <- os.NewSyscallError("read", errno)
continue
}
if n < syscall.SizeofInotifyEvent {
w.Error <- errors.New("inotify: short read in readEvents()")
continue
}
var offset uint32 = 0
// We don't know how many events we just read into the buffer
// While the offset points to at least one whole event...
for offset <= uint32(n-syscall.SizeofInotifyEvent) {
// Point "raw" to the event in the buffer
raw := (*syscall.InotifyEvent)(unsafe.Pointer(&buf[offset]))
event := new(FileEvent)
event.mask = uint32(raw.Mask)
event.cookie = uint32(raw.Cookie)
nameLen := uint32(raw.Len)
// If the event happened to the watched directory or the watched file, the kernel
// doesn't append the filename to the event, but we would like to always fill the
// the "Name" field with a valid filename. We retrieve the path of the watch from
// the "paths" map.
w.mu.Lock()
event.Name = w.paths[int(raw.Wd)]
w.mu.Unlock()
watchedName := event.Name
if nameLen > 0 {
// Point "bytes" at the first byte of the filename
bytes := (*[syscall.PathMax]byte)(unsafe.Pointer(&buf[offset+syscall.SizeofInotifyEvent]))
// The filename is padded with NUL bytes. TrimRight() gets rid of those.
event.Name += "/" + strings.TrimRight(string(bytes[0:nameLen]), "\000")
}
// Send the events that are not ignored on the events channel
if !event.ignoreLinux() {
// Setup FSNotify flags (inherit from directory watch)
w.fsnmut.Lock()
if _, fsnFound := w.fsnFlags[event.Name]; !fsnFound {
if fsnFlags, watchFound := w.fsnFlags[watchedName]; watchFound {
w.fsnFlags[event.Name] = fsnFlags
} else {
w.fsnFlags[event.Name] = FSN_ALL
}
}
w.fsnmut.Unlock()
w.internalEvent <- event
}
// Move to the next event in the buffer
offset += syscall.SizeofInotifyEvent + nameLen
}
}
}
// Certain types of events can be "ignored" and not sent over the Event
// channel. Such as events marked ignore by the kernel, or MODIFY events
// against files that do not exist.
func (e *FileEvent) ignoreLinux() bool {
// Ignore anything the inotify API says to ignore
if e.mask&sys_IN_IGNORED == sys_IN_IGNORED {
return true
}
// If the event is not a DELETE or RENAME, the file must exist.
// Otherwise the event is ignored.
// *Note*: this was put in place because it was seen that a MODIFY
// event was sent after the DELETE. This ignores that MODIFY and
// assumes a DELETE will come or has come if the file doesn't exist.
if !(e.IsDelete() || e.IsRename()) {
_, statErr := os.Lstat(e.Name)
return os.IsNotExist(statErr)
}
return false
}

Some files were not shown because too many files have changed in this diff Show More