Merge branch 'V3.6' of https://github.com/goodrain/rainbond into V3.6

Makefile

@@ -54,6 +54,10 @@ build-image-node:
 	@echo "🐳 $@"
 	@bash ./release.sh node
 #	@docker run -v `pwd`:${WORK_DIR} -w ${WORK_DIR} -it golang:1.8.3 go build  ${GO_LDFLAGS}  -o ${BASE_DOCKER}/node/${BASE_NAME}-node ./cmd/node
+build-image-monitor:
+	@echo "🐳 $@"
+	@bash ./release.sh monitor
+
 build-image-entrance:
 	@echo "🐳 $@"
 	@cp -r ${BASE_DOCKER}/dist ${BASE_DOCKER}/entrance/dist

cmd/monitor/main.go

@@ -0,0 +1,36 @@
+// Copyright (C) 2014-2018 Goodrain Co., Ltd.
+// RAINBOND, Application Management Platform
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version. For any non-GPL usage of Rainbond,
+// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
+// must be obtained first.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"github.com/goodrain/rainbond/cmd/monitor/option"
+	"github.com/spf13/pflag"
+	"github.com/goodrain/rainbond/monitor"
+)
+
+func main() {
+	c := option.NewConfig()
+	c.AddFlag(pflag.CommandLine)
+	pflag.Parse()
+
+	c.CompleteConfig()
+
+	m := monitor.NewMonitor(c)
+	m.Start()
+}

cmd/monitor/option/option.go

@@ -0,0 +1,63 @@
+// Copyright (C) 2014-2018 Goodrain Co., Ltd.
+// RAINBOND, Application Management Platform
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version. For any non-GPL usage of Rainbond,
+// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
+// must be obtained first.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package option
+
+import (
+	"github.com/spf13/pflag"
+	"github.com/Sirupsen/logrus"
+	"fmt"
+	"os"
+)
+
+type Config struct {
+	EtcdEndpoints []string
+	LogLevel      string
+	ConfigFile    string
+	BindIp        string
+	Port          int
+}
+
+func NewConfig() *Config {
+	h, _ := os.Hostname()
+	return &Config{
+		EtcdEndpoints: []string{"http://127.0.0.1:2379"},
+		LogLevel:      "info",
+		ConfigFile:    "/etc/prometheus/prometheus.yml",
+		BindIp:        h,
+		Port:          9999,
+	}
+}
+
+func (c *Config) AddFlag(cmd *pflag.FlagSet) {
+	cmd.StringArrayVar(&c.EtcdEndpoints, "etcd-endpoints", c.EtcdEndpoints, "etcd endpoints list")
+	cmd.StringVar(&c.LogLevel, "log-level", c.LogLevel, "log level")
+	cmd.StringVar(&c.ConfigFile, "config-file", c.ConfigFile, "prometheus config file path")
+	cmd.StringVar(&c.BindIp, "bind-ip", c.BindIp, "prometheus bind ip")
+	cmd.IntVar(&c.Port, "port", c.Port, "prometheus listen port")
+}
+
+func (c *Config) CompleteConfig() {
+	level, err := logrus.ParseLevel(c.LogLevel)
+	if err != nil {
+		fmt.Println("ERROR set log level:", err)
+		return
+	}
+	logrus.SetLevel(level)
+
+}

hack/contrib/docker/monitor/Dockerfile

@@ -0,0 +1,9 @@
+FROM prom/prometheus
+
+VOLUME ["/prometheusdata"]
+
+ENV RELEASE_DESC=__RELEASE_DESC__
+
+ADD /rainbond-monitor /usr/bin/monitor
+
+ENTRYPOINT /usr/bin/monitor

monitor/callback/app.go

@@ -0,0 +1,113 @@
+// Copyright (C) 2014-2018 Goodrain Co., Ltd.
+// RAINBOND, Application Management Platform
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version. For any non-GPL usage of Rainbond,
+// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
+// must be obtained first.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package callback
+
+import (
+	"github.com/Sirupsen/logrus"
+	"github.com/goodrain/rainbond/discover"
+	"github.com/goodrain/rainbond/discover/config"
+	"github.com/goodrain/rainbond/monitor/prometheus"
+	"github.com/goodrain/rainbond/util/watch"
+	"github.com/prometheus/common/model"
+	"github.com/tidwall/gjson"
+	"time"
+)
+
+// App 指app运行时信息，来源于所有子节点上的node
+// 127.0.0.1:6100/app/metrics
+type App struct {
+	discover.Callback
+	Prometheus      *prometheus.Manager
+	sortedEndpoints []string
+
+	endpoints []*config.Endpoint
+}
+
+func (e *App) UpdateEndpoints(endpoints ...*config.Endpoint) {
+
+	return
+}
+
+func (e *App) Error(err error) {
+	logrus.Error(err)
+}
+
+func (e *App) Name() string {
+	return "app"
+}
+
+func (e *App) toScrape() *prometheus.ScrapeConfig {
+	ts := make([]model.LabelSet, 0, len(e.sortedEndpoints))
+	for _, end := range e.sortedEndpoints {
+		ts = append(ts, model.LabelSet{model.AddressLabel: model.LabelValue(end)})
+	}
+
+	return &prometheus.ScrapeConfig{
+		JobName:        e.Name(),
+		ScrapeInterval: model.Duration(1 * time.Minute),
+		ScrapeTimeout:  model.Duration(30 * time.Second),
+		MetricsPath:    "/app/metrics",
+		HonorLabels:    true,
+		ServiceDiscoveryConfig: prometheus.ServiceDiscoveryConfig{
+			StaticConfigs: []*prometheus.Group{
+				{
+					Targets: ts,
+					Labels: map[model.LabelName]model.LabelValue{
+						"component": "acp_entrance",
+					},
+				},
+			},
+		},
+	}
+}
+
+func (e *App) AddEndpoint(end *config.Endpoint) {
+	e.endpoints = append(e.endpoints, end)
+	e.UpdateEndpoints(e.endpoints...)
+}
+
+func (e *App) Add(event *watch.Event) {
+	url := gjson.Get(event.GetValueString(), "external_ip").String() + ":6100"
+	end := &config.Endpoint{
+		URL: url,
+	}
+
+	e.AddEndpoint(end)
+}
+
+func (e *App) Modify(event *watch.Event) {
+	for i, end := range e.endpoints {
+		if end.URL == event.GetValueString() {
+			url := gjson.Get(event.GetValueString(), "external_ip").String() + ":6100"
+			e.endpoints[i].URL = url
+			e.UpdateEndpoints(e.endpoints...)
+			break
+		}
+	}
+}
+
+func (e *App) Delete(event *watch.Event) {
+	for i, end := range e.endpoints {
+		if end.URL == event.GetValueString() {
+			e.endpoints = append(e.endpoints[:i], e.endpoints[i+1:]...)
+			e.UpdateEndpoints(e.endpoints...)
+			break
+		}
+	}
+}

monitor/callback/appstatus.go

@@ -0,0 +1,89 @@
+// Copyright (C) 2014-2018 Goodrain Co., Ltd.
+// RAINBOND, Application Management Platform
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version. For any non-GPL usage of Rainbond,
+// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
+// must be obtained first.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package callback
+
+import (
+	"github.com/Sirupsen/logrus"
+	"github.com/goodrain/rainbond/discover"
+	"github.com/goodrain/rainbond/discover/config"
+	"github.com/goodrain/rainbond/monitor/prometheus"
+	"github.com/goodrain/rainbond/monitor/utils"
+	"github.com/prometheus/common/model"
+	"time"
+	"strings"
+)
+
+// AppStatus 指app性能数据，被选举为leader的worker，也就是app_sync_runtime_server所在的节点
+// 127.0.0.1:6369/metrics
+type AppStatus struct {
+	discover.Callback
+	Prometheus      *prometheus.Manager
+	sortedEndpoints []string
+}
+
+func (e *AppStatus) UpdateEndpoints(endpoints ...*config.Endpoint) {
+	newArr := utils.TrimAndSort(endpoints)
+
+	// change port
+	for i, end := range newArr {
+		newArr[i] = strings.Split(end, ":")[0] + ":6369"
+	}
+
+	if utils.ArrCompare(e.sortedEndpoints, newArr) {
+		return
+	}
+
+	e.sortedEndpoints = newArr
+
+	scrape := e.toScrape()
+	e.Prometheus.UpdateScrape(scrape)
+}
+
+func (e *AppStatus) Error(err error) {
+	logrus.Error(err)
+}
+
+func (e *AppStatus) Name() string {
+	return "app_status"
+}
+
+func (e *AppStatus) toScrape() *prometheus.ScrapeConfig {
+	ts := make([]model.LabelSet, 0, len(e.sortedEndpoints))
+	for _, end := range e.sortedEndpoints {
+		ts = append(ts, model.LabelSet{model.AddressLabel: model.LabelValue(end)})
+	}
+
+	return &prometheus.ScrapeConfig{
+		JobName:        e.Name(),
+		ScrapeInterval: model.Duration(5 * time.Minute),
+		ScrapeTimeout:  model.Duration(30 * time.Second),
+		MetricsPath:    "/metrics",
+		HonorLabels:    true,
+		ServiceDiscoveryConfig: prometheus.ServiceDiscoveryConfig{
+			StaticConfigs: []*prometheus.Group{
+				{
+					Targets: ts,
+					Labels: map[model.LabelName]model.LabelValue{
+						"component": "acp_entrance",
+					},
+				},
+			},
+		},
+	}
+}

monitor/callback/entrance.go

@@ -0,0 +1,81 @@
+// Copyright (C) 2014-2018 Goodrain Co., Ltd.
+// RAINBOND, Application Management Platform
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version. For any non-GPL usage of Rainbond,
+// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
+// must be obtained first.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package callback
+
+import (
+	"github.com/Sirupsen/logrus"
+	"github.com/goodrain/rainbond/discover"
+	"github.com/goodrain/rainbond/discover/config"
+	"github.com/goodrain/rainbond/monitor/prometheus"
+	"github.com/goodrain/rainbond/monitor/utils"
+	"github.com/prometheus/common/model"
+	"time"
+)
+
+type Entrance struct {
+	discover.Callback
+	Prometheus      *prometheus.Manager
+	sortedEndpoints []string
+}
+
+func (e *Entrance) UpdateEndpoints(endpoints ...*config.Endpoint) {
+	newArr := utils.TrimAndSort(endpoints)
+
+	if utils.ArrCompare(e.sortedEndpoints, newArr) {
+		return
+	}
+
+	e.sortedEndpoints = newArr
+
+	scrape := e.toScrape()
+	e.Prometheus.UpdateScrape(scrape)
+}
+
+func (e *Entrance) Error(err error) {
+	logrus.Error(err)
+}
+
+func (e *Entrance) Name() string {
+	return "entrance"
+}
+
+func (e *Entrance) toScrape() *prometheus.ScrapeConfig {
+	ts := make([]model.LabelSet, 0, len(e.sortedEndpoints))
+	for _, end := range e.sortedEndpoints {
+		ts = append(ts, model.LabelSet{model.AddressLabel: model.LabelValue(end)})
+	}
+
+	return &prometheus.ScrapeConfig{
+		JobName:        e.Name(),
+		ScrapeInterval: model.Duration(1 * time.Minute),
+		ScrapeTimeout:  model.Duration(30 * time.Second),
+		MetricsPath:    "/metrics",
+		HonorLabels:    true,
+		ServiceDiscoveryConfig: prometheus.ServiceDiscoveryConfig{
+			StaticConfigs: []*prometheus.Group{
+				{
+					Targets: ts,
+					Labels: map[model.LabelName]model.LabelValue{
+						"component": "acp_entrance",
+					},
+				},
+			},
+		},
+	}
+}

monitor/callback/etcd.go

@@ -0,0 +1,80 @@
+// Copyright (C) 2014-2018 Goodrain Co., Ltd.
+// RAINBOND, Application Management Platform
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version. For any non-GPL usage of Rainbond,
+// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
+// must be obtained first.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package callback
+
+import (
+	"github.com/Sirupsen/logrus"
+	"github.com/goodrain/rainbond/discover"
+	"github.com/goodrain/rainbond/discover/config"
+	"github.com/goodrain/rainbond/monitor/prometheus"
+	"github.com/goodrain/rainbond/monitor/utils"
+	"github.com/prometheus/common/model"
+	"time"
+)
+
+type Etcd struct {
+	discover.Callback
+	Prometheus      *prometheus.Manager
+	sortedEndpoints []string
+}
+
+func (e *Etcd) UpdateEndpoints(endpoints ...*config.Endpoint) {
+	newArr := utils.TrimAndSort(endpoints)
+
+	if utils.ArrCompare(e.sortedEndpoints, newArr) {
+		return
+	}
+
+	e.sortedEndpoints = newArr
+
+	scrape := e.toScrape()
+	e.Prometheus.UpdateScrape(scrape)
+}
+
+func (e *Etcd) Error(err error) {
+	logrus.Error(err)
+}
+
+func (e *Etcd) Name() string {
+	return "etcd"
+}
+
+func (e *Etcd) toScrape() *prometheus.ScrapeConfig {
+	ts := make([]model.LabelSet, 0, len(e.sortedEndpoints))
+	for _, end := range e.sortedEndpoints {
+		ts = append(ts, model.LabelSet{model.AddressLabel: model.LabelValue(end)})
+	}
+
+	return &prometheus.ScrapeConfig{
+		JobName:        e.Name(),
+		ScrapeInterval: model.Duration(1 * time.Minute),
+		ScrapeTimeout:  model.Duration(30 * time.Second),
+		MetricsPath:    "/metrics",
+		ServiceDiscoveryConfig: prometheus.ServiceDiscoveryConfig{
+			StaticConfigs: []*prometheus.Group{
+				{
+					Targets: ts,
+					Labels: map[model.LabelName]model.LabelValue{
+						"component": "acp_entrance",
+					},
+				},
+			},
+		},
+	}
+}

monitor/callback/eventlog.go

@@ -0,0 +1,81 @@
+// Copyright (C) 2014-2018 Goodrain Co., Ltd.
+// RAINBOND, Application Management Platform
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version. For any non-GPL usage of Rainbond,
+// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
+// must be obtained first.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package callback
+
+import (
+	"github.com/goodrain/rainbond/discover/config"
+	"github.com/Sirupsen/logrus"
+	"github.com/goodrain/rainbond/discover"
+	"github.com/goodrain/rainbond/monitor/prometheus"
+	"github.com/goodrain/rainbond/monitor/utils"
+	"github.com/prometheus/common/model"
+	"time"
+)
+
+type EventLog struct {
+	discover.Callback
+	Prometheus      *prometheus.Manager
+	sortedEndpoints []string
+}
+
+func (e *EventLog) UpdateEndpoints(endpoints ...*config.Endpoint) {
+	newArr := utils.TrimAndSort(endpoints)
+
+	if utils.ArrCompare(e.sortedEndpoints, newArr) {
+		return
+	}
+
+	e.sortedEndpoints = newArr
+
+	scrape := e.toScrape()
+	e.Prometheus.UpdateScrape(scrape)
+}
+
+func (e *EventLog) Error(err error) {
+	logrus.Error(err)
+}
+
+func (e *EventLog) Name() string {
+	return "eventlog"
+}
+
+func (e *EventLog) toScrape() *prometheus.ScrapeConfig {
+	ts := make([]model.LabelSet, 0, len(e.sortedEndpoints))
+	for _, end := range e.sortedEndpoints {
+		ts = append(ts, model.LabelSet{model.AddressLabel: model.LabelValue(end)})
+	}
+
+	return &prometheus.ScrapeConfig{
+		JobName:        e.Name(),
+		ScrapeInterval: model.Duration(1 * time.Minute),
+		ScrapeTimeout:  model.Duration(30 * time.Second),
+		MetricsPath:    "/metrics",
+		HonorLabels:    true,
+		ServiceDiscoveryConfig: prometheus.ServiceDiscoveryConfig{
+			StaticConfigs: []*prometheus.Group{
+				{
+					Targets: ts,
+					Labels: map[model.LabelName]model.LabelValue{
+						"component": "acp_event_log",
+					},
+				},
+			},
+		},
+	}
+}

monitor/callback/node.go

@@ -0,0 +1,120 @@
+// Copyright (C) 2014-2018 Goodrain Co., Ltd.
+// RAINBOND, Application Management Platform
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version. For any non-GPL usage of Rainbond,
+// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
+// must be obtained first.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package callback
+
+import (
+	"github.com/Sirupsen/logrus"
+	"github.com/goodrain/rainbond/discover"
+	"github.com/goodrain/rainbond/discover/config"
+	"github.com/goodrain/rainbond/monitor/prometheus"
+	"github.com/goodrain/rainbond/util/watch"
+	"github.com/goodrain/rainbond/monitor/utils"
+	"github.com/prometheus/common/model"
+	"time"
+	"github.com/tidwall/gjson"
+)
+
+type Node struct {
+	discover.Callback
+	Prometheus      *prometheus.Manager
+	sortedEndpoints []string
+
+	endpoints []*config.Endpoint
+}
+
+func (e *Node) UpdateEndpoints(endpoints ...*config.Endpoint) {
+	newArr := utils.TrimAndSort(endpoints)
+
+	if utils.ArrCompare(e.sortedEndpoints, newArr) {
+		return
+	}
+
+	e.sortedEndpoints = newArr
+
+	scrape := e.toScrape()
+	e.Prometheus.UpdateScrape(scrape)
+}
+
+func (e *Node) Error(err error) {
+	logrus.Error(err)
+}
+
+func (e *Node) Name() string {
+	return "node"
+}
+
+func (e *Node) toScrape() *prometheus.ScrapeConfig {
+	ts := make([]model.LabelSet, 0, len(e.sortedEndpoints))
+	for _, end := range e.sortedEndpoints {
+		ts = append(ts, model.LabelSet{model.AddressLabel: model.LabelValue(end)})
+	}
+
+	return &prometheus.ScrapeConfig{
+		JobName:        e.Name(),
+		ScrapeInterval: model.Duration(1 * time.Minute),
+		ScrapeTimeout:  model.Duration(30 * time.Second),
+		MetricsPath:    "/node/metrics",
+		HonorLabels:    true,
+		ServiceDiscoveryConfig: prometheus.ServiceDiscoveryConfig{
+			StaticConfigs: []*prometheus.Group{
+				{
+					Targets: ts,
+					Labels: map[model.LabelName]model.LabelValue{
+						"component": "acp_entrance",
+					},
+				},
+			},
+		},
+	}
+}
+
+func (e *Node) AddEndpoint(end *config.Endpoint) {
+	e.endpoints = append(e.endpoints, end)
+	e.UpdateEndpoints(e.endpoints...)
+}
+
+func (e *Node) Add(event *watch.Event) {
+	url := gjson.Get(event.GetValueString(), "external_ip").String() + ":6100"
+	end := &config.Endpoint{
+		URL: url,
+	}
+
+	e.AddEndpoint(end)
+}
+
+func (e *Node) Modify(event *watch.Event) {
+	for i, end := range e.endpoints {
+		if end.URL == event.GetValueString() {
+			url := gjson.Get(event.GetValueString(), "external_ip").String() + ":6100"
+			e.endpoints[i].URL = url
+			e.UpdateEndpoints(e.endpoints...)
+			break
+		}
+	}
+}
+
+func (e *Node) Delete(event *watch.Event) {
+	for i, end := range e.endpoints {
+		if end.URL == event.GetValueString() {
+			e.endpoints = append(e.endpoints[:i], e.endpoints[i+1:]...)
+			e.UpdateEndpoints(e.endpoints...)
+			break
+		}
+	}
+}

monitor/callback/promethes.go

@@ -0,0 +1,82 @@
+// Copyright (C) 2014-2018 Goodrain Co., Ltd.
+// RAINBOND, Application Management Platform
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version. For any non-GPL usage of Rainbond,
+// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
+// must be obtained first.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package callback
+
+import (
+	"github.com/Sirupsen/logrus"
+	"github.com/goodrain/rainbond/discover"
+	"github.com/goodrain/rainbond/discover/config"
+	"github.com/goodrain/rainbond/monitor/prometheus"
+	"github.com/goodrain/rainbond/monitor/utils"
+	"github.com/prometheus/common/model"
+	"time"
+)
+
+// Prometheus 指prometheus的运行指标，数据来源于prometheus自身API
+type Prometheus struct {
+	discover.Callback
+	Prometheus      *prometheus.Manager
+	sortedEndpoints []string
+}
+
+func (e *Prometheus) UpdateEndpoints(endpoints ...*config.Endpoint) {
+	newArr := utils.TrimAndSort(endpoints)
+
+	if utils.ArrCompare(e.sortedEndpoints, newArr) {
+		return
+	}
+
+	e.sortedEndpoints = newArr
+
+	scrape := e.toScrape()
+	e.Prometheus.UpdateScrape(scrape)
+}
+
+func (e *Prometheus) Error(err error) {
+	logrus.Error(err)
+}
+
+func (e *Prometheus) Name() string {
+	return "prometheus"
+}
+
+func (e *Prometheus) toScrape() *prometheus.ScrapeConfig {
+	ts := make([]model.LabelSet, 0, len(e.sortedEndpoints))
+	for _, end := range e.sortedEndpoints {
+		ts = append(ts, model.LabelSet{model.AddressLabel: model.LabelValue(end)})
+	}
+
+	return &prometheus.ScrapeConfig{
+		JobName:        e.Name(),
+		ScrapeInterval: model.Duration(5 * time.Minute),
+		ScrapeTimeout:  model.Duration(30 * time.Second),
+		MetricsPath:    "/metrics",
+		HonorLabels:    true,
+		ServiceDiscoveryConfig: prometheus.ServiceDiscoveryConfig{
+			StaticConfigs: []*prometheus.Group{
+				{
+					Targets: ts,
+					Labels: map[model.LabelName]model.LabelValue{
+						"component": "acp_entrance",
+					},
+				},
+			},
+		},
+	}
+}

monitor/monitor.go

@@ -0,0 +1,240 @@
+// Copyright (C) 2014-2018 Goodrain Co., Ltd.
+// RAINBOND, Application Management Platform
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version. For any non-GPL usage of Rainbond,
+// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
+// must be obtained first.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package monitor
+
+import (
+	"context"
+	v3 "github.com/coreos/etcd/clientv3"
+	"github.com/goodrain/rainbond/cmd/monitor/option"
+	discover1 "github.com/goodrain/rainbond/discover"
+	discover3 "github.com/goodrain/rainbond/discover.v2"
+	"github.com/goodrain/rainbond/discover/config"
+	"github.com/goodrain/rainbond/monitor/callback"
+	"github.com/goodrain/rainbond/util/watch"
+	"time"
+	"github.com/Sirupsen/logrus"
+	"os"
+	"syscall"
+	"os/signal"
+	"github.com/goodrain/rainbond/monitor/prometheus"
+	"github.com/tidwall/gjson"
+)
+
+type Monitor struct {
+	config      *option.Config
+	ctx         context.Context
+	cancel      context.CancelFunc
+	client      *v3.Client
+	timeout     time.Duration
+	stopperList []chan bool
+
+	discover1   discover1.Discover
+	discover3   discover3.Discover
+}
+
+func (d *Monitor) Start() {
+	// create prometheus manager
+	p := prometheus.NewManager(d.config)
+	// start prometheus daemon and watching tis status in all time, exit monitor process if start failed
+	p.StartDaemon(d.GetStopper())
+
+	d.discover1.AddProject("event_log_event_grpc", &callback.EventLog{Prometheus: p})
+	d.discover1.AddProject("acp_entrance", &callback.Entrance{Prometheus: p})
+	d.discover3.AddProject("app_sync_runtime_server", &callback.AppStatus{Prometheus: p})
+
+	// node and app runtime metrics needs to be monitored separately
+	go d.discoverNodes(&callback.Node{Prometheus: p}, &callback.App{Prometheus: p}, d.GetStopper())
+
+	d.listenStop()
+}
+
+func (d *Monitor) discoverNodes(node *callback.Node, app *callback.App, done chan bool) {
+	// get all exist nodes by etcd
+	resp, err := d.client.Get(d.ctx, "/rainbond/nodes/", v3.WithPrefix())
+	if err != nil {
+		logrus.Error("failed to get all nodes: ", err)
+		return
+	}
+
+	for _, kv := range resp.Kvs {
+		url := gjson.GetBytes(kv.Value, "external_ip").String() + ":6100"
+		end := &config.Endpoint{
+			URL: url,
+		}
+
+		node.AddEndpoint(end)
+
+		isSlave := gjson.GetBytes(kv.Value, "labels.rainbond_node_rule_compute").String()
+		if isSlave == "true" {
+			app.AddEndpoint(end)
+		}
+	}
+
+	// start listen node modified
+	watcher := watch.New(d.client, "")
+	w, err := watcher.WatchList(d.ctx, "/rainbond/nodes", "")
+	if err != nil {
+		logrus.Error("failed to watch list for discover all nodes: ", err)
+		return
+	}
+	defer w.Stop()
+
+	for {
+		select {
+		case event, ok := <-w.ResultChan():
+			if !ok {
+				logrus.Warn("the events channel is closed.")
+				return
+			}
+
+			switch event.Type {
+			case watch.Added:
+				node.Add(&event)
+
+				isSlave := gjson.Get(event.GetValueString(), "labels.rainbond_node_rule_compute").String()
+				if isSlave == "true" {
+					app.Add(&event)
+				}
+			case watch.Modified:
+				node.Modify(&event)
+
+				isSlave := gjson.Get(event.GetValueString(), "labels.rainbond_node_rule_compute").String()
+				if isSlave == "true" {
+					app.Modify(&event)
+				}
+			case watch.Deleted:
+				node.Delete(&event)
+
+				isSlave := gjson.Get(event.GetValueString(), "labels.rainbond_node_rule_compute").String()
+				if isSlave == "true" {
+					app.Delete(&event)
+				}
+			case watch.Error:
+				logrus.Error("error when read a event from result chan for discover all nodes: ", event.Error)
+			}
+		case <-done:
+			logrus.Info("stop discover nodes because received stop signal.")
+			close(done)
+			return
+		}
+
+	}
+
+}
+
+func (d *Monitor) discoverEtcd(e *callback.Etcd, done chan bool) {
+	t := time.Tick(time.Second * 5)
+	for {
+		select {
+		case <-done:
+			logrus.Info("stop discover etcd because received stop signal.")
+			close(done)
+			return
+		case <-t:
+			resp, err := d.client.MemberList(d.ctx)
+			if err != nil {
+				logrus.Error("Failed to list etcd members for discover etcd.")
+				continue
+			}
+
+			endpoints := make([]*config.Endpoint, 0, 5)
+			for _, member := range resp.Members {
+				url := member.GetName() + ":2379"
+				end := &config.Endpoint{
+					URL: url,
+				}
+				endpoints = append(endpoints, end)
+			}
+
+			e.UpdateEndpoints(endpoints...)
+		}
+	}
+}
+
+func (d *Monitor) Stop() {
+	logrus.Info("Stop all child process for monitor.")
+	for _, ch := range d.stopperList {
+		ch <- true
+	}
+
+	d.discover1.Stop()
+	d.discover3.Stop()
+	d.client.Close()
+	d.cancel()
+
+	time.Sleep(time.Second)
+}
+
+func (d *Monitor) GetStopper() chan bool {
+	ch := make(chan bool, 1)
+	d.stopperList = append(d.stopperList, ch)
+
+	return ch
+}
+
+func (d *Monitor) listenStop() {
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGKILL, syscall.SIGINT, syscall.SIGTERM)
+
+	sig := <- sigs
+	signal.Ignore(syscall.SIGKILL, syscall.SIGINT, syscall.SIGTERM)
+
+	close(sigs)
+	logrus.Warn("monitor manager received signal: ", sig)
+	d.Stop()
+}
+
+func NewMonitor(opt *option.Config) *Monitor {
+	ctx, cancel := context.WithCancel(context.Background())
+	defaultTimeout := time.Second * 3
+
+	cli, err := v3.New(v3.Config{
+		Endpoints:   opt.EtcdEndpoints,
+		DialTimeout: defaultTimeout,
+	})
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	dc1, err := discover1.GetDiscover(config.DiscoverConfig{
+		EtcdClusterEndpoints: opt.EtcdEndpoints,
+	})
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	dc3, err := discover3.GetDiscover(config.DiscoverConfig{
+		EtcdClusterEndpoints: opt.EtcdEndpoints,
+	})
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	d := &Monitor{
+		config:    opt,
+		ctx:       ctx,
+		cancel:    cancel,
+		client:    cli,
+		discover1: dc1,
+		discover3: dc3,
+		timeout:   defaultTimeout,
+	}
+
+	return d
+}

monitor/prometheus/config.go

@@ -0,0 +1,256 @@
+// Copyright (C) 2014-2018 Goodrain Co., Ltd.
+// RAINBOND, Application Management Platform
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version. For any non-GPL usage of Rainbond,
+// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
+// must be obtained first.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package prometheus
+
+import (
+	"net/url"
+	"github.com/prometheus/common/model"
+	"regexp"
+	"time"
+)
+
+// Config is the top-level configuration for Prometheus's config files.
+type Config struct {
+	GlobalConfig   GlobalConfig    `yaml:"global"`
+	AlertingConfig AlertingConfig  `yaml:"alerting,omitempty"`
+	RuleFiles      []string        `yaml:"rule_files,omitempty"`
+	ScrapeConfigs  []*ScrapeConfig `yaml:"scrape_configs,omitempty"`
+
+	RemoteWriteConfigs []*RemoteWriteConfig `yaml:"remote_write,omitempty"`
+	RemoteReadConfigs  []*RemoteReadConfig  `yaml:"remote_read,omitempty"`
+
+	// original is the input from which the config was parsed.
+	original string
+}
+
+// GlobalConfig configures values that are used across other configuration
+// objects.
+type GlobalConfig struct {
+	// How frequently to scrape targets by default.
+	ScrapeInterval model.Duration `yaml:"scrape_interval,omitempty"`
+	// The default timeout when scraping targets.
+	ScrapeTimeout model.Duration `yaml:"scrape_timeout,omitempty"`
+	// How frequently to evaluate rules by default.
+	EvaluationInterval model.Duration `yaml:"evaluation_interval,omitempty"`
+	// The labels to add to any timeseries that this Prometheus instance scrapes.
+	ExternalLabels model.LabelSet `yaml:"external_labels,omitempty"`
+}
+
+// AlertingConfig configures alerting and alertmanager related configs.
+type AlertingConfig struct {
+	AlertRelabelConfigs []*RelabelConfig      `yaml:"alert_relabel_configs,omitempty"`
+	AlertmanagerConfigs []*AlertmanagerConfig `yaml:"alertmanagers,omitempty"`
+}
+
+// AlertmanagerConfig configures how Alertmanagers can be discovered and communicated with.
+type AlertmanagerConfig struct {
+	// We cannot do proper Go type embedding below as the parser will then parse
+	// values arbitrarily into the overflow maps of further-down types.
+
+	ServiceDiscoveryConfig ServiceDiscoveryConfig `yaml:",inline"`
+	HTTPClientConfig       HTTPClientConfig     `yaml:",inline"`
+
+	// The URL scheme to use when talking to Alertmanagers.
+	Scheme string `yaml:"scheme,omitempty"`
+	// Path prefix to add in front of the push endpoint path.
+	PathPrefix string `yaml:"path_prefix,omitempty"`
+	// The timeout used when sending alerts.
+	Timeout time.Duration `yaml:"timeout,omitempty"`
+
+	// List of Alertmanager relabel configurations.
+	RelabelConfigs []*RelabelConfig `yaml:"relabel_configs,omitempty"`
+}
+
+// RemoteWriteConfig is the configuration for writing to remote storage.
+type RemoteWriteConfig struct {
+	URL                 *URL `yaml:"url"`
+	RemoteTimeout       model.Duration   `yaml:"remote_timeout,omitempty"`
+	WriteRelabelConfigs []*RelabelConfig `yaml:"write_relabel_configs,omitempty"`
+
+	// We cannot do proper Go type embedding below as the parser will then parse
+	// values arbitrarily into the overflow maps of further-down types.
+	HTTPClientConfig HTTPClientConfig `yaml:",inline"`
+	QueueConfig      QueueConfig                  `yaml:"queue_config,omitempty"`
+}
+
+// RemoteReadConfig is the configuration for reading from remote storage.
+type RemoteReadConfig struct {
+	URL           *URL `yaml:"url"`
+	RemoteTimeout model.Duration   `yaml:"remote_timeout,omitempty"`
+	ReadRecent    bool             `yaml:"read_recent,omitempty"`
+	// We cannot do proper Go type embedding below as the parser will then parse
+	// values arbitrarily into the overflow maps of further-down types.
+	HTTPClientConfig HTTPClientConfig `yaml:",inline"`
+
+	// RequiredMatchers is an optional list of equality matchers which have to
+	// be present in a selector to query the remote read endpoint.
+	RequiredMatchers model.LabelSet `yaml:"required_matchers,omitempty"`
+}
+
+// QueueConfig is the configuration for the queue used to write to remote
+// storage.
+type QueueConfig struct {
+	// Number of samples to buffer per shard before we start dropping them.
+	Capacity int `yaml:"capacity,omitempty"`
+
+	// Max number of shards, i.e. amount of concurrency.
+	MaxShards int `yaml:"max_shards,omitempty"`
+
+	// Maximum number of samples per send.
+	MaxSamplesPerSend int `yaml:"max_samples_per_send,omitempty"`
+
+	// Maximum time sample will wait in buffer.
+	BatchSendDeadline time.Duration `yaml:"batch_send_deadline,omitempty"`
+
+	// Max number of times to retry a batch on recoverable errors.
+	MaxRetries int `yaml:"max_retries,omitempty"`
+
+	// On recoverable errors, backoff exponentially.
+	MinBackoff time.Duration `yaml:"min_backoff,omitempty"`
+	MaxBackoff time.Duration `yaml:"max_backoff,omitempty"`
+}
+
+// ScrapeConfig configures a scraping unit for Prometheus.
+type ScrapeConfig struct {
+	// The job name to which the job label is set by default.
+	JobName string `yaml:"job_name"`
+	// Indicator whether the scraped metrics should remain unmodified.
+	HonorLabels bool `yaml:"honor_labels,omitempty"`
+	// A set of query parameters with which the target is scraped.
+	Params url.Values `yaml:"params,omitempty"`
+	// How frequently to scrape the targets of this scrape config.
+	ScrapeInterval model.Duration `yaml:"scrape_interval,omitempty"`
+	// The timeout for scraping targets of this config.
+	ScrapeTimeout model.Duration `yaml:"scrape_timeout,omitempty"`
+	// The HTTP resource path on which to fetch metrics from targets.
+	MetricsPath string `yaml:"metrics_path,omitempty"`
+	// The URL scheme with which to fetch metrics from targets.
+	Scheme string `yaml:"scheme,omitempty"`
+	// More than this many samples post metric-relabelling will cause the scrape to fail.
+	SampleLimit uint `yaml:"sample_limit,omitempty"`
+
+	// We cannot do proper Go type embedding below as the parser will then parse
+	// values arbitrarily into the overflow maps of further-down types.
+
+	ServiceDiscoveryConfig ServiceDiscoveryConfig `yaml:",inline"`
+	HTTPClientConfig       HTTPClientConfig     `yaml:",inline"`
+
+	// List of target relabel configurations.
+	RelabelConfigs []*RelabelConfig `yaml:"relabel_configs,omitempty"`
+	// List of metric relabel configurations.
+	MetricRelabelConfigs []*RelabelConfig `yaml:"metric_relabel_configs,omitempty"`
+}
+
+// RelabelConfig is the configuration for relabeling of target label sets.
+type RelabelConfig struct {
+	// A list of labels from which values are taken and concatenated
+	// with the configured separator in order.
+	SourceLabels model.LabelNames `yaml:"source_labels,flow,omitempty"`
+	// Separator is the string between concatenated values from the source labels.
+	Separator string `yaml:"separator,omitempty"`
+	// Regex against which the concatenation is matched.
+	Regex Regexp `yaml:"regex,omitempty"`
+	// Modulus to take of the hash of concatenated values from the source labels.
+	Modulus uint64 `yaml:"modulus,omitempty"`
+	// TargetLabel is the label to which the resulting string is written in a replacement.
+	// Regexp interpolation is allowed for the replace action.
+	TargetLabel string `yaml:"target_label,omitempty"`
+	// Replacement is the regex replacement pattern to be used.
+	Replacement string `yaml:"replacement,omitempty"`
+	// Action is the action to be performed for the relabeling.
+	Action RelabelAction `yaml:"action,omitempty"`
+}
+
+// ServiceDiscoveryConfig configures lists of different service discovery mechanisms.
+type ServiceDiscoveryConfig struct {
+	// List of labeled target groups for this job.
+	StaticConfigs []*Group `yaml:"static_configs,omitempty"`
+}
+
+type Group struct {
+	// Targets is a list of targets identified by a label set. Each target is
+	// uniquely identifiable in the group by its address label.
+	Targets []model.LabelSet
+	// Labels is a set of labels that is common across all targets in the group.
+	Labels model.LabelSet
+
+	// Source is an identifier that describes a group of targets.
+	Source string
+}
+
+// Regexp encapsulates a regexp.Regexp and makes it YAML marshallable.
+type Regexp struct {
+	*regexp.Regexp
+	original string
+}
+
+// RelabelAction is the action to be performed on relabeling.
+type RelabelAction string
+
+// HTTPClientConfig configures an HTTP client.
+type HTTPClientConfig struct {
+	// The HTTP basic authentication credentials for the targets.
+	BasicAuth *BasicAuth `yaml:"basic_auth,omitempty"`
+	// The bearer token for the targets.
+	BearerToken Secret `yaml:"bearer_token,omitempty"`
+	// The bearer token file for the targets.
+	BearerTokenFile string `yaml:"bearer_token_file,omitempty"`
+	// HTTP proxy server to use to connect to the targets.
+	ProxyURL URL `yaml:"proxy_url,omitempty"`
+	// TLSConfig to use to connect to the targets.
+	TLSConfig TLSConfig `yaml:"tls_config,omitempty"`
+
+	// Catches all undefined fields and must be empty after parsing.
+	XXX map[string]interface{} `yaml:",inline"`
+}
+
+// BasicAuth contains basic HTTP authentication credentials.
+type BasicAuth struct {
+	Username     string `yaml:"username"`
+	Password     Secret `yaml:"password,omitempty"`
+	PasswordFile string `yaml:"password_file,omitempty"`
+
+	// Catches all undefined fields and must be empty after parsing.
+	XXX map[string]interface{} `yaml:",inline"`
+}
+
+// URL is a custom URL type that allows validation at configuration load time.
+type URL struct {
+	*url.URL
+}
+
+// Secret special type for storing secrets.
+type Secret string
+
+// TLSConfig configures the options for TLS connections.
+type TLSConfig struct {
+	// The CA cert to use for the targets.
+	CAFile string `yaml:"ca_file,omitempty"`
+	// The client cert file for the targets.
+	CertFile string `yaml:"cert_file,omitempty"`
+	// The client key file for the targets.
+	KeyFile string `yaml:"key_file,omitempty"`
+	// Used to verify the hostname for the targets.
+	ServerName string `yaml:"server_name,omitempty"`
+	// Disable target certificate validation.
+	InsecureSkipVerify bool `yaml:"insecure_skip_verify"`
+
+	// Catches all undefined fields and must be empty after parsing.
+	XXX map[string]interface{} `yaml:",inline"`
+}

monitor/prometheus/manager.go

@@ -0,0 +1,162 @@
+// Copyright (C) 2014-2018 Goodrain Co., Ltd.
+// RAINBOND, Application Management Platform
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version. For any non-GPL usage of Rainbond,
+// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
+// must be obtained first.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package prometheus
+
+import (
+	"github.com/Sirupsen/logrus"
+	"github.com/goodrain/rainbond/cmd/monitor/option"
+	"gopkg.in/yaml.v2"
+	"io/ioutil"
+	"net/http"
+	"os/exec"
+	"sync"
+	"time"
+	discover3 "github.com/goodrain/rainbond/discover.v2"
+	"fmt"
+)
+
+type Manager struct {
+	ApiUrl     string
+	Opt        *option.Config
+	Config     *Config
+	Reg        *discover3.KeepAlive
+	httpClient *http.Client
+	l          *sync.Mutex
+}
+
+func NewManager(config *option.Config) *Manager {
+	client := &http.Client{
+		Timeout: time.Second * 3,
+	}
+
+	reg, err := discover3.CreateKeepAlive(config.EtcdEndpoints, "prometheus", "http", config.BindIp, config.Port)
+	if err != nil {
+		panic(err)
+	}
+
+	return &Manager{
+		ApiUrl:     fmt.Sprintf("http://127.0.0.1:%s", config.Port),
+		Opt:        config,
+		Config:     &Config{},
+		Reg:        reg,
+		httpClient: client,
+		l:          &sync.Mutex{},
+	}
+}
+
+func (p *Manager) LoadConfig() error {
+	context, err := ioutil.ReadFile(p.Opt.ConfigFile)
+	if err != nil {
+		logrus.Error("Failed to read prometheus config file: ", err)
+		return err
+	}
+
+	if err := yaml.Unmarshal(context, p.Config); err != nil {
+		logrus.Error("Unmarshal prometheus config string to object error.", err.Error())
+		return err
+	}
+
+	return nil
+}
+
+func (p *Manager) SaveConfig() error {
+	data, err := yaml.Marshal(p.Config)
+	if err != nil {
+		logrus.Error("Marshal prometheus config to yaml error.", err.Error())
+		return err
+	}
+
+	err = ioutil.WriteFile(p.Opt.ConfigFile, data, 0644)
+	if err != nil {
+		logrus.Error("Write prometheus config file error.", err.Error())
+		return err
+	}
+
+	return nil
+}
+
+func (p *Manager) StartDaemon(done chan bool) {
+	cmd := "which prometheus && " +
+		"prometheus " +
+		"--web.listen-address=:%s " +
+		"--storage.tsdb.path=/prometheusdata " +
+		"--storage.tsdb.retention=7d " +
+		"--config.file=%s &"
+
+	cmd = fmt.Sprintf(cmd, p.Opt.Port, p.Opt.ConfigFile)
+
+	err := exec.Command("sh", "-c", cmd).Run()
+	if err != nil {
+		logrus.Error("Can not start prometheus daemon: ", err)
+		panic(err)
+	}
+
+	p.Reg.Start()
+	defer p.Reg.Stop()
+
+	t := time.Tick(time.Second * 5)
+	for {
+		select {
+		case <-done:
+			exec.Command("sh", "-c", "kill `pgrep prometheus`").Run()
+			return
+		case <-t:
+			err := exec.Command("sh", "-c", "pgrep prometheus").Run()
+			if err != nil {
+				logrus.Error("the prometheus process is exited, ready to restart it.")
+				err := exec.Command("sh", "-c", cmd).Run()
+				if err == nil {
+					logrus.Error("Failed to restart the prometheus daemon: ", err)
+				}
+			}
+		}
+	}
+
+}
+
+func (p *Manager) RestartDaemon() error {
+	request, err := http.NewRequest("POST", p.ApiUrl+"/-/reload", nil)
+	if err != nil {
+		logrus.Error("Create request to load config error: ", err)
+		return err
+	}
+
+	_, err = p.httpClient.Do(request)
+	if err != nil {
+		logrus.Error("load config error: ", err)
+		return err
+	}
+
+	return nil
+}
+
+func (p *Manager) UpdateScrape(scrape *ScrapeConfig) {
+	p.l.Lock()
+	defer p.l.Unlock()
+
+	for i, s := range p.Config.ScrapeConfigs {
+		if s.JobName == scrape.JobName {
+			p.Config.ScrapeConfigs[i] = scrape
+			break
+		}
+	}
+
+	p.SaveConfig()
+	p.RestartDaemon()
+}

monitor/utils/utils.go

@@ -0,0 +1,51 @@
+// Copyright (C) 2014-2018 Goodrain Co., Ltd.
+// RAINBOND, Application Management Platform
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version. For any non-GPL usage of Rainbond,
+// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
+// must be obtained first.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package utils
+
+import (
+	"strings"
+	"sort"
+	"github.com/goodrain/rainbond/discover/config"
+)
+
+func TrimAndSort(endpoints []*config.Endpoint) []string {
+	arr := make([]string, 0, len(endpoints))
+	for _, end := range endpoints {
+		url := strings.TrimLeft(end.URL, "https://")
+		arr = append(arr, url)
+	}
+
+	sort.Strings(arr)
+
+	return arr
+}
+
+func ArrCompare(arr1, arr2 []string) bool {
+	if len(arr1) != len(arr2) {
+		return false
+	}
+
+	for i, item := range arr1 {
+		if item != arr2[i] {
+			return false
+		}
+	}
+
+	return true
+}

release.sh

@@ -80,4 +80,4 @@ case $1 in
 	*)
 		build::image $1
 	;;
-esac
+esac