[ADD] add default alerting rules

grctl/cmd/cluster.go

@@ -55,9 +55,21 @@ func getClusterInfo(c *cli.Context) error {
 	table.AddRow("DistributedDisk", fmt.Sprintf("%dGb/%dGb", clusterInfo.ReqDisk/1024/1024/1024, clusterInfo.CapDisk/1024/1024/1024),
 		fmt.Sprintf("%.2f", float32(clusterInfo.ReqDisk*100)/float32(clusterInfo.CapDisk))+"%")
 	fmt.Println(table)
-	//show node detail
+	//show services health status
 	list, err := clients.RegionClient.Nodes().List()
 	handleErr(err)
+	serviceTable2 := termtables.CreateTable()
+	serviceTable2.AddHeaders("Service", "Status", "Message")
+	serviceStatusInfo := getServicesHealthy(list)
+	for name, v := range serviceStatusInfo {
+		if name == string(client.NodeReady){
+			continue
+		}
+		status, message := summaryResult(v)
+		serviceTable2.AddRow(name, status, message)
+	}
+	fmt.Println(serviceTable2.Render())
+	//show node detail
 	serviceTable := termtables.CreateTable()
 	serviceTable.AddHeaders("Uid", "IP", "HostName", "NodeRole", "NodeMode", "Status", "Alived", "Schedulable", "Ready")
 	var rest []*client.HostNode
@@ -75,16 +87,6 @@ func getClusterInfo(c *cli.Context) error {
 		handleStatus(serviceTable, isNodeReady(v), v)
 	}
 	fmt.Println(serviceTable.Render())
-
-	serviceTable2 := termtables.CreateTable()
-	serviceTable2.AddHeaders("Service", "Status", "Message")
-	serviceStatusInfo := getServicesHealthy(list)
-	for name, v := range serviceStatusInfo {
-		status, message := summaryResult(v)
-		serviceTable2.AddRow(name, status, message)
-	}
-	fmt.Println(serviceTable2.Render())
-
 	return nil
 }
 

grctl/cmd/node.go

@@ -113,15 +113,29 @@ func handleStatus(serviceTable *termtables.Table, ready bool, v *client.HostNode
 }
 
 func handleResult(table *uitable.Table, v *client.HostNode) {
+	table.AddRow("Title", "Result", "Message")
 	table.AddRow("Uid:", v.ID)
 	table.AddRow("IP:", v.InternalIP)
 	table.AddRow("HostName:", v.HostName)
+	extractReady(table, v, "Ready")
+	fmt.Printf("------------------------%s----------------------------\n", "Service Health")
 
 	for _, v := range v.NodeStatus.Conditions {
+		if v.Type == client.NodeReady{
+			continue
+		}
 		table.AddRow(string(v.Type), string(v.Status), handleMessage(string(v.Status), v.Message))
 	}
 }
 
+func extractReady(table *uitable.Table, v *client.HostNode, name string)  {
+	for _, v := range v.NodeStatus.Conditions {
+		if string(v.Type) == name{
+			table.AddRow(string(v.Type), string(v.Status), handleMessage(string(v.Status), v.Message))
+		}
+	}
+}
+
 func handleMessage(status string, message string) string {
 	if status == "True" {
 		return "N/A"
@@ -216,7 +230,7 @@ func NewCmdNode() cli.Command {
 					handleErr(err)
 					table := uitable.New()
 					table.Wrap = true // wrap columns
-					fmt.Printf("-------------------%s-----------------------\n", v.HostName)
+					fmt.Printf("------------------------%s----------------------------\n", v.HostName)
 					handleResult(table, v)
 
 					fmt.Println(table)

monitor/prometheus/alerting_rules_config.go

@@ -0,0 +1,43 @@
+// Copyright (C) 2014-2018 Goodrain Co., Ltd.
+// RAINBOND, Application Management Platform
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version. For any non-GPL usage of Rainbond,
+// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
+// must be obtained first.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package prometheus
+
+type AlertingRulesConfig struct {
+	Groups []*AlertingNameConfig `yaml:"groups"`
+}
+
+type AlertingNameConfig struct {
+	Name  string         `yaml:"name"`
+	Rules []*RulesConfig `yaml:"rules"`
+}
+
+type RulesConfig struct {
+	Alert  string            `yaml:"alert"`
+	Expr   string            `yaml:"expr,omitempty"`
+	For    string            `yaml:"for,omitempty"`
+	Labels map[string]string `yaml:"labels,omitempty"`
+
+	Annotations *AnnotationsConfig `yaml:"annotations,omitempty"`
+}
+
+
+type AnnotationsConfig struct {
+	Summary     string `yaml:"summary"`
+	Description string `yaml:"description"`
+}

monitor/prometheus/config.go

@@ -34,6 +34,7 @@ type Config struct {
 
 	RemoteWriteConfigs []*RemoteWriteConfig `yaml:"remote_write,omitempty"`
 	RemoteReadConfigs  []*RemoteReadConfig  `yaml:"remote_read,omitempty"`
+	Templates          []string             `yaml:"templates,omitempty"`
 
 	// original is the input from which the config was parsed.
 	original string
@@ -64,7 +65,7 @@ type AlertmanagerConfig struct {
 	// values arbitrarily into the overflow maps of further-down types.
 
 	ServiceDiscoveryConfig ServiceDiscoveryConfig `yaml:",inline"`
-	HTTPClientConfig       HTTPClientConfig     `yaml:",inline"`
+	HTTPClientConfig       HTTPClientConfig       `yaml:",inline"`
 
 	// The URL scheme to use when talking to Alertmanagers.
 	Scheme string `yaml:"scheme,omitempty"`
@@ -79,21 +80,21 @@ type AlertmanagerConfig struct {
 
 // RemoteWriteConfig is the configuration for writing to remote storage.
 type RemoteWriteConfig struct {
-	URL                 *URL `yaml:"url"`
+	URL                 *URL             `yaml:"url"`
 	RemoteTimeout       model.Duration   `yaml:"remote_timeout,omitempty"`
 	WriteRelabelConfigs []*RelabelConfig `yaml:"write_relabel_configs,omitempty"`
 
 	// We cannot do proper Go type embedding below as the parser will then parse
 	// values arbitrarily into the overflow maps of further-down types.
 	HTTPClientConfig HTTPClientConfig `yaml:",inline"`
-	QueueConfig      QueueConfig                  `yaml:"queue_config,omitempty"`
+	QueueConfig      QueueConfig      `yaml:"queue_config,omitempty"`
 }
 
 // RemoteReadConfig is the configuration for reading from remote storage.
 type RemoteReadConfig struct {
-	URL           *URL `yaml:"url"`
-	RemoteTimeout model.Duration   `yaml:"remote_timeout,omitempty"`
-	ReadRecent    bool             `yaml:"read_recent,omitempty"`
+	URL           *URL           `yaml:"url"`
+	RemoteTimeout model.Duration `yaml:"remote_timeout,omitempty"`
+	ReadRecent    bool           `yaml:"read_recent,omitempty"`
 	// We cannot do proper Go type embedding below as the parser will then parse
 	// values arbitrarily into the overflow maps of further-down types.
 	HTTPClientConfig HTTPClientConfig `yaml:",inline"`
@@ -149,7 +150,7 @@ type ScrapeConfig struct {
 	// values arbitrarily into the overflow maps of further-down types.
 
 	ServiceDiscoveryConfig ServiceDiscoveryConfig `yaml:",inline"`
-	HTTPClientConfig       HTTPClientConfig     `yaml:",inline"`
+	HTTPClientConfig       HTTPClientConfig       `yaml:",inline"`
 
 	// List of target relabel configurations.
 	RelabelConfigs []*RelabelConfig `yaml:"relabel_configs,omitempty"`

monitor/prometheus/manager.go

@@ -73,11 +73,13 @@ func NewManager(config *option.Config) *Manager {
 				ScrapeInterval:     model.Duration(time.Second * 5),
 				EvaluationInterval: model.Duration(time.Second * 30),
 			},
+			Templates: []string{"/etc/prometheus/alerting/default_rules.yml"},
 		},
 		Registry:   reg,
 		httpClient: client,
 		l:          &sync.Mutex{},
 	}
+	SaveAlertingRulesConfig()
 	m.LoadConfig()
 
 	return m
@@ -204,3 +206,43 @@ func (p *Manager) UpdateScrape(scrape *ScrapeConfig) {
 	p.SaveConfig()
 	p.RestartDaemon()
 }
+
+func SaveAlertingRulesConfig() error {
+	logrus.Debug("Save alerting rules config file.")
+
+	a := &AlertingRulesConfig{
+
+		Groups: []*AlertingNameConfig{
+			&AlertingNameConfig{
+
+				Name: "test",
+				Rules: []*RulesConfig{
+					&RulesConfig{
+						Alert:  "MqHealth",
+						Expr:   "acp_mq_exporter_health_status{job='mq'} = 0",
+						For:    "5m",
+						Labels: map[string]string{"service_name": "mq"},
+						Annotations: &AnnotationsConfig{
+							Summary:     "Mq unhealthy",
+							Description: "Mq unhealthy",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	data, err := yaml.Marshal(a)
+	if err != nil {
+		logrus.Error("Marshal alerting rules config to yaml error.", err.Error())
+		return err
+	}
+
+	err = ioutil.WriteFile("/etc/prometheus/alerting/default_rules.yml", data, 0644)
+	if err != nil {
+		logrus.Error("Write alerting rules config file error.", err.Error())
+		return err
+	}
+
+	return nil
+}