Rainbond/monitor/prometheus/etcd_alert.yaml

groups:
    - name: etcd3_alert.rules
      rules:
      - alert: InsufficientMembers
        expr: count(up{job="etcd"} == 0) > (count(up{job="etcd"}) / 2 - 1)
        for: 3m
        labels:
          severity: critical
        annotations:
          description: If one more etcd member goes down the cluster will be unavailable
          summary: etcd cluster insufficient members
      - alert: NoLeader
        expr: etcd_server_has_leader{job="etcd"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          description: etcd member {{ $labels.instance }} has no leader
          summary: etcd member has no leader
      - alert: HighNumberOfLeaderChanges
        expr: increase(etcd_server_leader_changes_seen_total{job="etcd"}[1h]) > 3
        labels:
          severity: warning
        annotations:
          description: etcd instance {{ $labels.instance }} has seen {{ $value }} leader
            changes within the last hour
          summary: a high number of leader changes within the etcd cluster are happening
      - alert: HighNumberOfFailedGRPCRequests
        expr: sum(rate(etcd_grpc_requests_failed_total{job="etcd"}[5m])) BY (grpc_method)
          / sum(rate(etcd_grpc_total{job="etcd"}[5m])) BY (grpc_method) > 0.01
        for: 10m
        labels:
          severity: warning
        annotations:
          description: '{{ $value }}% of requests for {{ $labels.grpc_method }} failed
            on etcd instance {{ $labels.instance }}'
          summary: a high number of gRPC requests are failing
      - alert: HighNumberOfFailedGRPCRequests
        expr: sum(rate(etcd_grpc_requests_failed_total{job="etcd"}[5m])) BY (grpc_method)
          / sum(rate(etcd_grpc_total{job="etcd"}[5m])) BY (grpc_method) > 0.05
        for: 5m
        labels:
          severity: critical
        annotations:
          description: '{{ $value }}% of requests for {{ $labels.grpc_method }} failed
            on etcd instance {{ $labels.instance }}'
          summary: a high number of gRPC requests are failing
      - alert: GRPCRequestsSlow
        expr: histogram_quantile(0.99, rate(etcd_grpc_unary_requests_duration_seconds_bucket[5m])) > 0.15
        for: 10m
        labels:
          severity: critical
        annotations:
          description: on etcd instance {{ $labels.instance }} gRPC requests to {{ $labels.grpc_method
            }} are slow
          summary: slow gRPC requests
      - alert: HighNumberOfFailedHTTPRequests
        expr: sum(rate(etcd_http_failed_total{job="etcd"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job="etcd"}[5m]))
          BY (method) > 0.01
        for: 10m
        labels:
          severity: warning
        annotations:
          description: '{{ $value }}% of requests for {{ $labels.method }} failed on etcd
            instance {{ $labels.instance }}'
          summary: a high number of HTTP requests are failing
      - alert: HighNumberOfFailedHTTPRequests
        expr: sum(rate(etcd_http_failed_total{job="etcd"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job="etcd"}[5m]))
          BY (method) > 0.05
        for: 5m
        labels:
          severity: critical
        annotations:
          description: '{{ $value }}% of requests for {{ $labels.method }} failed on etcd
            instance {{ $labels.instance }}'
          summary: a high number of HTTP requests are failing
      - alert: HTTPRequestsSlow
        expr: histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m])) > 0.15
        for: 10m
        labels:
          severity: warning
        annotations:
          description: on etcd instance {{ $labels.instance }} HTTP requests to {{ $labels.method
            }} are slow
          summary: slow HTTP requests
      - alert: FdExhaustionClose
        expr: predict_linear(instance:fd_utilization[1h], 3600 * 4) > 1
        for: 10m
        labels:
          severity: warning
        annotations:
          description: '{{ $labels.job }} instance {{ $labels.instance }} will exhaust
            its file descriptors soon'
          summary: file descriptors soon exhausted
      - alert: FdExhaustionClose
        expr: predict_linear(instance:fd_utilization[10m], 3600) > 1
        for: 10m
        labels:
          severity: critical
        annotations:
          description: '{{ $labels.job }} instance {{ $labels.instance }} will exhaust
            its file descriptors soon'
          summary: file descriptors soon exhausted
      - alert: EtcdMemberCommunicationSlow
        expr: histogram_quantile(0.99, rate(etcd_network_member_round_trip_time_seconds_bucket[5m])) > 0.15
        for: 10m
        labels:
          severity: warning
        annotations:
          description: etcd instance {{ $labels.instance }} member communication with
            {{ $labels.To }} is slow
          summary: etcd member communication is slow
      - alert: HighNumberOfFailedProposals
        expr: increase(etcd_server_proposals_failed_total{job="etcd"}[1h]) > 5
        labels:
          severity: warning
        annotations:
          description: etcd instance {{ $labels.instance }} has seen {{ $value }} proposal
            failures within the last hour
          summary: a high number of proposals within the etcd cluster are failing
      - alert: HighFsyncDurations
        expr: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])) > 0.5
        for: 10m
        labels:
          severity: warning
        annotations:
          description: etcd instance {{ $labels.instance }} fync durations are high
          summary: high fsync durations
      - alert: HighCommitDurations
        expr: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[5m])) > 0.25
        for: 10m
        labels:
          severity: warning
        annotations:
          description: etcd instance {{ $labels.instance }} commit durations are high
          summary: high commit durations
      - alert: DatabaseSpaceExceeded
        expr: etcd_mvcc_db_total_size_in_bytes/etcd_server_quota_backend_bytes > 0.80
        for: 1m
        labels:
        severity: critical
        annotations:
        summary: "{{ $labels.instance }}, {{ $labels.job }} of etcd DB space uses more than 80%"
        runbook: "Please consider manual compaction and defrag. https://github.com/etcd-io/etcd/blob/master/Documentation/op-guide/maintenance.md"