优化一些配置文件

2024-11-29 18:38:34 +08:00 · 2021-07-09 16:45:21 +08:00 · 2021-07-09 16:45:21 +08:00 · ca34c0c0e4
commit ca34c0c0e4
parent 21311bddf0
15 changed files with 187 additions and 41 deletions
--- a/grafana/docker-compose.yml
+++ b/grafana/docker-compose.yml
@ -1,6 +1,6 @@
 version: '2'
 services:
-    app:
+  grafana:
    image: registry.cn-hangzhou.aliyuncs.com/kennylee/grafana:8.0.0
    ports:
      - 3000:3000
--- a/loki/README.md
+++ b/loki/README.md
@ -5,4 +5,4 @@

 ## 使用

-应用启动后，可以访问如 `http://localhost:3100/metrics` 看看是否有输出内容。
+应用启动后，可以访问如 `http://localhost:3100/ready` 看看是否有输出内容。
--- a/loki/docker-compose.yml
+++ b/loki/docker-compose.yml
@ -1,12 +1,12 @@
 version: '2'
 services:
-    app:
+  loki:
    image: registry.cn-hangzhou.aliyuncs.com/kennylee/loki:2.2.1
    ports:
      - 3100:3100
    container_name: loki
    restart: on-failure
-        command: '-config.file=/loki/config/loki-local-config.yaml'
+    command: '-config.file=/loki/config/loki-config.yaml'
    volumes:
      - ./config:/loki/config:z
      - ./data/loki:/tmp/loki
--- a/prometheus/Dockerfile
+++ b/prometheus/Dockerfile
@ -33,3 +33,6 @@ RUN ln -fs /usr/share/zoneinfo/$TZ /etc/localtime && \
 USER 1001

 EXPOSE 9090
+
+# CMD [ "--config.file=/opt/bitnami/prometheus/conf/prometheus.yml", "--storage.tsdb.path=/opt/bitnami/prometheus/data", "--web.console.libraries=/opt/bitnami/prometheus/conf/console_libraries", "--web.console.templates=/opt/bitnami/prometheus/conf/consoles" ]
+
--- a/prometheus/README.md
+++ b/prometheus/README.md
@ -4,7 +4,7 @@

 相关构建镜像

-* [prom/alertmanager](https://hub.docker.com/r/prom/alertmanager)
+* [bitnami/alertmanager](https://hub.docker.com/r/bitnami/alertmanager)
 * [prom/blackbox-exporter](https://hub.docker.com/r/prom/blackbox-exporter)
 * [prom/pushgateway](https://hub.docker.com/r/prom/pushgateway)

--- a/prometheus/alertmanager/config/alertmanager.yml
+++ b/prometheus/alertmanager/config/alertmanager.yml
@ -0,0 +1,36 @@
+global:
+  resolve_timeout: 5m
+  smtp_from: '{{ template "email.from" . }}'
+  smtp_smarthost: 'smtp.exmail.qq.com:25'
+  smtp_auth_username: 'xxxx@qq,com' # 不能使用 tmpl_string
+  smtp_auth_password: 'password123456'
+  smtp_require_tls: false
+
+templates:
+  - '/etc/alertmanager-tmpl/email.gohtml'
+
+route:
+  receiver: 'email'
+  group_by: [ 'alertname' ]
+  group_wait: 5s
+  group_interval: 5s
+  repeat_interval: 5m
+  routes:
+    - receiver: 'email'
+      group_wait: 10s
+      matchers:
+        - severity="critical"
+
+receivers: # 配置告警消息接受者信息，例如常用的 email、wechat、slack、webhook 等消息通知方式。
+  - name: 'email'
+    email_configs:
+      - to: '{{ template "email.to" . }}'
+        html: '{{ template "email.to.html" . }}'
+        send_resolved: true
+
+inhibit_rules:
+  - source_match:
+      severity: 'critical'
+    target_match:
+      severity: 'warning'
+    equal: [ 'alertname', 'instance' ]
--- a/prometheus/alertmanager/config/email.gohtml
+++ b/prometheus/alertmanager/config/email.gohtml
@ -0,0 +1,14 @@
+{{ define "email.from" }}noreply@qq.com{{ end }}
+{{ define "email.to" }}xxxxxxxx@qq.com{{ end }}
+{{ define "email.to.html" }}
+    {{ range .Alerts }}
+        =========start==========<br>
+        告警程序: prometheus_alert <br>
+        告警级别: {{ .Labels.severity }} 级 <br>
+        告警类型: {{ .Labels.alertname }} <br>
+        故障主机: {{ .Labels.instance }} <br>
+        告警主题: {{ .Annotations.summary }} <br>
+        告警详情: {{ .Annotations.description }} <br>
+        =========end==========<br>
+    {{ end }}
+{{ end }}
--- a/prometheus/alertmanager/docker-compose.yml
+++ b/prometheus/alertmanager/docker-compose.yml
@ -0,0 +1,17 @@
+version: '2'
+services:
+  alertmanager:
+    image: registry.cn-hangzhou.aliyuncs.com/kennylee/alertmanager:0.22.2
+    container_name: alertmanager
+    ports:
+      - '9093:9093'
+    restart: on-failure
+    volumes:
+      - ./config/alertmanager.yml:/opt/bitnami/alertmanager/conf/config.yml
+      - ./config/email.gohtml:/etc/alertmanager-tmpl/email.gohtml
+      - ./data:/opt/bitnami/data
+    logging:
+      driver: 'json-file'
+      options:
+        max-size: '10m'
+        max-file: '1'
--- a/prometheus/blackbox_exporter/blackbox_exporter.yml
+++ b/prometheus/blackbox_exporter/blackbox_exporter.yml
@ -0,0 +1,17 @@
+scrape_configs:
+  - job_name: 'blackbox'
+    metrics_path: /probe
+    params:
+      module: [ http_2xx ]  # Look for a HTTP 200 response.
+    static_configs:
+      - targets:
+          - http://prometheus.io    # Target to probe with http.
+          - https://prometheus.io   # Target to probe with https.
+          - http://example.com:8080 # Target to probe with http on port 8080.
+    relabel_configs:
+      - source_labels: [ __address__ ]
+        target_label: __param_target
+      - source_labels: [ __param_target ]
+        target_label: instance
+      - target_label: __address__
+        replacement: 127.0.0.1:9115  # The blackbox exporter's real hostname:port.
--- a/prometheus/blackbox_exporter/docker-compose.yml
+++ b/prometheus/blackbox_exporter/docker-compose.yml
@ -0,0 +1,15 @@
+version: '2'
+services:
+  blackbox_exporter:
+    image: registry.cn-hangzhou.aliyuncs.com/kennylee/blackbox_exporter
+    container_name: blackbox_exporter
+    restart: on-failure
+    ports:
+      - '9115:9115'
+    volumes:
+      - ./blackbox_exporter.yml:/opt/bitnami/prometheus/conf/prometheus.yml
+    logging:
+      driver: 'json-file'
+      options:
+        max-size: '10m'
+        max-file: '1'
--- a/prometheus/config/rules/app-rules.yml
+++ b/prometheus/config/rules/app-rules.yml
@ -0,0 +1,12 @@
+groups:
+  - name: app-error-logs
+    rules:
+      - alert: '应用发生错误日志'
+        expr: promtail_custom_total_error_logs > 0
+        for: 1m
+        labels:
+          serverity: critical
+        annotations:
+          summary: '错误日志量5分钟内大于0'
+          description: "Total error logs over {{ $value }}"
+          value: "{{ $value }}"
--- a/promtail/config/promtail-config.yaml
+++ b/promtail/config/promtail-config.yaml
@ -12,8 +12,20 @@ scrape_configs:
  - job_name: test-nginx-logs
    static_configs:
      - labels:
-          job: 'test-nginx-logs'
+          job: 'test-k8s-pod-nginx-logs'
          __path__: /tmp/logs/*.log
+    pipeline_stages:
+      - match:
+          selector: '{job="test-k8s-pod-nginx-logs"}'
+          stages:
+            - regex:
+                expression: '^.*F (?P<remote_addr>[\w\.]+) - (?P<remote_user>[^ ]*) \[(?P<time_local>.*)\] "(?P<method>[^ ]*) (?P<request>[^ ]*) (?P<protocol>[^ ]*)" (?P<status>[\d]+) (?P<body_bytes_sent>[\d]+) "(?P<http_referer>[^"]*)" "(?P<http_user_agent>[^"]*)"?'
+            - labels:
+                #remote_addr:
+                #time_local:
+                status:
+                method:
+                #request:
  - job_name: test-app-logs
    static_configs:
      - labels:
--- a/promtail/docker-compose.yml
+++ b/promtail/docker-compose.yml
@ -1,4 +1,4 @@
-version: '2.4'
+version: '2'
 services:
  promtail:
    image: registry.cn-hangzhou.aliyuncs.com/kennylee/promtail:2.2.1
--- a/promtail/logs/nginx.log
+++ b/promtail/logs/nginx.log
@ -1,5 +1,9 @@
-127.0.0.1 - - [18/Jun/2021:14:17:59 +0800] "GET /favicon.ico HTTP/1.1" 200 1150 "http://www.foo.com:31000/login" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.101 Safari/537.36" "-"
-127.0.0.1 - - [18/Jun/2021:14:18:02 +0800] "POST /api/v3/auth HTTP/1.1" 200 361 "http://www.foo.com:31000/login" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.101 Safari/537.36" "-"
-127.0.0.1 - - [18/Jun/2021:14:18:02 +0800] "GET /api/v1/sys/userInfo?dummy=1623997081923 HTTP/1.1" 200 746 "http://www.foo.com:31000/login" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.101 Safari/537.36" "-"
-127.0.0.1 - - [18/Jun/2021:14:18:02 +0800] "GET /api/v1/menu/my?dummy=1623997082017 HTTP/1.1" 200 2014 "http://www.foo.com:31000/dashboard/console" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.101 Safari/537.36" "-"
-127.0.0.1 - - [18/Jun/2021:14:18:06 +0800] "GET /api/v3/auth/captcha?dummy=1623997086527 HTTP/1.1" 404 246 "http://www.foo.com:31000/login" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.101 Safari/537.36" "-"
+2021-07-07T10:12:54.776155602+08:00 stdout F 10.100.195.51 - - [07/Jul/2021:10:12:54 +0800] "GET / HTTP/1.1" 200 912 "-" "Go-http-client/1.1" "-"
+2021-07-07T09:49:39.777318562+08:00 stdout F 10.100.195.51 - - [07/Jul/2021:09:49:39 +0800] "GET / HTTP/1.1" 200 912 "-" "Go-http-client/1.1" "-"
+2021-07-07T10:11:48.511185796+08:00 stdout F 192.168.1.206 - - [07/Jul/2021:10:11:48 +0800] "GET /api/v3/auth/captcha?dummy=1625623908350 HTTP/1.1" 404 246 "http://tksite3.dev.timekey.com.cn:31000/login?redirect=%2Fdashboard%2Fconsole" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36" "-"
+2021-07-07T10:11:47.914407384+08:00 stdout F 192.168.1.206 - - [07/Jul/2021:10:11:47 +0800] "GET /js/app.6eeea57f.js HTTP/1.1" 200 338870 "http://tksite3.dev.timekey.com.cn:31000/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36" "-"
+2021-07-07T10:11:47.900575964+08:00 stderr F 2021/07/07 10:11:47 [warn] 7#0: *3104 an upstream response is buffered to a temporary file /etc/nginx/proxy_temp/3/01/0000000013 while reading upstream, client: 192.168.1.206, server: localhost, request: "GET /js/chunk-vendors.9a2cbb8f.js HTTP/1.1", upstream: "http://10.96.233.9:80/js/chunk-vendors.9a2cbb8f.js", host: "tksite3.dev.timekey.com.cn:31000", referrer: "http://tksite3.dev.timekey.com.cn:31000/"
+2021-07-07T10:11:47.743676835+08:00 stderr F 2021/07/07 10:11:47 [warn] 7#0: *3104 an upstream response is buffered to a temporary file /etc/nginx/proxy_temp/1/01/0000000011 while reading upstream, client: 192.168.1.206, server: localhost, request: "GET /css/app.8fd2da49.css HTTP/1.1", upstream: "http://10.96.233.9:80/css/app.8fd2da49.css", host: "tksite3.dev.timekey.com.cn:31000", referrer: "http://tksite3.dev.timekey.com.cn:31000/"
+2021-06-28T16:00:06.577696021+08:00 stdout F 192.168.1.206 - - [28/Jun/2021:16:00:06 +0800] "GET /logout HTTP/1.1" 304 0 "http://tksite3.dev.timekey.com.cn:31000/dashboard/console" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36" "-"
+2021-06-28T16:00:33.01527833+08:00 stdout F 192.168.1.206 - - [28/Jun/2021:16:00:33 +0800] "GET /api/v1/user?page=1&rows=10&sort=createTime%3Adesc&sort=name%3Aacs&sort=cnName%3Aacs&sort=position&dummy=1624867232897 HTTP/1.1" 200 113 "http://tksite3.dev.timekey.com.cn:31000/system/user" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36" "-"
+2021-06-28T16:00:49.238698156+08:00 stdout F 192.168.1.206 - - [28/Jun/2021:16:00:49 +0800] "GET /api/v1/role?sort=position&sort=name&rows=1000&dummy=1624867249124 HTTP/1.1" 200 377 "http://tksite3.dev.timekey.com.cn:31000/system/menu" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36" "-"
--- a/tnginx/alpine/config/nginx.conf
+++ b/tnginx/alpine/config/nginx.conf
@ -14,11 +14,27 @@ http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

-    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
-                      '$status $body_bytes_sent "$http_referer" '
-                      '"$http_user_agent" "$http_x_forwarded_for"';
+#     log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+#                       '$status $body_bytes_sent "$http_referer" '
+#                       '"$http_user_agent" "$http_x_forwarded_for"';

-    access_log  /var/log/nginx/access.log  main;
+    log_format json_combined escape=json
+      '{'
+        '"time_local":"$time_local",'
+        '"remote_addr":"$remote_addr",'
+        '"remote_user":"$remote_user",'
+        '"path":"$request_uri",'
+        '"req":"$request",'
+        '"status": "$status",'
+        '"method": "$request_method",'
+        '"size":"$body_bytes_sent",'
+        '"request_time":"$request_time",'
+        '"referrer":"$http_referer",'
+        '"ua":"$http_user_agent",'
+        '"forwardedfor":"$http_x_forwarded_for"'
+      '}';
+
+    access_log  /var/log/nginx/access.log  json_combined;

    # List of application servers
    upstream app_servers {