添加elk的docker化使用容器

elk/README.md

@@ -0,0 +1,49 @@
+# Docker ELK stack
+
+参考的两个项目
+
+* [deviantony/docker-elk](https://github.com/deviantony/docker-elk)
+* [HackerWilson/docker-elk-deployment](https://github.com/HackerWilson/docker-elk-deployment)
+
+## 使用
+
+**注：要使用跨主机网络的话，记得先弄个好overlay的network的支持。**
+
+开发环境下，默认使用 `bridge` 方式，集群环境请改成 `overlay` 。
+
+### 一. ElasticSearch机
+
+1. 调整操作系统vm.max_map_count值
+
+	```
+	sudo sysctl -w vm.max_map_count=262144
+	```
+
+2. 创建本地data目录
+
+	根据配置，新建和赋予本地目录访问权限。如默认本地目录 `./data/` ，可参考命令 `chmod 777 -R ./data/`，特别是 `./data/elasticsearch` 目录如果没读写权限将启动失败。
+	
+3. 启动
+
+	```
+	docker-compose up -d
+	```
+
+### 二 集群的logstash-shipper
+
+建议每台docker容器上跑，在需要采集日志的集群机器上启动 `logstash-shipper` 容器。
+
+```
+docker-compose -f elk-logstash-shipper.yml up -d
+```
+
+### 三 初始化
+
+初始化kibana，这货启动稍慢..要耐心一点..
+	
+第一次启动时，需要建立索引，但没数据的话是无法创建，所以要先添加一些数据。
+
+```
+docker run --rm --log-driver gelf --log-opt gelf-address=udp://127.0.0.1:9500 -d registry.cn-hangzhou.aliyuncs.com/kennylee/alpine echo hello world
+```
+

elk/docker-compose-single.yml

@@ -0,0 +1,76 @@
+version: '2'
+
+services:
+    redis:
+        image: registry.cn-hangzhou.aliyuncs.com/kennylee/redis
+        container_name: elk_redis
+        restart: always
+        networks:
+            - elk
+        volumes:
+            - ./data/redis:/data:rw
+    elasticsearch:
+        build: elasticsearch/
+        container_name: elasticsearch 
+        restart: always
+        ports:
+            - "9200:9200"
+            - "9300:9300"
+        environment:
+            ES_JAVA_OPTS: "-Xmx256m -Xms256m"
+            # disable X-Pack
+            # see https://www.elastic.co/guide/en/x-pack/current/xpack-settings.html
+            #     https://www.elastic.co/guide/en/x-pack/current/installing-xpack.html#xpack-enabling
+            xpack.security.enabled: "false"
+            xpack.monitoring.enabled: "false"
+            xpack.graph.enabled: "false"
+            xpack.watcher.enabled: "false"
+        networks:
+            - elk
+        volumes:
+            - ./data/elasticsearch:/usr/share/elasticsearch/data:rw
+    logstash:
+        build: logstash/indexer/
+        container_name: logstash_indexer
+        restart: always
+        volumes:
+            - ./logstash/indexer/config/logstash.yml:/usr/share/logstash/config/logstash.yml
+            - ./logstash/indexer/pipeline:/usr/share/logstash/pipeline
+        environment:
+            LS_JAVA_OPTS: "-Xmx256m -Xms256m"
+        networks:
+            - elk
+        depends_on:
+            - elasticsearch
+            - redis
+    logstash-shipper:
+        image: registry.cn-hangzhou.aliyuncs.com/kennylee/logstash:5.3.0
+        container_name: logstash_shipper
+        restart: always
+        ports:
+            - 9500:9500/udp
+        volumes:
+            - ./logstash/shipper/config/logstash.yml:/usr/share/logstash/config/logstash.yml
+            - ./logstash/shipper/pipeline:/usr/share/logstash/pipeline
+        environment:
+            LS_JAVA_OPTS: "-Xmx256m -Xms256m"
+        networks:
+            - elk
+        depends_on:
+            - elasticsearch
+            - redis
+    kibana:
+        build: kibana/
+        container_name: kibana
+        restart: always
+        volumes:
+            - ./kibana/config/:/usr/share/kibana/config
+        ports:
+            - "5601:5601"
+        networks:
+            - elk
+        depends_on:
+            - elasticsearch
+networks:
+    elk:
+        driver: bridge

elk/docker-compose.yml

@@ -0,0 +1,56 @@
+version: '2'
+
+services:
+    redis:
+        image: registry.cn-hangzhou.aliyuncs.com/kennylee/redis
+        container_name: elk_redis
+        networks:
+            - elk
+        volumes:
+            - ./data/redis:/data:rw
+    elasticsearch:
+        build: elasticsearch/
+        container_name: elasticsearch            
+        ports:
+            - "9200:9200"
+            - "9300:9300"
+        environment:
+            ES_JAVA_OPTS: "-Xmx256m -Xms256m"
+            # disable X-Pack
+            # see https://www.elastic.co/guide/en/x-pack/current/xpack-settings.html
+            #     https://www.elastic.co/guide/en/x-pack/current/installing-xpack.html#xpack-enabling
+            xpack.security.enabled: "false"
+            xpack.monitoring.enabled: "false"
+            xpack.graph.enabled: "false"
+            xpack.watcher.enabled: "false"
+        networks:
+            - elk
+        volumes:
+            - ./data/elasticsearch:/usr/share/elasticsearch/data:rw
+    logstash:
+        build: logstash/indexer/
+        container_name: logstash_indexer
+        volumes:
+            - ./logstash/indexer/config/logstash.yml:/usr/share/logstash/config/logstash.yml
+            - ./logstash/indexer/pipeline:/usr/share/logstash/pipeline
+        environment:
+            LS_JAVA_OPTS: "-Xmx256m -Xms256m"
+        networks:
+            - elk
+        depends_on:
+            - elasticsearch
+            - redis
+    kibana:
+        build: kibana/
+        container_name: kibana
+        volumes:
+            - ./kibana/config/:/usr/share/kibana/config
+        ports:
+            - "5601:5601"
+        networks:
+            - elk
+        depends_on:
+            - elasticsearch
+networks:
+    elk:
+        driver: bridge

elk/elasticsearch/Dockerfile

@@ -0,0 +1,7 @@
+# https://github.com/elastic/elasticsearch-docker
+FROM registry.cn-hangzhou.aliyuncs.com/kennylee/elasticsearch:5.3.0
+
+# Add your elasticsearch plugins setup here
+# Example: RUN elasticsearch-plugin install analysis-icu
+
+

elk/elasticsearch/config/.placeholder

@@ -0,0 +1 @@
+Ensure the existence of the parent folder.

elk/elk-logstash-shipper.yml

@@ -0,0 +1,16 @@
+version: '2'
+
+services:
+    logstash-shipper:
+        image: registry.cn-hangzhou.aliyuncs.com/kennylee/logstash:5.3.0
+        ports:
+            - 9500:9500/udp
+        volumes:
+            - ./logstash/shipper/config/logstash.yml:/usr/share/logstash/config/logstash.yml
+            - ./logstash/shipper/pipeline:/usr/share/logstash/pipeline
+        environment:
+            LS_JAVA_OPTS: "-Xmx256m -Xms256m"
+networks:
+    default:
+        external: 
+            name: elk_elk

elk/kibana/Dockerfile

@@ -0,0 +1,5 @@
+# https://github.com/elastic/kibana-docker
+FROM registry.cn-hangzhou.aliyuncs.com/kennylee/kibana:5.3.0
+
+# Add your kibana plugins setup here
+# Example: RUN kibana-plugin install <name|url>

elk/kibana/config/kibana.yml

@@ -0,0 +1,16 @@
+---
+## Default Kibana configuration from kibana-docker.
+## from https://github.com/elastic/kibana-docker/blob/master/build/kibana/config/kibana.yml
+#
+server.name: kibana
+server.host: "0"
+elasticsearch.url: http://elasticsearch:9200
+
+## Disable X-Pack
+## see https://www.elastic.co/guide/en/x-pack/current/xpack-settings.html
+##     https://www.elastic.co/guide/en/x-pack/current/installing-xpack.html#xpack-enabling
+#
+xpack.security.enabled: false
+xpack.monitoring.enabled: false
+xpack.graph.enabled: false
+xpack.reporting.enabled: false

elk/logstash/indexer/Dockerfile

@@ -0,0 +1,5 @@
+# https://github.com/elastic/logstash-docker
+FROM registry.cn-hangzhou.aliyuncs.com/kennylee/logstash:5.3.0
+
+# Add your logstash plugins setup here
+# Example: RUN logstash-plugin install logstash-filter-json

elk/logstash/indexer/config/logstash.yml

@@ -0,0 +1,11 @@
+---
+## Default Logstash configuration from logstash-docker.
+## from https://github.com/elastic/logstash-docker/blob/master/build/logstash/config/logstash.yml
+#
+http.host: "0.0.0.0"
+
+## Disable X-Pack
+## see https://www.elastic.co/guide/en/x-pack/current/xpack-settings.html
+##     https://www.elastic.co/guide/en/x-pack/current/installing-xpack.html#xpack-enabling
+#
+xpack.monitoring.enabled: false

elk/logstash/indexer/pipeline/logstash.conf

@@ -0,0 +1,17 @@
+input {
+    redis { 
+        data_type => "channel" 
+        key => "logstash" 
+        host => "redis" 
+        port=> 6379 
+    } 
+}
+
+## Add your filters / logstash plugins configuration here
+
+output {
+	elasticsearch {
+		hosts => "elasticsearch:9200"
+	}
+    stdout { codec => rubydebug }
+}

elk/logstash/shipper-tcp/config/logstash.yml

@@ -0,0 +1,11 @@
+---
+## Default Logstash configuration from logstash-docker.
+## from https://github.com/elastic/logstash-docker/blob/master/build/logstash/config/logstash.yml
+#
+http.host: "0.0.0.0"
+
+## Disable X-Pack
+## see https://www.elastic.co/guide/en/x-pack/current/xpack-settings.html
+##     https://www.elastic.co/guide/en/x-pack/current/installing-xpack.html#xpack-enabling
+#
+xpack.monitoring.enabled: false

elk/logstash/shipper-tcp/pipeline/logstash.conf

@@ -0,0 +1,20 @@
+input { 
+  stdin{} 
+  tcp {
+    port => 1514
+    codec => json {
+        charset => "UTF-8"
+    }   
+  }
+} 
+
+output { 
+    redis { 
+        host => "elk_redis" 
+        port => 6379 
+        data_type => "channel" 
+        key => "logstash" 
+    } 
+    stdout { codec => rubydebug }
+}
+

elk/logstash/shipper/config/logstash.yml

@@ -0,0 +1,11 @@
+---
+## Default Logstash configuration from logstash-docker.
+## from https://github.com/elastic/logstash-docker/blob/master/build/logstash/config/logstash.yml
+#
+http.host: "0.0.0.0"
+
+## Disable X-Pack
+## see https://www.elastic.co/guide/en/x-pack/current/xpack-settings.html
+##     https://www.elastic.co/guide/en/x-pack/current/installing-xpack.html#xpack-enabling
+#
+xpack.monitoring.enabled: false

elk/logstash/shipper/pipeline/logstash.conf

@@ -0,0 +1,16 @@
+input { 
+    stdin{} 
+    gelf { 
+        port => 9500 
+    } 
+} 
+
+output { 
+    redis { 
+        host => "elk_redis" 
+        port => 6379 
+        data_type => "channel" 
+        key => "logstash" 
+    }
+    stdout { codec => rubydebug }
+}

lts/lts-center/lts-jobtracker/Dockerfile

@@ -1,18 +1,7 @@
-FROM registry.cn-hangzhou.aliyuncs.com/kennylee/java:oracle-java8
+FROM registry.cn-hangzhou.aliyuncs.com/kennylee/java:openjdk-8-jre
 
 MAINTAINER kennylee26 <kennylee26@gmail.com>
 
-RUN apt-get -y update && \
-    apt-get -y install language-pack-zh-hans && \
-    rm -rf /var/lib/apt/lists/* 
-RUN locale
-ENV LANG=zh_CN.UTF-8;\
-    LANGUAGE=zh_CN:zh:en_US:en;\
-    LC_ALL=zh_CN.UTF-8;\
-    TZ="Asia/Shanghai";\
-    TERM=xterm
-RUN echo "Asia/Shanghai"  | sudo tee /etc/timezone && sudo dpkg-reconfigure --frontend noninteractive tzdata
-
 COPY ./app/lts-jobtracker.jar /data/lts-jobtracker.jar
 
 CMD ["bash"]

lts/lts-tasktracker/Dockerfile

@@ -2,17 +2,9 @@ FROM registry.cn-hangzhou.aliyuncs.com/kennylee/java:openjdk-7-jre
 
 MAINTAINER kennylee26 <kennylee26@gmail.com>
 
-RUN apt-get -y update && \
-    apt-get -y install language-pack-zh-hans && \
-    rm -rf /var/lib/apt/lists/* 
-RUN locale
-ENV LANG=zh_CN.UTF-8;\
-    LANGUAGE=zh_CN:zh:en_US:en;\
-    LC_ALL=zh_CN.UTF-8;\
-    TZ="Asia/Shanghai";\
-    TERM=xterm
-RUN echo "Asia/Shanghai"  | sudo tee /etc/timezone && sudo dpkg-reconfigure --frontend noninteractive tzdata
+COPY ./app/lts-tasktracker.jar /data/app/lts-tasktracker.jar
 
-COPY ./app/lts-tasktracker.jar /data/lts-tasktracker.jar
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
 
-CMD ["bash"]
+ENTRYPOINT "/entrypoint.sh"

lts/lts-tasktracker/docker-compose.yml

@@ -6,7 +6,5 @@ services:
             - ./config/application.yml:/data/application.yml:ro
         network_mode: 'host'
         entrypoint: 
-            - java 
-            - -jar 
-            - /data/lts-tasktracker.jar
-            - --spring.config.location=/data/application.yml
+            - /entrypoint.sh
+            - --spring.config.location=/data/application.yml   

lts/lts-tasktracker/entrypoint.sh

@@ -0,0 +1,17 @@
+#! /bin/bash
+
+BASE_PATH=/data/app/
+ORI_APP_FILE=$(find $BASE_PATH -type f -name "*.jar" 2>/dev/null | head -1)
+APP_FILE=${BASE_PATH}app.jar
+
+if [ ${#ORI_APP_FILE} -gt 0 ]; then
+    echo "create symbolic link for $ORI_APP_FILE"
+    if [ -e $APP_FILE ]; then
+        #echo "$APP_FILE was existed, remove that."
+        rm -f $APP_FILE
+    fi
+    ln -s $ORI_APP_FILE $APP_FILE
+fi
+
+# if $APP_FILE not exist, let it throw error。
+java $JAVA_OPTS -jar $APP_FILE "$@"