diff --git a/elk/README.md b/elk/README.md new file mode 100755 index 0000000..e181809 --- /dev/null +++ b/elk/README.md @@ -0,0 +1,49 @@ +# Docker ELK stack + +参考的两个项目 + +* [deviantony/docker-elk](https://github.com/deviantony/docker-elk) +* [HackerWilson/docker-elk-deployment](https://github.com/HackerWilson/docker-elk-deployment) + +## 使用 + +**注：要使用跨主机网络的话，记得先弄个好overlay的network的支持。** + +开发环境下，默认使用 `bridge` 方式，集群环境请改成 `overlay` 。 + +### 一. ElasticSearch机 + +1. 调整操作系统vm.max_map_count值 + + ``` + sudo sysctl -w vm.max_map_count=262144 + ``` + +2. 创建本地data目录 + + 根据配置，新建和赋予本地目录访问权限。如默认本地目录 `./data/` ，可参考命令 `chmod 777 -R ./data/`，特别是 `./data/elasticsearch` 目录如果没读写权限将启动失败。 + +3. 启动 + + ``` + docker-compose up -d + ``` + +### 二 集群的logstash-shipper + +建议每台docker容器上跑，在需要采集日志的集群机器上启动 `logstash-shipper` 容器。 + +``` +docker-compose -f elk-logstash-shipper.yml up -d +``` + +### 三 初始化 + +初始化kibana，这货启动稍慢..要耐心一点.. + +第一次启动时，需要建立索引，但没数据的话是无法创建，所以要先添加一些数据。 + +``` +docker run --rm --log-driver gelf --log-opt gelf-address=udp://127.0.0.1:9500 -d registry.cn-hangzhou.aliyuncs.com/kennylee/alpine echo hello world +``` + diff --git a/elk/docker-compose-single.yml b/elk/docker-compose-single.yml new file mode 100755 index 0000000..7578a13 --- /dev/null +++ b/elk/docker-compose-single.yml @@ -0,0 +1,76 @@ +version: '2' + +services: + redis: + image: registry.cn-hangzhou.aliyuncs.com/kennylee/redis + container_name: elk_redis + restart: always + networks: + - elk + volumes: + - ./data/redis:/data:rw + elasticsearch: + build: elasticsearch/ + container_name: elasticsearch + restart: always + ports: + - "9200:9200" + - "9300:9300" + environment: + ES_JAVA_OPTS: "-Xmx256m -Xms256m" + # disable X-Pack + # see https://www.elastic.co/guide/en/x-pack/current/xpack-settings.html + # https://www.elastic.co/guide/en/x-pack/current/installing-xpack.html#xpack-enabling + xpack.security.enabled: "false" + xpack.monitoring.enabled: "false" + xpack.graph.enabled: "false" + xpack.watcher.enabled: "false" + networks: + - elk + volumes: + - ./data/elasticsearch:/usr/share/elasticsearch/data:rw + logstash: + build: logstash/indexer/ + container_name: logstash_indexer + restart: always + volumes: + - ./logstash/indexer/config/logstash.yml:/usr/share/logstash/config/logstash.yml + - ./logstash/indexer/pipeline:/usr/share/logstash/pipeline + environment: + LS_JAVA_OPTS: "-Xmx256m -Xms256m" + networks: + - elk + depends_on: + - elasticsearch + - redis + logstash-shipper: + image: registry.cn-hangzhou.aliyuncs.com/kennylee/logstash:5.3.0 + container_name: logstash_shipper + restart: always + ports: + - 9500:9500/udp + volumes: + - ./logstash/shipper/config/logstash.yml:/usr/share/logstash/config/logstash.yml + - ./logstash/shipper/pipeline:/usr/share/logstash/pipeline + environment: + LS_JAVA_OPTS: "-Xmx256m -Xms256m" + networks: + - elk + depends_on: + - elasticsearch + - redis + kibana: + build: kibana/ + container_name: kibana + restart: always + volumes: + - ./kibana/config/:/usr/share/kibana/config + ports: + - "5601:5601" + networks: + - elk + depends_on: + - elasticsearch +networks: + elk: + driver: bridge diff --git a/elk/docker-compose.yml b/elk/docker-compose.yml new file mode 100755 index 0000000..f908698 --- /dev/null +++ b/elk/docker-compose.yml @@ -0,0 +1,56 @@ +version: '2' + +services: + redis: + image: registry.cn-hangzhou.aliyuncs.com/kennylee/redis + container_name: elk_redis + networks: + - elk + volumes: + - ./data/redis:/data:rw + elasticsearch: + build: elasticsearch/ + container_name: elasticsearch + ports: + - "9200:9200" + - "9300:9300" + environment: + ES_JAVA_OPTS: "-Xmx256m -Xms256m" + # disable X-Pack + # see https://www.elastic.co/guide/en/x-pack/current/xpack-settings.html + # https://www.elastic.co/guide/en/x-pack/current/installing-xpack.html#xpack-enabling + xpack.security.enabled: "false" + xpack.monitoring.enabled: "false" + xpack.graph.enabled: "false" + xpack.watcher.enabled: "false" + networks: + - elk + volumes: + - ./data/elasticsearch:/usr/share/elasticsearch/data:rw + logstash: + build: logstash/indexer/ + container_name: logstash_indexer + volumes: + - ./logstash/indexer/config/logstash.yml:/usr/share/logstash/config/logstash.yml + - ./logstash/indexer/pipeline:/usr/share/logstash/pipeline + environment: + LS_JAVA_OPTS: "-Xmx256m -Xms256m" + networks: + - elk + depends_on: + - elasticsearch + - redis + kibana: + build: kibana/ + container_name: kibana + volumes: + - ./kibana/config/:/usr/share/kibana/config + ports: + - "5601:5601" + networks: + - elk + depends_on: + - elasticsearch +networks: + elk: + driver: bridge diff --git a/elk/elasticsearch/Dockerfile b/elk/elasticsearch/Dockerfile new file mode 100755 index 0000000..cf2c998 --- /dev/null +++ b/elk/elasticsearch/Dockerfile @@ -0,0 +1,7 @@ +# https://github.com/elastic/elasticsearch-docker +FROM registry.cn-hangzhou.aliyuncs.com/kennylee/elasticsearch:5.3.0 + +# Add your elasticsearch plugins setup here +# Example: RUN elasticsearch-plugin install analysis-icu + + diff --git a/elk/elasticsearch/config/.placeholder b/elk/elasticsearch/config/.placeholder new file mode 100755 index 0000000..9ad2662 --- /dev/null +++ b/elk/elasticsearch/config/.placeholder @@ -0,0 +1 @@ +Ensure the existence of the parent folder. diff --git a/elk/elk-logstash-shipper.yml b/elk/elk-logstash-shipper.yml new file mode 100755 index 0000000..43c414f --- /dev/null +++ b/elk/elk-logstash-shipper.yml @@ -0,0 +1,16 @@ +version: '2' + +services: + logstash-shipper: + image: registry.cn-hangzhou.aliyuncs.com/kennylee/logstash:5.3.0 + ports: + - 9500:9500/udp + volumes: + - ./logstash/shipper/config/logstash.yml:/usr/share/logstash/config/logstash.yml + - ./logstash/shipper/pipeline:/usr/share/logstash/pipeline + environment: + LS_JAVA_OPTS: "-Xmx256m -Xms256m" +networks: + default: + external: + name: elk_elk diff --git a/elk/kibana/Dockerfile b/elk/kibana/Dockerfile new file mode 100755 index 0000000..6ef55e6 --- /dev/null +++ b/elk/kibana/Dockerfile @@ -0,0 +1,5 @@ +# https://github.com/elastic/kibana-docker +FROM registry.cn-hangzhou.aliyuncs.com/kennylee/kibana:5.3.0 + +# Add your kibana plugins setup here +# Example: RUN kibana-plugin install diff --git a/elk/kibana/config/kibana.yml b/elk/kibana/config/kibana.yml new file mode 100755 index 0000000..166f255 --- /dev/null +++ b/elk/kibana/config/kibana.yml @@ -0,0 +1,16 @@ +--- +## Default Kibana configuration from kibana-docker. +## from https://github.com/elastic/kibana-docker/blob/master/build/kibana/config/kibana.yml +# +server.name: kibana +server.host: "0" +elasticsearch.url: http://elasticsearch:9200 + +## Disable X-Pack +## see https://www.elastic.co/guide/en/x-pack/current/xpack-settings.html +## https://www.elastic.co/guide/en/x-pack/current/installing-xpack.html#xpack-enabling +# +xpack.security.enabled: false +xpack.monitoring.enabled: false +xpack.graph.enabled: false +xpack.reporting.enabled: false diff --git a/elk/logstash/indexer/Dockerfile b/elk/logstash/indexer/Dockerfile new file mode 100755 index 0000000..ed7c7cf --- /dev/null +++ b/elk/logstash/indexer/Dockerfile @@ -0,0 +1,5 @@ +# https://github.com/elastic/logstash-docker +FROM registry.cn-hangzhou.aliyuncs.com/kennylee/logstash:5.3.0 + +# Add your logstash plugins setup here +# Example: RUN logstash-plugin install logstash-filter-json diff --git a/elk/logstash/indexer/config/logstash.yml b/elk/logstash/indexer/config/logstash.yml new file mode 100755 index 0000000..65400fb --- /dev/null +++ b/elk/logstash/indexer/config/logstash.yml @@ -0,0 +1,11 @@ +--- +## Default Logstash configuration from logstash-docker. +## from https://github.com/elastic/logstash-docker/blob/master/build/logstash/config/logstash.yml +# +http.host: "0.0.0.0" + +## Disable X-Pack +## see https://www.elastic.co/guide/en/x-pack/current/xpack-settings.html +## https://www.elastic.co/guide/en/x-pack/current/installing-xpack.html#xpack-enabling +# +xpack.monitoring.enabled: false diff --git a/elk/logstash/indexer/pipeline/logstash.conf b/elk/logstash/indexer/pipeline/logstash.conf new file mode 100755 index 0000000..96ca92d --- /dev/null +++ b/elk/logstash/indexer/pipeline/logstash.conf @@ -0,0 +1,17 @@ +input { + redis { + data_type => "channel" + key => "logstash" + host => "redis" + port=> 6379 + } +} + +## Add your filters / logstash plugins configuration here + +output { + elasticsearch { + hosts => "elasticsearch:9200" + } + stdout { codec => rubydebug } +} diff --git a/elk/logstash/shipper-tcp/config/logstash.yml b/elk/logstash/shipper-tcp/config/logstash.yml new file mode 100644 index 0000000..65400fb --- /dev/null +++ b/elk/logstash/shipper-tcp/config/logstash.yml @@ -0,0 +1,11 @@ +--- +## Default Logstash configuration from logstash-docker. +## from https://github.com/elastic/logstash-docker/blob/master/build/logstash/config/logstash.yml +# +http.host: "0.0.0.0" + +## Disable X-Pack +## see https://www.elastic.co/guide/en/x-pack/current/xpack-settings.html +## https://www.elastic.co/guide/en/x-pack/current/installing-xpack.html#xpack-enabling +# +xpack.monitoring.enabled: false diff --git a/elk/logstash/shipper-tcp/pipeline/logstash.conf b/elk/logstash/shipper-tcp/pipeline/logstash.conf new file mode 100644 index 0000000..a2b7d8f --- /dev/null +++ b/elk/logstash/shipper-tcp/pipeline/logstash.conf @@ -0,0 +1,20 @@ +input { + stdin{} + tcp { + port => 1514 + codec => json { + charset => "UTF-8" + } + } +} + +output { + redis { + host => "elk_redis" + port => 6379 + data_type => "channel" + key => "logstash" + } + stdout { codec => rubydebug } +} + diff --git a/elk/logstash/shipper/config/logstash.yml b/elk/logstash/shipper/config/logstash.yml new file mode 100755 index 0000000..65400fb --- /dev/null +++ b/elk/logstash/shipper/config/logstash.yml @@ -0,0 +1,11 @@ +--- +## Default Logstash configuration from logstash-docker. +## from https://github.com/elastic/logstash-docker/blob/master/build/logstash/config/logstash.yml +# +http.host: "0.0.0.0" + +## Disable X-Pack +## see https://www.elastic.co/guide/en/x-pack/current/xpack-settings.html +## https://www.elastic.co/guide/en/x-pack/current/installing-xpack.html#xpack-enabling +# +xpack.monitoring.enabled: false diff --git a/elk/logstash/shipper/pipeline/logstash.conf b/elk/logstash/shipper/pipeline/logstash.conf new file mode 100755 index 0000000..121416e --- /dev/null +++ b/elk/logstash/shipper/pipeline/logstash.conf @@ -0,0 +1,16 @@ +input { + stdin{} + gelf { + port => 9500 + } +} + +output { + redis { + host => "elk_redis" + port => 6379 + data_type => "channel" + key => "logstash" + } + stdout { codec => rubydebug } +} diff --git a/lts/lts-center/lts-jobtracker/Dockerfile b/lts/lts-center/lts-jobtracker/Dockerfile index c0a858f..bad6aed 100755 --- a/lts/lts-center/lts-jobtracker/Dockerfile +++ b/lts/lts-center/lts-jobtracker/Dockerfile @@ -1,18 +1,7 @@ -FROM registry.cn-hangzhou.aliyuncs.com/kennylee/java:oracle-java8 +FROM registry.cn-hangzhou.aliyuncs.com/kennylee/java:openjdk-8-jre MAINTAINER kennylee26 -RUN apt-get -y update && \ - apt-get -y install language-pack-zh-hans && \ - rm -rf /var/lib/apt/lists/* -RUN locale -ENV LANG=zh_CN.UTF-8;\ - LANGUAGE=zh_CN:zh:en_US:en;\ - LC_ALL=zh_CN.UTF-8;\ - TZ="Asia/Shanghai";\ - TERM=xterm -RUN echo "Asia/Shanghai" | sudo tee /etc/timezone && sudo dpkg-reconfigure --frontend noninteractive tzdata - COPY ./app/lts-jobtracker.jar /data/lts-jobtracker.jar CMD ["bash"] diff --git a/lts/lts-tasktracker/Dockerfile b/lts/lts-tasktracker/Dockerfile index e9093f3..82f90bf 100755 --- a/lts/lts-tasktracker/Dockerfile +++ b/lts/lts-tasktracker/Dockerfile @@ -2,17 +2,9 @@ FROM registry.cn-hangzhou.aliyuncs.com/kennylee/java:openjdk-7-jre MAINTAINER kennylee26 -RUN apt-get -y update && \ - apt-get -y install language-pack-zh-hans && \ - rm -rf /var/lib/apt/lists/* -RUN locale -ENV LANG=zh_CN.UTF-8;\ - LANGUAGE=zh_CN:zh:en_US:en;\ - LC_ALL=zh_CN.UTF-8;\ - TZ="Asia/Shanghai";\ - TERM=xterm -RUN echo "Asia/Shanghai" | sudo tee /etc/timezone && sudo dpkg-reconfigure --frontend noninteractive tzdata +COPY ./app/lts-tasktracker.jar /data/app/lts-tasktracker.jar -COPY ./app/lts-tasktracker.jar /data/lts-tasktracker.jar +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh -CMD ["bash"] +ENTRYPOINT "/entrypoint.sh" diff --git a/lts/lts-tasktracker/docker-compose.yml b/lts/lts-tasktracker/docker-compose.yml index ae38a58..e0fd90c 100644 --- a/lts/lts-tasktracker/docker-compose.yml +++ b/lts/lts-tasktracker/docker-compose.yml @@ -6,7 +6,5 @@ services: - ./config/application.yml:/data/application.yml:ro network_mode: 'host' entrypoint: - - java - - -jar - - /data/lts-tasktracker.jar - - --spring.config.location=/data/application.yml + - /entrypoint.sh + - --spring.config.location=/data/application.yml diff --git a/lts/lts-tasktracker/entrypoint.sh b/lts/lts-tasktracker/entrypoint.sh new file mode 100644 index 0000000..c81a59f --- /dev/null +++ b/lts/lts-tasktracker/entrypoint.sh @@ -0,0 +1,17 @@ +#! /bin/bash + +BASE_PATH=/data/app/ +ORI_APP_FILE=$(find $BASE_PATH -type f -name "*.jar" 2>/dev/null | head -1) +APP_FILE=${BASE_PATH}app.jar + +if [ ${#ORI_APP_FILE} -gt 0 ]; then + echo "create symbolic link for $ORI_APP_FILE" + if [ -e $APP_FILE ]; then + #echo "$APP_FILE was existed, remove that." + rm -f $APP_FILE + fi + ln -s $ORI_APP_FILE $APP_FILE +fi + +# if $APP_FILE not exist, let it throw error。 +java $JAVA_OPTS -jar $APP_FILE "$@"