fix zookeeper health check and merge volumes into logs volume (#2674)

* fix zookeeper health check and merge volumes into logs volume 1. Fix zookeeper health check 2. Merge volumes into logs volume 3. Modify `dockerfile/hooks/build` script * Modify ci_e2e and docker image registry * Modify ci_e2e * Modify ci_e2e * Modify ci_e2e and move ./dockerfile/hooks/check to ./docker/docker-swarm/
2024-11-30 03:08:01 +08:00 · 2020-05-11 23:22:04 +08:00 · 2020-05-11 23:22:04 +08:00 · 3c014c5a57
commit 3c014c5a57
parent 83ede1037e
7 changed files with 48 additions and 53 deletions
--- a/.github/workflows/ci_e2e.yml
+++ b/.github/workflows/ci_e2e.yml
@ -44,15 +44,14 @@ jobs:
            ${{ runner.os }}-maven-
      - name: Build Image
        run: |
-          export VERSION=`cat $(pwd)/pom.xml| grep "SNAPSHOT</version>" | awk -F "-SNAPSHOT" '{print $1}' | awk -F ">" '{print $2}'`
          sh ./dockerfile/hooks/build
      - name: Docker Run
        run: |
-          VERSION=`cat $(pwd)/pom.xml| grep "SNAPSHOT</version>" | awk -F "-SNAPSHOT" '{print $1}' | awk -F ">" '{print $2}'`
-          mkdir -p /tmp/logs
-          docker run -dit -e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -v /tmp/logs:/opt/dolphinscheduler/logs -p 8888:8888 dolphinscheduler:$VERSION all
+          export VERSION=$(cat $(pwd)/pom.xml | grep '<version>' -m 1 | awk '{print $1}' | sed 's/<version>//' | sed 's/<\/version>//')
+          sed -i "s/apache\/dolphinscheduler:latest/apache\/dolphinscheduler:${VERSION}/g" $(pwd)/docker/docker-swarm/docker-compose.yml
+          docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml up -d
      - name: Check Server Status
-        run: sh ./dockerfile/hooks/check
+        run: sh $(pwd)/docker/docker-swarm/check
      - name: Prepare e2e env
        run: |
          sudo apt-get install -y libxss1 libappindicator1 libindicator7 xvfb unzip libgbm1
@ -70,6 +69,6 @@ jobs:
        uses: actions/upload-artifact@v1
        with:
          name: dslogs
-          path: /tmp/logs
+          path: /var/lib/docker/volumes/docker-swarm_dolphinscheduler-logs/_data


--- a/docker/docker-swarm/check
+++ b/docker/docker-swarm/check
@ -17,7 +17,7 @@
 #
 echo "------ dolphinscheduler check - server - status -------"
 sleep 60
-server_num=$(docker top `docker container list | grep '/sbin/tini' | awk '{print $1}'`| grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l)
+server_num=$(docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml top | grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l)
 if [ $server_num -eq 5 ]
 then
    echo "Server all start successfully"
--- a/docker/docker-swarm/docker-compose.yml
+++ b/docker/docker-swarm/docker-compose.yml
@ -41,13 +41,14 @@ services:
    environment:
      TZ: Asia/Shanghai
      ALLOW_ANONYMOUS_LOGIN: "yes"
+      ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
    volumes:
    - dolphinscheduler-zookeeper:/bitnami/zookeeper
    networks:
    - dolphinscheduler

  dolphinscheduler-api:
-    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+    image: apache/dolphinscheduler:latest
    container_name: dolphinscheduler-api
    command: ["api-server"]
    ports:
@ -70,12 +71,12 @@ services:
    - dolphinscheduler-postgresql
    - dolphinscheduler-zookeeper
    volumes:
-    - dolphinscheduler-api:/opt/dolphinscheduler/logs
-    networks: 
+    - dolphinscheduler-logs:/opt/dolphinscheduler/logs
+    networks:
    - dolphinscheduler

  dolphinscheduler-frontend:
-    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+    image: apache/dolphinscheduler:latest
    container_name: dolphinscheduler-frontend
    command: ["frontend"]
    ports:
@ -93,12 +94,12 @@ services:
    depends_on:
    - dolphinscheduler-api
    volumes:
-    - dolphinscheduler-frontend:/var/log/nginx
+    - dolphinscheduler-logs:/var/log/nginx
    networks:
    - dolphinscheduler

  dolphinscheduler-alert:
-    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+    image: apache/dolphinscheduler:latest
    container_name: dolphinscheduler-alert
    command: ["alert-server"]
    environment:
@ -130,18 +131,18 @@ services:
      start_period: 30s
    depends_on:
    - dolphinscheduler-postgresql
-    volumes: 
-    - dolphinscheduler-alert:/opt/dolphinscheduler/logs
-    networks: 
+    volumes:
+    - dolphinscheduler-logs:/opt/dolphinscheduler/logs
+    networks:
    - dolphinscheduler

  dolphinscheduler-master:
-    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+    image: apache/dolphinscheduler:latest
    container_name: dolphinscheduler-master
    command: ["master-server"]
-    ports: 
+    ports:
    - 5678:5678
-    environment: 
+    environment:
      TZ: Asia/Shanghai
      MASTER_EXEC_THREADS: "100"
      MASTER_EXEC_TASK_NUM: "20"
@ -162,16 +163,16 @@ services:
      timeout: 5s
      retries: 3
      start_period: 30s
-    depends_on: 
+    depends_on:
    - dolphinscheduler-postgresql
    - dolphinscheduler-zookeeper
-    volumes: 
-    - dolphinscheduler-master:/opt/dolphinscheduler/logs
+    volumes:
+    - dolphinscheduler-logs:/opt/dolphinscheduler/logs
    networks:
    - dolphinscheduler

  dolphinscheduler-worker:
-    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+    image: apache/dolphinscheduler:latest
    container_name: dolphinscheduler-worker
    command: ["worker-server"]
    ports: 
@ -209,7 +210,7 @@ services:
      source: dolphinscheduler-worker-data
      target: /tmp/dolphinscheduler
    - type: volume
-      source: dolphinscheduler-worker-logs
+      source: dolphinscheduler-logs
      target: /opt/dolphinscheduler/logs
    networks:
    - dolphinscheduler
@ -221,12 +222,8 @@ networks:
 volumes:
  dolphinscheduler-postgresql:
  dolphinscheduler-zookeeper:
-  dolphinscheduler-api:
-  dolphinscheduler-frontend:
-  dolphinscheduler-alert:
-  dolphinscheduler-master:
  dolphinscheduler-worker-data:
-  dolphinscheduler-worker-logs:
+  dolphinscheduler-logs:

 configs:
  dolphinscheduler-worker-task-env:
--- a/docker/docker-swarm/docker-stack.yml
+++ b/docker/docker-swarm/docker-stack.yml
@ -42,6 +42,7 @@ services:
    environment:
      TZ: Asia/Shanghai
      ALLOW_ANONYMOUS_LOGIN: "yes"
+      ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
    volumes:
    - dolphinscheduler-zookeeper:/bitnami/zookeeper
    networks:
@ -51,7 +52,7 @@ services:
      replicas: 1

  dolphinscheduler-api:
-    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+    image: apache/dolphinscheduler:latest
    command: ["api-server"]
    ports:
    - 12345:12345
@ -70,7 +71,7 @@ services:
      retries: 3
      start_period: 30s
    volumes:
-    - dolphinscheduler-api:/opt/dolphinscheduler/logs
+    - dolphinscheduler-logs:/opt/dolphinscheduler/logs
    networks: 
    - dolphinscheduler
    deploy:
@ -78,7 +79,7 @@ services:
      replicas: 1

  dolphinscheduler-frontend:
-    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+    image: apache/dolphinscheduler:latest
    command: ["frontend"]
    ports:
    - 8888:8888
@ -93,7 +94,7 @@ services:
      retries: 3
      start_period: 30s
    volumes:
-    - dolphinscheduler-frontend:/var/log/nginx
+    - dolphinscheduler-logs:/var/log/nginx
    networks:
    - dolphinscheduler
    deploy:
@ -101,7 +102,7 @@ services:
      replicas: 1

  dolphinscheduler-alert:
-    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+    image: apache/dolphinscheduler:latest
    command: ["alert-server"]
    environment:
      TZ: Asia/Shanghai
@ -131,7 +132,7 @@ services:
      retries: 3
      start_period: 30s
    volumes:
-    - dolphinscheduler-alert:/opt/dolphinscheduler/logs
+    - dolphinscheduler-logs:/opt/dolphinscheduler/logs
    networks: 
    - dolphinscheduler
    deploy:
@ -139,7 +140,7 @@ services:
      replicas: 1

  dolphinscheduler-master:
-    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+    image: apache/dolphinscheduler:latest
    command: ["master-server"]
    ports: 
    - 5678:5678
@ -165,7 +166,7 @@ services:
      retries: 3
      start_period: 30s
    volumes:
-    - dolphinscheduler-master:/opt/dolphinscheduler/logs
+    - dolphinscheduler-logs:/opt/dolphinscheduler/logs
    networks:
    - dolphinscheduler
    deploy:
@ -173,7 +174,7 @@ services:
      replicas: 1

  dolphinscheduler-worker:
-    image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+    image: apache/dolphinscheduler:latest
    command: ["worker-server"]
    ports: 
    - 1234:1234
@ -201,7 +202,7 @@ services:
      start_period: 30s
    volumes:
    - dolphinscheduler-worker-data:/tmp/dolphinscheduler
-    - dolphinscheduler-worker-logs:/opt/dolphinscheduler/logs
+    - dolphinscheduler-logs:/opt/dolphinscheduler/logs
    configs:
    - source: dolphinscheduler-worker-task-env
      target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh
@ -218,12 +219,8 @@ networks:
 volumes:
  dolphinscheduler-postgresql:
  dolphinscheduler-zookeeper:
-  dolphinscheduler-api:
-  dolphinscheduler-frontend:
-  dolphinscheduler-alert:
-  dolphinscheduler-master:
  dolphinscheduler-worker-data:
-  dolphinscheduler-worker-logs:
+  dolphinscheduler-logs:

 configs:
  dolphinscheduler-worker-task-env:
--- a/docker/kubernetes/dolphinscheduler/values.yaml
+++ b/docker/kubernetes/dolphinscheduler/values.yaml
@ -25,9 +25,9 @@ fullnameOverride: ""
 timezone: "Asia/Shanghai"

 image:
-  registry: "docker.io"
+  registry: "apache"
  repository: "dolphinscheduler"
-  tag: "1.3.0"
+  tag: "latest"
  pullPolicy: "IfNotPresent"

 imagePullSecrets: []
@ -56,6 +56,8 @@ externalDatabase:
 zookeeper:
  enabled: true
  taskQueue: "zookeeper"
+  config:
+    ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
  service:
    port: "2181"
  persistence:
--- a/dockerfile/Dockerfile
+++ b/dockerfile/Dockerfile
@ -53,8 +53,8 @@ ENV PATH $ZK_HOME/bin:$PATH
 RUN apk add postgresql postgresql-contrib

 #5. add dolphinscheduler
-ADD ./apache-dolphinscheduler-incubating-${VERSION}-SNAPSHOT-dolphinscheduler-bin.tar.gz /opt/
-RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-SNAPSHOT-dolphinscheduler-bin/ /opt/dolphinscheduler/
+ADD ./apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz /opt/
+RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin/ /opt/dolphinscheduler/
 ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler

 #6. modify nginx
--- a/dockerfile/hooks/build
+++ b/dockerfile/hooks/build
@ -24,13 +24,13 @@ printenv
 if [ -z "${VERSION}" ]
 then
    echo "set default environment variable [VERSION]"
-    VERSION=$(cat $(pwd)/sql/soft_version)
+    export VERSION=$(cat $(pwd)/pom.xml | grep '<version>' -m 1 | awk '{print $1}' | sed 's/<version>//' | sed 's/<\/version>//')
 fi

 if [ "${DOCKER_REPO}x" = "x" ]
 then
    echo "set default environment variable [DOCKER_REPO]"
-    DOCKER_REPO='dolphinscheduler'
+    export DOCKER_REPO='apache/dolphinscheduler'
 fi

 echo "Version: $VERSION"
@ -43,11 +43,11 @@ echo -e "mvn -B clean compile package -Prelease -Dmaven.test.skip=true"
 mvn -B clean compile package -Prelease -Dmaven.test.skip=true

 # mv dolphinscheduler-bin.tar.gz file to dockerfile directory
-echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-SNAPSHOT-dolphinscheduler-bin.tar.gz $(pwd)/dockerfile/\n"
-mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-SNAPSHOT-dolphinscheduler-bin.tar.gz $(pwd)/dockerfile/
+echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz $(pwd)/dockerfile/\n"
+mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz $(pwd)/dockerfile/

 # docker build
 echo -e "docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/dockerfile/\n"
-docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/dockerfile/
+sudo docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/dockerfile/

 echo "------ dolphinscheduler end   - build -------"