ci: replace install-ext-services-via-docker.sh with docker-compose (#5200)

2024-12-02 03:58:02 +08:00 · 2021-10-13 17:25:38 +08:00 · 2021-10-13 17:25:38 +08:00 · b5818cced7
commit b5818cced7
parent d1c178daaa
21 changed files with 618 additions and 243 deletions
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@ -24,25 +24,14 @@ jobs:
        os_name:
          - linux_openresty
          - linux_openresty_1_17
-          - linux_tengine
          - linux_openresty_mtls
+          - linux_tengine

    runs-on: ${{ matrix.platform }}
    env:
      SERVER_NAME: ${{ matrix.os_name }}
      OPENRESTY_VERSION: default

-
-    services:
-      etcd:
-        image: bitnami/etcd:3.4.2
-        ports:
-          - 2379:2379
-          - 2380:2380
-        env:
-          ALLOW_NONE_AUTHENTICATION: yes
-          ETCD_ADVERTISE_CLIENT_URLS: http://0.0.0.0:2379
-
    steps:
      - name: Check out code
        uses: actions/checkout@v2.3.4
@ -65,6 +54,10 @@ jobs:
          echo "##[set-output name=version;]$(echo ${GITHUB_REF##*/})"
          echo "##[set-output name=fullname;]$(echo apache-apisix-${GITHUB_REF##*/}-src.tgz)"

+      - name: Linux launch common services
+        run: |
+          project_compose_ci=ci/pod/docker-compose.common.yml make ci-env-up
+
      - name: Create tarball
        if: ${{ startsWith(github.ref, 'refs/heads/release/') }}
        run: |
@ -92,34 +85,6 @@ jobs:
          cd dubbo-backend-provider/target
          java -Djava.net.preferIPv4Stack=true -jar dubbo-demo-provider.one-jar.jar > /tmp/java.log &

-      - name: Install Redis Cluster
-        if: startsWith(matrix.os_name, 'linux_openresty')
-        run: |
-          docker run -d -p ${MASTER1_PORT}:6379 -p ${MASTER2_PORT}:6380 -p ${MASTER3_PORT}:6381 -p ${SLAVE1_PORT}:6382 -p ${SLAVE2_PORT}:6383 -p ${SLAVE3_PORT}:6384 --name redis-cluster vishnunair/docker-redis-cluster:latest
-        env:
-          MASTER1_PORT: 5000
-          MASTER2_PORT: 5001
-          MASTER3_PORT: 5002
-          SLAVE1_PORT: 5003
-          SLAVE2_PORT: 5004
-          SLAVE3_PORT: 5005
-
-      - name: Running Redis Cluster Test
-        if: startsWith(matrix.os_name, 'linux_openresty')
-        run: |
-          sudo apt-get install -y redis-tools
-          docker ps -a
-          redis-cli -h 127.0.0.1 -p 5000 ping
-          redis-cli -h 127.0.0.1 -p 5000 cluster nodes
-
-      - name: Running etcd server with TLS
-        run: |
-          ./ci/start-etcd-tls.sh
-
-      - name: Running etcd server with mTLS
-        run: |
-          ./ci/start-etcd-mtls.sh
-
      - name: Linux Install
        run: |
            sudo --preserve-env=OPENRESTY_VERSION \
--- a/.github/workflows/centos7-ci.yml
+++ b/.github/workflows/centos7-ci.yml
@ -19,16 +19,6 @@ jobs:
    name: run ci on centos7
    runs-on: ubuntu-latest

-    services:
-      etcd:
-        image: bitnami/etcd:3.4.0
-        ports:
-          - 2379:2379
-          - 2380:2380
-        env:
-          ALLOW_NONE_AUTHENTICATION: yes
-          ETCD_ADVERTISE_CLIENT_URLS: http://0.0.0.0:2379
-
    steps:
    - name: Check out code
      uses: actions/checkout@v2.3.4
@ -42,6 +32,10 @@ jobs:
      run: |
        echo "##[set-output name=version;]$(echo ${GITHUB_REF##*/})"

+    - name: Linux launch common services
+      run: |
+        project_compose_ci=ci/pod/docker-compose.common.yml make ci-env-up
+
    - name: Build rpm package
      if: ${{ startsWith(github.ref, 'refs/heads/release/') }}
      run: |
@ -58,28 +52,6 @@ jobs:
        cd ..
        rm -rf $(ls -1 --ignore=apisix-build-tools --ignore=t --ignore=utils --ignore=ci --ignore=Makefile --ignore=rockspec)

-    - name: Install Redis Cluster
-      run: |
-        docker run -d -p ${MASTER1_PORT}:6379 -p ${MASTER2_PORT}:6380 -p ${MASTER3_PORT}:6381 -p ${SLAVE1_PORT}:6382 -p ${SLAVE2_PORT}:6383 -p ${SLAVE3_PORT}:6384 --name redis-cluster vishnunair/docker-redis-cluster:latest
-      env:
-        MASTER1_PORT: 5000
-        MASTER2_PORT: 5001
-        MASTER3_PORT: 5002
-        SLAVE1_PORT: 5003
-        SLAVE2_PORT: 5004
-        SLAVE3_PORT: 5005
-
-    - name: Running Redis Cluster Test
-      run: |
-        sudo apt-get install -y redis-tools
-        docker ps -a
-        redis-cli -h 127.0.0.1 -p 5000 ping
-        redis-cli -h 127.0.0.1 -p 5000 cluster nodes
-
-    - name: Running etcd server with TLS
-      run: |
-        ./ci/start-etcd-tls.sh /home/runner/work/apisix/apisix
-
    - name: Run centos7 docker and mapping apisix into container
      run: |
        docker run -itd -v /home/runner/work/apisix/apisix:/apisix --name centos7Instance --net="host" docker.io/centos:7 /bin/bash
@ -87,7 +59,8 @@ jobs:

    - name: Run other docker containers for test
      run: |
-        ./ci/install-ext-services-via-docker.sh
+        make ci-env-up
+        ./ci/linux-ci-init-service.sh

    - name: Install dependencies
      run: |
--- a/.github/workflows/cli.yml
+++ b/.github/workflows/cli.yml
@ -31,26 +31,6 @@ jobs:
      SERVER_NAME: ${{ matrix.job_name }}
      OPENRESTY_VERSION: default

-
-    services:
-      etcd:
-        image: bitnami/etcd:3.4.2
-        ports:
-          - 2379:2379
-          - 2380:2380
-        env:
-          ALLOW_NONE_AUTHENTICATION: yes
-          ETCD_ADVERTISE_CLIENT_URLS: http://0.0.0.0:2379
-
-      old_etcd:
-        image: bitnami/etcd:3.3.8
-        ports:
-          - 3379:2379
-          - 3380:2380
-        env:
-          ALLOW_NONE_AUTHENTICATION: yes
-          ETCD_ADVERTISE_CLIENT_URLS: http://0.0.0.0:2379
-
    steps:
      - name: Check out code
        uses: actions/checkout@v2.3.4
@ -65,27 +45,16 @@ jobs:
          path: deps
          key: ${{ runner.os }}-${{ env.cache-name }}-${{ matrix.job_name }}-${{ hashFiles('rockspec/apisix-master-0.rockspec') }}

+      - name: Linux launch common services
+        run: |
+          project_compose_ci=ci/pod/docker-compose.common.yml make ci-env-up
+
      - name: Linux Get dependencies
        run: sudo apt install -y cpanminus build-essential libncurses5-dev libreadline-dev libssl-dev perl libpcre3 libpcre3-dev libldap2-dev

      - name: Linux Before install
        run: sudo ./ci/${{ matrix.job_name }}_runner.sh before_install

-      - name: Running etcd server with TLS
-        run: |
-            sudo docker run -d -p 12379:12379 -p 12380:12380 \
-            -e ALLOW_NONE_AUTHENTICATION=yes \
-            -e ETCD_ADVERTISE_CLIENT_URLS=https://0.0.0.0:12379 \
-            -e ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:12379 \
-            -e ETCD_CERT_FILE=/certs/etcd.pem \
-            -e ETCD_KEY_FILE=/certs/etcd.key \
-            -v ${{ github.workspace }}/t/certs:/certs \
-            bitnami/etcd:3.4.0
-
-      - name: Running etcd server with mTLS
-        run: |
-          ./ci/start-etcd-mtls.sh
-
      - name: Linux Install
        run: |
            sudo --preserve-env=OPENRESTY_VERSION \
--- a/.github/workflows/fuzzing-ci.yaml
+++ b/.github/workflows/fuzzing-ci.yaml
@ -21,16 +21,6 @@ jobs:
    name: run fuzzing
    runs-on: ubuntu-latest

-    services:
-      etcd:
-        image: bitnami/etcd:3.4.0
-        ports:
-          - 2379:2379
-          - 2380:2380
-        env:
-          ALLOW_NONE_AUTHENTICATION: yes
-          ETCD_ADVERTISE_CLIENT_URLS: http://0.0.0.0:2379
-
    steps:
    - name: Check out code
      uses: actions/checkout@v2.3.4
@ -45,6 +35,10 @@ jobs:
        path: deps
        key: ${{ runner.os }}-${{ env.cache-name }}-${{ hashFiles('rockspec/apisix-master-0.rockspec') }}

+    - name: Linux launch common services
+      run: |
+        project_compose_ci=ci/pod/docker-compose.common.yml make ci-env-up
+
    - name: run apisix
      run: |
        wget -qO - https://openresty.org/package/pubkey.gpg | sudo apt-key add -
--- a/.licenserc.yaml
+++ b/.licenserc.yaml
@ -29,6 +29,8 @@ header:
    - '**/*.pem'
    - '.github/'
    - 'conf/mime.types'
+    # Exclude CI env_file
+    - 'ci/pod/**/*.env'
    # eyes has some limitation to handle git pattern
    - '**/*.log'
    # Exclude test toolkit files
--- a/ci/install-ext-services-via-docker.sh
+++ b/ci/install-ext-services-via-docker.sh
@ -1,97 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-docker run --rm -itd -p 6379:6379 --name apisix_redis redis:3.0-alpine
-docker run --rm -itd -e HTTP_PORT=8888 -e HTTPS_PORT=9999 -p 8888:8888 -p 9999:9999 mendhak/http-https-echo
-# Runs Keycloak version 10.0.2 with inbuilt policies for unit tests
-docker run --rm -itd -e KEYCLOAK_USER=admin -e KEYCLOAK_PASSWORD=123456 -p 8090:8080 -p 8443:8443 sshniro/keycloak-apisix:1.0.0
-# spin up kafka cluster for tests (2 zookeper and 2 kafka instance)
-docker network create kafka-net --driver bridge
-docker run --name zookeeper-server1 -d -p 2181:2181 --network kafka-net -e ALLOW_ANONYMOUS_LOGIN=yes bitnami/zookeeper:3.6.0
-docker run --name zookeeper-server2 -d -p 12181:2181 --network kafka-net -e ALLOW_ANONYMOUS_LOGIN=yes bitnami/zookeeper:3.6.0
-docker run --name kafka-server1 -d --network kafka-net -e ALLOW_PLAINTEXT_LISTENER=yes -e KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper-server1:2181 -e KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://127.0.0.1:9092 -p 9092:9092 -e KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true bitnami/kafka:latest
-docker run --name kafka-server2 -d --network kafka-net -e ALLOW_PLAINTEXT_LISTENER=yes -e KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper-server2:2181 -e KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://127.0.0.1:9092 -p 19092:9092 -e KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true bitnami/kafka:latest
-docker run --name eureka -d -p 8761:8761 --env ENVIRONMENT=apisix --env spring.application.name=apisix-eureka --env server.port=8761 --env eureka.instance.ip-address=127.0.0.1 --env eureka.client.registerWithEureka=true --env eureka.client.fetchRegistry=false --env eureka.client.serviceUrl.defaultZone=http://127.0.0.1:8761/eureka/ bitinit/eureka
-sleep 5
-docker exec -i kafka-server1 /opt/bitnami/kafka/bin/kafka-topics.sh --create --zookeeper zookeeper-server1:2181 --replication-factor 1 --partitions 1 --topic test2
-docker exec -i kafka-server1 /opt/bitnami/kafka/bin/kafka-topics.sh --create --zookeeper zookeeper-server1:2181 --replication-factor 1 --partitions 3 --topic test3
-docker exec -i kafka-server2 /opt/bitnami/kafka/bin/kafka-topics.sh --create --zookeeper zookeeper-server2:2181 --replication-factor 1 --partitions 1 --topic test4
-
-# start skywalking
-docker run --rm --name skywalking -d -p 1234:1234 -p 11800:11800 -p 12800:12800 apache/skywalking-oap-server:8.7.0-es6
-# start consul servers
-docker run --rm --name consul_1 -d -p 8500:8500 consul:1.7 consul agent -server -bootstrap-expect=1 -client 0.0.0.0 -log-level info -data-dir=/consul/data
-docker run --rm --name consul_2 -d -p 8600:8500 consul:1.7 consul agent -server -bootstrap-expect=1 -client 0.0.0.0 -log-level info -data-dir=/consul/data
-
-# start openldap server
-docker run -d --rm --name openldap -p 1389:1389 -p 1636:1636 --env LDAP_ADMIN_USERNAME=admin --env LDAP_ADMIN_PASSWORD=adminpassword --env LDAP_USERS=user01,user02 --env LDAP_PASSWORDS=password1,password2 bitnami/openldap:latest
-
-# start nacos server
-docker network rm nacos_net
-docker network create nacos_net
-# nacos no auth server - for test no auth
-docker run --rm -d --name nacos_no_auth --network nacos_net --hostname nacos2 --env NACOS_SERVERS="nacos1:8848 nacos2:8848" --env PREFER_HOST_MODE=hostname --env MODE=cluster --env EMBEDDED_STORAGE=embedded  --env JVM_XMS=512m --env JVM_XMX=512m --env JVM_XMN=256m -p8858:8848 nacos/nacos-server:1.4.1
-# nacos auth server - for test auth
-docker run --rm -d --name nacos_auth --network nacos_net --hostname nacos1 --env NACOS_AUTH_ENABLE=true --env NACOS_SERVERS="nacos1:8848 nacos2:8848" --env PREFER_HOST_MODE=hostname --env MODE=cluster --env EMBEDDED_STORAGE=embedded  --env JVM_XMS=512m --env JVM_XMX=512m --env JVM_XMN=256m -p8848:8848 nacos/nacos-server:1.4.1
-url="127.0.0.1:8858/nacos/v1/ns/service/list?pageNo=1&pageSize=2"
-until  [[ "$(curl -s -o /dev/null -w ''%{http_code}'' $url)"  == "200" ]]; do
-    echo 'wait nacos server...'
-    sleep 1;
-done
-# register nacos service
-rm -rf tmp
-mkdir tmp
-cd tmp
-wget https://raw.githubusercontent.com/api7/nacos-test-service/main/spring-nacos-1.0-SNAPSHOT.jar
-curl https://raw.githubusercontent.com/api7/nacos-test-service/main/Dockerfile | docker build -t nacos-test-service:1.0-SNAPSHOT -f - .
-docker run -d --rm --network nacos_net --env SERVICE_NAME=APISIX-NACOS --env NACOS_ADDR=nacos2:8848 --env SUFFIX_NUM=1 -p 18001:18001 --name nacos-service1 nacos-test-service:1.0-SNAPSHOT
-docker run -d --rm --network nacos_net --env SERVICE_NAME=APISIX-NACOS --env NACOS_ADDR=nacos2:8848 --env SUFFIX_NUM=2 -p 18002:18001 --name nacos-service2 nacos-test-service:1.0-SNAPSHOT
-# register nacos service with namespaceId=test_ns
-docker run -d --rm --network nacos_net --env SERVICE_NAME=APISIX-NACOS --env NACOS_ADDR=nacos2:8848 --env NAMESPACE=test_ns --env SUFFIX_NUM=1 -p 18003:18001 --name nacos-service3 nacos-test-service:1.0-SNAPSHOT
-# register nacos service with group=test_group
-docker run -d --rm --network nacos_net --env SERVICE_NAME=APISIX-NACOS --env NACOS_ADDR=nacos2:8848 --env GROUP=test_group --env SUFFIX_NUM=1 -p 18004:18001 --name nacos-service4 nacos-test-service:1.0-SNAPSHOT
-# register nacos service with namespaceId=test_ns and group=test_group
-docker run -d --rm --network nacos_net --env SERVICE_NAME=APISIX-NACOS --env NACOS_ADDR=nacos2:8848 --env NAMESPACE=test_ns --env GROUP=test_group --env SUFFIX_NUM=1 -p 18005:18001 --name nacos-service5 nacos-test-service:1.0-SNAPSHOT
-# register nacos service with namespaceId=test_ns and group=test_group2
-docker run -d --rm --network nacos_net --env SERVICE_NAME=APISIX-NACOS --env NACOS_ADDR=nacos2:8848 --env NAMESPACE=test_ns --env GROUP=test_group2 --env SUFFIX_NUM=3 -p 18006:18001 --name nacos-service6 nacos-test-service:1.0-SNAPSHOT
-# register nacos service with namespaceId=test_ns2 and group=test_group
-docker run -d --rm --network nacos_net --env SERVICE_NAME=APISIX-NACOS --env NACOS_ADDR=nacos2:8848 --env NAMESPACE=test_ns2 --env GROUP=test_group --env SUFFIX_NUM=4 -p 18007:18001 --name nacos-service7 nacos-test-service:1.0-SNAPSHOT
-
-url="127.0.0.1:18005/hello"
-until  [[ "$(curl -s -o /dev/null -w ''%{http_code}'' $url)"  == "200" ]]; do
-    echo 'wait nacos service...'
-    sleep 1;
-done
-until  [[ $(curl -s "127.0.0.1:8858/nacos/v1/ns/service/list?pageNo=1&pageSize=2" | grep "APISIX-NACOS") ]]; do
-    echo 'wait nacos reg...'
-    sleep 1;
-done
-until  [[ $(curl -s "127.0.0.1:8858/nacos/v1/ns/service/list?groupName=test_group&pageNo=1&pageSize=2" | grep "APISIX-NACOS") ]]; do
-    echo 'wait nacos reg...'
-    sleep 1;
-done
-until  [[ $(curl -s "127.0.0.1:8858/nacos/v1/ns/service/list?groupName=DEFAULT_GROUP&namespaceId=test_ns&pageNo=1&pageSize=2" | grep "APISIX-NACOS") ]]; do
-    echo 'wait nacos reg...'
-    sleep 1;
-done
-until  [[ $(curl -s "127.0.0.1:8858/nacos/v1/ns/service/list?groupName=test_group&namespaceId=test_ns&pageNo=1&pageSize=2" | grep "APISIX-NACOS") ]]; do
-    echo 'wait nacos reg...'
-    sleep 1;
-done
-
-cd ..
--- a/ci/linux-ci-init-service.sh
+++ b/ci/linux-ci-init-service.sh
@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+docker exec -i apache-apisix_kafka-server1_1 /opt/bitnami/kafka/bin/kafka-topics.sh --create --zookeeper zookeeper-server1:2181 --replication-factor 1 --partitions 1 --topic test2
+docker exec -i apache-apisix_kafka-server1_1 /opt/bitnami/kafka/bin/kafka-topics.sh --create --zookeeper zookeeper-server1:2181 --replication-factor 1 --partitions 3 --topic test3
+docker exec -i apache-apisix_kafka-server2_1 /opt/bitnami/kafka/bin/kafka-topics.sh --create --zookeeper zookeeper-server2:2181 --replication-factor 1 --partitions 1 --topic test4
--- a/ci/linux_openresty_common_runner.sh
+++ b/ci/linux_openresty_common_runner.sh
@ -21,7 +21,9 @@
 before_install() {
    sudo cpanm --notest Test::Nginx >build.log 2>&1 || (cat build.log && exit 1)

-    ./ci/install-ext-services-via-docker.sh
+    # launch deps env
+    make ci-env-up
+    ./ci/linux-ci-init-service.sh
 }

 do_install() {
--- a/ci/linux_tengine_runner.sh
+++ b/ci/linux_tengine_runner.sh
@ -21,7 +21,9 @@
 before_install() {
    sudo cpanm --notest Test::Nginx >build.log 2>&1 || (cat build.log && exit 1)

-    ./ci/install-ext-services-via-docker.sh
+    # launch deps env
+    make ci-env-up
+    ./ci/linux-ci-init-service.sh
 }

 tengine_install() {
--- a/ci/pod/docker-compose.common.yml
+++ b/ci/pod/docker-compose.common.yml
@ -0,0 +1,89 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: "3.8"
+
+services:
+  ## Etcd
+  etcd_old:
+    image: bitnami/etcd:3.3.8
+    restart: unless-stopped
+    env_file:
+      - ci/pod/etcd/env/common.env
+    environment:
+      ETCD_ADVERTISE_CLIENT_URLS: http://0.0.0.0:2379
+    ports:
+      - "3379:2379"
+      - "3380:2380"
+
+  etcd:
+    image: bitnami/etcd:3.4.0
+    restart: unless-stopped
+    env_file:
+      - ci/pod/etcd/env/common.env
+    environment:
+        ETCD_ADVERTISE_CLIENT_URLS: http://0.0.0.0:2379
+    ports:
+      - "2379:2379"
+      - "2380:2380"
+
+  etcd_tls:
+    image: bitnami/etcd:3.4.0
+    restart: unless-stopped
+    env_file:
+      - ci/pod/etcd/env/common.env
+    environment:
+      ETCD_ADVERTISE_CLIENT_URLS: https://0.0.0.0:12379
+      ETCD_LISTEN_CLIENT_URLS: https://0.0.0.0:12379
+      ETCD_CERT_FILE: /certs/etcd.pem
+      ETCD_KEY_FILE: /certs/etcd.key
+    ports:
+      - "12379:12379"
+      - "12380:12380"
+    volumes:
+      - ./t/certs:/certs
+
+  etcd_mtls:
+    image: bitnami/etcd:3.4.0
+    restart: unless-stopped
+    env_file:
+      - ci/pod/etcd/env/common.env
+    environment:
+      ETCD_ADVERTISE_CLIENT_URLS: https://0.0.0.0:22379
+      ETCD_LISTEN_CLIENT_URLS: https://0.0.0.0:22379
+      ETCD_CERT_FILE: /certs/mtls_server.crt
+      ETCD_KEY_FILE: /certs/mtls_server.key
+      ETCD_CLIENT_CERT_AUTH: "true"
+      ETCD_TRUSTED_CA_FILE: /certs/mtls_ca.crt
+    ports:
+      - "22379:22379"
+      - "22380:22380"
+    volumes:
+      - ./t/certs:/certs
+
+
+  ## Redis cluster
+  redis-cluster:
+    image: vishnunair/docker-redis-cluster:latest
+    restart: unless-stopped
+    ports:
+      - "5000:6379"
+      - "5002:6380"
+      - "5003:6381"
+      - "5004:6382"
+      - "5005:6383"
+      - "5006:6384"
--- a/ci/pod/docker-compose.yml
+++ b/ci/pod/docker-compose.yml
@ -0,0 +1,364 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: "3.8"
+
+services:
+  ## Redis
+  apisix_redis:
+    image: redis:3.0-alpine
+    restart: unless-stopped
+    ports:
+      - "6379:6379"
+    networks:
+      apisix_net:
+
+
+  ## keycloak
+  apisix_keycloak:
+    image: sshniro/keycloak-apisix:1.0.0
+    environment:
+      KEYCLOAK_USER: admin
+      KEYCLOAK_PASSWORD: 123456
+    restart: unless-stopped
+    ports:
+      - "8090:8080"
+      - "8443:8443"
+    networks:
+      apisix_net:
+
+
+  ## kafka-cluster
+  zookeeper-server1:
+    image: bitnami/zookeeper:3.6.0
+    env_file:
+      - ci/pod/kafka/zookeeper-server/env/common.env
+    restart: unless-stopped
+    ports:
+      - "2181:2181"
+    networks:
+      kafka_net:
+
+  zookeeper-server2:
+    image: bitnami/zookeeper:3.6.0
+    env_file:
+      - ci/pod/kafka/zookeeper-server/env/common.env
+    restart: unless-stopped
+    ports:
+      - "12181:12181"
+    networks:
+      kafka_net:
+
+  kafka-server1:
+    image: bitnami/kafka:latest
+    env_file:
+      - ci/pod/kafka/kafka-server/env/common.env
+    environment:
+      KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper-server1:2181
+    restart: unless-stopped
+    ports:
+      - "9092:9092"
+    depends_on:
+      - zookeeper-server1
+      - zookeeper-server2
+    networks:
+      kafka_net:
+
+  kafka-server2:
+    image: bitnami/kafka:latest
+    env_file:
+      - ci/pod/kafka/kafka-server/env/common.env
+    environment:
+      KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper-server2:2181
+    restart: unless-stopped
+    ports:
+      - "19092:9092"
+    depends_on:
+      - zookeeper-server1
+      - zookeeper-server2
+    networks:
+      kafka_net:
+
+
+  ## Eureka
+  eureka:
+    image: bitinit/eureka
+    env_file:
+      - ci/pod/eureka/env/common.env
+    restart: unless-stopped
+    ports:
+      - "8761:8761"
+
+
+  ## SkyWalking
+  skywalking:
+    image: apache/skywalking-oap-server:8.7.0-es6
+    restart: unless-stopped
+    ports:
+      - "1234:1234"
+      - "11800:11800"
+      - "12800:12800"
+    networks:
+      skywalk_net:
+
+
+  ## Consul
+  consul_1:
+    image: consul:1.7
+    restart: unless-stopped
+    ports:
+      - "8500:8500"
+    command: [ "consul", "agent", "-server", "-bootstrap-expect=1", "-client", "0.0.0.0", "-log-level", "info", "-data-dir=/consul/data" ]
+    networks:
+      consul_net:
+
+  consul_2:
+    image: consul:1.7
+    restart: unless-stopped
+    ports:
+      - "8600:8500"
+    command: [ "consul", "agent", "-server", "-bootstrap-expect=1", "-client", "0.0.0.0", "-log-level", "info", "-data-dir=/consul/data" ]
+    networks:
+      consul_net:
+
+
+  ## OpenLDAP
+  openldap:
+    image: bitnami/openldap:latest
+    environment:
+      LDAP_ADMIN_USERNAME: amdin
+      LDAP_ADMIN_PASSWORD: adminpassword
+      LDAP_USERS: user01,user02
+      LDAP_PASSWORDS: password1,password2
+    ports:
+      - "1389:1389"
+      - "1636:1636"
+
+
+  ## Nacos cluster
+  nacos_auth:
+    hostname: nacos1
+    image: nacos/nacos-server:1.4.1
+    env_file:
+      - ci/pod/nacos/env/common.env
+    environment:
+      NACOS_AUTH_ENABLE: "true"
+    restart: unless-stopped
+    ports:
+      - "8848:8848"
+    networks:
+      nacos_net:
+
+  nacos_no_auth:
+    hostname: nacos2
+    image: nacos/nacos-server:1.4.1
+    env_file:
+      - ci/pod/nacos/env/common.env
+    restart: unless-stopped
+    ports:
+      - "8858:8848"
+    networks:
+      nacos_net:
+
+  nacos_server_health_check:
+    build:
+      context: ci/pod/nacos/healthcheck
+      dockerfile: Dockerfile
+    environment:
+      CHECK_URI: "http://nacos2:8848/nacos/v1/ns/service/list?pageNo=1&pageSize=2"
+    tty: true
+    # debug healthcheck script
+#    volumes:
+#     - ./ci/pod/nacos/healthcheck/nacos-server-healthcheck.sh:/nacos-server-healthcheck.sh
+    healthcheck:
+      test: [ "CMD", "bash", "/nacos-server-healthcheck.sh" ]
+      interval: 5s
+      timeout: 5s
+      retries: 60
+      start_period: 10s
+    networks:
+      nacos_net:
+
+  nacos_service_health_check:
+    build:
+      context: ci/pod/nacos/healthcheck
+      dockerfile: Dockerfile
+    # debug healthcheck script
+#    volumes:
+#     - ./ci/pod/nacos/healthcheck/nacos-service-healthcheck.sh:/nacos-service-healthcheck.sh
+    tty: true
+    healthcheck:
+      test: [ "CMD", "bash", "/nacos-service-healthcheck.sh" ]
+      interval: 5s
+      timeout: 30s
+      retries: 60
+      start_period: 10s
+    networks:
+      nacos_net:
+
+  ### Nacos services
+  nacos-service1:
+    build:
+      context: ci/pod/nacos/service
+      dockerfile: Dockerfile
+    env_file:
+      - ci/pod/nacos/env/service.env
+    environment:
+      SUFFIX_NUM: 1
+    restart: unless-stopped
+    ports:
+      - "18001:18001"
+    depends_on:
+      nacos_server_health_check:
+        condition: service_healthy
+    networks:
+      nacos_net:
+
+  nacos-service2:
+    build:
+      context: ci/pod/nacos/service
+      dockerfile: Dockerfile
+    env_file:
+      - ci/pod/nacos/env/service.env
+    environment:
+      SUFFIX_NUM: 2
+    restart: unless-stopped
+    ports:
+      - "18002:18001"
+    depends_on:
+      nacos_server_health_check:
+        condition: service_healthy
+    networks:
+      nacos_net:
+
+  nacos-service3:
+    build:
+      context: ci/pod/nacos/service
+      dockerfile: Dockerfile
+    env_file:
+      - ci/pod/nacos/env/service.env
+    environment:
+      SUFFIX_NUM: 1
+      NAMESPACE: test_ns
+    restart: unless-stopped
+    ports:
+      - "18003:18001"
+    depends_on:
+      nacos_server_health_check:
+        condition: service_healthy
+    networks:
+      nacos_net:
+
+  nacos-service4:
+    build:
+      context: ci/pod/nacos/service
+      dockerfile: Dockerfile
+    env_file:
+      - ci/pod/nacos/env/service.env
+    environment:
+      SUFFIX_NUM: 1
+      GROUP: test_group
+    restart: unless-stopped
+    ports:
+      - "18004:18001"
+    depends_on:
+      nacos_server_health_check:
+        condition: service_healthy
+    networks:
+      nacos_net:
+
+  nacos-service5:
+    build:
+      context: ci/pod/nacos/service
+      dockerfile: Dockerfile
+    env_file:
+      - ci/pod/nacos/env/service.env
+    environment:
+      SUFFIX_NUM: 1
+      GROUP: test_group
+      NAMESPACE: test_ns
+    restart: unless-stopped
+    ports:
+      - "18005:18001"
+    depends_on:
+      nacos_server_health_check:
+        condition: service_healthy
+    networks:
+      nacos_net:
+
+  nacos-service6:
+    build:
+      context: ci/pod/nacos/service
+      dockerfile: Dockerfile
+    env_file:
+      - ci/pod/nacos/env/service.env
+    environment:
+      SUFFIX_NUM: 3
+      GROUP: test_group2
+      NAMESPACE: test_ns
+    restart: unless-stopped
+    ports:
+      - "18006:18001"
+    depends_on:
+      nacos_server_health_check:
+        condition: service_healthy
+    networks:
+      nacos_net:
+
+  nacos-service7:
+    build:
+      context: ci/pod/nacos/service
+      dockerfile: Dockerfile
+    env_file:
+      - ci/pod/nacos/env/service.env
+    environment:
+      SUFFIX_NUM: 4
+      GROUP: test_group
+      NAMESPACE: test_ns2
+    restart: unless-stopped
+    ports:
+      - "18007:18001"
+    depends_on:
+      nacos_server_health_check:
+        condition: service_healthy
+    networks:
+      nacos_net:
+
+
+  ## http_echo
+  apisix_http_echo:
+    image: mendhak/http-https-echo
+    environment:
+      HTTP_PORT: 8888
+      HTTPS_PORT: 9999
+    restart: unless-stopped
+    ports:
+      - "8888:8888"
+      - "9999:9999"
+    depends_on:
+      nacos_service_health_check:
+        condition: service_healthy
+    networks:
+      apisix_net:
+
+
+networks:
+  apisix_net:
+  consul_net:
+  kafka_net:
+  nacos_net:
+  skywalk_net:
--- a/ci/pod/etcd/env/common.env
+++ b/ci/pod/etcd/env/common.env
@ -0,0 +1 @@
+ALLOW_NONE_AUTHENTICATION=yes
--- a/ci/pod/eureka/env/common.env
+++ b/ci/pod/eureka/env/common.env
@ -0,0 +1,7 @@
+ENVIRONMENT=apisix
+spring.application.name=apisix-eureka
+server.port=8761
+eureka.instance.ip-address=127.0.0.1
+eureka.client.registerWithEureka=true
+eureka.client.fetchRegistry=false
+eureka.client.serviceUrl.defaultZone=http://127.0.0.1:8761/eureka/
--- a/ci/pod/kafka/kafka-server/env/common.env
+++ b/ci/pod/kafka/kafka-server/env/common.env
@ -0,0 +1,3 @@
+ALLOW_PLAINTEXT_LISTENER=yes
+KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true
+KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://127.0.0.1:9092
--- a/ci/pod/kafka/zookeeper-server/env/common.env
+++ b/ci/pod/kafka/zookeeper-server/env/common.env
@ -0,0 +1 @@
+ALLOW_ANONYMOUS_LOGIN=yes
--- a/ci/pod/nacos/env/common.env
+++ b/ci/pod/nacos/env/common.env
@ -0,0 +1,6 @@
+EMBEDDED_STORAGE=embedded
+PREFER_HOST_MODE=hostname
+MODE=cluster
+NACOS_SERVERS="nacos1:8848 nacos2:8848"
+JVM_XMS=512m
+JVM_XMX=512m
--- a/ci/pod/nacos/env/service.env
+++ b/ci/pod/nacos/env/service.env
@ -0,0 +1,2 @@
+SERVICE_NAME=APISIX-NACOS
+NACOS_ADDR=nacos2:8848
--- a/ci/pod/nacos/healthcheck/Dockerfile
+++ b/ci/pod/nacos/healthcheck/Dockerfile
@ -1,4 +1,3 @@
-#!/usr/bin/env bash
 #
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
@ -16,14 +15,16 @@
 # limitations under the License.
 #

-docker run -d --rm --name etcd_mtls \
-    -p 22379:22379 -p 22380:22380 \
-    -e ALLOW_NONE_AUTHENTICATION=yes \
-    -e ETCD_ADVERTISE_CLIENT_URLS=https://0.0.0.0:22379 \
-    -e ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:22379 \
-    -e ETCD_CERT_FILE=/certs/mtls_server.crt \
-    -e ETCD_KEY_FILE=/certs/mtls_server.key \
-    -e ETCD_CLIENT_CERT_AUTH=true \
-    -e ETCD_TRUSTED_CA_FILE=/certs/mtls_ca.crt \
-    -v "$PWD"/t/certs:/certs \
-    bitnami/etcd:3.4.0
+FROM alpine:latest
+
+# change workdir to /
+WORKDIR /
+
+# install curl
+RUN apk --no-cache add bash curl
+
+# add healthcheck script
+COPY *.sh /
+
+# add hosted process
+CMD ["cat"]
--- a/ci/pod/nacos/healthcheck/nacos-server-healthcheck.sh
+++ b/ci/pod/nacos/healthcheck/nacos-server-healthcheck.sh
@ -15,18 +15,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-if [ $# -gt 0 ]; then
-    APISIX_DIR="$1"
-else
-    APISIX_DIR="$PWD"
-fi

-docker run -d --rm --name etcd_tls \
-    -p 12379:12379 -p 12380:12380 \
-    -e ALLOW_NONE_AUTHENTICATION=yes \
-    -e ETCD_ADVERTISE_CLIENT_URLS=https://0.0.0.0:12379 \
-    -e ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:12379 \
-    -e ETCD_CERT_FILE=/certs/etcd.pem \
-    -e ETCD_KEY_FILE=/certs/etcd.key \
-    -v "$APISIX_DIR"/t/certs:/certs \
-    bitnami/etcd:3.4.0
+
+set -ex
+
+# nacos server healthcheck
+REQ_STATUS=$(curl -s -o /dev/null -w '%{http_code}' "${CHECK_URI}")
+
+if [ "${REQ_STATUS}" -ne "200" ]; then
+  exit 1;
+fi
--- a/ci/pod/nacos/healthcheck/nacos-service-healthcheck.sh
+++ b/ci/pod/nacos/healthcheck/nacos-service-healthcheck.sh
@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -ex
+
+# nacos service healthcheck
+URI_LIST=(
+  "http://nacos2:8848/nacos/v1/ns/service/list?pageNo=1&pageSize=2"
+  "http://nacos2:8848/nacos/v1/ns/service/list?groupName=test_group&pageNo=1&pageSize=2"
+  "http://nacos2:8848/nacos/v1/ns/service/list?groupName=DEFAULT_GROUP&namespaceId=test_ns&pageNo=1&pageSize=2"
+  "http://nacos2:8848/nacos/v1/ns/service/list?groupName=test_group&namespaceId=test_ns&pageNo=1&pageSize=2"
+)
+
+for URI in "${URI_LIST[@]}"; do
+  if [[ $(curl -s "${URI}" | grep "APISIX-NACOS") ]]; then
+    continue
+  else
+    exit 1;
+  fi
+done
+
+
+for IDX in {1..7..1}; do
+  REQ_STATUS=$(curl -s -o /dev/null -w '%{http_code}' "http://nacos-service${IDX}:18001/hello")
+  if [ "${REQ_STATUS}" -ne "200" ]; then
+    exit 1;
+  fi
+done
--- a/ci/pod/nacos/service/Dockerfile
+++ b/ci/pod/nacos/service/Dockerfile
@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM java
+
+ENV SUFFIX_NUM=${SUFFIX_NUM:-1}
+ENV NACOS_ADDR=${NACOS_ADDR:-127.0.0.1:8848}
+ENV SERVICE_NAME=${SERVICE_NAME:-gateway-service}
+ENV NAMESPACE=${NAMESPACE}
+ENV GROUP=${GROUP:-DEFAULT_GROUP}
+
+ADD https://raw.githubusercontent.com/api7/nacos-test-service/main/spring-nacos-1.0-SNAPSHOT.jar /app.jar
+
+ENTRYPOINT ["java","-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar",\
+            "--suffix.num=${SUFFIX_NUM}","--spring.cloud.nacos.discovery.server-addr=${NACOS_ADDR}",\
+            "--spring.application.name=${SERVICE_NAME}","--spring.cloud.nacos.discovery.group=${GROUP}",\
+            "--spring.cloud.nacos.discovery.namespace=${NAMESPACE}"]
+EXPOSE 18001