milvus/.github/workflows/deploy-test.yaml

name: Deploy Test

on:
  workflow_dispatch:
    inputs:
      old_image_repo:
        description: The image repository name to use for the deploy test
        required: true
        default: 'milvusdb/milvus'

      old_image_tag:
        description: The old image tag to use for the deploy test
        required: true
        default: 'v2.1.0'

      previous_release_version:
        description: The previous release version to use for the deploy test
        required: true
        default: 'v2.1.0'

      new_image_repo:
        description: The image repository name to use for the deploy test
        required: true
        default: 'milvusdb/milvus'

      new_image_tag:
        description: The new image tag to use for the deploy test
        required: true
        default: 'master-latest'

  schedule:
    # * is a special character in YAML so you have to quote this string
    #         ┌───────────── minute (0 - 59)
    #         │  ┌───────────── hour (0 - 23)
    #         │  │ ┌───────────── day of the month (1 - 31)
    #         │  │ │ ┌───────────── month (1 - 12 or JAN-DEC)
    #         │  │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
    #         │  │ │ │ │
    #         │  │ │ │ │
    #         │  │ │ │ │
    - cron: "30 20 * * *"

jobs:

  test-docker-compose:

    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        mode: [standalone, cluster]
        task: [reinstall, upgrade]

    steps:
      - name: Set env param
        env:
          DEFAULT_OLD_IMAGE_REPO: "milvusdb/milvus"
          DEFAULT_OLD_IMAGE_TAG: "latest"
          DEFAULT_PREVIOUS_RELEASE_VERSION: "v2.1.0"
          DEFAULT_NEW_IMAGE_REPO: "milvusdb/milvus"
          DEFAULT_NEW_IMAGE_TAG: "master-latest"
        run: |
          echo "OLD_IMAGE_REPO=${{ github.event.inputs.old_image_repo || env.DEFAULT_OLD_IMAGE_REPO }}" >> $GITHUB_ENV
          echo "OLD_IMAGE_TAG=${{ github.event.inputs.old_image_tag || env.DEFAULT_OLD_IMAGE_TAG }}" >> $GITHUB_ENV
          echo "PREVIOUS_RELEASE_VERSION=${{ github.event.inputs.previous_release_version || env.DEFAULT_PREVIOUS_RELEASE_VERSION }}" >> $GITHUB_ENV
          echo "NEW_IMAGE_REPO=${{ github.event.inputs.new_image_repo || env.DEFAULT_NEW_IMAGE_REPO }}" >> $GITHUB_ENV
          echo "NEW_IMAGE_TAG=${{ github.event.inputs.new_image_tag || env.DEFAULT_NEW_IMAGE_TAG }}" >> $GITHUB_ENV          

      - uses: actions/checkout@v2

      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8

      - name: Install dependency
        shell: bash
        working-directory: tests/python_client
        run: |
          pip install -r requirements.txt --trusted-host https://test.pypi.org
          sudo systemctl restart docker
          sleep 30s          
      - name: Download dataset
        shell: bash
        working-directory: tests/python_client/assets/ann_hdf5
        run: |
          echo "Downloading dataset..."
          bash download.sh          

      - name: First Milvus deployment
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/deploy/${{ matrix.mode }}
        run: |
          source ../utils.sh
          if [ ${{ matrix.task }} == "reinstall" ]; then
            wget https://raw.githubusercontent.com/milvus-io/milvus/master/deployments/docker/${{ matrix.mode }}/docker-compose.yml -O docker-compose.yml;
            replace_image_tag ${{ env.NEW_IMAGE_REPO }} ${{ env.NEW_IMAGE_TAG }};

          fi
          if [ ${{ matrix.task }} == "upgrade" ]; then
            wget https://github.com/milvus-io/milvus/releases/download/${{ env.PREVIOUS_RELEASE_VERSION }}/milvus-${{ matrix.mode }}-docker-compose.yml -O docker-compose.yml;
            replace_image_tag ${{ env.OLD_IMAGE_REPO }} ${{ env.OLD_IMAGE_TAG }};
          fi
          docker-compose up -d
          bash ../check_healthy.sh
          docker-compose ps -a
          sleep 10s          
      - name: Run first test
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/deploy
        run: |
          python scripts/first_recall_test.py
          if [ ${{ matrix.task }} == "reinstall" ]; then
            python3 scripts/action_before_reinstall.py
          fi
          if [ ${{ matrix.task }} == "upgrade" ]; then
            python3 scripts/action_before_upgrade.py
          fi
                    
      - name: Milvus Idle Time
        timeout-minutes: 5
        shell: bash
        working-directory: tests/python_client/deploy
        run: |
          sleep 60s          
      - name: Export logs
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client/deploy/${{ matrix.mode }}
        run: |
          docker-compose ps -a || true
          mkdir -p logs/first_deploy
          bash ../../../scripts/export_log_docker.sh ./logs/first_deploy || echo "export logs failed"          
      - name: Second Milvus deployment
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/deploy/${{ matrix.mode }}
        run: |
          source ../utils.sh
          if [ ${{ matrix.task }} == "reinstall" ]; then
            docker-compose restart
          fi
          if [ ${{ matrix.task }} == "upgrade" ]; then
            wget https://raw.githubusercontent.com/milvus-io/milvus/master/deployments/docker/${{ matrix.mode }}/docker-compose.yml -O docker-compose.yml;
            replace_image_tag ${{ env.NEW_IMAGE_REPO }} ${{ env.NEW_IMAGE_TAG }};
            docker-compose up -d;
          fi
          bash ../check_healthy.sh
          docker-compose ps -a

          echo "sleep 120s for the second deployment to be ready"
          sleep 120s          
      - name: Run second test
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/deploy
        run: |
          python scripts/second_recall_test.py
          if [ ${{ matrix.task }} == "reinstall" ]; then
            python3 scripts/action_after_reinstall.py
          fi
          if [ ${{ matrix.task }} == "upgrade" ]; then
            python3 scripts/action_after_upgrade.py
          fi          
      - name: Export logs
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client/deploy/${{ matrix.mode }}
        run: |
          docker-compose ps -a || true
          mkdir -p logs/second_deploy
          bash ../../../scripts/export_log_docker.sh ./logs/second_deploy || echo "export logs failed"          

      - name: Restart docker
        timeout-minutes: 5
        shell: bash
        working-directory: tests/python_client/deploy/${{ matrix.mode }}
        run: |
          echo "restart docker service"
          sudo systemctl restart docker
          sleep 20s
          docker-compose up -d
          bash ../check_healthy.sh
          docker-compose ps -a

          echo "sleep 120s for the deployment to be ready after docker restart"
          sleep 120s          


      - name: Run third test
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/deploy
        run: |
          python scripts/second_recall_test.py
          if [ ${{ matrix.task }} == "reinstall" ]; then
            python3 scripts/action_after_reinstall.py
          fi
          if [ ${{ matrix.task }} == "upgrade" ]; then
            python3 scripts/action_after_upgrade.py
          fi          
      - name: Export logs
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client/deploy/${{ matrix.mode }}
        run: |
          docker-compose ps -a || true
          mkdir -p logs/second_deploy
          bash ../../../scripts/export_log_docker.sh ./logs/third_deploy || echo "export logs failed"                
      
      - name: 'Send mail'
        if: ${{ failure() }}
        uses: dawidd6/action-send-mail@v3
        with:
          server_address: ${{ secrets.EMAIL_SERVICE_NAME }}
          server_port: 465
          username: ${{ secrets.TEST_EMAIL_USERNAME }}
          password: ${{ secrets.TEST_EMAIL_PASSWORD  }}
          subject: Deploy Test
          body: "test ${{ matrix.mode }} ${{ matrix.task }} failed \n You can view it at https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}"
          to: ${{ secrets.QA_EMAIL_ADDRESS }}
          from: GitHub Actions

      - name: Upload logs
        if: ${{ ! success() }}
        uses: actions/upload-artifact@v2
        with:
          name: docker-compose-logs-${{ matrix.mode }}-${{ matrix.task }}
          path: tests/python_client/deploy/${{ matrix.mode }}/logs

  test-helm-install:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        mq_type: [pulsar, kafka]
        mode: [standalone,cluster]
        task: [reinstall,upgrade]
        exclude:
          - mq_type: kafka
            task: upgrade

    steps:

      - name: Creating kind cluster
        uses: helm/kind-action@v1.2.0

      - name: Print cluster information
        run: |
          kubectl config view
          kubectl cluster-info
          kubectl get nodes
          kubectl get pods -n kube-system
          helm version
          kubectl version          

      - uses: actions/checkout@v2

      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8

      - name: Install dependency
        shell: bash
        working-directory: tests/python_client
        run: |
          pip install -r requirements.txt --trusted-host https://test.pypi.org          

      - name: Modify chart value config
        timeout-minutes: 1
        shell: bash
        working-directory: tests/python_client/deploy
        run: |
          yq -i ".kafka.enabled = false" cluster-values.yaml
          yq -i ".pulsar.enabled = false" cluster-values.yaml
          yq -i ".kafka.enabled = false" cluster-values-second.yaml
          yq -i ".pulsar.enabled = false" cluster-values-second.yaml
          yq -i ".${{ matrix.mq_type }}.enabled = true" cluster-values.yaml
          yq -i ".${{ matrix.mq_type }}.enabled = true" cluster-values-second.yaml
          if [ ${{ matrix.mq_type }} == "kafka" ]; then
              yq -i ".kafka.enabled = true" standalone-values.yaml;
          fi          

      - name: First Milvus Deployment
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/deploy
        run: |
          helm repo add milvus https://zilliztech.github.io/milvus-helm
          helm repo update

          # if the task is reinstall, install milvus with latest image in repo milvusdb/milvus
          # for cluster mode
          if [ ${{ matrix.task }} == "reinstall" ] && [ ${{ matrix.mode }} == "cluster" ]; then
              echo "task: ${{ matrix.task }} mode: ${{ matrix.mode }}";
              helm install --wait --timeout 720s deploy-testing milvus/milvus -f cluster-values.yaml;
          fi
          # for standalone mode
          if [ ${{ matrix.task }} == "reinstall" ] && [ ${{ matrix.mode }} == "standalone" ]; then
              echo "task: ${{ matrix.task }} mode: ${{ matrix.mode }}";
              helm install --wait --timeout 720s deploy-testing milvus/milvus -f standalone-values.yaml;
          fi

          # if the task is upgrade, install milvus with latest rc image in repo milvusdb/milvus
          if [ ${{ matrix.task }} == "upgrade" ] && [ ${{ matrix.mode }} == "cluster" ]; then
              echo "task: ${{ matrix.task }} mode: ${{ matrix.mode }}";
              helm install --wait --timeout 720s deploy-testing milvus/milvus --set image.all.repository=milvusdb/milvus --set image.all.tag=latest --set etcd.image.repository=milvusdb/etcd --set etcd.image.tag=3.5.0-r6 -f cluster-values.yaml;
          fi
          if [ ${{ matrix.task }} == "upgrade" ] && [ ${{ matrix.mode }} == "standalone" ]; then
              echo "task: ${{ matrix.task }} mode: ${{ matrix.mode }}";
              helm install --wait --timeout 720s deploy-testing milvus/milvus --set image.all.repository=milvusdb/milvus --set image.all.tag=latest --set etcd.image.repository=milvusdb/etcd --set etcd.image.tag=3.5.0-r6 -f standalone-values.yaml;
          fi

          kubectl get pods
          sleep 20s
          kubectl get pods
          kubectl port-forward service/deploy-testing-milvus 19530 >/dev/null 2>&1 &
          sleep 20s
          # check whether port-forward success
          nc -vz 127.0.0.1 19530          

      - name: Run first test
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/deploy
        run: |
          # first test
          if [ ${{ matrix.task }} == "reinstall" ]; then python scripts/action_before_reinstall.py; fi
          if [ ${{ matrix.task }} == "upgrade" ]; then python scripts/action_before_upgrade.py; fi          
      - name: Milvus Idle Time
        timeout-minutes: 5
        shell: bash
        working-directory: tests/python_client/deploy
        run: |
          sleep 60s          
      - name: Export logs
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client/deploy
        run: |
          kubectl get pod
          # export k8s log for milvus
          bash ../../scripts/export_log_k8s.sh default deploy-testing k8s_logs/first_deploy          
      - name: Restart Milvus
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/deploy
        run: |
          # uninstall milvus
          if [ ${{ matrix.mode }} == "standalone" ];
          then
            kubectl delete pod -l app.kubernetes.io/instance=deploy-testing --grace-period=0 --force;
            kubectl delete pod -l release=deploy-testing --grace-period=0 --force;
          else
            helm uninstall deploy-testing
          fi          

      - name: Seconde Milvus Deployment
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/deploy
        run: |
          if [ ${{ matrix.mode }} == "cluster" ]; then helm install --wait --timeout 720s deploy-testing milvus/milvus -f cluster-values-second.yaml; fi
          if [ ${{ matrix.mode }} == "standalone" ]; then helm upgrade --wait --timeout 720s deploy-testing milvus/milvus -f standalone-values.yaml; fi
          kubectl get pods
          sleep 20s
          kubectl get pods
          ps aux|grep forward|grep -v grep|awk '{print $2}'|xargs kill -9
          kubectl port-forward service/deploy-testing-milvus 19530 >/dev/null 2>&1 &
          sleep 120s
          # check whether port-forward success
          nc -vz 127.0.0.1 19530          

      - name: Run second test
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/deploy
        run: |
          # second test
          if [ ${{ matrix.task }} == "reinstall" ]; then python scripts/action_after_reinstall.py; fi
          if [ ${{ matrix.task }} == "upgrade" ]; then python scripts/action_after_upgrade.py; fi          

      - name: Export logs
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client/deploy
        run: |
          kubectl get pod
          # export k8s log for milvus
          bash ../../scripts/export_log_k8s.sh default deploy-testing k8s_logs/second_deploy          

      - name: Upload logs
        if: ${{ ! success() }}
        uses: actions/upload-artifact@v2
        with:
          name: helm-log-${{ matrix.mq_type }}-${{ matrix.mode }}-${{ matrix.task }}
          path: tests/python_client/deploy/k8s_logs