milvus/.github/workflows/network-partition-chaos-test.yaml

name: Network Partition Chaos Test

on:
  workflow_dispatch:

jobs:
  
  test-network-partition-chaos:
   
    runs-on: ubuntu-latest
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        pod: [datacoord, datanode, indexcoord, indexnode, proxy, pulsar, querycoord, querynode, rootcoord, etcd, minio]
    
    steps:

      - name: Creating kind cluster
        uses: helm/kind-action@v1.2.0
      - name: Print cluster information
        run: |
          kubectl config view
          kubectl cluster-info
          kubectl get nodes
          kubectl get pods -n kube-system
          helm version
          kubectl version
      
      - uses: actions/checkout@v2

      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8

      - name: Install dependency
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          pip install -r ../requirements.txt
          pip install --upgrade protobuf      

      - name: Deploy Chaos Mesh
        shell: bash
        run: |
          helm repo add chaos-mesh https://charts.chaos-mesh.org
          helm search repo chaos-mesh
          kubectl create ns chaos-testing
          helm install chaos-mesh chaos-mesh/chaos-mesh -n=chaos-testing --version v2.0.1 --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock
          # wait for all pod running 
          sleep 60s
          kubectl get po -n chaos-testing

      - name: Deploy Milvus
        shell: bash
        run: |
          helm repo add milvus https://milvus-io.github.io/milvus-helm
          helm repo update
          # install milvus with latest image
          helm install --wait --timeout 360s chaos-testing milvus/milvus --set cluster.enabled=true --set image.all.repository=milvusdb/milvus --set image.all.tag=master-latest -n=chaos-testing
          kubectl get pods -n chaos-testing
          sleep 20s
          kubectl get pods -n chaos-testing
          kubectl port-forward service/chaos-testing-milvus 19530 -n chaos-testing >/dev/null 2>&1 &
          sleep 20s
          # check whether port-forward success
          nc -vz 127.0.0.1 19530
      - name: Chaos Test
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          # replace chaos object
          sed -i "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/network_partition\/'/g" constants.py
          sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${{ matrix.pod }}_network_partition.yaml\'/g" constants.py
          cat constants.py
          pytest -s -v test_chaos.py --host 127.0.0.1

      - name: Export logs
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          #in this step, verify whether pod has been killed by pod's age
          kubectl get po -n chaos-testing
          # export k8s log for chaos mesh and milvus
          bash ../../scripts/export_log_k8s.sh chaos-testing chaos-testing
          bash ../../scripts/export_log_k8s.sh chaos-testing chaos-daemon
      
      - name: Upload logs
        if: ${{ always() }}
        uses: actions/upload-artifact@v2
        with:
          name: logs-${{ matrix.pod }}
          path: tests/python_client/chaos/k8s_logs