milvus/.github/workflows/network-partition-chaos-test.yaml

95 lines
3.3 KiB
YAML
Raw Normal View History

name: Network Partition Chaos Test
on:
workflow_dispatch:
jobs:
test-network-partition-chaos:
runs-on: ubuntu-latest
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
pod: [datacoord, datanode, indexcoord, indexnode, proxy, pulsar, querycoord, querynode, rootcoord, etcd, minio]
steps:
- name: Creating kind cluster
uses: helm/kind-action@v1.2.0
- name: Print cluster information
run: |
kubectl config view
kubectl cluster-info
kubectl get nodes
kubectl get pods -n kube-system
helm version
kubectl version
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: 3.8
- name: Install dependency
shell: bash
working-directory: tests/python_client/chaos
run: |
pip install -r ../requirements.txt
pip install --upgrade protobuf
- name: Deploy Chaos Mesh
shell: bash
run: |
helm repo add chaos-mesh https://charts.chaos-mesh.org
helm search repo chaos-mesh
kubectl create ns chaos-testing
helm install chaos-mesh chaos-mesh/chaos-mesh -n=chaos-testing --version v2.0.1 --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock
# wait for all pod running
sleep 60s
kubectl get po -n chaos-testing
- name: Deploy Milvus
shell: bash
run: |
helm repo add milvus https://milvus-io.github.io/milvus-helm
helm repo update
# install milvus with latest image
helm install --wait --timeout 360s chaos-testing milvus/milvus --set cluster.enabled=true --set image.all.repository=milvusdb/milvus --set image.all.tag=master-latest -n=chaos-testing
kubectl get pods -n chaos-testing
sleep 20s
kubectl get pods -n chaos-testing
kubectl port-forward service/chaos-testing-milvus 19530 -n chaos-testing >/dev/null 2>&1 &
sleep 20s
# check whether port-forward success
nc -vz 127.0.0.1 19530
- name: Chaos Test
shell: bash
working-directory: tests/python_client/chaos
run: |
# replace chaos object
sed -i "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/network_partition\/'/g" constants.py
sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${{ matrix.pod }}_network_partition.yaml\'/g" constants.py
cat constants.py
pytest -s -v test_chaos.py --host 127.0.0.1
- name: Export logs
if: ${{ always() }}
shell: bash
working-directory: tests/python_client/chaos
run: |
#in this step, verify whether pod has been killed by pod's age
kubectl get po -n chaos-testing
# export k8s log for chaos mesh and milvus
bash ../../scripts/export_log_k8s.sh chaos-testing chaos-testing
bash ../../scripts/export_log_k8s.sh chaos-testing chaos-daemon
- name: Upload logs
if: ${{ always() }}
uses: actions/upload-artifact@v2
with:
name: logs-${{ matrix.pod }}
path: tests/python_client/chaos/k8s_logs