Add jenkinsfile for run ci on QA cluster (#15351)

Signed-off-by: Jenny Li <jing.li@zilliz.com>
2024-11-29 18:38:44 +08:00 · 2022-01-24 13:18:29 +08:00 · 2022-01-24 13:18:29 +08:00 · de197e6eb8
commit de197e6eb8
parent 390cdd2515
5 changed files with 490 additions and 1 deletions
--- a/build/ci/jenkins/QaPR.groovy
+++ b/build/ci/jenkins/QaPR.groovy
@ -0,0 +1,226 @@
+#!/usr/bin/env groovy
+
+int total_timeout_minutes = 120
+int e2e_timeout_seconds = 70 * 60
+def imageTag=''
+int case_timeout_seconds = 10 * 60
+def chart_version='2.4.25'
+pipeline {
+    options {
+        timestamps()
+        timeout(time: total_timeout_minutes, unit: 'MINUTES')
+        buildDiscarder logRotator(artifactDaysToKeepStr: '30')
+        parallelsAlwaysFailFast()
+        preserveStashes(buildCount: 5)
+
+    }
+    agent {
+            kubernetes {
+                label 'milvus-e2e-test-pipeline'
+                inheritFrom 'default'
+                defaultContainer 'main'
+                yamlFile 'build/ci/jenkins/pod/qa/rte.yaml'
+                customWorkspace '/home/jenkins/agent/workspace'
+            }
+    }
+    environment {
+        PROJECT_NAME = 'milvus'
+        SEMVER = "${BRANCH_NAME.contains('/') ? BRANCH_NAME.substring(BRANCH_NAME.lastIndexOf('/') + 1) : BRANCH_NAME}"
+        DOCKER_BUILDKIT = 1
+        ARTIFACTS = "${env.WORKSPACE}/_artifacts"
+        DOCKER_CREDENTIALS_ID = "f0aacc8e-33f2-458a-ba9e-2c44f431b4d2"
+        TARGET_REPO = "milvusdb"
+        // CI_DOCKER_CREDENTIAL_ID = "ci-docker-registry"
+        CI_DOCKER_CREDENTIAL_ID = "qa-ci-docker-registry"
+        MILVUS_HELM_NAMESPACE = "milvus-ci"
+        DISABLE_KIND = true
+        // HUB = 'registry.milvus.io/milvus'
+        HUB = 'harbor.zilliz.cc/milvus-ci'
+        JENKINS_BUILD_ID = "${env.BUILD_ID}"
+        CI_MODE="pr"
+        MIRROR_URL="http://10.201.20.246:5000"
+    }
+
+    stages {
+        stage ('Build'){
+            steps {
+                container('main') {
+                    dir ('build'){
+                            sh './set_docker_mirror.sh'
+                    }
+                    dir ('tests/scripts') {
+                        script {
+                            sh 'printenv'
+                            def date = sh(returnStdout: true, script: 'date +%Y%m%d').trim()
+                            def gitShortCommit = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()    
+                            imageTag="${env.BRANCH_NAME}-${date}-${gitShortCommit}"
+                            withCredentials([usernamePassword(credentialsId: "${env.CI_DOCKER_CREDENTIAL_ID}", usernameVariable: 'CI_REGISTRY_USERNAME', passwordVariable: 'CI_REGISTRY_PASSWORD')]){
+                                sh """
+                                TAG="${imageTag}" \
+                                ./e2e-k8s.sh \
+                                --skip-export-logs \
+                                --skip-install \
+                                --skip-cleanup \
+                                --skip-setup \
+                                --skip-test
+                                """
+
+                                // stash imageTag info for rebuild install & E2E Test only
+                                sh "echo ${imageTag} > imageTag.txt"
+                                stash includes: 'imageTag.txt', name: 'imageTag'
+
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+
+        stage('Install & E2E Test') {
+            matrix {
+                axes {
+                    axis {
+                        name 'MILVUS_SERVER_TYPE'
+                        values 'standalone', 'distributed'
+                    }
+                    axis {
+                        name 'MILVUS_CLIENT'
+                        values 'pymilvus'
+                    }
+                }
+
+                stages {
+                    stage('Install') {
+                        steps {
+                            container('main') {
+                                dir ('tests/scripts') {
+                                    script {
+                                        sh 'printenv'
+                                        def clusterEnabled = "false"
+                                        if ("${MILVUS_SERVER_TYPE}" == 'distributed') {
+                                            clusterEnabled = "true"
+                                        }
+
+                                        if ("${MILVUS_CLIENT}" == "pymilvus") {
+                                            if ("${imageTag}"==''){
+                                                dir ("imageTag"){
+                                                    try{
+                                                        unstash 'imageTag'
+                                                        imageTag=sh(returnStdout: true, script: 'cat imageTag.txt | tr -d \'\n\r\'')
+                                                    }catch(e){
+                                                        print "No Image Tag info remained ,please rerun build to build new image."
+                                                        exit 1
+                                                    }
+                                                }
+                                            }
+                                            withCredentials([usernamePassword(credentialsId: "${env.CI_DOCKER_CREDENTIAL_ID}", usernameVariable: 'CI_REGISTRY_USERNAME', passwordVariable: 'CI_REGISTRY_PASSWORD')]){
+                                                sh """
+                                                MILVUS_CLUSTER_ENABLED=${clusterEnabled} \
+                                                TAG=${imageTag}\
+                                                ./e2e-k8s.sh \
+                                                --skip-export-logs \
+                                                --skip-cleanup \
+                                                --skip-setup \
+                                                --skip-test \
+                                                --skip-build \
+                                                --skip-build-image \
+                                                --install-extra-arg "--set etcd.persistence.storageClass=local-path \
+                                                --set minio.persistence.storageClass=local-path \
+                                                --set etcd.metrics.enabled=false \
+                                                --set etcd.metrics.podMonitor.enabled=false\
+                                                --set metrics.serviceMonitor.enabled=true \
+                                                --version ${chart_version} \
+                                                -f values/pr.yaml" 
+                                                """
+                                            }
+                                        } else {
+                                            error "Error: Unsupported Milvus client: ${MILVUS_CLIENT}"
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    stage('E2E Test'){
+                        agent {
+                                kubernetes {
+                                    label 'milvus-e2e-test-pr'
+                                    inheritFrom 'default'
+                                    defaultContainer 'main'
+                                    yamlFile 'build/ci/jenkins/pod/qa/rte.yaml'
+                                    customWorkspace '/home/jenkins/agent/workspace'
+                                }
+                        }
+                        steps {
+                            container('pytest') {
+                                dir ('tests/scripts') {
+                                    script {
+                                        def release_name=sh(returnStdout: true, script: './get_release_name.sh')
+                                        def clusterEnabled = 'false'
+                                        if ("${MILVUS_SERVER_TYPE}" == "distributed") {
+                                            clusterEnabled = "true"
+                                        }
+                                        if ("${MILVUS_CLIENT}" == "pymilvus") {
+                                            sh """
+                                            MILVUS_HELM_RELEASE_NAME="${release_name}" \
+                                            MILVUS_HELM_NAMESPACE="milvus-ci" \
+                                            MILVUS_CLUSTER_ENABLED="${clusterEnabled}" \
+                                            TEST_TIMEOUT="${e2e_timeout_seconds}" \
+                                            ./ci_e2e.sh  "-n 6 -x --tags L0 L1 --timeout ${case_timeout_seconds}"
+                                            """
+                            
+                                        } else {
+                                        error "Error: Unsupported Milvus client: ${MILVUS_CLIENT}"
+                                        }
+                                    }
+                                }
+                            }
+                        }
+
+                    }
+                }
+                post{
+                    always {
+                        container('pytest'){
+                            dir("${env.ARTIFACTS}") {
+                                    sh "tar -zcvf artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${MILVUS_CLIENT}-pytest-logs.tar.gz /tmp/ci_logs/test --remove-files || true"
+                                    archiveArtifacts artifacts: "artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${MILVUS_CLIENT}-pytest-logs.tar.gz ", allowEmptyArchive: true
+                            }
+                        }
+                        container('main') {
+                            dir ('tests/scripts') {  
+                                script {
+                                    def release_name=sh(returnStdout: true, script: './get_release_name.sh')
+                                    sh "./qa/uninstall_milvus.sh --release-name ${release_name}"
+                                    sh "./qa/ci_logs.sh --log-dir /ci-logs  --artifacts-name ${env.ARTIFACTS}/artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${SEMVER}-${env.BUILD_NUMBER}-${MILVUS_CLIENT}-e2e-logs \
+                                    --release-name ${release_name}"
+                                    dir("${env.ARTIFACTS}") {
+                                        archiveArtifacts artifacts: "artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${SEMVER}-${env.BUILD_NUMBER}-${MILVUS_CLIENT}-e2e-logs.tar.gz", allowEmptyArchive: true
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+
+        }
+    }
+    post{
+        unsuccessful {
+                container('jnlp') {
+                    dir ('tests/scripts') {
+                        script {
+                            def authorEmail = sh(returnStdout: true, script: './get_author_email.sh ')
+                            emailext subject: '$DEFAULT_SUBJECT',
+                            body: '$DEFAULT_CONTENT',
+                            recipientProviders: [developers(), culprits()],
+                            replyTo: '$DEFAULT_REPLYTO',
+                            to: "${authorEmail},devops@zilliz.com"
+                        }
+                    }
+                }
+            }
+        }
+}
--- a/build/ci/jenkins/pod/qa/rte.yaml
+++ b/build/ci/jenkins/pod/qa/rte.yaml
@ -0,0 +1,66 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: milvus-e2e
+  namespace: jenkins
+spec:
+  enableServiceLinks: false
+  containers:
+  - name: main
+    image: harbor.zilliz.cc/dockerhub/milvusdb/krte:20211213-dcc15e9
+    env:
+    - name: DOCKER_IN_DOCKER_ENABLED
+      value: "true"
+    - name: DOCKER_VOLUME_DIRECTORY
+      value: "/mnt/disk/.docker"
+    tty: true
+    securityContext:
+      privileged: true
+    args: ["cat"]
+    resources:
+      limits:
+        cpu: "6"
+        memory: 12Gi
+      requests:
+        cpu: "0.5"
+        memory: 5Gi
+    volumeMounts:
+    - mountPath: /docker-graph
+      name: docker-graph
+    - mountPath: /var/lib/docker
+      name: docker-root
+    - mountPath: /lib/modules
+      name: modules
+      readOnly: true
+    - mountPath: /sys/fs/cgroup
+      name: cgroup
+    - mountPath: /mnt/disk/.docker
+      name: build-cache
+      subPath: docker-volume
+  - name: pytest
+    image: harbor.zilliz.cc/dockerhub/milvusdb/pytest:20220118-3eb5cc4
+    resources:
+      limits:
+        cpu: "6"
+        memory: 12Gi
+      requests:
+        cpu: "0.5"
+        memory: 5Gi
+  volumes:
+  - emptyDir: {}
+    name: docker-graph
+  - emptyDir: {}
+    name: docker-root
+  - hostPath:
+      path: /tmp/krte/cache
+      type: DirectoryOrCreate
+    name: build-cache
+  - hostPath:
+      path: /lib/modules
+      type: Directory
+    name: modules
+  - hostPath:
+      path: /sys/fs/cgroup
+      type: Directory
+    name: cgroup
--- a/build/set_docker_mirror.sh
+++ b/build/set_docker_mirror.sh
@ -21,7 +21,8 @@
 set -e

 # Use nexus as docker mirror registry
-MIRROR_URL="http://nexus-nexus-repository-manager-docker-5000.nexus:5000"
+MIRROR_URL="${MIRROR_URL:-http://nexus-nexus-repository-manager-docker-5000.nexus:5000}"
+#MIRROR_URL="http://nexus-nexus-repository-manager-docker-5000.nexus:5000"

 # Add registry mirror config into docker daemon
 set_daemon_json_file(){
--- a/tests/scripts/qa/ci_logs.sh
+++ b/tests/scripts/qa/ci_logs.sh
@ -0,0 +1,84 @@
+#!/bin/bash
+
+# Licensed to the LF AI & Data foundation under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Exit immediately for non zero status
+set -e
+# Check unset variables
+# set -u
+# Print commands
+set -x
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+ROOT="$( cd -P "$( dirname "$SOURCE" )/../.." && pwd )"
+
+while (( "$#" )); do
+  case "$1" in
+
+    --release-name)
+      RELEASE_NAME=$2
+      shift 2
+    ;;
+
+    --artifacts-name)
+      ARTIFACTS_NAME=$2
+      shift 2
+    ;;
+
+    --log-dir)
+      LOG_DIR=$2
+      shift 2
+    ;;
+
+    -h|--help)
+      { set +x; } 2>/dev/null
+      HELP="
+Usage:
+  $0 [flags] [Arguments]
+    
+    --log-dir                  Log Path
+
+    --artifacts-name           Artifacts Name
+
+    --release-name             Milvus helm release name
+
+    -h or --help                Print help information
+
+
+Use \"$0  --help\" for more information about a given command.
+"
+      echo -e "${HELP}" ; exit 0
+    ;;
+    -*)
+      echo "Error: Unsupported flag $1" >&2
+      exit 1
+      ;;
+    *) # preserve positional arguments
+      PARAMS+=("$1")
+      shift
+      ;;
+  esac
+done
+
+RELEASE_NAME=${RELEASE_NAME:-milvus-testing}
+
+tar -zcvf ${ARTIFACTS_NAME:-artifacts}.tar.gz ${RELEASE_NAME}/*
--- a/tests/scripts/qa/uninstall_milvus.sh
+++ b/tests/scripts/qa/uninstall_milvus.sh
@ -0,0 +1,112 @@
+#!/bin/bash
+
+# Copyright (C) 2019-2020 Zilliz. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied. See the License for the specific language governing permissions and limitations under the License.
+
+# Exit immediately for non zero status
+set -e
+# Print commands
+# set -x
+
+while (( "$#" )); do
+  case "$1" in
+
+    --release-name)
+      RELEASE_NAME=$2
+      shift 2
+    ;;
+  
+    -h|--help)
+      { set +x; } 2>/dev/null
+      HELP="
+Usage:
+  $0 [flags] [Arguments]
+
+    --release-name             Milvus helm release name
+
+    -h or --help                Print help information
+
+
+Use \"$0  --help\" for more information about a given command.
+"
+      echo -e "${HELP}" ; exit 0
+    ;;
+    -*)
+      echo "Error: Unsupported flag $1" >&2
+      exit 1
+      ;;
+    *) # preserve positional arguments
+      PARAMS+=("$1")
+      shift
+      ;;
+  esac
+done
+
+MILVUS_HELM_RELEASE_NAME="${MILVUS_HELM_RELEASE_NAME:-milvus-testing}"
+MILVUS_HELM_NAMESPACE="${MILVUS_HELM_NAMESPACE:-default}"
+
+if [[ -n "${RELEASE_NAME:-}" ]]; then
+    MILVUS_HELM_RELEASE_NAME="${RELEASE_NAME}"
+    # List pod list before uninstall 
+    kubectl get pods -n ${MILVUS_HELM_NAMESPACE}  -o wide | grep "${MILVUS_HELM_RELEASE_NAME}-"
+    # Show restart pods last terminated reason
+    # restart_pods=$(kubectl get pods -n ${MILVUS_HELM_NAMESPACE} | grep "${MILVUS_HELM_RELEASE_NAME}-" | grep 'ago)' | awk '{print $1}')
+   
+    # for restart_pod in ${restart_pods}
+    # do 
+    #   reason=$(kubectl get pod ${restart_pod} -n milvus-ci -o json | jq .status.containerStatuses[0].lastState.terminated.reason )
+    #   restart_count=$(kubectl get pod ${restart_pod} -n milvus-ci -o json | jq .status.containerStatuses[0].restartCount )
+    #   echo "${restart_pod} restarts ${restart_count}, last terminateed reason is ${reason}"
+    # done
+    
+fi
+mkdir ${MILVUS_HELM_RELEASE_NAME}
+all_pods=$(kubectl get pods -n ${MILVUS_HELM_NAMESPACE} | grep "${MILVUS_HELM_RELEASE_NAME}-"  | awk '{print $1}')
+for pod_name in ${all_pods}
+do
+  kubectl logs ${pod_name} -n ${MILVUS_HELM_NAMESPACE} --all-containers=true > ${MILVUS_HELM_RELEASE_NAME}/${pod_name}.log
+  kubectl logs ${pod_name} -n ${MILVUS_HELM_NAMESPACE} --all-containers=true --previous=true > ${MILVUS_HELM_RELEASE_NAME}/${pod_name}_previous.log
+done
+
+
+# Uninstall Milvus Helm Release
+# Do not exit with error when release not found so that the script can also be used to clean up related pvc even helm release has been uninstalled already
+helm uninstall -n "${MILVUS_HELM_NAMESPACE}" "${MILVUS_HELM_RELEASE_NAME}" || true
+
+MILVUS_LABELS1="app.kubernetes.io/instance=${MILVUS_HELM_RELEASE_NAME}"
+MILVUS_LABELS2="release=${MILVUS_HELM_RELEASE_NAME}"
+
+# Clean up pvc
+kubectl delete pvc --wait -n "${MILVUS_HELM_NAMESPACE}" $(kubectl get pvc -n "${MILVUS_HELM_NAMESPACE}" -l "${MILVUS_LABELS1}" -o jsonpath='{range.items[*]}{.metadata.name} ') || true
+kubectl delete pvc --wait -n "${MILVUS_HELM_NAMESPACE}" $(kubectl get pvc -n "${MILVUS_HELM_NAMESPACE}" -l "${MILVUS_LABELS2}" -o jsonpath='{range.items[*]}{.metadata.name} ') || true
+
+# Add check & delete pvc again in case pvc need time to be deleted 
+# clean_label_pvc(){
+#     local label=${1?label expected as first argument.}
+
+#     for i in {1..10}
+#     do
+#         PVC=$(kubectl get pvc -n "${MILVUS_HELM_NAMESPACE}" -l "${label}" -o jsonpath='{range.items[*]}{.metadata.name} ')
+#         STATUS=$(echo ${PVC} | wc -w )
+#         echo "status is ${STATUS}"
+#         if [ $STATUS == 0 ]; then
+#             break
+#         else
+#             sleep 5
+#             echo "PVCs are ${PVC}"
+#             kubectl delete pvc --wait -n "${MILVUS_HELM_NAMESPACE}" ${PVC}
+#         fi
+#     done
+# }
+
+# echo "Check & Delete Persistent Volumes"
+# clean_label_pvc ${MILVUS_LABELS1}
+# clean_label_pvc ${MILVUS_LABELS2}