From e8c40e94ad5e32a6c08ce682dbc446a0dca380e4 Mon Sep 17 00:00:00 2001 From: yellow-shine Date: Thu, 10 Oct 2024 17:11:21 +0800 Subject: [PATCH] enhance: use new implementation of e2e pipeline (#36687) https://github.com/milvus-io/milvus/issues/36348 https://github.com/milvus-io/milvus/pull/36347 Signed-off-by: Yellow Shine --- ci/jenkins/PR.groovy | 356 +++++++++++------------------------------- ci/jenkins/PR2.groovy | 137 ---------------- 2 files changed, 91 insertions(+), 402 deletions(-) delete mode 100644 ci/jenkins/PR2.groovy diff --git a/ci/jenkins/PR.groovy b/ci/jenkins/PR.groovy index 7d21c8d1f8..aa217d96bb 100644 --- a/ci/jenkins/PR.groovy +++ b/ci/jenkins/PR.groovy @@ -1,311 +1,137 @@ -#!/usr/bin/env groovy +@Library('jenkins-shared-library@v0.57.0') _ + +def pod = libraryResource 'io/milvus/pod/tekton-4am.yaml' +def milvus_helm_chart_version = '4.2.8' -int total_timeout_minutes = 60 * 5 -int e2e_timeout_seconds = 120 * 60 -def imageTag='' -int case_timeout_seconds = 20 * 60 -def chart_version='4.1.8' pipeline { options { - timestamps() - timeout(time: total_timeout_minutes, unit: 'MINUTES') - buildDiscarder logRotator(artifactDaysToKeepStr: '30') + skipDefaultCheckout true parallelsAlwaysFailFast() + buildDiscarder logRotator(artifactDaysToKeepStr: '30') preserveStashes(buildCount: 5) disableConcurrentBuilds(abortPrevious: true) - } agent { - kubernetes { - cloud '4am' - inheritFrom 'milvus-e2e-4am' - defaultContainer 'main' - yamlFile 'ci/jenkins/pod/rte-build.yaml' - customWorkspace '/home/jenkins/agent/workspace' - } + kubernetes { + cloud '4am' + yaml pod + } } - environment { - PROJECT_NAME = 'milvus' - SEMVER = "${BRANCH_NAME.contains('/') ? BRANCH_NAME.substring(BRANCH_NAME.lastIndexOf('/') + 1) : BRANCH_NAME}" - DOCKER_BUILDKIT = 1 - ARTIFACTS = "${env.WORKSPACE}/_artifacts" - CI_DOCKER_CREDENTIAL_ID = "harbor-milvus-io-registry" - MILVUS_HELM_NAMESPACE = "milvus-ci" - DISABLE_KIND = true - HUB = 'harbor.milvus.io/milvus' - JENKINS_BUILD_ID = "${env.BUILD_ID}" - CI_MODE="pr" - SHOW_MILVUS_CONFIGMAP= true - } - stages { - stage ('Build'){ + stage('meta') { steps { - container('main') { - dir ('build'){ - sh """ - MIRROR_URL="https://docker-nexus-ci.zilliz.cc" ./set_docker_mirror.sh - """ + container('jnlp') { + script { + isPr = env.CHANGE_ID != null + gitMode = isPr ? 'merge' : 'fetch' + gitBaseRef = isPr ? "$env.CHANGE_TARGET" : "$env.BRANCH_NAME" + + get_helm_release_name = tekton.helm_release_name client: 'py', + changeId: "${env.CHANGE_ID}", + buildId:"${env.BUILD_ID}" } - dir ('tests/scripts') { + } + } + } + stage('build') { + steps { + container('tkn') { + script { + def job_name = tekton.run arch: 'amd64', + isPr: isPr, + gitMode: gitMode , + gitBaseRef: gitBaseRef, + pullRequestNumber: "$env.CHANGE_ID", + suppress_suffix_of_image_tag: true, + images: '["milvus","pytest","helm"]' + + milvus_image_tag = tekton.query_result job_name, 'milvus-image-tag' + pytest_image = tekton.query_result job_name, 'pytest-image-fqdn' + helm_image = tekton.query_result job_name, 'helm-image-fqdn' + } + } + } + post { + always { + container('tkn') { script { - sh 'printenv' - def date = sh(returnStdout: true, script: 'date +%Y%m%d').trim() - sh 'git config --global --add safe.directory /home/jenkins/agent/workspace' - def gitShortCommit = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim() - imageTag="${env.BRANCH_NAME}-${date}-${gitShortCommit}" - withCredentials([usernamePassword(credentialsId: "${env.CI_DOCKER_CREDENTIAL_ID}", usernameVariable: 'CI_REGISTRY_USERNAME', passwordVariable: 'CI_REGISTRY_PASSWORD')]){ - sh """ - TAG="${imageTag}" \ - ./e2e-k8s.sh \ - --skip-export-logs \ - --skip-install \ - --skip-cleanup \ - --skip-setup \ - --skip-test - """ - - // stash imageTag info for rebuild install & E2E Test only - sh "echo ${imageTag} > imageTag.txt" - stash includes: 'imageTag.txt', name: 'imageTag' - - } + tekton.sure_stop() } } } } } - - - stage('Install & E2E Test') { + stage('E2E Test') { matrix { + agent { + kubernetes { + cloud '4am' + yaml pod + } + } axes { axis { - name 'MILVUS_SERVER_TYPE' + name 'milvus_deployment_option' values 'standalone', 'distributed', 'standalone-kafka', 'standalone-one-pod' } - axis { - name 'MILVUS_CLIENT' - values 'pymilvus' - } } - stages { - stage('Install') { + stage('E2E Test') { steps { - container('main') { - stash includes: 'tests/**', name: 'testCode', useDefaultExcludes: false - dir ('tests/scripts') { - script { - sh 'printenv' - def clusterEnabled = "false" - def valuesFile = "pr-4am.yaml" - if ("${MILVUS_SERVER_TYPE}".contains('distributed')) { - clusterEnabled = "true" - } - if ("${MILVUS_SERVER_TYPE}".contains("kafka")) { - valuesFile = "pr_kafka.yaml" - } - if ("${MILVUS_SERVER_TYPE}" == "standalone-one-pod") { - valuesFile = "nightly-one-pod.yaml" - } - - if ("${MILVUS_CLIENT}" == "pymilvus") { - if ("${imageTag}"==''){ - dir ("imageTag"){ - try{ - unstash 'imageTag' - imageTag=sh(returnStdout: true, script: 'cat imageTag.txt | tr -d \'\n\r\'') - }catch(e){ - print "No Image Tag info remained ,please rerun build to build new image." - exit 1 - } - } - } - // modify values file to enable kafka - if ("${MILVUS_SERVER_TYPE}".contains("kafka")) { - sh ''' - apt-get update - apt-get install wget -y - wget https://github.com/mikefarah/yq/releases/download/v4.34.1/yq_linux_amd64 -O /usr/bin/yq - chmod +x /usr/bin/yq - ''' - sh """ - cp values/ci/pr-4am.yaml values/ci/pr_kafka.yaml - yq -i '.pulsar.enabled=false' values/ci/pr_kafka.yaml - yq -i '.kafka.enabled=true' values/ci/pr_kafka.yaml - yq -i '.kafka.metrics.kafka.enabled=true' values/ci/pr_kafka.yaml - yq -i '.standalone.messageQueue="kafka"' values/ci/pr_kafka.yaml - yq -i '.kafka.metrics.jmx.enabled=true' values/ci/pr_kafka.yaml - yq -i '.kafka.metrics.serviceMonitor.enabled=true' values/ci/pr_kafka.yaml - """ - } - withCredentials([usernamePassword(credentialsId: "${env.CI_DOCKER_CREDENTIAL_ID}", usernameVariable: 'CI_REGISTRY_USERNAME', passwordVariable: 'CI_REGISTRY_PASSWORD')]){ - if ("${MILVUS_SERVER_TYPE}" == "standalone-one-pod") { - try { - sh """ - MILVUS_CLUSTER_ENABLED=${clusterEnabled} \ - MILVUS_HELM_REPO="https://nexus-ci.zilliz.cc/repository/milvus-proxy" \ - TAG=${imageTag}\ - ./e2e-k8s.sh \ - --skip-export-logs \ - --skip-cleanup \ - --skip-setup \ - --skip-test \ - --skip-build \ - --skip-build-image \ - --install-extra-arg " - --set etcd.metrics.enabled=true \ - --set etcd.metrics.podMonitor.enabled=true \ - --set indexCoordinator.gc.interval=1 \ - --set indexNode.disk.enabled=true \ - --set queryNode.disk.enabled=true \ - --set standalone.disk.enabled=true \ - --version ${chart_version} \ - -f values/ci/${valuesFile}" - """ - } catch (Exception e) { - echo "Tests failed, but the build will not be marked as failed." - } - - }else{ - sh """ - MILVUS_CLUSTER_ENABLED=${clusterEnabled} \ - MILVUS_HELM_REPO="https://nexus-ci.zilliz.cc/repository/milvus-proxy" \ - TAG=${imageTag}\ - ./e2e-k8s.sh \ - --skip-export-logs \ - --skip-cleanup \ - --skip-setup \ - --skip-test \ - --skip-build \ - --skip-build-image \ - --install-extra-arg " - --set etcd.metrics.enabled=true \ - --set etcd.metrics.podMonitor.enabled=true \ - --set indexCoordinator.gc.interval=1 \ - --set indexNode.disk.enabled=true \ - --set queryNode.disk.enabled=true \ - --set standalone.disk.enabled=true \ - --version ${chart_version} \ - -f values/ci/${valuesFile}" - """ - } - } - } else { - error "Error: Unsupported Milvus client: ${MILVUS_CLIENT}" - } - } - } - } - - } - } - stage('E2E Test'){ - options { - skipDefaultCheckout() - } - agent { - kubernetes { - cloud '4am' - inheritFrom 'default' - defaultContainer 'main' - yamlFile 'ci/jenkins/pod/e2e.yaml' - customWorkspace '/home/jenkins/agent/workspace' - } - } - steps { - container('pytest') { - unstash('testCode') + container('tkn') { script { - sh 'ls -lah' - } - dir ('tests/scripts') { - script { - def release_name=sh(returnStdout: true, script: './get_release_name.sh') - def clusterEnabled = 'false' - if ("${MILVUS_SERVER_TYPE}".contains("distributed")) { - clusterEnabled = "true" - } - if ("${MILVUS_CLIENT}" == "pymilvus") { - if ("${MILVUS_SERVER_TYPE}" == "standalone-one-pod") { - try { - sh """ - MILVUS_HELM_RELEASE_NAME="${release_name}" \ - MILVUS_HELM_NAMESPACE="milvus-ci" \ - MILVUS_CLUSTER_ENABLED="${clusterEnabled}" \ - TEST_TIMEOUT="${e2e_timeout_seconds}" \ - ./ci_e2e_4am.sh "-n 6 -x --tags L0 L1 --timeout ${case_timeout_seconds}" - """ - } catch (Exception e) { - echo "Tests failed, but the build will not be marked as failed." - } - }else{ - sh """ - MILVUS_HELM_RELEASE_NAME="${release_name}" \ - MILVUS_HELM_NAMESPACE="milvus-ci" \ - MILVUS_CLUSTER_ENABLED="${clusterEnabled}" \ - TEST_TIMEOUT="${e2e_timeout_seconds}" \ - ./ci_e2e_4am.sh "-n 6 -x --tags L0 L1 --timeout ${case_timeout_seconds}" - """ - } - } else { - error "Error: Unsupported Milvus client: ${MILVUS_CLIENT}" + def helm_release_name = get_helm_release_name milvus_deployment_option + + if (milvus_deployment_option == 'distributed-streaming-service') { + try { + tekton.pytest helm_release_name: helm_release_name, + milvus_helm_version: milvus_helm_chart_version, + ciMode: 'e2e', + milvus_image_tag: milvus_image_tag, + pytest_image: pytest_image, + helm_image: helm_image, + milvus_deployment_option: milvus_deployment_option, + verbose: 'false' + } catch (Exception e) { + println e } + } else { + tekton.pytest helm_release_name: helm_release_name, + milvus_helm_version: milvus_helm_chart_version, + ciMode: 'e2e', + milvus_image_tag: milvus_image_tag, + pytest_image: pytest_image, + helm_image: helm_image, + milvus_deployment_option: milvus_deployment_option, + verbose: 'false' } } } } - post{ + + post { always { - container('pytest'){ - dir("${env.ARTIFACTS}") { - sh "tar -zcvf ${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${MILVUS_CLIENT}-pytest-logs.tar.gz /tmp/ci_logs/test --remove-files || true" - archiveArtifacts artifacts: "${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${MILVUS_CLIENT}-pytest-logs.tar.gz ", allowEmptyArchive: true + container('tkn') { + script { + tekton.sure_stop() } } - } - } - } - } - post{ - always { - container('main') { - dir ('tests/scripts') { - script { - def release_name=sh(returnStdout: true, script: './get_release_name.sh') - sh "kubectl get pods -n ${MILVUS_HELM_NAMESPACE} | grep ${release_name} " - sh "./uninstall_milvus.sh --release-name ${release_name}" - sh "./ci_logs.sh --log-dir /ci-logs --artifacts-name ${env.ARTIFACTS}/artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${SEMVER}-${env.BUILD_NUMBER}-${MILVUS_CLIENT}-e2e-logs \ - --release-name ${release_name}" - dir("${env.ARTIFACTS}") { - archiveArtifacts artifacts: "artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${SEMVER}-${env.BUILD_NUMBER}-${MILVUS_CLIENT}-e2e-logs.tar.gz", allowEmptyArchive: true + container('archive') { + script { + def helm_release_name = get_helm_release_name milvus_deployment_option + + tekton.archive milvus_deployment_option: milvus_deployment_option, + release_name: helm_release_name , + change_id: env.CHANGE_ID, + build_id: env.BUILD_ID } } } } } } - } - } } - post{ - unsuccessful { - container('jnlp') { - dir ('tests/scripts') { - script { - def authorEmail = sh(returnStdout: true, script: './get_author_email.sh ') - emailext subject: '$DEFAULT_SUBJECT', - body: '$DEFAULT_CONTENT', - recipientProviders: [developers(), culprits()], - replyTo: '$DEFAULT_REPLYTO', - to: "${authorEmail},devops@zilliz.com" - } - } - } - } - } } diff --git a/ci/jenkins/PR2.groovy b/ci/jenkins/PR2.groovy deleted file mode 100644 index 72864cb206..0000000000 --- a/ci/jenkins/PR2.groovy +++ /dev/null @@ -1,137 +0,0 @@ -@Library('jenkins-shared-library@v0.53.0') _ - -def pod = libraryResource 'io/milvus/pod/tekton-4am.yaml' -def milvus_helm_chart_version = '4.2.8' - -pipeline { - options { - skipDefaultCheckout true - parallelsAlwaysFailFast() - buildDiscarder logRotator(artifactDaysToKeepStr: '30') - preserveStashes(buildCount: 5) - disableConcurrentBuilds(abortPrevious: true) - } - agent { - kubernetes { - cloud '4am' - yaml pod - } - } - stages { - stage('meta') { - steps { - container('jnlp') { - script { - isPr = env.CHANGE_ID != null - gitMode = isPr ? 'merge' : 'fetch' - gitBaseRef = isPr ? "$env.CHANGE_TARGET" : "$env.BRANCH_NAME" - - get_helm_release_name = tekton.helm_release_name client: 'py', - changeId: "${env.CHANGE_ID}", - buildId:"${env.BUILD_ID}" - } - } - } - } - stage('build') { - steps { - container('tkn') { - script { - def job_name = tekton.run arch: 'amd64', - isPr: isPr, - gitMode: gitMode , - gitBaseRef: gitBaseRef, - pullRequestNumber: "$env.CHANGE_ID", - suppress_suffix_of_image_tag: true, - images: '["milvus","pytest","helm"]' - - milvus_image_tag = tekton.query_result job_name, 'milvus-image-tag' - pytest_image = tekton.query_result job_name, 'pytest-image-fqdn' - helm_image = tekton.query_result job_name, 'helm-image-fqdn' - } - } - } - post { - always { - container('tkn') { - script { - tekton.sure_stop() - } - } - } - } - } - stage('E2E Test') { - matrix { - agent { - kubernetes { - cloud '4am' - yaml pod - } - } - axes { - axis { - name 'milvus_deployment_option' - values 'standalone', 'distributed', 'standalone-kafka', 'distributed-streaming-service' - } - } - stages { - stage('E2E Test') { - steps { - container('tkn') { - script { - def helm_release_name = get_helm_release_name milvus_deployment_option - - if (milvus_deployment_option == 'distributed-streaming-service') { - try { - tekton.pytest helm_release_name: helm_release_name, - milvus_helm_version: milvus_helm_chart_version, - ciMode: 'e2e', - milvus_image_tag: milvus_image_tag, - pytest_image: pytest_image, - helm_image: helm_image, - milvus_deployment_option: milvus_deployment_option, - verbose: 'false' - } catch (Exception e) { - println e - } - } else { - tekton.pytest helm_release_name: helm_release_name, - milvus_helm_version: milvus_helm_chart_version, - ciMode: 'e2e', - milvus_image_tag: milvus_image_tag, - pytest_image: pytest_image, - helm_image: helm_image, - milvus_deployment_option: milvus_deployment_option, - verbose: 'false' - } - } - } - } - - post { - always { - container('tkn') { - script { - tekton.sure_stop() - } - } - - container('archive') { - script { - def helm_release_name = get_helm_release_name milvus_deployment_option - - tekton.archive milvus_deployment_option: milvus_deployment_option, - release_name: helm_release_name , - change_id: env.CHANGE_ID, - build_id: env.BUILD_ID - } - } - } - } - } - } - } - } - } -}