Run KinD Inside A Kubernetes Cluster For Continuous Integration (#5085)

**What type of PR is this?** - [x] Improvement **Which issue(s) this PR fixes:** Fixes #5038 **What this PR does / why we need it:** Run KinD Inside A Kubernetes Cluster For Continuous Integration
2024-11-30 02:48:45 +08:00 · 2021-04-29 16:45:17 +08:00 · 2021-04-29 16:45:17 +08:00 · 136cd7a704
commit 136cd7a704
parent 88e1d651cb
17 changed files with 137 additions and 579 deletions
--- a/.jenkins/modules/Build/Build.groovy
+++ b/.jenkins/modules/Build/Build.groovy
@ -1,20 +0,0 @@
-timeout(time: 20, unit: 'MINUTES') {
-
-    sh '. ./scripts/before-install.sh && ./scripts/check_cache.sh -l $CCACHE_ARTFACTORY_URL --cache_dir=\$CCACHE_DIR -f ccache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz || echo \"Ccache artfactory files not found!\"'
-    sh '. ./scripts/before-install.sh && ./scripts/check_cache.sh -l $GO_CACHE_ARTFACTORY_URL --cache_dir=\$(go env GOCACHE) -f go-cache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz || echo \"Go cache artfactory files not found!\"'
-    sh '. ./scripts/before-install.sh && ./scripts/check_cache.sh -l $THIRDPARTY_ARTFACTORY_URL --cache_dir=$CUSTOM_THIRDPARTY_PATH -f thirdparty-download.tar.gz || echo \"Thirdparty artfactory files not found!\"'
-    sh '. ./scripts/before-install.sh && go clean --modcache && ./scripts/check_cache.sh -l $GO_MOD_ARTFACTORY_URL --cache_dir=\$GOPATH/pkg/mod -f milvus-go-mod-\$(md5sum go.mod).tar.gz || echo \"Go mod artfactory files not found!\"'
-
-    // Zero the cache statistics (but not the configuration options)
-    sh 'ccache -z'
-    sh '. ./scripts/before-install.sh && make install'
-    sh 'echo -e "===\n=== ccache statistics after build\n===" && ccache --show-stats'
-
-
-    withCredentials([usernamePassword(credentialsId: "${env.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
-        sh '. ./scripts/before-install.sh && ./scripts/update_cache.sh -l $CCACHE_ARTFACTORY_URL --cache_dir=\$CCACHE_DIR -f ccache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz -u ${USERNAME} -p ${PASSWORD}'
-        sh '. ./scripts/before-install.sh && ./scripts/update_cache.sh -l $GO_CACHE_ARTFACTORY_URL --cache_dir=\$(go env GOCACHE) -f go-cache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz -u ${USERNAME} -p ${PASSWORD}'
-        sh '. ./scripts/before-install.sh && ./scripts/update_cache.sh -l $THIRDPARTY_ARTFACTORY_URL --cache_dir=$CUSTOM_THIRDPARTY_PATH -f thirdparty-download.tar.gz -u ${USERNAME} -p ${PASSWORD}'
-        sh '. ./scripts/before-install.sh && ./scripts/update_cache.sh -l $GO_MOD_ARTFACTORY_URL --cache_dir=\$GOPATH/pkg/mod -f milvus-go-mod-\$(md5sum go.mod).tar.gz -u ${USERNAME} -p ${PASSWORD}'
-    }
-}
--- a/.jenkins/modules/Package/Package.groovy
+++ b/.jenkins/modules/Package/Package.groovy
@ -1,7 +0,0 @@
-sh 'tar -zcvf ./${PACKAGE_NAME} ./bin ./configs ./lib'
-withCredentials([usernamePassword(credentialsId: "${env.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) {
-    def uploadStatus = sh(returnStatus: true, script: 'curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -T ./${PACKAGE_NAME} ${PACKAGE_ARTFACTORY_URL}')
-    if (uploadStatus != 0) {
-        error("\" ${PACKAGE_NAME} \" upload to \" ${PACKAGE_ARTFACTORY_URL} \" failed!")
-    }
-}
--- a/.jenkins/modules/Publish/Publish.groovy
+++ b/.jenkins/modules/Publish/Publish.groovy
@ -1,25 +0,0 @@
-withCredentials([usernamePassword(credentialsId: "${env.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) {
-    def downloadStatus = sh(returnStatus: true, script: 'curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -O ${PACKAGE_ARTFACTORY_URL}')
-
-    if (downloadStatus != 0) {
-        error("\" Download \" ${PACKAGE_ARTFACTORY_URL} \" failed!")
-    }
-}
-
-sh 'tar zxvf ${PACKAGE_NAME}'
-
-try {
-    withCredentials([usernamePassword(credentialsId: "${env.DOCKER_CREDENTIALS_ID}", usernameVariable: 'DOCKER_USERNAME', passwordVariable: 'DOCKER_PASSWORD')]) {
-        sh 'docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD} ${DOKCER_REGISTRY_URL}'
-
-        sh 'docker pull registry.zilliz.com/milvus/openblas:latest || true'
-        sh "docker build -f build/docker/milvus/Dockerfile -t ${TARGET_REPO}/milvus:${TARGET_TAG} ."
-        sh "docker push ${TARGET_REPO}/milvus:${TARGET_TAG}"
-    }
-} catch (exc) {
-    throw exc
-} finally {
-    sh 'docker logout ${DOKCER_REGISTRY_URL}'
-    sh "docker rmi -f \$(docker images | grep '<none>' | awk '{print \$3}') || true"
-    sh "docker rmi ${TARGET_REPO}/milvus:${TARGET_TAG}"
-}
--- a/.jenkins/modules/Regression/CleanupPythonRegression.groovy
+++ b/.jenkins/modules/Regression/CleanupPythonRegression.groovy
@ -1,8 +0,0 @@
-container('deploy-env') {
-    def helmStatus = sh script: "helm status -n ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME}", returnStatus: true
-    if (!helmStatus) {
-        sh "helm uninstall -n ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME}"
-        def labels = "app.kubernetes.io/instance=${env.HELM_RELEASE_NAME}"
-        sh "kubectl delete pvc -n ${env.HELM_RELEASE_NAMESPACE} \$(kubectl get pvc -n ${env.HELM_RELEASE_NAMESPACE} -l ${labels} -o jsonpath='{range.items[*]}{.metadata.name} ')"
-    }
-}
--- a/.jenkins/modules/Regression/PythonRegression.groovy
+++ b/.jenkins/modules/Regression/PythonRegression.groovy
@ -1,88 +0,0 @@
-def isTimeTriggeredBuild = currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause').size() != 0
-def regressionTimeout = isTimeTriggeredBuild ? "120" : "60"
-timeout(time: "${regressionTimeout}", unit: 'MINUTES') {
-    container('deploy-env') {
-        dir ('milvus-helm-chart') {
-            sh " helm version && \
-                 helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts && \
-                 helm repo add bitnami https://charts.bitnami.com/bitnami && \
-                 helm repo add minio https://helm.min.io/ && \
-                 helm repo update"
-
-            def milvusHelmURL = "https://github.com/zilliztech/milvus-helm-charts.git"
-            checkout([$class: 'GitSCM', branches: [[name: "${env.HELM_BRANCH}"]], userRemoteConfigs: [[url: "${milvusHelmURL}"]]])
-
-            dir ('charts/milvus-ha') {
-                sh script: "kubectl create namespace ${env.HELM_RELEASE_NAMESPACE}", returnStatus: true
-
-                def helmCMD = ""
-                if ("${REGRESSION_SERVICE_TYPE}" == "distributed") {
-                    helmCMD = "helm install --wait --timeout 300s \
-                                   --set standalone.enabled=false \
-                                   --set image.all.repository=${env.TARGET_REPO}/milvus \
-                                   --set image.all.tag=${env.TARGET_TAG} \
-                                   --set image.all.pullPolicy=Always \
-                                   --set logsPersistence.enabled=true \
-                                   --set logsPersistence.mountPath=/milvus/logs \
-                                   --namespace ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME} ."
-                } else {
-                    helmCMD = "helm install --wait --timeout 300s \
-                                   --set standalone.enabled=true \
-                                   --set image.all.repository=${env.TARGET_REPO}/milvus \
-                                   --set image.all.tag=${env.TARGET_TAG} \
-                                   --set image.all.pullPolicy=Always \
-                                   --set logsPersistence.enabled=true \
-                                   --set logsPersistence.mountPath=/milvus/logs \
-                                   --namespace ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME} ."
-                }
-
-                try {
-                    sh "${helmCMD}"
-                } catch (exc) {
-                    def helmStatusCMD = "helm get manifest -n ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME} | kubectl describe -n ${env.HELM_RELEASE_NAMESPACE} -f - && \
-                                         helm status -n ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME}"
-                    sh script: helmStatusCMD, returnStatus: true
-                    throw exc
-                }
-            }
-        }
-    }
-
-    container('test-env') {
-        try {
-            dir ('tests/python_test') {
-                sh "python3 -m pip install --no-cache-dir -r requirements.txt"
-                if (isTimeTriggeredBuild) {
-                    echo "This is Cron Job!"
-                    sh "pytest -n 4 --ip ${env.HELM_RELEASE_NAME}-milvus-ha.${env.HELM_RELEASE_NAMESPACE}.svc.cluster.local"
-                } else {
-                    sh "pytest --tags=smoke -n 4 --ip ${env.HELM_RELEASE_NAME}-milvus-ha.${env.HELM_RELEASE_NAMESPACE}.svc.cluster.local"
-                }
-            }
-        } catch (exc) {
-            echo 'PyTest Regression Failed !'
-            throw exc
-        } finally {
-            container('deploy-env') {
-                def milvusLabels = ""
-                if ("${REGRESSION_SERVICE_TYPE}" == "distributed") {
-                    milvusLabels = "app.kubernetes.io/instance=${env.HELM_RELEASE_NAME},component=proxyservice"
-                } else {
-                    milvusLabels = "app.kubernetes.io/instance=${env.HELM_RELEASE_NAME},component=standalone"
-                }
-                def etcdLabels = "app.kubernetes.io/instance=${env.HELM_RELEASE_NAME},app.kubernetes.io/name=etcd"
-                def pulsarLabels = "app.kubernetes.io/instance=${env.HELM_RELEASE_NAME},component=pulsar"
-                def componentLabels = "release=${env.HELM_RELEASE_NAME}"
-                def namespace = "${env.HELM_RELEASE_NAMESPACE}"
-                def artifactsPath = "${env.DEV_TEST_ARTIFACTS_PATH}"
-
-                sh "mkdir -p $artifactsPath"
-                sh "kubectl cp -n ${env.HELM_RELEASE_NAMESPACE} \$(kubectl get pod -n ${env.HELM_RELEASE_NAMESPACE} -l ${milvusLabels} -o jsonpath='{range.items[0]}{.metadata.name}'):logs $artifactsPath"
-                sh "for pod in \$(kubectl get pod -n $namespace -l ${etcdLabels} -o jsonpath='{range.items[*]}{.metadata.name} '); do kubectl logs --all-containers -n $namespace \$pod > $artifactsPath/\$pod.log; done"
-                sh "for pod in \$(kubectl get pod -n $namespace -l ${pulsarLabels} -o jsonpath='{range.items[*]}{.metadata.name} '); do kubectl logs --all-containers -n $namespace \$pod > $artifactsPath/\$pod.log; done"
-                sh "for pod in \$(kubectl get pod -n $namespace -l ${componentLabels} -o jsonpath='{range.items[*]}{.metadata.name} '); do kubectl logs --all-containers -n $namespace \$pod > $artifactsPath/\$pod.log; done"
-                archiveArtifacts artifacts: "$artifactsPath/**", allowEmptyArchive: true
-            }
-        }
-    }
-}
--- a/.jenkins/modules/UnitTest/UnitTest.groovy
+++ b/.jenkins/modules/UnitTest/UnitTest.groovy
@ -1,3 +0,0 @@
-timeout(time: 5, unit: 'MINUTES') {
-	sh 'make unittest'
-}
--- a/build/ci/jenkins/Jenkinsfile
+++ b/build/ci/jenkins/Jenkinsfile
@ -1,5 +1,4 @@
 #!/usr/bin/env groovy
-@Library('mpl') _

 String cron_timezone = "TZ=Asia/Shanghai"
 String cron_string = BRANCH_NAME == "r0.3-test" ? "50 20,22,0,2,4,6,11,16 * * * " : ""
@ -12,103 +11,75 @@ pipeline {
    }
    options {
        timestamps()
+        timeout(time: 1, unit: 'HOURS')
        // parallelsAlwaysFailFast()
    }
-    environment {
-        PROJECT_NAME = "milvus"
-        SEMVER = "${BRANCH_NAME.contains('/') ? BRANCH_NAME.substring(BRANCH_NAME.lastIndexOf('/') + 1) : BRANCH_NAME}"
-        BUILD_TYPE = "Release"
-        LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase()
-        PACKAGE_VERSION = "${SEMVER}-${LOWER_BUILD_TYPE}"
-        PACKAGE_NAME = "${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz"
-        JFROG_CREDENTIALS_ID = "1a527823-d2b7-44fd-834b-9844350baf14"
-        JFROG_ARTFACTORY_URL = "http://192.168.1.201/artifactory/milvus"
-        PACKAGE_ARTFACTORY_URL = "${JFROG_ARTFACTORY_URL}/${PROJECT_NAME}/package/${PACKAGE_NAME}"
-        DOCKER_CREDENTIALS_ID = "ba070c98-c8cc-4f7c-b657-897715f359fc"
-        DOKCER_REGISTRY_URL = "registry.zilliz.com"
-        SOURCE_REPO = "${DOKCER_REGISTRY_URL}/milvus"
-        TARGET_REPO = "${DOKCER_REGISTRY_URL}/milvus"
-        SOURCE_TAG = "${CHANGE_TARGET ? CHANGE_TARGET : SEMVER}-${LOWER_BUILD_TYPE}"
-        TARGET_TAG = "${SEMVER}-${LOWER_BUILD_TYPE}"
-        DOCKER_BUILDKIT = 1
-    }
    stages {
-        stage ('Build and UnitTest') {
-            agent {
-                kubernetes {
-                    label "${env.PROJECT_NAME}-${SEMVER}-${env.BUILD_NUMBER}-build"
-                    defaultContainer 'build-env'
-                    customWorkspace '/home/jenkins/agent/workspace'
-                    yamlFile "build/ci/jenkins/pod/build-env.yaml"
-                }
-            }
-            environment {
-                PULSAR_ADDRESS = "pulsar://127.0.0.1:6650"
-                ETCD_ADDRESS = "127.0.0.1:2379"
-                CCACHE_ARTFACTORY_URL = "${JFROG_ARTFACTORY_URL}/milvus/ccache"
-                THIRDPARTY_ARTFACTORY_URL = "${JFROG_ARTFACTORY_URL}/milvus/thirdparty"
-                CUSTOM_THIRDPARTY_PATH = "${WORKSPACE}/3rdparty_download"
-                GO_CACHE_ARTFACTORY_URL = "${JFROG_ARTFACTORY_URL}/milvus/go-cache"
-                GO_MOD_ARTFACTORY_URL = "${JFROG_ARTFACTORY_URL}/milvus/go-mod"
-            }
-            steps {
-                container('build-env') {
-                    MPLModule('Build')
-                    MPLModule('Package')
-                    // MPLModule('UnitTest')
-                }
-            }
-        }
-
-        stage ('Publish Docker Images') {
-            agent {
-                kubernetes {
-                    label "${env.PROJECT_NAME}-${SEMVER}-${env.BUILD_NUMBER}-publish"
-                    defaultContainer 'publish-images'
-                    yamlFile "build/ci/jenkins/pod/docker-pod.yaml"
-                }
-            }
-            steps {
-                container('publish-images') {
-                    MPLModule('Publish')
-                }
-            }
-        }
-
-        stage ('Dev Test') {
+        stage ('E2E Test') {
            matrix {
                axes {
                    axis {
-                        name 'REGRESSION_SERVICE_TYPE'
+                        name 'MILVUS_SERVER_TYPE'
                        values 'standalone', 'distributed'
                    }
                }
-                environment {
-                    HELM_BRANCH = "2.0"
-                    HELM_RELEASE_NAMESPACE = "dev"
-                    HELM_RELEASE_NAME = "${PROJECT_NAME}-${SEMVER}-${env.BUILD_NUMBER}-${REGRESSION_SERVICE_TYPE}".replaceAll("\\.", "-").replaceAll("_", "-").toLowerCase()
-                    DEV_TEST_ARTIFACTS_PATH = "artifacts"
-                }
                agent {
                    kubernetes {
-                        label "${env.PROJECT_NAME}-${SEMVER}-${env.BUILD_NUMBER}-${REGRESSION_SERVICE_TYPE}-devtest"
-                        defaultContainer 'test-env'
-                        yamlFile "build/ci/jenkins/pod/test-env.yaml"
+                        label "milvus-e2e-test-kind"
+                        defaultContainer 'main'
+                        yamlFile "build/ci/jenkins/pod/krte.yaml"
+                        customWorkspace '/home/jenkins/agent/workspace'
+                        // We allow this pod to remain active for a while, later jobs can
+                        // reuse cache in previous created nodes.
+                        idleMinutes 120
                    }
                }
+                environment {
+                    IMAGE_REPO = "registry.zilliz.com/milvus"
+                    DOCKER_BUILDKIT = 1
+                    DOCKER_VOLUME_DIRECTORY = "/mnt/disk/.docker"
+                    ARTIFACTS = "${env.WORKSPACE}/artifacts"
+                }
                stages {
                    stage('Test') {
                        steps {
-                            MPLModule('Python Regression')
+                            container('main') {
+                                dir ('tests/scripts') {
+                                    script {
+                                        def standaloneEnabled = "true"
+                                        if ("${MILVUS_SERVER_TYPE}" == "distributed") {
+                                            standaloneEnabled = "false"
                                        }
+
+                                        if (isTimeTriggeredBuild()) {
+                                            echo "This is Cron Job!"
+                                            sh "MILVUS_STANDALONE_ENABLED=${standaloneEnabled} ./e2e-k8s.sh --node-image registry.zilliz.com/kindest/node:v1.20.2"
+                                        } else {
+                                            sh "MILVUS_STANDALONE_ENABLED=${standaloneEnabled} ./e2e-k8s.sh --node-image registry.zilliz.com/kindest/node:v1.20.2 --test-extra-arg \"--tags=smoke\""
                                        }
                                    }
-                post {
-                    cleanup {
-                        MPLModule('Cleanup Python Regression')
-                    }
                                }
                            }
                        }
                    }
                }
+                post {
+                    always {
+                        container('main') {
+                            script {
+                                dir("${env.ARTIFACTS}") {
+                                    sh "tar -zcvf artifacts-milvus-${MILVUS_SERVER_TYPE}-e2e-kind.tar.gz ./kind --remove-files || true"
+                                    archiveArtifacts artifacts: "artifacts-milvus-${MILVUS_SERVER_TYPE}-e2e-kind.tar.gz", allowEmptyArchive: true
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+boolean isTimeTriggeredBuild() {
+    return (currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause').size() != 0) ? true : false;
+}
--- a/build/ci/jenkins/pod/build-env.yaml
+++ b/build/ci/jenkins/pod/build-env.yaml
@ -1,43 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: build-env
-  labels:
-    app: milvus
-    componet: build-env
-spec:
-  containers:
-  - name: build-env
-    image: milvusdb/milvus-dev:amd64-ubuntu18.04-20210426-055946
-    env:
-    - name: OS_NAME
-      value: "ubuntu18.04"
-    - name: BUILD_ENV_IMAGE_ID
-      value: "14fd5a2cff5485257e79421a119ece1d6613587c2336fc18c7a8d12c791db0ef"
-    securityContext:
-       runAsUser: 1000
-       runAsGroup: 1000
-    tty: true
-    resources:
-      limits:
-        memory: "16Gi"
-        cpu: "8.0"
-      requests:
-        memory: "8Gi"
-        cpu: "4.0"
-  - name: etcd
-    image: quay.io/coreos/etcd:v3.4.13
-    env:
-    - name: ETCD_LISTEN_CLIENT_URLS
-      value: "http://0.0.0.0:2379"
-    - name: ETCD_ADVERTISE_CLIENT_URLS
-      value: "http://0.0.0.0:2379"
-    ports:
-    - containerPort: 2379
-      name: etcd
-  - name: pulsar
-    image: apachepulsar/pulsar:2.6.1
-    ports:
-    - containerPort: 6650
-      name: pulsar
-    command: ["bin/pulsar", "standalone"]
--- a/build/ci/jenkins/pod/docker-pod.yaml
+++ b/build/ci/jenkins/pod/docker-pod.yaml
@ -1,34 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  labels:
-    app: publish
-    componet: docker
-spec:
-  containers:
-  - name: publish-images
-    image: registry.zilliz.com/library/dind-compose:v0.1
-    imagePullPolicy: Always
-    securityContext:
-      privileged: true
-    resources:
-      limits:
-        memory: "8Gi"
-        cpu: "2"
-      requests:
-        memory: "2Gi"
-        cpu: "1"
-    volumeMounts:
-    - name: docker-graph-storage
-      mountPath: /var/lib/docker
-  volumes:
-    - name: docker-graph-storage
-      emptyDir: {}
-  tolerations:
-  - key: dedicated
-    operator: Equal
-    value: milvus
-    effect: NoSchedule
-
-
-
--- a/build/ci/jenkins/pod/krte.yaml
+++ b/build/ci/jenkins/pod/krte.yaml
@ -0,0 +1,66 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: milvus-e2e
+spec:
+  containers:
+  - name: main
+    image: registry.zilliz.com/milvus/krte:v20210414-bd36693
+    env:
+    - name: DOCKER_IN_DOCKER_ENABLED
+      value: "true"
+    securityContext:
+      privileged: true
+    tty: true
+    args: ["cat"]
+    resources:
+      limits:
+        memory: 14Gi
+        ephemeral-storage: "20Gi"
+      requests:
+        cpu: "4"
+        memory: 8Gi
+        ephemeral-storage: "20Gi"
+    volumeMounts:
+    - mountPath: /docker-graph
+      name: docker-graph
+    - mountPath: /var/lib/docker
+      name: docker-root
+    - mountPath: /lib/modules
+      name: modules
+      readOnly: true
+    - mountPath: /sys/fs/cgroup
+      name: cgroup
+    - mountPath: /mnt/disk/.docker
+      name: build-cache
+      subPath: docker-volume
+  volumes:
+  - emptyDir: {}
+    name: docker-graph
+  - emptyDir: {}
+    name: docker-root
+  - hostPath:
+      path: /tmp/krte/cache
+      type: DirectoryOrCreate
+    name: build-cache
+  - hostPath:
+      path: /lib/modules
+      type: Directory
+    name: modules
+  - hostPath:
+      path: /sys/fs/cgroup
+      type: Directory
+    name: cgroup
+  affinity:
+    podAntiAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+      - weight: 100
+        podAffinityTerm:
+          labelSelector:
+            matchExpressions:
+            - key: app
+              operator: In
+              values:
+              - milvus-e2e
+          topologyKey: kubernetes.io/hostname
--- a/build/ci/jenkins/pod/test-env.yaml
+++ b/build/ci/jenkins/pod/test-env.yaml
@ -1,38 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  labels:
-    app: milvus
-    componet: test-env
-spec:
-  containers:
-  - name: test-env
-    image: milvusdb/pytest:20210322-201a528
-    command:
-    - cat
-    tty: true
-    resources:
-      limits:
-        memory: "8Gi"
-        cpu: "4.0"
-      requests:
-        memory: "4Gi"
-        cpu: "2.0"
-  - name: deploy-env
-    image: registry.zilliz.com/library/helm3:v0.1
-    command:
-    - cat
-    tty: true
-    volumeMounts:
-    - name: kubeconf
-      mountPath: /root/.kube/
-      readOnly: true
-  volumes:
-  - name: kubeconf
-    secret:
-      secretName: test-cluster-config
-  tolerations:
-  - key: dedicated
-    operator: Equal
-    value: milvus
-    effect: NoSchedule
--- a/build/config/topology/trustworthy-jwt.yaml
+++ b/build/config/topology/trustworthy-jwt.yaml
@ -4,11 +4,29 @@ apiVersion: kind.x-k8s.io/v1alpha4
 networking:
  ipFamily: "ipv4"
  kubeProxyMode: "iptables"
+  podSubnet: "10.245.0.0/16"
+  serviceSubnet: "10.246.0.0/16"
 nodes:
 - role: control-plane
 - role: worker
 - role: worker
 kubeadmConfigPatches:
+  - |
+    apiVersion: kubeadm.k8s.io/v1beta2
+    kind: JoinConfiguration
+    metadata:
+      name: config
+    nodeRegistration:
+      kubeletExtraArgs:
+        cgroup-root: "/kubelet"
+  - |
+    apiVersion: kubeadm.k8s.io/v1beta2
+    kind: InitConfiguration
+    metadata:
+      name: config
+    nodeRegistration:
+      kubeletExtraArgs:
+        cgroup-root: "/kubelet"
  - |
    apiVersion: kubeadm.k8s.io/v1beta2
    kind: ClusterConfiguration
--- a/cmd/standalone/main.go
+++ b/cmd/standalone/main.go
@ -42,13 +42,13 @@ func initLogCfg() log.Config {
 	logCfg.File.MaxDays = 10

 	// FIXME(wxyu): Load from config files
-	//logCfg.File.Filename = ""
-	ciFileDir := "/milvus/logs/"
-	if _, err := os.Stat(ciFileDir); err == nil {
-		logCfg.File.Filename = ciFileDir + "standalone.log"
-	} else {
-		logCfg.File.Filename = "/tmp/milvus/standalone.log"
-	}
+	logCfg.File.Filename = ""
+	// ciFileDir := "/milvus/logs/"
+	// if _, err := os.Stat(ciFileDir); err == nil {
+	// 	logCfg.File.Filename = ciFileDir + "standalone.log"
+	// } else {
+	// 	logCfg.File.Filename = "/tmp/milvus/standalone.log"
+	// }
 	return logCfg
 }

--- a/scripts/before-install.sh
+++ b/scripts/before-install.sh
@ -1,12 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-export CCACHE_COMPRESS=${CCACHE_COMPRESS:="1"}
-export CCACHE_COMPRESSLEVEL=${CCACHE_COMPRESSLEVEL:="5"}
-export CCACHE_COMPILERCHECK=${CCACHE_COMPILERCHECK:="content"}
-export CCACHE_MAXSIZE=${CCACHE_MAXSIZE:="2G"}
-export CCACHE_DIR=${CCACHE_DIR:="${HOME}/.ccache"}
-export GOPROXY="https://goproxy.cn"
-
-set +ex
--- a/scripts/check_cache.sh
+++ b/scripts/check_cache.sh
@ -1,114 +0,0 @@
-#!/bin/bash
-
-HELP="
-Usage:
-  $0 [flags] [Arguments]
-
-    -l [ARTIFACTORY_URL]          Artifactory URL
-    --cache_dir=[CACHE_DIR]       Cache directory
-    -f [FILE] or --file=[FILE]    Cache compress package file
-    -h or --help                  Print help information
-
-
-Use \"$0  --help\" for more information about a given command.
-"
-
-ARGS=$(getopt -o "l:f:h" -l "cache_dir::,file::,help" -n "$0" -- "$@")
-
-eval set -- "${ARGS}"
-
-while true ; do
-        case "$1" in
-                -l)
-                        # o has an optional argument. As we are in quoted mode,
-                        # an empty parameter will be generated if its optional
-                        # argument is not found.
-                        case "$2" in
-                                "") echo "Option Artifactory URL, no argument"; exit 1 ;;
-                                *)  ARTIFACTORY_URL=$2 ; shift 2 ;;
-                        esac ;;
-                --cache_dir)
-                        case "$2" in
-                                "") echo "Option cache_dir, no argument"; exit 1 ;;
-                                *)  CACHE_DIR=$2 ; shift 2 ;;
-                        esac ;;
-                -f|--file)
-                        case "$2" in
-                                "") echo "Option file, no argument"; exit 1 ;;
-                                *)  PACKAGE_FILE=$2 ; shift 2 ;;
-                        esac ;;
-                -h|--help) echo -e "${HELP}" ; exit 0 ;;
-                --) shift ; break ;;
-                *) echo "Internal error!" ; exit 1 ;;
-        esac
-done
-
-# Set defaults for vars modified by flags to this script
-BRANCH_NAMES=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's=[a-zA-Z]*\/==g' | awk -F", " '{$1=""; print $0}')
-
-if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then
-    echo "You have not input ARTIFACTORY_URL !"
-    exit 1
-fi
-
-if [[ -z "${CACHE_DIR}" ]]; then
-    echo "You have not input CACHE_DIR !"
-    exit 1
-fi
-
-if [[ -z "${PACKAGE_FILE}" ]]; then
-    echo "You have not input PACKAGE_FILE !"
-    exit 1
-fi
-
-function check_cache() {
-    BRANCH=$1
-    echo "fetching ${BRANCH}/${PACKAGE_FILE}"
-    wget -q --spider "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}"
-    return $?
-}
-
-function download_file() {
-    BRANCH=$1
-    wget -q "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}" && \
-    mkdir -p "${CACHE_DIR}" && \
-    tar zxf "${PACKAGE_FILE}" -C "${CACHE_DIR}" && \
-    rm ${PACKAGE_FILE}
-    return $?
-}
-
-if [[ -n "${CHANGE_TARGET}" && "${BRANCH_NAME}" =~ "PR-" ]];then
-    check_cache ${CHANGE_TARGET}
-    if [[ $? == 0 ]];then
-        download_file ${CHANGE_TARGET}
-        if [[ $? == 0 ]];then
-            echo "found cache"
-            exit 0
-        fi
-    fi
-
-    check_cache ${BRANCH_NAME}
-    if [[ $? == 0 ]];then
-        download_file ${BRANCH_NAME}
-        if [[ $? == 0 ]];then
-            echo "found cache"
-            exit 0
-        fi
-    fi
-fi
-
-for CURRENT_BRANCH in ${BRANCH_NAMES}
-do
-    if [[ "${CURRENT_BRANCH}" != "HEAD" ]];then
-        check_cache ${CURRENT_BRANCH}
-        if [[ $? == 0 ]];then
-            download_file ${CURRENT_BRANCH}
-            if [[ $? == 0 ]];then
-                echo "found cache"
-                exit 0
-            fi
-        fi
-    fi
-done
-
-echo "could not download cache" && exit 1
--- a/scripts/update_cache.sh
+++ b/scripts/update_cache.sh
@ -1,104 +0,0 @@
-#!/bin/bash
-
-HELP="
-Usage:
-  $0 [flags] [Arguments]
-
-    -l [ARTIFACTORY_URL]          Artifactory URL
-    --cache_dir=[CACHE_DIR]       Cache directory
-    -f [FILE] or --file=[FILE]    Cache compress package file
-    -u [USERNAME]                 Artifactory Username
-    -p [PASSWORD]                 Artifactory Password
-    -h or --help                  Print help information
-
-
-Use \"$0  --help\" for more information about a given command.
-"
-
-ARGS=$(getopt -o "l:f:u:p:h" -l "cache_dir::,file::,help" -n "$0" -- "$@")
-
-eval set -- "${ARGS}"
-
-while true ; do
-        case "$1" in
-                -l)
-                        # o has an optional argument. As we are in quoted mode,
-                        # an empty parameter will be generated if its optional
-                        # argument is not found.
-                        case "$2" in
-                                "") echo "Option Artifactory URL, no argument"; exit 1 ;;
-                                *)  ARTIFACTORY_URL=$2 ; shift 2 ;;
-                        esac ;;
-                --cache_dir)
-                        case "$2" in
-                                "") echo "Option cache_dir, no argument"; exit 1 ;;
-                                *)  CACHE_DIR=$2 ; shift 2 ;;
-                        esac ;;
-                -u)
-                        case "$2" in
-                                "") echo "Option Username, no argument"; exit 1 ;;
-                                *)  USERNAME=$2 ; shift 2 ;;
-                        esac ;;
-                -p)
-                        case "$2" in
-                                "") echo "Option Password, no argument"; exit 1 ;;
-                                *)  PASSWORD=$2 ; shift 2 ;;
-                        esac ;;
-                -f|--file)
-                        case "$2" in
-                                "") echo "Option file, no argument"; exit 1 ;;
-                                *)  PACKAGE_FILE=$2 ; shift 2 ;;
-                        esac ;;
-                -h|--help) echo -e "${HELP}" ; exit 0 ;;
-                --) shift ; break ;;
-                *) echo "Internal error!" ; exit 1 ;;
-        esac
-done
-
-# Set defaults for vars modified by flags to this script
-BRANCH_NAME=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's/.*, //' | sed 's=[a-zA-Z]*\/==g')
-
-if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then
-    echo "You have not input ARTIFACTORY_URL !"
-    exit 1
-fi
-
-if [[ ! -d "${CACHE_DIR}" ]]; then
-    echo "\"${CACHE_DIR}\" directory does not exist !"
-    exit 1
-fi
-
-if [[ -z "${PACKAGE_FILE}" ]]; then
-    echo "You have not input PACKAGE_FILE !"
-    exit 1
-fi
-
-function check_cache() {
-    BRANCH=$1
-    wget -q --spider "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}"
-    return $?
-}
-
-if [[ -n "${CHANGE_TARGET}" && "${BRANCH_NAME}" =~ "PR-" ]]; then
-    check_cache ${CHANGE_TARGET}
-    if [[ $? == 0 ]];then
-        echo "Skip Update cache package ..." && exit 0
-    fi
-fi
-
-if [[ "${BRANCH_NAME}" != "HEAD" ]];then
-    REMOTE_PACKAGE_PATH="${ARTIFACTORY_URL}/${BRANCH_NAME}"
-    echo "Updating cache package file: ${PACKAGE_FILE}"
-    tar zcf ./"${PACKAGE_FILE}" -C "${CACHE_DIR}" .
-    echo "Uploading cache package file ${PACKAGE_FILE} to ${REMOTE_PACKAGE_PATH}"
-    curl -u"${USERNAME}":"${PASSWORD}" -T "${PACKAGE_FILE}" "${REMOTE_PACKAGE_PATH}"/"${PACKAGE_FILE}"
-    if [[ $? == 0 ]];then
-        echo "Uploading cache package file success !"
-        exit 0
-    else
-        echo "Uploading cache package file fault !"
-        exit 1
-    fi
-fi
-
-echo "Skip Update cache package ..."
--- a/tests/scripts/e2e-k8s.sh
+++ b/tests/scripts/e2e-k8s.sh
@ -272,7 +272,6 @@ if [[ -z "${SKIP_BUILD:-}" ]]; then

  trace "setup kind registry" setup_kind_registry
  pushd "${ROOT}"
-    source "${ROOT}/scripts/before-install.sh"
    trace "build milvus" "${ROOT}/build/builder.sh" /bin/bash -c "${BUILD_COMMAND}"
    trace "build milvus image" docker build -f "${ROOT}/build/docker/milvus/Dockerfile" -t "${MILVUS_IMAGE_REPO}:${MILVUS_IMAGE_TAG}" .
    trace "push milvus image" docker push "${MILVUS_IMAGE_REPO}:${MILVUS_IMAGE_TAG}"