Optimize regression workflow

Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com>

.github/workflows/publish-test-images.yaml

@@ -6,12 +6,14 @@ on:
   push:
     # file paths to consider in the event. Optional; defaults to all.
     paths:
-      - 'build/docker/test/**'
+      - 'build/docker/test/Dockerfile'
+      - 'tests/python_test/requirements.txt'
       - '.github/workflows/publish-test-images.yaml'
   pull_request:
     # file paths to consider in the event. Optional; defaults to all.
     paths:
-      - 'build/docker/test/**'
+      - 'build/docker/test/Dockerfile'
+      - 'tests/python_test/requirements.txt'
       - '.github/workflows/publish-test-images.yaml'
 
 jobs:
@@ -33,24 +35,19 @@ jobs:
         run: |
           echo "::set-output name=version::$(date +%Y%m%d)"
           echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
-      - name: Docker Pull
-        shell: bash
-        working-directory: build/docker/test
-        run: |
-          docker-compose pull --ignore-pull-failures regression_standalone
       - name: Docker Build
         shell: bash
-        working-directory: build/docker/test
+        working-directory: tests/python_test
         run: |
-          TARGET_TAG=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} docker-compose build regression_standalone
+          docker build -t milvusdb/pytest:${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} .
-          TARGET_TAG=latest docker-compose build regression_standalone
+          docker tag milvusdb/pytest:${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} milvusdb/pytest:latest
       - name: Docker Push
         if: success() && github.event_name == 'push' && github.repository == 'zilliztech/milvus-distributed'
         continue-on-error: true
         shell: bash
-        working-directory: build/docker/test
         run: |
           docker login -u ${{ secrets.DOCKERHUB_USER }} \
                        -p ${{ secrets.DOCKERHUB_TOKEN }}
-          TARGET_TAG=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} docker-compose push regression_standalone
+          docker push milvusdb/pytest:${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}
-          TARGET_TAG=latest docker-compose push regression_standalone
+          docker push milvusdb/pytest:latest
+          echo "Push milvusdb/pytest:${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} Succeeded"

.jenkins/modules/Regression/PythonRegression.groovy

@@ -48,6 +48,7 @@ timeout(time: 60, unit: 'MINUTES') {
     container('test-env') {
         try {
             dir ('tests/python_test') {
+                sh "python3 -m pip install --no-cache-dir pymilvus-distributed==${env.PYMILVUS_VERSION}"
                 sh "pytest --tags=0331 --ip ${env.HELM_RELEASE_NAME}-milvus-ha.${env.HELM_RELEASE_NAMESPACE}.svc.cluster.local"
             }
         } catch (exc) {

build/ci/jenkins/Jenkinsfile

@@ -84,6 +84,7 @@ pipeline {
                 }
                 environment {
                     HELM_BRANCH = "main"
+                    PYMILVUS_VERSION = "0.0.42"
                     HELM_RELEASE_NAMESPACE = "dev"
                     HELM_RELEASE_NAME = "${PROJECT_NAME}-${SEMVER}-${env.BUILD_NUMBER}-${REGRESSION_SERVICE_TYPE}".replaceAll("\\.", "-").replaceAll("_", "-").toLowerCase()
                     DEV_TEST_ARTIFACTS_PATH = "artifacts"

build/ci/jenkins/pod/test-env.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   containers:
   - name: test-env
-    image: milvusdb/pytest:20210311-2afcc45
+    image: milvusdb/pytest:20210311-bfdffdd
     command:
     - cat
     tty: true

build/docker/test/.env

@@ -1,4 +0,0 @@
-SOURCE_REPO=milvusdb
-TARGET_REPO=milvusdb
-SOURCE_TAG=20210311-2afcc45
-TARGET_TAG=20210311-2afcc45

build/docker/test/Dockerfile

@@ -1,18 +0,0 @@
-# Copyright (C) 2019-2020 Zilliz. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-# or implied. See the License for the specific language governing permissions and limitations under the License.
-
-FROM python:3.6.8-jessie
-
-COPY ./tests/python_test/requirements.txt /requirements.txt
-
-RUN python3 -m pip install --no-cache-dir -r /requirements.txt
-
-CMD ["tail", "-f", "/dev/null"]

build/docker/test/docker-compose.yml

@@ -1,35 +0,0 @@
-version: '3.5'
-
-services:
-  regression_distributed:
-    image: ${TARGET_REPO}/pytest:${TARGET_TAG}
-    build:
-      context: ../../../
-      dockerfile: build/docker/test/Dockerfile
-      cache_from:
-        - ${SOURCE_REPO}/pytest:${SOURCE_TAG}
-    volumes:
-      - ../../..:/milvus-distributed:delegated
-    working_dir: "/milvus-distributed/tests/python_test"
-    command: >
-      /bin/bash -c "sleep 10s && pytest --tags=0331 --reruns=1 --ip proxynode"
-    networks:
-      - milvus
-
-  regression_standalone:
-    image: ${TARGET_REPO}/pytest:${TARGET_TAG}
-    build:
-      context: ../../../
-      dockerfile: build/docker/test/Dockerfile
-      cache_from:
-        - ${SOURCE_REPO}/pytest:${SOURCE_TAG}
-    volumes:
-      - ../../..:/milvus-distributed:delegated
-    working_dir: "/milvus-distributed/tests/python_test"
-    command: >
-      /bin/bash -c "sleep 10s && pytest --tags=0331 --reruns=1 --ip standalone"
-    networks:
-      - milvus
-
-networks:
-  milvus:

tests/python_test/Dockerfile

@@ -1,15 +1,18 @@
+# Copyright (C) 2019-2020 Zilliz. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied. See the License for the specific language governing permissions and limitations under the License.
+
 FROM python:3.6.8-jessie
 
-LABEL Name=megasearch_engine_test Version=0.0.1
+COPY ./requirements.txt /requirements.txt
 
-WORKDIR /app
+RUN python3 -m pip install --no-cache-dir -r /requirements.txt
-COPY . /app
 
-RUN apt-get update && apt-get install -y --no-install-recommends \
+CMD ["tail", "-f", "/dev/null"]
-    libc-dev build-essential && \
-    python3 -m pip install -r requirements.txt && \
-    apt-get remove --purge -y && \
-    rm -rf /var/lib/apt/lists/*
-
-ENTRYPOINT [ "/app/docker-entrypoint.sh" ]
-CMD [ "start" ]

tests/python_test/requirements.txt

@@ -2,7 +2,6 @@ grpcio==1.26.0
 grpcio-tools==1.26.0
 numpy==1.18.1
 pytest-cov==2.8.1
-pymilvus-distributed==0.0.42
 sklearn==0.0
 pytest==6.2.2
 pytest-timeout==1.3.3

tests/python_test/requirements_cluster.txt

@@ -1,25 +0,0 @@
-astroid==2.2.5
-atomicwrites==1.3.0
-attrs==19.1.0
-importlib-metadata==0.15
-isort==4.3.20
-lazy-object-proxy==1.4.1
-mccabe==0.6.1
-more-itertools==7.0.0
-numpy==1.16.3
-pluggy==0.12.0
-py==1.8.0
-pylint==2.5.0
-pytest==4.5.0
-pytest-timeout==1.3.3
-pytest-repeat==0.8.0
-allure-pytest==2.7.0
-pytest-print==0.1.2
-pytest-level==0.1.1
-six==1.12.0
-thrift==0.11.0
-typed-ast==1.3.5
-wcwidth==0.1.7
-wrapt==1.11.1
-zipp==0.5.1
-pymilvus>=0.2.0