test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-11-30 02:48:45 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

Add PyTest 0331 Regression pipeline

Browse Source 
Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com>
This commit is contained in:
quicksilver 2021-02-27 18:32:23 +08:00 committed by yefu.chen
parent 66146223ca
commit f0a0ae433f
5 changed files with 67 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
.jenkins/modules/Regression/PythonRegression.groovy
View File

@ -1,30 +1,30 @@
try {
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d etcd'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d pulsar'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d minio'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d etcd'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d pulsar'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d minio'
dir ('build/docker/deploy') {
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} pull'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d master'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d indexservice'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d indexnode'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d proxyservice'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d dataservice'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d queryservice'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} run -e DATA_NODE_ID=3 -d datanode'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d proxynode'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} run -e QUERY_NODE_ID=1 -d querynode'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} run -e QUERY_NODE_ID=2 -d querynode'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} pull'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d master'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d indexservice'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d indexnode'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d proxyservice'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d dataservice'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d queryservice'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} run -e DATA_NODE_ID=3 -d datanode'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d proxynode'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} run -e QUERY_NODE_ID=1 -d querynode'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} run -e QUERY_NODE_ID=2 -d querynode'
}
dir ('build/docker/test') {
sh 'docker pull ${SOURCE_REPO}/pytest:${SOURCE_TAG} || true'
sh 'docker-compose build --force-rm regression'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} run --rm regression'
sh 'docker-compose build --force-rm ${REGRESSION_SERVICE_NAME}'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} run --rm ${REGRESSION_SERVICE_NAME}'
try {
withCredentials([usernamePassword(credentialsId: "${env.DOCKER_CREDENTIALS_ID}", usernameVariable: 'DOCKER_USERNAME', passwordVariable: 'DOCKER_PASSWORD')]) {
sh 'docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD} ${DOKCER_REGISTRY_URL}'
sh 'docker-compose push regression || true'
sh 'docker-compose push ${REGRESSION_SERVICE_NAME} || true'
}
} catch (exc) {
throw exc
@ -36,15 +36,15 @@ try {
throw exc
} finally {
dir ('build/docker/deploy') {
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} ps -a | tail -n +3 | awk \'{ print $1 }\' | ( while read arg; do docker logs -t $arg > $arg.log 2>&1; done )'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} ps -a | tail -n +3 | awk \'{ print $1 }\' | ( while read arg; do docker logs -t $arg > $arg.log 2>&1; done )'
archiveArtifacts artifacts: "**.log", allowEmptyArchive: true
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} down --rmi all -v || true'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} down --rmi all -v || true'
}
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} rm -f -s -v pulsar'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} rm -f -s -v etcd'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} rm -f -s -v minio'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} rm -f -s -v pulsar'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} rm -f -s -v etcd'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} rm -f -s -v minio'
dir ('build/docker/test') {
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} run --rm regression /bin/bash -c "rm -rf __pycache__ && rm -rf .pytest_cache"'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} down --rmi all -v || true'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} run --rm ${REGRESSION_SERVICE_NAME} /bin/bash -c "rm -rf __pycache__ && rm -rf .pytest_cache"'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} down --rmi all -v || true'
}
}

36
build/ci/jenkins/Jenkinsfile vendored
View File

@ -68,18 +68,30 @@ pipeline {
}
stage ('Dev Test') {
agent {
label "performance"
}
environment {
DOCKER_COMPOSE_PROJECT_NAME = "${PROJECT_NAME}-${SEMVER}-${env.BUILD_NUMBER}".replaceAll("\\.", "-").replaceAll("_", "-")
}
steps {
MPLModule('Python Regression')
}
post {
cleanup {
deleteDir() /* clean up our workspace */
matrix {
axes {
axis {
name 'REGRESSION_SERVICE_NAME'
values 'regression'
}
}
environment {
DOCKER_COMPOSE_PROJECT_NAME = "${PROJECT_NAME}-${SEMVER}-${env.BUILD_NUMBER}".replaceAll("\\.", "-").replaceAll("_", "-")
}
agent {
label 'performance'
}
stages {
stage('Test') {
steps {
MPLModule('Python Regression')
}
}
}
post {
cleanup {
deleteDir() /* clean up our workspace */
}
}
}
}

8
build/docker/deploy/distributed/Dockerfile
View File

@ -11,6 +11,8 @@
FROM milvusdb/milvus-distributed-dev:amd64-ubuntu18.04-latest AS openblas
FROM hectormolinero/tini:v18 AS tini
#FROM alpine
FROM ubuntu:bionic-20200921
@ -19,11 +21,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends libtbb-dev gfor
rm -rf /var/lib/apt/lists/*
# Add Tini
ENV TINI_VERSION v0.19.0
ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini
RUN chmod +x /tini
COPY --from=tini /usr/bin/tini /tini
COPY --from=openblas /usr/lib/libopenblas-r0.3.9.so /usr/lib/

2
build/docker/test/Dockerfile
View File

@ -13,6 +13,6 @@ FROM python:3.6.8-jessie
COPY ./tests/python/requirements.txt /requirements.txt
RUN python3 -m pip install -r /requirements.txt
RUN python3 -m pip install -r /requirements.txt && python3 -m pip install git+https://gitee.com/quicksilver/pytest-tags.git
CMD ["tail", "-f", "/dev/null"]

15
build/docker/test/docker-compose.yml
View File

@ -16,5 +16,20 @@ services:
networks:
- milvus
regression_0331:
image: ${TARGET_REPO}/pytest:${TARGET_TAG}
build:
context: ../../../
dockerfile: build/docker/test/Dockerfile
cache_from:
- ${SOURCE_REPO}/pytest:${SOURCE_TAG}
volumes:
- ../../..:/milvus-distributed:delegated
working_dir: "/milvus-distributed/tests/python_test"
command: >
/bin/bash -c "pytest --tags=0331 --ip proxyservice -n 4"
networks:
- milvus
networks:
milvus: