mirror of
https://gitee.com/milvus-io/milvus.git
synced 2024-11-30 10:59:32 +08:00
enhance new jenkinsfile for nightly and e2e (#35715)
https://github.com/milvus-io/milvus/pull/35672 Signed-off-by: Yellow Shine <sammy.huang@zilliz.com>
This commit is contained in:
yellow-shine
GitHub
parent
ab261d0f8b
commit
03627150d3
126
ci/jenkins/Nightly2.groovy
Normal file
126
ci/jenkins/Nightly2.groovy
Normal file
@ -0,0 +1,126 @@
|
||||
@Library('jenkins-shared-library@v0.27.0') _
|
||||
|
||||
def pod = libraryResource 'io/milvus/pod/tekton-ci.yaml'
|
||||
|
||||
String cron_timezone = 'TZ=Asia/Shanghai'
|
||||
String cron_string = "50 3 * * * "
|
||||
|
||||
// Make timeout 4 hours so that we can run two nightly during the ci
|
||||
int total_timeout_minutes = 7 * 60
|
||||
|
||||
def milvus_helm_chart_version = '4.2.8'
|
||||
|
||||
pipeline {
|
||||
triggers {
|
||||
cron """${cron_timezone}
|
||||
${cron_string}"""
|
||||
}
|
||||
options {
|
||||
skipDefaultCheckout true
|
||||
timeout(time: total_timeout_minutes, unit: 'MINUTES')
|
||||
// parallelsAlwaysFailFast()
|
||||
buildDiscarder logRotator(artifactDaysToKeepStr: '30')
|
||||
preserveStashes(buildCount: 5)
|
||||
disableConcurrentBuilds(abortPrevious: true)
|
||||
}
|
||||
agent {
|
||||
kubernetes {
|
||||
yaml pod
|
||||
}
|
||||
}
|
||||
stages {
|
||||
stage('build') {
|
||||
steps {
|
||||
container('tkn') {
|
||||
script {
|
||||
isPr = env.CHANGE_ID != null
|
||||
gitMode = isPr ? 'merge' : 'fetch'
|
||||
gitBaseRef = isPr ? "$env.CHANGE_TARGET" : "$env.BRANCH_NAME"
|
||||
|
||||
job_name = tekton.run arch: 'amd64',
|
||||
isPr: isPr,
|
||||
gitMode: gitMode ,
|
||||
gitBaseRef: gitBaseRef,
|
||||
pullRequestNumber: "$env.CHANGE_ID",
|
||||
suppress_suffix_of_image_tag: true,
|
||||
test_client_type: '["pytest"]'
|
||||
|
||||
milvus_image_tag = tekton.query_result job_name, 'milvus-image-tag'
|
||||
pytest_image = tekton.query_result job_name, 'pytest-image-fqdn'
|
||||
}
|
||||
}
|
||||
}
|
||||
post {
|
||||
always {
|
||||
container('tkn') {
|
||||
script {
|
||||
tekton.sure_stop()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('E2E Test') {
|
||||
matrix {
|
||||
agent {
|
||||
kubernetes {
|
||||
yaml pod
|
||||
}
|
||||
}
|
||||
axes {
|
||||
axis {
|
||||
name 'milvus_deployment_option'
|
||||
values 'standalone', 'distributed-pulsar', 'distributed-kafka', 'standalone-authentication', 'standalone-one-pod', 'distributed-streaming-service'
|
||||
}
|
||||
}
|
||||
stages {
|
||||
stage('E2E Test') {
|
||||
steps {
|
||||
container('tkn') {
|
||||
script {
|
||||
def helm_release_name = tekton.release_name milvus_deployment_option: milvus_deployment_option,
|
||||
ciMode: 'nightly',
|
||||
client: 'py',
|
||||
changeId: "${env.CHANGE_ID}",
|
||||
buildId:"${env.BUILD_ID}"
|
||||
|
||||
tekton.pytest helm_release_name: helm_release_name,
|
||||
milvus_helm_version: milvus_helm_chart_version,
|
||||
ciMode: 'nightly',
|
||||
milvus_image_tag: milvus_image_tag,
|
||||
pytest_image: pytest_image,
|
||||
milvus_deployment_option: milvus_deployment_option
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
post {
|
||||
always {
|
||||
container('tkn') {
|
||||
script {
|
||||
tekton.sure_stop()
|
||||
}
|
||||
}
|
||||
|
||||
container('archive') {
|
||||
script {
|
||||
def helm_release_name = tekton.release_name milvus_deployment_option: milvus_deployment_option,
|
||||
ciMode: 'nightly',
|
||||
client: 'py',
|
||||
changeId: "${env.CHANGE_ID}",
|
||||
buildId:"${env.BUILD_ID}"
|
||||
|
||||
tekton.archive milvus_deployment_option: milvus_deployment_option,
|
||||
release_name: helm_release_name ,
|
||||
change_id: env.CHANGE_ID,
|
||||
build_id: env.BUILD_ID
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
127
ci/jenkins/PR2.groovy
Normal file
127
ci/jenkins/PR2.groovy
Normal file
@ -0,0 +1,127 @@
|
||||
@Library('jenkins-shared-library@v0.27.0') _
|
||||
|
||||
def pod = libraryResource 'io/milvus/pod/tekton-4am.yaml'
|
||||
def milvus_helm_chart_version = '4.2.8'
|
||||
|
||||
pipeline {
|
||||
options {
|
||||
skipDefaultCheckout true
|
||||
parallelsAlwaysFailFast()
|
||||
buildDiscarder logRotator(artifactDaysToKeepStr: '30')
|
||||
preserveStashes(buildCount: 5)
|
||||
disableConcurrentBuilds(abortPrevious: true)
|
||||
}
|
||||
agent {
|
||||
kubernetes {
|
||||
cloud '4am'
|
||||
yaml pod
|
||||
}
|
||||
}
|
||||
stages {
|
||||
stage('build') {
|
||||
steps {
|
||||
container('tkn') {
|
||||
script {
|
||||
isPr = env.CHANGE_ID != null
|
||||
gitMode = isPr ? 'merge' : 'fetch'
|
||||
gitBaseRef = isPr ? "$env.CHANGE_TARGET" : "$env.BRANCH_NAME"
|
||||
|
||||
job_name = tekton.run arch: 'amd64',
|
||||
isPr: isPr,
|
||||
gitMode: gitMode ,
|
||||
gitBaseRef: gitBaseRef,
|
||||
pullRequestNumber: "$env.CHANGE_ID",
|
||||
suppress_suffix_of_image_tag: true,
|
||||
test_client_type: '["pytest"]'
|
||||
|
||||
milvus_image_tag = tekton.query_result job_name, 'milvus-image-tag'
|
||||
pytest_image = tekton.query_result job_name, 'pytest-image-fqdn'
|
||||
}
|
||||
}
|
||||
}
|
||||
post {
|
||||
always {
|
||||
container('tkn') {
|
||||
script {
|
||||
tekton.sure_stop()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('E2E Test') {
|
||||
matrix {
|
||||
agent {
|
||||
kubernetes {
|
||||
cloud '4am'
|
||||
yaml pod
|
||||
}
|
||||
}
|
||||
axes {
|
||||
axis {
|
||||
name 'milvus_deployment_option'
|
||||
values 'standalone', 'distributed', 'standalone-kafka', 'distributed-streaming-service'
|
||||
}
|
||||
}
|
||||
stages {
|
||||
stage('E2E Test') {
|
||||
steps {
|
||||
container('tkn') {
|
||||
script {
|
||||
def helm_release_name = tekton.release_name milvus_deployment_option: milvus_deployment_option,
|
||||
client: 'py',
|
||||
changeId: "${env.CHANGE_ID}",
|
||||
buildId:"${env.BUILD_ID}"
|
||||
|
||||
if (milvus_deployment_option == 'distributed-streaming-service') {
|
||||
try {
|
||||
tekton.pytest helm_release_name: helm_release_name,
|
||||
milvus_helm_version: milvus_helm_chart_version,
|
||||
ciMode: 'e2e',
|
||||
milvus_image_tag: milvus_image_tag,
|
||||
pytest_image: pytest_image,
|
||||
milvus_deployment_option: milvus_deployment_option
|
||||
} catch (Exception e) {
|
||||
println e
|
||||
}
|
||||
} else {
|
||||
tekton.pytest helm_release_name: helm_release_name,
|
||||
milvus_helm_version: milvus_helm_chart_version,
|
||||
ciMode: 'e2e',
|
||||
milvus_image_tag: milvus_image_tag,
|
||||
pytest_image: pytest_image,
|
||||
milvus_deployment_option: milvus_deployment_option
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
post {
|
||||
always {
|
||||
container('tkn') {
|
||||
script {
|
||||
tekton.sure_stop()
|
||||
}
|
||||
}
|
||||
|
||||
container('archive') {
|
||||
script {
|
||||
def helm_release_name = tekton.release_name milvus_deployment_option: milvus_deployment_option,
|
||||
client: 'py',
|
||||
changeId: "${env.CHANGE_ID}",
|
||||
buildId:"${env.BUILD_ID}"
|
||||
|
||||
tekton.archive milvus_deployment_option: milvus_deployment_option,
|
||||
release_name: helm_release_name ,
|
||||
change_id: env.CHANGE_ID,
|
||||
build_id: env.BUILD_ID
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
29
tests/python_client/Dockerfile
Normal file
29
tests/python_client/Dockerfile
Normal file
@ -0,0 +1,29 @@
|
||||
FROM python:3.8-buster
|
||||
|
||||
RUN apt-get update && apt-get install -y jq
|
||||
|
||||
# Define the ARG variable
|
||||
ARG PIP_TRUSTED_HOST=""
|
||||
ARG PIP_INDEX_URL=""
|
||||
ARG PIP_INDEX=""
|
||||
ARG PIP_FIND_LINKS=""
|
||||
|
||||
# Set the ENV variable
|
||||
ENV PIP_TRUSTED_HOST=${PIP_TRUSTED_HOST}
|
||||
ENV PIP_INDEX_URL=${PIP_INDEX_URL}
|
||||
ENV PIP_INDEX=${PIP_INDEX}
|
||||
ENV PIP_FIND_LINKS=${PIP_FIND_LINKS}
|
||||
|
||||
|
||||
WORKDIR /milvus
|
||||
|
||||
COPY tests/python_client/requirements.txt tests/python_client/requirements.txt
|
||||
|
||||
RUN cd tests/python_client && python3 -m pip install --upgrade setuptools \
|
||||
&& python3 -m pip install --upgrade pip \
|
||||
&& python3 -m pip install --no-cache-dir -r requirements.txt --timeout 30 --retries 6
|
||||
|
||||
COPY tests/python_client tests/python_client
|
||||
COPY tests/restful_client tests/restful_client
|
||||
COPY tests/restful_client_v2 tests/restful_client_v2
|
||||
COPY tests/scripts tests/scripts
|
||||
@ -61,8 +61,11 @@ if [ ! -d "${CI_LOG_PATH}" ]; then
|
||||
mkdir -p ${CI_LOG_PATH}
|
||||
fi
|
||||
|
||||
echo "prepare e2e test"
|
||||
install_pytest_requirements
|
||||
# skip pip install when DISABLE_PIP_INSTALL is set
|
||||
if [ "${DISABLE_PIP_INSTALL:-}" = "" ]; then
|
||||
echo "prepare e2e test"
|
||||
install_pytest_requirements
|
||||
fi
|
||||
|
||||
|
||||
# Pytest is not able to have both --timeout & --workers, so do not add --timeout or --workers in the shell script
|
||||
|
||||
@ -62,8 +62,12 @@ if [ ! -d "${CI_LOG_PATH}" ]; then
|
||||
mkdir -p ${CI_LOG_PATH}
|
||||
fi
|
||||
|
||||
echo "prepare e2e test"
|
||||
install_pytest_requirements
|
||||
# skip pip install when DISABLE_PIP_INSTALL is set
|
||||
DISABLE_PIP_INSTALL=${DISABLE_PIP_INSTALL:-false}
|
||||
if [ "${DISABLE_PIP_INSTALL:-}" = "" ]; then
|
||||
echo "prepare e2e test"
|
||||
install_pytest_requirements
|
||||
fi
|
||||
|
||||
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user