test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-11-29 10:28:41 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

Add jenkinsfile for run ci on QA cluster (#15351)

Browse Source 
Signed-off-by: Jenny Li <jing.li@zilliz.com>
This commit is contained in:
Jenny Li 2022-01-24 13:18:29 +08:00 committed by GitHub
parent 390cdd2515
commit de197e6eb8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 490 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

226
build/ci/jenkins/QaPR.groovy Normal file
View File

@ -0,0 +1,226 @@
#!/usr/bin/env groovy
int total_timeout_minutes = 120
int e2e_timeout_seconds = 70 * 60
def imageTag=''
int case_timeout_seconds = 10 * 60
def chart_version='2.4.25'
pipeline {
options {
timestamps()
timeout(time: total_timeout_minutes, unit: 'MINUTES')
buildDiscarder logRotator(artifactDaysToKeepStr: '30')
parallelsAlwaysFailFast()
preserveStashes(buildCount: 5)
}
agent {
kubernetes {
label 'milvus-e2e-test-pipeline'
inheritFrom 'default'
defaultContainer 'main'
yamlFile 'build/ci/jenkins/pod/qa/rte.yaml'
customWorkspace '/home/jenkins/agent/workspace'
}
}
environment {
PROJECT_NAME = 'milvus'
SEMVER = "${BRANCH_NAME.contains('/') ? BRANCH_NAME.substring(BRANCH_NAME.lastIndexOf('/') + 1) : BRANCH_NAME}"
DOCKER_BUILDKIT = 1
ARTIFACTS = "${env.WORKSPACE}/_artifacts"
DOCKER_CREDENTIALS_ID = "f0aacc8e-33f2-458a-ba9e-2c44f431b4d2"
TARGET_REPO = "milvusdb"
// CI_DOCKER_CREDENTIAL_ID = "ci-docker-registry"
CI_DOCKER_CREDENTIAL_ID = "qa-ci-docker-registry"
MILVUS_HELM_NAMESPACE = "milvus-ci"
DISABLE_KIND = true
// HUB = 'registry.milvus.io/milvus'
HUB = 'harbor.zilliz.cc/milvus-ci'
JENKINS_BUILD_ID = "${env.BUILD_ID}"
CI_MODE="pr"
MIRROR_URL="http://10.201.20.246:5000"
}
stages {
stage ('Build'){
steps {
container('main') {
dir ('build'){
sh './set_docker_mirror.sh'
}
dir ('tests/scripts') {
script {
sh 'printenv'
def date = sh(returnStdout: true, script: 'date +%Y%m%d').trim()
def gitShortCommit = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()
imageTag="${env.BRANCH_NAME}-${date}-${gitShortCommit}"
withCredentials([usernamePassword(credentialsId: "${env.CI_DOCKER_CREDENTIAL_ID}", usernameVariable: 'CI_REGISTRY_USERNAME', passwordVariable: 'CI_REGISTRY_PASSWORD')]){
sh """
TAG="${imageTag}" \
./e2e-k8s.sh \
--skip-export-logs \
--skip-install \
--skip-cleanup \
--skip-setup \
--skip-test
"""
// stash imageTag info for rebuild install & E2E Test only
sh "echo ${imageTag} > imageTag.txt"
stash includes: 'imageTag.txt', name: 'imageTag'
}
}
}
}
}
}
stage('Install & E2E Test') {
matrix {
axes {
axis {
name 'MILVUS_SERVER_TYPE'
values 'standalone', 'distributed'
}
axis {
name 'MILVUS_CLIENT'
values 'pymilvus'
}
}
stages {
stage('Install') {
steps {
container('main') {
dir ('tests/scripts') {
script {
sh 'printenv'
def clusterEnabled = "false"
if ("${MILVUS_SERVER_TYPE}" == 'distributed') {
clusterEnabled = "true"
}
if ("${MILVUS_CLIENT}" == "pymilvus") {
if ("${imageTag}"==''){
dir ("imageTag"){
try{
unstash 'imageTag'
imageTag=sh(returnStdout: true, script: 'cat imageTag.txt | tr -d \'\n\r\'')
}catch(e){
print "No Image Tag info remained ,please rerun build to build new image."
exit 1
}
}
}
withCredentials([usernamePassword(credentialsId: "${env.CI_DOCKER_CREDENTIAL_ID}", usernameVariable: 'CI_REGISTRY_USERNAME', passwordVariable: 'CI_REGISTRY_PASSWORD')]){
sh """
MILVUS_CLUSTER_ENABLED=${clusterEnabled} \
TAG=${imageTag}\
./e2e-k8s.sh \
--skip-export-logs \
--skip-cleanup \
--skip-setup \
--skip-test \
--skip-build \
--skip-build-image \
--install-extra-arg "--set etcd.persistence.storageClass=local-path \
--set minio.persistence.storageClass=local-path \
--set etcd.metrics.enabled=false \
--set etcd.metrics.podMonitor.enabled=false\
--set metrics.serviceMonitor.enabled=true \
--version ${chart_version} \
-f values/pr.yaml"
"""
}
} else {
error "Error: Unsupported Milvus client: ${MILVUS_CLIENT}"
}
}
}
}
}
}
stage('E2E Test'){
agent {
kubernetes {
label 'milvus-e2e-test-pr'
inheritFrom 'default'
defaultContainer 'main'
yamlFile 'build/ci/jenkins/pod/qa/rte.yaml'
customWorkspace '/home/jenkins/agent/workspace'
}
}
steps {
container('pytest') {
dir ('tests/scripts') {
script {
def release_name=sh(returnStdout: true, script: './get_release_name.sh')
def clusterEnabled = 'false'
if ("${MILVUS_SERVER_TYPE}" == "distributed") {
clusterEnabled = "true"
}
if ("${MILVUS_CLIENT}" == "pymilvus") {
sh """
MILVUS_HELM_RELEASE_NAME="${release_name}" \
MILVUS_HELM_NAMESPACE="milvus-ci" \
MILVUS_CLUSTER_ENABLED="${clusterEnabled}" \
TEST_TIMEOUT="${e2e_timeout_seconds}" \
./ci_e2e.sh "-n 6 -x --tags L0 L1 --timeout ${case_timeout_seconds}"
"""
} else {
error "Error: Unsupported Milvus client: ${MILVUS_CLIENT}"
}
}
}
}
}
}
}
post{
always {
container('pytest'){
dir("${env.ARTIFACTS}") {
sh "tar -zcvf artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${MILVUS_CLIENT}-pytest-logs.tar.gz /tmp/ci_logs/test --remove-files || true"
archiveArtifacts artifacts: "artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${MILVUS_CLIENT}-pytest-logs.tar.gz ", allowEmptyArchive: true
}
}
container('main') {
dir ('tests/scripts') {
script {
def release_name=sh(returnStdout: true, script: './get_release_name.sh')
sh "./qa/uninstall_milvus.sh --release-name ${release_name}"
sh "./qa/ci_logs.sh --log-dir /ci-logs --artifacts-name ${env.ARTIFACTS}/artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${SEMVER}-${env.BUILD_NUMBER}-${MILVUS_CLIENT}-e2e-logs \
--release-name ${release_name}"
dir("${env.ARTIFACTS}") {
archiveArtifacts artifacts: "artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${SEMVER}-${env.BUILD_NUMBER}-${MILVUS_CLIENT}-e2e-logs.tar.gz", allowEmptyArchive: true
}
}
}
}
}
}
}
}
}
post{
unsuccessful {
container('jnlp') {
dir ('tests/scripts') {
script {
def authorEmail = sh(returnStdout: true, script: './get_author_email.sh ')
emailext subject: '$DEFAULT_SUBJECT',
body: '$DEFAULT_CONTENT',
recipientProviders: [developers(), culprits()],
replyTo: '$DEFAULT_REPLYTO',
to: "${authorEmail},devops@zilliz.com"
}
}
}
}
}
}

66
build/ci/jenkins/pod/qa/rte.yaml Normal file
View File

@ -0,0 +1,66 @@
apiVersion: v1
kind: Pod
metadata:
labels:
app: milvus-e2e
namespace: jenkins
spec:
enableServiceLinks: false
containers:
- name: main
image: harbor.zilliz.cc/dockerhub/milvusdb/krte:20211213-dcc15e9
env:
- name: DOCKER_IN_DOCKER_ENABLED
value: "true"
- name: DOCKER_VOLUME_DIRECTORY
value: "/mnt/disk/.docker"
tty: true
securityContext:
privileged: true
args: ["cat"]
resources:
limits:
cpu: "6"
memory: 12Gi
requests:
cpu: "0.5"
memory: 5Gi
volumeMounts:
- mountPath: /docker-graph
name: docker-graph
- mountPath: /var/lib/docker
name: docker-root
- mountPath: /lib/modules
name: modules
readOnly: true
- mountPath: /sys/fs/cgroup
name: cgroup
- mountPath: /mnt/disk/.docker
name: build-cache
subPath: docker-volume
- name: pytest
image: harbor.zilliz.cc/dockerhub/milvusdb/pytest:20220118-3eb5cc4
resources:
limits:
cpu: "6"
memory: 12Gi
requests:
cpu: "0.5"
memory: 5Gi
volumes:
- emptyDir: {}
name: docker-graph
- emptyDir: {}
name: docker-root
- hostPath:
path: /tmp/krte/cache
type: DirectoryOrCreate
name: build-cache
- hostPath:
path: /lib/modules
type: Directory
name: modules
- hostPath:
path: /sys/fs/cgroup
type: Directory
name: cgroup

3
build/set_docker_mirror.sh
View File

@ -21,7 +21,8 @@
set -e
# Use nexus as docker mirror registry
MIRROR_URL="http://nexus-nexus-repository-manager-docker-5000.nexus:5000"
MIRROR_URL="${MIRROR_URL:-http://nexus-nexus-repository-manager-docker-5000.nexus:5000}"
#MIRROR_URL="http://nexus-nexus-repository-manager-docker-5000.nexus:5000"
# Add registry mirror config into docker daemon
set_daemon_json_file(){

84
tests/scripts/qa/ci_logs.sh Executable file
View File

@ -0,0 +1,84 @@
#!/bin/bash
# Licensed to the LF AI & Data foundation under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Exit immediately for non zero status
set -e
# Check unset variables
# set -u
# Print commands
set -x
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
SOURCE="$(readlink "$SOURCE")"
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
done
ROOT="$( cd -P "$( dirname "$SOURCE" )/../.." && pwd )"
while (( "$#" )); do
case "$1" in
--release-name)
RELEASE_NAME=$2
shift 2
;;
--artifacts-name)
ARTIFACTS_NAME=$2
shift 2
;;
--log-dir)
LOG_DIR=$2
shift 2
;;
-h|--help)
{ set +x; } 2>/dev/null
HELP="
Usage:
$0 [flags] [Arguments]
--log-dir Log Path
--artifacts-name Artifacts Name
--release-name Milvus helm release name
-h or --help Print help information
Use \"$0 --help\" for more information about a given command.
"
echo -e "${HELP}" ; exit 0
;;
-*)
echo "Error: Unsupported flag $1" >&2
exit 1
;;
*) # preserve positional arguments
PARAMS+=("$1")
shift
;;
esac
done
RELEASE_NAME=${RELEASE_NAME:-milvus-testing}
tar -zcvf ${ARTIFACTS_NAME:-artifacts}.tar.gz ${RELEASE_NAME}/*

112
tests/scripts/qa/uninstall_milvus.sh Executable file
View File

@ -0,0 +1,112 @@
#!/bin/bash
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
# Exit immediately for non zero status
set -e
# Print commands
# set -x
while (( "$#" )); do
case "$1" in
--release-name)
RELEASE_NAME=$2
shift 2
;;
-h|--help)
{ set +x; } 2>/dev/null
HELP="
Usage:
$0 [flags] [Arguments]
--release-name Milvus helm release name
-h or --help Print help information
Use \"$0 --help\" for more information about a given command.
"
echo -e "${HELP}" ; exit 0
;;
-*)
echo "Error: Unsupported flag $1" >&2
exit 1
;;
*) # preserve positional arguments
PARAMS+=("$1")
shift
;;
esac
done
MILVUS_HELM_RELEASE_NAME="${MILVUS_HELM_RELEASE_NAME:-milvus-testing}"
MILVUS_HELM_NAMESPACE="${MILVUS_HELM_NAMESPACE:-default}"
if [[ -n "${RELEASE_NAME:-}" ]]; then
MILVUS_HELM_RELEASE_NAME="${RELEASE_NAME}"
# List pod list before uninstall
kubectl get pods -n ${MILVUS_HELM_NAMESPACE} -o wide | grep "${MILVUS_HELM_RELEASE_NAME}-"
# Show restart pods last terminated reason
# restart_pods=$(kubectl get pods -n ${MILVUS_HELM_NAMESPACE} | grep "${MILVUS_HELM_RELEASE_NAME}-" | grep 'ago)' | awk '{print $1}')
# for restart_pod in ${restart_pods}
# do
# reason=$(kubectl get pod ${restart_pod} -n milvus-ci -o json | jq .status.containerStatuses[0].lastState.terminated.reason )
# restart_count=$(kubectl get pod ${restart_pod} -n milvus-ci -o json | jq .status.containerStatuses[0].restartCount )
# echo "${restart_pod} restarts ${restart_count}, last terminateed reason is ${reason}"
# done
fi
mkdir ${MILVUS_HELM_RELEASE_NAME}
all_pods=$(kubectl get pods -n ${MILVUS_HELM_NAMESPACE} | grep "${MILVUS_HELM_RELEASE_NAME}-" | awk '{print $1}')
for pod_name in ${all_pods}
do
kubectl logs ${pod_name} -n ${MILVUS_HELM_NAMESPACE} --all-containers=true > ${MILVUS_HELM_RELEASE_NAME}/${pod_name}.log
kubectl logs ${pod_name} -n ${MILVUS_HELM_NAMESPACE} --all-containers=true --previous=true > ${MILVUS_HELM_RELEASE_NAME}/${pod_name}_previous.log
done
# Uninstall Milvus Helm Release
# Do not exit with error when release not found so that the script can also be used to clean up related pvc even helm release has been uninstalled already
helm uninstall -n "${MILVUS_HELM_NAMESPACE}" "${MILVUS_HELM_RELEASE_NAME}" || true
MILVUS_LABELS1="app.kubernetes.io/instance=${MILVUS_HELM_RELEASE_NAME}"
MILVUS_LABELS2="release=${MILVUS_HELM_RELEASE_NAME}"
# Clean up pvc
kubectl delete pvc --wait -n "${MILVUS_HELM_NAMESPACE}" $(kubectl get pvc -n "${MILVUS_HELM_NAMESPACE}" -l "${MILVUS_LABELS1}" -o jsonpath='{range.items[*]}{.metadata.name} ') || true
kubectl delete pvc --wait -n "${MILVUS_HELM_NAMESPACE}" $(kubectl get pvc -n "${MILVUS_HELM_NAMESPACE}" -l "${MILVUS_LABELS2}" -o jsonpath='{range.items[*]}{.metadata.name} ') || true
# Add check & delete pvc again in case pvc need time to be deleted
# clean_label_pvc(){
# local label=${1?label expected as first argument.}
# for i in {1..10}
# do
# PVC=$(kubectl get pvc -n "${MILVUS_HELM_NAMESPACE}" -l "${label}" -o jsonpath='{range.items[*]}{.metadata.name} ')
# STATUS=$(echo ${PVC} | wc -w )
# echo "status is ${STATUS}"
# if [ $STATUS == 0 ]; then
# break
# else
# sleep 5
# echo "PVCs are ${PVC}"
# kubectl delete pvc --wait -n "${MILVUS_HELM_NAMESPACE}" ${PVC}
# fi
# done
# }
# echo "Check & Delete Persistent Volumes"
# clean_label_pvc ${MILVUS_LABELS1}
# clean_label_pvc ${MILVUS_LABELS2}