Merge pull request #540 from ZhifengZhang-CN/0.6.0

Add ccache in Jenkins CI
2024-12-04 04:49:08 +08:00 · 2019-11-26 14:30:53 +08:00 · 2019-11-26 14:30:53 +08:00 · 6937c7eb01
commit 6937c7eb01
parent 4ccf5bac54 c111c5361f
8 changed files with 147 additions and 8 deletions
--- a/ci/jenkins/step/build.groovy
+++ b/ci/jenkins/step/build.groovy
@ -2,10 +2,10 @@ timeout(time: 60, unit: 'MINUTES') {
    dir ("ci/scripts") {
        withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
            if ("${env.BINRARY_VERSION}" == "gpu") {
-            	sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -g -j -u -c"
-        	} else {
-        		sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -m -j -u -c"
-        	}
+                sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -g -j -u -c"
+            } else {
+                sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -m -j -u -c"
+            }
        }
    }
 }
--- a/ci/scripts/before-install.sh
+++ b/ci/scripts/before-install.sh
@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -ex
+
+export CCACHE_COMPRESS=1
+export CCACHE_COMPRESSLEVEL=5
+export CCACHE_COMPILERCHECK=content
+export PATH=/usr/lib/ccache/:$PATH
+ccache --show-stats
+
+set +ex
--- a/ci/scripts/check_ccache.sh
+++ b/ci/scripts/check_ccache.sh
@ -0,0 +1,61 @@
+#!/bin/bash
+
+OS_NAME="linux"
+CODE_NAME=$(lsb_release -sc)
+OS_MD5=$(lsb_release -s | md5sum | cut -d " " -f 1)
+BRANCH_NAMES=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's=[a-zA-Z]*\/==g' | awk -F", " '{$1=""; print $0}')
+ARTIFACTORY_URL=""
+CCACHE_DIRECTORY="${HOME}/.ccache"
+
+while getopts "l:d:h" arg
+do
+        case $arg in
+             l)
+                ARTIFACTORY_URL=$OPTARG
+                ;;
+             d)
+                CCACHE_DIRECTORY=$OPTARG
+                ;;
+             h) # help
+                echo "
+
+parameter:
+-l: artifactory url
+-d: ccache directory
+-h: help
+
+usage:
+./build.sh -l \${ARTIFACTORY_URL} -d \${CCACHE_DIRECTORY} [-h]
+                "
+                exit 0
+                ;;
+             ?)
+                echo "ERROR! unknown argument"
+        exit 1
+        ;;
+        esac
+done
+
+if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then
+    echo "you have not input ARTIFACTORY_URL !"
+    exit 1
+fi
+
+for BRANCH_NAME in ${BRANCH_NAMES}
+do
+    echo "fetching ${BRANCH_NAME}/ccache-${OS_NAME}-${CODE_NAME}-${OS_MD5}.tar.gz"
+    wget -q --method HEAD "${ARTIFACTORY_URL}/${BRANCH_NAME}/ccache-${OS_NAME}-${CODE_NAME}-${OS_MD5}.tar.gz"
+    if [[ $? == 0 ]];then
+        wget "${ARTIFACTORY_URL}/${BRANCH_NAME}/ccache-${OS_NAME}-${CODE_NAME}-${OS_MD5}.tar.gz" && \
+        mkdir -p ${CCACHE_DIRECTORY} && \
+        tar zxf ccache-${OS_NAME}-${CODE_NAME}-${OS_MD5}.tar.gz -C ${CCACHE_DIRECTORY} && \
+        rm ccache-${OS_NAME}-${CODE_NAME}-${OS_MD5}.tar.gz
+        if [[ $? == 0 ]];then
+            echo "found cache"
+            exit 0
+        fi
+    fi
+done
+
+echo "could not download cache" && exit 1
+
--- a/ci/scripts/update_ccache.sh
+++ b/ci/scripts/update_ccache.sh
@ -0,0 +1,67 @@
+#!/bin/bash
+
+OS_NAME="linux"
+CODE_NAME=$(lsb_release -sc)
+OS_MD5=$(lsb_release -s | md5sum | cut -d " " -f 1)
+BRANCH_NAME=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's/.* \(.*\),.*/\1/' | sed 's=[a-zA-Z]*\/==g')
+ARTIFACTORY_URL=""
+ARTIFACTORY_USER=""
+ARTIFACTORY_PASSWORD=""
+CCACHE_DIRECTORY="${HOME}/.ccache"
+
+while getopts "l:u:p:d:h" arg
+do
+        case $arg in
+             l)
+                ARTIFACTORY_URL=$OPTARG
+                ;;
+             u)
+                ARTIFACTORY_USER=$OPTARG
+                ;;
+             p)
+                ARTIFACTORY_PASSWORD=$OPTARG
+                ;;
+             d)
+                CCACHE_DIRECTORY=$OPTARG
+                ;;
+             h) # help
+                echo "
+
+parameter:
+-l: artifactory url
+-u: artifactory user
+-p: artifactory password
+-d: ccache directory
+-h: help
+
+usage:
+./build.sh -l \${ARTIFACTORY_URL} -u \${ARTIFACTORY_USER} -p \${ARTIFACTORY_PASSWORD} -d \${CCACHE_DIRECTORY} [-h]
+                "
+                exit 0
+                ;;
+             ?)
+                echo "ERROR! unknown argument"
+        exit 1
+        ;;
+        esac
+done
+
+if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then
+    echo "you have not input ARTIFACTORY_URL !"
+    exit 1
+fi
+
+PACKAGE_FILE="ccache-${OS_NAME}-${CODE_NAME}-${OS_MD5}.tar.gz"
+REMOTE_PACKAGE_PATH="${ARTIFACTORY_URL}/${BRANCH_NAME}"
+
+echo "Updating ccache package file: ${PACKAGE_FILE}"
+tar zcf ./${PACKAGE_FILE} -C ${HOME}/.ccache .
+echo "Uploading ccache package file ${PACKAGE_FILE} to ${REMOTE_PACKAGE_PATH}"
+curl -u${ARTIFACTORY_USER}:${ARTIFACTORY_PASSWORD} -T ${PACKAGE_FILE} ${REMOTE_PACKAGE_PATH}
+if [[ $? == 0 ]];then
+    echo "Uploading ccache package file success !"
+    exit 0
+else
+    echo "Uploading ccache package file fault !"
+    exit 1
+fi
--- a/docker/build_env/cpu/ubuntu16.04/Dockerfile
+++ b/docker/build_env/cpu/ubuntu16.04/Dockerfile
@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends wget ca-certifi
    sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' && \
    wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \
    apt-get update && apt-get install -y --no-install-recommends \
-    g++ git gfortran lsb-core \
+    g++ git gfortran lsb-core ccache \
    libboost-serialization-dev libboost-filesystem-dev libboost-system-dev libboost-regex-dev \
    curl libtool automake libssl-dev pkg-config libcurl4-openssl-dev python3-pip \
    clang-format-6.0 clang-tidy-6.0 \
--- a/docker/build_env/cpu/ubuntu18.04/Dockerfile
+++ b/docker/build_env/cpu/ubuntu18.04/Dockerfile
@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends wget ca-certifi
    sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' && \
    wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \
    apt-get update && apt-get install -y --no-install-recommends \
-    g++ git gfortran lsb-core \
+    g++ git gfortran lsb-core ccache \
    libboost-serialization-dev libboost-filesystem-dev libboost-system-dev libboost-regex-dev \
    curl libtool automake libssl-dev pkg-config libcurl4-openssl-dev python3-pip \
    clang-format-6.0 clang-tidy-6.0 \
--- a/docker/build_env/gpu/ubuntu16.04/Dockerfile
+++ b/docker/build_env/gpu/ubuntu16.04/Dockerfile
@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends wget && \
    apt-key add /tmp/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \
    sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' && \
    apt-get update && apt-get install -y --no-install-recommends \
-    git flex bison gfortran lsb-core \
+    git flex bison gfortran lsb-core ccache \
    curl libtool automake libboost1.58-all-dev libssl-dev pkg-config libcurl4-openssl-dev python3-pip \
    clang-format-6.0 clang-tidy-6.0 \
    lcov mysql-client libmysqlclient-dev intel-mkl-gnu-2019.5-281 intel-mkl-core-2019.5-281 && \
--- a/docker/build_env/gpu/ubuntu18.04/Dockerfile
+++ b/docker/build_env/gpu/ubuntu18.04/Dockerfile
@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends wget && \
    apt-key add /tmp/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \
    sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' && \
    apt-get update && apt-get install -y --no-install-recommends \
-    git flex bison gfortran lsb-core \
+    git flex bison gfortran lsb-core ccache \
    curl libtool automake libboost-all-dev libssl-dev pkg-config libcurl4-openssl-dev python3-pip \
    clang-format-6.0 clang-tidy-6.0 \
    lcov mysql-client libmysqlclient-dev intel-mkl-gnu-2019.5-281 intel-mkl-core-2019.5-281 && \