test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-11-29 18:38:44 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

Add Github Actions and remove Travis CI (#752)

Browse Source 
* add github actions

* add amd64-ubuntu-18.04-cache volumes

* update core.yml in github actions

* update ci/docker/ubuntu-18.04-core.dockerfile

* update CI scripts file

* update github actions

* update github actions

* update github actions

* remove Travis CI

* update docker-compose.yml

* update ci scripts files

* update docker-compose.yml
This commit is contained in:
quicksilver 2019-12-12 14:19:13 +08:00 committed by Jin Hai
parent 5e045af51f
commit d1b79da6d8
13 changed files with 128 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.env Normal file
View File

@ -0,0 +1,5 @@
REPO=milvusdb/milvus-dev
MILVUS_INSTALL_PREFIX=/var/lib/milvus
ARCH=amd64
UBUNTU=18.04
CUDA=10.1

42
.github/workflows/core.yml vendored Normal file
View File

@ -0,0 +1,42 @@
name: Core
on:
push:
paths:
- 'ci/**'
- 'core/**'
pull_request:
paths:
- 'ci/**'
- 'core/**'
jobs:
ubuntu:
name: AMD64 Ubuntu ${{ matrix.ubuntu }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
ubuntu: [18.04]
env:
UBUNTU: ${{ matrix.ubuntu }}
steps:
- name: Checkout Milvus
uses: actions/checkout@v1
- name: Docker Pull
shell: bash
run: |
docker-compose pull --ignore-pull-failures db
docker-compose pull --ignore-pull-failures ubuntu-core
- name: Docker Build
run: |
docker-compose run -d db
docker-compose run ubuntu-core
- name: Docker Push
if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus'
continue-on-error: true
shell: bash
run: |
docker login -u ${{ secrets.DOCKERHUB_USER }} \
-p ${{ secrets.DOCKERHUB_TOKEN }}
docker-compose push ubuntu-core

1
.gitignore vendored
View File

@ -12,7 +12,6 @@ __pycache__
# vscode generated files
.vscode
.env
build
cmake-build-debug
cmake-build-release

20
.travis.yml
View File

@ -1,20 +0,0 @@
language: cpp
sudo: required
dist: bionic
cache:
directories:
- $HOME/.ccache
addons:
apt:
update: true
before_install:
- source ci/travis/before-install.sh
install:
- source $TRAVIS_BUILD_DIR/ci/travis/install_dependency.sh
script:
- $TRAVIS_BUILD_DIR/ci/travis/travis_build.sh

27
ci/docker/ubuntu-18.04-core.dockerfile Normal file
View File

@ -0,0 +1,27 @@
ARG arch=amd64
FROM ${arch}/ubuntu:18.04
# pipefail is enabled for proper error detection in the `wget | apt-key add`
# step
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update && apt-get install -y --no-install-recommends wget ca-certificates gnupg2 && \
wget -P /tmp https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \
apt-key add /tmp/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \
sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' && \
wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \
apt-get update && apt-get install -y --no-install-recommends \
g++ git gfortran lsb-core ccache \
libboost-serialization-dev libboost-filesystem-dev libboost-system-dev libboost-regex-dev \
curl libtool automake libssl-dev pkg-config libcurl4-openssl-dev python3-pip \
clang-format-6.0 clang-tidy-6.0 \
lcov mysql-client libmysqlclient-dev intel-mkl-gnu-2019.5-281 intel-mkl-core-2019.5-281 libopenblas-dev liblapack3 && \
apt-get remove --purge -y && \
rm -rf /var/lib/apt/lists/*
RUN ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so \
/usr/lib/x86_64-linux-gnu/libmysqlclient_r.so
RUN sh -c 'echo export LD_LIBRARY_PATH=/opt/intel/compilers_and_libraries_2019.5.281/linux/mkl/lib/intel64:\$LD_LIBRARY_PATH > /etc/profile.d/mkl.sh'

2
ci/jenkins/step/build.groovy
View File

@ -3,7 +3,7 @@ timeout(time: 60, unit: 'MINUTES') {
withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
def checkResult = sh(script: "./check_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache", returnStatus: true)
if ("${env.BINRARY_VERSION}" == "gpu") {
sh ". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -g -x -u -c"
sh ". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -g -u -c"
} else {
sh ". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -u -c"
}

1
ci/scripts/before-install.sh
View File

@ -6,6 +6,5 @@ export CCACHE_COMPRESS=1
export CCACHE_COMPRESSLEVEL=5
export CCACHE_COMPILERCHECK=content
export PATH=/usr/lib/ccache/:$PATH
ccache --show-stats
set +ex

13
ci/scripts/build.sh
View File

@ -17,7 +17,6 @@ BUILD_UNITTEST="OFF"
INSTALL_PREFIX="/var/lib/milvus"
FAISS_ROOT=""
PRIVILEGES="OFF"
CUSTOMIZATION="OFF" # default use origin faiss
BUILD_COVERAGE="OFF"
USE_JFROG_CACHE="OFF"
RUN_CPPLINT="OFF"
@ -25,7 +24,7 @@ GPU_VERSION="OFF"
WITH_MKL="OFF"
CUDA_COMPILER=/usr/local/cuda/bin/nvcc
while getopts "o:t:b:f:pgxulcjmh" arg
while getopts "o:t:b:f:pgulcjmh" arg
do
case $arg in
o)
@ -46,9 +45,6 @@ do
g)
GPU_VERSION="ON";
;;
x)
CUSTOMIZATION="ON";
;;
u)
echo "Build and run unittest cases" ;
BUILD_UNITTEST="ON";
@ -75,7 +71,6 @@ parameter:
-f: faiss root path
-p: install command with elevated privileges
-g: gpu version
-x: milvus customization (default: OFF)
-u: building unit test options(default: OFF)
-l: run cpplint, clang-format and clang-tidy(default: OFF)
-c: code coverage(default: OFF)
@ -84,7 +79,7 @@ parameter:
-h: help
usage:
./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-p] [-g] [-x] [-u] [-l] [-c] [-j] [-m] [-h]
./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-p] [-g] [-u] [-l] [-c] [-j] [-m] [-h]
"
exit 0
;;
@ -95,6 +90,9 @@ usage:
esac
done
echo -e "===\n=== ccache statistics before build\n==="
ccache --show-stats
if [[ ! -d ${CORE_BUILD_DIR} ]]; then
mkdir ${CORE_BUILD_DIR}
fi
@ -106,7 +104,6 @@ CMAKE_CMD="cmake \
-DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
-DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
-DMILVUS_GPU_VERSION=${GPU_VERSION} \
-DCUSTOMIZATION=${CUSTOMIZATION} \
-DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
-DBUILD_COVERAGE=${BUILD_COVERAGE} \
-DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \

13
ci/travis/before-install.sh
View File

@ -1,13 +0,0 @@
#!/bin/bash
set -ex
if [[ "${TRAVIS_OS_NAME}" == "linux" ]]; then
export CCACHE_COMPRESS=1
export CCACHE_COMPRESSLEVEL=5
export CCACHE_COMPILERCHECK=content
export PATH=/usr/lib/ccache/:$PATH
ccache --show-stats
fi
set +ex

44
ci/travis/install_dependency.sh
View File

@ -1,44 +0,0 @@
#!/usr/bin/env bash
set -e
wget -P /tmp https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
sudo apt-key add /tmp/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
echo "deb https://apt.repos.intel.com/mkl all main" | \
sudo tee /etc/apt/sources.list.d/intel-mkl.list
sudo wget -O /usr/share/keyrings/apache-arrow-keyring.gpg https://dl.bintray.com/apache/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-keyring.gpg
sudo tee /etc/apt/sources.list.d/apache-arrow.list <<APT_LINE
deb [arch=amd64 signed-by=/usr/share/keyrings/apache-arrow-keyring.gpg] https://dl.bintray.com/apache/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/ $(lsb_release --codename --short) main
deb-src [signed-by=/usr/share/keyrings/apache-arrow-keyring.gpg] https://dl.bintray.com/apache/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/ $(lsb_release --codename --short) main
APT_LINE
sudo apt-get update -qq
sudo apt-get install -y -q --no-install-recommends \
gfortran \
lsb-core \
libtool \
automake \
ccache \
pkg-config \
libarrow-dev \
libjemalloc-dev \
libboost-serialization-dev \
libboost-filesystem-dev \
libboost-system-dev \
libboost-regex-dev \
intel-mkl-gnu-2019.5-281 \
intel-mkl-core-2019.5-281 \
libmysqlclient-dev \
clang-format-6.0 \
clang-tidy-6.0 \
lcov
sudo ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so \
/usr/lib/x86_64-linux-gnu/libmysqlclient_r.so
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/intel/compilers_and_libraries_2019.5.281/linux/mkl/lib/intel64

24
ci/travis/travis_build.sh
View File

@ -1,24 +0,0 @@
#!/usr/bin/env bash
set -ex
source $TRAVIS_BUILD_DIR/ci/travis/travis_env_common.sh
only_library_mode=no
while true; do
case "$1" in
--only-library)
only_library_mode=yes
shift ;;
*) break ;;
esac
done
BUILD_COMMON_FLAGS="-t ${MILVUS_BUILD_TYPE} -o ${MILVUS_INSTALL_PREFIX} -b ${MILVUS_BUILD_DIR}"
if [ $only_library_mode == "yes" ]; then
${TRAVIS_BUILD_DIR}/ci/scripts/build.sh ${BUILD_COMMON_FLAGS} -m -p
else
${TRAVIS_BUILD_DIR}/ci/scripts/build.sh ${BUILD_COMMON_FLAGS} -m -p -u -c
fi

10
ci/travis/travis_env_common.sh
View File

@ -1,10 +0,0 @@
export MILVUS_CORE_DIR=${TRAVIS_BUILD_DIR}/core
export MILVUS_BUILD_DIR=${TRAVIS_BUILD_DIR}/core/cmake_build
export MILVUS_INSTALL_PREFIX=/var/lib/milvus
export MILVUS_TRAVIS_COVERAGE=${MILVUS_TRAVIS_COVERAGE:=0}
if [ "${MILVUS_TRAVIS_COVERAGE}" == "1" ]; then
export MILVUS_CPP_COVERAGE_FILE=${TRAVIS_BUILD_DIR}/output_new.info
fi
export MILVUS_BUILD_TYPE=${MILVUS_BUILD_TYPE:=Release}

48
docker-compose.yml Normal file
View File

@ -0,0 +1,48 @@
version: '3.5'
volumes:
amd64-ubuntu-18.04-cache:
x-ccache: &ccache
CCACHE_COMPILERCHECK: content
CCACHE_COMPRESS: 1
CCACHE_COMPRESSLEVEL: 5
CCACHE_MAXSIZE: 2G
CCACHE_DIR: /build/ccache
x-command: &cpp-command >
/bin/bash -c "
/milvus/ci/scripts/build.sh -o ${MILVUS_INSTALL_PREFIX} -l -u -c
/milvus/ci/scripts/coverage.sh -o ${MILVUS_INSTALL_PREFIX} -u root -p 123456 -t mysql"
services:
db:
image: mysql:5.6
ports:
- "3306:3306"
environment:
MYSQL_ROOT_PASSWORD: 123456
networks:
milvus:
aliases:
- "mysql"
ubuntu-core:
image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-core
build:
context: .
dockerfile: ci/docker/ubuntu-${UBUNTU}-core.dockerfile
cache_from:
- ${REPO}:${ARCH}-ubuntu-${UBUNTU}-core
shm_size: 2G
environment:
<<: *ccache
volumes: &ubuntu-volumes
- .:/milvus:delegated
- ${ARCH}-ubuntu-${UBUNTU}-cache:/build:delegated
networks:
- milvus
command: *cpp-command
networks:
milvus: