test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-02 20:09:57 +08:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

delete jfrog cache

Browse Source
This commit is contained in:
quicksilver 2019-12-25 12:04:00 +08:00
parent ac77e075e4
commit 24fd7ceede
8 changed files with 189 additions and 1441 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
core/build.sh
View File

@ -8,7 +8,6 @@ MAKE_CLEAN="OFF"
BUILD_COVERAGE="OFF"
DB_PATH="/tmp/milvus"
PROFILING="OFF"
USE_JFROG_CACHE="OFF"
RUN_CPPLINT="OFF"
CUSTOMIZATION="OFF" # default use ori faiss
CUDA_COMPILER=/usr/local/cuda/bin/nvcc
@ -19,7 +18,7 @@ FAISS_SOURCE="BUNDLED"
WITH_PROMETHEUS="ON"
FIU_ENABLE="OFF"
while getopts "p:d:t:f:ulrcgjhxzmei" arg; do
while getopts "p:d:t:f:ulrcghxzmei" arg; do
case $arg in
p)
INSTALL_PREFIX=$OPTARG
@ -53,9 +52,6 @@ while getopts "p:d:t:f:ulrcgjhxzmei" arg; do
z)
PROFILING="ON"
;;
j)
USE_JFROG_CACHE="ON"
;;
x)
CUSTOMIZATION="ON"
;;
@ -86,7 +82,6 @@ parameter:
-r: remove previous build directory(default: OFF)
-c: code coverage(default: OFF)
-z: profiling(default: OFF)
-j: use jfrog cache build directory(default: OFF)
-g: build GPU version(default: OFF)
-m: build with MKL(default: OFF)
-e: build without prometheus(default: OFF)
@ -94,7 +89,7 @@ parameter:
-h: help
usage:
./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -f \${FAISS_ROOT} [-u] [-l] [-r] [-c] [-z] [-j] [-g] [-m] [-e] [-h]
./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -f \${FAISS_ROOT} [-u] [-l] [-r] [-c] [-z] [-g] [-m] [-e] [-h]
"
exit 0
;;
@ -125,7 +120,6 @@ CMAKE_CMD="cmake \
-DBUILD_COVERAGE=${BUILD_COVERAGE} \
-DMILVUS_DB_PATH=${DB_PATH} \
-DMILVUS_ENABLE_PROFILING=${PROFILING} \
-DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
-DCUSTOMIZATION=${CUSTOMIZATION} \
-DMILVUS_GPU_VERSION=${GPU_VERSION} \
-DFAISS_WITH_MKL=${WITH_MKL} \

623
core/cmake/ThirdPartyPackages.cmake
View File

@ -131,43 +131,6 @@ endif (UNIX)
# thirdparty directory
set(THIRDPARTY_DIR "${MILVUS_SOURCE_DIR}/thirdparty")
# ----------------------------------------------------------------------
# JFrog
if (NOT DEFINED USE_JFROG_CACHE)
set(USE_JFROG_CACHE "OFF")
endif ()
if (USE_JFROG_CACHE STREQUAL "ON")
if (DEFINED ENV{JFROG_ARTFACTORY_URL})
set(JFROG_ARTFACTORY_URL "$ENV{JFROG_ARTFACTORY_URL}")
endif ()
if (NOT DEFINED JFROG_ARTFACTORY_URL)
message(FATAL_ERROR "JFROG_ARTFACTORY_URL is not set")
endif ()
if (UBUNTU_FOUND)
set(JFROG_ARTFACTORY_CACHE_URL "${JFROG_ARTFACTORY_URL}/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${UBUNTU_VERSION}/${MILVUS_BUILD_ARCH}/${BUILD_TYPE}")
else ()
set(JFROG_ARTFACTORY_CACHE_URL "${JFROG_ARTFACTORY_URL}/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${MILVUS_BUILD_ARCH}/${BUILD_TYPE}")
endif ()
if (DEFINED ENV{JFROG_USER_NAME})
set(JFROG_USER_NAME "$ENV{JFROG_USER_NAME}")
endif ()
if (NOT DEFINED JFROG_USER_NAME)
message(FATAL_ERROR "JFROG_USER_NAME is not set")
endif ()
if (DEFINED ENV{JFROG_PASSWORD})
set(JFROG_PASSWORD "$ENV{JFROG_PASSWORD}")
endif ()
if (NOT DEFINED JFROG_PASSWORD)
message(FATAL_ERROR "JFROG_PASSWORD is not set")
endif ()
set(THIRDPARTY_PACKAGE_CACHE "${THIRDPARTY_DIR}/cache")
if (NOT EXISTS ${THIRDPARTY_PACKAGE_CACHE})
message(STATUS "Will create cached directory: ${THIRDPARTY_PACKAGE_CACHE}")
file(MAKE_DIRECTORY ${THIRDPARTY_PACKAGE_CACHE})
endif ()
endif ()
macro(resolve_dependency DEPENDENCY_NAME)
if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO")
find_package(${DEPENDENCY_NAME} MODULE)
@ -400,48 +363,19 @@ macro(build_gtest)
"${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
if (USE_JFROG_CACHE STREQUAL "ON")
set(GTEST_CACHE_PACKAGE_NAME "googletest_${GTEST_MD5}.tar.gz")
set(GTEST_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${GTEST_CACHE_PACKAGE_NAME}")
set(GTEST_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${GTEST_CACHE_PACKAGE_NAME}")
file(DOWNLOAD ${GTEST_CACHE_URL} ${GTEST_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${GTEST_CACHE_URL} TO ${GTEST_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (NOT status_code EQUAL 0)
ExternalProject_Add(googletest_ep
URL
${GTEST_SOURCE_URL}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
BUILD_BYPRODUCTS
${GTEST_STATIC_LIB}
${GTEST_MAIN_STATIC_LIB}
${GMOCK_STATIC_LIB}
CMAKE_ARGS
${GTEST_CMAKE_ARGS}
${EP_LOG_OPTIONS})
ExternalProject_Create_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GTEST_CACHE_URL})
else ()
ExternalProject_Use_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif ()
else ()
ExternalProject_Add(googletest_ep
URL
${GTEST_SOURCE_URL}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
BUILD_BYPRODUCTS
${GTEST_STATIC_LIB}
${GTEST_MAIN_STATIC_LIB}
${GMOCK_STATIC_LIB}
CMAKE_ARGS
${GTEST_CMAKE_ARGS}
${EP_LOG_OPTIONS})
endif ()
ExternalProject_Add(googletest_ep
URL
${GTEST_SOURCE_URL}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
BUILD_BYPRODUCTS
${GTEST_STATIC_LIB}
${GTEST_MAIN_STATIC_LIB}
${GMOCK_STATIC_LIB}
CMAKE_ARGS
${GTEST_CMAKE_ARGS}
${EP_LOG_OPTIONS})
# The include directory must exist before it is referenced by a target.
file(MAKE_DIRECTORY "${GTEST_INCLUDE_DIR}")
@ -495,52 +429,19 @@ macro(build_mysqlpp)
"CXXFLAGS=${EP_CXX_FLAGS}"
"LDFLAGS=-pthread")
if (USE_JFROG_CACHE STREQUAL "ON")
set(MYSQLPP_CACHE_PACKAGE_NAME "mysqlpp_${MYSQLPP_MD5}.tar.gz")
set(MYSQLPP_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${MYSQLPP_CACHE_PACKAGE_NAME}")
set(MYSQLPP_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${MYSQLPP_CACHE_PACKAGE_NAME}")
execute_process(COMMAND wget -q --method HEAD ${MYSQLPP_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${MYSQLPP_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(mysqlpp_ep
URL
${MYSQLPP_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${MYSQLPP_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS}
BUILD_IN_SOURCE
1
BUILD_BYPRODUCTS
${MYSQLPP_SHARED_LIB})
ExternalProject_Create_Cache(mysqlpp_ep ${MYSQLPP_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/mysqlpp_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${MYSQLPP_CACHE_URL})
else ()
file(DOWNLOAD ${MYSQLPP_CACHE_URL} ${MYSQLPP_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${MYSQLPP_CACHE_URL} TO ${MYSQLPP_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(mysqlpp_ep ${MYSQLPP_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif ()
endif ()
else ()
externalproject_add(mysqlpp_ep
URL
${MYSQLPP_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${MYSQLPP_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS}
BUILD_IN_SOURCE
1
BUILD_BYPRODUCTS
${MYSQLPP_SHARED_LIB})
endif ()
externalproject_add(mysqlpp_ep
URL
${MYSQLPP_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${MYSQLPP_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS}
BUILD_IN_SOURCE
1
BUILD_BYPRODUCTS
${MYSQLPP_SHARED_LIB})
file(MAKE_DIRECTORY "${MYSQLPP_INCLUDE_DIR}")
add_library(mysqlpp SHARED IMPORTED)
@ -586,78 +487,29 @@ macro(build_prometheus)
"-DCMAKE_INSTALL_PREFIX=${PROMETHEUS_PREFIX}"
-DCMAKE_BUILD_TYPE=Release)
if (USE_JFROG_CACHE STREQUAL "ON")
execute_process(COMMAND sh -c "git ls-remote --heads --tags ${PROMETHEUS_SOURCE_URL} ${PROMETHEUS_VERSION} | cut -f 1" OUTPUT_VARIABLE PROMETHEUS_LAST_COMMIT_ID)
if (${PROMETHEUS_LAST_COMMIT_ID} MATCHES "^[^#][a-z0-9]+")
string(MD5 PROMETHEUS_COMBINE_MD5 "${PROMETHEUS_LAST_COMMIT_ID}")
set(PROMETHEUS_CACHE_PACKAGE_NAME "prometheus_${PROMETHEUS_COMBINE_MD5}.tar.gz")
set(PROMETHEUS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${PROMETHEUS_CACHE_PACKAGE_NAME}")
set(PROMETHEUS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${PROMETHEUS_CACHE_PACKAGE_NAME}")
execute_process(COMMAND wget -q --method HEAD ${PROMETHEUS_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${PROMETHEUS_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(prometheus_ep
GIT_REPOSITORY
${PROMETHEUS_SOURCE_URL}
GIT_TAG
${PROMETHEUS_VERSION}
GIT_SHALLOW
TRUE
${EP_LOG_OPTIONS}
CMAKE_ARGS
${PROMETHEUS_CMAKE_ARGS}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE}
"DESTDIR=${PROMETHEUS_PREFIX}"
install
BUILD_BYPRODUCTS
"${PROMETHEUS_CORE_STATIC_LIB}"
"${PROMETHEUS_PUSH_STATIC_LIB}"
"${PROMETHEUS_PULL_STATIC_LIB}")
ExternalProject_Create_Cache(prometheus_ep ${PROMETHEUS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/prometheus_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${PROMETHEUS_CACHE_URL})
else ()
file(DOWNLOAD ${PROMETHEUS_CACHE_URL} ${PROMETHEUS_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${PROMETHEUS_CACHE_URL} TO ${PROMETHEUS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(prometheus_ep ${PROMETHEUS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif ()
endif ()
else ()
message(FATAL_ERROR "The last commit ID of \"${PROMETHEUS_SOURCE_URL}\" repository don't match!")
endif ()
else ()
externalproject_add(prometheus_ep
GIT_REPOSITORY
${PROMETHEUS_SOURCE_URL}
GIT_TAG
${PROMETHEUS_VERSION}
GIT_SHALLOW
TRUE
${EP_LOG_OPTIONS}
CMAKE_ARGS
${PROMETHEUS_CMAKE_ARGS}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE}
"DESTDIR=${PROMETHEUS_PREFIX}"
install
BUILD_BYPRODUCTS
"${PROMETHEUS_CORE_STATIC_LIB}"
"${PROMETHEUS_PUSH_STATIC_LIB}"
"${PROMETHEUS_PULL_STATIC_LIB}")
endif ()
externalproject_add(prometheus_ep
GIT_REPOSITORY
${PROMETHEUS_SOURCE_URL}
GIT_TAG
${PROMETHEUS_VERSION}
GIT_SHALLOW
TRUE
${EP_LOG_OPTIONS}
CMAKE_ARGS
${PROMETHEUS_CMAKE_ARGS}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE}
"DESTDIR=${PROMETHEUS_PREFIX}"
install
BUILD_BYPRODUCTS
"${PROMETHEUS_CORE_STATIC_LIB}"
"${PROMETHEUS_PUSH_STATIC_LIB}"
"${PROMETHEUS_PULL_STATIC_LIB}")
file(MAKE_DIRECTORY "${PROMETHEUS_PREFIX}/push/include")
add_library(prometheus-cpp-push STATIC IMPORTED)
@ -713,54 +565,20 @@ macro(build_sqlite)
"CFLAGS=${EP_C_FLAGS}"
"CXXFLAGS=${EP_CXX_FLAGS}")
if (USE_JFROG_CACHE STREQUAL "ON")
set(SQLITE_CACHE_PACKAGE_NAME "sqlite_${SQLITE_MD5}.tar.gz")
set(SQLITE_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${SQLITE_CACHE_PACKAGE_NAME}")
set(SQLITE_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${SQLITE_CACHE_PACKAGE_NAME}")
execute_process(COMMAND wget -q --method HEAD ${SQLITE_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${SQLITE_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(sqlite_ep
URL
${SQLITE_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${SQLITE_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
BUILD_IN_SOURCE
1
BUILD_BYPRODUCTS
"${SQLITE_STATIC_LIB}")
ExternalProject_Create_Cache(sqlite_ep ${SQLITE_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/sqlite_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${SQLITE_CACHE_URL})
else ()
file(DOWNLOAD ${SQLITE_CACHE_URL} ${SQLITE_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${SQLITE_CACHE_URL} TO ${SQLITE_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(sqlite_ep ${SQLITE_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif ()
endif ()
else ()
externalproject_add(sqlite_ep
URL
${SQLITE_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${SQLITE_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
BUILD_IN_SOURCE
1
BUILD_BYPRODUCTS
"${SQLITE_STATIC_LIB}")
endif ()
externalproject_add(sqlite_ep
URL
${SQLITE_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${SQLITE_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
BUILD_IN_SOURCE
1
BUILD_BYPRODUCTS
"${SQLITE_STATIC_LIB}")
file(MAKE_DIRECTORY "${SQLITE_INCLUDE_DIR}")
add_library(sqlite STATIC IMPORTED)
@ -851,48 +669,17 @@ macro(build_yamlcpp)
-DYAML_CPP_BUILD_TESTS=OFF
-DYAML_CPP_BUILD_TOOLS=OFF)
if (USE_JFROG_CACHE STREQUAL "ON")
set(YAMLCPP_CACHE_PACKAGE_NAME "yaml-cpp_${YAMLCPP_MD5}.tar.gz")
set(YAMLCPP_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${YAMLCPP_CACHE_PACKAGE_NAME}")
set(YAMLCPP_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${YAMLCPP_CACHE_PACKAGE_NAME}")
execute_process(COMMAND wget -q --method HEAD ${YAMLCPP_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${YAMLCPP_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(yaml-cpp_ep
URL
${YAMLCPP_SOURCE_URL}
${EP_LOG_OPTIONS}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
BUILD_BYPRODUCTS
"${YAMLCPP_STATIC_LIB}"
CMAKE_ARGS
${YAMLCPP_CMAKE_ARGS})
ExternalProject_Create_Cache(yaml-cpp_ep ${YAMLCPP_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/yaml-cpp_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${YAMLCPP_CACHE_URL})
else ()
file(DOWNLOAD ${YAMLCPP_CACHE_URL} ${YAMLCPP_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${YAMLCPP_CACHE_URL} TO ${YAMLCPP_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(yaml-cpp_ep ${YAMLCPP_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif ()
endif ()
else ()
externalproject_add(yaml-cpp_ep
URL
${YAMLCPP_SOURCE_URL}
${EP_LOG_OPTIONS}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
BUILD_BYPRODUCTS
"${YAMLCPP_STATIC_LIB}"
CMAKE_ARGS
${YAMLCPP_CMAKE_ARGS})
endif ()
externalproject_add(yaml-cpp_ep
URL
${YAMLCPP_SOURCE_URL}
${EP_LOG_OPTIONS}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
BUILD_BYPRODUCTS
"${YAMLCPP_STATIC_LIB}"
CMAKE_ARGS
${YAMLCPP_CMAKE_ARGS})
file(MAKE_DIRECTORY "${YAMLCPP_INCLUDE_DIR}")
add_library(yaml-cpp STATIC IMPORTED)
@ -921,56 +708,21 @@ macro(build_libunwind)
set(LIBUNWIND_SHARED_LIB "${LIBUNWIND_PREFIX}/lib/libunwind${CMAKE_SHARED_LIBRARY_SUFFIX}")
set(LIBUNWIND_CONFIGURE_ARGS "--prefix=${LIBUNWIND_PREFIX}")
if (USE_JFROG_CACHE STREQUAL "ON")
set(LIBUNWIND_CACHE_PACKAGE_NAME "libunwind_${LIBUNWIND_MD5}.tar.gz")
set(LIBUNWIND_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${LIBUNWIND_CACHE_PACKAGE_NAME}")
set(LIBUNWIND_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${LIBUNWIND_CACHE_PACKAGE_NAME}")
execute_process(COMMAND wget -q --method HEAD ${LIBUNWIND_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${LIBUNWIND_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(libunwind_ep
URL
${LIBUNWIND_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${LIBUNWIND_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS}
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE} install
BUILD_BYPRODUCTS
${LIBUNWIND_SHARED_LIB})
ExternalProject_Create_Cache(libunwind_ep ${LIBUNWIND_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/libunwind_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${LIBUNWIND_CACHE_URL})
else ()
file(DOWNLOAD ${LIBUNWIND_CACHE_URL} ${LIBUNWIND_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${LIBUNWIND_CACHE_URL} TO ${LIBUNWIND_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(libunwind_ep ${LIBUNWIND_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif ()
endif ()
else ()
externalproject_add(libunwind_ep
URL
${LIBUNWIND_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${LIBUNWIND_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS}
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE} install
BUILD_BYPRODUCTS
${LIBUNWIND_SHARED_LIB})
endif ()
externalproject_add(libunwind_ep
URL
${LIBUNWIND_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${LIBUNWIND_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS}
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE} install
BUILD_BYPRODUCTS
${LIBUNWIND_SHARED_LIB})
file(MAKE_DIRECTORY "${LIBUNWIND_INCLUDE_DIR}")
@ -999,56 +751,21 @@ macro(build_gperftools)
set(GPERFTOOLS_STATIC_LIB "${GPERFTOOLS_PREFIX}/lib/libprofiler${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(GPERFTOOLS_CONFIGURE_ARGS "--prefix=${GPERFTOOLS_PREFIX}")
if (USE_JFROG_CACHE STREQUAL "ON")
set(GPERFTOOLS_CACHE_PACKAGE_NAME "gperftools_${GPERFTOOLS_MD5}.tar.gz")
set(GPERFTOOLS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${GPERFTOOLS_CACHE_PACKAGE_NAME}")
set(GPERFTOOLS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${GPERFTOOLS_CACHE_PACKAGE_NAME}")
execute_process(COMMAND wget -q --method HEAD ${GPERFTOOLS_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${GPERFTOOLS_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(gperftools_ep
URL
${GPERFTOOLS_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${GPERFTOOLS_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS}
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE} install
BUILD_BYPRODUCTS
${GPERFTOOLS_STATIC_LIB})
ExternalProject_Create_Cache(gperftools_ep ${GPERFTOOLS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/gperftools_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GPERFTOOLS_CACHE_URL})
else ()
file(DOWNLOAD ${GPERFTOOLS_CACHE_URL} ${GPERFTOOLS_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${GPERFTOOLS_CACHE_URL} TO ${GPERFTOOLS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(gperftools_ep ${GPERFTOOLS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif ()
endif ()
else ()
externalproject_add(gperftools_ep
URL
${GPERFTOOLS_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${GPERFTOOLS_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS}
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE} install
BUILD_BYPRODUCTS
${GPERFTOOLS_STATIC_LIB})
endif ()
externalproject_add(gperftools_ep
URL
${GPERFTOOLS_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${GPERFTOOLS_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS}
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE} install
BUILD_BYPRODUCTS
${GPERFTOOLS_STATIC_LIB})
ExternalProject_Add_StepDependencies(gperftools_ep build libunwind_ep)
@ -1086,67 +803,26 @@ macro(build_grpc)
set(GRPC_PROTOBUF_STATIC_LIB "${GRPC_PROTOBUF_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}protobuf${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(GRPC_PROTOC_STATIC_LIB "${GRPC_PROTOBUF_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}protoc${CMAKE_STATIC_LIBRARY_SUFFIX}")
if (USE_JFROG_CACHE STREQUAL "ON")
set(GRPC_CACHE_PACKAGE_NAME "grpc_${GRPC_MD5}.tar.gz")
set(GRPC_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${GRPC_CACHE_PACKAGE_NAME}")
set(GRPC_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${GRPC_CACHE_PACKAGE_NAME}")
externalproject_add(grpc_ep
URL
${GRPC_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
""
BUILD_IN_SOURCE
1
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS} prefix=${GRPC_PREFIX}
INSTALL_COMMAND
${MAKE} install prefix=${GRPC_PREFIX}
BUILD_BYPRODUCTS
${GRPC_STATIC_LIB}
${GRPC++_STATIC_LIB}
${GRPCPP_CHANNELZ_STATIC_LIB}
${GRPC_PROTOBUF_STATIC_LIB}
${GRPC_PROTOC_STATIC_LIB})
execute_process(COMMAND wget -q --method HEAD ${GRPC_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${GRPC_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(grpc_ep
URL
${GRPC_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
""
BUILD_IN_SOURCE
1
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS} prefix=${GRPC_PREFIX}
INSTALL_COMMAND
${MAKE} install prefix=${GRPC_PREFIX}
BUILD_BYPRODUCTS
${GRPC_STATIC_LIB}
${GRPC++_STATIC_LIB}
${GRPCPP_CHANNELZ_STATIC_LIB}
${GRPC_PROTOBUF_STATIC_LIB}
${GRPC_PROTOC_STATIC_LIB})
ExternalProject_Add_StepDependencies(grpc_ep build zlib_ep)
ExternalProject_Create_Cache(grpc_ep ${GRPC_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GRPC_CACHE_URL})
else ()
file(DOWNLOAD ${GRPC_CACHE_URL} ${GRPC_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${GRPC_CACHE_URL} TO ${GRPC_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(grpc_ep ${GRPC_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif ()
endif ()
else ()
externalproject_add(grpc_ep
URL
${GRPC_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
""
BUILD_IN_SOURCE
1
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS} prefix=${GRPC_PREFIX}
INSTALL_COMMAND
${MAKE} install prefix=${GRPC_PREFIX}
BUILD_BYPRODUCTS
${GRPC_STATIC_LIB}
${GRPC++_STATIC_LIB}
${GRPCPP_CHANNELZ_STATIC_LIB}
${GRPC_PROTOBUF_STATIC_LIB}
${GRPC_PROTOC_STATIC_LIB})
ExternalProject_Add_StepDependencies(grpc_ep build zlib_ep)
endif ()
ExternalProject_Add_StepDependencies(grpc_ep build zlib_ep)
file(MAKE_DIRECTORY "${GRPC_INCLUDE_DIR}")
@ -1209,48 +885,17 @@ macro(build_zlib)
set(ZLIB_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX}"
-DBUILD_SHARED_LIBS=OFF)
if (USE_JFROG_CACHE STREQUAL "ON")
set(ZLIB_CACHE_PACKAGE_NAME "zlib_${ZLIB_MD5}.tar.gz")
set(ZLIB_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${ZLIB_CACHE_PACKAGE_NAME}")
set(ZLIB_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${ZLIB_CACHE_PACKAGE_NAME}")
execute_process(COMMAND wget -q --method HEAD ${ZLIB_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${ZLIB_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(zlib_ep
URL
${ZLIB_SOURCE_URL}
${EP_LOG_OPTIONS}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
BUILD_BYPRODUCTS
"${ZLIB_STATIC_LIB}"
CMAKE_ARGS
${ZLIB_CMAKE_ARGS})
ExternalProject_Create_Cache(zlib_ep ${ZLIB_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/zlib_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ZLIB_CACHE_URL})
else ()
file(DOWNLOAD ${ZLIB_CACHE_URL} ${ZLIB_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${ZLIB_CACHE_URL} TO ${ZLIB_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(zlib_ep ${ZLIB_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif ()
endif ()
else ()
externalproject_add(zlib_ep
URL
${ZLIB_SOURCE_URL}
${EP_LOG_OPTIONS}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
BUILD_BYPRODUCTS
"${ZLIB_STATIC_LIB}"
CMAKE_ARGS
${ZLIB_CMAKE_ARGS})
endif ()
externalproject_add(zlib_ep
URL
${ZLIB_SOURCE_URL}
${EP_LOG_OPTIONS}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
BUILD_BYPRODUCTS
"${ZLIB_STATIC_LIB}"
CMAKE_ARGS
${ZLIB_CMAKE_ARGS})
file(MAKE_DIRECTORY "${ZLIB_INCLUDE_DIR}")
add_library(zlib STATIC IMPORTED)

7
core/src/index/build.sh
View File

@ -6,9 +6,8 @@ INSTALL_PREFIX=$(pwd)/cmake_build
MAKE_CLEAN="OFF"
PROFILING="OFF"
FAISS_WITH_MKL="OFF"
USE_JFROG_CACHE="OFF"
while getopts "p:d:t:uhrcgmj" arg
while getopts "p:d:t:uhrcgm" arg
do
case $arg in
t)
@ -33,9 +32,6 @@ do
m)
FAISS_WITH_MKL="ON"
;;
j)
USE_JFROG_CACHE="ON"
;;
h) # help
echo "
@ -75,7 +71,6 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then
-DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
-DMILVUS_ENABLE_PROFILING=${PROFILING} \
-DFAISS_WITH_MKL=${FAISS_WITH_MKL} \
-DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
../"
echo ${CMAKE_CMD}

244
core/src/index/cmake/ThirdPartyPackagesCore.cmake
View File

@ -115,20 +115,6 @@ endif (UNIX)
# thirdparty directory
set(THIRDPARTY_DIR "${INDEX_SOURCE_DIR}/thirdparty")
# ----------------------------------------------------------------------
# JFrog
if (NOT DEFINED USE_JFROG_CACHE)
set(USE_JFROG_CACHE "OFF")
endif ()
if (USE_JFROG_CACHE STREQUAL "ON")
set(JFROG_ARTFACTORY_CACHE_URL "${JFROG_ARTFACTORY_URL}/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${KNOWHERE_BUILD_ARCH}/${BUILD_TYPE}")
set(THIRDPARTY_PACKAGE_CACHE "${THIRDPARTY_DIR}/cache")
if (NOT EXISTS ${THIRDPARTY_PACKAGE_CACHE})
message(STATUS "Will create cached directory: ${THIRDPARTY_PACKAGE_CACHE}")
file(MAKE_DIRECTORY ${THIRDPARTY_PACKAGE_CACHE})
endif ()
endif ()
# ----------------------------------------------------------------------
# ExternalProject options
@ -284,71 +270,25 @@ macro(build_arrow)
-DBOOST_SOURCE=AUTO #try to find BOOST in the system default locations and build from source if not found
)
if (USE_JFROG_CACHE STREQUAL "ON")
execute_process(COMMAND sh -c "git ls-remote --heads --tags ${ARROW_SOURCE_URL} ${ARROW_VERSION} | cut -f 1" OUTPUT_VARIABLE ARROW_LAST_COMMIT_ID)
if (${ARROW_LAST_COMMIT_ID} MATCHES "^[^#][a-z0-9]+")
string(MD5 ARROW_COMBINE_MD5 "${ARROW_LAST_COMMIT_ID}")
set(ARROW_CACHE_PACKAGE_NAME "arrow_${ARROW_COMBINE_MD5}.tar.gz")
set(ARROW_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${ARROW_CACHE_PACKAGE_NAME}")
set(ARROW_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${ARROW_CACHE_PACKAGE_NAME}")
execute_process(COMMAND wget -q --method HEAD ${ARROW_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${ARROW_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(arrow_ep
GIT_REPOSITORY
${ARROW_SOURCE_URL}
GIT_TAG
${ARROW_VERSION}
GIT_SHALLOW
TRUE
SOURCE_SUBDIR
cpp
${EP_LOG_OPTIONS}
CMAKE_ARGS
${ARROW_CMAKE_ARGS}
BUILD_COMMAND
""
INSTALL_COMMAND
${MAKE} ${MAKE_BUILD_ARGS} install
BUILD_BYPRODUCTS
"${ARROW_STATIC_LIB}"
)
ExternalProject_Create_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} "${INDEX_BINARY_DIR}/arrow_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ARROW_CACHE_URL})
else ()
file(DOWNLOAD ${ARROW_CACHE_URL} ${ARROW_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${ARROW_CACHE_URL} TO ${ARROW_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} ${INDEX_BINARY_DIR})
endif ()
endif ()
else ()
message(FATAL_ERROR "The last commit ID of \"${ARROW_SOURCE_URL}\" repository don't match!")
endif ()
else ()
externalproject_add(arrow_ep
GIT_REPOSITORY
${ARROW_SOURCE_URL}
GIT_TAG
${ARROW_VERSION}
GIT_SHALLOW
TRUE
SOURCE_SUBDIR
cpp
${EP_LOG_OPTIONS}
CMAKE_ARGS
${ARROW_CMAKE_ARGS}
BUILD_COMMAND
""
INSTALL_COMMAND
${MAKE} ${MAKE_BUILD_ARGS} install
BUILD_BYPRODUCTS
"${ARROW_STATIC_LIB}"
)
endif ()
externalproject_add(arrow_ep
GIT_REPOSITORY
${ARROW_SOURCE_URL}
GIT_TAG
${ARROW_VERSION}
GIT_SHALLOW
TRUE
SOURCE_SUBDIR
cpp
${EP_LOG_OPTIONS}
CMAKE_ARGS
${ARROW_CMAKE_ARGS}
BUILD_COMMAND
""
INSTALL_COMMAND
${MAKE} ${MAKE_BUILD_ARGS} install
BUILD_BYPRODUCTS
"${ARROW_STATIC_LIB}"
)
file(MAKE_DIRECTORY "${ARROW_INCLUDE_DIR}")
add_library(arrow STATIC IMPORTED)
@ -409,39 +349,6 @@ macro(build_gtest)
"${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
if (USE_JFROG_CACHE STREQUAL "ON")
set(GTEST_CACHE_PACKAGE_NAME "googletest_${GTEST_MD5}.tar.gz")
set(GTEST_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${GTEST_CACHE_PACKAGE_NAME}")
set(GTEST_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${GTEST_CACHE_PACKAGE_NAME}")
execute_process(COMMAND wget -q --method HEAD ${GTEST_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${GTEST_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
ExternalProject_Add(googletest_ep
URL
${GTEST_SOURCE_URL}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
BUILD_BYPRODUCTS
${GTEST_STATIC_LIB}
${GTEST_MAIN_STATIC_LIB}
${GMOCK_STATIC_LIB}
CMAKE_ARGS
${GTEST_CMAKE_ARGS}
${EP_LOG_OPTIONS})
ExternalProject_Create_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} "${INDEX_BINARY_DIR}/googletest_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GTEST_CACHE_URL})
else ()
file(DOWNLOAD ${GTEST_CACHE_URL} ${GTEST_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${GTEST_CACHE_URL} TO ${GTEST_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} ${INDEX_BINARY_DIR})
endif ()
endif ()
else ()
ExternalProject_Add(googletest_ep
URL
${GTEST_SOURCE_URL}
@ -455,7 +362,6 @@ macro(build_gtest)
CMAKE_ARGS
${GTEST_CMAKE_ARGS}
${EP_LOG_OPTIONS})
endif ()
# The include directory must exist before it is referenced by a target.
file(MAKE_DIRECTORY "${GTEST_INCLUDE_DIR}")
@ -564,86 +470,40 @@ macro(build_faiss)
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} --without-cuda)
endif ()
if (USE_JFROG_CACHE STREQUAL "ON")
string(MD5 FAISS_COMBINE_MD5 "${FAISS_MD5}${LAPACK_MD5}${OPENBLAS_MD5}")
if (KNOWHERE_GPU_VERSION)
set(FAISS_COMPUTE_TYPE "gpu")
else ()
set(FAISS_COMPUTE_TYPE "cpu")
endif ()
if (FAISS_WITH_MKL)
set(FAISS_CACHE_PACKAGE_NAME "faiss_${FAISS_COMPUTE_TYPE}_mkl_${FAISS_COMBINE_MD5}.tar.gz")
else ()
set(FAISS_CACHE_PACKAGE_NAME "faiss_${FAISS_COMPUTE_TYPE}_openblas_${FAISS_COMBINE_MD5}.tar.gz")
endif ()
set(FAISS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${FAISS_CACHE_PACKAGE_NAME}")
set(FAISS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${FAISS_CACHE_PACKAGE_NAME}")
execute_process(COMMAND wget -q --method HEAD ${FAISS_CACHE_URL} RESULT_VARIABLE return_code)
message(STATUS "Check the remote file ${FAISS_CACHE_URL}. return code = ${return_code}")
if (NOT return_code EQUAL 0)
externalproject_add(faiss_ep
URL
${FAISS_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${FAISS_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS} all
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE} install
BUILD_BYPRODUCTS
${FAISS_STATIC_LIB})
ExternalProject_Create_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} "${INDEX_BINARY_DIR}/faiss_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${FAISS_CACHE_URL})
else ()
file(DOWNLOAD ${FAISS_CACHE_URL} ${FAISS_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${FAISS_CACHE_URL} TO ${FAISS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} ${INDEX_BINARY_DIR})
endif ()
endif ()
if (CUSTOMIZATION)
externalproject_add(faiss_ep
DOWNLOAD_COMMAND
""
SOURCE_DIR
${FAISS_SOURCE_DIR}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${FAISS_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS} all
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE} install
BUILD_BYPRODUCTS
${FAISS_STATIC_LIB})
else ()
if (CUSTOMIZATION)
externalproject_add(faiss_ep
DOWNLOAD_COMMAND
""
SOURCE_DIR
${FAISS_SOURCE_DIR}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${FAISS_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS} all
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE} install
BUILD_BYPRODUCTS
${FAISS_STATIC_LIB})
else ()
externalproject_add(faiss_ep
URL
${FAISS_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${FAISS_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS} all
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE} install
BUILD_BYPRODUCTS
${FAISS_STATIC_LIB})
endif ()
externalproject_add(faiss_ep
URL
${FAISS_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${FAISS_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS} all
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE} install
BUILD_BYPRODUCTS
${FAISS_STATIC_LIB})
endif ()
file(MAKE_DIRECTORY "${FAISS_INCLUDE_DIR}")

579
core/src/index/thirdparty/faiss/ci/jenkins/Jenkinsfile vendored
View File

@ -1,579 +0,0 @@
pipeline {
agent none
options {
timestamps()
}
parameters{
choice choices: ['Release', 'Debug'], description: 'Build Type', name: 'BUILD_TYPE'
choice choices: ['False', 'True'], description: 'Whether it is origin Faiss', name: 'IS_ORIGIN_FAISS'
string defaultValue: 'registry.zilliz.com', description: 'DOCKER REGISTRY URL', name: 'DOKCER_REGISTRY_URL', trim: true
string defaultValue: 'a54e38ef-c424-4ea9-9224-b25fc20e3924', description: 'DOCKER CREDENTIALS ID', name: 'DOCKER_CREDENTIALS_ID', trim: true
string defaultValue: 'http://192.168.1.201/artifactory/milvus', description: 'JFROG ARTFACTORY URL', name: 'JFROG_ARTFACTORY_URL', trim: true
string defaultValue: '76fd48ab-2b8e-4eed-834d-2eefd23bb3a6', description: 'JFROG CREDENTIALS ID', name: 'JFROG_CREDENTIALS_ID', trim: true
}
environment {
PROJECT_NAME = "milvus"
LOWER_BUILD_TYPE = params.BUILD_TYPE.toLowerCase()
SEMVER = "0.6.0"
JOBNAMES = env.JOB_NAME.split('/')
PIPELINE_NAME = "${JOBNAMES[0]}"
FAISS_ROOT_PATH="/usr/local/faiss"
NATIVE_FAISS_VERSION="1.6.0"
}
stages {
stage("Ubuntu 18.04 x86_64") {
environment {
OS_NAME = "ubuntu18.04"
CPU_ARCH = "amd64"
}
parallel {
stage("GPU Version") {
environment {
BINRARY_VERSION = "gpu"
PACKAGE_VERSION = VersionNumber([
versionNumberString : '${SEMVER}-gpu-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}'
]);
DOCKER_VERSION = "${SEMVER}-gpu-${OS_NAME}-${LOWER_BUILD_TYPE}"
}
stages {
stage("Run Build") {
agent {
kubernetes {
label "${env.BINRARY_VERSION}-build"
defaultContainer 'jnlp'
yaml """
apiVersion: v1
kind: Pod
metadata:
name: milvus-gpu-build-env
labels:
app: milvus
componet: gpu-build-env
spec:
containers:
- name: milvus-gpu-build-env
image: registry.zilliz.com/milvus/milvus-gpu-build-env:v0.6.0-ubuntu18.04
env:
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: BUILD_ENV_IMAGE_ID
value: "da9023b0f858f072672f86483a869aa87e90a5140864f89e5a012ec766d96dea"
command:
- cat
tty: true
resources:
limits:
memory: "24Gi"
cpu: "8.0"
nvidia.com/gpu: 1
requests:
memory: "16Gi"
cpu: "4.0"
- name: milvus-mysql
image: mysql:5.6
env:
- name: MYSQL_ROOT_PASSWORD
value: 123456
ports:
- containerPort: 3306
name: mysql
"""
}
}
stages {
stage('Prepare') {
steps {
container("milvus-${env.BINRARY_VERSION}-build-env") {
script {
dir ("milvus") {
checkout([$class: 'GitSCM', branches: [[name: "${env.SEMVER}"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus.git", name: 'origin', refspec: "+refs/heads/${env.SEMVER}:refs/remotes/origin/${env.SEMVER}"]]])
}
}
}
}
}
stage('Build') {
steps {
container("milvus-${env.BINRARY_VERSION}-build-env") {
script {
load "${env.WORKSPACE}/ci/jenkins/step/build.groovy"
}
}
}
}
stage('Code Coverage') {
steps {
container("milvus-${env.BINRARY_VERSION}-build-env") {
script {
load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy"
}
}
}
}
stage('Upload Package') {
steps {
container("milvus-${env.BINRARY_VERSION}-build-env") {
script {
dir ("milvus") {
load "ci/jenkins/step/package.groovy"
}
}
}
}
}
}
}
stage("Publish docker images") {
agent {
kubernetes {
label "${BINRARY_VERSION}-publish"
defaultContainer 'jnlp'
yaml """
apiVersion: v1
kind: Pod
metadata:
labels:
app: publish
componet: docker
spec:
containers:
- name: publish-images
image: registry.zilliz.com/library/docker:v1.0.0
securityContext:
privileged: true
command:
- cat
tty: true
volumeMounts:
- name: docker-sock
mountPath: /var/run/docker.sock
volumes:
- name: docker-sock
hostPath:
path: /var/run/docker.sock
"""
}
}
stages {
stage('Prepare') {
steps {
container('publish-images') {
script {
dir ("milvus") {
checkout([$class: 'GitSCM', branches: [[name: "${env.SEMVER}"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus.git", name: 'origin', refspec: "+refs/heads/${env.SEMVER}:refs/remotes/origin/${env.SEMVER}"]]])
}
}
}
}
}
stage('Publish') {
steps {
container('publish-images') {
script {
dir ("milvus") {
load "ci/jenkins/step/publishImages.groovy"
}
}
}
}
}
}
}
stage("Deploy to Development") {
environment {
FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-")
HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}".toLowerCase()
}
agent {
kubernetes {
label "${env.BINRARY_VERSION}-dev-test"
defaultContainer 'jnlp'
yaml """
apiVersion: v1
kind: Pod
metadata:
labels:
app: milvus
componet: test-env
spec:
containers:
- name: milvus-test-env
image: registry.zilliz.com/milvus/milvus-test-env:v0.1
command:
- cat
tty: true
volumeMounts:
- name: kubeconf
mountPath: /root/.kube/
readOnly: true
volumes:
- name: kubeconf
secret:
secretName: test-cluster-config
"""
}
}
stages {
stage('Prepare') {
steps {
container('milvus-test-env') {
script {
dir ("milvus") {
checkout([$class: 'GitSCM', branches: [[name: "${env.SEMVER}"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus.git", name: 'origin', refspec: "+refs/heads/${env.SEMVER}:refs/remotes/origin/${env.SEMVER}"]]])
}
}
}
}
}
stage("Deploy to Dev") {
steps {
container('milvus-test-env') {
script {
dir ("milvus") {
load "ci/jenkins/step/deploySingle2Dev.groovy"
}
}
}
}
}
stage("Dev Test") {
steps {
container('milvus-test-env') {
script {
boolean isNightlyTest = isTimeTriggeredBuild()
if (isNightlyTest) {
dir ("milvus") {
load "ci/jenkins/step/singleDevNightlyTest.groovy"
}
} else {
dir ("milvus") {
load "ci/jenkins/step/singleDevTest.groovy"
}
}
}
}
}
}
stage ("Cleanup Dev") {
steps {
container('milvus-test-env') {
script {
dir ("milvus") {
load "ci/jenkins/step/cleanupSingleDev.groovy"
}
}
}
}
}
}
post {
unsuccessful {
container('milvus-test-env') {
script {
dir ("milvus") {
load "ci/jenkins/step/cleanupSingleDev.groovy"
}
}
}
}
}
}
}
}
stage("CPU Version") {
environment {
BINRARY_VERSION = "cpu"
PACKAGE_VERSION = VersionNumber([
versionNumberString : '${SEMVER}-cpu-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}'
]);
DOCKER_VERSION = "${SEMVER}-cpu-${OS_NAME}-${LOWER_BUILD_TYPE}"
}
stages {
stage("Run Build") {
agent {
kubernetes {
label "${env.BINRARY_VERSION}-build"
defaultContainer 'jnlp'
yaml """
apiVersion: v1
kind: Pod
metadata:
name: milvus-cpu-build-env
labels:
app: milvus
componet: cpu-build-env
spec:
containers:
- name: milvus-cpu-build-env
image: registry.zilliz.com/milvus/milvus-cpu-build-env:v0.6.0-ubuntu18.04
env:
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: BUILD_ENV_IMAGE_ID
value: "23476391bec80c64f10d44a6370c73c71f011a6b95114b10ff82a60e771e11c7"
command:
- cat
tty: true
resources:
limits:
memory: "24Gi"
cpu: "8.0"
requests:
memory: "16Gi"
cpu: "4.0"
- name: milvus-mysql
image: mysql:5.6
env:
- name: MYSQL_ROOT_PASSWORD
value: 123456
ports:
- containerPort: 3306
name: mysql
"""
}
}
stages {
stage('Prepare') {
steps {
container("milvus-${env.BINRARY_VERSION}-build-env") {
script {
dir ("milvus") {
checkout([$class: 'GitSCM', branches: [[name: "${env.SEMVER}"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus.git", name: 'origin', refspec: "+refs/heads/${env.SEMVER}:refs/remotes/origin/${env.SEMVER}"]]])
}
}
}
}
}
stage('Build') {
steps {
container("milvus-${env.BINRARY_VERSION}-build-env") {
script {
load "${env.WORKSPACE}/ci/jenkins/step/build.groovy"
}
}
}
}
stage('Code Coverage') {
steps {
container("milvus-${env.BINRARY_VERSION}-build-env") {
script {
load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy"
}
}
}
}
stage('Upload Package') {
steps {
container("milvus-${env.BINRARY_VERSION}-build-env") {
script {
dir ("milvus") {
load "ci/jenkins/step/package.groovy"
}
}
}
}
}
}
}
stage("Publish docker images") {
agent {
kubernetes {
label "${BINRARY_VERSION}-publish"
defaultContainer 'jnlp'
yaml """
apiVersion: v1
kind: Pod
metadata:
labels:
app: publish
componet: docker
spec:
containers:
- name: publish-images
image: registry.zilliz.com/library/docker:v1.0.0
securityContext:
privileged: true
command:
- cat
tty: true
volumeMounts:
- name: docker-sock
mountPath: /var/run/docker.sock
volumes:
- name: docker-sock
hostPath:
path: /var/run/docker.sock
"""
}
}
stages {
stage('Prepare') {
steps {
container("publish-images") {
script {
dir ("milvus") {
checkout([$class: 'GitSCM', branches: [[name: "${env.SEMVER}"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus.git", name: 'origin', refspec: "+refs/heads/${env.SEMVER}:refs/remotes/origin/${env.SEMVER}"]]])
}
}
}
}
}
stage('Publish') {
steps {
container('publish-images'){
script {
dir ("milvus") {
load "ci/jenkins/step/publishImages.groovy"
}
}
}
}
}
}
}
stage("Deploy to Development") {
environment {
FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-")
HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}".toLowerCase()
}
agent {
kubernetes {
label "${env.BINRARY_VERSION}-dev-test"
defaultContainer 'jnlp'
yaml """
apiVersion: v1
kind: Pod
metadata:
labels:
app: milvus
componet: test-env
spec:
containers:
- name: milvus-test-env
image: registry.zilliz.com/milvus/milvus-test-env:v0.1
command:
- cat
tty: true
volumeMounts:
- name: kubeconf
mountPath: /root/.kube/
readOnly: true
volumes:
- name: kubeconf
secret:
secretName: test-cluster-config
"""
}
}
stages {
stage('Prepare') {
steps {
container('milvus-test-env') {
script {
dir ("milvus") {
checkout([$class: 'GitSCM', branches: [[name: "${env.SEMVER}"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus.git", name: 'origin', refspec: "+refs/heads/${env.SEMVER}:refs/remotes/origin/${env.SEMVER}"]]])
}
}
}
}
}
stage("Deploy to Dev") {
steps {
container('milvus-test-env') {
script {
dir ("milvus") {
load "ci/jenkins/step/deploySingle2Dev.groovy"
}
}
}
}
}
stage("Dev Test") {
steps {
container('milvus-test-env') {
script {
boolean isNightlyTest = isTimeTriggeredBuild()
if (isNightlyTest) {
dir ("milvus") {
load "ci/jenkins/step/singleDevNightlyTest.groovy"
}
} else {
dir ("milvus") {
load "ci/jenkins/step/singleDevTest.groovy"
}
}
}
}
}
}
stage ("Cleanup Dev") {
steps {
container('milvus-test-env') {
script {
dir ("milvus") {
load "ci/jenkins/step/cleanupSingleDev.groovy"
}
}
}
}
}
}
post {
unsuccessful {
container('milvus-test-env') {
script {
dir ("milvus") {
load "ci/jenkins/step/cleanupSingleDev.groovy"
}
}
}
}
}
}
}
}
}
}
}
}
boolean isTimeTriggeredBuild() {
if (currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause').size() != 0) {
return true
}
return false
}

35
core/src/index/thirdparty/faiss/ci/jenkins/step/build.groovy vendored
View File

@ -1,35 +0,0 @@
timeout(time: 60, unit: 'MINUTES') {
dir ("ci/scripts") {
if ("${env.BINRARY_VERSION}" == "gpu") {
if ("${params.IS_ORIGIN_FAISS}" == "False") {
sh "./build.sh -o ${env.FAISS_ROOT_PATH} -i -g"
} else {
sh "wget https://github.com/facebookresearch/faiss/archive/v${env.NATIVE_FAISS_VERSION}.tar.gz && \
tar zxvf v${env.NATIVE_FAISS_VERSION}.tar.gz"
sh "./build.sh -o ${env.FAISS_ROOT_PATH} -s ./faiss-${env.NATIVE_FAISS_VERSION} -i -g"
}
} else {
sh "wget https://github.com/facebookresearch/faiss/archive/v${env.NATIVE_FAISS_VERSION}.tar.gz && \
tar zxvf v${env.NATIVE_FAISS_VERSION}.tar.gz"
sh "./build.sh -o ${env.FAISS_ROOT_PATH} -s ./faiss-${env.NATIVE_FAISS_VERSION} -i"
}
}
dir ("milvus") {
dir ("ci/scripts") {
withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
def checkResult = sh(script: "./check_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache", returnStatus: true)
if ("${env.BINRARY_VERSION}" == "gpu") {
if ("${params.IS_ORIGIN_FAISS}" == "False") {
sh ". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -f ${env.FAISS_ROOT_PATH} -l -m -g -x -u -c"
} else {
sh ". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -f ${env.FAISS_ROOT_PATH} -l -m -g -u -c"
}
} else {
sh ". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -f ${env.FAISS_ROOT_PATH} -l -m -u -c"
}
sh "./update_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache -u ${USERNAME} -p ${PASSWORD}"
}
}
}
}

5
core/src/index/thirdparty/faiss/ci/jenkins/step/coverage.groovy vendored
View File

@ -1,5 +0,0 @@
timeout(time: 30, unit: 'MINUTES') {
dir ("milvus/ci/scripts") {
sh "./coverage.sh -o /opt/milvus -u root -p 123456 -t \$POD_IP"
}
}

127
core/src/index/thirdparty/faiss/ci/scripts/build.sh vendored
View File

@ -1,127 +0,0 @@
#!/bin/bash
set -e
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
SOURCE="$(readlink "$SOURCE")"
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
done
SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
FAISS_SOURCE_DIR="${SCRIPTS_DIR}/../.."
FAISS_WITH_MKL="False"
FAISS_GPU_VERSION="False"
FAISS_COMMON_CONFIGURE_ARGS="CXXFLAGS=\"-mavx2 -mf16c\" --without-python"
FAISS_CONFIGURE_ARGS="${FAISS_COMMON_CONFIGURE_ARGS}"
CUDA_TOOLKIT_ROOT_DIR="/usr/local/cuda"
FAISS_CUDA_ARCH="-gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75"
MKL_ROOT_DIR="/opt/intel/compilers_and_libraries_2019.5.281/linux/mkl"
while getopts "o:s:m:b:l:c:a:igh" arg
do
case $arg in
o)
FAISS_INSTALL_PREFIX=$OPTARG
;;
s)
FAISS_SOURCE_DIR=$OPTARG
;;
m)
MKL_ROOT_DIR=$OPTARG
;;
b)
OPENBLAS_PREFIX=$OPTARG
;;
l)
LAPACK_PREFIX=$OPTARG
;;
c)
CUDA_TOOLKIT_ROOT_DIR=$OPTARG
;;
a)
FAISS_CUDA_ARCH=$OPTARG
;;
i)
FAISS_WITH_MKL="True"
;;
g)
FAISS_GPU_VERSION="True"
;;
h) # help
echo "
parameter:
-o: faiss install prefix path
-s: faiss source directory
-m: mkl root directory
-b: openblas install prefix path
-l: lapack install prefix path
-c: CUDA toolkit root directory
-a: faiss CUDA compute architecture
-i: faiss with mkl
-g: faiss gpu version
-h: help
usage:
./build.sh -o \${FAISS_INSTALL_PREFIX} -s \${FAISS_SOURCE_DIR} -m \${MKL_ROOT_DIR} -b \${OPENBLAS_PREFIX} -l \${LAPACK_PREFIX} -c \${CUDA_TOOLKIT_ROOT_DIR} -a \${FAISS_CUDA_ARCH} [-i] [-g] [-h]
"
exit 0
;;
?)
echo "ERROR! unknown argument"
exit 1
;;
esac
done
if [[ -n "${FAISS_INSTALL_PREFIX}" ]];then
FAISS_CONFIGURE_ARGS="${FAISS_CONFIGURE_ARGS} --prefix=${FAISS_INSTALL_PREFIX}"
fi
if [[ "${FAISS_GPU_VERSION}" == "True" ]];then
if [[ ! -n "${FAISS_CUDA_ARCH}" ]];then
echo "FAISS_CUDA_ARCH: \"${FAISS_CUDA_ARCH}\" is empty!"
exit 1
fi
if [[ ! -d "${CUDA_TOOLKIT_ROOT_DIR}" ]];then
echo "CUDA_TOOLKIT_ROOT_DIR: \"${CUDA_TOOLKIT_ROOT_DIR}\" directory doesn't exist!"
exit 1
fi
FAISS_CONFIGURE_ARGS="${FAISS_CONFIGURE_ARGS} --with-cuda=${CUDA_TOOLKIT_ROOT_DIR} --with-cuda-arch='${FAISS_CUDA_ARCH}'"
else
FAISS_CONFIGURE_ARGS="${FAISS_CONFIGURE_ARGS} --without-cuda"
fi
if [[ "${FAISS_WITH_MKL}" == "True" ]];then
if [[ ! -d "${MKL_ROOT_DIR}" ]];then
echo "MKL_ROOT_DIR: \"${MKL_ROOT_DIR}\" directory doesn't exist!"
exit 1
fi
FAISS_CONFIGURE_ARGS="${FAISS_CONFIGURE_ARGS} CPPFLAGS='-DFINTEGER=long -DMKL_ILP64 -m64 -I${MKL_ROOT_DIR}/include' LDFLAGS='-L${MKL_ROOT_DIR}/lib/intel64'"
else
if [[ -n "${LAPACK_PREFIX}" ]];then
if [[ ! -d "${LAPACK_PREFIX}" ]];then
echo "LAPACK_PREFIX: \"${LAPACK_PREFIX}\" directory doesn't exist!"
exit 1
fi
FAISS_CONFIGURE_ARGS="${FAISS_CONFIGURE_ARGS} LDFLAGS=-L${LAPACK_PREFIX}/lib"
fi
if [[ -n "${OPENBLAS_PREFIX}" ]];then
if [[ ! -d "${OPENBLAS_PREFIX}" ]];then
echo "OPENBLAS_PREFIX: \"${OPENBLAS_PREFIX}\" directory doesn't exist!"
exit 1
fi
FAISS_CONFIGURE_ARGS="${FAISS_CONFIGURE_ARGS} LDFLAGS=-L${OPENBLAS_PREFIX}/lib"
fi
fi
cd ${FAISS_SOURCE_DIR}
sh -c "./configure ${FAISS_CONFIGURE_ARGS}"
# compile and build
make -j8 || exit 1
make install || exit 1