test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-11 09:46:26 +08:00
Code Issues Actions 16 Packages Projects Releases Wiki Activity

Merge branch 'branch-0.3.1' into 'branch-0.3.1'

Browse Source 
MS-261: update faiss version to 1.5.3 and add BUILD_FAISS_WITH_MKL as an option

See merge request megasearch/milvus!255

Former-commit-id: 4a78da360bcdb222f68899db3b7035c5224c1130
This commit is contained in:
jinhai 2019-07-21 21:04:31 +08:00
commit b404ba8a6d
9 changed files with 30 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cpp/CHANGELOG.md
View File

@ -35,6 +35,7 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-248 - Support AddVector/SearchVector profiling
- MS-256 - Add more cache config
- MS-260 - Refine log
- MS-261 - update faiss version to 1.5.3 and add BUILD_FAISS_WITH_MKL as an option
## New Feature
- MS-180 - Add new mem manager

10
cpp/build.sh
View File

@ -8,8 +8,9 @@ MAKE_CLEAN="OFF"
BUILD_COVERAGE="OFF"
DB_PATH="/opt/milvus"
PROFILING="OFF"
BUILD_FAISS_WITH_MKL="OFF"
while getopts "p:d:t:uhlrcg" arg
while getopts "p:d:t:uhlrcgm" arg
do
case $arg in
t)
@ -40,6 +41,9 @@ do
g)
PROFILING="ON"
;;
m)
BUILD_FAISS_WITH_MKL="ON"
;;
h) # help
echo "
@ -52,9 +56,10 @@ parameter:
-r: remove previous build directory(default: OFF)
-c: code coverage(default: OFF)
-g: profiling(default: OFF)
-m: build faiss with MKL(default: OFF)
usage:
./build.sh -t \${BUILD_TYPE} [-u] [-h] [-g] [-r] [-c]
./build.sh -t \${BUILD_TYPE} [-u] [-h] [-g] [-r] [-c] [-m]
"
exit 0
;;
@ -83,6 +88,7 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then
-DBUILD_COVERAGE=${BUILD_COVERAGE} \
-DMILVUS_DB_PATH=${DB_PATH} \
-DMILVUS_ENABLE_PROFILING=${PROFILING} \
-DBUILD_FAISS_WITH_MKL=${BUILD_FAISS_WITH_MKL} \
$@ ../"
echo ${CMAKE_CMD}

35
cpp/cmake/ThirdPartyPackages.cmake
View File

@ -699,7 +699,9 @@ endmacro()
# ----------------------------------------------------------------------
# FAISS
set(BUILD_FAISS_WITH_MKL false)
if(NOT DEFINED BUILD_FAISS_WITH_MKL)
set(BUILD_FAISS_WITH_MKL OFF)
endif()
if(EXISTS "/proc/cpuinfo")
FILE(READ /proc/cpuinfo PROC_CPUINFO)
@ -708,8 +710,8 @@ if(EXISTS "/proc/cpuinfo")
STRING(REGEX MATCH "${VENDOR_ID_RX}" VENDOR_ID "${PROC_CPUINFO}")
STRING(REGEX REPLACE "${VENDOR_ID_RX}" "\\1" VENDOR_ID "${VENDOR_ID}")
if(${VENDOR_ID} STREQUAL "GenuineIntel")
set(BUILD_FAISS_WITH_MKL true)
if(NOT ${VENDOR_ID} STREQUAL "GenuineIntel")
set(BUILD_FAISS_WITH_MKL OFF)
endif()
endif()
@ -729,7 +731,7 @@ macro(build_faiss)
set(FAISS_CFLAGS ${EP_C_FLAGS})
set(FAISS_CXXFLAGS ${EP_CXX_FLAGS})
if(${BUILD_FAISS_WITH_MKL} STREQUAL "true")
if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
message(STATUS "Build Faiss with MKL")
if(NOT DEFINED MKL_LIB_PATH)
set(MKL_LIB_PATH "/opt/intel/compilers_and_libraries_${MKL_VERSION}/linux/mkl/lib/intel64")
@ -750,10 +752,10 @@ macro(build_faiss)
if(${MILVUS_WITH_FAISS_GPU_VERSION} STREQUAL "ON")
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
"--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}"
"--with-cuda-arch=\"-gencode=arch=compute_35,code=compute_35\""
"--with-cuda-arch=\"-gencode=arch=compute_52,code=compute_52\""
"--with-cuda-arch=\"-gencode=arch=compute_60,code=compute_60\""
"--with-cuda-arch=\"-gencode=arch=compute_61,code=compute_61\""
"--with-cuda-arch=\"-gencode=arch=compute_35,code=sm_35\""
"--with-cuda-arch=\"-gencode=arch=compute_52,code=sm_52\""
"--with-cuda-arch=\"-gencode=arch=compute_60,code=sm_60\""
"--with-cuda-arch=\"-gencode=arch=compute_61,code=sm_61\""
)
else()
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} --without-cuda)
@ -767,26 +769,22 @@ macro(build_faiss)
"./configure"
${FAISS_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS} all
COMMAND
cd gpu && ${MAKE} ${MAKE_BUILD_ARGS}
${MAKE} ${MAKE_BUILD_ARGS}
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE} install
COMMAND
ln -s faiss_ep ../faiss
BUILD_BYPRODUCTS
${FAISS_STATIC_LIB})
if(${BUILD_FAISS_WITH_MKL} STREQUAL "false")
if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF")
ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep)
endif()
file(MAKE_DIRECTORY "${FAISS_INCLUDE_DIR}")
add_library(faiss SHARED IMPORTED)
if(${BUILD_FAISS_WITH_MKL} STREQUAL "true")
if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
set(MKL_LIBS ${MKL_LIB_PATH}/libmkl_intel_ilp64.a
${MKL_LIB_PATH}/libmkl_gnu_thread.a
${MKL_LIB_PATH}/libmkl_core.a)
@ -806,7 +804,7 @@ macro(build_faiss)
add_dependencies(faiss faiss_ep)
if(${BUILD_FAISS_WITH_MKL} STREQUAL "false")
if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF")
add_dependencies(faiss openblas_ep)
add_dependencies(faiss lapack_ep)
endif()
@ -815,7 +813,7 @@ endmacro()
if(MILVUS_WITH_FAISS)
if(${BUILD_FAISS_WITH_MKL} STREQUAL "false")
if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF")
resolve_dependency(OpenBLAS)
get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}")
@ -830,10 +828,7 @@ if(MILVUS_WITH_FAISS)
resolve_dependency(FAISS)
get_target_property(FAISS_INCLUDE_DIR faiss INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM "${FAISS_INCLUDE_DIR}")
include_directories(SYSTEM "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix/src/")
link_directories(SYSTEM ${FAISS_PREFIX}/)
link_directories(SYSTEM ${FAISS_PREFIX}/lib/)
link_directories(SYSTEM ${FAISS_PREFIX}/gpu/)
endif()
# ----------------------------------------------------------------------

3
cpp/src/CMakeLists.txt
View File

@ -72,7 +72,6 @@ set(third_party_libs
sqlite
thrift
yaml-cpp
libgpufaiss.a
faiss
prometheus-cpp-push
prometheus-cpp-pull
@ -93,7 +92,7 @@ set(third_party_libs
if (MEGASEARCH_WITH_ARROW STREQUAL "ON")
set(third_party_libs ${third_party_libs} arrow)
endif()
if(${BUILD_FAISS_WITH_MKL} STREQUAL "true")
if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
set(third_party_libs ${third_party_libs}
${MKL_LIBS}
${MKL_LIBS})

2
cpp/thirdparty/versions.txt vendored
View File

@ -2,7 +2,7 @@ ARROW_VERSION=zilliz
BOOST_VERSION=1.70.0
BZIP2_VERSION=1.0.6
EASYLOGGINGPP_VERSION=v9.96.7
FAISS_VERSION=7b07685
FAISS_VERSION=v1.5.3
MKL_VERSION=2019.4.243
GTEST_VERSION=1.8.1
JSONCONS_VERSION=0.126.0

3
cpp/unittest/db/CMakeLists.txt
View File

@ -35,7 +35,6 @@ set(db_test_src
cuda_add_executable(db_test ${db_test_src})
set(db_libs
libgpufaiss.a
faiss
cudart
cublas
@ -46,7 +45,7 @@ set(db_libs
mysqlpp
)
if(${BUILD_FAISS_WITH_MKL} STREQUAL "true")
if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
set(db_libs ${db_libs} ${MKL_LIBS} ${MKL_LIBS})
endif()

3
cpp/unittest/faiss_wrapper/CMakeLists.txt
View File

@ -24,7 +24,6 @@ set(wrapper_libs
stdc++
boost_system_static
boost_filesystem_static
libgpufaiss.a
faiss
cudart
cublas
@ -35,7 +34,7 @@ set(wrapper_libs
zstd
lz4
)
if(${BUILD_FAISS_WITH_MKL} STREQUAL "true")
if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
set(wrapper_libs ${wrapper_libs} ${MKL_LIBS} ${MKL_LIBS})
endif()

3
cpp/unittest/metrics/CMakeLists.txt
View File

@ -49,7 +49,6 @@ set(count_test_src
add_executable(metrics_test ${count_test_src} ${require_files} )
target_link_libraries(metrics_test
libgpufaiss.a
faiss
cudart
cublas
@ -64,7 +63,7 @@ target_link_libraries(metrics_test
mysqlpp
${unittest_libs}
)
if(${BUILD_FAISS_WITH_MKL} STREQUAL "true")
if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
target_link_libraries(metrics_test ${MKL_LIBS} ${MKL_LIBS})
endif()

3
cpp/unittest/server/CMakeLists.txt
View File

@ -33,7 +33,6 @@ cuda_add_executable(server_test
set(require_libs
stdc++
libgpufaiss.a
faiss
cudart
cublas
@ -48,7 +47,7 @@ set(require_libs
pthread
)
if(${BUILD_FAISS_WITH_MKL} STREQUAL "true")
if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
set(require_libs ${require_libs} ${MKL_LIBS} ${MKL_LIBS})
endif()