test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-04 21:09:06 +08:00
Code Issues Actions 14 Packages Projects Releases Wiki Activity

Merge branch '0.5.1' into 0.5.1-yzb

Browse Source 
Former-commit-id: 1e1a78a79db59c10c3ede8fd3187aea41c582c59
This commit is contained in:
Heisenberg 2019-10-31 14:43:11 +08:00
commit 90e961649c
9 changed files with 272 additions and 186 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -5,11 +5,13 @@ Please mark all change in change log and use the ticket from JIRA.
# Milvus 0.5.1 (TODO)
## Bug
- \#134 - JFrog cache error
## Feature
- \#90 - The server start error messages could be improved to enhance user experience
- \#104 - test_scheduler core dump
- \#115 - Using new structure for tasktable
- \#139 - New config opion use_gpu_threshold
## Improvement
- \#64 - Improvement dump function in scheduler

6
core/CMakeLists.txt
View File

@ -32,10 +32,14 @@ string(REGEX REPLACE "\n" "" BUILD_TIME ${BUILD_TIME})
message(STATUS "Build time = ${BUILD_TIME}")
MACRO (GET_GIT_BRANCH_NAME GIT_BRANCH_NAME)
execute_process(COMMAND "git" symbolic-ref --short HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
execute_process(COMMAND "git" rev-parse --abbrev-ref HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
if(GIT_BRANCH_NAME STREQUAL "")
execute_process(COMMAND "git" symbolic-ref --short -q HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
endif()
ENDMACRO (GET_GIT_BRANCH_NAME)
GET_GIT_BRANCH_NAME(GIT_BRANCH_NAME)
message(STATUS "GIT_BRANCH_NAME = ${GIT_BRANCH_NAME}")
if(NOT GIT_BRANCH_NAME STREQUAL "")
string(REGEX REPLACE "\n" "" GIT_BRANCH_NAME ${GIT_BRANCH_NAME})
endif()

375
core/cmake/ThirdPartyPackages.cmake
View File

@ -30,11 +30,11 @@ set(MILVUS_THIRDPARTY_DEPENDENCIES
message(STATUS "Using ${MILVUS_DEPENDENCY_SOURCE} approach to find dependencies")
# For each dependency, set dependency source to global default, if unset
foreach(DEPENDENCY ${MILVUS_THIRDPARTY_DEPENDENCIES})
if("${${DEPENDENCY}_SOURCE}" STREQUAL "")
foreach (DEPENDENCY ${MILVUS_THIRDPARTY_DEPENDENCIES})
if ("${${DEPENDENCY}_SOURCE}" STREQUAL "")
set(${DEPENDENCY}_SOURCE ${MILVUS_DEPENDENCY_SOURCE})
endif()
endforeach()
endif ()
endforeach ()
macro(build_dependency DEPENDENCY_NAME)
if ("${DEPENDENCY_NAME}" STREQUAL "GTest")
@ -47,17 +47,17 @@ macro(build_dependency DEPENDENCY_NAME)
build_sqlite()
elseif ("${DEPENDENCY_NAME}" STREQUAL "SQLite_ORM")
build_sqlite_orm()
elseif("${DEPENDENCY_NAME}" STREQUAL "yaml-cpp")
elseif ("${DEPENDENCY_NAME}" STREQUAL "yaml-cpp")
build_yamlcpp()
elseif("${DEPENDENCY_NAME}" STREQUAL "libunwind")
elseif ("${DEPENDENCY_NAME}" STREQUAL "libunwind")
build_libunwind()
elseif("${DEPENDENCY_NAME}" STREQUAL "gperftools")
elseif ("${DEPENDENCY_NAME}" STREQUAL "gperftools")
build_gperftools()
elseif("${DEPENDENCY_NAME}" STREQUAL "GRPC")
elseif ("${DEPENDENCY_NAME}" STREQUAL "GRPC")
build_grpc()
elseif("${DEPENDENCY_NAME}" STREQUAL "ZLIB")
elseif ("${DEPENDENCY_NAME}" STREQUAL "ZLIB")
build_zlib()
else()
else ()
message(FATAL_ERROR "Unknown thirdparty dependency to build: ${DEPENDENCY_NAME}")
endif ()
endmacro()
@ -66,28 +66,28 @@ endmacro()
# Identify OS
if (UNIX)
if (APPLE)
set (CMAKE_OS_NAME "osx" CACHE STRING "Operating system name" FORCE)
set(CMAKE_OS_NAME "osx" CACHE STRING "Operating system name" FORCE)
else (APPLE)
## Check for Debian GNU/Linux ________________
find_file (DEBIAN_FOUND debian_version debconf.conf
PATHS /etc
)
find_file(DEBIAN_FOUND debian_version debconf.conf
PATHS /etc
)
if (DEBIAN_FOUND)
set (CMAKE_OS_NAME "debian" CACHE STRING "Operating system name" FORCE)
set(CMAKE_OS_NAME "debian" CACHE STRING "Operating system name" FORCE)
endif (DEBIAN_FOUND)
## Check for Fedora _________________________
find_file (FEDORA_FOUND fedora-release
PATHS /etc
)
find_file(FEDORA_FOUND fedora-release
PATHS /etc
)
if (FEDORA_FOUND)
set (CMAKE_OS_NAME "fedora" CACHE STRING "Operating system name" FORCE)
set(CMAKE_OS_NAME "fedora" CACHE STRING "Operating system name" FORCE)
endif (FEDORA_FOUND)
## Check for RedHat _________________________
find_file (REDHAT_FOUND redhat-release inittab.RH
PATHS /etc
)
find_file(REDHAT_FOUND redhat-release inittab.RH
PATHS /etc
)
if (REDHAT_FOUND)
set (CMAKE_OS_NAME "redhat" CACHE STRING "Operating system name" FORCE)
set(CMAKE_OS_NAME "redhat" CACHE STRING "Operating system name" FORCE)
endif (REDHAT_FOUND)
## Extra check for Ubuntu ____________________
if (DEBIAN_FOUND)
@ -96,18 +96,25 @@ if (UNIX)
## a first superficial inspection a system will
## be considered as Debian, which signifies an
## extra check is required.
find_file (UBUNTU_EXTRA legal issue
PATHS /etc
)
find_file(UBUNTU_EXTRA legal issue
PATHS /etc
)
if (UBUNTU_EXTRA)
## Scan contents of file
file (STRINGS ${UBUNTU_EXTRA} UBUNTU_FOUND
REGEX Ubuntu
)
file(STRINGS ${UBUNTU_EXTRA} UBUNTU_FOUND
REGEX Ubuntu
)
## Check result of string search
if (UBUNTU_FOUND)
set (CMAKE_OS_NAME "ubuntu" CACHE STRING "Operating system name" FORCE)
set (DEBIAN_FOUND FALSE)
set(CMAKE_OS_NAME "ubuntu" CACHE STRING "Operating system name" FORCE)
set(DEBIAN_FOUND FALSE)
find_program(LSB_RELEASE_EXEC lsb_release)
execute_process(COMMAND ${LSB_RELEASE_EXEC} -rs
OUTPUT_VARIABLE LSB_RELEASE_ID_SHORT
OUTPUT_STRIP_TRAILING_WHITESPACE
)
STRING(REGEX REPLACE "\\." "_" UBUNTU_VERSION ${LSB_RELEASE_ID_SHORT})
endif (UBUNTU_FOUND)
endif (UBUNTU_EXTRA)
endif (DEBIAN_FOUND)
@ -120,36 +127,40 @@ set(THIRDPARTY_DIR "${MILVUS_SOURCE_DIR}/thirdparty")
# ----------------------------------------------------------------------
# JFrog
if(NOT DEFINED USE_JFROG_CACHE)
if (NOT DEFINED USE_JFROG_CACHE)
set(USE_JFROG_CACHE "OFF")
endif()
if(USE_JFROG_CACHE STREQUAL "ON")
if(DEFINED ENV{JFROG_ARTFACTORY_URL})
endif ()
if (USE_JFROG_CACHE STREQUAL "ON")
if (DEFINED ENV{JFROG_ARTFACTORY_URL})
set(JFROG_ARTFACTORY_URL "$ENV{JFROG_ARTFACTORY_URL}")
endif()
if(NOT DEFINED JFROG_ARTFACTORY_URL)
endif ()
if (NOT DEFINED JFROG_ARTFACTORY_URL)
message(FATAL_ERROR "JFROG_ARTFACTORY_URL is not set")
endif()
set(JFROG_ARTFACTORY_CACHE_URL "${JFROG_ARTFACTORY_URL}/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${MILVUS_BUILD_ARCH}/${BUILD_TYPE}")
if(DEFINED ENV{JFROG_USER_NAME})
endif ()
if (UBUNTU_FOUND)
set(JFROG_ARTFACTORY_CACHE_URL "${JFROG_ARTFACTORY_URL}/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${UBUNTU_VERSION}/${MILVUS_BUILD_ARCH}/${BUILD_TYPE}")
else ()
set(JFROG_ARTFACTORY_CACHE_URL "${JFROG_ARTFACTORY_URL}/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${MILVUS_BUILD_ARCH}/${BUILD_TYPE}")
endif ()
if (DEFINED ENV{JFROG_USER_NAME})
set(JFROG_USER_NAME "$ENV{JFROG_USER_NAME}")
endif()
if(NOT DEFINED JFROG_USER_NAME)
endif ()
if (NOT DEFINED JFROG_USER_NAME)
message(FATAL_ERROR "JFROG_USER_NAME is not set")
endif()
if(DEFINED ENV{JFROG_PASSWORD})
endif ()
if (DEFINED ENV{JFROG_PASSWORD})
set(JFROG_PASSWORD "$ENV{JFROG_PASSWORD}")
endif()
if(NOT DEFINED JFROG_PASSWORD)
endif ()
if (NOT DEFINED JFROG_PASSWORD)
message(FATAL_ERROR "JFROG_PASSWORD is not set")
endif()
endif ()
set(THIRDPARTY_PACKAGE_CACHE "${THIRDPARTY_DIR}/cache")
if(NOT EXISTS ${THIRDPARTY_PACKAGE_CACHE})
if (NOT EXISTS ${THIRDPARTY_PACKAGE_CACHE})
message(STATUS "Will create cached directory: ${THIRDPARTY_PACKAGE_CACHE}")
file(MAKE_DIRECTORY ${THIRDPARTY_PACKAGE_CACHE})
endif()
endif()
endif ()
endif ()
macro(resolve_dependency DEPENDENCY_NAME)
if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO")
@ -182,13 +193,13 @@ set(EP_C_FLAGS "${EP_C_FLAGS} -fPIC")
set(EP_COMMON_TOOLCHAIN -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER})
if(CMAKE_AR)
if (CMAKE_AR)
set(EP_COMMON_TOOLCHAIN ${EP_COMMON_TOOLCHAIN} -DCMAKE_AR=${CMAKE_AR})
endif()
endif ()
if(CMAKE_RANLIB)
if (CMAKE_RANLIB)
set(EP_COMMON_TOOLCHAIN ${EP_COMMON_TOOLCHAIN} -DCMAKE_RANLIB=${CMAKE_RANLIB})
endif()
endif ()
# External projects are still able to override the following declarations.
# cmake command line will favor the last defined variable when a duplicate is
@ -202,20 +213,20 @@ set(EP_COMMON_CMAKE_ARGS
-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}
-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS})
if(NOT MILVUS_VERBOSE_THIRDPARTY_BUILD)
if (NOT MILVUS_VERBOSE_THIRDPARTY_BUILD)
set(EP_LOG_OPTIONS LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1 LOG_DOWNLOAD 1)
else()
else ()
set(EP_LOG_OPTIONS)
endif()
endif ()
# Ensure that a default make is set
if("${MAKE}" STREQUAL "")
if ("${MAKE}" STREQUAL "")
find_program(MAKE make)
endif()
endif ()
if (NOT DEFINED MAKE_BUILD_ARGS)
set(MAKE_BUILD_ARGS "-j8")
endif()
endif ()
message(STATUS "Third Party MAKE_BUILD_ARGS = ${MAKE_BUILD_ARGS}")
# ----------------------------------------------------------------------
@ -230,39 +241,39 @@ find_package(Threads REQUIRED)
# Read toolchain versions from cpp/thirdparty/versions.txt
file(STRINGS "${THIRDPARTY_DIR}/versions.txt" TOOLCHAIN_VERSIONS_TXT)
foreach(_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT})
foreach (_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT})
# Exclude comments
if(NOT _VERSION_ENTRY MATCHES "^[^#][A-Za-z0-9-_]+_VERSION=")
if (NOT _VERSION_ENTRY MATCHES "^[^#][A-Za-z0-9-_]+_VERSION=")
continue()
endif()
endif ()
string(REGEX MATCH "^[^=]*" _LIB_NAME ${_VERSION_ENTRY})
string(REPLACE "${_LIB_NAME}=" "" _LIB_VERSION ${_VERSION_ENTRY})
# Skip blank or malformed lines
if(${_LIB_VERSION} STREQUAL "")
if (${_LIB_VERSION} STREQUAL "")
continue()
endif()
endif ()
# For debugging
#message(STATUS "${_LIB_NAME}: ${_LIB_VERSION}")
set(${_LIB_NAME} "${_LIB_VERSION}")
endforeach()
endforeach ()
if (DEFINED ENV{MILVUS_GTEST_URL})
set(GTEST_SOURCE_URL "$ENV{MILVUS_GTEST_URL}")
else ()
set(GTEST_SOURCE_URL
"https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz")
endif()
endif ()
set(GTEST_MD5 "2e6fbeb6a91310a16efe181886c59596")
if(DEFINED ENV{MILVUS_MYSQLPP_URL})
if (DEFINED ENV{MILVUS_MYSQLPP_URL})
set(MYSQLPP_SOURCE_URL "$ENV{MILVUS_MYSQLPP_URL}")
else()
else ()
set(MYSQLPP_SOURCE_URL "https://tangentsoft.com/mysqlpp/releases/mysql++-${MYSQLPP_VERSION}.tar.gz")
endif()
endif ()
set(MYSQLPP_MD5 "cda38b5ecc0117de91f7c42292dd1e79")
if (DEFINED ENV{MILVUS_PROMETHEUS_URL})
@ -270,60 +281,60 @@ if (DEFINED ENV{MILVUS_PROMETHEUS_URL})
else ()
set(PROMETHEUS_SOURCE_URL
https://github.com/jupp0r/prometheus-cpp.git)
endif()
endif ()
if(DEFINED ENV{MILVUS_SQLITE_URL})
if (DEFINED ENV{MILVUS_SQLITE_URL})
set(SQLITE_SOURCE_URL "$ENV{MILVUS_SQLITE_URL}")
else()
else ()
set(SQLITE_SOURCE_URL
"https://www.sqlite.org/2019/sqlite-autoconf-${SQLITE_VERSION}.tar.gz")
endif()
endif ()
set(SQLITE_MD5 "3c68eb400f8354605736cd55400e1572")
if(DEFINED ENV{MILVUS_SQLITE_ORM_URL})
if (DEFINED ENV{MILVUS_SQLITE_ORM_URL})
set(SQLITE_ORM_SOURCE_URL "$ENV{MILVUS_SQLITE_ORM_URL}")
else()
else ()
set(SQLITE_ORM_SOURCE_URL
"https://github.com/fnc12/sqlite_orm/archive/${SQLITE_ORM_VERSION}.zip")
endif()
endif ()
set(SQLITE_ORM_MD5 "ba9a405a8a1421c093aa8ce988ff8598")
if(DEFINED ENV{MILVUS_YAMLCPP_URL})
if (DEFINED ENV{MILVUS_YAMLCPP_URL})
set(YAMLCPP_SOURCE_URL "$ENV{MILVUS_YAMLCPP_URL}")
else()
else ()
set(YAMLCPP_SOURCE_URL "https://github.com/jbeder/yaml-cpp/archive/yaml-cpp-${YAMLCPP_VERSION}.tar.gz")
endif()
endif ()
set(YAMLCPP_MD5 "5b943e9af0060d0811148b037449ef82")
if(DEFINED ENV{MILVUS_LIBUNWIND_URL})
if (DEFINED ENV{MILVUS_LIBUNWIND_URL})
set(LIBUNWIND_SOURCE_URL "$ENV{MILVUS_LIBUNWIND_URL}")
else()
else ()
set(LIBUNWIND_SOURCE_URL
"https://github.com/libunwind/libunwind/releases/download/v${LIBUNWIND_VERSION}/libunwind-${LIBUNWIND_VERSION}.tar.gz")
endif()
endif ()
set(LIBUNWIND_MD5 "a04f69d66d8e16f8bf3ab72a69112cd6")
if(DEFINED ENV{MILVUS_GPERFTOOLS_URL})
if (DEFINED ENV{MILVUS_GPERFTOOLS_URL})
set(GPERFTOOLS_SOURCE_URL "$ENV{MILVUS_GPERFTOOLS_URL}")
else()
else ()
set(GPERFTOOLS_SOURCE_URL
"https://github.com/gperftools/gperftools/releases/download/gperftools-${GPERFTOOLS_VERSION}/gperftools-${GPERFTOOLS_VERSION}.tar.gz")
endif()
endif ()
set(GPERFTOOLS_MD5 "c6a852a817e9160c79bdb2d3101b4601")
if(DEFINED ENV{MILVUS_GRPC_URL})
if (DEFINED ENV{MILVUS_GRPC_URL})
set(GRPC_SOURCE_URL "$ENV{MILVUS_GRPC_URL}")
else()
else ()
set(GRPC_SOURCE_URL
"https://github.com/youny626/grpc-milvus/archive/${GRPC_VERSION}.zip")
endif()
endif ()
set(GRPC_MD5 "0362ba219f59432c530070b5f5c3df73")
if(DEFINED ENV{MILVUS_ZLIB_URL})
if (DEFINED ENV{MILVUS_ZLIB_URL})
set(ZLIB_SOURCE_URL "$ENV{MILVUS_ZLIB_URL}")
else()
else ()
set(ZLIB_SOURCE_URL "https://github.com/madler/zlib/archive/${ZLIB_VERSION}.tar.gz")
endif()
endif ()
set(ZLIB_MD5 "0095d2d2d1f3442ce1318336637b695f")
# ----------------------------------------------------------------------
@ -334,13 +345,13 @@ macro(build_gtest)
set(GTEST_VENDORED TRUE)
set(GTEST_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS}")
if(APPLE)
if (APPLE)
set(GTEST_CMAKE_CXX_FLAGS
${GTEST_CMAKE_CXX_FLAGS}
-DGTEST_USE_OWN_TR1_TUPLE=1
-Wno-unused-value
-Wno-ignored-attributes)
endif()
endif ()
set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep")
set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include")
@ -359,9 +370,9 @@ macro(build_gtest)
set(GMOCK_INCLUDE_DIR "${GTEST_PREFIX}/include")
set(GMOCK_STATIC_LIB
"${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
)
if(USE_JFROG_CACHE STREQUAL "ON")
if (USE_JFROG_CACHE STREQUAL "ON")
set(GTEST_CACHE_PACKAGE_NAME "googletest_${GTEST_MD5}.tar.gz")
set(GTEST_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${GTEST_CACHE_PACKAGE_NAME}")
set(GTEST_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${GTEST_CACHE_PACKAGE_NAME}")
@ -385,10 +396,10 @@ macro(build_gtest)
${EP_LOG_OPTIONS})
ExternalProject_Create_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GTEST_CACHE_URL})
else()
else ()
ExternalProject_Use_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
else()
endif ()
else ()
ExternalProject_Add(googletest_ep
URL
${GTEST_SOURCE_URL}
@ -402,20 +413,20 @@ macro(build_gtest)
CMAKE_ARGS
${GTEST_CMAKE_ARGS}
${EP_LOG_OPTIONS})
endif()
endif ()
# The include directory must exist before it is referenced by a target.
file(MAKE_DIRECTORY "${GTEST_INCLUDE_DIR}")
add_library(gtest STATIC IMPORTED)
set_target_properties(gtest
PROPERTIES IMPORTED_LOCATION "${GTEST_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}")
PROPERTIES IMPORTED_LOCATION "${GTEST_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}")
add_library(gtest_main STATIC IMPORTED)
set_target_properties(gtest_main
PROPERTIES IMPORTED_LOCATION "${GTEST_MAIN_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}")
PROPERTIES IMPORTED_LOCATION "${GTEST_MAIN_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}")
add_library(gmock STATIC IMPORTED)
set_target_properties(gmock
@ -431,13 +442,13 @@ endmacro()
if (MILVUS_BUILD_TESTS)
resolve_dependency(GTest)
if(NOT GTEST_VENDORED)
endif()
if (NOT GTEST_VENDORED)
endif ()
get_target_property(GTEST_INCLUDE_DIR gtest INTERFACE_INCLUDE_DIRECTORIES)
link_directories(SYSTEM "${GTEST_PREFIX}/lib")
include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
endif()
endif ()
# ----------------------------------------------------------------------
# MySQL++
@ -456,7 +467,7 @@ macro(build_mysqlpp)
"CXXFLAGS=${EP_CXX_FLAGS}"
"LDFLAGS=-pthread")
if(USE_JFROG_CACHE STREQUAL "ON")
if (USE_JFROG_CACHE STREQUAL "ON")
set(MYSQLPP_CACHE_PACKAGE_NAME "mysqlpp_${MYSQLPP_MD5}.tar.gz")
set(MYSQLPP_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${MYSQLPP_CACHE_PACKAGE_NAME}")
set(MYSQLPP_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${MYSQLPP_CACHE_PACKAGE_NAME}")
@ -479,15 +490,15 @@ macro(build_mysqlpp)
${MYSQLPP_SHARED_LIB})
ExternalProject_Create_Cache(mysqlpp_ep ${MYSQLPP_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/mysqlpp_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${MYSQLPP_CACHE_URL})
else()
else ()
file(DOWNLOAD ${MYSQLPP_CACHE_URL} ${MYSQLPP_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${MYSQLPP_CACHE_URL} TO ${MYSQLPP_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(mysqlpp_ep ${MYSQLPP_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
endif ()
endif ()
else ()
externalproject_add(mysqlpp_ep
URL
${MYSQLPP_SOURCE_URL}
@ -501,7 +512,7 @@ macro(build_mysqlpp)
1
BUILD_BYPRODUCTS
${MYSQLPP_SHARED_LIB})
endif()
endif ()
file(MAKE_DIRECTORY "${MYSQLPP_INCLUDE_DIR}")
add_library(mysqlpp SHARED IMPORTED)
@ -515,13 +526,13 @@ macro(build_mysqlpp)
endmacro()
if(MILVUS_WITH_MYSQLPP)
if (MILVUS_WITH_MYSQLPP)
resolve_dependency(MySQLPP)
get_target_property(MYSQLPP_INCLUDE_DIR mysqlpp INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM "${MYSQLPP_INCLUDE_DIR}")
link_directories(SYSTEM ${MYSQLPP_PREFIX}/lib)
endif()
endif ()
# ----------------------------------------------------------------------
# Prometheus
@ -547,9 +558,9 @@ macro(build_prometheus)
"-DCMAKE_INSTALL_PREFIX=${PROMETHEUS_PREFIX}"
-DCMAKE_BUILD_TYPE=Release)
if(USE_JFROG_CACHE STREQUAL "ON")
if (USE_JFROG_CACHE STREQUAL "ON")
execute_process(COMMAND sh -c "git ls-remote --heads --tags ${PROMETHEUS_SOURCE_URL} ${PROMETHEUS_VERSION} | cut -f 1" OUTPUT_VARIABLE PROMETHEUS_LAST_COMMIT_ID)
if(${PROMETHEUS_LAST_COMMIT_ID} MATCHES "^[^#][a-z0-9]+")
if (${PROMETHEUS_LAST_COMMIT_ID} MATCHES "^[^#][a-z0-9]+")
string(MD5 PROMETHEUS_COMBINE_MD5 "${PROMETHEUS_LAST_COMMIT_ID}")
set(PROMETHEUS_CACHE_PACKAGE_NAME "prometheus_${PROMETHEUS_COMBINE_MD5}.tar.gz")
set(PROMETHEUS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${PROMETHEUS_CACHE_PACKAGE_NAME}")
@ -583,18 +594,18 @@ macro(build_prometheus)
"${PROMETHEUS_PULL_STATIC_LIB}")
ExternalProject_Create_Cache(prometheus_ep ${PROMETHEUS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/prometheus_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${PROMETHEUS_CACHE_URL})
else()
else ()
file(DOWNLOAD ${PROMETHEUS_CACHE_URL} ${PROMETHEUS_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${PROMETHEUS_CACHE_URL} TO ${PROMETHEUS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(prometheus_ep ${PROMETHEUS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
endif ()
endif ()
else ()
message(FATAL_ERROR "The last commit ID of \"${PROMETHEUS_SOURCE_URL}\" repository don't match!")
endif()
else()
endif ()
else ()
externalproject_add(prometheus_ep
GIT_REPOSITORY
${PROMETHEUS_SOURCE_URL}
@ -618,7 +629,7 @@ macro(build_prometheus)
"${PROMETHEUS_CORE_STATIC_LIB}"
"${PROMETHEUS_PUSH_STATIC_LIB}"
"${PROMETHEUS_PULL_STATIC_LIB}")
endif()
endif ()
file(MAKE_DIRECTORY "${PROMETHEUS_PREFIX}/push/include")
add_library(prometheus-cpp-push STATIC IMPORTED)
@ -642,7 +653,7 @@ macro(build_prometheus)
add_dependencies(prometheus-cpp-core prometheus_ep)
endmacro()
if(MILVUS_WITH_PROMETHEUS)
if (MILVUS_WITH_PROMETHEUS)
resolve_dependency(Prometheus)
@ -655,7 +666,7 @@ if(MILVUS_WITH_PROMETHEUS)
link_directories(SYSTEM ${PROMETHEUS_PREFIX}/core/)
include_directories(SYSTEM ${PROMETHEUS_PREFIX}/core/include)
endif()
endif ()
# ----------------------------------------------------------------------
# SQLite
@ -674,7 +685,7 @@ macro(build_sqlite)
"CFLAGS=${EP_C_FLAGS}"
"CXXFLAGS=${EP_CXX_FLAGS}")
if(USE_JFROG_CACHE STREQUAL "ON")
if (USE_JFROG_CACHE STREQUAL "ON")
set(SQLITE_CACHE_PACKAGE_NAME "sqlite_${SQLITE_MD5}.tar.gz")
set(SQLITE_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${SQLITE_CACHE_PACKAGE_NAME}")
set(SQLITE_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${SQLITE_CACHE_PACKAGE_NAME}")
@ -698,15 +709,15 @@ macro(build_sqlite)
"${SQLITE_STATIC_LIB}")
ExternalProject_Create_Cache(sqlite_ep ${SQLITE_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/sqlite_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${SQLITE_CACHE_URL})
else()
else ()
file(DOWNLOAD ${SQLITE_CACHE_URL} ${SQLITE_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${SQLITE_CACHE_URL} TO ${SQLITE_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(sqlite_ep ${SQLITE_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
endif ()
endif ()
else ()
externalproject_add(sqlite_ep
URL
${SQLITE_SOURCE_URL}
@ -721,7 +732,7 @@ macro(build_sqlite)
1
BUILD_BYPRODUCTS
"${SQLITE_STATIC_LIB}")
endif()
endif ()
file(MAKE_DIRECTORY "${SQLITE_INCLUDE_DIR}")
add_library(sqlite STATIC IMPORTED)
@ -733,11 +744,11 @@ macro(build_sqlite)
add_dependencies(sqlite sqlite_ep)
endmacro()
if(MILVUS_WITH_SQLITE)
if (MILVUS_WITH_SQLITE)
resolve_dependency(SQLite)
include_directories(SYSTEM "${SQLITE_INCLUDE_DIR}")
link_directories(SYSTEM ${SQLITE_PREFIX}/lib/)
endif()
endif ()
# ----------------------------------------------------------------------
# SQLite_ORM
@ -753,16 +764,16 @@ macro(build_sqlite_orm)
file(DOWNLOAD ${SQLITE_ORM_SOURCE_URL}
${SQLITE_ORM_TAR_NAME})
execute_process(COMMAND ${CMAKE_COMMAND} -E tar -xf ${SQLITE_ORM_TAR_NAME}
WORKING_DIRECTORY ${SQLITE_ORM_PREFIX})
WORKING_DIRECTORY ${SQLITE_ORM_PREFIX})
endif ()
endmacro()
if(MILVUS_WITH_SQLITE_ORM)
if (MILVUS_WITH_SQLITE_ORM)
resolve_dependency(SQLite_ORM)
include_directories(SYSTEM "${SQLITE_ORM_INCLUDE_DIR}")
endif()
endif ()
# ----------------------------------------------------------------------
# yaml-cpp
@ -779,7 +790,7 @@ macro(build_yamlcpp)
-DYAML_CPP_BUILD_TESTS=OFF
-DYAML_CPP_BUILD_TOOLS=OFF)
if(USE_JFROG_CACHE STREQUAL "ON")
if (USE_JFROG_CACHE STREQUAL "ON")
set(YAMLCPP_CACHE_PACKAGE_NAME "yaml-cpp_${YAMLCPP_MD5}.tar.gz")
set(YAMLCPP_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${YAMLCPP_CACHE_PACKAGE_NAME}")
set(YAMLCPP_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${YAMLCPP_CACHE_PACKAGE_NAME}")
@ -800,15 +811,15 @@ macro(build_yamlcpp)
${YAMLCPP_CMAKE_ARGS})
ExternalProject_Create_Cache(yaml-cpp_ep ${YAMLCPP_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/yaml-cpp_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${YAMLCPP_CACHE_URL})
else()
else ()
file(DOWNLOAD ${YAMLCPP_CACHE_URL} ${YAMLCPP_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${YAMLCPP_CACHE_URL} TO ${YAMLCPP_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(yaml-cpp_ep ${YAMLCPP_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
endif ()
endif ()
else ()
externalproject_add(yaml-cpp_ep
URL
${YAMLCPP_SOURCE_URL}
@ -820,7 +831,7 @@ macro(build_yamlcpp)
"${YAMLCPP_STATIC_LIB}"
CMAKE_ARGS
${YAMLCPP_CMAKE_ARGS})
endif()
endif ()
file(MAKE_DIRECTORY "${YAMLCPP_INCLUDE_DIR}")
add_library(yaml-cpp STATIC IMPORTED)
@ -831,13 +842,13 @@ macro(build_yamlcpp)
add_dependencies(yaml-cpp yaml-cpp_ep)
endmacro()
if(MILVUS_WITH_YAMLCPP)
if (MILVUS_WITH_YAMLCPP)
resolve_dependency(yaml-cpp)
get_target_property(YAMLCPP_INCLUDE_DIR yaml-cpp INTERFACE_INCLUDE_DIRECTORIES)
link_directories(SYSTEM ${YAMLCPP_PREFIX}/lib/)
include_directories(SYSTEM ${YAMLCPP_INCLUDE_DIR})
endif()
endif ()
# ----------------------------------------------------------------------
# libunwind
@ -849,7 +860,7 @@ macro(build_libunwind)
set(LIBUNWIND_SHARED_LIB "${LIBUNWIND_PREFIX}/lib/libunwind${CMAKE_SHARED_LIBRARY_SUFFIX}")
set(LIBUNWIND_CONFIGURE_ARGS "--prefix=${LIBUNWIND_PREFIX}")
if(USE_JFROG_CACHE STREQUAL "ON")
if (USE_JFROG_CACHE STREQUAL "ON")
set(LIBUNWIND_CACHE_PACKAGE_NAME "libunwind_${LIBUNWIND_MD5}.tar.gz")
set(LIBUNWIND_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${LIBUNWIND_CACHE_PACKAGE_NAME}")
set(LIBUNWIND_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${LIBUNWIND_CACHE_PACKAGE_NAME}")
@ -874,15 +885,15 @@ macro(build_libunwind)
${LIBUNWIND_SHARED_LIB})
ExternalProject_Create_Cache(libunwind_ep ${LIBUNWIND_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/libunwind_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${LIBUNWIND_CACHE_URL})
else()
else ()
file(DOWNLOAD ${LIBUNWIND_CACHE_URL} ${LIBUNWIND_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${LIBUNWIND_CACHE_URL} TO ${LIBUNWIND_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(libunwind_ep ${LIBUNWIND_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
endif ()
endif ()
else ()
externalproject_add(libunwind_ep
URL
${LIBUNWIND_SOURCE_URL}
@ -898,7 +909,7 @@ macro(build_libunwind)
${MAKE} install
BUILD_BYPRODUCTS
${LIBUNWIND_SHARED_LIB})
endif()
endif ()
file(MAKE_DIRECTORY "${LIBUNWIND_INCLUDE_DIR}")
@ -910,12 +921,12 @@ macro(build_libunwind)
add_dependencies(libunwind libunwind_ep)
endmacro()
if(MILVUS_WITH_LIBUNWIND)
if (MILVUS_WITH_LIBUNWIND)
resolve_dependency(libunwind)
get_target_property(LIBUNWIND_INCLUDE_DIR libunwind INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM ${LIBUNWIND_INCLUDE_DIR})
endif()
endif ()
# ----------------------------------------------------------------------
# gperftools
@ -927,7 +938,7 @@ macro(build_gperftools)
set(GPERFTOOLS_STATIC_LIB "${GPERFTOOLS_PREFIX}/lib/libprofiler${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(GPERFTOOLS_CONFIGURE_ARGS "--prefix=${GPERFTOOLS_PREFIX}")
if(USE_JFROG_CACHE STREQUAL "ON")
if (USE_JFROG_CACHE STREQUAL "ON")
set(GPERFTOOLS_CACHE_PACKAGE_NAME "gperftools_${GPERFTOOLS_MD5}.tar.gz")
set(GPERFTOOLS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${GPERFTOOLS_CACHE_PACKAGE_NAME}")
set(GPERFTOOLS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${GPERFTOOLS_CACHE_PACKAGE_NAME}")
@ -952,15 +963,15 @@ macro(build_gperftools)
${GPERFTOOLS_STATIC_LIB})
ExternalProject_Create_Cache(gperftools_ep ${GPERFTOOLS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/gperftools_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GPERFTOOLS_CACHE_URL})
else()
else ()
file(DOWNLOAD ${GPERFTOOLS_CACHE_URL} ${GPERFTOOLS_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${GPERFTOOLS_CACHE_URL} TO ${GPERFTOOLS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(gperftools_ep ${GPERFTOOLS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
endif ()
endif ()
else ()
externalproject_add(gperftools_ep
URL
${GPERFTOOLS_SOURCE_URL}
@ -976,7 +987,7 @@ macro(build_gperftools)
${MAKE} install
BUILD_BYPRODUCTS
${GPERFTOOLS_STATIC_LIB})
endif()
endif ()
ExternalProject_Add_StepDependencies(gperftools_ep build libunwind_ep)
@ -992,13 +1003,13 @@ macro(build_gperftools)
add_dependencies(gperftools libunwind_ep)
endmacro()
if(MILVUS_WITH_GPERFTOOLS)
if (MILVUS_WITH_GPERFTOOLS)
resolve_dependency(gperftools)
get_target_property(GPERFTOOLS_INCLUDE_DIR gperftools INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM ${GPERFTOOLS_INCLUDE_DIR})
link_directories(SYSTEM ${GPERFTOOLS_PREFIX}/lib)
endif()
endif ()
# ----------------------------------------------------------------------
# GRPC
@ -1014,7 +1025,7 @@ macro(build_grpc)
set(GRPC_PROTOBUF_STATIC_LIB "${GRPC_PROTOBUF_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}protobuf${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(GRPC_PROTOC_STATIC_LIB "${GRPC_PROTOBUF_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}protoc${CMAKE_STATIC_LIBRARY_SUFFIX}")
if(USE_JFROG_CACHE STREQUAL "ON")
if (USE_JFROG_CACHE STREQUAL "ON")
set(GRPC_CACHE_PACKAGE_NAME "grpc_${GRPC_MD5}.tar.gz")
set(GRPC_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${GRPC_CACHE_PACKAGE_NAME}")
set(GRPC_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${GRPC_CACHE_PACKAGE_NAME}")
@ -1044,15 +1055,15 @@ macro(build_grpc)
ExternalProject_Add_StepDependencies(grpc_ep build zlib_ep)
ExternalProject_Create_Cache(grpc_ep ${GRPC_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GRPC_CACHE_URL})
else()
else ()
file(DOWNLOAD ${GRPC_CACHE_URL} ${GRPC_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${GRPC_CACHE_URL} TO ${GRPC_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(grpc_ep ${GRPC_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
endif ()
endif ()
else ()
externalproject_add(grpc_ep
URL
${GRPC_SOURCE_URL}
@ -1074,7 +1085,7 @@ macro(build_grpc)
ExternalProject_Add_StepDependencies(grpc_ep build zlib_ep)
endif()
endif ()
file(MAKE_DIRECTORY "${GRPC_INCLUDE_DIR}")
@ -1082,29 +1093,29 @@ macro(build_grpc)
set_target_properties(grpc
PROPERTIES IMPORTED_LOCATION "${GRPC_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${GRPC_INCLUDE_DIR}"
INTERFACE_LINK_LIBRARIES "zlib" )
INTERFACE_LINK_LIBRARIES "zlib")
add_library(grpc++ STATIC IMPORTED)
set_target_properties(grpc++
PROPERTIES IMPORTED_LOCATION "${GRPC++_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${GRPC_INCLUDE_DIR}"
INTERFACE_LINK_LIBRARIES "zlib" )
INTERFACE_LINK_LIBRARIES "zlib")
add_library(grpcpp_channelz STATIC IMPORTED)
set_target_properties(grpcpp_channelz
PROPERTIES IMPORTED_LOCATION "${GRPCPP_CHANNELZ_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${GRPC_INCLUDE_DIR}"
INTERFACE_LINK_LIBRARIES "zlib" )
INTERFACE_LINK_LIBRARIES "zlib")
add_library(grpc_protobuf STATIC IMPORTED)
set_target_properties(grpc_protobuf
PROPERTIES IMPORTED_LOCATION "${GRPC_PROTOBUF_STATIC_LIB}"
INTERFACE_LINK_LIBRARIES "zlib" )
INTERFACE_LINK_LIBRARIES "zlib")
add_library(grpc_protoc STATIC IMPORTED)
set_target_properties(grpc_protoc
PROPERTIES IMPORTED_LOCATION "${GRPC_PROTOC_STATIC_LIB}"
INTERFACE_LINK_LIBRARIES "zlib" )
INTERFACE_LINK_LIBRARIES "zlib")
add_dependencies(grpc grpc_ep)
add_dependencies(grpc++ grpc_ep)
@ -1113,7 +1124,7 @@ macro(build_grpc)
add_dependencies(grpc_protoc grpc_ep)
endmacro()
if(MILVUS_WITH_GRPC)
if (MILVUS_WITH_GRPC)
resolve_dependency(GRPC)
get_target_property(GRPC_INCLUDE_DIR grpc INTERFACE_INCLUDE_DIRECTORIES)
@ -1123,7 +1134,7 @@ if(MILVUS_WITH_GRPC)
set(GRPC_THIRD_PARTY_DIR ${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix/src/grpc_ep/third_party)
include_directories(SYSTEM ${GRPC_THIRD_PARTY_DIR}/protobuf/src)
link_directories(SYSTEM ${GRPC_PROTOBUF_LIB_DIR})
endif()
endif ()
# ----------------------------------------------------------------------
# zlib
@ -1137,7 +1148,7 @@ macro(build_zlib)
set(ZLIB_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX}"
-DBUILD_SHARED_LIBS=OFF)
if(USE_JFROG_CACHE STREQUAL "ON")
if (USE_JFROG_CACHE STREQUAL "ON")
set(ZLIB_CACHE_PACKAGE_NAME "zlib_${ZLIB_MD5}.tar.gz")
set(ZLIB_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${ZLIB_CACHE_PACKAGE_NAME}")
set(ZLIB_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${ZLIB_CACHE_PACKAGE_NAME}")
@ -1158,15 +1169,15 @@ macro(build_zlib)
${ZLIB_CMAKE_ARGS})
ExternalProject_Create_Cache(zlib_ep ${ZLIB_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/zlib_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ZLIB_CACHE_URL})
else()
else ()
file(DOWNLOAD ${ZLIB_CACHE_URL} ${ZLIB_CACHE_PACKAGE_PATH} STATUS status)
list(GET status 0 status_code)
message(STATUS "DOWNLOADING FROM ${ZLIB_CACHE_URL} TO ${ZLIB_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
if (status_code EQUAL 0)
ExternalProject_Use_Cache(zlib_ep ${ZLIB_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif()
else()
endif ()
endif ()
else ()
externalproject_add(zlib_ep
URL
${ZLIB_SOURCE_URL}
@ -1178,7 +1189,7 @@ macro(build_zlib)
"${ZLIB_STATIC_LIB}"
CMAKE_ARGS
${ZLIB_CMAKE_ARGS})
endif()
endif ()
file(MAKE_DIRECTORY "${ZLIB_INCLUDE_DIR}")
add_library(zlib STATIC IMPORTED)
@ -1189,9 +1200,9 @@ macro(build_zlib)
add_dependencies(zlib zlib_ep)
endmacro()
if(MILVUS_WITH_ZLIB)
if (MILVUS_WITH_ZLIB)
resolve_dependency(ZLIB)
get_target_property(ZLIB_INCLUDE_DIR zlib INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM ${ZLIB_INCLUDE_DIR})
endif()
endif ()

1
core/conf/server_config.template
View File

@ -36,6 +36,7 @@ cache_config:
engine_config:
use_blas_threshold: 20 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times
# if nq >= use_blas_threshold, use OpenBlas, slower with stable response times
use_gpu_threshold: 1000
resource_config:
search_resources: # define the GPUs used for search computation, must be in format: gpux

3
core/src/index/cmake/ThirdPartyPackagesCore.cmake
View File

@ -245,7 +245,8 @@ if(CUSTOMIZATION)
# set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1
# set(FAISS_MD5 "c89ea8e655f5cdf58f42486f13614714") # commit-id 9c28a1cbb88f41fa03b03d7204106201ad33276b branch-0.2.1
# set(FAISS_MD5 "87fdd86351ffcaf3f80dc26ade63c44b") # commit-id 841a156e67e8e22cd8088e1b58c00afbf2efc30b branch-0.2.1
set(FAISS_MD5 "f3b2ce3364c3fa7febd3aa7fdd0fe380") # commit-id 694e03458e6b69ce8a62502f71f69a614af5af8f branch-0.3.0
# set(FAISS_MD5 "f3b2ce3364c3fa7febd3aa7fdd0fe380") # commit-id 694e03458e6b69ce8a62502f71f69a614af5af8f branch-0.3.0
set(FAISS_MD5 "bb30722c22390ce5f6759ccb216c1b2a") # commit-id d324db297475286afe107847c7fb7a0f9dc7e90e branch-0.3.0
endif()
else()
set(FAISS_SOURCE_URL "https://github.com/milvus-io/faiss/archive/1.6.0.tar.gz")

12
core/src/scheduler/optimizer/LargeSQ8HPass.cpp
View File

@ -21,11 +21,20 @@
#include "scheduler/Utils.h"
#include "scheduler/task/SearchTask.h"
#include "scheduler/tasklabel/SpecResLabel.h"
#include "server/Config.h"
#include "utils/Log.h"
namespace milvus {
namespace scheduler {
LargeSQ8HPass::LargeSQ8HPass() {
server::Config& config = server::Config::GetInstance();
Status s = config.GetEngineConfigUseGpuThreshold(threshold_);
if (!s.ok()) {
threshold_ = std::numeric_limits<int32_t>::max();
}
}
bool
LargeSQ8HPass::Run(const TaskPtr& task) {
if (task->Type() != TaskType::SearchTask) {
@ -40,7 +49,8 @@ LargeSQ8HPass::Run(const TaskPtr& task) {
auto search_job = std::static_pointer_cast<SearchJob>(search_task->job_.lock());
// TODO: future, Index::IVFSQ8H, if nq < threshold set cpu, else set gpu
if (search_job->nq() < 100) {
if (search_job->nq() < threshold_) {
return false;
}

6
core/src/scheduler/optimizer/LargeSQ8HPass.h
View File

@ -18,6 +18,7 @@
#include <condition_variable>
#include <deque>
#include <limits>
#include <list>
#include <memory>
#include <mutex>
@ -34,11 +35,14 @@ namespace scheduler {
class LargeSQ8HPass : public Pass {
public:
LargeSQ8HPass() = default;
LargeSQ8HPass();
public:
bool
Run(const TaskPtr& task) override;
private:
int32_t threshold_ = std::numeric_limits<int32_t>::max();
};
using LargeSQ8HPassPtr = std::shared_ptr<LargeSQ8HPass>;

45
core/src/server/Config.cpp
View File

@ -193,6 +193,12 @@ Config::ValidateConfig() {
return s;
}
int32_t engine_use_gpu_threshold;
s = GetEngineConfigUseGpuThreshold(engine_use_gpu_threshold);
if (!s.ok()) {
return s;
}
/* resource config */
std::string resource_mode;
s = GetResourceConfigMode(resource_mode);
@ -324,6 +330,11 @@ Config::ResetDefaultConfig() {
return s;
}
s = SetEngineConfigUseGpuThreshold(CONFIG_ENGINE_USE_GPU_THRESHOLD_DEFAULT);
if (!s.ok()) {
return s;
}
/* resource config */
s = SetResourceConfigMode(CONFIG_RESOURCE_MODE_DEFAULT);
if (!s.ok()) {
@ -656,6 +667,16 @@ Config::CheckEngineConfigOmpThreadNum(const std::string& value) {
return Status::OK();
}
Status
Config::CheckEngineConfigUseGpuThreshold(const std::string& value) {
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid gpu threshold: " + value +
". Possible reason: engine_config.use_gpu_threshold is not a positive integer.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
}
Status
Config::CheckResourceConfigMode(const std::string& value) {
if (value != "simple") {
@ -951,6 +972,19 @@ Config::GetEngineConfigOmpThreadNum(int32_t& value) {
return Status::OK();
}
Status
Config::GetEngineConfigUseGpuThreshold(int32_t& value) {
std::string str =
GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_USE_GPU_THRESHOLD, CONFIG_ENGINE_USE_GPU_THRESHOLD_DEFAULT);
Status s = CheckEngineConfigUseGpuThreshold(str);
if (!s.ok()) {
return s;
}
value = std::stoi(str);
return Status::OK();
}
Status
Config::GetResourceConfigMode(std::string& value) {
value = GetConfigStr(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, CONFIG_RESOURCE_MODE_DEFAULT);
@ -1203,6 +1237,17 @@ Config::SetEngineConfigOmpThreadNum(const std::string& value) {
return Status::OK();
}
Status
Config::SetEngineConfigUseGpuThreshold(const std::string& value) {
Status s = CheckEngineConfigUseGpuThreshold(value);
if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_USE_GPU_THRESHOLD, value);
return Status::OK();
}
/* resource config */
Status
Config::SetResourceConfigMode(const std::string& value) {

8
core/src/server/Config.h
View File

@ -84,6 +84,8 @@ static const char* CONFIG_ENGINE_USE_BLAS_THRESHOLD = "use_blas_threshold";
static const char* CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT = "20";
static const char* CONFIG_ENGINE_OMP_THREAD_NUM = "omp_thread_num";
static const char* CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0";
static const char* CONFIG_ENGINE_USE_GPU_THRESHOLD = "use_gpu_threshold";
static const char* CONFIG_ENGINE_USE_GPU_THRESHOLD_DEFAULT = "1000";
/* resource config */
static const char* CONFIG_RESOURCE = "resource_config";
@ -166,6 +168,8 @@ class Config {
CheckEngineConfigUseBlasThreshold(const std::string& value);
Status
CheckEngineConfigOmpThreadNum(const std::string& value);
Status
CheckEngineConfigUseGpuThreshold(const std::string& value);
/* resource config */
Status
@ -230,6 +234,8 @@ class Config {
GetEngineConfigUseBlasThreshold(int32_t& value);
Status
GetEngineConfigOmpThreadNum(int32_t& value);
Status
GetEngineConfigUseGpuThreshold(int32_t& value);
/* resource config */
Status
@ -289,6 +295,8 @@ class Config {
SetEngineConfigUseBlasThreshold(const std::string& value);
Status
SetEngineConfigOmpThreadNum(const std::string& value);
Status
SetEngineConfigUseGpuThreshold(const std::string& value);
/* resource config */
Status