From 2aa7176d462b5e6c634d7b8e954d03f1007cc4c1 Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 30 Oct 2019 11:21:28 +0800 Subject: [PATCH 1/7] show branch name Former-commit-id: 8c5eca7a3372f553b13a349ad0e5407ec3683bf2 --- core/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 51cb2270a1..ae4366af2f 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -36,6 +36,7 @@ MACRO (GET_GIT_BRANCH_NAME GIT_BRANCH_NAME) ENDMACRO (GET_GIT_BRANCH_NAME) GET_GIT_BRANCH_NAME(GIT_BRANCH_NAME) +message(STATUS "GIT_BRANCH_NAME = ${GIT_BRANCH_NAME}") if(NOT GIT_BRANCH_NAME STREQUAL "") string(REGEX REPLACE "\n" "" GIT_BRANCH_NAME ${GIT_BRANCH_NAME}) endif() From c41ebb7e7db13ec6eecfa4bcead2b1650196c05d Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Wed, 30 Oct 2019 15:27:05 +0800 Subject: [PATCH 2/7] fix JFrog cache error Former-commit-id: 8b75351080e9201d6f9f47c643dfb7eec32d5a86 --- CHANGELOG.md | 1 + core/cmake/ThirdPartyPackages.cmake | 376 ++++++++++++++-------------- 2 files changed, 195 insertions(+), 182 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b28c7bd3a7..4580c6d55e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ Please mark all change in change log and use the ticket from JIRA. # Milvus 0.5.1 (TODO) ## Bug +- \#134 - JFrog cache error ## Feature - \#90 - The server start error messages could be improved to enhance user experience diff --git a/core/cmake/ThirdPartyPackages.cmake b/core/cmake/ThirdPartyPackages.cmake index d0057d3c22..788c09ef2e 100644 --- a/core/cmake/ThirdPartyPackages.cmake +++ b/core/cmake/ThirdPartyPackages.cmake @@ -30,11 +30,11 @@ set(MILVUS_THIRDPARTY_DEPENDENCIES message(STATUS "Using ${MILVUS_DEPENDENCY_SOURCE} approach to find dependencies") # For each dependency, set dependency source to global default, if unset -foreach(DEPENDENCY ${MILVUS_THIRDPARTY_DEPENDENCIES}) - if("${${DEPENDENCY}_SOURCE}" STREQUAL "") +foreach (DEPENDENCY ${MILVUS_THIRDPARTY_DEPENDENCIES}) + if ("${${DEPENDENCY}_SOURCE}" STREQUAL "") set(${DEPENDENCY}_SOURCE ${MILVUS_DEPENDENCY_SOURCE}) - endif() -endforeach() + endif () +endforeach () macro(build_dependency DEPENDENCY_NAME) if ("${DEPENDENCY_NAME}" STREQUAL "GTest") @@ -47,17 +47,17 @@ macro(build_dependency DEPENDENCY_NAME) build_sqlite() elseif ("${DEPENDENCY_NAME}" STREQUAL "SQLite_ORM") build_sqlite_orm() - elseif("${DEPENDENCY_NAME}" STREQUAL "yaml-cpp") + elseif ("${DEPENDENCY_NAME}" STREQUAL "yaml-cpp") build_yamlcpp() - elseif("${DEPENDENCY_NAME}" STREQUAL "libunwind") + elseif ("${DEPENDENCY_NAME}" STREQUAL "libunwind") build_libunwind() - elseif("${DEPENDENCY_NAME}" STREQUAL "gperftools") + elseif ("${DEPENDENCY_NAME}" STREQUAL "gperftools") build_gperftools() - elseif("${DEPENDENCY_NAME}" STREQUAL "GRPC") + elseif ("${DEPENDENCY_NAME}" STREQUAL "GRPC") build_grpc() - elseif("${DEPENDENCY_NAME}" STREQUAL "ZLIB") + elseif ("${DEPENDENCY_NAME}" STREQUAL "ZLIB") build_zlib() - else() + else () message(FATAL_ERROR "Unknown thirdparty dependency to build: ${DEPENDENCY_NAME}") endif () endmacro() @@ -66,28 +66,28 @@ endmacro() # Identify OS if (UNIX) if (APPLE) - set (CMAKE_OS_NAME "osx" CACHE STRING "Operating system name" FORCE) + set(CMAKE_OS_NAME "osx" CACHE STRING "Operating system name" FORCE) else (APPLE) ## Check for Debian GNU/Linux ________________ - find_file (DEBIAN_FOUND debian_version debconf.conf - PATHS /etc - ) + find_file(DEBIAN_FOUND debian_version debconf.conf + PATHS /etc + ) if (DEBIAN_FOUND) - set (CMAKE_OS_NAME "debian" CACHE STRING "Operating system name" FORCE) + set(CMAKE_OS_NAME "debian" CACHE STRING "Operating system name" FORCE) endif (DEBIAN_FOUND) ## Check for Fedora _________________________ - find_file (FEDORA_FOUND fedora-release - PATHS /etc - ) + find_file(FEDORA_FOUND fedora-release + PATHS /etc + ) if (FEDORA_FOUND) - set (CMAKE_OS_NAME "fedora" CACHE STRING "Operating system name" FORCE) + set(CMAKE_OS_NAME "fedora" CACHE STRING "Operating system name" FORCE) endif (FEDORA_FOUND) ## Check for RedHat _________________________ - find_file (REDHAT_FOUND redhat-release inittab.RH - PATHS /etc - ) + find_file(REDHAT_FOUND redhat-release inittab.RH + PATHS /etc + ) if (REDHAT_FOUND) - set (CMAKE_OS_NAME "redhat" CACHE STRING "Operating system name" FORCE) + set(CMAKE_OS_NAME "redhat" CACHE STRING "Operating system name" FORCE) endif (REDHAT_FOUND) ## Extra check for Ubuntu ____________________ if (DEBIAN_FOUND) @@ -96,18 +96,25 @@ if (UNIX) ## a first superficial inspection a system will ## be considered as Debian, which signifies an ## extra check is required. - find_file (UBUNTU_EXTRA legal issue - PATHS /etc - ) + find_file(UBUNTU_EXTRA legal issue + PATHS /etc + ) if (UBUNTU_EXTRA) ## Scan contents of file - file (STRINGS ${UBUNTU_EXTRA} UBUNTU_FOUND - REGEX Ubuntu - ) + file(STRINGS ${UBUNTU_EXTRA} UBUNTU_FOUND + REGEX Ubuntu + ) ## Check result of string search if (UBUNTU_FOUND) - set (CMAKE_OS_NAME "ubuntu" CACHE STRING "Operating system name" FORCE) - set (DEBIAN_FOUND FALSE) + set(CMAKE_OS_NAME "ubuntu" CACHE STRING "Operating system name" FORCE) + set(DEBIAN_FOUND FALSE) + + find_program(LSB_RELEASE_EXEC lsb_release) + execute_process(COMMAND ${LSB_RELEASE_EXEC} -rs + OUTPUT_VARIABLE LSB_RELEASE_ID_SHORT + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + STRING(REGEX REPLACE "\\." "_" UBUNTU_VERSION ${LSB_RELEASE_ID_SHORT}) endif (UBUNTU_FOUND) endif (UBUNTU_EXTRA) endif (DEBIAN_FOUND) @@ -120,36 +127,41 @@ set(THIRDPARTY_DIR "${MILVUS_SOURCE_DIR}/thirdparty") # ---------------------------------------------------------------------- # JFrog -if(NOT DEFINED USE_JFROG_CACHE) +if (NOT DEFINED USE_JFROG_CACHE) set(USE_JFROG_CACHE "OFF") -endif() -if(USE_JFROG_CACHE STREQUAL "ON") - if(DEFINED ENV{JFROG_ARTFACTORY_URL}) +endif () +if (USE_JFROG_CACHE STREQUAL "ON") + if (DEFINED ENV{JFROG_ARTFACTORY_URL}) set(JFROG_ARTFACTORY_URL "$ENV{JFROG_ARTFACTORY_URL}") - endif() - if(NOT DEFINED JFROG_ARTFACTORY_URL) + endif () + if (NOT DEFINED JFROG_ARTFACTORY_URL) message(FATAL_ERROR "JFROG_ARTFACTORY_URL is not set") - endif() - set(JFROG_ARTFACTORY_CACHE_URL "${JFROG_ARTFACTORY_URL}/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${MILVUS_BUILD_ARCH}/${BUILD_TYPE}") - if(DEFINED ENV{JFROG_USER_NAME}) + endif () + if (UBUNTU_FOUND) + set(JFROG_ARTFACTORY_CACHE_URL "${JFROG_ARTFACTORY_URL}/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${UBUNTU_VERSION}/${MILVUS_BUILD_ARCH}/${BUILD_TYPE}") + message(${JFROG_ARTFACTORY_CACHE_URL}) + else () + set(JFROG_ARTFACTORY_CACHE_URL "${JFROG_ARTFACTORY_URL}/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${MILVUS_BUILD_ARCH}/${BUILD_TYPE}") + endif () + if (DEFINED ENV{JFROG_USER_NAME}) set(JFROG_USER_NAME "$ENV{JFROG_USER_NAME}") - endif() - if(NOT DEFINED JFROG_USER_NAME) + endif () + if (NOT DEFINED JFROG_USER_NAME) message(FATAL_ERROR "JFROG_USER_NAME is not set") - endif() - if(DEFINED ENV{JFROG_PASSWORD}) + endif () + if (DEFINED ENV{JFROG_PASSWORD}) set(JFROG_PASSWORD "$ENV{JFROG_PASSWORD}") - endif() - if(NOT DEFINED JFROG_PASSWORD) + endif () + if (NOT DEFINED JFROG_PASSWORD) message(FATAL_ERROR "JFROG_PASSWORD is not set") - endif() + endif () set(THIRDPARTY_PACKAGE_CACHE "${THIRDPARTY_DIR}/cache") - if(NOT EXISTS ${THIRDPARTY_PACKAGE_CACHE}) + if (NOT EXISTS ${THIRDPARTY_PACKAGE_CACHE}) message(STATUS "Will create cached directory: ${THIRDPARTY_PACKAGE_CACHE}") file(MAKE_DIRECTORY ${THIRDPARTY_PACKAGE_CACHE}) - endif() -endif() + endif () +endif () macro(resolve_dependency DEPENDENCY_NAME) if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO") @@ -182,13 +194,13 @@ set(EP_C_FLAGS "${EP_C_FLAGS} -fPIC") set(EP_COMMON_TOOLCHAIN -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}) -if(CMAKE_AR) +if (CMAKE_AR) set(EP_COMMON_TOOLCHAIN ${EP_COMMON_TOOLCHAIN} -DCMAKE_AR=${CMAKE_AR}) -endif() +endif () -if(CMAKE_RANLIB) +if (CMAKE_RANLIB) set(EP_COMMON_TOOLCHAIN ${EP_COMMON_TOOLCHAIN} -DCMAKE_RANLIB=${CMAKE_RANLIB}) -endif() +endif () # External projects are still able to override the following declarations. # cmake command line will favor the last defined variable when a duplicate is @@ -202,20 +214,20 @@ set(EP_COMMON_CMAKE_ARGS -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS} -DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS}) -if(NOT MILVUS_VERBOSE_THIRDPARTY_BUILD) +if (NOT MILVUS_VERBOSE_THIRDPARTY_BUILD) set(EP_LOG_OPTIONS LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1 LOG_DOWNLOAD 1) -else() +else () set(EP_LOG_OPTIONS) -endif() +endif () # Ensure that a default make is set -if("${MAKE}" STREQUAL "") +if ("${MAKE}" STREQUAL "") find_program(MAKE make) -endif() +endif () if (NOT DEFINED MAKE_BUILD_ARGS) set(MAKE_BUILD_ARGS "-j8") -endif() +endif () message(STATUS "Third Party MAKE_BUILD_ARGS = ${MAKE_BUILD_ARGS}") # ---------------------------------------------------------------------- @@ -230,39 +242,39 @@ find_package(Threads REQUIRED) # Read toolchain versions from cpp/thirdparty/versions.txt file(STRINGS "${THIRDPARTY_DIR}/versions.txt" TOOLCHAIN_VERSIONS_TXT) -foreach(_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT}) +foreach (_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT}) # Exclude comments - if(NOT _VERSION_ENTRY MATCHES "^[^#][A-Za-z0-9-_]+_VERSION=") + if (NOT _VERSION_ENTRY MATCHES "^[^#][A-Za-z0-9-_]+_VERSION=") continue() - endif() + endif () string(REGEX MATCH "^[^=]*" _LIB_NAME ${_VERSION_ENTRY}) string(REPLACE "${_LIB_NAME}=" "" _LIB_VERSION ${_VERSION_ENTRY}) # Skip blank or malformed lines - if(${_LIB_VERSION} STREQUAL "") + if (${_LIB_VERSION} STREQUAL "") continue() - endif() + endif () # For debugging #message(STATUS "${_LIB_NAME}: ${_LIB_VERSION}") set(${_LIB_NAME} "${_LIB_VERSION}") -endforeach() +endforeach () if (DEFINED ENV{MILVUS_GTEST_URL}) set(GTEST_SOURCE_URL "$ENV{MILVUS_GTEST_URL}") else () set(GTEST_SOURCE_URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz") -endif() +endif () set(GTEST_MD5 "2e6fbeb6a91310a16efe181886c59596") -if(DEFINED ENV{MILVUS_MYSQLPP_URL}) +if (DEFINED ENV{MILVUS_MYSQLPP_URL}) set(MYSQLPP_SOURCE_URL "$ENV{MILVUS_MYSQLPP_URL}") -else() +else () set(MYSQLPP_SOURCE_URL "https://tangentsoft.com/mysqlpp/releases/mysql++-${MYSQLPP_VERSION}.tar.gz") -endif() +endif () set(MYSQLPP_MD5 "cda38b5ecc0117de91f7c42292dd1e79") if (DEFINED ENV{MILVUS_PROMETHEUS_URL}) @@ -270,60 +282,60 @@ if (DEFINED ENV{MILVUS_PROMETHEUS_URL}) else () set(PROMETHEUS_SOURCE_URL https://github.com/jupp0r/prometheus-cpp.git) -endif() +endif () -if(DEFINED ENV{MILVUS_SQLITE_URL}) +if (DEFINED ENV{MILVUS_SQLITE_URL}) set(SQLITE_SOURCE_URL "$ENV{MILVUS_SQLITE_URL}") -else() +else () set(SQLITE_SOURCE_URL "https://www.sqlite.org/2019/sqlite-autoconf-${SQLITE_VERSION}.tar.gz") -endif() +endif () set(SQLITE_MD5 "3c68eb400f8354605736cd55400e1572") -if(DEFINED ENV{MILVUS_SQLITE_ORM_URL}) +if (DEFINED ENV{MILVUS_SQLITE_ORM_URL}) set(SQLITE_ORM_SOURCE_URL "$ENV{MILVUS_SQLITE_ORM_URL}") -else() +else () set(SQLITE_ORM_SOURCE_URL "https://github.com/fnc12/sqlite_orm/archive/${SQLITE_ORM_VERSION}.zip") -endif() +endif () set(SQLITE_ORM_MD5 "ba9a405a8a1421c093aa8ce988ff8598") -if(DEFINED ENV{MILVUS_YAMLCPP_URL}) +if (DEFINED ENV{MILVUS_YAMLCPP_URL}) set(YAMLCPP_SOURCE_URL "$ENV{MILVUS_YAMLCPP_URL}") -else() +else () set(YAMLCPP_SOURCE_URL "https://github.com/jbeder/yaml-cpp/archive/yaml-cpp-${YAMLCPP_VERSION}.tar.gz") -endif() +endif () set(YAMLCPP_MD5 "5b943e9af0060d0811148b037449ef82") -if(DEFINED ENV{MILVUS_LIBUNWIND_URL}) +if (DEFINED ENV{MILVUS_LIBUNWIND_URL}) set(LIBUNWIND_SOURCE_URL "$ENV{MILVUS_LIBUNWIND_URL}") -else() +else () set(LIBUNWIND_SOURCE_URL "https://github.com/libunwind/libunwind/releases/download/v${LIBUNWIND_VERSION}/libunwind-${LIBUNWIND_VERSION}.tar.gz") -endif() +endif () set(LIBUNWIND_MD5 "a04f69d66d8e16f8bf3ab72a69112cd6") -if(DEFINED ENV{MILVUS_GPERFTOOLS_URL}) +if (DEFINED ENV{MILVUS_GPERFTOOLS_URL}) set(GPERFTOOLS_SOURCE_URL "$ENV{MILVUS_GPERFTOOLS_URL}") -else() +else () set(GPERFTOOLS_SOURCE_URL "https://github.com/gperftools/gperftools/releases/download/gperftools-${GPERFTOOLS_VERSION}/gperftools-${GPERFTOOLS_VERSION}.tar.gz") -endif() +endif () set(GPERFTOOLS_MD5 "c6a852a817e9160c79bdb2d3101b4601") -if(DEFINED ENV{MILVUS_GRPC_URL}) +if (DEFINED ENV{MILVUS_GRPC_URL}) set(GRPC_SOURCE_URL "$ENV{MILVUS_GRPC_URL}") -else() +else () set(GRPC_SOURCE_URL "https://github.com/youny626/grpc-milvus/archive/${GRPC_VERSION}.zip") -endif() +endif () set(GRPC_MD5 "0362ba219f59432c530070b5f5c3df73") -if(DEFINED ENV{MILVUS_ZLIB_URL}) +if (DEFINED ENV{MILVUS_ZLIB_URL}) set(ZLIB_SOURCE_URL "$ENV{MILVUS_ZLIB_URL}") -else() +else () set(ZLIB_SOURCE_URL "https://github.com/madler/zlib/archive/${ZLIB_VERSION}.tar.gz") -endif() +endif () set(ZLIB_MD5 "0095d2d2d1f3442ce1318336637b695f") # ---------------------------------------------------------------------- @@ -334,13 +346,13 @@ macro(build_gtest) set(GTEST_VENDORED TRUE) set(GTEST_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS}") - if(APPLE) + if (APPLE) set(GTEST_CMAKE_CXX_FLAGS ${GTEST_CMAKE_CXX_FLAGS} -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-value -Wno-ignored-attributes) - endif() + endif () set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep") set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include") @@ -359,9 +371,9 @@ macro(build_gtest) set(GMOCK_INCLUDE_DIR "${GTEST_PREFIX}/include") set(GMOCK_STATIC_LIB "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) + ) - if(USE_JFROG_CACHE STREQUAL "ON") + if (USE_JFROG_CACHE STREQUAL "ON") set(GTEST_CACHE_PACKAGE_NAME "googletest_${GTEST_MD5}.tar.gz") set(GTEST_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${GTEST_CACHE_PACKAGE_NAME}") set(GTEST_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${GTEST_CACHE_PACKAGE_NAME}") @@ -385,10 +397,10 @@ macro(build_gtest) ${EP_LOG_OPTIONS}) ExternalProject_Create_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GTEST_CACHE_URL}) - else() + else () ExternalProject_Use_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - else() + endif () + else () ExternalProject_Add(googletest_ep URL ${GTEST_SOURCE_URL} @@ -402,20 +414,20 @@ macro(build_gtest) CMAKE_ARGS ${GTEST_CMAKE_ARGS} ${EP_LOG_OPTIONS}) - endif() + endif () # The include directory must exist before it is referenced by a target. file(MAKE_DIRECTORY "${GTEST_INCLUDE_DIR}") add_library(gtest STATIC IMPORTED) set_target_properties(gtest - PROPERTIES IMPORTED_LOCATION "${GTEST_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}") + PROPERTIES IMPORTED_LOCATION "${GTEST_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}") add_library(gtest_main STATIC IMPORTED) set_target_properties(gtest_main - PROPERTIES IMPORTED_LOCATION "${GTEST_MAIN_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}") + PROPERTIES IMPORTED_LOCATION "${GTEST_MAIN_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}") add_library(gmock STATIC IMPORTED) set_target_properties(gmock @@ -431,13 +443,13 @@ endmacro() if (MILVUS_BUILD_TESTS) resolve_dependency(GTest) - if(NOT GTEST_VENDORED) - endif() + if (NOT GTEST_VENDORED) + endif () get_target_property(GTEST_INCLUDE_DIR gtest INTERFACE_INCLUDE_DIRECTORIES) link_directories(SYSTEM "${GTEST_PREFIX}/lib") include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) -endif() +endif () # ---------------------------------------------------------------------- # MySQL++ @@ -456,7 +468,7 @@ macro(build_mysqlpp) "CXXFLAGS=${EP_CXX_FLAGS}" "LDFLAGS=-pthread") - if(USE_JFROG_CACHE STREQUAL "ON") + if (USE_JFROG_CACHE STREQUAL "ON") set(MYSQLPP_CACHE_PACKAGE_NAME "mysqlpp_${MYSQLPP_MD5}.tar.gz") set(MYSQLPP_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${MYSQLPP_CACHE_PACKAGE_NAME}") set(MYSQLPP_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${MYSQLPP_CACHE_PACKAGE_NAME}") @@ -479,15 +491,15 @@ macro(build_mysqlpp) ${MYSQLPP_SHARED_LIB}) ExternalProject_Create_Cache(mysqlpp_ep ${MYSQLPP_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/mysqlpp_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${MYSQLPP_CACHE_URL}) - else() + else () file(DOWNLOAD ${MYSQLPP_CACHE_URL} ${MYSQLPP_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${MYSQLPP_CACHE_URL} TO ${MYSQLPP_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) ExternalProject_Use_Cache(mysqlpp_ep ${MYSQLPP_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - endif() - else() + endif () + endif () + else () externalproject_add(mysqlpp_ep URL ${MYSQLPP_SOURCE_URL} @@ -501,7 +513,7 @@ macro(build_mysqlpp) 1 BUILD_BYPRODUCTS ${MYSQLPP_SHARED_LIB}) - endif() + endif () file(MAKE_DIRECTORY "${MYSQLPP_INCLUDE_DIR}") add_library(mysqlpp SHARED IMPORTED) @@ -515,13 +527,13 @@ macro(build_mysqlpp) endmacro() -if(MILVUS_WITH_MYSQLPP) +if (MILVUS_WITH_MYSQLPP) resolve_dependency(MySQLPP) get_target_property(MYSQLPP_INCLUDE_DIR mysqlpp INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM "${MYSQLPP_INCLUDE_DIR}") link_directories(SYSTEM ${MYSQLPP_PREFIX}/lib) -endif() +endif () # ---------------------------------------------------------------------- # Prometheus @@ -547,9 +559,9 @@ macro(build_prometheus) "-DCMAKE_INSTALL_PREFIX=${PROMETHEUS_PREFIX}" -DCMAKE_BUILD_TYPE=Release) - if(USE_JFROG_CACHE STREQUAL "ON") + if (USE_JFROG_CACHE STREQUAL "ON") execute_process(COMMAND sh -c "git ls-remote --heads --tags ${PROMETHEUS_SOURCE_URL} ${PROMETHEUS_VERSION} | cut -f 1" OUTPUT_VARIABLE PROMETHEUS_LAST_COMMIT_ID) - if(${PROMETHEUS_LAST_COMMIT_ID} MATCHES "^[^#][a-z0-9]+") + if (${PROMETHEUS_LAST_COMMIT_ID} MATCHES "^[^#][a-z0-9]+") string(MD5 PROMETHEUS_COMBINE_MD5 "${PROMETHEUS_LAST_COMMIT_ID}") set(PROMETHEUS_CACHE_PACKAGE_NAME "prometheus_${PROMETHEUS_COMBINE_MD5}.tar.gz") set(PROMETHEUS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${PROMETHEUS_CACHE_PACKAGE_NAME}") @@ -583,18 +595,18 @@ macro(build_prometheus) "${PROMETHEUS_PULL_STATIC_LIB}") ExternalProject_Create_Cache(prometheus_ep ${PROMETHEUS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/prometheus_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${PROMETHEUS_CACHE_URL}) - else() + else () file(DOWNLOAD ${PROMETHEUS_CACHE_URL} ${PROMETHEUS_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${PROMETHEUS_CACHE_URL} TO ${PROMETHEUS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) ExternalProject_Use_Cache(prometheus_ep ${PROMETHEUS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - endif() - else() + endif () + endif () + else () message(FATAL_ERROR "The last commit ID of \"${PROMETHEUS_SOURCE_URL}\" repository don't match!") - endif() - else() + endif () + else () externalproject_add(prometheus_ep GIT_REPOSITORY ${PROMETHEUS_SOURCE_URL} @@ -618,7 +630,7 @@ macro(build_prometheus) "${PROMETHEUS_CORE_STATIC_LIB}" "${PROMETHEUS_PUSH_STATIC_LIB}" "${PROMETHEUS_PULL_STATIC_LIB}") - endif() + endif () file(MAKE_DIRECTORY "${PROMETHEUS_PREFIX}/push/include") add_library(prometheus-cpp-push STATIC IMPORTED) @@ -642,7 +654,7 @@ macro(build_prometheus) add_dependencies(prometheus-cpp-core prometheus_ep) endmacro() -if(MILVUS_WITH_PROMETHEUS) +if (MILVUS_WITH_PROMETHEUS) resolve_dependency(Prometheus) @@ -655,7 +667,7 @@ if(MILVUS_WITH_PROMETHEUS) link_directories(SYSTEM ${PROMETHEUS_PREFIX}/core/) include_directories(SYSTEM ${PROMETHEUS_PREFIX}/core/include) -endif() +endif () # ---------------------------------------------------------------------- # SQLite @@ -674,7 +686,7 @@ macro(build_sqlite) "CFLAGS=${EP_C_FLAGS}" "CXXFLAGS=${EP_CXX_FLAGS}") - if(USE_JFROG_CACHE STREQUAL "ON") + if (USE_JFROG_CACHE STREQUAL "ON") set(SQLITE_CACHE_PACKAGE_NAME "sqlite_${SQLITE_MD5}.tar.gz") set(SQLITE_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${SQLITE_CACHE_PACKAGE_NAME}") set(SQLITE_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${SQLITE_CACHE_PACKAGE_NAME}") @@ -698,15 +710,15 @@ macro(build_sqlite) "${SQLITE_STATIC_LIB}") ExternalProject_Create_Cache(sqlite_ep ${SQLITE_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/sqlite_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${SQLITE_CACHE_URL}) - else() + else () file(DOWNLOAD ${SQLITE_CACHE_URL} ${SQLITE_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${SQLITE_CACHE_URL} TO ${SQLITE_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) ExternalProject_Use_Cache(sqlite_ep ${SQLITE_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - endif() - else() + endif () + endif () + else () externalproject_add(sqlite_ep URL ${SQLITE_SOURCE_URL} @@ -721,7 +733,7 @@ macro(build_sqlite) 1 BUILD_BYPRODUCTS "${SQLITE_STATIC_LIB}") - endif() + endif () file(MAKE_DIRECTORY "${SQLITE_INCLUDE_DIR}") add_library(sqlite STATIC IMPORTED) @@ -733,11 +745,11 @@ macro(build_sqlite) add_dependencies(sqlite sqlite_ep) endmacro() -if(MILVUS_WITH_SQLITE) +if (MILVUS_WITH_SQLITE) resolve_dependency(SQLite) include_directories(SYSTEM "${SQLITE_INCLUDE_DIR}") link_directories(SYSTEM ${SQLITE_PREFIX}/lib/) -endif() +endif () # ---------------------------------------------------------------------- # SQLite_ORM @@ -753,16 +765,16 @@ macro(build_sqlite_orm) file(DOWNLOAD ${SQLITE_ORM_SOURCE_URL} ${SQLITE_ORM_TAR_NAME}) execute_process(COMMAND ${CMAKE_COMMAND} -E tar -xf ${SQLITE_ORM_TAR_NAME} - WORKING_DIRECTORY ${SQLITE_ORM_PREFIX}) + WORKING_DIRECTORY ${SQLITE_ORM_PREFIX}) endif () endmacro() -if(MILVUS_WITH_SQLITE_ORM) +if (MILVUS_WITH_SQLITE_ORM) resolve_dependency(SQLite_ORM) include_directories(SYSTEM "${SQLITE_ORM_INCLUDE_DIR}") -endif() +endif () # ---------------------------------------------------------------------- # yaml-cpp @@ -779,7 +791,7 @@ macro(build_yamlcpp) -DYAML_CPP_BUILD_TESTS=OFF -DYAML_CPP_BUILD_TOOLS=OFF) - if(USE_JFROG_CACHE STREQUAL "ON") + if (USE_JFROG_CACHE STREQUAL "ON") set(YAMLCPP_CACHE_PACKAGE_NAME "yaml-cpp_${YAMLCPP_MD5}.tar.gz") set(YAMLCPP_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${YAMLCPP_CACHE_PACKAGE_NAME}") set(YAMLCPP_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${YAMLCPP_CACHE_PACKAGE_NAME}") @@ -800,15 +812,15 @@ macro(build_yamlcpp) ${YAMLCPP_CMAKE_ARGS}) ExternalProject_Create_Cache(yaml-cpp_ep ${YAMLCPP_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/yaml-cpp_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${YAMLCPP_CACHE_URL}) - else() + else () file(DOWNLOAD ${YAMLCPP_CACHE_URL} ${YAMLCPP_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${YAMLCPP_CACHE_URL} TO ${YAMLCPP_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) ExternalProject_Use_Cache(yaml-cpp_ep ${YAMLCPP_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - endif() - else() + endif () + endif () + else () externalproject_add(yaml-cpp_ep URL ${YAMLCPP_SOURCE_URL} @@ -820,7 +832,7 @@ macro(build_yamlcpp) "${YAMLCPP_STATIC_LIB}" CMAKE_ARGS ${YAMLCPP_CMAKE_ARGS}) - endif() + endif () file(MAKE_DIRECTORY "${YAMLCPP_INCLUDE_DIR}") add_library(yaml-cpp STATIC IMPORTED) @@ -831,13 +843,13 @@ macro(build_yamlcpp) add_dependencies(yaml-cpp yaml-cpp_ep) endmacro() -if(MILVUS_WITH_YAMLCPP) +if (MILVUS_WITH_YAMLCPP) resolve_dependency(yaml-cpp) get_target_property(YAMLCPP_INCLUDE_DIR yaml-cpp INTERFACE_INCLUDE_DIRECTORIES) link_directories(SYSTEM ${YAMLCPP_PREFIX}/lib/) include_directories(SYSTEM ${YAMLCPP_INCLUDE_DIR}) -endif() +endif () # ---------------------------------------------------------------------- # libunwind @@ -849,7 +861,7 @@ macro(build_libunwind) set(LIBUNWIND_SHARED_LIB "${LIBUNWIND_PREFIX}/lib/libunwind${CMAKE_SHARED_LIBRARY_SUFFIX}") set(LIBUNWIND_CONFIGURE_ARGS "--prefix=${LIBUNWIND_PREFIX}") - if(USE_JFROG_CACHE STREQUAL "ON") + if (USE_JFROG_CACHE STREQUAL "ON") set(LIBUNWIND_CACHE_PACKAGE_NAME "libunwind_${LIBUNWIND_MD5}.tar.gz") set(LIBUNWIND_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${LIBUNWIND_CACHE_PACKAGE_NAME}") set(LIBUNWIND_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${LIBUNWIND_CACHE_PACKAGE_NAME}") @@ -874,15 +886,15 @@ macro(build_libunwind) ${LIBUNWIND_SHARED_LIB}) ExternalProject_Create_Cache(libunwind_ep ${LIBUNWIND_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/libunwind_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${LIBUNWIND_CACHE_URL}) - else() + else () file(DOWNLOAD ${LIBUNWIND_CACHE_URL} ${LIBUNWIND_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${LIBUNWIND_CACHE_URL} TO ${LIBUNWIND_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) ExternalProject_Use_Cache(libunwind_ep ${LIBUNWIND_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - endif() - else() + endif () + endif () + else () externalproject_add(libunwind_ep URL ${LIBUNWIND_SOURCE_URL} @@ -898,7 +910,7 @@ macro(build_libunwind) ${MAKE} install BUILD_BYPRODUCTS ${LIBUNWIND_SHARED_LIB}) - endif() + endif () file(MAKE_DIRECTORY "${LIBUNWIND_INCLUDE_DIR}") @@ -910,12 +922,12 @@ macro(build_libunwind) add_dependencies(libunwind libunwind_ep) endmacro() -if(MILVUS_WITH_LIBUNWIND) +if (MILVUS_WITH_LIBUNWIND) resolve_dependency(libunwind) get_target_property(LIBUNWIND_INCLUDE_DIR libunwind INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM ${LIBUNWIND_INCLUDE_DIR}) -endif() +endif () # ---------------------------------------------------------------------- # gperftools @@ -927,7 +939,7 @@ macro(build_gperftools) set(GPERFTOOLS_STATIC_LIB "${GPERFTOOLS_PREFIX}/lib/libprofiler${CMAKE_STATIC_LIBRARY_SUFFIX}") set(GPERFTOOLS_CONFIGURE_ARGS "--prefix=${GPERFTOOLS_PREFIX}") - if(USE_JFROG_CACHE STREQUAL "ON") + if (USE_JFROG_CACHE STREQUAL "ON") set(GPERFTOOLS_CACHE_PACKAGE_NAME "gperftools_${GPERFTOOLS_MD5}.tar.gz") set(GPERFTOOLS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${GPERFTOOLS_CACHE_PACKAGE_NAME}") set(GPERFTOOLS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${GPERFTOOLS_CACHE_PACKAGE_NAME}") @@ -952,15 +964,15 @@ macro(build_gperftools) ${GPERFTOOLS_STATIC_LIB}) ExternalProject_Create_Cache(gperftools_ep ${GPERFTOOLS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/gperftools_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GPERFTOOLS_CACHE_URL}) - else() + else () file(DOWNLOAD ${GPERFTOOLS_CACHE_URL} ${GPERFTOOLS_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${GPERFTOOLS_CACHE_URL} TO ${GPERFTOOLS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) ExternalProject_Use_Cache(gperftools_ep ${GPERFTOOLS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - endif() - else() + endif () + endif () + else () externalproject_add(gperftools_ep URL ${GPERFTOOLS_SOURCE_URL} @@ -976,7 +988,7 @@ macro(build_gperftools) ${MAKE} install BUILD_BYPRODUCTS ${GPERFTOOLS_STATIC_LIB}) - endif() + endif () ExternalProject_Add_StepDependencies(gperftools_ep build libunwind_ep) @@ -992,13 +1004,13 @@ macro(build_gperftools) add_dependencies(gperftools libunwind_ep) endmacro() -if(MILVUS_WITH_GPERFTOOLS) +if (MILVUS_WITH_GPERFTOOLS) resolve_dependency(gperftools) get_target_property(GPERFTOOLS_INCLUDE_DIR gperftools INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM ${GPERFTOOLS_INCLUDE_DIR}) link_directories(SYSTEM ${GPERFTOOLS_PREFIX}/lib) -endif() +endif () # ---------------------------------------------------------------------- # GRPC @@ -1014,7 +1026,7 @@ macro(build_grpc) set(GRPC_PROTOBUF_STATIC_LIB "${GRPC_PROTOBUF_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}protobuf${CMAKE_STATIC_LIBRARY_SUFFIX}") set(GRPC_PROTOC_STATIC_LIB "${GRPC_PROTOBUF_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}protoc${CMAKE_STATIC_LIBRARY_SUFFIX}") - if(USE_JFROG_CACHE STREQUAL "ON") + if (USE_JFROG_CACHE STREQUAL "ON") set(GRPC_CACHE_PACKAGE_NAME "grpc_${GRPC_MD5}.tar.gz") set(GRPC_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${GRPC_CACHE_PACKAGE_NAME}") set(GRPC_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${GRPC_CACHE_PACKAGE_NAME}") @@ -1044,15 +1056,15 @@ macro(build_grpc) ExternalProject_Add_StepDependencies(grpc_ep build zlib_ep) ExternalProject_Create_Cache(grpc_ep ${GRPC_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GRPC_CACHE_URL}) - else() + else () file(DOWNLOAD ${GRPC_CACHE_URL} ${GRPC_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${GRPC_CACHE_URL} TO ${GRPC_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) ExternalProject_Use_Cache(grpc_ep ${GRPC_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - endif() - else() + endif () + endif () + else () externalproject_add(grpc_ep URL ${GRPC_SOURCE_URL} @@ -1074,7 +1086,7 @@ macro(build_grpc) ExternalProject_Add_StepDependencies(grpc_ep build zlib_ep) - endif() + endif () file(MAKE_DIRECTORY "${GRPC_INCLUDE_DIR}") @@ -1082,29 +1094,29 @@ macro(build_grpc) set_target_properties(grpc PROPERTIES IMPORTED_LOCATION "${GRPC_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${GRPC_INCLUDE_DIR}" - INTERFACE_LINK_LIBRARIES "zlib" ) + INTERFACE_LINK_LIBRARIES "zlib") add_library(grpc++ STATIC IMPORTED) set_target_properties(grpc++ PROPERTIES IMPORTED_LOCATION "${GRPC++_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${GRPC_INCLUDE_DIR}" - INTERFACE_LINK_LIBRARIES "zlib" ) + INTERFACE_LINK_LIBRARIES "zlib") add_library(grpcpp_channelz STATIC IMPORTED) set_target_properties(grpcpp_channelz PROPERTIES IMPORTED_LOCATION "${GRPCPP_CHANNELZ_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${GRPC_INCLUDE_DIR}" - INTERFACE_LINK_LIBRARIES "zlib" ) + INTERFACE_LINK_LIBRARIES "zlib") add_library(grpc_protobuf STATIC IMPORTED) set_target_properties(grpc_protobuf PROPERTIES IMPORTED_LOCATION "${GRPC_PROTOBUF_STATIC_LIB}" - INTERFACE_LINK_LIBRARIES "zlib" ) + INTERFACE_LINK_LIBRARIES "zlib") add_library(grpc_protoc STATIC IMPORTED) set_target_properties(grpc_protoc PROPERTIES IMPORTED_LOCATION "${GRPC_PROTOC_STATIC_LIB}" - INTERFACE_LINK_LIBRARIES "zlib" ) + INTERFACE_LINK_LIBRARIES "zlib") add_dependencies(grpc grpc_ep) add_dependencies(grpc++ grpc_ep) @@ -1113,7 +1125,7 @@ macro(build_grpc) add_dependencies(grpc_protoc grpc_ep) endmacro() -if(MILVUS_WITH_GRPC) +if (MILVUS_WITH_GRPC) resolve_dependency(GRPC) get_target_property(GRPC_INCLUDE_DIR grpc INTERFACE_INCLUDE_DIRECTORIES) @@ -1123,7 +1135,7 @@ if(MILVUS_WITH_GRPC) set(GRPC_THIRD_PARTY_DIR ${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix/src/grpc_ep/third_party) include_directories(SYSTEM ${GRPC_THIRD_PARTY_DIR}/protobuf/src) link_directories(SYSTEM ${GRPC_PROTOBUF_LIB_DIR}) -endif() +endif () # ---------------------------------------------------------------------- # zlib @@ -1137,7 +1149,7 @@ macro(build_zlib) set(ZLIB_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX}" -DBUILD_SHARED_LIBS=OFF) - if(USE_JFROG_CACHE STREQUAL "ON") + if (USE_JFROG_CACHE STREQUAL "ON") set(ZLIB_CACHE_PACKAGE_NAME "zlib_${ZLIB_MD5}.tar.gz") set(ZLIB_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${ZLIB_CACHE_PACKAGE_NAME}") set(ZLIB_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${ZLIB_CACHE_PACKAGE_NAME}") @@ -1158,15 +1170,15 @@ macro(build_zlib) ${ZLIB_CMAKE_ARGS}) ExternalProject_Create_Cache(zlib_ep ${ZLIB_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/zlib_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ZLIB_CACHE_URL}) - else() + else () file(DOWNLOAD ${ZLIB_CACHE_URL} ${ZLIB_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${ZLIB_CACHE_URL} TO ${ZLIB_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) ExternalProject_Use_Cache(zlib_ep ${ZLIB_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - endif() - else() + endif () + endif () + else () externalproject_add(zlib_ep URL ${ZLIB_SOURCE_URL} @@ -1178,7 +1190,7 @@ macro(build_zlib) "${ZLIB_STATIC_LIB}" CMAKE_ARGS ${ZLIB_CMAKE_ARGS}) - endif() + endif () file(MAKE_DIRECTORY "${ZLIB_INCLUDE_DIR}") add_library(zlib STATIC IMPORTED) @@ -1189,9 +1201,9 @@ macro(build_zlib) add_dependencies(zlib zlib_ep) endmacro() -if(MILVUS_WITH_ZLIB) +if (MILVUS_WITH_ZLIB) resolve_dependency(ZLIB) get_target_property(ZLIB_INCLUDE_DIR zlib INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM ${ZLIB_INCLUDE_DIR}) -endif() +endif () From c99f407099748295356aed5d01a412f1801dabc3 Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Wed, 30 Oct 2019 15:30:28 +0800 Subject: [PATCH 3/7] delete message Former-commit-id: 577b8693eca42bf3537c8691f754dd1876d579a3 --- core/cmake/ThirdPartyPackages.cmake | 1 - 1 file changed, 1 deletion(-) diff --git a/core/cmake/ThirdPartyPackages.cmake b/core/cmake/ThirdPartyPackages.cmake index 788c09ef2e..62d7713024 100644 --- a/core/cmake/ThirdPartyPackages.cmake +++ b/core/cmake/ThirdPartyPackages.cmake @@ -139,7 +139,6 @@ if (USE_JFROG_CACHE STREQUAL "ON") endif () if (UBUNTU_FOUND) set(JFROG_ARTFACTORY_CACHE_URL "${JFROG_ARTFACTORY_URL}/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${UBUNTU_VERSION}/${MILVUS_BUILD_ARCH}/${BUILD_TYPE}") - message(${JFROG_ARTFACTORY_CACHE_URL}) else () set(JFROG_ARTFACTORY_CACHE_URL "${JFROG_ARTFACTORY_URL}/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${MILVUS_BUILD_ARCH}/${BUILD_TYPE}") endif () From a4f0f4bd13844d183f4b1e5c4c7c3f961ebe6ce1 Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 30 Oct 2019 17:37:44 +0800 Subject: [PATCH 4/7] get branch name Former-commit-id: eb75eb1a41845c294479a8e15176feac629be9a8 --- core/CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index ae4366af2f..df7cd9183b 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -32,7 +32,10 @@ string(REGEX REPLACE "\n" "" BUILD_TIME ${BUILD_TIME}) message(STATUS "Build time = ${BUILD_TIME}") MACRO (GET_GIT_BRANCH_NAME GIT_BRANCH_NAME) - execute_process(COMMAND "git" symbolic-ref --short HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) + execute_process(COMMAND "git" rev-parse --abbrev-ref HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) + if(GIT_BRANCH_NAME STREQUAL "") + execute_process(COMMAND "git" symbolic-ref --short -q HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) + endif() ENDMACRO (GET_GIT_BRANCH_NAME) GET_GIT_BRANCH_NAME(GIT_BRANCH_NAME) From 7dc552fdea6f488e8e18d97a0ed8053a4948cf31 Mon Sep 17 00:00:00 2001 From: wxyu Date: Wed, 30 Oct 2019 19:57:55 +0800 Subject: [PATCH 5/7] New config opion use_gpu_threshold Former-commit-id: 3d33145d8fa473f2b81cc98022c1bb93853a2fbc --- CHANGELOG.md | 1 + core/conf/server_config.template | 1 + .../src/scheduler/optimizer/LargeSQ8HPass.cpp | 12 ++++- core/src/scheduler/optimizer/LargeSQ8HPass.h | 5 ++- core/src/server/Config.cpp | 45 +++++++++++++++++++ core/src/server/Config.h | 8 ++++ 6 files changed, 70 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b28c7bd3a7..5176ce2b79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#90 - The server start error messages could be improved to enhance user experience - \#104 - test_scheduler core dump - \#115 - Using new structure for tasktable +- \#139 - New config opion use_gpu_threshold ## Improvement - \#64 - Improvement dump function in scheduler diff --git a/core/conf/server_config.template b/core/conf/server_config.template index 3feb16fd63..a23707be5d 100644 --- a/core/conf/server_config.template +++ b/core/conf/server_config.template @@ -36,6 +36,7 @@ cache_config: engine_config: use_blas_threshold: 20 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times + use_gpu_threshold: 1000 resource_config: search_resources: # define the GPUs used for search computation, must be in format: gpux diff --git a/core/src/scheduler/optimizer/LargeSQ8HPass.cpp b/core/src/scheduler/optimizer/LargeSQ8HPass.cpp index 62d0e57902..ddcee414a7 100644 --- a/core/src/scheduler/optimizer/LargeSQ8HPass.cpp +++ b/core/src/scheduler/optimizer/LargeSQ8HPass.cpp @@ -21,11 +21,20 @@ #include "scheduler/Utils.h" #include "scheduler/task/SearchTask.h" #include "scheduler/tasklabel/SpecResLabel.h" +#include "server/Config.h" #include "utils/Log.h" namespace milvus { namespace scheduler { +LargeSQ8HPass::LargeSQ8HPass() { + server::Config& config = server::Config::GetInstance(); + Status s = config.GetEngineConfigUseGpuThreshold(threshold_); + if (!s.ok()) { + threshold_ = std::numeric_limits::max(); + } +} + bool LargeSQ8HPass::Run(const TaskPtr& task) { if (task->Type() != TaskType::SearchTask) { @@ -40,7 +49,8 @@ LargeSQ8HPass::Run(const TaskPtr& task) { auto search_job = std::static_pointer_cast(search_task->job_.lock()); // TODO: future, Index::IVFSQ8H, if nq < threshold set cpu, else set gpu - if (search_job->nq() < 100) { + + if (search_job->nq() < threshold_) { return false; } diff --git a/core/src/scheduler/optimizer/LargeSQ8HPass.h b/core/src/scheduler/optimizer/LargeSQ8HPass.h index 49e658002f..f293e1db73 100644 --- a/core/src/scheduler/optimizer/LargeSQ8HPass.h +++ b/core/src/scheduler/optimizer/LargeSQ8HPass.h @@ -34,11 +34,14 @@ namespace scheduler { class LargeSQ8HPass : public Pass { public: - LargeSQ8HPass() = default; + LargeSQ8HPass(); public: bool Run(const TaskPtr& task) override; + + private: + int32_t threshold_ = std::numeric_limits::max(); }; using LargeSQ8HPassPtr = std::shared_ptr; diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index 111cc26f9c..86caf6dd37 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -193,6 +193,12 @@ Config::ValidateConfig() { return s; } + int32_t engine_use_gpu_threshold; + s = GetEngineConfigUseGpuThreshold(engine_use_gpu_threshold); + if (!s.ok()) { + return s; + } + /* resource config */ std::string resource_mode; s = GetResourceConfigMode(resource_mode); @@ -324,6 +330,11 @@ Config::ResetDefaultConfig() { return s; } + s = SetEngineConfigUseGpuThreshold(CONFIG_ENGINE_USE_GPU_THRESHOLD_DEFAULT); + if (!s.ok()) { + return s; + } + /* resource config */ s = SetResourceConfigMode(CONFIG_RESOURCE_MODE_DEFAULT); if (!s.ok()) { @@ -656,6 +667,16 @@ Config::CheckEngineConfigOmpThreadNum(const std::string& value) { return Status::OK(); } +Status +Config::CheckEngineConfigUseGpuThreshold(const std::string& value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + std::string msg = "Invalid gpu threshold: " + value + + ". Possible reason: engine_config.use_gpu_threshold is not a positive integer."; + return Status(SERVER_INVALID_ARGUMENT, msg); + } + return Status::OK(); +} + Status Config::CheckResourceConfigMode(const std::string& value) { if (value != "simple") { @@ -951,6 +972,19 @@ Config::GetEngineConfigOmpThreadNum(int32_t& value) { return Status::OK(); } +Status +Config::GetEngineConfigUseGpuThreshold(int32_t& value) { + std::string str = + GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_USE_GPU_THRESHOLD, CONFIG_ENGINE_USE_GPU_THRESHOLD_DEFAULT); + Status s = CheckEngineConfigUseGpuThreshold(str); + if (!s.ok()) { + return s; + } + + value = std::stoi(str); + return Status::OK(); +} + Status Config::GetResourceConfigMode(std::string& value) { value = GetConfigStr(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, CONFIG_RESOURCE_MODE_DEFAULT); @@ -1203,6 +1237,17 @@ Config::SetEngineConfigOmpThreadNum(const std::string& value) { return Status::OK(); } +Status +Config::SetEngineConfigUseGpuThreshold(const std::string& value) { + Status s = CheckEngineConfigUseGpuThreshold(value); + if (!s.ok()) { + return s; + } + + SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_USE_GPU_THRESHOLD, value); + return Status::OK(); +} + /* resource config */ Status Config::SetResourceConfigMode(const std::string& value) { diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 4cab25a1c6..3e7ae0c818 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -84,6 +84,8 @@ static const char* CONFIG_ENGINE_USE_BLAS_THRESHOLD = "use_blas_threshold"; static const char* CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT = "20"; static const char* CONFIG_ENGINE_OMP_THREAD_NUM = "omp_thread_num"; static const char* CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0"; +static const char* CONFIG_ENGINE_USE_GPU_THRESHOLD = "use_gpu_threshold"; +static const char* CONFIG_ENGINE_USE_GPU_THRESHOLD_DEFAULT = "1000"; /* resource config */ static const char* CONFIG_RESOURCE = "resource_config"; @@ -166,6 +168,8 @@ class Config { CheckEngineConfigUseBlasThreshold(const std::string& value); Status CheckEngineConfigOmpThreadNum(const std::string& value); + Status + CheckEngineConfigUseGpuThreshold(const std::string& value); /* resource config */ Status @@ -230,6 +234,8 @@ class Config { GetEngineConfigUseBlasThreshold(int32_t& value); Status GetEngineConfigOmpThreadNum(int32_t& value); + Status + GetEngineConfigUseGpuThreshold(int32_t& value); /* resource config */ Status @@ -289,6 +295,8 @@ class Config { SetEngineConfigUseBlasThreshold(const std::string& value); Status SetEngineConfigOmpThreadNum(const std::string& value); + Status + SetEngineConfigUseGpuThreshold(const std::string& value); /* resource config */ Status From 87963408bf24e73d1f655c7011f2b97ed8fe34b7 Mon Sep 17 00:00:00 2001 From: wxyu Date: Wed, 30 Oct 2019 20:24:10 +0800 Subject: [PATCH 6/7] fix lint Former-commit-id: 2f5166b4c2a461c379c55eb5a49d431e878aa19a --- core/src/scheduler/optimizer/LargeSQ8HPass.h | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/scheduler/optimizer/LargeSQ8HPass.h b/core/src/scheduler/optimizer/LargeSQ8HPass.h index f293e1db73..1e00aa4a1b 100644 --- a/core/src/scheduler/optimizer/LargeSQ8HPass.h +++ b/core/src/scheduler/optimizer/LargeSQ8HPass.h @@ -18,6 +18,7 @@ #include #include +#include #include #include #include From eeb503f7c548a5d5c373ea08debc0cd2f5bf4339 Mon Sep 17 00:00:00 2001 From: JinHai-CN Date: Thu, 31 Oct 2019 14:07:23 +0800 Subject: [PATCH 7/7] Update faiss link Former-commit-id: 22febba9b19f8de8422eae922ec75d7106e37e7a --- core/src/index/cmake/ThirdPartyPackagesCore.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/index/cmake/ThirdPartyPackagesCore.cmake b/core/src/index/cmake/ThirdPartyPackagesCore.cmake index e8a5c8a995..c066f7b392 100644 --- a/core/src/index/cmake/ThirdPartyPackagesCore.cmake +++ b/core/src/index/cmake/ThirdPartyPackagesCore.cmake @@ -245,7 +245,8 @@ if(CUSTOMIZATION) # set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1 # set(FAISS_MD5 "c89ea8e655f5cdf58f42486f13614714") # commit-id 9c28a1cbb88f41fa03b03d7204106201ad33276b branch-0.2.1 # set(FAISS_MD5 "87fdd86351ffcaf3f80dc26ade63c44b") # commit-id 841a156e67e8e22cd8088e1b58c00afbf2efc30b branch-0.2.1 - set(FAISS_MD5 "f3b2ce3364c3fa7febd3aa7fdd0fe380") # commit-id 694e03458e6b69ce8a62502f71f69a614af5af8f branch-0.3.0 + # set(FAISS_MD5 "f3b2ce3364c3fa7febd3aa7fdd0fe380") # commit-id 694e03458e6b69ce8a62502f71f69a614af5af8f branch-0.3.0 + set(FAISS_MD5 "bb30722c22390ce5f6759ccb216c1b2a") # commit-id d324db297475286afe107847c7fb7a0f9dc7e90e branch-0.3.0 endif() else() set(FAISS_SOURCE_URL "https://github.com/milvus-io/faiss/archive/1.6.0.tar.gz")