From b43b9bac27b0bd0187dc0e04c0488a2613319e9e Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Tue, 29 Oct 2019 19:15:54 +0800 Subject: [PATCH 01/18] #89 do normalize() for IP test Former-commit-id: 11be41ee43e3dd0e2ecbcd50ba70bab2df58688d --- .../faiss_benchmark/faiss_benchmark_test.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp index e80b85e024..dfce6970cc 100644 --- a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp +++ b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp @@ -54,6 +54,20 @@ elapsed() { return tv.tv_sec + tv.tv_usec * 1e-6; } +void normalize(float* arr, size_t nq, size_t dim) { + for (size_t i = 0; i < nq; i++) { + double vecLen = 0.0; + for (size_t j = 0; j < dim; j++) { + double val = arr[i * dim + j]; + vecLen += val * val; + } + vecLen = std::sqrt(vecLen); + for (size_t j = 0; j < dim; j++) { + arr[i * dim + j] = (float) (arr[i * dim + j] / vecLen); + } + } +} + void* hdf5_read(const char* file_name, const char* dataset_name, H5T_class_t dataset_class, size_t& d_out, size_t& n_out) { hid_t file, dataset, datatype, dataspace, memspace; @@ -237,6 +251,11 @@ test_ann_hdf5(const std::string& ann_test_name, const std::string& index_key, in float* xb = (float*)hdf5_read(ann_file_name.c_str(), "train", H5T_FLOAT, d, nb); assert(d == dim || !"dataset does not have correct dimension"); + if (metric_type == faiss::METRIC_INNER_PRODUCT) { + printf("[%.3f s] Normalizing data set \n", elapsed() - t0); + normalize(xb, nb, d); + } + printf("[%.3f s] Preparing index \"%s\" d=%ld\n", elapsed() - t0, index_key.c_str(), d); index = faiss::index_factory(d, index_key.c_str(), metric_type); From b6484475d891620d639c2d5e3240a2fdc5d87a5b Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Tue, 29 Oct 2019 19:22:21 +0800 Subject: [PATCH 02/18] #89 fix clang-format Former-commit-id: 0c7feb7370d58e7511a1089d181d256e9c9e6905 --- .../index/unittest/faiss_benchmark/faiss_benchmark_test.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp index dfce6970cc..618ee00b28 100644 --- a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp +++ b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp @@ -54,7 +54,8 @@ elapsed() { return tv.tv_sec + tv.tv_usec * 1e-6; } -void normalize(float* arr, size_t nq, size_t dim) { +void +normalize(float* arr, size_t nq, size_t dim) { for (size_t i = 0; i < nq; i++) { double vecLen = 0.0; for (size_t j = 0; j < dim; j++) { @@ -63,7 +64,7 @@ void normalize(float* arr, size_t nq, size_t dim) { } vecLen = std::sqrt(vecLen); for (size_t j = 0; j < dim; j++) { - arr[i * dim + j] = (float) (arr[i * dim + j] / vecLen); + arr[i * dim + j] = (float)(arr[i * dim + j] / vecLen); } } } From c41ebb7e7db13ec6eecfa4bcead2b1650196c05d Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Wed, 30 Oct 2019 15:27:05 +0800 Subject: [PATCH 03/18] fix JFrog cache error Former-commit-id: 8b75351080e9201d6f9f47c643dfb7eec32d5a86 --- CHANGELOG.md | 1 + core/cmake/ThirdPartyPackages.cmake | 376 ++++++++++++++-------------- 2 files changed, 195 insertions(+), 182 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b28c7bd3a7..4580c6d55e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ Please mark all change in change log and use the ticket from JIRA. # Milvus 0.5.1 (TODO) ## Bug +- \#134 - JFrog cache error ## Feature - \#90 - The server start error messages could be improved to enhance user experience diff --git a/core/cmake/ThirdPartyPackages.cmake b/core/cmake/ThirdPartyPackages.cmake index d0057d3c22..788c09ef2e 100644 --- a/core/cmake/ThirdPartyPackages.cmake +++ b/core/cmake/ThirdPartyPackages.cmake @@ -30,11 +30,11 @@ set(MILVUS_THIRDPARTY_DEPENDENCIES message(STATUS "Using ${MILVUS_DEPENDENCY_SOURCE} approach to find dependencies") # For each dependency, set dependency source to global default, if unset -foreach(DEPENDENCY ${MILVUS_THIRDPARTY_DEPENDENCIES}) - if("${${DEPENDENCY}_SOURCE}" STREQUAL "") +foreach (DEPENDENCY ${MILVUS_THIRDPARTY_DEPENDENCIES}) + if ("${${DEPENDENCY}_SOURCE}" STREQUAL "") set(${DEPENDENCY}_SOURCE ${MILVUS_DEPENDENCY_SOURCE}) - endif() -endforeach() + endif () +endforeach () macro(build_dependency DEPENDENCY_NAME) if ("${DEPENDENCY_NAME}" STREQUAL "GTest") @@ -47,17 +47,17 @@ macro(build_dependency DEPENDENCY_NAME) build_sqlite() elseif ("${DEPENDENCY_NAME}" STREQUAL "SQLite_ORM") build_sqlite_orm() - elseif("${DEPENDENCY_NAME}" STREQUAL "yaml-cpp") + elseif ("${DEPENDENCY_NAME}" STREQUAL "yaml-cpp") build_yamlcpp() - elseif("${DEPENDENCY_NAME}" STREQUAL "libunwind") + elseif ("${DEPENDENCY_NAME}" STREQUAL "libunwind") build_libunwind() - elseif("${DEPENDENCY_NAME}" STREQUAL "gperftools") + elseif ("${DEPENDENCY_NAME}" STREQUAL "gperftools") build_gperftools() - elseif("${DEPENDENCY_NAME}" STREQUAL "GRPC") + elseif ("${DEPENDENCY_NAME}" STREQUAL "GRPC") build_grpc() - elseif("${DEPENDENCY_NAME}" STREQUAL "ZLIB") + elseif ("${DEPENDENCY_NAME}" STREQUAL "ZLIB") build_zlib() - else() + else () message(FATAL_ERROR "Unknown thirdparty dependency to build: ${DEPENDENCY_NAME}") endif () endmacro() @@ -66,28 +66,28 @@ endmacro() # Identify OS if (UNIX) if (APPLE) - set (CMAKE_OS_NAME "osx" CACHE STRING "Operating system name" FORCE) + set(CMAKE_OS_NAME "osx" CACHE STRING "Operating system name" FORCE) else (APPLE) ## Check for Debian GNU/Linux ________________ - find_file (DEBIAN_FOUND debian_version debconf.conf - PATHS /etc - ) + find_file(DEBIAN_FOUND debian_version debconf.conf + PATHS /etc + ) if (DEBIAN_FOUND) - set (CMAKE_OS_NAME "debian" CACHE STRING "Operating system name" FORCE) + set(CMAKE_OS_NAME "debian" CACHE STRING "Operating system name" FORCE) endif (DEBIAN_FOUND) ## Check for Fedora _________________________ - find_file (FEDORA_FOUND fedora-release - PATHS /etc - ) + find_file(FEDORA_FOUND fedora-release + PATHS /etc + ) if (FEDORA_FOUND) - set (CMAKE_OS_NAME "fedora" CACHE STRING "Operating system name" FORCE) + set(CMAKE_OS_NAME "fedora" CACHE STRING "Operating system name" FORCE) endif (FEDORA_FOUND) ## Check for RedHat _________________________ - find_file (REDHAT_FOUND redhat-release inittab.RH - PATHS /etc - ) + find_file(REDHAT_FOUND redhat-release inittab.RH + PATHS /etc + ) if (REDHAT_FOUND) - set (CMAKE_OS_NAME "redhat" CACHE STRING "Operating system name" FORCE) + set(CMAKE_OS_NAME "redhat" CACHE STRING "Operating system name" FORCE) endif (REDHAT_FOUND) ## Extra check for Ubuntu ____________________ if (DEBIAN_FOUND) @@ -96,18 +96,25 @@ if (UNIX) ## a first superficial inspection a system will ## be considered as Debian, which signifies an ## extra check is required. - find_file (UBUNTU_EXTRA legal issue - PATHS /etc - ) + find_file(UBUNTU_EXTRA legal issue + PATHS /etc + ) if (UBUNTU_EXTRA) ## Scan contents of file - file (STRINGS ${UBUNTU_EXTRA} UBUNTU_FOUND - REGEX Ubuntu - ) + file(STRINGS ${UBUNTU_EXTRA} UBUNTU_FOUND + REGEX Ubuntu + ) ## Check result of string search if (UBUNTU_FOUND) - set (CMAKE_OS_NAME "ubuntu" CACHE STRING "Operating system name" FORCE) - set (DEBIAN_FOUND FALSE) + set(CMAKE_OS_NAME "ubuntu" CACHE STRING "Operating system name" FORCE) + set(DEBIAN_FOUND FALSE) + + find_program(LSB_RELEASE_EXEC lsb_release) + execute_process(COMMAND ${LSB_RELEASE_EXEC} -rs + OUTPUT_VARIABLE LSB_RELEASE_ID_SHORT + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + STRING(REGEX REPLACE "\\." "_" UBUNTU_VERSION ${LSB_RELEASE_ID_SHORT}) endif (UBUNTU_FOUND) endif (UBUNTU_EXTRA) endif (DEBIAN_FOUND) @@ -120,36 +127,41 @@ set(THIRDPARTY_DIR "${MILVUS_SOURCE_DIR}/thirdparty") # ---------------------------------------------------------------------- # JFrog -if(NOT DEFINED USE_JFROG_CACHE) +if (NOT DEFINED USE_JFROG_CACHE) set(USE_JFROG_CACHE "OFF") -endif() -if(USE_JFROG_CACHE STREQUAL "ON") - if(DEFINED ENV{JFROG_ARTFACTORY_URL}) +endif () +if (USE_JFROG_CACHE STREQUAL "ON") + if (DEFINED ENV{JFROG_ARTFACTORY_URL}) set(JFROG_ARTFACTORY_URL "$ENV{JFROG_ARTFACTORY_URL}") - endif() - if(NOT DEFINED JFROG_ARTFACTORY_URL) + endif () + if (NOT DEFINED JFROG_ARTFACTORY_URL) message(FATAL_ERROR "JFROG_ARTFACTORY_URL is not set") - endif() - set(JFROG_ARTFACTORY_CACHE_URL "${JFROG_ARTFACTORY_URL}/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${MILVUS_BUILD_ARCH}/${BUILD_TYPE}") - if(DEFINED ENV{JFROG_USER_NAME}) + endif () + if (UBUNTU_FOUND) + set(JFROG_ARTFACTORY_CACHE_URL "${JFROG_ARTFACTORY_URL}/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${UBUNTU_VERSION}/${MILVUS_BUILD_ARCH}/${BUILD_TYPE}") + message(${JFROG_ARTFACTORY_CACHE_URL}) + else () + set(JFROG_ARTFACTORY_CACHE_URL "${JFROG_ARTFACTORY_URL}/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${MILVUS_BUILD_ARCH}/${BUILD_TYPE}") + endif () + if (DEFINED ENV{JFROG_USER_NAME}) set(JFROG_USER_NAME "$ENV{JFROG_USER_NAME}") - endif() - if(NOT DEFINED JFROG_USER_NAME) + endif () + if (NOT DEFINED JFROG_USER_NAME) message(FATAL_ERROR "JFROG_USER_NAME is not set") - endif() - if(DEFINED ENV{JFROG_PASSWORD}) + endif () + if (DEFINED ENV{JFROG_PASSWORD}) set(JFROG_PASSWORD "$ENV{JFROG_PASSWORD}") - endif() - if(NOT DEFINED JFROG_PASSWORD) + endif () + if (NOT DEFINED JFROG_PASSWORD) message(FATAL_ERROR "JFROG_PASSWORD is not set") - endif() + endif () set(THIRDPARTY_PACKAGE_CACHE "${THIRDPARTY_DIR}/cache") - if(NOT EXISTS ${THIRDPARTY_PACKAGE_CACHE}) + if (NOT EXISTS ${THIRDPARTY_PACKAGE_CACHE}) message(STATUS "Will create cached directory: ${THIRDPARTY_PACKAGE_CACHE}") file(MAKE_DIRECTORY ${THIRDPARTY_PACKAGE_CACHE}) - endif() -endif() + endif () +endif () macro(resolve_dependency DEPENDENCY_NAME) if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO") @@ -182,13 +194,13 @@ set(EP_C_FLAGS "${EP_C_FLAGS} -fPIC") set(EP_COMMON_TOOLCHAIN -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}) -if(CMAKE_AR) +if (CMAKE_AR) set(EP_COMMON_TOOLCHAIN ${EP_COMMON_TOOLCHAIN} -DCMAKE_AR=${CMAKE_AR}) -endif() +endif () -if(CMAKE_RANLIB) +if (CMAKE_RANLIB) set(EP_COMMON_TOOLCHAIN ${EP_COMMON_TOOLCHAIN} -DCMAKE_RANLIB=${CMAKE_RANLIB}) -endif() +endif () # External projects are still able to override the following declarations. # cmake command line will favor the last defined variable when a duplicate is @@ -202,20 +214,20 @@ set(EP_COMMON_CMAKE_ARGS -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS} -DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS}) -if(NOT MILVUS_VERBOSE_THIRDPARTY_BUILD) +if (NOT MILVUS_VERBOSE_THIRDPARTY_BUILD) set(EP_LOG_OPTIONS LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1 LOG_DOWNLOAD 1) -else() +else () set(EP_LOG_OPTIONS) -endif() +endif () # Ensure that a default make is set -if("${MAKE}" STREQUAL "") +if ("${MAKE}" STREQUAL "") find_program(MAKE make) -endif() +endif () if (NOT DEFINED MAKE_BUILD_ARGS) set(MAKE_BUILD_ARGS "-j8") -endif() +endif () message(STATUS "Third Party MAKE_BUILD_ARGS = ${MAKE_BUILD_ARGS}") # ---------------------------------------------------------------------- @@ -230,39 +242,39 @@ find_package(Threads REQUIRED) # Read toolchain versions from cpp/thirdparty/versions.txt file(STRINGS "${THIRDPARTY_DIR}/versions.txt" TOOLCHAIN_VERSIONS_TXT) -foreach(_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT}) +foreach (_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT}) # Exclude comments - if(NOT _VERSION_ENTRY MATCHES "^[^#][A-Za-z0-9-_]+_VERSION=") + if (NOT _VERSION_ENTRY MATCHES "^[^#][A-Za-z0-9-_]+_VERSION=") continue() - endif() + endif () string(REGEX MATCH "^[^=]*" _LIB_NAME ${_VERSION_ENTRY}) string(REPLACE "${_LIB_NAME}=" "" _LIB_VERSION ${_VERSION_ENTRY}) # Skip blank or malformed lines - if(${_LIB_VERSION} STREQUAL "") + if (${_LIB_VERSION} STREQUAL "") continue() - endif() + endif () # For debugging #message(STATUS "${_LIB_NAME}: ${_LIB_VERSION}") set(${_LIB_NAME} "${_LIB_VERSION}") -endforeach() +endforeach () if (DEFINED ENV{MILVUS_GTEST_URL}) set(GTEST_SOURCE_URL "$ENV{MILVUS_GTEST_URL}") else () set(GTEST_SOURCE_URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz") -endif() +endif () set(GTEST_MD5 "2e6fbeb6a91310a16efe181886c59596") -if(DEFINED ENV{MILVUS_MYSQLPP_URL}) +if (DEFINED ENV{MILVUS_MYSQLPP_URL}) set(MYSQLPP_SOURCE_URL "$ENV{MILVUS_MYSQLPP_URL}") -else() +else () set(MYSQLPP_SOURCE_URL "https://tangentsoft.com/mysqlpp/releases/mysql++-${MYSQLPP_VERSION}.tar.gz") -endif() +endif () set(MYSQLPP_MD5 "cda38b5ecc0117de91f7c42292dd1e79") if (DEFINED ENV{MILVUS_PROMETHEUS_URL}) @@ -270,60 +282,60 @@ if (DEFINED ENV{MILVUS_PROMETHEUS_URL}) else () set(PROMETHEUS_SOURCE_URL https://github.com/jupp0r/prometheus-cpp.git) -endif() +endif () -if(DEFINED ENV{MILVUS_SQLITE_URL}) +if (DEFINED ENV{MILVUS_SQLITE_URL}) set(SQLITE_SOURCE_URL "$ENV{MILVUS_SQLITE_URL}") -else() +else () set(SQLITE_SOURCE_URL "https://www.sqlite.org/2019/sqlite-autoconf-${SQLITE_VERSION}.tar.gz") -endif() +endif () set(SQLITE_MD5 "3c68eb400f8354605736cd55400e1572") -if(DEFINED ENV{MILVUS_SQLITE_ORM_URL}) +if (DEFINED ENV{MILVUS_SQLITE_ORM_URL}) set(SQLITE_ORM_SOURCE_URL "$ENV{MILVUS_SQLITE_ORM_URL}") -else() +else () set(SQLITE_ORM_SOURCE_URL "https://github.com/fnc12/sqlite_orm/archive/${SQLITE_ORM_VERSION}.zip") -endif() +endif () set(SQLITE_ORM_MD5 "ba9a405a8a1421c093aa8ce988ff8598") -if(DEFINED ENV{MILVUS_YAMLCPP_URL}) +if (DEFINED ENV{MILVUS_YAMLCPP_URL}) set(YAMLCPP_SOURCE_URL "$ENV{MILVUS_YAMLCPP_URL}") -else() +else () set(YAMLCPP_SOURCE_URL "https://github.com/jbeder/yaml-cpp/archive/yaml-cpp-${YAMLCPP_VERSION}.tar.gz") -endif() +endif () set(YAMLCPP_MD5 "5b943e9af0060d0811148b037449ef82") -if(DEFINED ENV{MILVUS_LIBUNWIND_URL}) +if (DEFINED ENV{MILVUS_LIBUNWIND_URL}) set(LIBUNWIND_SOURCE_URL "$ENV{MILVUS_LIBUNWIND_URL}") -else() +else () set(LIBUNWIND_SOURCE_URL "https://github.com/libunwind/libunwind/releases/download/v${LIBUNWIND_VERSION}/libunwind-${LIBUNWIND_VERSION}.tar.gz") -endif() +endif () set(LIBUNWIND_MD5 "a04f69d66d8e16f8bf3ab72a69112cd6") -if(DEFINED ENV{MILVUS_GPERFTOOLS_URL}) +if (DEFINED ENV{MILVUS_GPERFTOOLS_URL}) set(GPERFTOOLS_SOURCE_URL "$ENV{MILVUS_GPERFTOOLS_URL}") -else() +else () set(GPERFTOOLS_SOURCE_URL "https://github.com/gperftools/gperftools/releases/download/gperftools-${GPERFTOOLS_VERSION}/gperftools-${GPERFTOOLS_VERSION}.tar.gz") -endif() +endif () set(GPERFTOOLS_MD5 "c6a852a817e9160c79bdb2d3101b4601") -if(DEFINED ENV{MILVUS_GRPC_URL}) +if (DEFINED ENV{MILVUS_GRPC_URL}) set(GRPC_SOURCE_URL "$ENV{MILVUS_GRPC_URL}") -else() +else () set(GRPC_SOURCE_URL "https://github.com/youny626/grpc-milvus/archive/${GRPC_VERSION}.zip") -endif() +endif () set(GRPC_MD5 "0362ba219f59432c530070b5f5c3df73") -if(DEFINED ENV{MILVUS_ZLIB_URL}) +if (DEFINED ENV{MILVUS_ZLIB_URL}) set(ZLIB_SOURCE_URL "$ENV{MILVUS_ZLIB_URL}") -else() +else () set(ZLIB_SOURCE_URL "https://github.com/madler/zlib/archive/${ZLIB_VERSION}.tar.gz") -endif() +endif () set(ZLIB_MD5 "0095d2d2d1f3442ce1318336637b695f") # ---------------------------------------------------------------------- @@ -334,13 +346,13 @@ macro(build_gtest) set(GTEST_VENDORED TRUE) set(GTEST_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS}") - if(APPLE) + if (APPLE) set(GTEST_CMAKE_CXX_FLAGS ${GTEST_CMAKE_CXX_FLAGS} -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-value -Wno-ignored-attributes) - endif() + endif () set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep") set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include") @@ -359,9 +371,9 @@ macro(build_gtest) set(GMOCK_INCLUDE_DIR "${GTEST_PREFIX}/include") set(GMOCK_STATIC_LIB "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) + ) - if(USE_JFROG_CACHE STREQUAL "ON") + if (USE_JFROG_CACHE STREQUAL "ON") set(GTEST_CACHE_PACKAGE_NAME "googletest_${GTEST_MD5}.tar.gz") set(GTEST_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${GTEST_CACHE_PACKAGE_NAME}") set(GTEST_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${GTEST_CACHE_PACKAGE_NAME}") @@ -385,10 +397,10 @@ macro(build_gtest) ${EP_LOG_OPTIONS}) ExternalProject_Create_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GTEST_CACHE_URL}) - else() + else () ExternalProject_Use_Cache(googletest_ep ${GTEST_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - else() + endif () + else () ExternalProject_Add(googletest_ep URL ${GTEST_SOURCE_URL} @@ -402,20 +414,20 @@ macro(build_gtest) CMAKE_ARGS ${GTEST_CMAKE_ARGS} ${EP_LOG_OPTIONS}) - endif() + endif () # The include directory must exist before it is referenced by a target. file(MAKE_DIRECTORY "${GTEST_INCLUDE_DIR}") add_library(gtest STATIC IMPORTED) set_target_properties(gtest - PROPERTIES IMPORTED_LOCATION "${GTEST_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}") + PROPERTIES IMPORTED_LOCATION "${GTEST_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}") add_library(gtest_main STATIC IMPORTED) set_target_properties(gtest_main - PROPERTIES IMPORTED_LOCATION "${GTEST_MAIN_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}") + PROPERTIES IMPORTED_LOCATION "${GTEST_MAIN_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}") add_library(gmock STATIC IMPORTED) set_target_properties(gmock @@ -431,13 +443,13 @@ endmacro() if (MILVUS_BUILD_TESTS) resolve_dependency(GTest) - if(NOT GTEST_VENDORED) - endif() + if (NOT GTEST_VENDORED) + endif () get_target_property(GTEST_INCLUDE_DIR gtest INTERFACE_INCLUDE_DIRECTORIES) link_directories(SYSTEM "${GTEST_PREFIX}/lib") include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) -endif() +endif () # ---------------------------------------------------------------------- # MySQL++ @@ -456,7 +468,7 @@ macro(build_mysqlpp) "CXXFLAGS=${EP_CXX_FLAGS}" "LDFLAGS=-pthread") - if(USE_JFROG_CACHE STREQUAL "ON") + if (USE_JFROG_CACHE STREQUAL "ON") set(MYSQLPP_CACHE_PACKAGE_NAME "mysqlpp_${MYSQLPP_MD5}.tar.gz") set(MYSQLPP_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${MYSQLPP_CACHE_PACKAGE_NAME}") set(MYSQLPP_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${MYSQLPP_CACHE_PACKAGE_NAME}") @@ -479,15 +491,15 @@ macro(build_mysqlpp) ${MYSQLPP_SHARED_LIB}) ExternalProject_Create_Cache(mysqlpp_ep ${MYSQLPP_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/mysqlpp_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${MYSQLPP_CACHE_URL}) - else() + else () file(DOWNLOAD ${MYSQLPP_CACHE_URL} ${MYSQLPP_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${MYSQLPP_CACHE_URL} TO ${MYSQLPP_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) ExternalProject_Use_Cache(mysqlpp_ep ${MYSQLPP_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - endif() - else() + endif () + endif () + else () externalproject_add(mysqlpp_ep URL ${MYSQLPP_SOURCE_URL} @@ -501,7 +513,7 @@ macro(build_mysqlpp) 1 BUILD_BYPRODUCTS ${MYSQLPP_SHARED_LIB}) - endif() + endif () file(MAKE_DIRECTORY "${MYSQLPP_INCLUDE_DIR}") add_library(mysqlpp SHARED IMPORTED) @@ -515,13 +527,13 @@ macro(build_mysqlpp) endmacro() -if(MILVUS_WITH_MYSQLPP) +if (MILVUS_WITH_MYSQLPP) resolve_dependency(MySQLPP) get_target_property(MYSQLPP_INCLUDE_DIR mysqlpp INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM "${MYSQLPP_INCLUDE_DIR}") link_directories(SYSTEM ${MYSQLPP_PREFIX}/lib) -endif() +endif () # ---------------------------------------------------------------------- # Prometheus @@ -547,9 +559,9 @@ macro(build_prometheus) "-DCMAKE_INSTALL_PREFIX=${PROMETHEUS_PREFIX}" -DCMAKE_BUILD_TYPE=Release) - if(USE_JFROG_CACHE STREQUAL "ON") + if (USE_JFROG_CACHE STREQUAL "ON") execute_process(COMMAND sh -c "git ls-remote --heads --tags ${PROMETHEUS_SOURCE_URL} ${PROMETHEUS_VERSION} | cut -f 1" OUTPUT_VARIABLE PROMETHEUS_LAST_COMMIT_ID) - if(${PROMETHEUS_LAST_COMMIT_ID} MATCHES "^[^#][a-z0-9]+") + if (${PROMETHEUS_LAST_COMMIT_ID} MATCHES "^[^#][a-z0-9]+") string(MD5 PROMETHEUS_COMBINE_MD5 "${PROMETHEUS_LAST_COMMIT_ID}") set(PROMETHEUS_CACHE_PACKAGE_NAME "prometheus_${PROMETHEUS_COMBINE_MD5}.tar.gz") set(PROMETHEUS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${PROMETHEUS_CACHE_PACKAGE_NAME}") @@ -583,18 +595,18 @@ macro(build_prometheus) "${PROMETHEUS_PULL_STATIC_LIB}") ExternalProject_Create_Cache(prometheus_ep ${PROMETHEUS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/prometheus_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${PROMETHEUS_CACHE_URL}) - else() + else () file(DOWNLOAD ${PROMETHEUS_CACHE_URL} ${PROMETHEUS_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${PROMETHEUS_CACHE_URL} TO ${PROMETHEUS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) ExternalProject_Use_Cache(prometheus_ep ${PROMETHEUS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - endif() - else() + endif () + endif () + else () message(FATAL_ERROR "The last commit ID of \"${PROMETHEUS_SOURCE_URL}\" repository don't match!") - endif() - else() + endif () + else () externalproject_add(prometheus_ep GIT_REPOSITORY ${PROMETHEUS_SOURCE_URL} @@ -618,7 +630,7 @@ macro(build_prometheus) "${PROMETHEUS_CORE_STATIC_LIB}" "${PROMETHEUS_PUSH_STATIC_LIB}" "${PROMETHEUS_PULL_STATIC_LIB}") - endif() + endif () file(MAKE_DIRECTORY "${PROMETHEUS_PREFIX}/push/include") add_library(prometheus-cpp-push STATIC IMPORTED) @@ -642,7 +654,7 @@ macro(build_prometheus) add_dependencies(prometheus-cpp-core prometheus_ep) endmacro() -if(MILVUS_WITH_PROMETHEUS) +if (MILVUS_WITH_PROMETHEUS) resolve_dependency(Prometheus) @@ -655,7 +667,7 @@ if(MILVUS_WITH_PROMETHEUS) link_directories(SYSTEM ${PROMETHEUS_PREFIX}/core/) include_directories(SYSTEM ${PROMETHEUS_PREFIX}/core/include) -endif() +endif () # ---------------------------------------------------------------------- # SQLite @@ -674,7 +686,7 @@ macro(build_sqlite) "CFLAGS=${EP_C_FLAGS}" "CXXFLAGS=${EP_CXX_FLAGS}") - if(USE_JFROG_CACHE STREQUAL "ON") + if (USE_JFROG_CACHE STREQUAL "ON") set(SQLITE_CACHE_PACKAGE_NAME "sqlite_${SQLITE_MD5}.tar.gz") set(SQLITE_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${SQLITE_CACHE_PACKAGE_NAME}") set(SQLITE_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${SQLITE_CACHE_PACKAGE_NAME}") @@ -698,15 +710,15 @@ macro(build_sqlite) "${SQLITE_STATIC_LIB}") ExternalProject_Create_Cache(sqlite_ep ${SQLITE_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/sqlite_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${SQLITE_CACHE_URL}) - else() + else () file(DOWNLOAD ${SQLITE_CACHE_URL} ${SQLITE_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${SQLITE_CACHE_URL} TO ${SQLITE_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) ExternalProject_Use_Cache(sqlite_ep ${SQLITE_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - endif() - else() + endif () + endif () + else () externalproject_add(sqlite_ep URL ${SQLITE_SOURCE_URL} @@ -721,7 +733,7 @@ macro(build_sqlite) 1 BUILD_BYPRODUCTS "${SQLITE_STATIC_LIB}") - endif() + endif () file(MAKE_DIRECTORY "${SQLITE_INCLUDE_DIR}") add_library(sqlite STATIC IMPORTED) @@ -733,11 +745,11 @@ macro(build_sqlite) add_dependencies(sqlite sqlite_ep) endmacro() -if(MILVUS_WITH_SQLITE) +if (MILVUS_WITH_SQLITE) resolve_dependency(SQLite) include_directories(SYSTEM "${SQLITE_INCLUDE_DIR}") link_directories(SYSTEM ${SQLITE_PREFIX}/lib/) -endif() +endif () # ---------------------------------------------------------------------- # SQLite_ORM @@ -753,16 +765,16 @@ macro(build_sqlite_orm) file(DOWNLOAD ${SQLITE_ORM_SOURCE_URL} ${SQLITE_ORM_TAR_NAME}) execute_process(COMMAND ${CMAKE_COMMAND} -E tar -xf ${SQLITE_ORM_TAR_NAME} - WORKING_DIRECTORY ${SQLITE_ORM_PREFIX}) + WORKING_DIRECTORY ${SQLITE_ORM_PREFIX}) endif () endmacro() -if(MILVUS_WITH_SQLITE_ORM) +if (MILVUS_WITH_SQLITE_ORM) resolve_dependency(SQLite_ORM) include_directories(SYSTEM "${SQLITE_ORM_INCLUDE_DIR}") -endif() +endif () # ---------------------------------------------------------------------- # yaml-cpp @@ -779,7 +791,7 @@ macro(build_yamlcpp) -DYAML_CPP_BUILD_TESTS=OFF -DYAML_CPP_BUILD_TOOLS=OFF) - if(USE_JFROG_CACHE STREQUAL "ON") + if (USE_JFROG_CACHE STREQUAL "ON") set(YAMLCPP_CACHE_PACKAGE_NAME "yaml-cpp_${YAMLCPP_MD5}.tar.gz") set(YAMLCPP_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${YAMLCPP_CACHE_PACKAGE_NAME}") set(YAMLCPP_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${YAMLCPP_CACHE_PACKAGE_NAME}") @@ -800,15 +812,15 @@ macro(build_yamlcpp) ${YAMLCPP_CMAKE_ARGS}) ExternalProject_Create_Cache(yaml-cpp_ep ${YAMLCPP_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/yaml-cpp_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${YAMLCPP_CACHE_URL}) - else() + else () file(DOWNLOAD ${YAMLCPP_CACHE_URL} ${YAMLCPP_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${YAMLCPP_CACHE_URL} TO ${YAMLCPP_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) ExternalProject_Use_Cache(yaml-cpp_ep ${YAMLCPP_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - endif() - else() + endif () + endif () + else () externalproject_add(yaml-cpp_ep URL ${YAMLCPP_SOURCE_URL} @@ -820,7 +832,7 @@ macro(build_yamlcpp) "${YAMLCPP_STATIC_LIB}" CMAKE_ARGS ${YAMLCPP_CMAKE_ARGS}) - endif() + endif () file(MAKE_DIRECTORY "${YAMLCPP_INCLUDE_DIR}") add_library(yaml-cpp STATIC IMPORTED) @@ -831,13 +843,13 @@ macro(build_yamlcpp) add_dependencies(yaml-cpp yaml-cpp_ep) endmacro() -if(MILVUS_WITH_YAMLCPP) +if (MILVUS_WITH_YAMLCPP) resolve_dependency(yaml-cpp) get_target_property(YAMLCPP_INCLUDE_DIR yaml-cpp INTERFACE_INCLUDE_DIRECTORIES) link_directories(SYSTEM ${YAMLCPP_PREFIX}/lib/) include_directories(SYSTEM ${YAMLCPP_INCLUDE_DIR}) -endif() +endif () # ---------------------------------------------------------------------- # libunwind @@ -849,7 +861,7 @@ macro(build_libunwind) set(LIBUNWIND_SHARED_LIB "${LIBUNWIND_PREFIX}/lib/libunwind${CMAKE_SHARED_LIBRARY_SUFFIX}") set(LIBUNWIND_CONFIGURE_ARGS "--prefix=${LIBUNWIND_PREFIX}") - if(USE_JFROG_CACHE STREQUAL "ON") + if (USE_JFROG_CACHE STREQUAL "ON") set(LIBUNWIND_CACHE_PACKAGE_NAME "libunwind_${LIBUNWIND_MD5}.tar.gz") set(LIBUNWIND_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${LIBUNWIND_CACHE_PACKAGE_NAME}") set(LIBUNWIND_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${LIBUNWIND_CACHE_PACKAGE_NAME}") @@ -874,15 +886,15 @@ macro(build_libunwind) ${LIBUNWIND_SHARED_LIB}) ExternalProject_Create_Cache(libunwind_ep ${LIBUNWIND_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/libunwind_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${LIBUNWIND_CACHE_URL}) - else() + else () file(DOWNLOAD ${LIBUNWIND_CACHE_URL} ${LIBUNWIND_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${LIBUNWIND_CACHE_URL} TO ${LIBUNWIND_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) ExternalProject_Use_Cache(libunwind_ep ${LIBUNWIND_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - endif() - else() + endif () + endif () + else () externalproject_add(libunwind_ep URL ${LIBUNWIND_SOURCE_URL} @@ -898,7 +910,7 @@ macro(build_libunwind) ${MAKE} install BUILD_BYPRODUCTS ${LIBUNWIND_SHARED_LIB}) - endif() + endif () file(MAKE_DIRECTORY "${LIBUNWIND_INCLUDE_DIR}") @@ -910,12 +922,12 @@ macro(build_libunwind) add_dependencies(libunwind libunwind_ep) endmacro() -if(MILVUS_WITH_LIBUNWIND) +if (MILVUS_WITH_LIBUNWIND) resolve_dependency(libunwind) get_target_property(LIBUNWIND_INCLUDE_DIR libunwind INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM ${LIBUNWIND_INCLUDE_DIR}) -endif() +endif () # ---------------------------------------------------------------------- # gperftools @@ -927,7 +939,7 @@ macro(build_gperftools) set(GPERFTOOLS_STATIC_LIB "${GPERFTOOLS_PREFIX}/lib/libprofiler${CMAKE_STATIC_LIBRARY_SUFFIX}") set(GPERFTOOLS_CONFIGURE_ARGS "--prefix=${GPERFTOOLS_PREFIX}") - if(USE_JFROG_CACHE STREQUAL "ON") + if (USE_JFROG_CACHE STREQUAL "ON") set(GPERFTOOLS_CACHE_PACKAGE_NAME "gperftools_${GPERFTOOLS_MD5}.tar.gz") set(GPERFTOOLS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${GPERFTOOLS_CACHE_PACKAGE_NAME}") set(GPERFTOOLS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${GPERFTOOLS_CACHE_PACKAGE_NAME}") @@ -952,15 +964,15 @@ macro(build_gperftools) ${GPERFTOOLS_STATIC_LIB}) ExternalProject_Create_Cache(gperftools_ep ${GPERFTOOLS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/gperftools_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GPERFTOOLS_CACHE_URL}) - else() + else () file(DOWNLOAD ${GPERFTOOLS_CACHE_URL} ${GPERFTOOLS_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${GPERFTOOLS_CACHE_URL} TO ${GPERFTOOLS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) ExternalProject_Use_Cache(gperftools_ep ${GPERFTOOLS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - endif() - else() + endif () + endif () + else () externalproject_add(gperftools_ep URL ${GPERFTOOLS_SOURCE_URL} @@ -976,7 +988,7 @@ macro(build_gperftools) ${MAKE} install BUILD_BYPRODUCTS ${GPERFTOOLS_STATIC_LIB}) - endif() + endif () ExternalProject_Add_StepDependencies(gperftools_ep build libunwind_ep) @@ -992,13 +1004,13 @@ macro(build_gperftools) add_dependencies(gperftools libunwind_ep) endmacro() -if(MILVUS_WITH_GPERFTOOLS) +if (MILVUS_WITH_GPERFTOOLS) resolve_dependency(gperftools) get_target_property(GPERFTOOLS_INCLUDE_DIR gperftools INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM ${GPERFTOOLS_INCLUDE_DIR}) link_directories(SYSTEM ${GPERFTOOLS_PREFIX}/lib) -endif() +endif () # ---------------------------------------------------------------------- # GRPC @@ -1014,7 +1026,7 @@ macro(build_grpc) set(GRPC_PROTOBUF_STATIC_LIB "${GRPC_PROTOBUF_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}protobuf${CMAKE_STATIC_LIBRARY_SUFFIX}") set(GRPC_PROTOC_STATIC_LIB "${GRPC_PROTOBUF_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}protoc${CMAKE_STATIC_LIBRARY_SUFFIX}") - if(USE_JFROG_CACHE STREQUAL "ON") + if (USE_JFROG_CACHE STREQUAL "ON") set(GRPC_CACHE_PACKAGE_NAME "grpc_${GRPC_MD5}.tar.gz") set(GRPC_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${GRPC_CACHE_PACKAGE_NAME}") set(GRPC_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${GRPC_CACHE_PACKAGE_NAME}") @@ -1044,15 +1056,15 @@ macro(build_grpc) ExternalProject_Add_StepDependencies(grpc_ep build zlib_ep) ExternalProject_Create_Cache(grpc_ep ${GRPC_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GRPC_CACHE_URL}) - else() + else () file(DOWNLOAD ${GRPC_CACHE_URL} ${GRPC_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${GRPC_CACHE_URL} TO ${GRPC_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) ExternalProject_Use_Cache(grpc_ep ${GRPC_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - endif() - else() + endif () + endif () + else () externalproject_add(grpc_ep URL ${GRPC_SOURCE_URL} @@ -1074,7 +1086,7 @@ macro(build_grpc) ExternalProject_Add_StepDependencies(grpc_ep build zlib_ep) - endif() + endif () file(MAKE_DIRECTORY "${GRPC_INCLUDE_DIR}") @@ -1082,29 +1094,29 @@ macro(build_grpc) set_target_properties(grpc PROPERTIES IMPORTED_LOCATION "${GRPC_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${GRPC_INCLUDE_DIR}" - INTERFACE_LINK_LIBRARIES "zlib" ) + INTERFACE_LINK_LIBRARIES "zlib") add_library(grpc++ STATIC IMPORTED) set_target_properties(grpc++ PROPERTIES IMPORTED_LOCATION "${GRPC++_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${GRPC_INCLUDE_DIR}" - INTERFACE_LINK_LIBRARIES "zlib" ) + INTERFACE_LINK_LIBRARIES "zlib") add_library(grpcpp_channelz STATIC IMPORTED) set_target_properties(grpcpp_channelz PROPERTIES IMPORTED_LOCATION "${GRPCPP_CHANNELZ_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${GRPC_INCLUDE_DIR}" - INTERFACE_LINK_LIBRARIES "zlib" ) + INTERFACE_LINK_LIBRARIES "zlib") add_library(grpc_protobuf STATIC IMPORTED) set_target_properties(grpc_protobuf PROPERTIES IMPORTED_LOCATION "${GRPC_PROTOBUF_STATIC_LIB}" - INTERFACE_LINK_LIBRARIES "zlib" ) + INTERFACE_LINK_LIBRARIES "zlib") add_library(grpc_protoc STATIC IMPORTED) set_target_properties(grpc_protoc PROPERTIES IMPORTED_LOCATION "${GRPC_PROTOC_STATIC_LIB}" - INTERFACE_LINK_LIBRARIES "zlib" ) + INTERFACE_LINK_LIBRARIES "zlib") add_dependencies(grpc grpc_ep) add_dependencies(grpc++ grpc_ep) @@ -1113,7 +1125,7 @@ macro(build_grpc) add_dependencies(grpc_protoc grpc_ep) endmacro() -if(MILVUS_WITH_GRPC) +if (MILVUS_WITH_GRPC) resolve_dependency(GRPC) get_target_property(GRPC_INCLUDE_DIR grpc INTERFACE_INCLUDE_DIRECTORIES) @@ -1123,7 +1135,7 @@ if(MILVUS_WITH_GRPC) set(GRPC_THIRD_PARTY_DIR ${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix/src/grpc_ep/third_party) include_directories(SYSTEM ${GRPC_THIRD_PARTY_DIR}/protobuf/src) link_directories(SYSTEM ${GRPC_PROTOBUF_LIB_DIR}) -endif() +endif () # ---------------------------------------------------------------------- # zlib @@ -1137,7 +1149,7 @@ macro(build_zlib) set(ZLIB_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX}" -DBUILD_SHARED_LIBS=OFF) - if(USE_JFROG_CACHE STREQUAL "ON") + if (USE_JFROG_CACHE STREQUAL "ON") set(ZLIB_CACHE_PACKAGE_NAME "zlib_${ZLIB_MD5}.tar.gz") set(ZLIB_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${ZLIB_CACHE_PACKAGE_NAME}") set(ZLIB_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${ZLIB_CACHE_PACKAGE_NAME}") @@ -1158,15 +1170,15 @@ macro(build_zlib) ${ZLIB_CMAKE_ARGS}) ExternalProject_Create_Cache(zlib_ep ${ZLIB_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/zlib_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ZLIB_CACHE_URL}) - else() + else () file(DOWNLOAD ${ZLIB_CACHE_URL} ${ZLIB_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${ZLIB_CACHE_URL} TO ${ZLIB_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") if (status_code EQUAL 0) ExternalProject_Use_Cache(zlib_ep ${ZLIB_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif() - endif() - else() + endif () + endif () + else () externalproject_add(zlib_ep URL ${ZLIB_SOURCE_URL} @@ -1178,7 +1190,7 @@ macro(build_zlib) "${ZLIB_STATIC_LIB}" CMAKE_ARGS ${ZLIB_CMAKE_ARGS}) - endif() + endif () file(MAKE_DIRECTORY "${ZLIB_INCLUDE_DIR}") add_library(zlib STATIC IMPORTED) @@ -1189,9 +1201,9 @@ macro(build_zlib) add_dependencies(zlib zlib_ep) endmacro() -if(MILVUS_WITH_ZLIB) +if (MILVUS_WITH_ZLIB) resolve_dependency(ZLIB) get_target_property(ZLIB_INCLUDE_DIR zlib INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM ${ZLIB_INCLUDE_DIR}) -endif() +endif () From c99f407099748295356aed5d01a412f1801dabc3 Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Wed, 30 Oct 2019 15:30:28 +0800 Subject: [PATCH 04/18] delete message Former-commit-id: 577b8693eca42bf3537c8691f754dd1876d579a3 --- core/cmake/ThirdPartyPackages.cmake | 1 - 1 file changed, 1 deletion(-) diff --git a/core/cmake/ThirdPartyPackages.cmake b/core/cmake/ThirdPartyPackages.cmake index 788c09ef2e..62d7713024 100644 --- a/core/cmake/ThirdPartyPackages.cmake +++ b/core/cmake/ThirdPartyPackages.cmake @@ -139,7 +139,6 @@ if (USE_JFROG_CACHE STREQUAL "ON") endif () if (UBUNTU_FOUND) set(JFROG_ARTFACTORY_CACHE_URL "${JFROG_ARTFACTORY_URL}/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${UBUNTU_VERSION}/${MILVUS_BUILD_ARCH}/${BUILD_TYPE}") - message(${JFROG_ARTFACTORY_CACHE_URL}) else () set(JFROG_ARTFACTORY_CACHE_URL "${JFROG_ARTFACTORY_URL}/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${MILVUS_BUILD_ARCH}/${BUILD_TYPE}") endif () From 3dd61fee0608b058015657ba7a5ae45eec418e21 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 30 Oct 2019 17:17:38 +0800 Subject: [PATCH 05/18] #89 re-org faiss_benchmark_test Former-commit-id: 0fb0c3f91e9c864b301f5a9b8d29049e456d99e4 --- .../faiss_benchmark/faiss_benchmark_test.cpp | 579 ++++++++---------- 1 file changed, 243 insertions(+), 336 deletions(-) diff --git a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp index 618ee00b28..14e22aa55f 100644 --- a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp +++ b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp @@ -28,7 +28,9 @@ #include #include #include +#ifdef CUSTOMIZATION #include +#endif #include #include #include @@ -46,6 +48,19 @@ * https://support.hdfgroup.org/ftp/HDF5/releases/ * and install it to /usr/local/hdf5 . *****************************************************/ +#define DEBUG_VERBOSE 0 + +const std::string HDF5_POSTFIX = ".hdf5"; +const std::string HDF5_DATASET_TRAIN = "train"; +const std::string HDF5_DATASET_TEST = "test"; +const std::string HDF5_DATASET_NEIGHBORS = "neighbors"; +const std::string HDF5_DATASET_DISTANCES = "distances"; + +enum QueryMode { + MODE_CPU = 0, + MODE_MIX, + MODE_GPU +}; double elapsed() { @@ -57,67 +72,45 @@ elapsed() { void normalize(float* arr, size_t nq, size_t dim) { for (size_t i = 0; i < nq; i++) { - double vecLen = 0.0; + double vecLen = 0.0, inv_vecLen = 0.0; for (size_t j = 0; j < dim; j++) { double val = arr[i * dim + j]; vecLen += val * val; } - vecLen = std::sqrt(vecLen); + inv_vecLen = 1.0 / std::sqrt(vecLen); for (size_t j = 0; j < dim; j++) { - arr[i * dim + j] = (float)(arr[i * dim + j] / vecLen); + arr[i * dim + j] = (float)(arr[i * dim + j] * inv_vecLen); } } } void* -hdf5_read(const char* file_name, const char* dataset_name, H5T_class_t dataset_class, size_t& d_out, size_t& n_out) { +hdf5_read(const std::string& file_name, const std::string& dataset_name, H5T_class_t dataset_class, + size_t& d_out, size_t& n_out) { hid_t file, dataset, datatype, dataspace, memspace; H5T_class_t t_class; /* data type class */ - H5T_order_t order; /* data order */ - size_t size; /* size of the data element stored in file */ hsize_t dimsm[3]; /* memory space dimensions */ hsize_t dims_out[2]; /* dataset dimensions */ hsize_t count[2]; /* size of the hyperslab in the file */ hsize_t offset[2]; /* hyperslab offset in the file */ hsize_t count_out[3]; /* size of the hyperslab in memory */ hsize_t offset_out[3]; /* hyperslab offset in memory */ - int rank; void* data_out; /* output buffer */ /* Open the file and the dataset. */ - file = H5Fopen(file_name, H5F_ACC_RDONLY, H5P_DEFAULT); - dataset = H5Dopen2(file, dataset_name, H5P_DEFAULT); + file = H5Fopen(file_name.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT); + dataset = H5Dopen2(file, dataset_name.c_str(), H5P_DEFAULT); - /* - * Get datatype and dataspace handles and then query - * dataset class, order, size, rank and dimensions. - */ + /* Get datatype and dataspace handles and then query + * dataset class, order, size, rank and dimensions. */ datatype = H5Dget_type(dataset); /* datatype handle */ t_class = H5Tget_class(datatype); assert(t_class == dataset_class || !"Illegal dataset class type"); - order = H5Tget_order(datatype); - switch (order) { - case H5T_ORDER_LE: - printf("Little endian order \n"); - break; - case H5T_ORDER_BE: - printf("Big endian order \n"); - break; - default: - printf("Illegal endian order \n"); - break; - } - - size = H5Tget_size(datatype); - printf("Data size is %d \n", (int)size); - dataspace = H5Dget_space(dataset); /* dataspace handle */ - rank = H5Sget_simple_extent_ndims(dataspace); H5Sget_simple_extent_dims(dataspace, dims_out, NULL); n_out = dims_out[0]; d_out = dims_out[1]; - printf("rank %d, dimensions %lu x %lu \n", rank, n_out, d_out); /* Define hyperslab in the dataset. */ offset[0] = offset[1] = 0; @@ -209,8 +202,7 @@ GetResultHitCount(const faiss::Index::idx_t* ground_index, const faiss::Index::i // each result replicates INDEX_ADD_LOOPS times for (int j_c = 0; j_c < ground_k; j_c++) { int r_c = index[i * k + j_c]; - int j_g = 0; - for (; j_g < ground_k / index_add_loops; j_g++) { + for (int j_g = 0; j_g < ground_k / index_add_loops; j_g++) { if (ground_index[i * ground_k + j_g] == r_c) { hit++; continue; @@ -221,106 +213,211 @@ GetResultHitCount(const faiss::Index::idx_t* ground_index, const faiss::Index::i return hit; } +#if DEBUG_VERBOSE void -test_ann_hdf5(const std::string& ann_test_name, const std::string& index_key, int32_t index_add_loops, - const std::vector& nprobes, int32_t search_loops) { +print_array(const char* header, bool is_integer, const void* arr, size_t nq, size_t k) { + const int ROW = 10; + const int COL = 10; + assert(ROW <= nq); + assert(COL <= k); + printf("%s\n", header); + printf("==============================================\n"); + for (int i = 0; i < 10; i++) { + for (int j = 0; j < 10; j++) { + if (is_integer) { + printf("%7ld ", ((int64_t*)arr)[i * k + j]); + } else { + printf("%.6f ", ((float*)arr)[i * k + j]); + } + } + printf("\n"); + } + printf("\n"); +} +#endif + +void +load_base_data(faiss::Index* &index, const std::string& ann_test_name, const std::string& index_key, + faiss::gpu::StandardGpuResources& res, const faiss::MetricType metric_type, const size_t dim, + int32_t index_add_loops, QueryMode mode = MODE_CPU) { double t0 = elapsed(); - const std::string ann_file_name = ann_test_name + ".hdf5"; + const std::string ann_file_name = ann_test_name + HDF5_POSTFIX; + const int GPU_DEVICE_IDX = 0; - faiss::MetricType metric_type; - size_t dim; - - if (!parse_ann_test_name(ann_test_name, dim, metric_type)) { - printf("Invalid ann test name: %s\n", ann_test_name.c_str()); - return; - } - - faiss::Index* index; - size_t d; + faiss::Index *cpu_index = nullptr, *gpu_index = nullptr; + faiss::distance_compute_blas_threshold = 800; std::string index_file_name = get_index_file_name(ann_test_name, index_key, index_add_loops); + try { - index = faiss::read_index(index_file_name.c_str()); - d = dim; + printf("[%.3f s] Reading index file: %s\n", elapsed() - t0, index_file_name.c_str()); + cpu_index = faiss::read_index(index_file_name.c_str()); + + if (mode != MODE_CPU) { + faiss::gpu::GpuClonerOptions option; + option.allInGpu = true; + + faiss::IndexComposition index_composition; + index_composition.index = cpu_index; + index_composition.quantizer = nullptr; + + switch (mode) { + case MODE_CPU: + assert(false); + break; + case MODE_MIX: + index_composition.mode = 1; // 0: all data, 1: copy quantizer, 2: copy data + break; + case MODE_GPU: + index_composition.mode = 0; // 0: all data, 1: copy quantizer, 2: copy data + break; + } + + printf("[%.3f s] Cloning CPU index to GPU\n", elapsed() - t0); + gpu_index = faiss::gpu::index_cpu_to_gpu(&res, GPU_DEVICE_IDX, &index_composition, &option); + } } catch (...) { - printf("Cannot read index file: %s\n", index_file_name.c_str()); - - printf("[%.3f s] Loading train set\n", elapsed() - t0); - - size_t nb; - float* xb = (float*)hdf5_read(ann_file_name.c_str(), "train", H5T_FLOAT, d, nb); + size_t nb, d; + printf("[%.3f s] Loading HDF5 file: %s\n", elapsed() - t0, ann_file_name.c_str()); + float* xb = (float*)hdf5_read(ann_file_name, HDF5_DATASET_TRAIN, H5T_FLOAT, d, nb); assert(d == dim || !"dataset does not have correct dimension"); if (metric_type == faiss::METRIC_INNER_PRODUCT) { - printf("[%.3f s] Normalizing data set \n", elapsed() - t0); + printf("[%.3f s] Normalizing base data set \n", elapsed() - t0); normalize(xb, nb, d); } - printf("[%.3f s] Preparing index \"%s\" d=%ld\n", elapsed() - t0, index_key.c_str(), d); + printf("[%.3f s] Creating CPU index \"%s\" d=%ld\n", elapsed() - t0, index_key.c_str(), d); + cpu_index = faiss::index_factory(d, index_key.c_str(), metric_type); - index = faiss::index_factory(d, index_key.c_str(), metric_type); + printf("[%.3f s] Cloning CPU index to GPU\n", elapsed() - t0); + gpu_index = faiss::gpu::index_cpu_to_gpu(&res, GPU_DEVICE_IDX, cpu_index); printf("[%.3f s] Training on %ld vectors\n", elapsed() - t0, nb); - - index->train(nb, xb); - - printf("[%.3f s] Loading database\n", elapsed() - t0); + gpu_index->train(nb, xb); // add index multiple times to get ~1G data set for (int i = 0; i < index_add_loops; i++) { - printf("[%.3f s] Indexing database, size %ld*%ld\n", elapsed() - t0, nb, d); - index->add(nb, xb); + printf("[%.3f s] No.%d Indexing database, size %ld*%ld\n", elapsed() - t0, i, nb, d); + gpu_index->add(nb, xb); } - faiss::write_index(index, index_file_name.c_str()); + printf("[%.3f s] Coping GPU index to CPU\n", elapsed() - t0); + delete cpu_index; + cpu_index = faiss::gpu::index_gpu_to_cpu(gpu_index); + + faiss::IndexIVF *cpu_ivf_index = dynamic_cast(cpu_index); + if (cpu_ivf_index != nullptr) { + cpu_ivf_index->to_readonly(); + } + + printf("[%.3f s] Writing index file: %s\n", elapsed() - t0, index_file_name.c_str()); + faiss::write_index(cpu_index, index_file_name.c_str()); delete[] xb; } - size_t nq; - float* xq; - { - printf("[%.3f s] Loading queries\n", elapsed() - t0); - - size_t d2; - xq = (float*)hdf5_read(ann_file_name.c_str(), "test", H5T_FLOAT, d2, nq); - assert(d == d2 || !"query does not have same dimension as train set"); + switch (mode) { + case MODE_CPU: + case MODE_MIX: + index = cpu_index; + if (gpu_index) { + delete gpu_index; + } + break; + case MODE_GPU: + index = gpu_index; + if (cpu_index) { + delete cpu_index; + } + break; } +} - size_t k; // nb of results per query in the GT - faiss::Index::idx_t* gt; // nq * k matrix of ground-truth nearest-neighbors - { - printf("[%.3f s] Loading ground truth for %ld queries\n", elapsed() - t0, nq); +void +load_query_data(faiss::Index::distance_t* &xq, size_t& nq, const std::string& ann_test_name, + const faiss::MetricType metric_type, const size_t dim) { + double t0 = elapsed(); + size_t d; - // load ground-truth and convert int to long - size_t nq2; - int* gt_int = (int*)hdf5_read(ann_file_name.c_str(), "neighbors", H5T_INTEGER, k, nq2); - assert(nq2 == nq || !"incorrect nb of ground truth entries"); + const std::string ann_file_name = ann_test_name + HDF5_POSTFIX; - gt = new faiss::Index::idx_t[k * nq]; - for (int i = 0; i < k * nq; i++) { - gt[i] = gt_int[i]; - } - delete[] gt_int; + xq = (float *) hdf5_read(ann_file_name, HDF5_DATASET_TEST, H5T_FLOAT, d, nq); + assert(d == dim || !"query does not have same dimension as train set"); + + if (metric_type == faiss::METRIC_INNER_PRODUCT) { + printf("[%.3f s] Normalizing query data \n", elapsed() - t0); + normalize(xq, nq, d); } +} + +void +load_ground_truth(faiss::Index::idx_t* >, size_t& k, const std::string& ann_test_name, const size_t nq) { + const std::string ann_file_name = ann_test_name + HDF5_POSTFIX; + + // load ground-truth and convert int to long + size_t nq2; + int *gt_int = (int *) hdf5_read(ann_file_name, HDF5_DATASET_NEIGHBORS, H5T_INTEGER, k, nq2); + assert(nq2 == nq || !"incorrect nb of ground truth index"); + + gt = new faiss::Index::idx_t[k * nq]; + for (int i = 0; i < k * nq; i++) { + gt[i] = gt_int[i]; + } + delete[] gt_int; + +#if DEBUG_VERBOSE + faiss::Index::distance_t* gt_dist; // nq * k matrix of ground-truth nearest-neighbors distances + gt_dist = (float*)hdf5_read(ann_file_name, HDF5_DATASET_DISTANCES, H5T_FLOAT, k, nq2); + assert(nq2 == nq || !"incorrect nb of ground truth distance"); + + std::string str; + str = ann_test_name + " ground truth index"; + print_array(str.c_str(), true, gt, nq, k); + str = ann_test_name + " ground truth distance"; + print_array(str.c_str(), false, gt_dist, nq, k); + + delete gt_dist; +#endif +} + +void +test_with_nprobes(const std::string& ann_test_name, const std::string& index_key, faiss::Index* index, + faiss::gpu::StandardGpuResources& res, const QueryMode query_mode, + const faiss::Index::distance_t *xq, const faiss::Index::idx_t *gt, const std::vector nprobes, + const int32_t index_add_loops, const int32_t search_loops) { + const size_t NQ = 1000, NQ_START = 10, NQ_STEP = 10; + const size_t K = 1000, K_START = 100, K_STEP = 10; + const size_t GK = 100; // topk of ground truth + + std::unordered_map mode_str_map = + {{MODE_CPU, "MODE_CPU"}, {MODE_MIX, "MODE_MIX"}, {MODE_GPU, "MODE_GPU"}}; for (auto nprobe : nprobes) { - faiss::ParameterSpace params; - - std::string nprobe_str = "nprobe=" + std::to_string(nprobe); - params.set_index_parameters(index, nprobe_str.c_str()); + switch (query_mode) { + case MODE_CPU: + case MODE_MIX: { + faiss::ParameterSpace params; + std::string nprobe_str = "nprobe=" + std::to_string(nprobe); + params.set_index_parameters(index, nprobe_str.c_str()); + break; + } + case MODE_GPU: { + faiss::gpu::GpuIndexIVF *gpu_index_ivf = dynamic_cast(index); + gpu_index_ivf->setNumProbes(nprobe); + } + } // output buffers -#if 1 - const size_t NQ = 1000, K = 1000; - faiss::Index::idx_t* I = new faiss::Index::idx_t[NQ * K]; - float* D = new float[NQ * K]; + faiss::Index::idx_t *I = new faiss::Index::idx_t[NQ * K]; + faiss::Index::distance_t *D = new faiss::Index::distance_t[NQ * K]; - printf("\n%s | %s | nprobe=%lu\n", ann_test_name.c_str(), index_key.c_str(), nprobe); + printf("\n%s | %s - %s | nprobe=%lu\n", ann_test_name.c_str(), index_key.c_str(), + mode_str_map[query_mode].c_str(), nprobe); printf("======================================================================================\n"); - for (size_t t_nq = 10; t_nq <= NQ; t_nq *= 10) { // nq = {10, 100, 1000} - for (size_t t_k = 100; t_k <= K; t_k *= 10) { // k = {100, 1000} + for (size_t t_nq = NQ_START; t_nq <= NQ; t_nq *= NQ_STEP) { // nq = {10, 100, 1000} + for (size_t t_k = K_START; t_k <= K; t_k *= K_STEP) { // k = {100, 1000} faiss::indexIVF_stats.quantization_time = 0.0; faiss::indexIVF_stats.search_time = 0.0; @@ -330,271 +427,71 @@ test_ann_hdf5(const std::string& ann_test_name, const std::string& index_key, in } t_end = elapsed(); +#if DEBUG_VERBOSE + std::string str; + str = "I (" + index_key + ", nq=" + std::to_string(t_nq) + ", k=" + std::to_string(t_k) + ")"; + print_array(str.c_str(), true, I, t_nq, t_k); + str = "D (" + index_key + ", nq=" + std::to_string(t_nq) + ", k=" + std::to_string(t_k) + ")"; + print_array(str.c_str(), false, D, t_nq, t_k); +#endif + // k = 100 for ground truth - int32_t hit = GetResultHitCount(gt, I, k, t_k, t_nq, index_add_loops); + int32_t hit = GetResultHitCount(gt, I, GK, t_k, t_nq, index_add_loops); printf("nq = %4ld, k = %4ld, elapse = %.4fs (quant = %.4fs, search = %.4fs), R@ = %.4f\n", t_nq, t_k, (t_end - t_start) / search_loops, faiss::indexIVF_stats.quantization_time / 1000 / search_loops, faiss::indexIVF_stats.search_time / 1000 / search_loops, - (hit / float(t_nq * k / index_add_loops))); + (hit / float(t_nq * GK / index_add_loops))); } } printf("======================================================================================\n"); -#else - printf("[%.3f s] Perform a search on %ld queries\n", elapsed() - t0, nq); - - faiss::Index::idx_t* I = new faiss::Index::idx_t[nq * k]; - float* D = new float[nq * k]; - - index->search(nq, xq, k, D, I); - - printf("[%.3f s] Compute recalls\n", elapsed() - t0); - - // evaluate result by hand. - int n_1 = 0, n_10 = 0, n_100 = 0; - for (int i = 0; i < nq; i++) { - int gt_nn = gt[i * k]; - for (int j = 0; j < k; j++) { - if (I[i * k + j] == gt_nn) { - if (j < 1) - n_1++; - if (j < 10) - n_10++; - if (j < 100) - n_100++; - } - } - } - printf("R@1 = %.4f\n", n_1 / float(nq)); - printf("R@10 = %.4f\n", n_10 / float(nq)); - printf("R@100 = %.4f\n", n_100 / float(nq)); -#endif - - printf("[%.3f s] Search test done\n\n", elapsed() - t0); delete[] I; delete[] D; } - - delete[] xq; - delete[] gt; - delete index; } -#ifdef CUSTOMIZATION void -test_ivfsq8h(const std::string& ann_test_name, int32_t index_add_loops, const std::vector& nprobes, - bool pure_gpu_mode, int32_t search_loops) { +test_ann_hdf5(const std::string& ann_test_name, const std::string& index_key, const QueryMode query_mode, + int32_t index_add_loops, const std::vector& nprobes, int32_t search_loops) { double t0 = elapsed(); - const std::string ann_file_name = ann_test_name + ".hdf5"; + faiss::gpu::StandardGpuResources res; faiss::MetricType metric_type; size_t dim; + if (query_mode == MODE_MIX && index_key.find("SQ8Hybrid") == std::string::npos) { + printf("Only SQ8Hybrid support MODE_MIX\n"); + return; + } + if (!parse_ann_test_name(ann_test_name, dim, metric_type)) { printf("Invalid ann test name: %s\n", ann_test_name.c_str()); return; } - faiss::distance_compute_blas_threshold = 800; - faiss::gpu::StandardGpuResources res; - - const std::string index_key = "IVF16384,SQ8Hybrid"; - - faiss::Index* cpu_index = nullptr; - size_t d; - - std::string index_file_name = get_index_file_name(ann_test_name, index_key, index_add_loops); - try { - cpu_index = faiss::read_index(index_file_name.c_str()); - d = dim; - } catch (...) { - printf("Cannot read index file: %s\n", index_file_name.c_str()); - - printf("[%.3f s] Loading train set\n", elapsed() - t0); - - size_t nb; - float* xb = (float*)hdf5_read(ann_file_name.c_str(), "train", H5T_FLOAT, d, nb); - assert(d == dim || !"dataset does not have correct dimension"); - - printf("[%.3f s] Preparing index \"%s\" d=%ld\n", elapsed() - t0, index_key.c_str(), d); - - faiss::Index* ori_index = faiss::index_factory(d, index_key.c_str(), metric_type); - - auto device_index = faiss::gpu::index_cpu_to_gpu(&res, 0, ori_index); - - printf("[%.3f s] Training on %ld vectors\n", elapsed() - t0, nb); - - device_index->train(nb, xb); - - printf("[%.3f s] Loading database\n", elapsed() - t0); - - for (int i = 0; i < index_add_loops; i++) { - printf("[%.3f s] Indexing database, size %ld*%ld\n", elapsed() - t0, nb, d); - device_index->add(nb, xb); - } - - cpu_index = faiss::gpu::index_gpu_to_cpu(device_index); - faiss::write_index(cpu_index, index_file_name.c_str()); - - delete[] xb; - } - - faiss::IndexIVF* cpu_ivf_index = dynamic_cast(cpu_index); - if (cpu_ivf_index != nullptr) { - cpu_ivf_index->to_readonly(); - } - - size_t nq; - float* xq; - { - printf("[%.3f s] Loading queries\n", elapsed() - t0); - - size_t d2; - xq = (float*)hdf5_read(ann_file_name.c_str(), "test", H5T_FLOAT, d2, nq); - assert(d == d2 || !"query does not have same dimension as train set"); - } - - size_t k; - faiss::Index::idx_t* gt; - { - printf("[%.3f s] Loading ground truth for %ld queries\n", elapsed() - t0, nq); - - size_t nq2; - int* gt_int = (int*)hdf5_read(ann_file_name.c_str(), "neighbors", H5T_INTEGER, k, nq2); - assert(nq2 == nq || !"incorrect nb of ground truth entries"); - - gt = new faiss::Index::idx_t[k * nq]; - for (uint64_t i = 0; i < k * nq; ++i) { - gt[i] = gt_int[i]; - } - delete[] gt_int; - } - - faiss::gpu::GpuClonerOptions option; - option.allInGpu = true; - - faiss::IndexComposition index_composition; - index_composition.index = cpu_index; - index_composition.quantizer = nullptr; - + size_t nq, k; faiss::Index* index; - double copy_time; + faiss::Index::distance_t* xq; + faiss::Index::idx_t* gt; // ground-truth index - if (!pure_gpu_mode) { - index_composition.mode = 1; // 0: all data, 1: copy quantizer, 2: copy data - index = faiss::gpu::index_cpu_to_gpu(&res, 0, &index_composition, &option); - delete index; + printf("[%.3f s] Loading base data\n", elapsed() - t0); + load_base_data(index, ann_test_name, index_key, res, metric_type, dim, index_add_loops, query_mode); - copy_time = elapsed(); - index = faiss::gpu::index_cpu_to_gpu(&res, 0, &index_composition, &option); - delete index; - } else { - index_composition.mode = 2; - index = faiss::gpu::index_cpu_to_gpu(&res, 0, &index_composition, &option); - delete index; + printf("[%.3f s] Loading queries\n", elapsed() - t0); + load_query_data(xq, nq, ann_test_name, metric_type, dim); - copy_time = elapsed(); - index = faiss::gpu::index_cpu_to_gpu(&res, 0, &index_composition, &option); - } + printf("[%.3f s] Loading ground truth for %ld queries\n", elapsed() - t0, nq); + load_ground_truth(gt, k, ann_test_name, nq); - copy_time = elapsed() - copy_time; - printf("[%.3f s] Copy quantizer completed, cost %f s\n", elapsed() - t0, copy_time); - - const size_t NQ = 1000, K = 1000; - if (!pure_gpu_mode) { - for (auto nprobe : nprobes) { - auto ivf_index = dynamic_cast(cpu_index); - ivf_index->nprobe = nprobe; - - auto is_gpu_flat_index = dynamic_cast(ivf_index->quantizer); - if (is_gpu_flat_index == nullptr) { - delete ivf_index->quantizer; - ivf_index->quantizer = index_composition.quantizer; - } - - int64_t* I = new faiss::Index::idx_t[NQ * K]; - float* D = new float[NQ * K]; - - printf("\n%s | %s-MIX | nprobe=%lu\n", ann_test_name.c_str(), index_key.c_str(), nprobe); - printf("======================================================================================\n"); - for (size_t t_nq = 10; t_nq <= NQ; t_nq *= 10) { // nq = {10, 100, 1000} - for (size_t t_k = 100; t_k <= K; t_k *= 10) { // k = {100, 1000} - faiss::indexIVF_stats.quantization_time = 0.0; - faiss::indexIVF_stats.search_time = 0.0; - - double t_start = elapsed(), t_end; - for (int32_t i = 0; i < search_loops; i++) { - cpu_index->search(t_nq, xq, t_k, D, I); - } - t_end = elapsed(); - - // k = 100 for ground truth - int32_t hit = GetResultHitCount(gt, I, k, t_k, t_nq, index_add_loops); - - printf("nq = %4ld, k = %4ld, elapse = %.4fs (quant = %.4fs, search = %.4fs), R@ = %.4f\n", t_nq, - t_k, (t_end - t_start) / search_loops, - faiss::indexIVF_stats.quantization_time / 1000 / search_loops, - faiss::indexIVF_stats.search_time / 1000 / search_loops, - (hit / float(t_nq * k / index_add_loops))); - } - } - printf("======================================================================================\n"); - - printf("[%.3f s] Search test done\n\n", elapsed() - t0); - - delete[] I; - delete[] D; - } - } else { - std::shared_ptr gpu_index_ivf_ptr = std::shared_ptr(index); - - for (auto nprobe : nprobes) { - faiss::gpu::GpuIndexIVFSQHybrid* gpu_index_ivf_hybrid = - dynamic_cast(gpu_index_ivf_ptr.get()); - gpu_index_ivf_hybrid->setNumProbes(nprobe); - - int64_t* I = new faiss::Index::idx_t[NQ * K]; - float* D = new float[NQ * K]; - - printf("\n%s | %s-GPU | nprobe=%lu\n", ann_test_name.c_str(), index_key.c_str(), nprobe); - printf("======================================================================================\n"); - for (size_t t_nq = 10; t_nq <= NQ; t_nq *= 10) { // nq = {10, 100, 1000} - for (size_t t_k = 100; t_k <= K; t_k *= 10) { // k = {100, 1000} - faiss::indexIVF_stats.quantization_time = 0.0; - faiss::indexIVF_stats.search_time = 0.0; - - double t_start = elapsed(), t_end; - for (int32_t i = 0; i < search_loops; i++) { - gpu_index_ivf_ptr->search(nq, xq, k, D, I); - } - t_end = elapsed(); - - // k = 100 for ground truth - int32_t hit = GetResultHitCount(gt, I, k, t_k, t_nq, index_add_loops); - - printf("nq = %4ld, k = %4ld, elapse = %.4fs (quant = %.4fs, search = %.4fs), R@ = %.4f\n", t_nq, - t_k, (t_end - t_start) / search_loops, - faiss::indexIVF_stats.quantization_time / 1000 / search_loops, - faiss::indexIVF_stats.search_time / 1000 / search_loops, - (hit / float(t_nq * k / index_add_loops))); - } - } - printf("======================================================================================\n"); - - printf("[%.3f s] Search test done\n\n", elapsed() - t0); - - delete[] I; - delete[] D; - } - } + test_with_nprobes(ann_test_name, index_key, index, res, query_mode, xq, gt, nprobes, index_add_loops, search_loops); + printf("[%.3f s] Search test done\n\n", elapsed() - t0); delete[] xq; delete[] gt; - delete cpu_index; + delete index; } -#endif /************************************************************************************ * https://github.com/erikbern/ann-benchmarks @@ -616,19 +513,29 @@ TEST(FAISSTEST, BENCHMARK) { const int32_t SIFT_INSERT_LOOPS = 2; // insert twice to get ~1G data set const int32_t GLOVE_INSERT_LOOPS = 1; - test_ann_hdf5("sift-128-euclidean", "IVF4096,Flat", SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); - test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8", SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + test_ann_hdf5("sift-128-euclidean", "IVF16384,Flat", MODE_CPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); + test_ann_hdf5("sift-128-euclidean", "IVF16384,Flat", MODE_GPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); + + test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8", MODE_CPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); + test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8", MODE_GPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); + #ifdef CUSTOMIZATION - test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8Hybrid", SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); - test_ivfsq8h("sift-128-euclidean", SIFT_INSERT_LOOPS, param_nprobes, false, SEARCH_LOOPS); - test_ivfsq8h("sift-128-euclidean", SIFT_INSERT_LOOPS, param_nprobes, true, SEARCH_LOOPS); + test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8Hybrid", MODE_CPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); + test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8Hybrid", MODE_GPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); +// test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8Hybrid", MODE_MIX, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); #endif - test_ann_hdf5("glove-200-angular", "IVF4096,Flat", GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); - test_ann_hdf5("glove-200-angular", "IVF16384,SQ8", GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + test_ann_hdf5("glove-200-angular", "IVF16384,Flat", MODE_CPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); + test_ann_hdf5("glove-200-angular", "IVF16384,Flat", MODE_GPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); + + test_ann_hdf5("glove-200-angular", "IVF16384,SQ8", MODE_CPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); + test_ann_hdf5("glove-200-angular", "IVF16384,SQ8", MODE_GPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); + #ifdef CUSTOMIZATION - test_ann_hdf5("glove-200-angular", "IVF16384,SQ8Hybrid", GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); - test_ivfsq8h("glove-200-angular", GLOVE_INSERT_LOOPS, param_nprobes, false, SEARCH_LOOPS); - test_ivfsq8h("glove-200-angular", GLOVE_INSERT_LOOPS, param_nprobes, true, SEARCH_LOOPS); + test_ann_hdf5("glove-200-angular", "IVF16384,SQ8Hybrid", MODE_CPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); + test_ann_hdf5("glove-200-angular", "IVF16384,SQ8Hybrid", MODE_GPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); +// test_ann_hdf5("glove-200-angular", "IVF16384,SQ8Hybrid", MODE_MIX, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); #endif } From 12b8d14a10f31bb80100449779503a8de01a4b6f Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 30 Oct 2019 17:22:30 +0800 Subject: [PATCH 06/18] #89 clang-format Former-commit-id: ddfd7fe63f48b0822d536b1079a129b84ea537cc --- .../faiss_benchmark/faiss_benchmark_test.cpp | 66 +++++++++---------- 1 file changed, 32 insertions(+), 34 deletions(-) diff --git a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp index 14e22aa55f..3c5a396fef 100644 --- a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp +++ b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp @@ -50,17 +50,13 @@ *****************************************************/ #define DEBUG_VERBOSE 0 -const std::string HDF5_POSTFIX = ".hdf5"; -const std::string HDF5_DATASET_TRAIN = "train"; -const std::string HDF5_DATASET_TEST = "test"; -const std::string HDF5_DATASET_NEIGHBORS = "neighbors"; -const std::string HDF5_DATASET_DISTANCES = "distances"; +const char HDF5_POSTFIX[] = ".hdf5"; +const char HDF5_DATASET_TRAIN[] = "train"; +const char HDF5_DATASET_TEST[] = "test"; +const char HDF5_DATASET_NEIGHBORS[] = "neighbors"; +const char HDF5_DATASET_DISTANCES[] = "distances"; -enum QueryMode { - MODE_CPU = 0, - MODE_MIX, - MODE_GPU -}; +enum QueryMode { MODE_CPU = 0, MODE_MIX, MODE_GPU }; double elapsed() { @@ -85,8 +81,8 @@ normalize(float* arr, size_t nq, size_t dim) { } void* -hdf5_read(const std::string& file_name, const std::string& dataset_name, H5T_class_t dataset_class, - size_t& d_out, size_t& n_out) { +hdf5_read(const std::string& file_name, const std::string& dataset_name, H5T_class_t dataset_class, size_t& d_out, + size_t& n_out) { hid_t file, dataset, datatype, dataspace, memspace; H5T_class_t t_class; /* data type class */ hsize_t dimsm[3]; /* memory space dimensions */ @@ -95,7 +91,7 @@ hdf5_read(const std::string& file_name, const std::string& dataset_name, H5T_cla hsize_t offset[2]; /* hyperslab offset in the file */ hsize_t count_out[3]; /* size of the hyperslab in memory */ hsize_t offset_out[3]; /* hyperslab offset in memory */ - void* data_out; /* output buffer */ + void* data_out; /* output buffer */ /* Open the file and the dataset. */ file = H5Fopen(file_name.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT); @@ -237,7 +233,7 @@ print_array(const char* header, bool is_integer, const void* arr, size_t nq, siz #endif void -load_base_data(faiss::Index* &index, const std::string& ann_test_name, const std::string& index_key, +load_base_data(faiss::Index*& index, const std::string& ann_test_name, const std::string& index_key, faiss::gpu::StandardGpuResources& res, const faiss::MetricType metric_type, const size_t dim, int32_t index_add_loops, QueryMode mode = MODE_CPU) { double t0 = elapsed(); @@ -307,7 +303,7 @@ load_base_data(faiss::Index* &index, const std::string& ann_test_name, const std delete cpu_index; cpu_index = faiss::gpu::index_gpu_to_cpu(gpu_index); - faiss::IndexIVF *cpu_ivf_index = dynamic_cast(cpu_index); + faiss::IndexIVF* cpu_ivf_index = dynamic_cast(cpu_index); if (cpu_ivf_index != nullptr) { cpu_ivf_index->to_readonly(); } @@ -336,14 +332,14 @@ load_base_data(faiss::Index* &index, const std::string& ann_test_name, const std } void -load_query_data(faiss::Index::distance_t* &xq, size_t& nq, const std::string& ann_test_name, +load_query_data(faiss::Index::distance_t*& xq, size_t& nq, const std::string& ann_test_name, const faiss::MetricType metric_type, const size_t dim) { double t0 = elapsed(); size_t d; const std::string ann_file_name = ann_test_name + HDF5_POSTFIX; - xq = (float *) hdf5_read(ann_file_name, HDF5_DATASET_TEST, H5T_FLOAT, d, nq); + xq = (float*)hdf5_read(ann_file_name, HDF5_DATASET_TEST, H5T_FLOAT, d, nq); assert(d == dim || !"query does not have same dimension as train set"); if (metric_type == faiss::METRIC_INNER_PRODUCT) { @@ -353,12 +349,12 @@ load_query_data(faiss::Index::distance_t* &xq, size_t& nq, const std::string& an } void -load_ground_truth(faiss::Index::idx_t* >, size_t& k, const std::string& ann_test_name, const size_t nq) { +load_ground_truth(faiss::Index::idx_t*& gt, size_t& k, const std::string& ann_test_name, const size_t nq) { const std::string ann_file_name = ann_test_name + HDF5_POSTFIX; // load ground-truth and convert int to long size_t nq2; - int *gt_int = (int *) hdf5_read(ann_file_name, HDF5_DATASET_NEIGHBORS, H5T_INTEGER, k, nq2); + int* gt_int = (int*)hdf5_read(ann_file_name, HDF5_DATASET_NEIGHBORS, H5T_INTEGER, k, nq2); assert(nq2 == nq || !"incorrect nb of ground truth index"); gt = new faiss::Index::idx_t[k * nq]; @@ -384,15 +380,15 @@ load_ground_truth(faiss::Index::idx_t* >, size_t& k, const std::string& ann_te void test_with_nprobes(const std::string& ann_test_name, const std::string& index_key, faiss::Index* index, - faiss::gpu::StandardGpuResources& res, const QueryMode query_mode, - const faiss::Index::distance_t *xq, const faiss::Index::idx_t *gt, const std::vector nprobes, - const int32_t index_add_loops, const int32_t search_loops) { + faiss::gpu::StandardGpuResources& res, const QueryMode query_mode, const faiss::Index::distance_t* xq, + const faiss::Index::idx_t* gt, const std::vector nprobes, const int32_t index_add_loops, + const int32_t search_loops) { const size_t NQ = 1000, NQ_START = 10, NQ_STEP = 10; const size_t K = 1000, K_START = 100, K_STEP = 10; const size_t GK = 100; // topk of ground truth - std::unordered_map mode_str_map = - {{MODE_CPU, "MODE_CPU"}, {MODE_MIX, "MODE_MIX"}, {MODE_GPU, "MODE_GPU"}}; + std::unordered_map mode_str_map = { + {MODE_CPU, "MODE_CPU"}, {MODE_MIX, "MODE_MIX"}, {MODE_GPU, "MODE_GPU"}}; for (auto nprobe : nprobes) { switch (query_mode) { @@ -404,20 +400,20 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key break; } case MODE_GPU: { - faiss::gpu::GpuIndexIVF *gpu_index_ivf = dynamic_cast(index); + faiss::gpu::GpuIndexIVF* gpu_index_ivf = dynamic_cast(index); gpu_index_ivf->setNumProbes(nprobe); } } // output buffers - faiss::Index::idx_t *I = new faiss::Index::idx_t[NQ * K]; - faiss::Index::distance_t *D = new faiss::Index::distance_t[NQ * K]; + faiss::Index::idx_t* I = new faiss::Index::idx_t[NQ * K]; + faiss::Index::distance_t* D = new faiss::Index::distance_t[NQ * K]; printf("\n%s | %s - %s | nprobe=%lu\n", ann_test_name.c_str(), index_key.c_str(), - mode_str_map[query_mode].c_str(), nprobe); + mode_str_map[query_mode].c_str(), nprobe); printf("======================================================================================\n"); - for (size_t t_nq = NQ_START; t_nq <= NQ; t_nq *= NQ_STEP) { // nq = {10, 100, 1000} - for (size_t t_k = K_START; t_k <= K; t_k *= K_STEP) { // k = {100, 1000} + for (size_t t_nq = NQ_START; t_nq <= NQ; t_nq *= NQ_STEP) { // nq = {10, 100, 1000} + for (size_t t_k = K_START; t_k <= K; t_k *= K_STEP) { // k = {100, 1000} faiss::indexIVF_stats.quantization_time = 0.0; faiss::indexIVF_stats.search_time = 0.0; @@ -513,7 +509,7 @@ TEST(FAISSTEST, BENCHMARK) { const int32_t SIFT_INSERT_LOOPS = 2; // insert twice to get ~1G data set const int32_t GLOVE_INSERT_LOOPS = 1; -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// test_ann_hdf5("sift-128-euclidean", "IVF16384,Flat", MODE_CPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); test_ann_hdf5("sift-128-euclidean", "IVF16384,Flat", MODE_GPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); @@ -523,10 +519,11 @@ TEST(FAISSTEST, BENCHMARK) { #ifdef CUSTOMIZATION test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8Hybrid", MODE_CPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8Hybrid", MODE_GPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); -// test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8Hybrid", MODE_MIX, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); +// test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8Hybrid", MODE_MIX, SIFT_INSERT_LOOPS, param_nprobes, +// SEARCH_LOOPS); #endif -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// test_ann_hdf5("glove-200-angular", "IVF16384,Flat", MODE_CPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); test_ann_hdf5("glove-200-angular", "IVF16384,Flat", MODE_GPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); @@ -536,6 +533,7 @@ TEST(FAISSTEST, BENCHMARK) { #ifdef CUSTOMIZATION test_ann_hdf5("glove-200-angular", "IVF16384,SQ8Hybrid", MODE_CPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); test_ann_hdf5("glove-200-angular", "IVF16384,SQ8Hybrid", MODE_GPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); -// test_ann_hdf5("glove-200-angular", "IVF16384,SQ8Hybrid", MODE_MIX, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); +// test_ann_hdf5("glove-200-angular", "IVF16384,SQ8Hybrid", MODE_MIX, GLOVE_INSERT_LOOPS, param_nprobes, +// SEARCH_LOOPS); #endif } From a4f0f4bd13844d183f4b1e5c4c7c3f961ebe6ce1 Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 30 Oct 2019 17:37:44 +0800 Subject: [PATCH 07/18] get branch name Former-commit-id: eb75eb1a41845c294479a8e15176feac629be9a8 --- core/CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index ae4366af2f..df7cd9183b 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -32,7 +32,10 @@ string(REGEX REPLACE "\n" "" BUILD_TIME ${BUILD_TIME}) message(STATUS "Build time = ${BUILD_TIME}") MACRO (GET_GIT_BRANCH_NAME GIT_BRANCH_NAME) - execute_process(COMMAND "git" symbolic-ref --short HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) + execute_process(COMMAND "git" rev-parse --abbrev-ref HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) + if(GIT_BRANCH_NAME STREQUAL "") + execute_process(COMMAND "git" symbolic-ref --short -q HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME}) + endif() ENDMACRO (GET_GIT_BRANCH_NAME) GET_GIT_BRANCH_NAME(GIT_BRANCH_NAME) From 74f7b359ae3728ac24185a6b8dfc716e74a4d370 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 30 Oct 2019 17:41:59 +0800 Subject: [PATCH 08/18] #89 update Former-commit-id: 924960514c582b1c53f2115861c88e90252b63ac --- .../faiss_benchmark/faiss_benchmark_test.cpp | 54 ++++++++++--------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp index 3c5a396fef..5d38a6c1bc 100644 --- a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp +++ b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp @@ -448,7 +448,7 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key } void -test_ann_hdf5(const std::string& ann_test_name, const std::string& index_key, const QueryMode query_mode, +test_ann_hdf5(const std::string& ann_test_name, const std::string& index_type, const QueryMode query_mode, int32_t index_add_loops, const std::vector& nprobes, int32_t search_loops) { double t0 = elapsed(); @@ -457,11 +457,13 @@ test_ann_hdf5(const std::string& ann_test_name, const std::string& index_key, co faiss::MetricType metric_type; size_t dim; - if (query_mode == MODE_MIX && index_key.find("SQ8Hybrid") == std::string::npos) { - printf("Only SQ8Hybrid support MODE_MIX\n"); + if (query_mode == MODE_MIX && index_type != "SQ8Hybrid") { + assert(index_type == "SQ8Hybrid" || !"Only SQ8Hybrid support MODE_MIX"); return; } + std::string index_key = "IVF16384," + index_type; + if (!parse_ann_test_name(ann_test_name, dim, metric_type)) { printf("Invalid ann test name: %s\n", ann_test_name.c_str()); return; @@ -506,34 +508,34 @@ test_ann_hdf5(const std::string& ann_test_name, const std::string& index_key, co TEST(FAISSTEST, BENCHMARK) { std::vector param_nprobes = {8, 128}; const int32_t SEARCH_LOOPS = 5; + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// const int32_t SIFT_INSERT_LOOPS = 2; // insert twice to get ~1G data set + + test_ann_hdf5("sift-128-euclidean", "Flat", MODE_CPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); + test_ann_hdf5("sift-128-euclidean", "Flat", MODE_GPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); + + test_ann_hdf5("sift-128-euclidean", "SQ8", MODE_CPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); + test_ann_hdf5("sift-128-euclidean", "SQ8", MODE_GPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); + +#ifdef CUSTOMIZATION + test_ann_hdf5("sift-128-euclidean", "SQ8Hybrid", MODE_CPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); + test_ann_hdf5("sift-128-euclidean", "SQ8Hybrid", MODE_GPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); +// test_ann_hdf5("sift-128-euclidean", "SQ8Hybrid", MODE_MIX, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); +#endif + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// const int32_t GLOVE_INSERT_LOOPS = 1; - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - test_ann_hdf5("sift-128-euclidean", "IVF16384,Flat", MODE_CPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); - test_ann_hdf5("sift-128-euclidean", "IVF16384,Flat", MODE_GPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); + test_ann_hdf5("glove-200-angular", "Flat", MODE_CPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); + test_ann_hdf5("glove-200-angular", "Flat", MODE_GPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); - test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8", MODE_CPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); - test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8", MODE_GPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); + test_ann_hdf5("glove-200-angular", "SQ8", MODE_CPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); + test_ann_hdf5("glove-200-angular", "SQ8", MODE_GPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); #ifdef CUSTOMIZATION - test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8Hybrid", MODE_CPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); - test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8Hybrid", MODE_GPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); -// test_ann_hdf5("sift-128-euclidean", "IVF16384,SQ8Hybrid", MODE_MIX, SIFT_INSERT_LOOPS, param_nprobes, -// SEARCH_LOOPS); -#endif - - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - test_ann_hdf5("glove-200-angular", "IVF16384,Flat", MODE_CPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); - test_ann_hdf5("glove-200-angular", "IVF16384,Flat", MODE_GPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); - - test_ann_hdf5("glove-200-angular", "IVF16384,SQ8", MODE_CPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); - test_ann_hdf5("glove-200-angular", "IVF16384,SQ8", MODE_GPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); - -#ifdef CUSTOMIZATION - test_ann_hdf5("glove-200-angular", "IVF16384,SQ8Hybrid", MODE_CPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); - test_ann_hdf5("glove-200-angular", "IVF16384,SQ8Hybrid", MODE_GPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); -// test_ann_hdf5("glove-200-angular", "IVF16384,SQ8Hybrid", MODE_MIX, GLOVE_INSERT_LOOPS, param_nprobes, -// SEARCH_LOOPS); + test_ann_hdf5("glove-200-angular", "SQ8Hybrid", MODE_CPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); + test_ann_hdf5("glove-200-angular", "SQ8Hybrid", MODE_GPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); +// test_ann_hdf5("glove-200-angular", "SQ8Hybrid", MODE_MIX, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); #endif } From e3e22cedc5b0665600a399be0a211bc092eaf0c8 Mon Sep 17 00:00:00 2001 From: milvus-ci-robot Date: Thu, 31 Oct 2019 10:34:55 +0800 Subject: [PATCH 09/18] delete GIT_CREDENTIALS_ID parameter Former-commit-id: 8308271362483b047c3b406f6cbe0de32734e73d --- ci/jenkins/Jenkinsfile | 19 +++++++++---------- ci/jenkins/{jenkinsfile => step}/build.groovy | 0 .../cleanupSingleDev.groovy | 0 .../{jenkinsfile => step}/coverage.groovy | 0 .../deploySingle2Dev.groovy | 2 +- .../{jenkinsfile => step}/package.groovy | 0 .../publishImages.groovy | 0 .../singleDevNightlyTest.groovy | 2 +- .../singleDevTest.groovy | 2 +- 9 files changed, 12 insertions(+), 13 deletions(-) rename ci/jenkins/{jenkinsfile => step}/build.groovy (100%) rename ci/jenkins/{jenkinsfile => step}/cleanupSingleDev.groovy (100%) rename ci/jenkins/{jenkinsfile => step}/coverage.groovy (100%) rename ci/jenkins/{jenkinsfile => step}/deploySingle2Dev.groovy (57%) rename ci/jenkins/{jenkinsfile => step}/package.groovy (100%) rename ci/jenkins/{jenkinsfile => step}/publishImages.groovy (100%) rename ci/jenkins/{jenkinsfile => step}/singleDevNightlyTest.groovy (79%) rename ci/jenkins/{jenkinsfile => step}/singleDevTest.groovy (81%) diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index 67bff5ac1e..24f972748a 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -16,7 +16,6 @@ pipeline { parameters{ choice choices: ['Release', 'Debug'], description: '', name: 'BUILD_TYPE' - string defaultValue: 'cf1434e7-5a4b-4d25-82e8-88d667aef9e5', description: 'GIT CREDENTIALS ID', name: 'GIT_CREDENTIALS_ID', trim: true string defaultValue: 'registry.zilliz.com', description: 'DOCKER REGISTRY URL', name: 'DOKCER_REGISTRY_URL', trim: true string defaultValue: 'ba070c98-c8cc-4f7c-b657-897715f359fc', description: 'DOCKER CREDENTIALS ID', name: 'DOCKER_CREDENTIALS_ID', trim: true string defaultValue: 'http://192.168.1.202/artifactory/milvus', description: 'JFROG ARTFACTORY URL', name: 'JFROG_ARTFACTORY_URL', trim: true @@ -56,7 +55,7 @@ pipeline { steps { container('milvus-build-env') { script { - load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/build.groovy" + load "${env.WORKSPACE}/ci/jenkins/step/build.groovy" } } } @@ -65,7 +64,7 @@ pipeline { steps { container('milvus-build-env') { script { - load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/coverage.groovy" + load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy" } } } @@ -74,7 +73,7 @@ pipeline { steps { container('milvus-build-env') { script { - load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/package.groovy" + load "${env.WORKSPACE}/ci/jenkins/step/package.groovy" } } } @@ -96,7 +95,7 @@ pipeline { steps { container('publish-images'){ script { - load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/publishImages.groovy" + load "${env.WORKSPACE}/ci/jenkins/step/publishImages.groovy" } } } @@ -118,7 +117,7 @@ pipeline { steps { container('milvus-test-env') { script { - load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/deploySingle2Dev.groovy" + load "${env.WORKSPACE}/ci/jenkins/step/deploySingle2Dev.groovy" } } } @@ -130,9 +129,9 @@ pipeline { script { boolean isNightlyTest = isTimeTriggeredBuild() if (isNightlyTest) { - load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/singleDevNightlyTest.groovy" + load "${env.WORKSPACE}/ci/jenkins/step/singleDevNightlyTest.groovy" } else { - load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/singleDevTest.groovy" + load "${env.WORKSPACE}/ci/jenkins/step/singleDevTest.groovy" } } } @@ -143,7 +142,7 @@ pipeline { steps { container('milvus-test-env') { script { - load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy" + load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" } } } @@ -153,7 +152,7 @@ pipeline { unsuccessful { container('milvus-test-env') { script { - load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy" + load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" } } } diff --git a/ci/jenkins/jenkinsfile/build.groovy b/ci/jenkins/step/build.groovy similarity index 100% rename from ci/jenkins/jenkinsfile/build.groovy rename to ci/jenkins/step/build.groovy diff --git a/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy b/ci/jenkins/step/cleanupSingleDev.groovy similarity index 100% rename from ci/jenkins/jenkinsfile/cleanupSingleDev.groovy rename to ci/jenkins/step/cleanupSingleDev.groovy diff --git a/ci/jenkins/jenkinsfile/coverage.groovy b/ci/jenkins/step/coverage.groovy similarity index 100% rename from ci/jenkins/jenkinsfile/coverage.groovy rename to ci/jenkins/step/coverage.groovy diff --git a/ci/jenkins/jenkinsfile/deploySingle2Dev.groovy b/ci/jenkins/step/deploySingle2Dev.groovy similarity index 57% rename from ci/jenkins/jenkinsfile/deploySingle2Dev.groovy rename to ci/jenkins/step/deploySingle2Dev.groovy index bc6c6f4438..f4964df5e2 100644 --- a/ci/jenkins/jenkinsfile/deploySingle2Dev.groovy +++ b/ci/jenkins/step/deploySingle2Dev.groovy @@ -1,7 +1,7 @@ sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' sh 'helm repo update' dir ('milvus-helm') { - checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_CREDENTIALS_ID}", url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]]) + checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]]) dir ("milvus-gpu") { sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/db_backend/sqlite_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." } diff --git a/ci/jenkins/jenkinsfile/package.groovy b/ci/jenkins/step/package.groovy similarity index 100% rename from ci/jenkins/jenkinsfile/package.groovy rename to ci/jenkins/step/package.groovy diff --git a/ci/jenkins/jenkinsfile/publishImages.groovy b/ci/jenkins/step/publishImages.groovy similarity index 100% rename from ci/jenkins/jenkinsfile/publishImages.groovy rename to ci/jenkins/step/publishImages.groovy diff --git a/ci/jenkins/jenkinsfile/singleDevNightlyTest.groovy b/ci/jenkins/step/singleDevNightlyTest.groovy similarity index 79% rename from ci/jenkins/jenkinsfile/singleDevNightlyTest.groovy rename to ci/jenkins/step/singleDevNightlyTest.groovy index 5140ad858f..9aeab2eb4e 100644 --- a/ci/jenkins/jenkinsfile/singleDevNightlyTest.groovy +++ b/ci/jenkins/step/singleDevNightlyTest.groovy @@ -8,7 +8,7 @@ timeout(time: 90, unit: 'MINUTES') { if (!fileExists('milvus-helm')) { dir ("milvus-helm") { - checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_CREDENTIALS_ID}", url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]]) + checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]]) } } dir ("milvus-helm") { diff --git a/ci/jenkins/jenkinsfile/singleDevTest.groovy b/ci/jenkins/step/singleDevTest.groovy similarity index 81% rename from ci/jenkins/jenkinsfile/singleDevTest.groovy rename to ci/jenkins/step/singleDevTest.groovy index 16fe65a9b3..86e6f126d9 100644 --- a/ci/jenkins/jenkinsfile/singleDevTest.groovy +++ b/ci/jenkins/step/singleDevTest.groovy @@ -10,7 +10,7 @@ timeout(time: 60, unit: 'MINUTES') { // if (!fileExists('milvus-helm')) { // dir ("milvus-helm") { - // checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_CREDENTIALS_ID}", url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]]) + // checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]]) // } // } // dir ("milvus-helm") { From eeb503f7c548a5d5c373ea08debc0cd2f5bf4339 Mon Sep 17 00:00:00 2001 From: JinHai-CN Date: Thu, 31 Oct 2019 14:07:23 +0800 Subject: [PATCH 10/18] Update faiss link Former-commit-id: 22febba9b19f8de8422eae922ec75d7106e37e7a --- core/src/index/cmake/ThirdPartyPackagesCore.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/index/cmake/ThirdPartyPackagesCore.cmake b/core/src/index/cmake/ThirdPartyPackagesCore.cmake index e8a5c8a995..c066f7b392 100644 --- a/core/src/index/cmake/ThirdPartyPackagesCore.cmake +++ b/core/src/index/cmake/ThirdPartyPackagesCore.cmake @@ -245,7 +245,8 @@ if(CUSTOMIZATION) # set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1 # set(FAISS_MD5 "c89ea8e655f5cdf58f42486f13614714") # commit-id 9c28a1cbb88f41fa03b03d7204106201ad33276b branch-0.2.1 # set(FAISS_MD5 "87fdd86351ffcaf3f80dc26ade63c44b") # commit-id 841a156e67e8e22cd8088e1b58c00afbf2efc30b branch-0.2.1 - set(FAISS_MD5 "f3b2ce3364c3fa7febd3aa7fdd0fe380") # commit-id 694e03458e6b69ce8a62502f71f69a614af5af8f branch-0.3.0 + # set(FAISS_MD5 "f3b2ce3364c3fa7febd3aa7fdd0fe380") # commit-id 694e03458e6b69ce8a62502f71f69a614af5af8f branch-0.3.0 + set(FAISS_MD5 "bb30722c22390ce5f6759ccb216c1b2a") # commit-id d324db297475286afe107847c7fb7a0f9dc7e90e branch-0.3.0 endif() else() set(FAISS_SOURCE_URL "https://github.com/milvus-io/faiss/archive/1.6.0.tar.gz") From 892bd4daacd24d5a04e6f2e9e9f9ed061464b73d Mon Sep 17 00:00:00 2001 From: wxyu Date: Thu, 31 Oct 2019 14:30:17 +0800 Subject: [PATCH 11/18] Improve large query optimizer pass Former-commit-id: 1f0c283ec4dc5560c14d4e17d76719d38dfc2280 --- CHANGELOG.md | 1 + core/src/db/engine/ExecutionEngineImpl.cpp | 2 +- .../src/scheduler/optimizer/LargeSQ8HPass.cpp | 22 ++++++++++--------- core/src/scheduler/optimizer/LargeSQ8HPass.h | 1 + 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5176ce2b79..a0381852bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#118 - Using shared_ptr instead of weak_ptr to avoid performance loss - \#122 - Add unique id for Job - \#130 - Set task state MOVED after resource copy it completed +- \#149 - Improve large query optimizer pass ## Task diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 66e9795ff3..b6db2616cb 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -258,7 +258,7 @@ Status ExecutionEngineImpl::CopyToGpu(uint64_t device_id, bool hybrid) { if (hybrid) { const std::string key = location_ + ".quantizer"; - std::vector gpus = scheduler::get_gpu_pool(); + std::vector gpus{device_id}; const int64_t NOT_FOUND = -1; int64_t device_id = NOT_FOUND; diff --git a/core/src/scheduler/optimizer/LargeSQ8HPass.cpp b/core/src/scheduler/optimizer/LargeSQ8HPass.cpp index ddcee414a7..cacedd6208 100644 --- a/core/src/scheduler/optimizer/LargeSQ8HPass.cpp +++ b/core/src/scheduler/optimizer/LargeSQ8HPass.cpp @@ -55,16 +55,18 @@ LargeSQ8HPass::Run(const TaskPtr& task) { } std::vector gpus = scheduler::get_gpu_pool(); - std::vector all_free_mem; - for (auto& gpu : gpus) { - auto cache = cache::GpuCacheMgr::GetInstance(gpu); - auto free_mem = cache->CacheCapacity() - cache->CacheUsage(); - all_free_mem.push_back(free_mem); - } - - auto max_e = std::max_element(all_free_mem.begin(), all_free_mem.end()); - auto best_index = std::distance(all_free_mem.begin(), max_e); - auto best_device_id = gpus[best_index]; + // std::vector all_free_mem; + // for (auto& gpu : gpus) { + // auto cache = cache::GpuCacheMgr::GetInstance(gpu); + // auto free_mem = cache->CacheCapacity() - cache->CacheUsage(); + // all_free_mem.push_back(free_mem); + // } + // + // auto max_e = std::max_element(all_free_mem.begin(), all_free_mem.end()); + // auto best_index = std::distance(all_free_mem.begin(), max_e); + // auto best_device_id = gpus[best_index]; + auto best_device_id = count_ % gpus.size(); + count_++; ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, best_device_id); if (not res_ptr) { diff --git a/core/src/scheduler/optimizer/LargeSQ8HPass.h b/core/src/scheduler/optimizer/LargeSQ8HPass.h index 1e00aa4a1b..9d135d413a 100644 --- a/core/src/scheduler/optimizer/LargeSQ8HPass.h +++ b/core/src/scheduler/optimizer/LargeSQ8HPass.h @@ -43,6 +43,7 @@ class LargeSQ8HPass : public Pass { private: int32_t threshold_ = std::numeric_limits::max(); + int64_t count_ = 0; }; using LargeSQ8HPassPtr = std::shared_ptr; From 28ac93a22653b44bccfaad9b2ce50ff2d3d6f12c Mon Sep 17 00:00:00 2001 From: milvus-ci-robot Date: Thu, 31 Oct 2019 14:30:55 +0800 Subject: [PATCH 12/18] make sure to put #!/usr/bin/env groovy at the top of the Jenkinsfile Former-commit-id: 5af0774e12ea9d10f3fe10b77336df22d748c7de --- ci/jenkins/Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index 24f972748a..40d9686415 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -1,3 +1,5 @@ +#!/usr/bin/env groovy + String cron_timezone = "TZ=Asia/Shanghai" String cron_string = BRANCH_NAME == "master" ? "H 0 * * * " : "" cron_string = BRANCH_NAME == "0.5.1" ? "H 1 * * * " : cron_string From 49105e44f329242f9bf175ec369c10d8db78cbcd Mon Sep 17 00:00:00 2001 From: fishpenguin Date: Thu, 31 Oct 2019 14:39:02 +0800 Subject: [PATCH 13/18] Add only GPU and only CPU version for IVF_SQ8 and IVF_FLAT Former-commit-id: 14f5b095df2988d5f65ecd1cb2628fd3a3791a14 --- CHANGELOG.md | 1 + core/src/scheduler/SchedInst.h | 4 ++ core/src/scheduler/optimizer/OnlyCPUPass.cpp | 48 ++++++++++++++ core/src/scheduler/optimizer/OnlyCPUPass.h | 47 ++++++++++++++ core/src/scheduler/optimizer/OnlyGPUPass.cpp | 66 ++++++++++++++++++++ core/src/scheduler/optimizer/OnlyGPUPass.h | 50 +++++++++++++++ 6 files changed, 216 insertions(+) create mode 100644 core/src/scheduler/optimizer/OnlyCPUPass.cpp create mode 100644 core/src/scheduler/optimizer/OnlyCPUPass.h create mode 100644 core/src/scheduler/optimizer/OnlyGPUPass.cpp create mode 100644 core/src/scheduler/optimizer/OnlyGPUPass.h diff --git a/CHANGELOG.md b/CHANGELOG.md index 5176ce2b79..f277ec2a59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#104 - test_scheduler core dump - \#115 - Using new structure for tasktable - \#139 - New config opion use_gpu_threshold +- \#146 - Add only GPU and only CPU version for IVF_SQ8 and IVF_FLAT ## Improvement - \#64 - Improvement dump function in scheduler diff --git a/core/src/scheduler/SchedInst.h b/core/src/scheduler/SchedInst.h index b9153d3bc3..e758f37851 100644 --- a/core/src/scheduler/SchedInst.h +++ b/core/src/scheduler/SchedInst.h @@ -23,6 +23,8 @@ #include "Scheduler.h" #include "optimizer/HybridPass.h" #include "optimizer/LargeSQ8HPass.h" +#include "optimizer/OnlyCPUPass.h" +#include "optimizer/OnlyGPUPass.h" #include "optimizer/Optimizer.h" #include @@ -96,6 +98,8 @@ class OptimizerInst { std::vector pass_list; pass_list.push_back(std::make_shared()); pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared()); instance = std::make_shared(pass_list); } } diff --git a/core/src/scheduler/optimizer/OnlyCPUPass.cpp b/core/src/scheduler/optimizer/OnlyCPUPass.cpp new file mode 100644 index 0000000000..d974887bb7 --- /dev/null +++ b/core/src/scheduler/optimizer/OnlyCPUPass.cpp @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "scheduler/optimizer/OnlyCPUPass.h" +#include "scheduler/SchedInst.h" +#include "scheduler/task/SearchTask.h" +#include "scheduler/tasklabel/SpecResLabel.h" +#include "scheduler/Utils.h" + +namespace milvus { +namespace scheduler { + +bool +OnlyCPUPass::Run(const TaskPtr& task) { + if (task->Type() != TaskType::SearchTask) + return false; + auto search_task = std::static_pointer_cast(task); + if (search_task->file_->engine_type_ != (int) engine::EngineType::FAISS_IVFSQ8 && + search_task->file_->engine_type_ != (int) engine::EngineType::FAISS_IVFFLAT) { + return false; + } + + auto gpu_id = get_gpu_pool(); + if (gpu_id.empty()) { + ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); + auto label = std::make_shared(std::weak_ptr(res_ptr)); + task->label() = label; + return true; + } + return false; +} + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/optimizer/OnlyCPUPass.h b/core/src/scheduler/optimizer/OnlyCPUPass.h new file mode 100644 index 0000000000..edc3848ed4 --- /dev/null +++ b/core/src/scheduler/optimizer/OnlyCPUPass.h @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Pass.h" + +namespace milvus { +namespace scheduler { + +class OnlyCPUPass : public Pass { +public: + OnlyCPUPass() = default; + +public: + bool + Run(const TaskPtr &task) override; +}; + +using OnlyCPUPassPtr = std::shared_ptr; + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/optimizer/OnlyGPUPass.cpp b/core/src/scheduler/optimizer/OnlyGPUPass.cpp new file mode 100644 index 0000000000..20127aefb8 --- /dev/null +++ b/core/src/scheduler/optimizer/OnlyGPUPass.cpp @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "scheduler/optimizer/OnlyGPUPass.h" +#include "scheduler/SchedInst.h" +#include "scheduler/task/SearchTask.h" +#include "scheduler/tasklabel/SpecResLabel.h" +#include "scheduler/Utils.h" +#include "server/Config.h" + +namespace milvus { +namespace scheduler { + +bool +OnlyGPUPass::Run(const TaskPtr& task) { + if (task->Type() != TaskType::SearchTask) + return false; + + auto search_task = std::static_pointer_cast(task); + if (search_task->file_->engine_type_ != (int) engine::EngineType::FAISS_IVFSQ8 && + search_task->file_->engine_type_ != (int) engine::EngineType::FAISS_IVFFLAT) { + return false; + } + + server::Config& config = server::Config::GetInstance(); + std::vector search_resources; + config.GetResourceConfigSearchResources(search_resources); + for (auto &resource : search_resources) { + if (resource == "cpu") { + return false; + } + } + + auto gpu_id = get_gpu_pool(); + if (!gpu_id.empty()) { + ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, gpu_id[specified_gpu_id_]); + auto label = std::make_shared(std::weak_ptr(res_ptr)); + task->label() = label; + } else { + return false; + } + + if (specified_gpu_id_ < gpu_id.size() - 1) { + ++specified_gpu_id_; + } else { + specified_gpu_id_ = 0; + } + return true; +} + +} // namespace scheduler +} // namespace milvus diff --git a/core/src/scheduler/optimizer/OnlyGPUPass.h b/core/src/scheduler/optimizer/OnlyGPUPass.h new file mode 100644 index 0000000000..9220bab11d --- /dev/null +++ b/core/src/scheduler/optimizer/OnlyGPUPass.h @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Pass.h" + +namespace milvus { +namespace scheduler { + +class OnlyGPUPass : public Pass { +public: + OnlyGPUPass() = default; + +public: + bool + Run(const TaskPtr &task) override; + +private: + uint64_t specified_gpu_id_ = 0; +}; + +using OnlyGPUPassPtr = std::shared_ptr; + +} // namespace scheduler +} // namespace milvus From a68255f9d005a53bf0769a3bb3610f6c85322173 Mon Sep 17 00:00:00 2001 From: fishpenguin Date: Thu, 31 Oct 2019 14:44:59 +0800 Subject: [PATCH 14/18] clang format Former-commit-id: 3f99d7df6b042683696dd70c9dc0f14fee57b388 --- core/src/scheduler/optimizer/OnlyCPUPass.cpp | 6 +++--- core/src/scheduler/optimizer/OnlyCPUPass.h | 8 ++++---- core/src/scheduler/optimizer/OnlyGPUPass.cpp | 8 ++++---- core/src/scheduler/optimizer/OnlyGPUPass.h | 10 +++++----- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/core/src/scheduler/optimizer/OnlyCPUPass.cpp b/core/src/scheduler/optimizer/OnlyCPUPass.cpp index d974887bb7..2651a6e1a5 100644 --- a/core/src/scheduler/optimizer/OnlyCPUPass.cpp +++ b/core/src/scheduler/optimizer/OnlyCPUPass.cpp @@ -17,9 +17,9 @@ #include "scheduler/optimizer/OnlyCPUPass.h" #include "scheduler/SchedInst.h" +#include "scheduler/Utils.h" #include "scheduler/task/SearchTask.h" #include "scheduler/tasklabel/SpecResLabel.h" -#include "scheduler/Utils.h" namespace milvus { namespace scheduler { @@ -29,8 +29,8 @@ OnlyCPUPass::Run(const TaskPtr& task) { if (task->Type() != TaskType::SearchTask) return false; auto search_task = std::static_pointer_cast(task); - if (search_task->file_->engine_type_ != (int) engine::EngineType::FAISS_IVFSQ8 && - search_task->file_->engine_type_ != (int) engine::EngineType::FAISS_IVFFLAT) { + if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFSQ8 && + search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFFLAT) { return false; } diff --git a/core/src/scheduler/optimizer/OnlyCPUPass.h b/core/src/scheduler/optimizer/OnlyCPUPass.h index edc3848ed4..76b42e3766 100644 --- a/core/src/scheduler/optimizer/OnlyCPUPass.h +++ b/core/src/scheduler/optimizer/OnlyCPUPass.h @@ -32,13 +32,13 @@ namespace milvus { namespace scheduler { -class OnlyCPUPass : public Pass { -public: +class OnlyCPUPass : public Pass { + public: OnlyCPUPass() = default; -public: + public: bool - Run(const TaskPtr &task) override; + Run(const TaskPtr& task) override; }; using OnlyCPUPassPtr = std::shared_ptr; diff --git a/core/src/scheduler/optimizer/OnlyGPUPass.cpp b/core/src/scheduler/optimizer/OnlyGPUPass.cpp index 20127aefb8..f39ca1a042 100644 --- a/core/src/scheduler/optimizer/OnlyGPUPass.cpp +++ b/core/src/scheduler/optimizer/OnlyGPUPass.cpp @@ -17,9 +17,9 @@ #include "scheduler/optimizer/OnlyGPUPass.h" #include "scheduler/SchedInst.h" +#include "scheduler/Utils.h" #include "scheduler/task/SearchTask.h" #include "scheduler/tasklabel/SpecResLabel.h" -#include "scheduler/Utils.h" #include "server/Config.h" namespace milvus { @@ -31,15 +31,15 @@ OnlyGPUPass::Run(const TaskPtr& task) { return false; auto search_task = std::static_pointer_cast(task); - if (search_task->file_->engine_type_ != (int) engine::EngineType::FAISS_IVFSQ8 && - search_task->file_->engine_type_ != (int) engine::EngineType::FAISS_IVFFLAT) { + if (search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFSQ8 && + search_task->file_->engine_type_ != (int)engine::EngineType::FAISS_IVFFLAT) { return false; } server::Config& config = server::Config::GetInstance(); std::vector search_resources; config.GetResourceConfigSearchResources(search_resources); - for (auto &resource : search_resources) { + for (auto& resource : search_resources) { if (resource == "cpu") { return false; } diff --git a/core/src/scheduler/optimizer/OnlyGPUPass.h b/core/src/scheduler/optimizer/OnlyGPUPass.h index 9220bab11d..75a5f9e4f1 100644 --- a/core/src/scheduler/optimizer/OnlyGPUPass.h +++ b/core/src/scheduler/optimizer/OnlyGPUPass.h @@ -32,15 +32,15 @@ namespace milvus { namespace scheduler { -class OnlyGPUPass : public Pass { -public: +class OnlyGPUPass : public Pass { + public: OnlyGPUPass() = default; -public: + public: bool - Run(const TaskPtr &task) override; + Run(const TaskPtr& task) override; -private: + private: uint64_t specified_gpu_id_ = 0; }; From c8da2c3d64c418f4c558ffd51f291a688a79dcb5 Mon Sep 17 00:00:00 2001 From: Heisenberg Date: Thu, 31 Oct 2019 14:46:13 +0800 Subject: [PATCH 15/18] add mix mode for faiss baseline make baseline compatible for different faiss version Former-commit-id: 90f96b4d437136539e7aa805f58c49e2d8a23ed1 --- .../faiss_benchmark/faiss_benchmark_test.cpp | 146 +++++++++++------- 1 file changed, 88 insertions(+), 58 deletions(-) diff --git a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp index 5d38a6c1bc..780e620e6a 100644 --- a/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp +++ b/core/src/index/unittest/faiss_benchmark/faiss_benchmark_test.cpp @@ -17,31 +17,40 @@ #include +#include +#include +#include +#include #include #include #include +#include + +#define USE_FAISS_V1_5_3 0 + +#if USE_FAISS_V1_5_3 +#include +#include +#include #include #include +#else +#include +#include +#include +#endif + #include #include #include -#include #include +#include +#include + #ifdef CUSTOMIZATION #include #endif -#include -#include -#include - -#include - -#include -#include -#include -#include -#include /***************************************************** * To run this test, please download the HDF5 from @@ -56,6 +65,8 @@ const char HDF5_DATASET_TEST[] = "test"; const char HDF5_DATASET_NEIGHBORS[] = "neighbors"; const char HDF5_DATASET_DISTANCES[] = "distances"; +const int32_t GPU_DEVICE_IDX = 0; + enum QueryMode { MODE_CPU = 0, MODE_MIX, MODE_GPU }; double @@ -196,7 +207,7 @@ GetResultHitCount(const faiss::Index::idx_t* ground_index, const faiss::Index::i for (int i = 0; i < nq; i++) { // count the num of results exist in ground truth result set // each result replicates INDEX_ADD_LOOPS times - for (int j_c = 0; j_c < ground_k; j_c++) { + for (int j_c = 0; j_c < k; j_c++) { int r_c = index[i * k + j_c]; for (int j_g = 0; j_g < ground_k / index_add_loops; j_g++) { if (ground_index[i * ground_k + j_g] == r_c) { @@ -239,7 +250,6 @@ load_base_data(faiss::Index*& index, const std::string& ann_test_name, const std double t0 = elapsed(); const std::string ann_file_name = ann_test_name + HDF5_POSTFIX; - const int GPU_DEVICE_IDX = 0; faiss::Index *cpu_index = nullptr, *gpu_index = nullptr; faiss::distance_compute_blas_threshold = 800; @@ -249,30 +259,6 @@ load_base_data(faiss::Index*& index, const std::string& ann_test_name, const std try { printf("[%.3f s] Reading index file: %s\n", elapsed() - t0, index_file_name.c_str()); cpu_index = faiss::read_index(index_file_name.c_str()); - - if (mode != MODE_CPU) { - faiss::gpu::GpuClonerOptions option; - option.allInGpu = true; - - faiss::IndexComposition index_composition; - index_composition.index = cpu_index; - index_composition.quantizer = nullptr; - - switch (mode) { - case MODE_CPU: - assert(false); - break; - case MODE_MIX: - index_composition.mode = 1; // 0: all data, 1: copy quantizer, 2: copy data - break; - case MODE_GPU: - index_composition.mode = 0; // 0: all data, 1: copy quantizer, 2: copy data - break; - } - - printf("[%.3f s] Cloning CPU index to GPU\n", elapsed() - t0); - gpu_index = faiss::gpu::index_cpu_to_gpu(&res, GPU_DEVICE_IDX, &index_composition, &option); - } } catch (...) { size_t nb, d; printf("[%.3f s] Loading HDF5 file: %s\n", elapsed() - t0, ann_file_name.c_str()); @@ -289,6 +275,7 @@ load_base_data(faiss::Index*& index, const std::string& ann_test_name, const std printf("[%.3f s] Cloning CPU index to GPU\n", elapsed() - t0); gpu_index = faiss::gpu::index_cpu_to_gpu(&res, GPU_DEVICE_IDX, cpu_index); + delete cpu_index; printf("[%.3f s] Training on %ld vectors\n", elapsed() - t0, nb); gpu_index->train(nb, xb); @@ -300,8 +287,9 @@ load_base_data(faiss::Index*& index, const std::string& ann_test_name, const std } printf("[%.3f s] Coping GPU index to CPU\n", elapsed() - t0); - delete cpu_index; + cpu_index = faiss::gpu::index_gpu_to_cpu(gpu_index); + delete gpu_index; faiss::IndexIVF* cpu_ivf_index = dynamic_cast(cpu_index); if (cpu_ivf_index != nullptr) { @@ -314,21 +302,7 @@ load_base_data(faiss::Index*& index, const std::string& ann_test_name, const std delete[] xb; } - switch (mode) { - case MODE_CPU: - case MODE_MIX: - index = cpu_index; - if (gpu_index) { - delete gpu_index; - } - break; - case MODE_GPU: - index = gpu_index; - if (cpu_index) { - delete cpu_index; - } - break; - } + index = cpu_index; } void @@ -379,10 +353,12 @@ load_ground_truth(faiss::Index::idx_t*& gt, size_t& k, const std::string& ann_te } void -test_with_nprobes(const std::string& ann_test_name, const std::string& index_key, faiss::Index* index, +test_with_nprobes(const std::string& ann_test_name, const std::string& index_key, faiss::Index* cpu_index, faiss::gpu::StandardGpuResources& res, const QueryMode query_mode, const faiss::Index::distance_t* xq, const faiss::Index::idx_t* gt, const std::vector nprobes, const int32_t index_add_loops, const int32_t search_loops) { + double t0 = elapsed(); + const size_t NQ = 1000, NQ_START = 10, NQ_STEP = 10; const size_t K = 1000, K_START = 100, K_STEP = 10; const size_t GK = 100; // topk of ground truth @@ -390,6 +366,59 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key std::unordered_map mode_str_map = { {MODE_CPU, "MODE_CPU"}, {MODE_MIX, "MODE_MIX"}, {MODE_GPU, "MODE_GPU"}}; + faiss::Index *gpu_index, *index; + if (query_mode != MODE_CPU) { + faiss::gpu::GpuClonerOptions option; + option.allInGpu = true; + + faiss::IndexComposition index_composition; + index_composition.index = cpu_index; + index_composition.quantizer = nullptr; + + double copy_time; + switch (query_mode) { + case MODE_MIX: { + index_composition.mode = 1; // 0: all data, 1: copy quantizer, 2: copy data + + // warm up the transmission + gpu_index = faiss::gpu::index_cpu_to_gpu(&res, GPU_DEVICE_IDX, &index_composition, &option); + delete gpu_index; + + copy_time = elapsed(); + gpu_index = faiss::gpu::index_cpu_to_gpu(&res, GPU_DEVICE_IDX, &index_composition, &option); + delete gpu_index; + copy_time = elapsed() - copy_time; + printf("[%.3f s] Copy quantizer completed, cost %f s\n", elapsed() - t0, copy_time); + + auto ivf_index = dynamic_cast(cpu_index); + auto is_gpu_flat_index = dynamic_cast(ivf_index->quantizer); + if (is_gpu_flat_index == nullptr) { + delete ivf_index->quantizer; + ivf_index->quantizer = index_composition.quantizer; + } + index = cpu_index; + break; + } + case MODE_GPU: + index_composition.mode = 0; // 0: all data, 1: copy quantizer, 2: copy data + + // warm up the transmission + gpu_index = faiss::gpu::index_cpu_to_gpu(&res, GPU_DEVICE_IDX, &index_composition, &option); + delete gpu_index; + + copy_time = elapsed(); + gpu_index = faiss::gpu::index_cpu_to_gpu(&res, GPU_DEVICE_IDX, &index_composition, &option); + copy_time = elapsed() - copy_time; + printf("[%.3f s] Copy data completed, cost %f s\n", elapsed() - t0, copy_time); + + delete cpu_index; + index = gpu_index; + break; + } + } else { + index = cpu_index; + } + for (auto nprobe : nprobes) { switch (query_mode) { case MODE_CPU: @@ -445,6 +474,8 @@ test_with_nprobes(const std::string& ann_test_name, const std::string& index_key delete[] I; delete[] D; } + + delete index; } void @@ -488,7 +519,6 @@ test_ann_hdf5(const std::string& ann_test_name, const std::string& index_type, c delete[] xq; delete[] gt; - delete index; } /************************************************************************************ @@ -520,8 +550,8 @@ TEST(FAISSTEST, BENCHMARK) { #ifdef CUSTOMIZATION test_ann_hdf5("sift-128-euclidean", "SQ8Hybrid", MODE_CPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); + test_ann_hdf5("sift-128-euclidean", "SQ8Hybrid", MODE_MIX, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); test_ann_hdf5("sift-128-euclidean", "SQ8Hybrid", MODE_GPU, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); -// test_ann_hdf5("sift-128-euclidean", "SQ8Hybrid", MODE_MIX, SIFT_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); #endif /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -535,7 +565,7 @@ TEST(FAISSTEST, BENCHMARK) { #ifdef CUSTOMIZATION test_ann_hdf5("glove-200-angular", "SQ8Hybrid", MODE_CPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); + test_ann_hdf5("glove-200-angular", "SQ8Hybrid", MODE_MIX, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); test_ann_hdf5("glove-200-angular", "SQ8Hybrid", MODE_GPU, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); -// test_ann_hdf5("glove-200-angular", "SQ8Hybrid", MODE_MIX, GLOVE_INSERT_LOOPS, param_nprobes, SEARCH_LOOPS); #endif } From 530b2a0f280306348b1437a15aa71028b4436088 Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Thu, 31 Oct 2019 14:57:09 +0800 Subject: [PATCH 16/18] fix cmake error: STRING sub-command REGEX, mode REPLACE needs at least 6 arguments total to command Former-commit-id: 773f938125ed03582fc8242af0a23076677bf848 --- core/cmake/ThirdPartyPackages.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/cmake/ThirdPartyPackages.cmake b/core/cmake/ThirdPartyPackages.cmake index 62d7713024..657efa2eef 100644 --- a/core/cmake/ThirdPartyPackages.cmake +++ b/core/cmake/ThirdPartyPackages.cmake @@ -114,7 +114,7 @@ if (UNIX) OUTPUT_VARIABLE LSB_RELEASE_ID_SHORT OUTPUT_STRIP_TRAILING_WHITESPACE ) - STRING(REGEX REPLACE "\\." "_" UBUNTU_VERSION ${LSB_RELEASE_ID_SHORT}) + STRING(REGEX REPLACE "\\." "_" UBUNTU_VERSION "${LSB_RELEASE_ID_SHORT}") endif (UBUNTU_FOUND) endif (UBUNTU_EXTRA) endif (DEBIAN_FOUND) From 1dc7b321a45a178813350ea7c87d2d0ce57103c8 Mon Sep 17 00:00:00 2001 From: fishpenguin Date: Thu, 31 Oct 2019 16:41:48 +0800 Subject: [PATCH 17/18] Read config once in optimizer Former-commit-id: 8fee2a3cfa20cdc8b51e86867b7b3eb71dd2e759 --- core/src/scheduler/SchedInst.h | 14 ++++++++- core/src/scheduler/optimizer/OnlyCPUPass.cpp | 14 ++++----- core/src/scheduler/optimizer/OnlyGPUPass.cpp | 32 ++++++-------------- core/src/scheduler/optimizer/OnlyGPUPass.h | 3 +- 4 files changed, 32 insertions(+), 31 deletions(-) diff --git a/core/src/scheduler/SchedInst.h b/core/src/scheduler/SchedInst.h index e758f37851..a3048069f9 100644 --- a/core/src/scheduler/SchedInst.h +++ b/core/src/scheduler/SchedInst.h @@ -26,9 +26,11 @@ #include "optimizer/OnlyCPUPass.h" #include "optimizer/OnlyGPUPass.h" #include "optimizer/Optimizer.h" +#include "server/Config.h" #include #include +#include #include namespace milvus { @@ -95,11 +97,21 @@ class OptimizerInst { if (instance == nullptr) { std::lock_guard lock(mutex_); if (instance == nullptr) { + server::Config& config = server::Config::GetInstance(); + std::vector search_resources; + bool has_cpu = false; + config.GetResourceConfigSearchResources(search_resources); + for (auto& resource : search_resources) { + if (resource == "cpu") { + has_cpu = true; + } + } + std::vector pass_list; pass_list.push_back(std::make_shared()); pass_list.push_back(std::make_shared()); pass_list.push_back(std::make_shared()); - pass_list.push_back(std::make_shared()); + pass_list.push_back(std::make_shared(has_cpu)); instance = std::make_shared(pass_list); } } diff --git a/core/src/scheduler/optimizer/OnlyCPUPass.cpp b/core/src/scheduler/optimizer/OnlyCPUPass.cpp index 2651a6e1a5..238a91a82c 100644 --- a/core/src/scheduler/optimizer/OnlyCPUPass.cpp +++ b/core/src/scheduler/optimizer/OnlyCPUPass.cpp @@ -35,13 +35,13 @@ OnlyCPUPass::Run(const TaskPtr& task) { } auto gpu_id = get_gpu_pool(); - if (gpu_id.empty()) { - ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); - auto label = std::make_shared(std::weak_ptr(res_ptr)); - task->label() = label; - return true; - } - return false; + if (not gpu_id.empty()) + return false; + + ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource("cpu"); + auto label = std::make_shared(std::weak_ptr(res_ptr)); + task->label() = label; + return true; } } // namespace scheduler diff --git a/core/src/scheduler/optimizer/OnlyGPUPass.cpp b/core/src/scheduler/optimizer/OnlyGPUPass.cpp index f39ca1a042..3fcda0e8a3 100644 --- a/core/src/scheduler/optimizer/OnlyGPUPass.cpp +++ b/core/src/scheduler/optimizer/OnlyGPUPass.cpp @@ -20,14 +20,16 @@ #include "scheduler/Utils.h" #include "scheduler/task/SearchTask.h" #include "scheduler/tasklabel/SpecResLabel.h" -#include "server/Config.h" namespace milvus { namespace scheduler { +OnlyGPUPass::OnlyGPUPass(bool has_cpu) : has_cpu_(has_cpu) { +} + bool OnlyGPUPass::Run(const TaskPtr& task) { - if (task->Type() != TaskType::SearchTask) + if (task->Type() != TaskType::SearchTask || has_cpu_) return false; auto search_task = std::static_pointer_cast(task); @@ -36,29 +38,15 @@ OnlyGPUPass::Run(const TaskPtr& task) { return false; } - server::Config& config = server::Config::GetInstance(); - std::vector search_resources; - config.GetResourceConfigSearchResources(search_resources); - for (auto& resource : search_resources) { - if (resource == "cpu") { - return false; - } - } - auto gpu_id = get_gpu_pool(); - if (!gpu_id.empty()) { - ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, gpu_id[specified_gpu_id_]); - auto label = std::make_shared(std::weak_ptr(res_ptr)); - task->label() = label; - } else { + if (gpu_id.empty()) return false; - } - if (specified_gpu_id_ < gpu_id.size() - 1) { - ++specified_gpu_id_; - } else { - specified_gpu_id_ = 0; - } + ResourcePtr res_ptr = ResMgrInst::GetInstance()->GetResource(ResourceType::GPU, gpu_id[specified_gpu_id_]); + auto label = std::make_shared(std::weak_ptr(res_ptr)); + task->label() = label; + + specified_gpu_id_ = specified_gpu_id_++ % gpu_id.size(); return true; } diff --git a/core/src/scheduler/optimizer/OnlyGPUPass.h b/core/src/scheduler/optimizer/OnlyGPUPass.h index 75a5f9e4f1..10d909d30e 100644 --- a/core/src/scheduler/optimizer/OnlyGPUPass.h +++ b/core/src/scheduler/optimizer/OnlyGPUPass.h @@ -34,7 +34,7 @@ namespace scheduler { class OnlyGPUPass : public Pass { public: - OnlyGPUPass() = default; + explicit OnlyGPUPass(bool has_cpu); public: bool @@ -42,6 +42,7 @@ class OnlyGPUPass : public Pass { private: uint64_t specified_gpu_id_ = 0; + bool has_cpu_ = false; }; using OnlyGPUPassPtr = std::shared_ptr; From 4df6f4b5a93fa9c6cfc54479d52edf0371224ddd Mon Sep 17 00:00:00 2001 From: wxyu Date: Thu, 31 Oct 2019 16:58:47 +0800 Subject: [PATCH 18/18] Not return error when search_resources and index_build_device set cpu Former-commit-id: 0e0313e59cfbf8f8b17fc62db02f2e21a25d25a2 --- CHANGELOG.md | 1 + core/src/server/Config.cpp | 12 +++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f399b87f6..e2ed8f4b7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#122 - Add unique id for Job - \#130 - Set task state MOVED after resource copy it completed - \#149 - Improve large query optimizer pass +- \#156 - Not return error when search_resources and index_build_device set cpu ## Task diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index 86caf6dd37..2a04128411 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -714,9 +714,12 @@ Config::CheckResourceConfigSearchResources(const std::vector& value return Status(SERVER_INVALID_ARGUMENT, msg); } - for (auto& gpu_device : value) { - if (!CheckGpuDevice(gpu_device).ok()) { - std::string msg = "Invalid search resource: " + gpu_device + + for (auto& device : value) { + if (device == "cpu") { + continue; + } + if (!CheckGpuDevice(device).ok()) { + std::string msg = "Invalid search resource: " + device + ". Possible reason: resource_config.search_resources does not match your hardware."; return Status(SERVER_INVALID_ARGUMENT, msg); } @@ -726,6 +729,9 @@ Config::CheckResourceConfigSearchResources(const std::vector& value Status Config::CheckResourceConfigIndexBuildDevice(const std::string& value) { + if (value == "cpu") { + return Status::OK(); + } if (!CheckGpuDevice(value).ok()) { std::string msg = "Invalid index build device: " + value + ". Possible reason: resource_config.index_build_device does not match your hardware.";