test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-01 11:29:48 +08:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

Make knowhere compile independently (#7606)

Browse Source 
Make knowhere compile independently

1. Make knowhere compile independently
    * Add gtest, arrow, and some other libraries to index.
    * Add cache, log and some other files to knowhere.
    * Add CMakeLists files to index's thirdparty.

2. Modified the compilation content of knowhere
    * Delete some content of compile library.
    * Add IMPORTED_GLOBAL property to faiss.

3.  Change the compilation location of some libraries
    * Make OpenBLas compiled in thirdpartycore.cmake.
    * Make faiss compiled in thirdparty/CMakeLists.

Change the content of knowhere/CMakeLists

1. Change easyloggingpp and nlohmann into index/thirdparty.
2. Change MILVUS_THIRDPARTY_SRC into KNOWHERE_THIRDPARTY_SRC.

Delete FindOpenBLAS

1. Delete Openblas.cmake.

2. The search task for openBlas is assigned to ThirdpartyCore.

3. Some changes were made to build.sh in index.

Fix the openBLas compilation problem

Delete the if-else in compilation of faiss;

Now when complie faiss, it will find the Openblas as we wish.

Fix some problem:

1. delete arrow

2. set openblas_source to AUTO

3. change a include_dir

4. delete MKL

5. delete the CMakeLists in index/utils,cache,log

Change variable build_test to knowhere_build_test in index/build.sh

Change the include location of  GNUInstallDirs

set CMAKE_INSTALL_LIBDIR

Resolves: milvus-io#5183
See also: milvus-io#6604

Signed-off-by: Shen Zhi <m13120163046@163.com>
This commit is contained in:
ZhiShen 2021-10-13 17:06:33 +08:00 committed by GitHub
parent c90a97f95a
commit 6a770e5c38
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
60 changed files with 38232 additions and 995 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
internal/core/src/index/CMakeLists.txt
View File

@ -42,10 +42,17 @@ else ()
endif ()
message(STATUS "Build type = ${BUILD_TYPE}")
include(GNUInstallDirs)
set(CMAKE_INSTALL_LIBDIR lib)
set(INDEX_SOURCE_DIR ${PROJECT_SOURCE_DIR})
set(INDEX_BINARY_DIR ${PROJECT_BINARY_DIR})
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${INDEX_SOURCE_DIR}/cmake")
# This will set RPATH to all excutable TARGET
# self-installed dynamic libraries will be correctly linked by excutable
set( CMAKE_INSTALL_RPATH "/usr/lib" "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}" )
set( CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE )
include(ExternalProject)
include(DefineOptionsCore)
include(BuildUtilsCore)
@ -72,6 +79,7 @@ if (MILVUS_SUPPORT_SPTAG)
endif ()
include(ThirdPartyPackagesCore)
add_subdirectory(thirdparty)
if (CMAKE_BUILD_TYPE STREQUAL "Release")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp")
@ -93,7 +101,7 @@ endif ()
set(INDEX_INCLUDE_DIRS ${INDEX_INCLUDE_DIRS} PARENT_SCOPE)
if (BUILD_UNIT_TEST)
if (KNOWHERE_BUILD_TESTS)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DELPP_DISABLE_LOGS")
add_subdirectory(unittest)
endif ()

12
internal/core/src/index/build.sh
View File

@ -21,8 +21,8 @@ do
;;
r)
if [[ -d cmake_build ]]; then
rm ./cmake_build -r
MAKE_CLEAN="ON"
cd cmake_build
make clean
fi
;;
g)
@ -52,15 +52,13 @@ done
if [[ ! -d cmake_build ]]; then
mkdir cmake_build
MAKE_CLEAN="ON"
fi
cd cmake_build
CUDA_COMPILER=/usr/local/cuda/bin/nvcc
if [[ ${MAKE_CLEAN} == "ON" ]]; then
CMAKE_CMD="cmake -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
CMAKE_CMD="cmake -DKNOWHERE_BUILD_TESTS=${BUILD_UNITTEST} \
-DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}
-DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
-DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
@ -69,9 +67,9 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then
echo ${CMAKE_CMD}
${CMAKE_CMD}
make clean
fi
make -j 8 ||exit 1
make install || exit 1

4
internal/core/src/index/cmake/DefineOptionsCore.cmake
View File

@ -55,15 +55,15 @@ endmacro()
#----------------------------------------------------------------------
set_option_category("Thirdparty")
set(KNOWHERE_DEPENDENCY_SOURCE_DEFAULT "BUNDLED")
set(KNOWHERE_DEPENDENCY_SOURCE "AUTO")
define_option_string(KNOWHERE_DEPENDENCY_SOURCE
"Method to use for acquiring KNOWHERE's build dependencies"
"${KNOWHERE_DEPENDENCY_SOURCE_DEFAULT}"
"AUTO"
"BUNDLED"
"SYSTEM")
define_option(KNOWHERE_USE_CCACHE "Use ccache when compiling (if available)" ON)
define_option(KNOWHERE_VERBOSE_THIRDPARTY_BUILD

431
internal/core/src/index/cmake/FindArrow.cmake
View File

@ -1,431 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# - Find Arrow (arrow/api.h, libarrow.a, libarrow.so)
# This module defines
# ARROW_FOUND, whether Arrow has been found
# ARROW_FULL_SO_VERSION, full shared object version of found Arrow "100.0.0"
# ARROW_IMPORT_LIB, path to libarrow's import library (Windows only)
# ARROW_INCLUDE_DIR, directory containing headers
# ARROW_LIBS, deprecated. Use ARROW_LIB_DIR instead
# ARROW_LIB_DIR, directory containing Arrow libraries
# ARROW_SHARED_IMP_LIB, deprecated. Use ARROW_IMPORT_LIB instead
# ARROW_SHARED_LIB, path to libarrow's shared library
# ARROW_SO_VERSION, shared object version of found Arrow such as "100"
# ARROW_STATIC_LIB, path to libarrow.a
# ARROW_VERSION, version of found Arrow
# ARROW_VERSION_MAJOR, major version of found Arrow
# ARROW_VERSION_MINOR, minor version of found Arrow
# ARROW_VERSION_PATCH, patch version of found Arrow
include(FindPkgConfig)
include(FindPackageHandleStandardArgs)
set(ARROW_SEARCH_LIB_PATH_SUFFIXES)
if(CMAKE_LIBRARY_ARCHITECTURE)
list(APPEND ARROW_SEARCH_LIB_PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}")
endif()
list(APPEND ARROW_SEARCH_LIB_PATH_SUFFIXES
"lib64"
"lib32"
"lib"
"bin")
set(ARROW_CONFIG_SUFFIXES
"_RELEASE"
"_RELWITHDEBINFO"
"_MINSIZEREL"
"_DEBUG"
"")
if(CMAKE_BUILD_TYPE)
string(TOUPPER ${CMAKE_BUILD_TYPE} ARROW_CONFIG_SUFFIX_PREFERRED)
set(ARROW_CONFIG_SUFFIX_PREFERRED "_${ARROW_CONFIG_SUFFIX_PREFERRED}")
list(INSERT ARROW_CONFIG_SUFFIXES 0 "${ARROW_CONFIG_SUFFIX_PREFERRED}")
endif()
if(NOT DEFINED ARROW_MSVC_STATIC_LIB_SUFFIX)
if(MSVC)
set(ARROW_MSVC_STATIC_LIB_SUFFIX "_static")
else()
set(ARROW_MSVC_STATIC_LIB_SUFFIX "")
endif()
endif()
# Internal function.
#
# Set shared library name for ${base_name} to ${output_variable}.
#
# Example:
# arrow_build_shared_library_name(ARROW_SHARED_LIBRARY_NAME arrow)
# # -> ARROW_SHARED_LIBRARY_NAME=libarrow.so on Linux
# # -> ARROW_SHARED_LIBRARY_NAME=libarrow.dylib on macOS
# # -> ARROW_SHARED_LIBRARY_NAME=arrow.dll with MSVC on Windows
# # -> ARROW_SHARED_LIBRARY_NAME=libarrow.dll with MinGW on Windows
function(arrow_build_shared_library_name output_variable base_name)
set(${output_variable}
"${CMAKE_SHARED_LIBRARY_PREFIX}${base_name}${CMAKE_SHARED_LIBRARY_SUFFIX}"
PARENT_SCOPE)
endfunction()
# Internal function.
#
# Set import library name for ${base_name} to ${output_variable}.
# This is useful only for MSVC build. Import library is used only
# with MSVC build.
#
# Example:
# arrow_build_import_library_name(ARROW_IMPORT_LIBRARY_NAME arrow)
# # -> ARROW_IMPORT_LIBRARY_NAME=arrow on Linux (meaningless)
# # -> ARROW_IMPORT_LIBRARY_NAME=arrow on macOS (meaningless)
# # -> ARROW_IMPORT_LIBRARY_NAME=arrow.lib with MSVC on Windows
# # -> ARROW_IMPORT_LIBRARY_NAME=libarrow.dll.a with MinGW on Windows
function(arrow_build_import_library_name output_variable base_name)
set(${output_variable}
"${CMAKE_IMPORT_LIBRARY_PREFIX}${base_name}${CMAKE_IMPORT_LIBRARY_SUFFIX}"
PARENT_SCOPE)
endfunction()
# Internal function.
#
# Set static library name for ${base_name} to ${output_variable}.
#
# Example:
# arrow_build_static_library_name(ARROW_STATIC_LIBRARY_NAME arrow)
# # -> ARROW_STATIC_LIBRARY_NAME=libarrow.a on Linux
# # -> ARROW_STATIC_LIBRARY_NAME=libarrow.a on macOS
# # -> ARROW_STATIC_LIBRARY_NAME=arrow.lib with MSVC on Windows
# # -> ARROW_STATIC_LIBRARY_NAME=libarrow.dll.a with MinGW on Windows
function(arrow_build_static_library_name output_variable base_name)
set(
${output_variable}
"${CMAKE_STATIC_LIBRARY_PREFIX}${base_name}${ARROW_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}"
PARENT_SCOPE)
endfunction()
# Internal function.
#
# Set macro value for ${macro_name} in ${header_content} to ${output_variable}.
#
# Example:
# arrow_extract_macro_value(version_major
# "ARROW_VERSION_MAJOR"
# "#define ARROW_VERSION_MAJOR 1.0.0")
# # -> version_major=1.0.0
function(arrow_extract_macro_value output_variable macro_name header_content)
string(REGEX MATCH "#define +${macro_name} +[^\r\n]+" macro_definition
"${header_content}")
string(REGEX
REPLACE "^#define +${macro_name} +(.+)$" "\\1" macro_value "${macro_definition}")
set(${output_variable} "${macro_value}" PARENT_SCOPE)
endfunction()
# Internal macro only for arrow_find_package.
#
# Find package in HOME.
macro(arrow_find_package_home)
find_path(${prefix}_include_dir "${header_path}"
PATHS "${home}"
PATH_SUFFIXES "include"
NO_DEFAULT_PATH)
set(include_dir "${${prefix}_include_dir}")
set(${prefix}_INCLUDE_DIR "${include_dir}" PARENT_SCOPE)
if(MSVC)
set(CMAKE_SHARED_LIBRARY_SUFFIXES_ORIGINAL ${CMAKE_FIND_LIBRARY_SUFFIXES})
# .dll isn't found by find_library with MSVC because .dll isn't included in
# CMAKE_FIND_LIBRARY_SUFFIXES.
list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES "${CMAKE_SHARED_LIBRARY_SUFFIX}")
endif()
find_library(${prefix}_shared_lib
NAMES "${shared_lib_name}"
PATHS "${home}"
PATH_SUFFIXES ${ARROW_SEARCH_LIB_PATH_SUFFIXES}
NO_DEFAULT_PATH)
if(MSVC)
set(CMAKE_SHARED_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_ORIGINAL})
endif()
set(shared_lib "${${prefix}_shared_lib}")
set(${prefix}_SHARED_LIB "${shared_lib}" PARENT_SCOPE)
if(shared_lib)
add_library(${target_shared} SHARED IMPORTED)
set_target_properties(${target_shared} PROPERTIES IMPORTED_LOCATION "${shared_lib}")
if(include_dir)
set_target_properties(${target_shared}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${include_dir}")
endif()
find_library(${prefix}_import_lib
NAMES "${import_lib_name}"
PATHS "${home}"
PATH_SUFFIXES ${ARROW_SEARCH_LIB_PATH_SUFFIXES}
NO_DEFAULT_PATH)
set(import_lib "${${prefix}_import_lib}")
set(${prefix}_IMPORT_LIB "${import_lib}" PARENT_SCOPE)
if(import_lib)
set_target_properties(${target_shared} PROPERTIES IMPORTED_IMPLIB "${import_lib}")
endif()
endif()
find_library(${prefix}_static_lib
NAMES "${static_lib_name}"
PATHS "${home}"
PATH_SUFFIXES ${ARROW_SEARCH_LIB_PATH_SUFFIXES}
NO_DEFAULT_PATH)
set(static_lib "${${prefix}_static_lib}")
set(${prefix}_STATIC_LIB "${static_lib}" PARENT_SCOPE)
if(static_lib)
add_library(${target_static} STATIC IMPORTED)
set_target_properties(${target_static} PROPERTIES IMPORTED_LOCATION "${static_lib}")
if(include_dir)
set_target_properties(${target_static}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${include_dir}")
endif()
endif()
endmacro()
# Internal macro only for arrow_find_package.
#
# Find package by CMake package configuration.
macro(arrow_find_package_cmake_package_configuration)
# ARROW-5575: We need to split target files for each component
if(TARGET ${target_shared} OR TARGET ${target_static})
set(${cmake_package_name}_FOUND TRUE)
else()
find_package(${cmake_package_name} CONFIG)
endif()
if(${cmake_package_name}_FOUND)
set(${prefix}_USE_CMAKE_PACKAGE_CONFIG TRUE PARENT_SCOPE)
if(TARGET ${target_shared})
foreach(suffix ${ARROW_CONFIG_SUFFIXES})
get_target_property(shared_lib ${target_shared} IMPORTED_LOCATION${suffix})
if(shared_lib)
# Remove shared library version:
# libarrow.so.100.0.0 -> libarrow.so
# Because ARROW_HOME and pkg-config approaches don't add
# shared library version.
string(REGEX
REPLACE "(${CMAKE_SHARED_LIBRARY_SUFFIX})[.0-9]+$" "\\1" shared_lib
"${shared_lib}")
set(${prefix}_SHARED_LIB "${shared_lib}" PARENT_SCOPE)
break()
endif()
endforeach()
endif()
if(TARGET ${target_static})
foreach(suffix ${ARROW_CONFIG_SUFFIXES})
get_target_property(static_lib ${target_static} IMPORTED_LOCATION${suffix})
if(static_lib)
set(${prefix}_STATIC_LIB "${static_lib}" PARENT_SCOPE)
break()
endif()
endforeach()
endif()
endif()
endmacro()
# Internal macro only for arrow_find_package.
#
# Find package by pkg-config.
macro(arrow_find_package_pkg_config)
pkg_check_modules(${prefix}_PC ${pkg_config_name})
if(${prefix}_PC_FOUND)
set(${prefix}_USE_PKG_CONFIG TRUE PARENT_SCOPE)
set(include_dir "${${prefix}_PC_INCLUDEDIR}")
set(lib_dir "${${prefix}_PC_LIBDIR}")
set(shared_lib_paths "${${prefix}_PC_LINK_LIBRARIES}")
# Use the first shared library path as the IMPORTED_LOCATION
# for ${target_shared}. This assumes that the first shared library
# path is the shared library path for this module.
list(GET shared_lib_paths 0 first_shared_lib_path)
# Use the rest shared library paths as the INTERFACE_LINK_LIBRARIES
# for ${target_shared}. This assumes that the rest shared library
# paths are dependency library paths for this module.
list(LENGTH shared_lib_paths n_shared_lib_paths)
if(n_shared_lib_paths LESS_EQUAL 1)
set(rest_shared_lib_paths)
else()
list(SUBLIST
shared_lib_paths
1
-1
rest_shared_lib_paths)
endif()
set(${prefix}_VERSION "${${prefix}_PC_VERSION}" PARENT_SCOPE)
set(${prefix}_INCLUDE_DIR "${include_dir}" PARENT_SCOPE)
set(${prefix}_SHARED_LIB "${first_shared_lib_path}" PARENT_SCOPE)
add_library(${target_shared} SHARED IMPORTED)
set_target_properties(${target_shared}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
"${include_dir}"
INTERFACE_LINK_LIBRARIES
"${rest_shared_lib_paths}"
IMPORTED_LOCATION
"${first_shared_lib_path}")
find_library(${prefix}_static_lib
NAMES "${static_lib_name}"
PATHS "${lib_dir}"
NO_DEFAULT_PATH)
set(static_lib "${${prefix}_static_lib}")
set(${prefix}_STATIC_LIB "${static_lib}" PARENT_SCOPE)
if(static_lib)
add_library(${target_static} STATIC IMPORTED)
set_target_properties(${target_static}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${include_dir}"
IMPORTED_LOCATION "${static_lib}")
endif()
endif()
endmacro()
function(arrow_find_package
prefix
home
base_name
header_path
cmake_package_name
pkg_config_name)
arrow_build_shared_library_name(shared_lib_name ${base_name})
arrow_build_import_library_name(import_lib_name ${base_name})
arrow_build_static_library_name(static_lib_name ${base_name})
set(target_shared ${base_name}_shared)
set(target_static ${base_name}_static)
if(home)
arrow_find_package_home()
set(${prefix}_FIND_APPROACH "HOME: ${home}" PARENT_SCOPE)
else()
arrow_find_package_cmake_package_configuration()
if(${cmake_package_name}_FOUND)
set(${prefix}_FIND_APPROACH
"CMake package configuration: ${cmake_package_name}"
PARENT_SCOPE)
else()
arrow_find_package_pkg_config()
set(${prefix}_FIND_APPROACH "pkg-config: ${pkg_config_name}" PARENT_SCOPE)
endif()
endif()
if(NOT include_dir)
if(TARGET ${target_shared})
get_target_property(include_dir ${target_shared} INTERFACE_INCLUDE_DIRECTORIES)
elseif(TARGET ${target_static})
get_target_property(include_dir ${target_static} INTERFACE_INCLUDE_DIRECTORIES)
endif()
endif()
if(include_dir)
set(${prefix}_INCLUDE_DIR "${include_dir}" PARENT_SCOPE)
endif()
if(shared_lib)
get_filename_component(lib_dir "${shared_lib}" DIRECTORY)
elseif(static_lib)
get_filename_component(lib_dir "${static_lib}" DIRECTORY)
else()
set(lib_dir NOTFOUND)
endif()
set(${prefix}_LIB_DIR "${lib_dir}" PARENT_SCOPE)
# For backward compatibility
set(${prefix}_LIBS "${lib_dir}" PARENT_SCOPE)
endfunction()
if(NOT "$ENV{ARROW_HOME}" STREQUAL "")
file(TO_CMAKE_PATH "$ENV{ARROW_HOME}" ARROW_HOME)
endif()
arrow_find_package(ARROW
"${ARROW_HOME}"
arrow
arrow/api.h
Arrow
arrow)
if(ARROW_HOME)
if(ARROW_INCLUDE_DIR)
file(READ "${ARROW_INCLUDE_DIR}/arrow/util/config.h" ARROW_CONFIG_H_CONTENT)
arrow_extract_macro_value(ARROW_VERSION_MAJOR "ARROW_VERSION_MAJOR"
"${ARROW_CONFIG_H_CONTENT}")
arrow_extract_macro_value(ARROW_VERSION_MINOR "ARROW_VERSION_MINOR"
"${ARROW_CONFIG_H_CONTENT}")
arrow_extract_macro_value(ARROW_VERSION_PATCH "ARROW_VERSION_PATCH"
"${ARROW_CONFIG_H_CONTENT}")
if("${ARROW_VERSION_MAJOR}" STREQUAL ""
OR "${ARROW_VERSION_MINOR}" STREQUAL ""
OR "${ARROW_VERSION_PATCH}" STREQUAL "")
set(ARROW_VERSION "0.0.0")
else()
set(ARROW_VERSION
"${ARROW_VERSION_MAJOR}.${ARROW_VERSION_MINOR}.${ARROW_VERSION_PATCH}")
endif()
arrow_extract_macro_value(ARROW_SO_VERSION_QUOTED "ARROW_SO_VERSION"
"${ARROW_CONFIG_H_CONTENT}")
string(REGEX REPLACE "^\"(.+)\"$" "\\1" ARROW_SO_VERSION "${ARROW_SO_VERSION_QUOTED}")
arrow_extract_macro_value(ARROW_FULL_SO_VERSION_QUOTED "ARROW_FULL_SO_VERSION"
"${ARROW_CONFIG_H_CONTENT}")
string(REGEX
REPLACE "^\"(.+)\"$" "\\1" ARROW_FULL_SO_VERSION
"${ARROW_FULL_SO_VERSION_QUOTED}")
endif()
else()
if(ARROW_USE_CMAKE_PACKAGE_CONFIG)
find_package(Arrow CONFIG)
elseif(ARROW_USE_PKG_CONFIG)
pkg_get_variable(ARROW_SO_VERSION arrow so_version)
pkg_get_variable(ARROW_FULL_SO_VERSION arrow full_so_version)
endif()
endif()
set(ARROW_ABI_VERSION ${ARROW_SO_VERSION})
mark_as_advanced(ARROW_ABI_VERSION
ARROW_CONFIG_SUFFIXES
ARROW_FULL_SO_VERSION
ARROW_IMPORT_LIB
ARROW_INCLUDE_DIR
ARROW_LIBS
ARROW_LIB_DIR
ARROW_SEARCH_LIB_PATH_SUFFIXES
ARROW_SHARED_IMP_LIB
ARROW_SHARED_LIB
ARROW_SO_VERSION
ARROW_STATIC_LIB
ARROW_VERSION
ARROW_VERSION_MAJOR
ARROW_VERSION_MINOR
ARROW_VERSION_PATCH)
find_package_handle_standard_args(Arrow REQUIRED_VARS
# The first required variable is shown
# in the found message. So this list is
# not sorted alphabetically.
ARROW_INCLUDE_DIR
ARROW_LIB_DIR
ARROW_FULL_SO_VERSION
ARROW_SO_VERSION
VERSION_VAR
ARROW_VERSION)
set(ARROW_FOUND ${Arrow_FOUND})
if(Arrow_FOUND AND NOT Arrow_FIND_QUIETLY)
message(STATUS "Arrow version: ${ARROW_VERSION} (${ARROW_FIND_APPROACH})")
message(STATUS "Arrow SO and ABI version: ${ARROW_SO_VERSION}")
message(STATUS "Arrow full SO version: ${ARROW_FULL_SO_VERSION}")
message(STATUS "Found the Arrow core shared library: ${ARROW_SHARED_LIB}")
message(STATUS "Found the Arrow core import library: ${ARROW_IMPORT_LIB}")
message(STATUS "Found the Arrow core static library: ${ARROW_STATIC_LIB}")
endif()

93
internal/core/src/index/cmake/FindOpenBLAS.cmake
View File

@ -1,93 +0,0 @@
if (OpenBLAS_FOUND) # the git version propose a OpenBLASConfig.cmake
message(STATUS "OpenBLASConfig found")
set(OpenBLAS_INCLUDE_DIR ${OpenBLAS_INCLUDE_DIRS})
else()
message("OpenBLASConfig not found")
unset(OpenBLAS_DIR CACHE)
set(OpenBLAS_INCLUDE_SEARCH_PATHS
/usr/local/openblas/include
/usr/include
/usr/include/openblas
/usr/include/openblas-base
/usr/local/include
/usr/local/include/openblas
/usr/local/include/openblas-base
/opt/OpenBLAS/include
/usr/local/opt/openblas/include
$ENV{OpenBLAS_HOME}
$ENV{OpenBLAS_HOME}/include
)
set(OpenBLAS_LIB_SEARCH_PATHS
/usr/local/openblas/lib
/lib/
/lib/openblas-base
/lib64/
/usr/lib
/usr/lib/openblas-base
/usr/lib64
/usr/local/lib
/usr/local/lib64
/usr/local/opt/openblas/lib
/opt/OpenBLAS/lib
$ENV{OpenBLAS}
$ENV{OpenBLAS}/lib
$ENV{OpenBLAS_HOME}
$ENV{OpenBLAS_HOME}/lib
)
set(DEFAULT_OpenBLAS_LIB_PATH
/usr/local/openblas/lib
${OPENBLAS_PREFIX}/lib)
message("DEFAULT_OpenBLAS_LIB_PATH: ${DEFAULT_OpenBLAS_LIB_PATH}")
find_path(OpenBLAS_INCLUDE_DIR NAMES openblas_config.h lapacke.h PATHS ${OpenBLAS_INCLUDE_SEARCH_PATHS})
find_library(OpenBLAS_LIB NAMES openblas PATHS ${DEFAULT_OpenBLAS_LIB_PATH} NO_DEFAULT_PATH)
find_library(OpenBLAS_LIB NAMES openblas PATHS ${OpenBLAS_LIB_SEARCH_PATHS})
# mostly for debian
find_library(Lapacke_LIB NAMES lapacke PATHS ${DEFAULT_OpenBLAS_LIB_PATH} NO_DEFAULT_PATH)
find_library(Lapacke_LIB NAMES lapacke PATHS ${OpenBLAS_LIB_SEARCH_PATHS})
set(OpenBLAS_FOUND ON)
# Check include files
if(NOT OpenBLAS_INCLUDE_DIR)
set(OpenBLAS_FOUND OFF)
message(STATUS "Could not find OpenBLAS include. Turning OpenBLAS_FOUND off")
else()
message(STATUS "find OpenBLAS include:${OpenBLAS_INCLUDE_DIR} ")
endif()
# Check libraries
if(NOT OpenBLAS_LIB)
set(OpenBLAS_FOUND OFF)
message(STATUS "Could not find OpenBLAS lib. Turning OpenBLAS_FOUND off")
else()
message(STATUS "find OpenBLAS lib:${OpenBLAS_LIB} ")
endif()
if (OpenBLAS_FOUND)
set(FOUND_OPENBLAS "true" PARENT_SCOPE)
set(OpenBLAS_LIBRARIES ${OpenBLAS_LIB})
STRING(REGEX REPLACE "/libopenblas.so" "" OpenBLAS_LIB_DIR ${OpenBLAS_LIBRARIES})
message(STATUS "find OpenBLAS libraries:${OpenBLAS_LIBRARIES} ")
if (Lapacke_LIB)
set(OpenBLAS_LIBRARIES ${OpenBLAS_LIBRARIES} ${Lapacke_LIB})
endif()
if (NOT OpenBLAS_FIND_QUIETLY)
message(STATUS "Found OpenBLAS libraries: ${OpenBLAS_LIBRARIES}")
message(STATUS "Found OpenBLAS include: ${OpenBLAS_INCLUDE_DIR}")
endif()
else()
set(FOUND_OPENBLAS "false" PARENT_SCOPE)
if (OpenBLAS_FIND_REQUIRED)
message(FATAL_ERROR "Could not find OpenBLAS")
endif()
endif()
endif()
mark_as_advanced(
OpenBLAS_INCLUDE_DIR
OpenBLAS_LIBRARIES
OpenBLAS_LIB_DIR
)

404
internal/core/src/index/cmake/ThirdPartyPackagesCore.cmake
View File

@ -10,13 +10,9 @@
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
#-------------------------------------------------------------------------------
set(KNOWHERE_THIRDPARTY_DEPENDENCIES
Arrow
FAISS
GTest
OpenBLAS
MKL
FAISS
)
message(STATUS "Using ${KNOWHERE_DEPENDENCY_SOURCE} approach to find dependencies")
@ -28,39 +24,9 @@ foreach (DEPENDENCY ${KNOWHERE_THIRDPARTY_DEPENDENCIES})
endif ()
endforeach ()
macro(build_dependency DEPENDENCY_NAME)
if ("${DEPENDENCY_NAME}" STREQUAL "Arrow")
build_arrow()
elseif ("${DEPENDENCY_NAME}" STREQUAL "OpenBLAS")
build_openblas()
elseif ("${DEPENDENCY_NAME}" STREQUAL "FAISS")
build_faiss()
elseif ("${DEPENDENCY_NAME}" STREQUAL "MKL")
build_mkl()
else ()
message(FATAL_ERROR "Unknown thirdparty dependency to build: ${DEPENDENCY_NAME}")
endif ()
endmacro()
macro(resolve_dependency DEPENDENCY_NAME)
if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO")
find_package(${DEPENDENCY_NAME} MODULE)
if (NOT ${${DEPENDENCY_NAME}_FOUND})
build_dependency(${DEPENDENCY_NAME})
endif ()
elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED")
build_dependency(${DEPENDENCY_NAME})
elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM")
find_package(${DEPENDENCY_NAME} REQUIRED)
endif ()
endmacro()
# ----------------------------------------------------------------------
# Identify OS
if (UNIX)
if (APPLE)
set(CMAKE_OS_NAME "osx" CACHE STRING "Operating system name" FORCE)
else (APPLE)
## Check for Debian GNU/Linux ________________
find_file(DEBIAN_FOUND debian_version debconf.conf
PATHS /etc
@ -104,7 +70,6 @@ if (UNIX)
endif (UBUNTU_FOUND)
endif (UBUNTU_EXTRA)
endif (DEBIAN_FOUND)
endif (APPLE)
endif (UNIX)
@ -170,13 +135,6 @@ endif ()
set(MAKE_BUILD_ARGS "-j6")
# ----------------------------------------------------------------------
# Find pthreads
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
# ----------------------------------------------------------------------
# Versions and URLs for toolchain builds, which also can be used to configure
# offline builds
@ -203,121 +161,19 @@ foreach (_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT})
set(${_LIB_NAME} "${_LIB_VERSION}")
endforeach ()
set(FAISS_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/faiss)
if (DEFINED ENV{KNOWHERE_ARROW_URL})
set(ARROW_SOURCE_URL "$ENV{KNOWHERE_ARROW_URL}")
else ()
set(ARROW_SOURCE_URL
"https://github.com/apache/arrow.git"
)
endif ()
if (DEFINED ENV{KNOWHERE_OPENBLAS_URL})
set(OPENBLAS_SOURCE_URL "$ENV{KNOWHERE_OPENBLAS_URL}")
else ()
set(OPENBLAS_SOURCE_URL
"https://github.com/xianyi/OpenBLAS/archive/v${OPENBLAS_VERSION}.tar.gz")
"https://github.com.cnpmjs.org/xianyi/OpenBLAS/archive/v${OPENBLAS_VERSION}.tar.gz")
endif ()
# ----------------------------------------------------------------------
# ARROW
set(ARROW_PREFIX "${INDEX_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp")
macro(build_arrow)
message(STATUS "Building Apache ARROW-${ARROW_VERSION} from source")
set(ARROW_STATIC_LIB_NAME arrow)
set(ARROW_LIB_DIR "${ARROW_PREFIX}/lib")
set(ARROW_STATIC_LIB
"${ARROW_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include")
set(ARROW_CMAKE_ARGS
${EP_COMMON_CMAKE_ARGS}
-DARROW_BUILD_STATIC=ON
-DARROW_BUILD_SHARED=OFF
-DARROW_USE_GLOG=OFF
-DCMAKE_INSTALL_PREFIX=${ARROW_PREFIX}
-DCMAKE_INSTALL_LIBDIR=${ARROW_LIB_DIR}
-DARROW_CUDA=OFF
-DARROW_FLIGHT=OFF
-DARROW_GANDIVA=OFF
-DARROW_GANDIVA_JAVA=OFF
-DARROW_HDFS=OFF
-DARROW_HIVESERVER2=OFF
-DARROW_ORC=OFF
-DARROW_PARQUET=OFF
-DARROW_PLASMA=OFF
-DARROW_PLASMA_JAVA_CLIENT=OFF
-DARROW_PYTHON=OFF
-DARROW_WITH_BZ2=OFF
-DARROW_WITH_ZLIB=OFF
-DARROW_WITH_LZ4=OFF
-DARROW_WITH_SNAPPY=OFF
-DARROW_WITH_ZSTD=OFF
-DARROW_WITH_BROTLI=OFF
-DCMAKE_BUILD_TYPE=Release
-DARROW_DEPENDENCY_SOURCE=BUNDLED #Build all arrow dependencies from source instead of calling find_package first
-DBOOST_SOURCE=AUTO #try to find BOOST in the system default locations and build from source if not found
)
externalproject_add(arrow_ep
GIT_REPOSITORY
${ARROW_SOURCE_URL}
GIT_TAG
${ARROW_VERSION}
GIT_SHALLOW
TRUE
SOURCE_SUBDIR
cpp
${EP_LOG_OPTIONS}
CMAKE_ARGS
${ARROW_CMAKE_ARGS}
BUILD_COMMAND
""
INSTALL_COMMAND
${MAKE} ${MAKE_BUILD_ARGS} install
BUILD_BYPRODUCTS
"${ARROW_STATIC_LIB}"
)
file(MAKE_DIRECTORY "${ARROW_INCLUDE_DIR}")
add_library(arrow STATIC IMPORTED)
set_target_properties(arrow
PROPERTIES IMPORTED_LOCATION "${ARROW_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}")
add_dependencies(arrow arrow_ep)
set(JEMALLOC_PREFIX "${INDEX_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep-build/jemalloc_ep-prefix/src/jemalloc_ep")
add_custom_command(TARGET arrow_ep POST_BUILD
COMMAND ${CMAKE_COMMAND} -E make_directory ${ARROW_LIB_DIR}
COMMAND ${CMAKE_COMMAND} -E copy ${JEMALLOC_PREFIX}/lib/libjemalloc_pic.a ${ARROW_LIB_DIR}
DEPENDS ${JEMALLOC_PREFIX}/lib/libjemalloc_pic.a)
endmacro()
if (KNOWHERE_WITH_ARROW AND NOT TARGET arrow_ep)
resolve_dependency(Arrow)
link_directories(SYSTEM ${ARROW_LIB_DIR})
include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
endif ()
# ----------------------------------------------------------------------
# OpenBLAS
set(OPENBLAS_PREFIX "${INDEX_BINARY_DIR}/openblas_ep-prefix/src/openblas_ep")
# Openblas
macro(build_openblas)
message(STATUS "Building OpenBLAS-${OPENBLAS_VERSION} from source")
set(OpenBLAS_INCLUDE_DIR "${OPENBLAS_PREFIX}/include")
set(OpenBLAS_LIB_DIR "${OPENBLAS_PREFIX}/lib")
set(OPENBLAS_SHARED_LIB
"${OPENBLAS_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas${CMAKE_SHARED_LIBRARY_SUFFIX}")
set(OPENBLAS_STATIC_LIB
"${OPENBLAS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}openblas${CMAKE_STATIC_LIBRARY_SUFFIX}")
set (KNOWHERE_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX})
set(OPENBLAS_CMAKE_ARGS
${EP_COMMON_CMAKE_ARGS}
-DCMAKE_BUILD_TYPE=Release
@ -332,233 +188,57 @@ macro(build_openblas)
-DCC=gcc
-DINTERFACE64=0
-DNUM_THREADS=128
-DNO_LAPACKE=1
-DNO_LAPACKE=0
"-DVERSION=${OPENBLAS_VERSION}"
"-DCMAKE_INSTALL_PREFIX=${OPENBLAS_PREFIX}"
-DCMAKE_INSTALL_LIBDIR=lib)
"-DCMAKE_INSTALL_PREFIX=${KNOWHERE_INSTALL_PREFIX}"
)
externalproject_add(openblas_ep
URL
${OPENBLAS_SOURCE_URL}
${EP_LOG_OPTIONS}
CMAKE_ARGS
${OPENBLAS_CMAKE_ARGS}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE}
PREFIX=${OPENBLAS_PREFIX}
install
BUILD_BYPRODUCTS
${OPENBLAS_SHARED_LIB}
${OPENBLAS_STATIC_LIB})
URL ${OPENBLAS_SOURCE_URL}
URL_MD5 "28cc19a6acbf636f5aab5f10b9a0dfe1"
CMAKE_ARGS ${OPENBLAS_CMAKE_ARGS}
BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS}
PREFIX ${CMAKE_BINARY_DIR}/3rdparty_download/openblas-subbuild
BINARY_DIR openblas-bin
INSTALL_DIR ${KNOWHERE_INSTALL_PREFIX}
)
ExternalProject_Get_Property(openblas_ep INSTALL_DIR)
if( NOT IS_DIRECTORY ${INSTALL_DIR}/include )
file( MAKE_DIRECTORY "${INSTALL_DIR}/include" )
endif()
file(MAKE_DIRECTORY "${OpenBLAS_INCLUDE_DIR}")
add_library(openblas SHARED IMPORTED)
set_target_properties(
openblas
set_target_properties( openblas
PROPERTIES
IMPORTED_LOCATION "${OPENBLAS_SHARED_LIB}"
LIBRARY_OUTPUT_NAME "openblas"
INTERFACE_INCLUDE_DIRECTORIES "${OpenBLAS_INCLUDE_DIR}")
IMPORTED_GLOBAL TRUE
IMPORTED_LOCATION ${INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/libopenblas.so
INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/${CMAKE_INSTALL_INCLUDEDIR})
add_dependencies(openblas openblas_ep)
get_target_property(OpenBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES)
set(OpenBLAS_LIBRARIES "${OPENBLAS_SHARED_LIB}")
endmacro()
if (KNOWHERE_WITH_OPENBLAS)
resolve_dependency(OpenBLAS)
include_directories(SYSTEM "${OpenBLAS_INCLUDE_DIR}")
link_directories(SYSTEM "${OpenBLAS_LIB_DIR}")
endif()
if (OpenBLAS_SOURCE STREQUAL "AUTO")
set (BLA_VENDOR OpenBLAS)
find_package(BLAS)
message(STATUS "Knowhere openblas libraries: ${BLAS_LIBRARIES}")
message(STATUS "Knowhere openblas found: ${BLAS_FOUND}")
# ----------------------------------------------------------------------
# MKL
macro(build_mkl)
if (FAISS_WITH_MKL)
if (EXISTS "/proc/cpuinfo")
FILE(READ /proc/cpuinfo PROC_CPUINFO)
SET(VENDOR_ID_RX "vendor_id[ \t]*:[ \t]*([a-zA-Z]+)\n")
STRING(REGEX MATCH "${VENDOR_ID_RX}" VENDOR_ID "${PROC_CPUINFO}")
STRING(REGEX REPLACE "${VENDOR_ID_RX}" "\\1" VENDOR_ID "${VENDOR_ID}")
if (NOT ${VENDOR_ID} STREQUAL "GenuineIntel")
set(FAISS_WITH_MKL OFF)
endif ()
endif ()
find_path(MKL_LIB_PATH
NAMES "libmkl_intel_ilp64.a" "libmkl_gnu_thread.a" "libmkl_core.a"
PATH_SUFFIXES "intel/compilers_and_libraries_${MKL_VERSION}/linux/mkl/lib/intel64/")
if (${MKL_LIB_PATH} STREQUAL "MKL_LIB_PATH-NOTFOUND")
message(FATAL_ERROR "Could not find MKL libraries")
endif ()
message(STATUS "MKL lib path = ${MKL_LIB_PATH}")
set(MKL_LIBS
${MKL_LIB_PATH}/libmkl_intel_ilp64.a
${MKL_LIB_PATH}/libmkl_gnu_thread.a
${MKL_LIB_PATH}/libmkl_core.a
)
endif ()
endmacro()
# ----------------------------------------------------------------------
# FAISS
macro(build_faiss)
message(STATUS "Building FAISS-${FAISS_VERSION} from source")
set(FAISS_PREFIX "${INDEX_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep")
set(FAISS_INCLUDE_DIR "${FAISS_PREFIX}/include")
set(FAISS_STATIC_LIB
"${FAISS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX}")
if (CCACHE_FOUND)
set(FAISS_C_COMPILER "${CCACHE_FOUND} ${CMAKE_C_COMPILER}")
if (MILVUS_GPU_VERSION)
set(FAISS_CXX_COMPILER "${CMAKE_CXX_COMPILER}")
set(FAISS_CUDA_COMPILER "${CCACHE_FOUND} ${CMAKE_CUDA_COMPILER}")
if (BLAS_FOUND)
add_library(openblas ALIAS BLAS::BLAS)
else()
set(FAISS_CXX_COMPILER "${CCACHE_FOUND} ${CMAKE_CXX_COMPILER}")
endif()
else ()
set(FAISS_C_COMPILER "${CMAKE_C_COMPILER}")
set(FAISS_CXX_COMPILER "${CMAKE_CXX_COMPILER}")
build_openblas()
endif()
set(FAISS_CONFIGURE_ARGS
"--prefix=${FAISS_PREFIX}"
"CC=${FAISS_C_COMPILER}"
"CXX=${FAISS_CXX_COMPILER}"
"NVCC=${FAISS_CUDA_COMPILER}"
"CFLAGS=${EP_C_FLAGS}"
"CXXFLAGS=${EP_CXX_FLAGS} -mf16c -O3"
--without-python)
if (FAISS_WITH_MKL)
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
"CPPFLAGS=-DFINTEGER=long -DMKL_ILP64 -m64 -I${MKL_LIB_PATH}/../../include"
"LDFLAGS=-L${MKL_LIB_PATH}"
)
else ()
message(STATUS "Build Faiss with OpenBlas/LAPACK")
if(OpenBLAS_FOUND)
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
"LDFLAGS=-L${OpenBLAS_LIB_DIR}")
else()
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
"LDFLAGS=-L${OPENBLAS_PREFIX}/lib")
elseif (OpenBLAS_SOURCE STREQUAL "BUNDLED")
build_openblas()
elseif (OpenBLAS_SOURCE STREQUAL "SYSTEM")
set (BLA_VENDOR OpenBLAS)
find_package(BLAS REQUIRED)
add_library(openblas ALIAS BLAS::BLAS)
endif ()
endif()
if (MILVUS_GPU_VERSION)
if (NOT MILVUS_CUDA_ARCH OR MILVUS_CUDA_ARCH STREQUAL "DEFAULT")
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
"--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}"
"--with-cuda-arch=-gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75"
)
else()
STRING(REPLACE ";" " " MILVUS_CUDA_ARCH "${MILVUS_CUDA_ARCH}")
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
"--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}"
"--with-cuda-arch=${MILVUS_CUDA_ARCH}"
)
endif ()
else ()
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
"CPPFLAGS=-DUSE_CPU"
--without-cuda)
endif ()
message(STATUS "Building FAISS with configure args -${FAISS_CONFIGURE_ARGS}")
if (DEFINED ENV{FAISS_SOURCE_URL})
set(FAISS_SOURCE_URL "$ENV{FAISS_SOURCE_URL}")
externalproject_add(faiss_ep
URL
${FAISS_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${FAISS_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS} all
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE} install
BUILD_BYPRODUCTS
${FAISS_STATIC_LIB})
else ()
externalproject_add(faiss_ep
DOWNLOAD_COMMAND
""
SOURCE_DIR
${FAISS_SOURCE_DIR}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${FAISS_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS} all
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE} install
BUILD_BYPRODUCTS
${FAISS_STATIC_LIB})
endif ()
if(NOT OpenBLAS_FOUND)
message("add faiss dependencies: openblas_ep")
ExternalProject_Add_StepDependencies(faiss_ep configure openblas_ep)
endif()
file(MAKE_DIRECTORY "${FAISS_INCLUDE_DIR}")
add_library(faiss STATIC IMPORTED)
set_target_properties(
faiss
PROPERTIES
IMPORTED_LOCATION "${FAISS_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}"
)
if (FAISS_WITH_MKL)
set_target_properties(
faiss
PROPERTIES
INTERFACE_LINK_LIBRARIES "${MKL_LIBS}")
else ()
set_target_properties(
faiss
PROPERTIES
INTERFACE_LINK_LIBRARIES "${OpenBLAS_LIBRARIES}")
endif ()
add_dependencies(faiss faiss_ep)
endmacro()
if (KNOWHERE_WITH_FAISS AND NOT TARGET faiss_ep)
if (FAISS_WITH_MKL)
resolve_dependency(MKL)
else ()
message("faiss with no mkl")
endif ()
resolve_dependency(FAISS)
get_target_property(FAISS_INCLUDE_DIR faiss INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM "${FAISS_INCLUDE_DIR}")
link_directories(SYSTEM ${FAISS_PREFIX}/lib/)
endif ()
add_subdirectory(thirdparty/NGT)

4
internal/core/src/index/knowhere/CMakeLists.txt
View File

@ -15,6 +15,7 @@ include_directories(${INDEX_SOURCE_DIR}/knowhere)
include_directories(${INDEX_SOURCE_DIR}/thirdparty)
include_directories(${INDEX_SOURCE_DIR}/thirdparty/NGT/lib)
set(KNOWHERE_THIRDPARTY_SRC ${INDEX_SOURCE_DIR}/thirdparty)
if (MILVUS_SUPPORT_SPTAG)
include_directories(${INDEX_SOURCE_DIR}/thirdparty/SPTAG/AnnService)
@ -45,12 +46,11 @@ set(external_srcs
set (LOG_SRC
knowhere/common/Log.cpp
${MILVUS_THIRDPARTY_SRC}/easyloggingpp/easylogging++.cc
${KNOWHERE_THIRDPARTY_SRC}/easyloggingpp/easylogging++.cc
)
add_library(index_log STATIC ${LOG_SRC})
set_target_properties(index_log PROPERTIES RULE_LAUNCH_COMPILE "")
set_target_properties(index_log PROPERTIES RULE_LAUNCH_LINK "")
include_directories(${MILVUS_THIRDPARTY_SRC})
set(config_srcs
knowhere/archive/KnowhereConfig.cpp

104
internal/core/src/index/knowhere/cache/Cache.h vendored Normal file
View File

@ -0,0 +1,104 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include "LRU.h"
#include "utils/Log.h"
#include <atomic>
#include <mutex>
#include <set>
#include <string>
namespace milvus {
namespace cache {
template <typename ItemObj>
class Cache {
public:
// mem_capacity, units:GB
Cache(int64_t capacity_gb, int64_t cache_max_count, const std::string& header = "");
~Cache() = default;
int64_t
usage() const {
return usage_;
}
// unit: BYTE
int64_t
capacity() const {
return capacity_;
}
// unit: BYTE
void
set_capacity(int64_t capacity);
double
freemem_percent() const {
return freemem_percent_;
}
void
set_freemem_percent(double percent) {
freemem_percent_ = percent;
}
size_t
size() const;
bool
exists(const std::string& key);
ItemObj
get(const std::string& key);
void
insert(const std::string& key, const ItemObj& item);
void
erase(const std::string& key);
bool
reserve(const int64_t size);
void
print();
void
clear();
private:
void
insert_internal(const std::string& key, const ItemObj& item);
void
erase_internal(const std::string& key);
void
free_memory_internal(const int64_t target_size);
private:
std::string header_;
int64_t usage_;
int64_t capacity_;
double freemem_percent_;
LRU<std::string, ItemObj> lru_;
mutable std::mutex mutex_;
};
} // namespace cache
} // namespace milvus
#include "cache/Cache.inl"

191
internal/core/src/index/knowhere/cache/Cache.inl vendored Normal file
View File

@ -0,0 +1,191 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
namespace milvus {
namespace cache {
constexpr double DEFAULT_THRESHOLD_PERCENT = 0.7;
template <typename ItemObj>
Cache<ItemObj>::Cache(int64_t capacity, int64_t cache_max_count, const std::string& header)
: header_(header),
usage_(0),
capacity_(capacity),
freemem_percent_(DEFAULT_THRESHOLD_PERCENT),
lru_(cache_max_count) {
}
template <typename ItemObj>
void
Cache<ItemObj>::set_capacity(int64_t capacity) {
std::lock_guard<std::mutex> lock(mutex_);
if (capacity > 0) {
capacity_ = capacity;
free_memory_internal(capacity);
}
}
template <typename ItemObj>
size_t
Cache<ItemObj>::size() const {
std::lock_guard<std::mutex> lock(mutex_);
return lru_.size();
}
template <typename ItemObj>
bool
Cache<ItemObj>::exists(const std::string& key) {
std::lock_guard<std::mutex> lock(mutex_);
return lru_.exists(key);
}
template <typename ItemObj>
ItemObj
Cache<ItemObj>::get(const std::string& key) {
std::lock_guard<std::mutex> lock(mutex_);
if (!lru_.exists(key)) {
return nullptr;
}
return lru_.get(key);
}
template <typename ItemObj>
void
Cache<ItemObj>::insert(const std::string& key, const ItemObj& item) {
std::lock_guard<std::mutex> lock(mutex_);
insert_internal(key, item);
}
template <typename ItemObj>
void
Cache<ItemObj>::erase(const std::string& key) {
std::lock_guard<std::mutex> lock(mutex_);
erase_internal(key);
}
template <typename ItemObj>
bool
Cache<ItemObj>::reserve(const int64_t item_size) {
std::lock_guard<std::mutex> lock(mutex_);
if (item_size > capacity_) {
LOG_SERVER_ERROR_ << header_ << " item size " << (item_size >> 20) << "MB too big to insert into cache capacity"
<< (capacity_ >> 20) << "MB";
return false;
}
if (item_size > capacity_ - usage_) {
free_memory_internal(capacity_ - item_size);
}
return true;
}
template <typename ItemObj>
void
Cache<ItemObj>::clear() {
std::lock_guard<std::mutex> lock(mutex_);
lru_.clear();
usage_ = 0;
LOG_SERVER_DEBUG_ << header_ << " Clear cache !";
}
template <typename ItemObj>
void
Cache<ItemObj>::print() {
std::lock_guard<std::mutex> lock(mutex_);
size_t cache_count = lru_.size();
// for (auto it = lru_.begin(); it != lru_.end(); ++it) {
// LOG_SERVER_DEBUG_ << it->first;
// }
LOG_SERVER_DEBUG_ << header_ << " [item count]: " << cache_count << ", [usage] " << (usage_ >> 20)
<< "MB, [capacity] " << (capacity_ >> 20) << "MB";
}
template <typename ItemObj>
void
Cache<ItemObj>::insert_internal(const std::string& key, const ItemObj& item) {
if (item == nullptr) {
return;
}
size_t item_size = item->Size();
// if key already exist, subtract old item size
if (lru_.exists(key)) {
const ItemObj& old_item = lru_.get(key);
usage_ -= old_item->Size();
}
// plus new item size
usage_ += item_size;
// if usage exceed capacity, free some items
if (usage_ > capacity_) {
LOG_SERVER_DEBUG_ << header_ << " Current usage " << (usage_ >> 20) << "MB is too high for capacity "
<< (capacity_ >> 20) << "MB, start free memory";
free_memory_internal(capacity_);
}
// insert new item
lru_.put(key, item);
LOG_SERVER_DEBUG_ << header_ << " Insert " << key << " size: " << (item_size >> 20) << "MB into cache";
LOG_SERVER_DEBUG_ << header_ << " Count: " << lru_.size() << ", Usage: " << (usage_ >> 20) << "MB, Capacity: "
<< (capacity_ >> 20) << "MB";
}
template <typename ItemObj>
void
Cache<ItemObj>::erase_internal(const std::string& key) {
if (!lru_.exists(key)) {
return;
}
const ItemObj& item = lru_.get(key);
size_t item_size = item->Size();
lru_.erase(key);
usage_ -= item_size;
LOG_SERVER_DEBUG_ << header_ << " Erase " << key << " size: " << (item_size >> 20) << "MB from cache";
LOG_SERVER_DEBUG_ << header_ << " Count: " << lru_.size() << ", Usage: " << (usage_ >> 20) << "MB, Capacity: "
<< (capacity_ >> 20) << "MB";
}
template <typename ItemObj>
void
Cache<ItemObj>::free_memory_internal(const int64_t target_size) {
int64_t threshold = std::min((int64_t)(capacity_ * freemem_percent_), target_size);
int64_t delta_size = usage_ - threshold;
if (delta_size <= 0) {
delta_size = 1; // ensure at least one item erased
}
std::set<std::string> key_array;
int64_t released_size = 0;
auto it = lru_.rbegin();
while (it != lru_.rend() && released_size < delta_size) {
auto& key = it->first;
auto& obj_ptr = it->second;
key_array.emplace(key);
released_size += obj_ptr->Size();
++it;
}
LOG_SERVER_DEBUG_ << header_ << " To be released memory size: " << (released_size >> 20) << "MB";
for (auto& key : key_array) {
erase_internal(key);
}
}
} // namespace cache
} // namespace milvus

71
internal/core/src/index/knowhere/cache/CacheMgr.h vendored Normal file
View File

@ -0,0 +1,71 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include "Cache.h"
#include "utils/Log.h"
#include <memory>
#include <string>
namespace milvus {
namespace cache {
template <typename ItemObj>
class CacheMgr {
public:
virtual uint64_t
ItemCount() const;
virtual bool
ItemExists(const std::string& key);
virtual ItemObj
GetItem(const std::string& key);
virtual void
InsertItem(const std::string& key, const ItemObj& data);
virtual void
EraseItem(const std::string& key);
virtual bool
Reserve(const int64_t size);
virtual void
PrintInfo();
virtual void
ClearCache();
int64_t
CacheUsage() const;
int64_t
CacheCapacity() const;
void
SetCapacity(int64_t capacity);
protected:
CacheMgr();
virtual ~CacheMgr();
protected:
std::shared_ptr<Cache<ItemObj>> cache_;
};
} // namespace cache
} // namespace milvus
#include "cache/CacheMgr.inl"

134
internal/core/src/index/knowhere/cache/CacheMgr.inl vendored Normal file
View File

@ -0,0 +1,134 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
namespace milvus {
namespace cache {
template <typename ItemObj>
CacheMgr<ItemObj>::CacheMgr() {
}
template <typename ItemObj>
CacheMgr<ItemObj>::~CacheMgr() {
}
template <typename ItemObj>
uint64_t
CacheMgr<ItemObj>::ItemCount() const {
if (cache_ == nullptr) {
LOG_SERVER_ERROR_ << "Cache doesn't exist";
return 0;
}
return (uint64_t)(cache_->size());
}
template <typename ItemObj>
bool
CacheMgr<ItemObj>::ItemExists(const std::string& key) {
if (cache_ == nullptr) {
LOG_SERVER_ERROR_ << "Cache doesn't exist";
return false;
}
return cache_->exists(key);
}
template <typename ItemObj>
ItemObj
CacheMgr<ItemObj>::GetItem(const std::string& key) {
if (cache_ == nullptr) {
LOG_SERVER_ERROR_ << "Cache doesn't exist";
return nullptr;
}
return cache_->get(key);
}
template <typename ItemObj>
void
CacheMgr<ItemObj>::InsertItem(const std::string& key, const ItemObj& data) {
if (cache_ == nullptr) {
LOG_SERVER_ERROR_ << "Cache doesn't exist";
return;
}
cache_->insert(key, data);
}
template <typename ItemObj>
void
CacheMgr<ItemObj>::EraseItem(const std::string& key) {
if (cache_ == nullptr) {
LOG_SERVER_ERROR_ << "Cache doesn't exist";
return;
}
cache_->erase(key);
}
template <typename ItemObj>
bool
CacheMgr<ItemObj>::Reserve(const int64_t size) {
if (cache_ == nullptr) {
LOG_SERVER_ERROR_ << "Cache doesn't exist";
return false;
}
return cache_->reserve(size);
}
template <typename ItemObj>
void
CacheMgr<ItemObj>::PrintInfo() {
if (cache_ == nullptr) {
LOG_SERVER_ERROR_ << "Cache doesn't exist";
return;
}
cache_->print();
}
template <typename ItemObj>
void
CacheMgr<ItemObj>::ClearCache() {
if (cache_ == nullptr) {
LOG_SERVER_ERROR_ << "Cache doesn't exist";
return;
}
cache_->clear();
}
template <typename ItemObj>
int64_t
CacheMgr<ItemObj>::CacheUsage() const {
if (cache_ == nullptr) {
LOG_SERVER_ERROR_ << "Cache doesn't exist";
return 0;
}
return cache_->usage();
}
template <typename ItemObj>
int64_t
CacheMgr<ItemObj>::CacheCapacity() const {
if (cache_ == nullptr) {
LOG_SERVER_ERROR_ << "Cache doesn't exist";
return 0;
}
return cache_->capacity();
}
template <typename ItemObj>
void
CacheMgr<ItemObj>::SetCapacity(int64_t capacity) {
if (cache_ == nullptr) {
LOG_SERVER_ERROR_ << "Cache doesn't exist";
return;
}
cache_->set_capacity(capacity);
}
} // namespace cache
} // namespace milvus

55
internal/core/src/index/knowhere/cache/CpuCacheMgr.cpp vendored Normal file
View File

@ -0,0 +1,55 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "cache/CpuCacheMgr.h"
#include <utility>
#include <fiu/fiu-local.h>
#include "utils/Log.h"
#include "value/config/ServerConfig.h"
namespace milvus {
namespace cache {
CpuCacheMgr&
CpuCacheMgr::GetInstance() {
static CpuCacheMgr s_mgr;
return s_mgr;
}
CpuCacheMgr::CpuCacheMgr() {
cache_ = std::make_shared<Cache<DataObjPtr>>(config.cache.cache_size(), 1UL << 32, "[CACHE CPU]");
if (config.cache.cpu_cache_threshold() > 0.0) {
cache_->set_freemem_percent(config.cache.cpu_cache_threshold());
}
ConfigMgr::GetInstance().Attach("cache.cache_size", this);
}
CpuCacheMgr::~CpuCacheMgr() {
ConfigMgr::GetInstance().Detach("cache.cache_size", this);
}
DataObjPtr
CpuCacheMgr::GetItem(const std::string& key) {
auto ret = CacheMgr<DataObjPtr>::GetItem(key);
return ret;
}
void
CpuCacheMgr::ConfigUpdate(const std::string& name) {
SetCapacity(config.cache.cache_size());
}
} // namespace cache
} // namespace milvus

44
internal/core/src/index/knowhere/cache/CpuCacheMgr.h vendored Normal file
View File

@ -0,0 +1,44 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <memory>
#include <string>
#include "cache/CacheMgr.h"
#include "cache/DataObj.h"
#include "value/config/ConfigMgr.h"
namespace milvus {
namespace cache {
class CpuCacheMgr : public CacheMgr<DataObjPtr>, public ConfigObserver {
public:
static CpuCacheMgr&
GetInstance();
private:
CpuCacheMgr();
~CpuCacheMgr();
public:
DataObjPtr
GetItem(const std::string& key) override;
public:
void
ConfigUpdate(const std::string& name) override;
};
} // namespace cache
} // namespace milvus

29
internal/core/src/index/knowhere/cache/DataObj.h vendored Normal file
View File

@ -0,0 +1,29 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <memory>
namespace milvus {
namespace cache {
class DataObj {
public:
virtual int64_t
Size() = 0;
virtual ~DataObj() = default;
};
using DataObjPtr = std::shared_ptr<DataObj>;
} // namespace cache
} // namespace milvus

63
internal/core/src/index/knowhere/cache/GpuCacheMgr.cpp vendored Normal file
View File

@ -0,0 +1,63 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "cache/GpuCacheMgr.h"
#include "utils/Log.h"
#include "value/config/ServerConfig.h"
#include <fiu/fiu-local.h>
#include <sstream>
#include <utility>
namespace milvus {
namespace cache {
#ifdef MILVUS_GPU_VERSION
std::mutex GpuCacheMgr::global_mutex_;
std::unordered_map<int64_t, GpuCacheMgrPtr> GpuCacheMgr::instance_;
GpuCacheMgr::GpuCacheMgr(int64_t gpu_id) : gpu_id_(gpu_id) {
std::string header = "[CACHE GPU" + std::to_string(gpu_id) + "]";
cache_ = std::make_shared<Cache<DataObjPtr>>(config.gpu.cache_size(), 1UL << 32, header);
if (config.gpu.cache_threshold() > 0.0) {
cache_->set_freemem_percent(config.gpu.cache_threshold());
}
ConfigMgr::GetInstance().Attach("gpu.cache_threshold", this);
}
GpuCacheMgr::~GpuCacheMgr() {
ConfigMgr::GetInstance().Detach("gpu.cache_threshold", this);
}
GpuCacheMgrPtr
GpuCacheMgr::GetInstance(int64_t gpu_id) {
if (instance_.find(gpu_id) == instance_.end()) {
std::lock_guard<std::mutex> lock(global_mutex_);
if (instance_.find(gpu_id) == instance_.end()) {
instance_[gpu_id] = std::make_shared<GpuCacheMgr>(gpu_id);
}
}
return instance_[gpu_id];
}
void
GpuCacheMgr::ConfigUpdate(const std::string& name) {
std::lock_guard<std::mutex> lock(global_mutex_);
for (auto& it : instance_) {
it.second->SetCapacity(config.gpu.cache_size());
}
}
#endif
} // namespace cache
} // namespace milvus

51
internal/core/src/index/knowhere/cache/GpuCacheMgr.h vendored Normal file
View File

@ -0,0 +1,51 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include <memory>
#include <mutex>
#include <string>
#include <unordered_map>
#include <utility>
#include "cache/CacheMgr.h"
#include "cache/DataObj.h"
#include "value/config/ConfigMgr.h"
namespace milvus {
namespace cache {
#ifdef MILVUS_GPU_VERSION
class GpuCacheMgr;
using GpuCacheMgrPtr = std::shared_ptr<GpuCacheMgr>;
using MutexPtr = std::shared_ptr<std::mutex>;
class GpuCacheMgr : public CacheMgr<DataObjPtr>, public ConfigObserver {
public:
explicit GpuCacheMgr(int64_t gpu_id);
~GpuCacheMgr();
static GpuCacheMgrPtr
GetInstance(int64_t gpu_id);
public:
void
ConfigUpdate(const std::string& name) override;
private:
int64_t gpu_id_;
static std::mutex global_mutex_;
static std::unordered_map<int64_t, GpuCacheMgrPtr> instance_;
};
#endif
} // namespace cache
} // namespace milvus

116
internal/core/src/index/knowhere/cache/LRU.h vendored Normal file
View File

@ -0,0 +1,116 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <cstddef>
#include <list>
#include <stdexcept>
#include <unordered_map>
#include <utility>
namespace milvus {
namespace cache {
template <typename key_t, typename value_t>
class LRU {
public:
typedef typename std::pair<key_t, value_t> key_value_pair_t;
typedef typename std::list<key_value_pair_t>::iterator list_iterator_t;
typedef typename std::list<key_value_pair_t>::reverse_iterator reverse_list_iterator_t;
explicit LRU(size_t max_size) : max_size_(max_size) {
}
void
put(const key_t& key, const value_t& value) {
auto it = cache_items_map_.find(key);
cache_items_list_.push_front(key_value_pair_t(key, value));
if (it != cache_items_map_.end()) {
cache_items_list_.erase(it->second);
cache_items_map_.erase(it);
}
cache_items_map_[key] = cache_items_list_.begin();
if (cache_items_map_.size() > max_size_) {
auto last = cache_items_list_.end();
last--;
cache_items_map_.erase(last->first);
cache_items_list_.pop_back();
}
}
const value_t&
get(const key_t& key) {
auto it = cache_items_map_.find(key);
if (it == cache_items_map_.end()) {
throw std::range_error("There is no such key in cache");
} else {
cache_items_list_.splice(cache_items_list_.begin(), cache_items_list_, it->second);
return it->second->second;
}
}
void
erase(const key_t& key) {
auto it = cache_items_map_.find(key);
if (it != cache_items_map_.end()) {
cache_items_list_.erase(it->second);
cache_items_map_.erase(it);
}
}
bool
exists(const key_t& key) const {
return cache_items_map_.find(key) != cache_items_map_.end();
}
size_t
size() const {
return cache_items_map_.size();
}
list_iterator_t
begin() {
iter_ = cache_items_list_.begin();
return iter_;
}
list_iterator_t
end() {
return cache_items_list_.end();
}
reverse_list_iterator_t
rbegin() {
return cache_items_list_.rbegin();
}
reverse_list_iterator_t
rend() {
return cache_items_list_.rend();
}
void
clear() {
cache_items_list_.clear();
cache_items_map_.clear();
}
private:
std::list<key_value_pair_t> cache_items_list_;
std::unordered_map<key_t, list_iterator_t> cache_items_map_;
size_t max_size_;
list_iterator_t iter_;
};
} // namespace cache
} // namespace milvus

2
internal/core/src/index/knowhere/knowhere/archive/KnowhereConfig.cpp
View File

@ -27,7 +27,7 @@
#include "utils/ConfigUtils.h"
#include "utils/Error.h"
#include "utils/Log.h"
#include "index/knowhere/knowhere/common/Exception.h"
#include "knowhere/common/Exception.h"
#include <string>
#include <vector>

119
internal/core/src/index/knowhere/log/DeprecatedLog.h Normal file
View File

@ -0,0 +1,119 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
/*
* Please use LOG_MODULE_LEVEL_C macro in member function of class
* and LOG_MODULE_LEVEL_ macro in other functions.
*/
/////////////////////////////////////////////////////////////////////////////////////////////////
#define SERVER_MODULE_NAME "SERVER"
#define SERVER_MODULE_CLASS_FUNCTION \
LogOut("[%s][%s::%s][%s] ", SERVER_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, GetThreadName().c_str())
#define SERVER_MODULE_FUNCTION LogOut("[%s][%s][%s] ", SERVER_MODULE_NAME, __FUNCTION__, GetThreadName().c_str())
#define LOG_SERVER_TRACE_C LOG(TRACE) << SERVER_MODULE_CLASS_FUNCTION
#define LOG_SERVER_DEBUG_C LOG(DEBUG) << SERVER_MODULE_CLASS_FUNCTION
#define LOG_SERVER_INFO_C LOG(INFO) << SERVER_MODULE_CLASS_FUNCTION
#define LOG_SERVER_WARNING_C LOG(WARNING) << SERVER_MODULE_CLASS_FUNCTION
#define LOG_SERVER_ERROR_C LOG(ERROR) << SERVER_MODULE_CLASS_FUNCTION
#define LOG_SERVER_FATAL_C LOG(FATAL) << SERVER_MODULE_CLASS_FUNCTION
#define LOG_SERVER_TRACE_ LOG(TRACE) << SERVER_MODULE_FUNCTION
#define LOG_SERVER_DEBUG_ LOG(DEBUG) << SERVER_MODULE_FUNCTION
#define LOG_SERVER_INFO_ LOG(INFO) << SERVER_MODULE_FUNCTION
#define LOG_SERVER_WARNING_ LOG(WARNING) << SERVER_MODULE_FUNCTION
#define LOG_SERVER_ERROR_ LOG(ERROR) << SERVER_MODULE_FUNCTION
#define LOG_SERVER_FATAL_ LOG(FATAL) << SERVER_MODULE_FUNCTION
/////////////////////////////////////////////////////////////////////////////////////////////////
#define ENGINE_MODULE_NAME "ENGINE"
#define ENGINE_MODULE_CLASS_FUNCTION \
LogOut("[%s][%s::%s][%s] ", ENGINE_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, GetThreadName().c_str())
#define ENGINE_MODULE_FUNCTION LogOut("[%s][%s][%s] ", ENGINE_MODULE_NAME, __FUNCTION__, GetThreadName().c_str())
#define LOG_ENGINE_TRACE_C LOG(TRACE) << ENGINE_MODULE_CLASS_FUNCTION
#define LOG_ENGINE_DEBUG_C LOG(DEBUG) << ENGINE_MODULE_CLASS_FUNCTION
#define LOG_ENGINE_INFO_C LOG(INFO) << ENGINE_MODULE_CLASS_FUNCTION
#define LOG_ENGINE_WARNING_C LOG(WARNING) << ENGINE_MODULE_CLASS_FUNCTION
#define LOG_ENGINE_ERROR_C LOG(ERROR) << ENGINE_MODULE_CLASS_FUNCTION
#define LOG_ENGINE_FATAL_C LOG(FATAL) << ENGINE_MODULE_CLASS_FUNCTION
#define LOG_ENGINE_TRACE_ LOG(TRACE) << ENGINE_MODULE_FUNCTION
#define LOG_ENGINE_DEBUG_ LOG(DEBUG) << ENGINE_MODULE_FUNCTION
#define LOG_ENGINE_INFO_ LOG(INFO) << ENGINE_MODULE_FUNCTION
#define LOG_ENGINE_WARNING_ LOG(WARNING) << ENGINE_MODULE_FUNCTION
#define LOG_ENGINE_ERROR_ LOG(ERROR) << ENGINE_MODULE_FUNCTION
#define LOG_ENGINE_FATAL_ LOG(FATAL) << ENGINE_MODULE_FUNCTION
/////////////////////////////////////////////////////////////////////////////////////////////////
#define WRAPPER_MODULE_NAME "WRAPPER"
#define WRAPPER_MODULE_CLASS_FUNCTION \
LogOut("[%s][%s::%s][%s] ", WRAPPER_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, GetThreadName().c_str())
#define WRAPPER_MODULE_FUNCTION LogOut("[%s][%s][%s] ", WRAPPER_MODULE_NAME, __FUNCTION__, GetThreadName().c_str())
#define LOG_WRAPPER_TRACE_C LOG(TRACE) << WRAPPER_MODULE_CLASS_FUNCTION
#define LOG_WRAPPER_DEBUG_C LOG(DEBUG) << WRAPPER_MODULE_CLASS_FUNCTION
#define LOG_WRAPPER_INFO_C LOG(INFO) << WRAPPER_MODULE_CLASS_FUNCTION
#define LOG_WRAPPER_WARNING_C LOG(WARNING) << WRAPPER_MODULE_CLASS_FUNCTION
#define LOG_WRAPPER_ERROR_C LOG(ERROR) << WRAPPER_MODULE_CLASS_FUNCTION
#define LOG_WRAPPER_FATAL_C LOG(FATAL) << WRAPPER_MODULE_CLASS_FUNCTION
#define LOG_WRAPPER_TRACE_ LOG(TRACE) << WRAPPER_MODULE_FUNCTION
#define LOG_WRAPPER_DEBUG_ LOG(DEBUG) << WRAPPER_MODULE_FUNCTION
#define LOG_WRAPPER_INFO_ LOG(INFO) << WRAPPER_MODULE_FUNCTION
#define LOG_WRAPPER_WARNING_ LOG(WARNING) << WRAPPER_MODULE_FUNCTION
#define LOG_WRAPPER_ERROR_ LOG(ERROR) << WRAPPER_MODULE_FUNCTION
#define LOG_WRAPPER_FATAL_ LOG(FATAL) << WRAPPER_MODULE_FUNCTION
/////////////////////////////////////////////////////////////////////////////////////////////////
#define STORAGE_MODULE_NAME "STORAGE"
#define STORAGE_MODULE_CLASS_FUNCTION \
LogOut("[%s][%s::%s][%s] ", STORAGE_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, GetThreadName().c_str())
#define STORAGE_MODULE_FUNCTION LogOut("[%s][%s][%s] ", STORAGE_MODULE_NAME, __FUNCTION__, GetThreadName().c_str())
#define LOG_STORAGE_TRACE_C LOG(TRACE) << STORAGE_MODULE_CLASS_FUNCTION
#define LOG_STORAGE_DEBUG_C LOG(DEBUG) << STORAGE_MODULE_CLASS_FUNCTION
#define LOG_STORAGE_INFO_C LOG(INFO) << STORAGE_MODULE_CLASS_FUNCTION
#define LOG_STORAGE_WARNING_C LOG(WARNING) << STORAGE_MODULE_CLASS_FUNCTION
#define LOG_STORAGE_ERROR_C LOG(ERROR) << STORAGE_MODULE_CLASS_FUNCTION
#define LOG_STORAGE_FATAL_C LOG(FATAL) << STORAGE_MODULE_CLASS_FUNCTION
#define LOG_STORAGE_TRACE_ LOG(TRACE) << STORAGE_MODULE_FUNCTION
#define LOG_STORAGE_DEBUG_ LOG(DEBUG) << STORAGE_MODULE_FUNCTION
#define LOG_STORAGE_INFO_ LOG(INFO) << STORAGE_MODULE_FUNCTION
#define LOG_STORAGE_WARNING_ LOG(WARNING) << STORAGE_MODULE_FUNCTION
#define LOG_STORAGE_ERROR_ LOG(ERROR) << STORAGE_MODULE_FUNCTION
#define LOG_STORAGE_FATAL_ LOG(FATAL) << STORAGE_MODULE_FUNCTION
/////////////////////////////////////////////////////////////////////////////////////////////////
#define WAL_MODULE_NAME "WAL"
#define WAL_MODULE_CLASS_FUNCTION \
LogOut("[%s][%s::%s][%s] ", WAL_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, GetThreadName().c_str())
#define WAL_MODULE_FUNCTION LogOut("[%s][%s][%s] ", WAL_MODULE_NAME, __FUNCTION__, GetThreadName().c_str())
#define LOG_WAL_TRACE_C LOG(TRACE) << WAL_MODULE_CLASS_FUNCTION
#define LOG_WAL_DEBUG_C LOG(DEBUG) << WAL_MODULE_CLASS_FUNCTION
#define LOG_WAL_INFO_C LOG(INFO) << WAL_MODULE_CLASS_FUNCTION
#define LOG_WAL_WARNING_C LOG(WARNING) << WAL_MODULE_CLASS_FUNCTION
#define LOG_WAL_ERROR_C LOG(ERROR) << WAL_MODULE_CLASS_FUNCTION
#define LOG_WAL_FATAL_C LOG(FATAL) << WAL_MODULE_CLASS_FUNCTION
#define LOG_WAL_TRACE_ LOG(TRACE) << WAL_MODULE_FUNCTION
#define LOG_WAL_DEBUG_ LOG(DEBUG) << WAL_MODULE_FUNCTION
#define LOG_WAL_INFO_ LOG(INFO) << WAL_MODULE_FUNCTION
#define LOG_WAL_WARNING_ LOG(WARNING) << WAL_MODULE_FUNCTION
#define LOG_WAL_ERROR_ LOG(ERROR) << WAL_MODULE_FUNCTION
#define LOG_WAL_FATAL_ LOG(FATAL) << WAL_MODULE_FUNCTION
/////////////////////////////////////////////////////////////////////////////////////////////////////

108
internal/core/src/index/knowhere/log/Log.cpp Normal file
View File

@ -0,0 +1,108 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "log/Log.h"
INITIALIZE_EASYLOGGINGPP
#include <chrono>
#include <cstdarg>
#include <cstdio>
#include <memory>
#include <stdexcept>
#include <string>
namespace milvus {
std::string
LogOut(const char* pattern, ...) {
size_t len = strnlen(pattern, 1024) + 256;
auto str_p = std::make_unique<char[]>(len);
memset(str_p.get(), 0, len);
va_list vl;
va_start(vl, pattern);
vsnprintf(str_p.get(), len, pattern, vl); // NOLINT
va_end(vl);
return std::string(str_p.get());
}
void
SetThreadName(const std::string& name) {
// Note: the name cannot exceed 16 bytes
pthread_setname_np(pthread_self(), name.c_str());
}
std::string
GetThreadName() {
std::string thread_name = "unamed";
char name[16];
size_t len = 16;
auto err = pthread_getname_np(pthread_self(), name, len);
if (not err) {
thread_name = name;
}
return thread_name;
}
int64_t
get_now_timestamp() {
auto now = std::chrono::system_clock::now().time_since_epoch();
return std::chrono::duration_cast<std::chrono::seconds>(now).count();
}
int64_t
get_system_boottime() {
FILE* uptime = fopen("/proc/uptime", "r");
float since_sys_boot, _;
auto ret = fscanf(uptime, "%f %f", &since_sys_boot, &_);
fclose(uptime);
if (ret != 2) {
throw std::runtime_error("read /proc/uptime failed.");
}
return static_cast<int64_t>(since_sys_boot);
}
int64_t
get_thread_starttime() {
int64_t tid = gettid();
int64_t pid = getpid();
char filename[256];
snprintf(filename, sizeof(filename), "/proc/%ld/task/%ld/stat", pid, tid);
int64_t val = 0;
char comm[16], state;
FILE* thread_stat = fopen(filename, "r");
auto ret = fscanf(thread_stat, "%ld %s %s ", &val, comm, &state);
for (auto i = 4; i < 23; i++) {
ret = fscanf(thread_stat, "%ld ", &val);
if (i == 22) {
break;
}
}
fclose(thread_stat);
if (ret != 1) {
throw std::runtime_error("read " + std::string(filename) + " failed.");
}
return val / sysconf(_SC_CLK_TCK);
}
int64_t
get_thread_start_timestamp() {
try {
return get_now_timestamp() - get_system_boottime() + get_thread_starttime();
} catch (...) {
return 0;
}
}
} // namespace milvus

77
internal/core/src/index/knowhere/log/Log.h Normal file
View File

@ -0,0 +1,77 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <string>
#include <sys/types.h>
#include <unistd.h>
#include "easyloggingpp/easylogging++.h"
#include "log/DeprecatedLog.h"
namespace milvus {
#if __GLIBC__ == 2 && __GLIBC_MINOR__ < 30
#include <sys/syscall.h>
#define gettid() syscall(SYS_gettid)
#endif
// Log message format: %timestamp | %request_id | %level | %collection_name | %client_id | %client_tag | %client_ipport
// | %thread_id | %thread_start_timestamp | %command_tag | %module | %error_code | %message
#define VAR_REQUEST_ID (context->request_id())
#define VAR_COLLECTION_NAME (context->collection_name())
#define VAR_CLIENT_ID ("")
#define VAR_CLIENT_TAG (context->client_tag())
#define VAR_CLIENT_IPPORT (context->client_ipport())
#define VAR_THREAD_ID (gettid())
#define VAR_THREAD_START_TIMESTAMP (get_thread_start_timestamp())
#define VAR_COMMAND_TAG (context->command_tag())
// Use this macro whenever possible
// Depends variables: context Context
#define MLOG(level, module, error_code) \
LOG(level) << " | " << VAR_REQUEST_ID << " | " << #level << " | " << VAR_COLLECTION_NAME << " | " << VAR_CLIENT_ID \
<< " | " << VAR_CLIENT_TAG << " | " << VAR_CLIENT_IPPORT << " | " << VAR_THREAD_ID << " | " \
<< VAR_THREAD_START_TIMESTAMP << " | " << VAR_COMMAND_TAG << " | " << #module << " | " << error_code \
<< " | "
// Use in some background process only
#define MLOG_(level, module, error_code) \
LOG(level) << " | " \
<< "" \
<< " | " << #level << " | " \
<< "" \
<< " | " \
<< "" \
<< " | " \
<< "" \
<< " | " \
<< "" \
<< " | " << VAR_THREAD_ID << " | " << VAR_THREAD_START_TIMESTAMP << " | " \
<< "" \
<< " | " << #module << " | " << error_code << " | "
std::string
LogOut(const char* pattern, ...);
void
SetThreadName(const std::string& name);
std::string
GetThreadName();
int64_t
get_thread_start_timestamp();
} // namespace milvus

262
internal/core/src/index/knowhere/log/LogMgr.cpp Normal file
View File

@ -0,0 +1,262 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include <fiu/fiu-local.h>
#include <libgen.h>
#include <cctype>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
#include <boost/filesystem.hpp>
#include "log/LogMgr.h"
#include "utils/Status.h"
#include "value/config/ServerConfig.h"
namespace milvus {
int LogMgr::trace_idx = 0;
int LogMgr::global_idx = 0;
int LogMgr::debug_idx = 0;
int LogMgr::info_idx = 0;
int LogMgr::warning_idx = 0;
int LogMgr::error_idx = 0;
int LogMgr::fatal_idx = 0;
int64_t LogMgr::logs_delete_exceeds = 1;
bool LogMgr::enable_log_delete = false;
Status
LogMgr::InitLog(bool trace_enable,
const std::string& level,
const std::string& logs_path,
const std::string& filename,
int64_t max_log_file_size,
int64_t log_rotate_num,
bool log_to_stdout,
bool log_to_file) {
try {
auto enables = parse_level(level);
enables["trace"] = trace_enable;
LogMgr log_mgr(logs_path);
log_mgr.Default()
.Filename(filename)
.Level(enables)
.To(log_to_stdout, log_to_file)
.Rotate(max_log_file_size, log_rotate_num)
.Setup();
} catch (std::exception& ex) {
return Status(SERVER_UNEXPECTED_ERROR, ex.what());
}
return Status::OK();
}
// TODO(yzb) : change the easylogging library to get the log level from parameter rather than filename
void
LogMgr::RolloutHandler(const char* filename, std::size_t size, el::Level level) {
char* dirc = strdup(filename);
char* basec = strdup(filename);
char* dir = dirname(dirc);
char* base = basename(basec);
std::string s(base);
std::vector<std::string> list = {"\\", " ", "\'", "\"", "*", "\?", "{", "}", ";", "<",
">", "|", "^", "&", "$", "#", "!", "`", "~"};
std::string::size_type position;
for (auto& substr : list) {
position = 0;
while ((position = s.find_first_of(substr, position)) != std::string::npos) {
s.insert(position, "\\");
position += 2;
}
}
std::string m(std::string(dir) + "/" + s);
try {
switch (level) {
case el::Level::Trace: {
rename_and_delete(m, ++trace_idx);
break;
}
case el::Level::Global: {
rename_and_delete(m, ++global_idx);
break;
}
case el::Level::Debug: {
rename_and_delete(m, ++debug_idx);
break;
}
case el::Level::Info: {
rename_and_delete(m, ++info_idx);
break;
}
case el::Level::Warning: {
rename_and_delete(m, ++warning_idx);
break;
}
case el::Level::Error: {
rename_and_delete(m, ++error_idx);
break;
}
case el::Level::Fatal: {
rename_and_delete(m, ++fatal_idx);
break;
}
default: {
break;
}
}
} catch (const std::exception& exc) {
std::cerr << exc.what() << ". Exception throws from RolloutHandler." << std::endl;
}
}
LogMgr::LogMgr(std::string log_path) : logs_path_(std::move(log_path)) {
}
LogMgr&
LogMgr::Default() {
el_config_.setToDefault();
el_config_.setGlobally(el::ConfigurationType::Format, "[%datetime][%level]%msg");
el_config_.setGlobally(el::ConfigurationType::SubsecondPrecision, "3");
el_config_.setGlobally(el::ConfigurationType::PerformanceTracking, "false");
return *this;
}
LogMgr&
LogMgr::Filename(const std::string& filename) {
std::string logs_reg_path = logs_path_.rfind('/') == logs_path_.length() - 1 ? logs_path_ : logs_path_ + "/";
std::string log_file = logs_reg_path + filename;
/* Set set log file at Global level to make all level log output to the same log file*/
el_config_.set(el::Level::Global, el::ConfigurationType::Filename, log_file.c_str());
return *this;
}
LogMgr&
LogMgr::Level(std::unordered_map<std::string, bool>& enables) {
fiu_do_on("LogMgr.Level.trace_enable_to_false", enables["trace"] = false);
enable(el_config_, el::Level::Trace, enables["trace"]);
fiu_do_on("LogMgr.Level.info_enable_to_false", enables["info"] = false);
enable(el_config_, el::Level::Info, enables["info"]);
fiu_do_on("LogMgr.Level.debug_enable_to_false", enables["debug"] = false);
enable(el_config_, el::Level::Debug, enables["debug"]);
fiu_do_on("LogMgr.Level.warning_enable_to_false", enables["warning"] = false);
enable(el_config_, el::Level::Warning, enables["warning"]);
fiu_do_on("LogMgr.Level.error_enable_to_false", enables["error"] = false);
enable(el_config_, el::Level::Error, enables["error"]);
fiu_do_on("LogMgr.Level.fatal_enable_to_false", enables["fatal"] = false);
enable(el_config_, el::Level::Fatal, enables["fatal"]);
return *this;
}
LogMgr&
LogMgr::To(bool log_to_stdout, bool log_to_file) {
el_config_.setGlobally(el::ConfigurationType::ToStandardOutput, (log_to_stdout ? "true" : "false"));
el_config_.setGlobally(el::ConfigurationType::ToFile, (log_to_file ? "true" : "false"));
return *this;
}
LogMgr&
LogMgr::Rotate(int64_t max_log_file_size, int64_t log_rotate_num) {
fiu_do_on("LogMgr.Rotate.set_max_log_size_small_than_min", max_log_file_size = MAX_LOG_FILE_SIZE_MIN - 1);
if (max_log_file_size < MAX_LOG_FILE_SIZE_MIN || max_log_file_size > MAX_LOG_FILE_SIZE_MAX) {
std::string msg = "max_log_file_size must in range[" + std::to_string(MAX_LOG_FILE_SIZE_MIN) + ", " +
std::to_string(MAX_LOG_FILE_SIZE_MAX) + "], now is " + std::to_string(max_log_file_size);
throw std::runtime_error(msg);
}
el_config_.setGlobally(el::ConfigurationType::MaxLogFileSize, std::to_string(max_log_file_size));
el::Loggers::addFlag(el::LoggingFlag::StrictLogFileSizeCheck);
el::Helpers::installPreRollOutCallback(LogMgr::RolloutHandler);
el::Loggers::addFlag(el::LoggingFlag::DisableApplicationAbortOnFatalLog);
// set delete_exceeds = 0 means disable throw away log file even they reach certain limit.
if (log_rotate_num != 0) {
fiu_do_on("LogMgr.Rotate.delete_exceeds_small_than_min", log_rotate_num = LOG_ROTATE_NUM_MIN - 1);
if (log_rotate_num < LOG_ROTATE_NUM_MIN || log_rotate_num > LOG_ROTATE_NUM_MAX) {
std::string msg = "log_rotate_num must in range[" + std::to_string(LOG_ROTATE_NUM_MIN) + ", " +
std::to_string(LOG_ROTATE_NUM_MAX) + "], now is " + std::to_string(log_rotate_num);
throw std::runtime_error(msg);
}
/* global variable */
enable_log_delete = true;
logs_delete_exceeds = log_rotate_num;
}
return *this;
}
void
LogMgr::Setup() {
el::Loggers::reconfigureLogger("default", el_config_);
}
void
LogMgr::rename_and_delete(const std::string& filename, int64_t idx) {
std::string target_filename = filename + "." + std::to_string(idx);
rename(filename.c_str(), target_filename.c_str());
if (enable_log_delete && idx - logs_delete_exceeds > 0) {
std::string to_delete = filename + "." + std::to_string(trace_idx - logs_delete_exceeds);
boost::filesystem::remove(to_delete);
}
}
std::unordered_map<std::string, bool>
LogMgr::parse_level(const std::string& level) {
std::unordered_map<std::string, bool> enables{
{"debug", false}, {"info", false}, {"warning", false}, {"error", false}, {"fatal", false},
};
std::unordered_map<std::string, int64_t> level_to_int{
{"debug", 5}, {"info", 4}, {"warning", 3}, {"error", 2}, {"fatal", 1},
};
switch (level_to_int[level]) {
case 5:
enables["debug"] = true;
case 4:
enables["info"] = true;
case 3:
enables["warning"] = true;
case 2:
enables["error"] = true;
case 1:
enables["fatal"] = true;
break;
default: {
std::string msg = "Cannot parse level " + level +
": invalid log level, must be one of debug, info, warning, error, fatal.";
throw std::runtime_error(msg);
}
}
return enables;
}
void
LogMgr::enable(el::Configurations& default_conf, el::Level level, bool enable) {
std::string enable_str = enable ? "true" : "false";
default_conf.set(level, el::ConfigurationType::Enabled, enable_str);
}
} // namespace milvus

95
internal/core/src/index/knowhere/log/LogMgr.h Normal file
View File

@ -0,0 +1,95 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include "easyloggingpp/easylogging++.h"
#include "utils/Status.h"
#include "value/config/ServerConfig.h"
#include <sstream>
#include <string>
#include <unordered_map>
namespace milvus {
class LogMgr {
public:
static Status
InitLog(bool trace_enable,
const std::string& level,
const std::string& logs_path,
const std::string& filename,
int64_t max_log_file_size,
int64_t delete_exceeds,
bool log_to_stdout,
bool log_to_file);
static void
RolloutHandler(const char* filename, std::size_t size, el::Level level);
private:
explicit LogMgr(std::string log_path);
LogMgr&
Default();
LogMgr&
Filename(const std::string& filename);
/* Non-const for fiu to injecting error */
LogMgr&
Level(std::unordered_map<std::string, bool>& enables);
LogMgr&
To(bool log_to_stdout, bool log_to_file);
LogMgr&
Rotate(int64_t max_log_file_size, int64_t log_rotate_num);
void
Setup();
private:
static void
rename_and_delete(const std::string& filename, int64_t idx);
static std::unordered_map<std::string, bool>
parse_level(const std::string& level);
/**
* @brief Configures if output corresponding level log
*/
static void
enable(el::Configurations& default_conf, el::Level level, bool enable);
private:
el::Configurations el_config_;
std::string logs_path_;
private:
static int trace_idx;
static int global_idx;
static int debug_idx;
static int info_idx;
static int warning_idx;
static int error_idx;
static int fatal_idx;
static int64_t logs_delete_exceeds;
static bool enable_log_delete;
const int64_t MAX_LOG_FILE_SIZE_MIN = 536870912; /* 512 MB */
const int64_t MAX_LOG_FILE_SIZE_MAX = 4294967296; /* 4 GB */
const int64_t LOG_ROTATE_NUM_MIN = 0;
const int64_t LOG_ROTATE_NUM_MAX = 1024;
};
} // namespace milvus

95
internal/core/src/index/knowhere/utils/BlockingQueue.h Normal file
View File

@ -0,0 +1,95 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <assert.h>
#include <condition_variable>
#include <iostream>
#include <queue>
#include <vector>
namespace milvus {
template <typename T>
class BlockingQueue {
public:
BlockingQueue() : mtx(), full_(), empty_() {
}
virtual ~BlockingQueue() {
}
BlockingQueue(const BlockingQueue& rhs) = delete;
BlockingQueue&
operator=(const BlockingQueue& rhs) = delete;
void
Put(const T& task) {
std::unique_lock<std::mutex> lock(mtx);
full_.wait(lock, [this] { return (queue_.size() < capacity_); });
queue_.push(task);
empty_.notify_all();
}
T
Take() {
std::unique_lock<std::mutex> lock(mtx);
empty_.wait(lock, [this] { return !queue_.empty(); });
T front(queue_.front());
queue_.pop();
full_.notify_all();
return front;
}
T
Front() {
std::unique_lock<std::mutex> lock(mtx);
empty_.wait(lock, [this] { return !queue_.empty(); });
T front(queue_.front());
return front;
}
T
Back() {
std::unique_lock<std::mutex> lock(mtx);
empty_.wait(lock, [this] { return !queue_.empty(); });
T back(queue_.back());
return back;
}
size_t
Size() const {
std::lock_guard<std::mutex> lock(mtx);
return queue_.size();
}
bool
Empty() const {
std::unique_lock<std::mutex> lock(mtx);
return queue_.empty();
}
void
SetCapacity(const size_t capacity) {
capacity_ = (capacity > 0 ? capacity : capacity_);
}
protected:
mutable std::mutex mtx;
std::condition_variable full_;
std::condition_variable empty_;
std::queue<T> queue_;
size_t capacity_ = 32;
};
} // namespace milvus

243
internal/core/src/index/knowhere/utils/CommonUtil.cpp Normal file
View File

@ -0,0 +1,243 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "utils/CommonUtil.h"
#include "utils/Log.h"
#include <dirent.h>
#include <fiu/fiu-local.h>
#include <pwd.h>
#include <sys/stat.h>
#include <unistd.h>
#include <boost/filesystem.hpp>
#include <iostream>
#include <vector>
namespace milvus {
namespace fs = boost::filesystem;
bool
CommonUtil::IsDirectoryExist(const std::string& path) {
DIR* dp = nullptr;
if ((dp = opendir(path.c_str())) == nullptr) {
return false;
}
closedir(dp);
return true;
}
Status
CommonUtil::CreateDirectory(const std::string& path) {
if (path.empty()) {
return Status::OK();
}
struct stat directory_stat;
int status = stat(path.c_str(), &directory_stat);
if (status == 0) {
return Status::OK(); // already exist
}
fs::path fs_path(path);
fs::path parent_path = fs_path.parent_path();
Status err_status = CreateDirectory(parent_path.string());
fiu_do_on("CommonUtil.CreateDirectory.create_parent_fail", err_status = Status(SERVER_INVALID_ARGUMENT, ""));
if (!err_status.ok()) {
return err_status;
}
status = stat(path.c_str(), &directory_stat);
if (status == 0) {
return Status::OK(); // already exist
}
int makeOK = mkdir(path.c_str(), S_IRWXU | S_IRGRP | S_IROTH);
fiu_do_on("CommonUtil.CreateDirectory.create_dir_fail", makeOK = 1);
if (makeOK != 0) {
return Status(SERVER_UNEXPECTED_ERROR, "failed to create directory: " + path);
}
return Status::OK();
}
namespace {
void
RemoveDirectory(const std::string& path) {
DIR* dir = nullptr;
const int32_t buf_size = 256;
char file_name[buf_size];
std::string folder_name = path + "/%s";
if ((dir = opendir(path.c_str())) != nullptr) {
struct dirent* dmsg;
while ((dmsg = readdir(dir)) != nullptr) {
if (strcmp(dmsg->d_name, ".") != 0 && strcmp(dmsg->d_name, "..") != 0) {
snprintf(file_name, buf_size, folder_name.c_str(), dmsg->d_name);
std::string tmp = file_name;
if (tmp.find('.') == std::string::npos) {
RemoveDirectory(file_name);
}
remove(file_name);
}
}
}
if (dir != nullptr) {
closedir(dir);
}
remove(path.c_str());
}
} // namespace
Status
CommonUtil::DeleteDirectory(const std::string& path) {
if (path.empty()) {
return Status::OK();
}
struct stat directory_stat;
int statOK = stat(path.c_str(), &directory_stat);
if (statOK != 0) {
return Status::OK();
}
RemoveDirectory(path);
return Status::OK();
}
bool
CommonUtil::IsFileExist(const std::string& path) {
return (access(path.c_str(), F_OK) == 0);
}
uint64_t
CommonUtil::GetFileSize(const std::string& path) {
struct stat file_info;
if (stat(path.c_str(), &file_info) < 0) {
return 0;
}
return static_cast<uint64_t>(file_info.st_size);
}
std::string
CommonUtil::GetFileName(std::string filename) {
int pos = filename.find_last_of('/');
return filename.substr(pos + 1);
}
std::string
CommonUtil::GetExePath() {
const int64_t buf_len = 1024;
char buf[buf_len];
int64_t cnt = readlink("/proc/self/exe", buf, buf_len);
fiu_do_on("CommonUtil.GetExePath.readlink_fail", cnt = -1);
if (cnt < 0 || cnt >= buf_len) {
return "";
}
buf[cnt] = '\0';
std::string exe_path = buf;
fiu_do_on("CommonUtil.GetExePath.exe_path_error", exe_path = "/");
if (exe_path.rfind('/') != exe_path.length() - 1) {
std::string sub_str = exe_path.substr(0, exe_path.rfind('/'));
return sub_str + "/";
}
return exe_path;
}
bool
CommonUtil::TimeStrToTime(const std::string& time_str,
time_t& time_integer,
tm& time_struct,
const std::string& format) {
time_integer = 0;
memset(&time_struct, 0, sizeof(tm));
int ret = sscanf(time_str.c_str(), format.c_str(), &(time_struct.tm_year), &(time_struct.tm_mon),
&(time_struct.tm_mday), &(time_struct.tm_hour), &(time_struct.tm_min), &(time_struct.tm_sec));
if (ret <= 0) {
return false;
}
time_struct.tm_year -= 1900;
time_struct.tm_mon--;
time_integer = mktime(&time_struct);
return true;
}
void
CommonUtil::GetCurrentTimeStr(std::string& time_str) {
auto t = std::time(nullptr);
struct tm ltm;
localtime_r(&t, &ltm);
time_str = "";
time_str += std::to_string(ltm.tm_year + 1900);
time_str += "-";
time_str += std::to_string(ltm.tm_mon + 1);
time_str += "-";
time_str += std::to_string(ltm.tm_mday);
time_str += "_";
time_str += std::to_string(ltm.tm_hour);
time_str += ":";
time_str += std::to_string(ltm.tm_min);
time_str += ":";
time_str += std::to_string(ltm.tm_sec);
}
void
CommonUtil::ConvertTime(time_t time_integer, tm& time_struct) {
localtime_r(&time_integer, &time_struct);
}
void
CommonUtil::ConvertTime(tm time_struct, time_t& time_integer) {
time_integer = mktime(&time_struct);
}
std::string
CommonUtil::ConvertSize(int64_t size) {
const int64_t gb = 1024ll * 1024 * 1024;
const int64_t mb = 1024ll * 1024;
const int64_t kb = 1024ll;
if (size % gb == 0) {
return std::to_string(size / gb) + "GB";
} else if (size % mb == 0) {
return std::to_string(size / mb) + "MB";
} else if (size % kb == 0) {
return std::to_string(size / kb) + "KB";
} else {
return std::to_string(size);
}
}
#ifdef ENABLE_CPU_PROFILING
std::string
CommonUtil::GetCurrentTimeStr() {
time_t tt;
time(&tt);
tt = tt + 8 * 60;
tm t;
gmtime_r(&tt, &t);
std::string str = std::to_string(t.tm_year + 1900) + "_" + std::to_string(t.tm_mon + 1) + "_" +
std::to_string(t.tm_mday) + "_" + std::to_string(t.tm_hour) + "_" + std::to_string(t.tm_min) +
"_" + std::to_string(t.tm_sec);
return str;
}
#endif
} // namespace milvus

62
internal/core/src/index/knowhere/utils/CommonUtil.h Normal file
View File

@ -0,0 +1,62 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include "utils/Status.h"
#include <time.h>
#include <string>
namespace milvus {
class CommonUtil {
public:
static bool
IsFileExist(const std::string& path);
static uint64_t
GetFileSize(const std::string& path);
static bool
IsDirectoryExist(const std::string& path);
static Status
CreateDirectory(const std::string& path);
static Status
DeleteDirectory(const std::string& path);
static std::string
GetFileName(std::string filename);
static std::string
GetExePath();
static bool
TimeStrToTime(const std::string& time_str,
time_t& time_integer,
tm& time_struct,
const std::string& format = "%d-%d-%d %d:%d:%d");
static void
GetCurrentTimeStr(std::string& time_str);
static void
ConvertTime(time_t time_integer, tm& time_struct);
static void
ConvertTime(tm time_struct, time_t& time_integer);
static std::string
ConvertSize(int64_t size);
#ifdef ENABLE_CPU_PROFILING
static std::string
GetCurrentTimeStr();
#endif
};
} // namespace milvus

320
internal/core/src/index/knowhere/utils/ConfigUtils.cpp Normal file
View File

@ -0,0 +1,320 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "utils/ConfigUtils.h"
#include "utils/Log.h"
#include "utils/StringHelpFunctions.h"
#include <arpa/inet.h>
#include <algorithm>
#include <cmath>
#ifdef MILVUS_GPU_VERSION
#include <cuda_runtime.h>
#endif
#include <fiu/fiu-local.h>
#include <sys/stat.h>
#include <sys/sysinfo.h>
#include <unistd.h>
#include <limits>
#include <regex>
#include <set>
#include <unordered_map>
#if defined(__x86_64__)
#define THREAD_MULTIPLY_CPU 1
#elif defined(__powerpc64__)
#define THREAD_MULTIPLY_CPU 4
#else
#define THREAD_MULTIPLY_CPU 1
#endif
namespace milvus {
namespace server {
std::unordered_map<std::string, int64_t> BYTE_UNITS = {
{"b", 1},
{"k", 1024},
{"m", 1024 * 1024},
{"g", 1024 * 1024 * 1024},
};
bool
is_number(const std::string& s) {
return !s.empty() && std::find_if(s.begin(), s.end(), [](unsigned char c) { return !std::isdigit(c); }) == s.end();
}
bool
is_alpha(const std::string& s) {
return !s.empty() && std::find_if(s.begin(), s.end(), [](unsigned char c) { return !std::isalpha(c); }) == s.end();
}
std::string
str_tolower(std::string s) {
std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return std::tolower(c); });
return s;
}
int64_t
parse_bytes(const std::string& str, std::string& err) {
try {
std::string s = str;
if (is_number(s)) {
return std::stoll(s);
}
if (s.length() == 0) {
return 0;
}
auto last_two = s.substr(s.length() - 2, 2);
auto last_one = s.substr(s.length() - 1);
if (is_alpha(last_two) && is_alpha(last_one)) {
if (last_one == "b" or last_one == "B") {
s = s.substr(0, s.length() - 1);
}
}
auto& units = BYTE_UNITS;
auto suffix = str_tolower(s.substr(s.length() - 1));
std::string digits_part;
if (is_number(suffix)) {
digits_part = s;
suffix = 'b';
} else {
digits_part = s.substr(0, s.length() - 1);
}
if (units.find(suffix) != units.end() or is_number(suffix)) {
auto digits = std::stoll(digits_part);
return digits * units[suffix];
} else {
std::stringstream ss;
ss << "The specified value for memory (" << str << ") should specify the units."
<< "The postfix should be one of the `b` `k` `m` `g` characters";
err = ss.str();
}
} catch (...) {
err = "Unknown error happened on parse bytes.";
}
return 0;
}
bool
GetSystemMemInfo(int64_t& total_mem, int64_t& free_mem) {
struct sysinfo info;
int ret = sysinfo(&info);
total_mem = info.totalram;
free_mem = info.freeram;
return ret == 0; // succeed 0, failed -1
}
bool
GetSystemAvailableThreads(int64_t& thread_count) {
// threadCnt = std::thread::hardware_concurrency();
thread_count = sysconf(_SC_NPROCESSORS_CONF);
thread_count *= THREAD_MULTIPLY_CPU;
fiu_do_on("GetSystemAvailableThreads.zero_thread", thread_count = 0);
if (thread_count == 0) {
thread_count = 8;
}
return true;
}
Status
ValidateGpuIndex(int32_t gpu_index) {
#ifdef MILVUS_GPU_VERSION
int num_devices = 0;
auto cuda_err = cudaGetDeviceCount(&num_devices);
fiu_do_on("config.ValidateGpuIndex.get_device_count_fail", cuda_err = cudaError::cudaErrorUnknown);
if (cuda_err != cudaSuccess) {
std::string msg = "Failed to get gpu card number, cuda error:" + std::to_string(cuda_err);
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_UNEXPECTED_ERROR, msg);
}
if (gpu_index >= num_devices) {
std::string msg = "Invalid gpu index: " + std::to_string(gpu_index);
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_ARGUMENT, msg);
}
#endif
return Status::OK();
}
#ifdef MILVUS_GPU_VERSION
Status
GetGpuMemory(int32_t gpu_index, int64_t& memory) {
fiu_return_on("config.GetGpuMemory.return_error", Status(SERVER_UNEXPECTED_ERROR, ""));
cudaDeviceProp deviceProp;
auto cuda_err = cudaGetDeviceProperties(&deviceProp, gpu_index);
if (cuda_err) {
std::string msg = "Failed to get gpu properties for gpu" + std::to_string(gpu_index) +
" , cuda error:" + std::to_string(cuda_err);
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_UNEXPECTED_ERROR, msg);
}
memory = deviceProp.totalGlobalMem;
return Status::OK();
}
#endif
Status
ValidateIpAddress(const std::string& ip_address) {
struct in_addr address;
int result = inet_pton(AF_INET, ip_address.c_str(), &address);
fiu_do_on("config.ValidateIpAddress.error_ip_result", result = 2);
switch (result) {
case 1:
return Status::OK();
case 0: {
std::string msg = "Invalid IP address: " + ip_address;
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_ARGUMENT, msg);
}
default: {
std::string msg = "IP address conversion error: " + ip_address;
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_UNEXPECTED_ERROR, msg);
}
}
}
Status
ValidateStringIsNumber(const std::string& str) {
if (str.empty() || !std::all_of(str.begin(), str.end(), ::isdigit)) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid number");
}
try {
int64_t value = std::stol(str);
fiu_do_on("config.ValidateStringIsNumber.throw_exception", throw std::exception());
if (value < 0) {
return Status(SERVER_INVALID_ARGUMENT, "Negative number");
}
} catch (...) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid number");
}
return Status::OK();
}
Status
ValidateStringIsBool(const std::string& str) {
fiu_return_on("ValidateStringNotBool", Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str));
std::string s = str;
std::transform(s.begin(), s.end(), s.begin(), ::tolower);
if (s == "true" || s == "on" || s == "yes" || s == "1" || s == "false" || s == "off" || s == "no" || s == "0" ||
s.empty()) {
return Status::OK();
}
return Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str);
}
Status
ValidateStringIsFloat(const std::string& str) {
try {
float val = std::stof(str);
if (val < 0.0) {
return Status(SERVER_INVALID_ARGUMENT, "Negative float: " + str);
}
} catch (...) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid float: " + str);
}
return Status::OK();
}
Status
ValidateDbURI(const std::string& uri) {
std::string dialectRegex = "(.*)";
std::string usernameRegex = "(.*)";
std::string passwordRegex = "(.*)";
std::string hostRegex = "(.*)";
std::string portRegex = "(.*)";
std::string dbNameRegex = "(.*)";
std::string uriRegexStr = dialectRegex + R"(\:\/\/)" + usernameRegex + R"(\:)" + passwordRegex + R"(\@)" +
hostRegex + R"(\:)" + portRegex + R"(\/)" + dbNameRegex;
std::regex uriRegex(uriRegexStr);
std::smatch pieces_match;
bool okay = true;
if (std::regex_match(uri, pieces_match, uriRegex)) {
std::string dialect = pieces_match[1].str();
std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower);
if (dialect.find("mysql") == std::string::npos && dialect.find("sqlite") == std::string::npos &&
dialect.find("mock") == std::string::npos) {
LOG_SERVER_ERROR_ << "Invalid dialect in URI: dialect = " << dialect;
okay = false;
}
/*
* Could be DNS, skip checking
*
std::string host = pieces_match[4].str();
if (!host.empty() && host != "localhost") {
if (ValidateIpAddress(host) != SERVER_SUCCESS) {
LOG_SERVER_ERROR_ << "Invalid host ip address in uri = " << host;
okay = false;
}
}
*/
std::string port = pieces_match[5].str();
if (!port.empty()) {
auto status = ValidateStringIsNumber(port);
if (!status.ok()) {
LOG_SERVER_ERROR_ << "Invalid port in uri = " << port;
okay = false;
}
}
} else {
LOG_SERVER_ERROR_ << "Wrong URI format: URI = " << uri;
okay = false;
}
return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Invalid db backend uri"));
}
Status
ValidateStoragePath(const std::string& path) {
// Validate storage path if is valid, only correct absolute path will be validated pass
// Invalid path only contain character[a-zA-Z], number[0-9], '-', and '_',
// and path must start with '/'.
// examples below are invalid
// '/a//a', '/a--/a', '/-a/a', '/a@#/a', 'aaa/sfs'
std::string path_pattern = R"(^\/(\w+-?\/?)+$)";
std::regex regex(path_pattern);
return std::regex_match(path, regex) ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Invalid file path");
}
Status
ValidateLogLevel(const std::string& level) {
std::set<std::string> supported_level{"debug", "info", "warning", "error", "fatal"};
return supported_level.find(level) != supported_level.end()
? Status::OK()
: Status(SERVER_INVALID_ARGUMENT, "Log level must be one of debug, info, warning, error and fatal.");
}
bool
IsNumber(const std::string& s) {
return !s.empty() && std::all_of(s.begin(), s.end(), ::isdigit);
}
} // namespace server
} // namespace milvus

63
internal/core/src/index/knowhere/utils/ConfigUtils.h Normal file
View File

@ -0,0 +1,63 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <string>
#include <vector>
#include "utils/Status.h"
namespace milvus {
namespace server {
extern int64_t
parse_bytes(const std::string& str, std::string& err);
extern bool
GetSystemMemInfo(int64_t& total_mem, int64_t& free_mem);
extern bool
GetSystemAvailableThreads(int64_t& thread_count);
extern Status
ValidateGpuIndex(int32_t gpu_index);
#ifdef MILVUS_GPU_VERSION
extern Status
GetGpuMemory(int32_t gpu_index, int64_t& memory);
#endif
extern Status
ValidateIpAddress(const std::string& ip_address);
extern Status
ValidateStringIsNumber(const std::string& str);
extern Status
ValidateStringIsBool(const std::string& str);
extern Status
ValidateStringIsFloat(const std::string& str);
extern Status
ValidateDbURI(const std::string& uri);
extern Status
ValidateStoragePath(const std::string& path);
extern Status
ValidateLogLevel(const std::string& level);
extern bool
IsNumber(const std::string& s);
} // namespace server
} // namespace milvus

145
internal/core/src/index/knowhere/utils/Error.h Normal file
View File

@ -0,0 +1,145 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <cstdint>
#include <exception>
#include <string>
namespace milvus {
using ErrorCode = int32_t;
constexpr ErrorCode SERVER_SUCCESS = 0;
constexpr ErrorCode SERVER_ERROR_CODE_BASE = 30000;
constexpr ErrorCode
ToServerErrorCode(const ErrorCode error_code) {
return SERVER_ERROR_CODE_BASE + error_code;
}
constexpr ErrorCode DB_SUCCESS = 0;
constexpr ErrorCode DB_ERROR_CODE_BASE = 40000;
constexpr ErrorCode
ToDbErrorCode(const ErrorCode error_code) {
return DB_ERROR_CODE_BASE + error_code;
}
constexpr ErrorCode KNOWHERE_SUCCESS = 0;
constexpr ErrorCode KNOWHERE_ERROR_CODE_BASE = 50000;
constexpr ErrorCode
ToKnowhereErrorCode(const ErrorCode error_code) {
return KNOWHERE_ERROR_CODE_BASE + error_code;
}
constexpr ErrorCode WAL_SUCCESS = 0;
constexpr ErrorCode WAL_ERROR_CODE_BASE = 60000;
constexpr ErrorCode
ToWalErrorCode(const ErrorCode error_code) {
return WAL_ERROR_CODE_BASE + error_code;
}
constexpr ErrorCode SS_SUCCESS = 0;
constexpr ErrorCode SS_ERROR_CODE_BASE = 70000;
constexpr ErrorCode
ToSSErrorCode(const ErrorCode error_code) {
return SS_ERROR_CODE_BASE + error_code;
}
// server error code
constexpr ErrorCode SERVER_UNEXPECTED_ERROR = ToServerErrorCode(1);
constexpr ErrorCode SERVER_UNSUPPORTED_ERROR = ToServerErrorCode(2);
constexpr ErrorCode SERVER_NULL_POINTER = ToServerErrorCode(3);
constexpr ErrorCode SERVER_INVALID_ARGUMENT = ToServerErrorCode(4);
constexpr ErrorCode SERVER_FILE_NOT_FOUND = ToServerErrorCode(5);
constexpr ErrorCode SERVER_NOT_IMPLEMENT = ToServerErrorCode(6);
constexpr ErrorCode SERVER_CANNOT_CREATE_FOLDER = ToServerErrorCode(8);
constexpr ErrorCode SERVER_CANNOT_CREATE_FILE = ToServerErrorCode(9);
constexpr ErrorCode SERVER_CANNOT_DELETE_FOLDER = ToServerErrorCode(10);
constexpr ErrorCode SERVER_CANNOT_DELETE_FILE = ToServerErrorCode(11);
constexpr ErrorCode SERVER_BUILD_INDEX_ERROR = ToServerErrorCode(12);
constexpr ErrorCode SERVER_CANNOT_OPEN_FILE = ToServerErrorCode(13);
constexpr ErrorCode SERVER_FILE_MAGIC_BYTES_ERROR = ToServerErrorCode(14);
constexpr ErrorCode SERVER_FILE_SUM_BYTES_ERROR = ToServerErrorCode(15);
constexpr ErrorCode SERVER_CANNOT_READ_FILE = ToServerErrorCode(16);
constexpr ErrorCode SERVER_COLLECTION_NOT_EXIST = ToServerErrorCode(100);
constexpr ErrorCode SERVER_INVALID_COLLECTION_NAME = ToServerErrorCode(101);
constexpr ErrorCode SERVER_INVALID_COLLECTION_DIMENSION = ToServerErrorCode(102);
constexpr ErrorCode SERVER_INVALID_VECTOR_DIMENSION = ToServerErrorCode(104);
constexpr ErrorCode SERVER_INVALID_INDEX_TYPE = ToServerErrorCode(105);
constexpr ErrorCode SERVER_INVALID_ROWRECORD = ToServerErrorCode(106);
constexpr ErrorCode SERVER_INVALID_ROWRECORD_ARRAY = ToServerErrorCode(107);
constexpr ErrorCode SERVER_INVALID_TOPK = ToServerErrorCode(108);
constexpr ErrorCode SERVER_ILLEGAL_VECTOR_ID = ToServerErrorCode(109);
constexpr ErrorCode SERVER_ILLEGAL_SEARCH_RESULT = ToServerErrorCode(110);
constexpr ErrorCode SERVER_CACHE_FULL = ToServerErrorCode(111);
constexpr ErrorCode SERVER_WRITE_ERROR = ToServerErrorCode(112);
constexpr ErrorCode SERVER_INVALID_NPROBE = ToServerErrorCode(113);
constexpr ErrorCode SERVER_INVALID_INDEX_NLIST = ToServerErrorCode(114);
constexpr ErrorCode SERVER_INVALID_INDEX_METRIC_TYPE = ToServerErrorCode(115);
constexpr ErrorCode SERVER_INVALID_SEGMENT_ROW_COUNT = ToServerErrorCode(116);
constexpr ErrorCode SERVER_OUT_OF_MEMORY = ToServerErrorCode(117);
constexpr ErrorCode SERVER_INVALID_PARTITION_TAG = ToServerErrorCode(118);
constexpr ErrorCode SERVER_INVALID_BINARY_QUERY = ToServerErrorCode(119);
constexpr ErrorCode SERVER_INVALID_DSL_PARAMETER = ToServerErrorCode(120);
constexpr ErrorCode SERVER_INVALID_FIELD_NAME = ToServerErrorCode(121);
constexpr ErrorCode SERVER_INVALID_FIELD_NUM = ToServerErrorCode(122);
// db error code
constexpr ErrorCode DB_META_TRANSACTION_FAILED = ToDbErrorCode(1);
constexpr ErrorCode DB_ERROR = ToDbErrorCode(2);
constexpr ErrorCode DB_NOT_FOUND = ToDbErrorCode(3);
constexpr ErrorCode DB_ALREADY_EXIST = ToDbErrorCode(4);
constexpr ErrorCode DB_INVALID_PATH = ToDbErrorCode(5);
constexpr ErrorCode DB_INCOMPATIB_META = ToDbErrorCode(6);
constexpr ErrorCode DB_INVALID_META_URI = ToDbErrorCode(7);
constexpr ErrorCode DB_EMPTY_COLLECTION = ToDbErrorCode(8);
constexpr ErrorCode DB_BLOOM_FILTER_ERROR = ToDbErrorCode(9);
constexpr ErrorCode DB_PARTITION_NOT_FOUND = ToDbErrorCode(10);
constexpr ErrorCode DB_OUT_OF_STORAGE = ToDbErrorCode(11);
constexpr ErrorCode DB_META_QUERY_FAILED = ToDbErrorCode(12);
constexpr ErrorCode DB_FILE_NOT_FOUND = ToDbErrorCode(13);
constexpr ErrorCode DB_PERMISSION_ERROR = ToDbErrorCode(14);
// knowhere error code
constexpr ErrorCode KNOWHERE_ERROR = ToKnowhereErrorCode(1);
constexpr ErrorCode KNOWHERE_INVALID_ARGUMENT = ToKnowhereErrorCode(2);
constexpr ErrorCode KNOWHERE_UNEXPECTED_ERROR = ToKnowhereErrorCode(3);
constexpr ErrorCode KNOWHERE_NO_SPACE = ToKnowhereErrorCode(4);
// knowhere error code
constexpr ErrorCode WAL_ERROR = ToWalErrorCode(1);
constexpr ErrorCode WAL_META_ERROR = ToWalErrorCode(2);
constexpr ErrorCode WAL_FILE_ERROR = ToWalErrorCode(3);
constexpr ErrorCode WAL_PATH_ERROR = ToWalErrorCode(4);
// Snapshot error code
constexpr ErrorCode SS_ERROR = ToSSErrorCode(1);
constexpr ErrorCode SS_STALE_ERROR = ToSSErrorCode(2);
constexpr ErrorCode SS_NOT_FOUND_ERROR = ToSSErrorCode(3);
constexpr ErrorCode SS_INVALID_CONTEX_ERROR = ToSSErrorCode(4);
constexpr ErrorCode SS_DUPLICATED_ERROR = ToSSErrorCode(5);
constexpr ErrorCode SS_NOT_ACTIVE_ERROR = ToSSErrorCode(6);
constexpr ErrorCode SS_CONSTRAINT_CHECK_ERROR = ToSSErrorCode(7);
constexpr ErrorCode SS_INVALID_ARGUMENT_ERROR = ToSSErrorCode(8);
constexpr ErrorCode SS_OPERATION_PENDING = ToSSErrorCode(9);
constexpr ErrorCode SS_TIMEOUT = ToSSErrorCode(10);
constexpr ErrorCode SS_NOT_COMMITED = ToSSErrorCode(11);
constexpr ErrorCode SS_COLLECTION_DROPPED = ToSSErrorCode(12);
constexpr ErrorCode SS_EMPTY_HOLDER = ToSSErrorCode(13);
} // namespace milvus

61
internal/core/src/index/knowhere/utils/Exception.h Normal file
View File

@ -0,0 +1,61 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include "utils/Error.h"
#include <exception>
#include <string>
#include <utility>
namespace milvus {
#define THROW_ERROR(err_code, err_msg) \
LOG_ENGINE_ERROR_ << err_msg; \
throw Exception(err_code, err_msg);
class Exception : public std::exception {
public:
Exception(ErrorCode code, std::string msg) : code_(code), message_(std::move(msg)) {
}
ErrorCode
code() const noexcept {
return code_;
}
const char*
what() const noexcept override {
if (message_.empty()) {
return "Default Exception.";
} else {
return message_.c_str();
}
}
~Exception() noexcept override = default;
protected:
ErrorCode code_;
std::string message_;
};
class InvalidArgumentException : public Exception {
public:
InvalidArgumentException() : Exception(SERVER_INVALID_ARGUMENT, "Invalid Argument") {
}
explicit InvalidArgumentException(const std::string& message) : Exception(SERVER_INVALID_ARGUMENT, message) {
}
};
} // namespace milvus

34
internal/core/src/index/knowhere/utils/Json.h Normal file
View File

@ -0,0 +1,34 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include "nlohmann/json.hpp"
namespace milvus {
using json = nlohmann::json;
#define JSON_NULL_CHECK(json) \
do { \
if (json.empty()) { \
return Status{SERVER_INVALID_ARGUMENT, "Json is null"}; \
} \
} while (false)
#define JSON_OBJECT_CHECK(json) \
do { \
if (!json.is_object()) { \
return Status{SERVER_INVALID_ARGUMENT, "Json is not a json object"}; \
} \
} while (false)
} // namespace milvus

14
internal/core/src/index/knowhere/utils/Log.h Normal file
View File

@ -0,0 +1,14 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include "log/Log.h"

60
internal/core/src/index/knowhere/utils/SignalHandler.cpp Normal file
View File

@ -0,0 +1,60 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "utils/SignalHandler.h"
#include "utils/Log.h"
#include <execinfo.h>
#include <string>
namespace milvus {
signal_func_ptr signal_routine_func = nullptr;
void
HandleSignal(int signum) {
int32_t exit_code = 1; /* 0: normal exit; 1: exception */
switch (signum) {
case SIGINT:
case SIGUSR2:
exit_code = 0;
/* no break */
default: {
if (exit_code == 0) {
LOG_SERVER_INFO_ << "Server received signal: " << signum;
} else {
LOG_SERVER_INFO_ << "Server received critical signal: " << signum;
PrintStacktrace();
}
if (signal_routine_func != nullptr) {
(*signal_routine_func)(exit_code);
}
}
}
}
void
PrintStacktrace() {
const int bt_depth = 128;
void* array[bt_depth];
int stack_num = backtrace(array, bt_depth);
char** stacktrace = backtrace_symbols(array, stack_num);
LOG_SERVER_INFO_ << "Call stack:";
for (int i = 0; i < stack_num; ++i) {
std::string info = stacktrace[i];
std::cout << "No." << i << ": " << info << std::endl;
LOG_SERVER_INFO_ << info;
}
free(stacktrace);
}
} // namespace milvus

28
internal/core/src/index/knowhere/utils/SignalHandler.h Normal file
View File

@ -0,0 +1,28 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <cstdint>
namespace milvus {
typedef void (*signal_func_ptr)(int32_t);
extern signal_func_ptr signal_routine_func;
extern void
HandleSignal(int signum);
extern void
PrintStacktrace();
} // namespace milvus

127
internal/core/src/index/knowhere/utils/Status.cpp Normal file
View File

@ -0,0 +1,127 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "utils/Status.h"
#include "memory"
#include <cstring>
namespace milvus {
constexpr int CODE_WIDTH = sizeof(StatusCode);
Status::Status(StatusCode code, const std::string& msg) {
// 4 bytes store code
// 4 bytes store message length
// the left bytes store message string
auto length = static_cast<uint32_t>(msg.size());
// auto result = new char[length + sizeof(length) + CODE_WIDTH];
state_.resize(length + sizeof(length) + CODE_WIDTH);
std::memcpy(state_.data(), &code, CODE_WIDTH);
std::memcpy(state_.data() + CODE_WIDTH, &length, sizeof(length));
memcpy(state_.data() + sizeof(length) + CODE_WIDTH, msg.data(), length);
}
Status::~Status() {
}
Status::Status(const Status& s) {
CopyFrom(s);
}
Status::Status(Status&& s) noexcept {
MoveFrom(s);
}
Status&
Status::operator=(const Status& s) {
CopyFrom(s);
return *this;
}
Status&
Status::operator=(Status&& s) noexcept {
MoveFrom(s);
return *this;
}
void
Status::CopyFrom(const Status& s) {
state_.clear();
if (s.state_.empty()) {
return;
}
uint32_t length = 0;
memcpy(&length, s.state_.data() + CODE_WIDTH, sizeof(length));
int buff_len = length + sizeof(length) + CODE_WIDTH;
state_.resize(buff_len);
memcpy(state_.data(), s.state_.data(), buff_len);
}
void
Status::MoveFrom(Status& s) {
state_ = s.state_;
s.state_.clear();
}
std::string
Status::message() const {
if (state_.empty()) {
return "OK";
}
std::string msg;
uint32_t length = 0;
memcpy(&length, state_.data() + CODE_WIDTH, sizeof(length));
if (length > 0) {
msg.append(state_.data() + sizeof(length) + CODE_WIDTH, length);
}
return msg;
}
std::string
Status::ToString() const {
if (state_.empty()) {
return "OK";
}
std::string result;
switch (code()) {
case DB_SUCCESS:
result = "OK ";
break;
case DB_ERROR:
result = "Error: ";
break;
case DB_META_TRANSACTION_FAILED:
result = "Database error: ";
break;
case DB_NOT_FOUND:
result = "Not found: ";
break;
case DB_ALREADY_EXIST:
result = "Already exist: ";
break;
case DB_INVALID_PATH:
result = "Invalid path: ";
break;
default:
result = "Error code(" + std::to_string(code()) + "): ";
break;
}
result += message();
return result;
}
} // namespace milvus

79
internal/core/src/index/knowhere/utils/Status.h Normal file
View File

@ -0,0 +1,79 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include "utils/Error.h"
#include <string>
namespace milvus {
class Status;
#define STATUS_CHECK(func) \
do { \
Status s = func; \
if (!s.ok()) { \
return s; \
} \
} while (false)
using StatusCode = ErrorCode;
class Status {
public:
Status(StatusCode code, const std::string& msg);
Status() = default;
virtual ~Status();
Status(const Status& s);
Status(Status&& s) noexcept;
Status&
operator=(const Status& s);
Status&
operator=(Status&& s) noexcept;
static Status
OK() {
return Status();
}
bool
ok() const {
return state_.empty() || code() == 0;
}
StatusCode
code() const {
return (state_.empty()) ? 0 : *(StatusCode*)(state_.data());
}
std::string
message() const;
std::string
ToString() const;
private:
inline void
CopyFrom(const Status& s);
inline void
MoveFrom(Status& s);
private:
std::string state_;
}; // Status
} // namespace milvus

175
internal/core/src/index/knowhere/utils/StringHelpFunctions.cpp Normal file
View File

@ -0,0 +1,175 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "utils/StringHelpFunctions.h"
#include "utils/Log.h"
#include <fiu/fiu-local.h>
#include <algorithm>
#include <regex>
#include <string>
namespace milvus {
void
StringHelpFunctions::TrimStringBlank(std::string& string) {
if (!string.empty()) {
static std::string s_format(" \n\r\t");
string.erase(0, string.find_first_not_of(s_format));
string.erase(string.find_last_not_of(s_format) + 1);
}
}
void
StringHelpFunctions::TrimStringQuote(std::string& string, const std::string& qoute) {
if (!string.empty()) {
string.erase(0, string.find_first_not_of(qoute));
string.erase(string.find_last_not_of(qoute) + 1);
}
}
void
StringHelpFunctions::SplitStringByDelimeter(const std::string& str,
const std::string& delimeter,
std::vector<std::string>& result) {
if (str.empty()) {
return;
}
size_t prev = 0;
while (true) {
size_t pos = str.find_first_of(delimeter, prev);
if (pos == std::string::npos) {
result.emplace_back(str.substr(prev));
break;
} else {
result.emplace_back(str.substr(prev, pos - prev));
prev = pos + 1;
}
}
}
void
StringHelpFunctions::MergeStringWithDelimeter(const std::vector<std::string>& strs,
const std::string& delimeter,
std::string& result) {
if (strs.empty()) {
result = "";
return;
}
result = strs[0];
for (size_t i = 1; i < strs.size(); i++) {
result = result + delimeter + strs[i];
}
}
Status
StringHelpFunctions::SplitStringByQuote(const std::string& str,
const std::string& delimeter,
const std::string& quote,
std::vector<std::string>& result) {
if (quote.empty()) {
SplitStringByDelimeter(str, delimeter, result);
return Status::OK();
}
size_t last = 0;
size_t index = str.find_first_of(quote, last);
if (index == std::string::npos) {
SplitStringByDelimeter(str, delimeter, result);
return Status::OK();
}
std::string process_str = str;
while (index != std::string::npos) {
std::string prefix = process_str.substr(last, index - last);
std::string append_prefix;
if (!prefix.empty()) {
std::vector<std::string> prefix_split;
SplitStringByDelimeter(prefix, delimeter, prefix_split);
for (size_t i = 0; i < prefix_split.size() - 1; i++) {
result.push_back(prefix_split[i]);
}
append_prefix = prefix_split[prefix_split.size() - 1];
}
last = index + 1;
std::string postfix = process_str.substr(last);
index = postfix.find_first_of(quote, 0);
fiu_do_on("StringHelpFunctions.SplitStringByQuote.invalid_index", index = std::string::npos);
if (index == std::string::npos) {
return Status(SERVER_UNEXPECTED_ERROR, "");
}
std::string quoted_text = postfix.substr(0, index);
append_prefix += quoted_text;
last = index + 1;
index = postfix.find_first_of(delimeter, last);
fiu_do_on("StringHelpFunctions.SplitStringByQuote.index_gt_last", last = 0);
fiu_do_on("StringHelpFunctions.SplitStringByQuote.invalid_index2", index = std::string::npos);
if (index != std::string::npos) {
if (index > last) {
append_prefix += postfix.substr(last, index - last);
}
} else {
append_prefix += postfix.substr(last);
}
result.emplace_back(append_prefix);
fiu_do_on("StringHelpFunctions.SplitStringByQuote.last_is_end", last = postfix.length());
if (last == postfix.length()) {
return Status::OK();
}
process_str = postfix.substr(index + 1);
last = 0;
index = process_str.find_first_of(quote, last);
}
if (!process_str.empty()) {
SplitStringByDelimeter(process_str, delimeter, result);
}
return Status::OK();
}
bool
StringHelpFunctions::IsRegexMatch(const std::string& target_str, const std::string& pattern_str) {
// if target_str equals pattern_str, return true
if (target_str == pattern_str) {
return true;
}
// regex match
// for illegal regex expression, the std::regex will throw exception, regard as unmatch
try {
std::regex pattern(pattern_str);
std::smatch results;
return std::regex_match(target_str, results, pattern);
} catch (std::exception& e) {
LOG_SERVER_ERROR_ << "Regex exception: " << e.what();
}
return false;
}
Status
StringHelpFunctions::ConvertToBoolean(const std::string& str, bool& value) {
std::string s = str;
std::transform(s.begin(), s.end(), s.begin(), ::tolower);
value = s == "true" || s == "on" || s == "yes" || s == "1";
return Status::OK();
}
} // namespace milvus

76
internal/core/src/index/knowhere/utils/StringHelpFunctions.h Normal file
View File

@ -0,0 +1,76 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include "utils/Status.h"
#include <string>
#include <vector>
namespace milvus {
class StringHelpFunctions {
private:
StringHelpFunctions() = default;
public:
// trim blanks from begin and end
// " a b c " => "a b c"
static void
TrimStringBlank(std::string& string);
// trim quotes from begin and end
// "'abc'" => "abc"
static void
TrimStringQuote(std::string& string, const std::string& qoute);
// split string by delimeter ','
// a,b,c a | b | c
// a,b, a | b |
// ,b,c | b | c
// ,b, | b |
// ,, | |
// a a
static void
SplitStringByDelimeter(const std::string& str, const std::string& delimeter, std::vector<std::string>& result);
// merge strings with delimeter
// "a", "b", "c" => "a,b,c"
static void
MergeStringWithDelimeter(const std::vector<std::string>& strs, const std::string& delimeter, std::string& result);
// assume the collection has two columns, quote='\"', delimeter=','
// a,b a | b
// "aa,gg,yy",b aa,gg,yy | b
// aa"dd,rr"kk,pp aadd,rrkk | pp
// "aa,bb" aa,bb
// 55,1122\"aa,bb\",yyy,\"kkk\" 55 | 1122aa,bb | yyy | kkk
// "55,1122"aa,bb",yyy,"kkk" illegal
static Status
SplitStringByQuote(const std::string& str,
const std::string& delimeter,
const std::string& quote,
std::vector<std::string>& result);
// std regex match function
// regex grammar reference: http://www.cplusplus.com/reference/regex/ECMAScript/
static bool
IsRegexMatch(const std::string& target_str, const std::string& pattern);
// conversion rules refer to ValidationUtil::ValidateStringIsBool()
// "true", "on", "yes", "1" ==> true
// "false", "off", "no", "0", "" ==> false
static Status
ConvertToBoolean(const std::string& str, bool& value);
};
} // namespace milvus

128
internal/core/src/index/knowhere/utils/ThreadPool.h Normal file
View File

@ -0,0 +1,128 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <fiu/fiu-local.h>
#include <atomic>
#include <condition_variable>
#include <functional>
#include <future>
#include <memory>
#include <mutex>
#include <queue>
#include <stdexcept>
#include <thread>
#include <utility>
#include <vector>
#define MAX_THREADS_NUM 32
namespace milvus {
class ThreadPool {
public:
explicit ThreadPool(size_t threads, size_t queue_size = 1000);
template <class F, class... Args>
auto
enqueue(F&& f, Args&&... args) -> std::future<typename std::result_of<F(Args...)>::type>;
void
Stop();
~ThreadPool();
private:
// need to keep track of threads so we can join them
std::vector<std::thread> workers_;
// the task queue
std::queue<std::function<void()>> tasks_;
size_t max_queue_size_;
// synchronization
std::mutex queue_mutex_;
std::condition_variable condition_;
std::atomic_bool stop_;
};
// the constructor just launches some amount of workers
inline ThreadPool::ThreadPool(size_t threads, size_t queue_size) : max_queue_size_(queue_size), stop_(false) {
for (size_t i = 0; i < threads; ++i)
workers_.emplace_back([this] {
for (;;) {
std::function<void()> task;
{
std::unique_lock<std::mutex> lock(this->queue_mutex_);
this->condition_.wait(lock, [this] { return this->stop_ || !this->tasks_.empty(); });
if (this->stop_ && this->tasks_.empty())
return;
task = std::move(this->tasks_.front());
this->tasks_.pop();
}
this->condition_.notify_all();
task();
}
});
}
// add new work item to the pool
template <class F, class... Args>
auto
ThreadPool::enqueue(F&& f, Args&&... args) -> std::future<typename std::result_of<F(Args...)>::type> {
using return_type = typename std::result_of<F(Args...)>::type;
auto task =
std::make_shared<std::packaged_task<return_type()>>(std::bind(std::forward<F>(f), std::forward<Args>(args)...));
fiu_do_on("ThreadPool.enqueue.stop_is_true", stop_ = true);
std::future<return_type> res = task->get_future();
{
std::unique_lock<std::mutex> lock(queue_mutex_);
this->condition_.wait(lock, [this] { return this->tasks_.size() < max_queue_size_; });
// don't allow enqueueing after stopping the pool
if (stop_)
throw std::runtime_error("enqueue on stopped ThreadPool");
tasks_.emplace([task]() { (*task)(); });
}
condition_.notify_all();
return res;
}
inline void
ThreadPool::Stop() {
if (stop_) {
return;
}
{
std::unique_lock<std::mutex> lock(queue_mutex_);
stop_ = true;
}
condition_.notify_all();
for (std::thread& worker : workers_) {
worker.join();
}
}
// the destructor joins all threads
inline ThreadPool::~ThreadPool() {
Stop();
}
using ThreadPoolPtr = std::shared_ptr<ThreadPool>;
} // namespace milvus

93
internal/core/src/index/knowhere/utils/TimeRecorder.cpp Normal file
View File

@ -0,0 +1,93 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include <utility>
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
namespace milvus {
TimeRecorder::TimeRecorder(std::string hdr, int64_t log_level) : header_(std::move(hdr)), log_level_(log_level) {
start_ = last_ = stdclock::now();
}
std::string
TimeRecorder::GetTimeSpanStr(double span) {
std::string str_sec = std::to_string(span * 0.000001) + ((span > 1000000) ? " seconds" : " second");
std::string str_ms = std::to_string(span * 0.001) + " ms";
return str_sec + " [" + str_ms + "]";
}
void
TimeRecorder::PrintTimeRecord(const std::string& msg, double span) {
std::string str_log;
if (!header_.empty()) {
str_log += header_ + ": ";
}
str_log += msg;
str_log += " (";
str_log += TimeRecorder::GetTimeSpanStr(span);
str_log += ")";
switch (log_level_) {
case 0:
LOG_SERVER_TRACE_ << str_log;
break;
case 1:
LOG_SERVER_DEBUG_ << str_log;
break;
case 2:
LOG_SERVER_INFO_ << str_log;
break;
case 3:
LOG_SERVER_WARNING_ << str_log;
break;
case 4:
LOG_SERVER_ERROR_ << str_log;
break;
case 5:
LOG_SERVER_FATAL_ << str_log;
break;
default:
LOG_SERVER_INFO_ << str_log;
break;
}
}
double
TimeRecorder::RecordSection(const std::string& msg) {
stdclock::time_point curr = stdclock::now();
double span = (std::chrono::duration<double, std::micro>(curr - last_)).count();
last_ = curr;
PrintTimeRecord(msg, span);
return span;
}
double
TimeRecorder::ElapseFromBegin(const std::string& msg) {
stdclock::time_point curr = stdclock::now();
double span = (std::chrono::duration<double, std::micro>(curr - start_)).count();
PrintTimeRecord(msg, span);
return span;
}
TimeRecorderAuto::TimeRecorderAuto(std::string hdr, int64_t log_level) : TimeRecorder(hdr, log_level) {
}
TimeRecorderAuto::~TimeRecorderAuto() {
ElapseFromBegin("totally cost");
}
} // namespace milvus

65
internal/core/src/index/knowhere/utils/TimeRecorder.h Normal file
View File

@ -0,0 +1,65 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <chrono>
#include <string>
#include "utils/Log.h"
namespace milvus {
inline void
print_timestamp(const std::string& message) {
std::chrono::time_point<std::chrono::system_clock> now = std::chrono::system_clock::now();
auto duration = now.time_since_epoch();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>(duration).count();
micros %= 1000000;
double millisecond = (double)micros / 1000.0;
LOG_SERVER_DEBUG_ << std::fixed << " " << millisecond << "(ms) [timestamp]" << message;
}
class TimeRecorder {
using stdclock = std::chrono::high_resolution_clock;
public:
// trace = 0, debug = 1, info = 2, warn = 3, error = 4, critical = 5
explicit TimeRecorder(std::string hdr, int64_t log_level = 1);
virtual ~TimeRecorder() = default;
double
RecordSection(const std::string& msg);
double
ElapseFromBegin(const std::string& msg);
static std::string
GetTimeSpanStr(double span);
private:
void
PrintTimeRecord(const std::string& msg, double span);
private:
std::string header_;
stdclock::time_point start_;
stdclock::time_point last_;
int64_t log_level_;
};
class TimeRecorderAuto : public TimeRecorder {
public:
explicit TimeRecorderAuto(std::string hdr, int64_t log_level = 1);
~TimeRecorderAuto() override;
};
} // namespace milvus

76
internal/core/src/index/knowhere/utils/TimerContext.h Normal file
View File

@ -0,0 +1,76 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <boost/asio.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <functional>
#include <memory>
#include <string>
#include <vector>
#include "utils/Log.h"
#include "utils/ThreadPool.h"
namespace milvus {
struct TimerContext {
using HandlerT = std::function<void(const boost::system::error_code&)>;
struct Context {
/* Context(int interval_us, HandlerT& handler, ThreadPoolPtr pool = nullptr) */
/* : interval_(interval_us), handler_(handler), timer_(io, interval_), pool_(pool) { */
/* } */
int interval_us;
HandlerT handler;
ThreadPoolPtr pool = nullptr;
};
TimerContext(boost::asio::io_service& io, int interval_us, HandlerT& handler, ThreadPoolPtr pool)
: io_(io), interval_(interval_us), handler_(handler), timer_(io, interval_), pool_(pool) {
}
TimerContext(boost::asio::io_service& io, Context& context)
: io_(io),
interval_(context.interval_us),
handler_(context.handler),
timer_(io, interval_),
pool_(context.pool) {
}
void
Reschedule(const boost::system::error_code& ec);
boost::asio::io_service& io_;
boost::posix_time::microseconds interval_;
boost::asio::deadline_timer timer_;
HandlerT handler_;
ThreadPoolPtr pool_;
};
inline void
TimerContext::Reschedule(const boost::system::error_code& ec) {
try {
pool_->enqueue(handler_, ec);
} catch (std::exception& ex) {
LOG_SERVER_ERROR_ << "Fail to enqueue handler: " << std::string(ex.what());
}
boost::system::error_code e;
auto new_expires = timer_.expires_at() + interval_;
timer_.expires_at(new_expires, e);
if (e) {
LOG_SERVER_ERROR_ << "Fail to Reschedule: " << e;
}
timer_.async_wait(std::bind(&TimerContext::Reschedule, this, std::placeholders::_1));
}
using TimerContextPtr = std::shared_ptr<TimerContext>;
} // namespace milvus

85
internal/core/src/index/knowhere/utils/TimerManager.cpp Normal file
View File

@ -0,0 +1,85 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include <utility>
#include "utils/Log.h"
#include "utils/TimerManager.h"
namespace milvus {
TimerManager::TimerManager(unsigned int pool_size) : pool_size_(pool_size) {
}
TimerManager::~TimerManager() {
}
Status
TimerManager::SetPoolSize(unsigned int pool_size) {
if (timer_exeutors_) {
return Status(SERVER_UNEXPECTED_ERROR, "Cannot set pool size since it has been set already");
}
pool_size_ = pool_size;
return Status::OK();
}
Status
TimerManager::Run() {
boost::system::error_code ec;
io_.run(ec);
Status status;
if (ec) {
status = Status(SERVER_UNEXPECTED_ERROR, ec.message());
}
return status;
}
Status
TimerManager::Start() {
for (auto& timer : timers_) {
timer->timer_.async_wait(std::bind(&TimerContext::Reschedule, timer, std::placeholders::_1));
}
return Status::OK();
}
void
TimerManager::Stop() {
boost::system::error_code ec;
for (auto& timer : timers_) {
timer->timer_.cancel(ec);
if (ec) {
LOG_SERVER_ERROR_ << "Fail to cancel timer: " << ec;
}
}
if (timer_exeutors_) {
timer_exeutors_->Stop();
}
}
void
TimerManager::AddTimer(int interval_us, TimerContext::HandlerT handler) {
if (!timer_exeutors_) {
timer_exeutors_ = std::make_shared<ThreadPool>(pool_size_);
}
timers_.emplace_back(std::make_shared<TimerContext>(io_, interval_us, handler, timer_exeutors_));
}
void
TimerManager::AddTimer(const TimerContext::Context& ctx) {
if (!timer_exeutors_) {
timer_exeutors_ = std::make_shared<ThreadPool>(pool_size_);
}
TimerContext::Context context(ctx);
context.pool = timer_exeutors_;
timers_.emplace_back(std::make_shared<TimerContext>(io_, context));
}
} // namespace milvus

57
internal/core/src/index/knowhere/utils/TimerManager.h Normal file
View File

@ -0,0 +1,57 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <boost/asio.hpp>
#include <functional>
#include <memory>
#include <vector>
#include "utils/Status.h"
#include "utils/ThreadPool.h"
#include "utils/TimerContext.h"
namespace milvus {
class TimerManager {
public:
TimerManager() = default;
explicit TimerManager(unsigned int pool_size);
Status
SetPoolSize(unsigned int pool_size);
void
AddTimer(int interval_us, TimerContext::HandlerT handler);
void
AddTimer(const TimerContext::Context& ctx);
virtual Status
Run();
virtual Status
Start();
virtual void
Stop();
virtual ~TimerManager();
protected:
boost::asio::io_service io_;
ThreadPoolPtr timer_exeutors_;
unsigned int pool_size_;
std::vector<TimerContextPtr> timers_;
};
using TimerManagerPtr = std::shared_ptr<TimerManager>;
} // namespace milvus

12
internal/core/src/index/knowhere/utils/Types.cpp Normal file
View File

@ -0,0 +1,12 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#include "Types.h"

51
internal/core/src/index/knowhere/utils/Types.h Normal file
View File

@ -0,0 +1,51 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
// #include <faiss/Index.h>
#include <cstdint>
#include <map>
#include <memory>
#include <set>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
namespace milvus::engine {
using idx_t = int64_t;
using offset_t = int32_t;
using date_t = int32_t;
using distance_t = float;
using IDNumbers = std::vector<idx_t>;
enum class DataType {
NONE = 0,
BOOL = 1,
INT8 = 2,
INT16 = 3,
INT32 = 4,
INT64 = 5,
FLOAT = 10,
DOUBLE = 11,
STRING = 20,
VECTOR_BINARY = 100,
VECTOR_FLOAT = 101,
};
} // namespace milvus::engine

30
internal/core/src/index/knowhere/utils/tools.h Normal file
View File

@ -0,0 +1,30 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#pragma once
#include <cstdint>
#include "exceptions/EasyAssert.h"
namespace milvus {
inline int64_t
upper_align(int64_t value, int64_t align) {
Assert(align > 0);
auto groups = (value + align - 1) / align;
return groups * align;
}
inline int64_t
upper_div(int64_t value, int64_t align) {
Assert(align > 0);
auto groups = (value + align - 1) / align;
return groups;
}
} // namespace milvus

179
internal/core/src/index/thirdparty/CMakeLists.txt vendored Normal file
View File

@ -0,0 +1,179 @@
add_compile_options( -O3 -fPIC -Wno-error -fopenmp )
if ( NOT KNOWHERE_VERBOSE_THIRDPARTY_BUILD )
set( EP_LOG_OPTIONS LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1 LOG_DOWNLOAD 1 )
else ()
set( EP_LOG_OPTIONS )
endif ()
set( MAKE_BUILD_ARGS "-j6" )
include( FetchContent )
set( FETCHCONTENT_BASE_DIR ${INDEX_BINARY_DIR}/3rdparty_download )
set( FETCHCONTENT_QUIET OFF )
if( CUSTOM_THIRDPARTY_DOWNLOAD_PATH )
set( THIRDPARTY_DOWNLOAD_PATH ${CUSTOM_THIRDPARTY_DOWNLOAD_PATH} )
else()
set( THIRDPARTY_DOWNLOAD_PATH ${CMAKE_BINARY_DIR}/3rdparty_download/download )
endif()
message( STATUS "Thirdparty downloaded file path: ${THIRDPARTY_DOWNLOAD_PATH}" )
# ----------------------------------------------------------------------
# Find pthreads
set( THREADS_PREFER_PTHREAD_FLAG ON )
find_package( Threads REQUIRED )
# ****************************** Thirdparty googletest ***************************************
if ( KNOWHERE_BUILD_TESTS )
if(NOT TARGET gtest::gtest_main)
add_subdirectory( gtest )
endif()
if(NOT TARGET benchmark::benchmark_main)
add_subdirectory( google_benchmark )
endif()
endif()
if(NOT TARGET fiu)
add_subdirectory(fiu)
endif()
# ****************************** Thirdparty NGT ***************************************
add_subdirectory(NGT)
# ****************************** Thirdparty Faiss ***************************************
set(FAISS_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/faiss)
macro(build_faiss)
message(STATUS "Building FAISS-${FAISS_VERSION} from source")
set(FAISS_PREFIX "${INDEX_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep")
set(FAISS_INCLUDE_DIR "${FAISS_PREFIX}/include")
set(FAISS_STATIC_LIB
"${FAISS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX}")
if (CCACHE_FOUND)
set(FAISS_C_COMPILER "${CCACHE_FOUND} ${CMAKE_C_COMPILER}")
if (MILVUS_GPU_VERSION)
set(FAISS_CXX_COMPILER "${CMAKE_CXX_COMPILER}")
set(FAISS_CUDA_COMPILER "${CCACHE_FOUND} ${CMAKE_CUDA_COMPILER}")
else ()
set(FAISS_CXX_COMPILER "${CCACHE_FOUND} ${CMAKE_CXX_COMPILER}")
endif()
else ()
set(FAISS_C_COMPILER "${CMAKE_C_COMPILER}")
set(FAISS_CXX_COMPILER "${CMAKE_CXX_COMPILER}")
endif()
set(FAISS_CONFIGURE_ARGS
"--prefix=${FAISS_PREFIX}"
"CC=${FAISS_C_COMPILER}"
"CXX=${FAISS_CXX_COMPILER}"
"NVCC=${FAISS_CUDA_COMPILER}"
"CFLAGS=${EP_C_FLAGS}"
"CXXFLAGS=${EP_CXX_FLAGS} -mf16c -O3"
--without-python)
message(STATUS "${FAISS_CONFIGURE_ARGS}")
message(STATUS "Build Faiss with OpenBlas/LAPACK")
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
"LDFLAGS=-L${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}"
)
if (MILVUS_GPU_VERSION)
if (NOT MILVUS_CUDA_ARCH OR MILVUS_CUDA_ARCH STREQUAL "DEFAULT")
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
"--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}"
"--with-cuda-arch=-gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75"
)
else()
STRING(REPLACE ";" " " MILVUS_CUDA_ARCH "${MILVUS_CUDA_ARCH}")
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
"--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}"
"--with-cuda-arch=${MILVUS_CUDA_ARCH}"
)
endif ()
else ()
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
"CPPFLAGS=-DUSE_CPU"
--without-cuda)
endif ()
message(STATUS "Building FAISS with configure args -${FAISS_CONFIGURE_ARGS}")
if (DEFINED ENV{FAISS_SOURCE_URL})
set(FAISS_SOURCE_URL "$ENV{FAISS_SOURCE_URL}")
externalproject_add(faiss_ep
URL
${FAISS_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${FAISS_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS} all
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE} install
BUILD_BYPRODUCTS
${FAISS_STATIC_LIB})
else ()
externalproject_add(faiss_ep
DOWNLOAD_COMMAND
""
SOURCE_DIR
${FAISS_SOURCE_DIR}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${FAISS_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS} all
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE} install
BUILD_BYPRODUCTS
${FAISS_STATIC_LIB})
endif ()
if(NOT BLAS_FOUND)
message(STATUS "OpenBLAS BOUNDED")
ExternalProject_Add_StepDependencies(faiss_ep configure openblas_ep)
endif()
file(MAKE_DIRECTORY "${FAISS_INCLUDE_DIR}")
add_library(faiss STATIC IMPORTED)
set_target_properties( faiss
PROPERTIES
IMPORTED_GLOBAL TRUE
IMPORTED_LOCATION "${FAISS_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}")
target_link_libraries( faiss INTERFACE openblas )
add_dependencies(faiss faiss_ep)
endmacro()
if (KNOWHERE_WITH_FAISS AND NOT TARGET faiss_ep)
if (FAISS_SOURCE STREQUAL "AUTO")
find_package(FAISS MODULE)
if (NOT FAISS_FOUND})
build_faiss()
endif ()
elseif (FAISS_SOURCE STREQUAL "BUNDLED")
build_faiss()
elseif (FAISS_SOURCE STREQUAL "SYSTEM")
find_package(FAISS REQUIRED)
endif ()
get_target_property(FAISS_INCLUDE_DIR faiss INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM "${FAISS_INCLUDE_DIR}")
link_directories(SYSTEM ${FAISS_PREFIX}/lib/)
endif ()

86
internal/core/src/index/thirdparty/OpenBLAS/CMakeLists.txt vendored Normal file
View File

@ -0,0 +1,86 @@
if (DEFINED ENV{KNOWHERE_OPENBLAS_URL})
set(OPENBLAS_SOURCE_URL "$ENV{KNOWHERE_OPENBLAS_URL}")
else ()
set(OPENBLAS_SOURCE_URL
"https://github.com.cnpmjs.org/xianyi/OpenBLAS/archive/v${OPENBLAS_VERSION}.tar.gz")
endif ()
set(OPENBLAS_PREFIX "${INDEX_BINARY_DIR}/openblas_ep-prefix/src/openblas_ep")
macro(build_openblas)
message(STATUS "Building OpenBLAS-${OPENBLAS_VERSION} from source")
set(OpenBLAS_INCLUDE_DIR "${OPENBLAS_PREFIX}/include")
set(OpenBLAS_LIB_DIR "${OPENBLAS_PREFIX}/lib")
set(OPENBLAS_SHARED_LIB
"${OPENBLAS_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas${CMAKE_SHARED_LIBRARY_SUFFIX}")
set(OPENBLAS_STATIC_LIB
"${OPENBLAS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}openblas${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(OPENBLAS_CMAKE_ARGS
${EP_COMMON_CMAKE_ARGS}
-DCMAKE_BUILD_TYPE=Release
-DBUILD_SHARED_LIBS=ON
-DBUILD_STATIC_LIBS=ON
-DTARGET=CORE2
-DDYNAMIC_ARCH=1
-DDYNAMIC_OLDER=1
-DUSE_THREAD=0
-DUSE_OPENMP=OPENBLAS_SOURCE_URL
-DFC=gfortran
-DCC=gcc
-DINTERFACE64=0
-DNUM_THREADS=128
-DNO_LAPACKE=1
"-DVERSION=${OPENBLAS_VERSION}"
"-DCMAKE_INSTALL_PREFIX=${OPENBLAS_PREFIX}"
-DCMAKE_INSTALL_LIBDIR=lib)
externalproject_add(openblas_ep
URL
${OPENBLAS_SOURCE_URL}
URL_MD5
"28cc19a6acbf636f5aab5f10b9a0dfe1"
${EP_LOG_OPTIONS}
CMAKE_ARGS
${OPENBLAS_CMAKE_ARGS}
BUILD_COMMAND
${MAKE}
${MAKE_BUILD_ARGS}
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE}
PREFIX=${OPENBLAS_PREFIX}
install
BUILD_BYPRODUCTS
${OPENBLAS_SHARED_LIB}
${OPENBLAS_STATIC_LIB})
file(MAKE_DIRECTORY "${OpenBLAS_INCLUDE_DIR}")
add_library(openblas SHARED IMPORTED)
set_target_properties(
openblas
PROPERTIES
IMPORTED_LOCATION "${OPENBLAS_SHARED_LIB}"
LIBRARY_OUTPUT_NAME "openblas"
INTERFACE_INCLUDE_DIRECTORIES "${OpenBLAS_INCLUDE_DIR}")
add_dependencies(openblas openblas_ep)
get_target_property(OpenBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES)
set(OpenBLAS_LIBRARIES "${OPENBLAS_SHARED_LIB}")
endmacro()
if (KNOWHERE_WITH_OPENBLAS)
if (OpenBLAS_SOURCE STREQUAL "AUTO")
find_package(OpenBLAS MODULE)
if (NOT ${OpenBLAS_FOUND})
build_openblas()
endif ()
elseif (OpenBLAS_SOURCE STREQUAL "BUNDLED")
build_openblas()
elseif (OpenBLAS_SOURCE STREQUAL "SYSTEM")
find_package(OpenBLAS REQUIRED)
endif ()
include_directories(SYSTEM "${OpenBLAS_INCLUDE_DIR}")
link_directories(SYSTEM "${OpenBLAS_LIB_DIR}")
endif()

3299
internal/core/src/index/thirdparty/easyloggingpp/easylogging++.cc vendored Normal file
View File

File diff suppressed because it is too large Load Diff

5193
internal/core/src/index/thirdparty/easyloggingpp/easylogging++.h vendored Normal file
View File

File diff suppressed because it is too large Load Diff

61
internal/core/src/index/thirdparty/fiu/CMakeLists.txt vendored Normal file
View File

@ -0,0 +1,61 @@
#-------------------------------------------------------------------------------
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
#-------------------------------------------------------------------------------
if(NOT DEFINED FIU_VERSION)
set(FIU_VERSION 1.00)
endif()
if ( DEFINED ENV{Knowhere_FIU_URL} )
set( FIU_SOURCE_URL "$ENV{MILVUS_FIU_URL}" )
else ()
set( FIU_SOURCE_URL "https://github.com.cnpmjs.org/albertito/libfiu/archive/${FIU_VERSION}.tar.gz" )
endif ()
macro( build_fiu )
message( STATUS "Building FIU-${FIU_VERSION} from source" )
ExternalProject_Add(
fiu_ep
PREFIX ${CMAKE_BINARY_DIR}/3rdparty_download/fiu-subbuild
DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH}
INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}
URL ${FIU_SOURCE_URL}
URL_MD5 "75f9d076daf964c9410611701f07c61b"
CONFIGURE_COMMAND ""
BUILD_IN_SOURCE 1
BUILD_COMMAND ${MAKE}
INSTALL_COMMAND ${MAKE} "PREFIX=<INSTALL_DIR>" install
${EP_LOG_OPTIONS}
)
ExternalProject_Get_Property( fiu_ep INSTALL_DIR )
if( NOT IS_DIRECTORY ${INSTALL_DIR}/include )
file( MAKE_DIRECTORY "${INSTALL_DIR}/include" )
endif()
add_library( fiu SHARED IMPORTED )
set_target_properties( fiu
PROPERTIES
IMPORTED_GLOBAL TRUE
IMPORTED_LOCATION ${INSTALL_DIR}/lib/libfiu.so
INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include )
add_dependencies(fiu fiu_ep)
endmacro()
build_fiu()
install( FILES ${INSTALL_DIR}/lib/libfiu.so
${INSTALL_DIR}/lib/libfiu.so.0
${INSTALL_DIR}/lib/libfiu.so.1.00
DESTINATION lib )
get_target_property( var fiu INTERFACE_INCLUDE_DIRECTORIES )
message( STATUS ${var} )
set_directory_properties( PROPERTY INCLUDE_DIRECTORIES ${var} )

37
internal/core/src/index/thirdparty/fiu/fiu-local.h vendored Normal file
View File

@ -0,0 +1,37 @@
/* libfiu - Fault Injection in Userspace
*
* This header, part of libfiu, is meant to be included in your project to
* avoid having libfiu as a mandatory build-time dependency.
*
* You can add it to your project, and #include it instead of fiu.h.
* The real fiu.h will be used only when FIU_ENABLE is defined.
*
* This header, as the rest of libfiu, is in the public domain.
*
* You can find more information about libfiu at
* http://blitiri.com.ar/p/libfiu.
*/
#ifndef _FIU_LOCAL_H
#define _FIU_LOCAL_H
/* Only define the stubs when fiu is disabled, otherwise use the real fiu.h
* header */
#ifndef FIU_ENABLE
#define fiu_init(flags) 0
#define fiu_fail(name) 0
#define fiu_failinfo() NULL
#define fiu_do_on(name, action)
#define fiu_exit_on(name)
#define fiu_return_on(name, retval)
#else
#include <fiu.h>
#endif /* FIU_ENABLE */
#endif /* _FIU_LOCAL_H */

20
internal/core/src/index/thirdparty/google_benchmark/CMakeLists.txt vendored Normal file
View File

@ -0,0 +1,20 @@
FetchContent_Declare(google_benchmark
URL https://github.com/google/benchmark/archive/v1.5.2.tar.gz
URL_MD5 084b34aceaeac11a6607d35220ca2efa
DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH}
SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/google_benchmark
BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/google_benchmark
)
FetchContent_GetProperties( google_benchmark )
if ( NOT google_benchmark_POPULATED )
FetchContent_Populate( google_benchmark )
# Adding the following targets:
# gtest, gtest_main, gmock, gmock_main
message("gb=${google_benchmark_SOURCE_DIR}")
add_subdirectory( ${google_benchmark_SOURCE_DIR}
${google_benchmark_BINARY_DIR}
EXCLUDE_FROM_ALL )
endif()

66
internal/core/src/index/thirdparty/gtest/CMakeLists.txt vendored Normal file
View File

@ -0,0 +1,66 @@
#-------------------------------------------------------------------------------
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
#-------------------------------------------------------------------------------
set( GTEST_VERSION 1.8.1 )
if ( DEFINED ENV{MILVUS_GTEST_URL} )
set( GTEST_SOURCE_URL "$ENV{MILVUS_GTEST_URL}" )
else()
set( GTEST_SOURCE_URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.zip" )
endif()
message( STATUS "Building gtest-${GTEST_VERSION} from source" )
set( CMAKE_POLICY_DEFAULT_CMP0022 NEW ) # for googletest only
FetchContent_Declare(
googletest
URL ${GTEST_SOURCE_URL}
URL_MD5 "ad6868782b5952b7476a7c1c72d5a714"
SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/googletest-src
BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/googletest-build
DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH} )
FetchContent_GetProperties( googletest )
if ( NOT googletest_POPULATED )
FetchContent_Populate( googletest )
# Adding the following targets:
# gtest, gtest_main, gmock, gmock_main
add_subdirectory( ${googletest_SOURCE_DIR}
${googletest_BINARY_DIR}
EXCLUDE_FROM_ALL )
endif()
# include(GoogleTest)
# ****************************************************************
# Create ALIAS Target
# ****************************************************************
# if (NOT TARGET GTest:gtest)
# add_library( GTest::gtest ALIAS gtest )
# endif()
# if (NOT TARGET GTest:main)
# add_library( GTest::main ALIAS gtest_main )
# endif()
# if (NOT TARGET GMock:gmock)
# target_link_libraries( gmock INTERFACE GTest::gtest )
# add_library( GMock::gmock ALIAS gmock )
# endif()
# if (NOT TARGET GMock:main)
# target_link_libraries( gmock_main INTERFACE GTest::gtest )
# add_library( GMock::main ALIAS gmock_main )
# endif()
get_property( var DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/googletest-src" PROPERTY COMPILE_OPTIONS )
message( STATUS "gtest compile options: ${var}" )

25447
internal/core/src/index/thirdparty/nlohmann/json.hpp vendored Normal file
View File

File diff suppressed because it is too large Load Diff

63
internal/core/src/index/unittest/CMakeLists.txt
View File

@ -15,32 +15,21 @@ include_directories(${INDEX_SOURCE_DIR}/thirdparty/NGT/lib)
include_directories(${INDEX_SOURCE_DIR}/knowhere)
include_directories(${INDEX_SOURCE_DIR})
set(depend_libs
gtest gmock gtest_main gmock_main
faiss fiu
set(DEPEND_LIBS
gtest
gmock
gtest_main
gmock_main
faiss
fiu
ngt
index_log
)
if (FAISS_WITH_MKL)
set(depend_libs ${depend_libs}
"-Wl,--start-group \
${MKL_LIB_PATH}/libmkl_intel_ilp64.a \
${MKL_LIB_PATH}/libmkl_gnu_thread.a \
${MKL_LIB_PATH}/libmkl_core.a \
-Wl,--end-group -lgomp -lpthread -lm -ldl"
)
else ()
set(depend_libs ${depend_libs}
${OpenBLAS_LIBRARIES}
${LAPACK_LIBRARIES}
)
endif ()
set(basic_libs
gomp gfortran pthread
openblas
)
set(util_srcs
set(BASIC_LIBS gomp gfortran pthread)
set(UTIL_SRCS
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/adapter/VectorAdapter.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/FaissIO.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp
@ -56,19 +45,17 @@ set(util_srcs
if (MILVUS_GPU_VERSION)
include_directories(${CUDA_INCLUDE_DIRS})
link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
set(cuda_lib
set(CUDA_LIB
cudart
cublas
)
set(basic_libs ${basic_libs}
${cuda_lib}
)
set(util_srcs ${util_srcs}
set(BASIC_LIBS ${basic_libs} ${cuda_lib})
set(UTIL_SRCS ${util_srcs}
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/FaissGpuResourceMgr.cpp
)
endif ()
set(faiss_srcs
set(FAISS_SRCS
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/FaissBaseBinaryIndex.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.cpp
@ -81,8 +68,9 @@ set(faiss_srcs
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_offset_index/OffsetBaseIndex.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_offset_index/IndexIVF_NM.cpp
)
if (MILVUS_GPU_VERSION)
set(faiss_srcs ${faiss_srcs}
set(FAISS_SRCS ${faiss_srcs}
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/ConfAdapter.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexGPUIDMAP.cpp
@ -106,8 +94,8 @@ else ()
message(FATAL_ERROR "no OpenMP supprot")
endif ()
set(all_indexing_srcs
${util_srcs}
set(ALL_INDEXING_SRCS
${UTIL_SRCS}
test_instructionset.cpp
test_common.cpp
test_customized_index.cpp
@ -127,21 +115,22 @@ set(all_indexing_srcs
test_ngtpanng.cpp
test_ngtonng.cpp
)
if (MILVUS_GPU_VERSION)
set(all_indexing_srcs
${all_indexing_srcs}
set(ALL_INDEXING_SRCS
${ALL_INDEXING_SRCS}
test_ivf_gpu_nm.cpp
test_gpuresource.cpp
)
endif ()
if (MILVUS_SUPPORT_SPTAG)
set(all_indexing_srcs
set(ALL_INDEXING_SRCS
test_sptag.cpp
)
endif()
add_executable(test_knowhere ${all_indexing_srcs})
target_link_libraries(test_knowhere knowhere ${depend_libs})
add_executable(test_knowhere ${ALL_INDEXING_SRCS})
target_link_libraries(test_knowhere knowhere ${DEPEND_LIBS})
install(TARGETS test_knowhere DESTINATION unittest)
install(TARGETS test_knowhere)

4
internal/core/src/index/unittest/test_common.cpp
View File

@ -14,8 +14,8 @@
#include "knowhere/common/Timer.h"
#include "knowhere/knowhere/common/Exception.h"
#include "unittest/utils.h"
#include "index/thirdparty/faiss/utils/BitsetView.h"
#include "index/thirdparty/faiss/utils/ConcurrentBitset.h"
#include "faiss/utils/BitsetView.h"
#include "faiss/utils/ConcurrentBitset.h"
#include <boost/dynamic_bitset.hpp>
/*Some unittest for knowhere/common, mainly for improve code coverage.*/

2
internal/core/thirdparty/google_benchmark/CMakeLists.txt vendored
View File

@ -12,7 +12,7 @@ if ( NOT google_benchmark_POPULATED )
FetchContent_Populate( google_benchmark )
# Adding the following targets:
# gtest, gtest_main, gmock, gmock_main
# benchmark::benchmark_main
message("gb=${google_benchmark_SOURCE_DIR}")
add_subdirectory( ${google_benchmark_SOURCE_DIR}
${google_benchmark_BINARY_DIR}