Make knowhere compile independently (#7606)

Make knowhere compile independently

1. Make knowhere compile independently
    * Add gtest, arrow, and some other libraries to index.
    * Add cache, log and some other files to knowhere.
    * Add CMakeLists files to index's thirdparty.

2. Modified the compilation content of knowhere
    * Delete some content of compile library.
    * Add IMPORTED_GLOBAL property to faiss.

3.  Change the compilation location of some libraries
    * Make OpenBLas compiled in thirdpartycore.cmake.
    * Make faiss compiled in thirdparty/CMakeLists.

Change the content of knowhere/CMakeLists

1. Change easyloggingpp and nlohmann into index/thirdparty.
2. Change MILVUS_THIRDPARTY_SRC into KNOWHERE_THIRDPARTY_SRC.

Delete FindOpenBLAS

1. Delete Openblas.cmake.

2. The search task for openBlas is assigned to ThirdpartyCore.

3. Some changes were made to build.sh in index.

Fix the openBLas compilation problem

Delete the if-else in compilation of faiss;

Now when complie faiss, it will find the Openblas as we wish.

Fix some problem:

1. delete arrow

2. set openblas_source to AUTO

3. change a include_dir

4. delete MKL

5. delete the CMakeLists in index/utils,cache,log

Change variable build_test to knowhere_build_test in index/build.sh

Change the include location of  GNUInstallDirs

set CMAKE_INSTALL_LIBDIR

Resolves: milvus-io#5183
See also: milvus-io#6604

Signed-off-by: Shen Zhi <m13120163046@163.com>

internal/core/src/index/CMakeLists.txt

@@ -42,10 +42,17 @@ else ()
 endif ()
 message(STATUS "Build type = ${BUILD_TYPE}")
 
+include(GNUInstallDirs)
+set(CMAKE_INSTALL_LIBDIR lib)
 set(INDEX_SOURCE_DIR ${PROJECT_SOURCE_DIR})
 set(INDEX_BINARY_DIR ${PROJECT_BINARY_DIR})
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${INDEX_SOURCE_DIR}/cmake")
 
+# This will set RPATH to all excutable TARGET
+# self-installed dynamic libraries will be correctly linked by excutable
+set( CMAKE_INSTALL_RPATH "/usr/lib" "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}" )
+set( CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE )
+
 include(ExternalProject)
 include(DefineOptionsCore)
 include(BuildUtilsCore)
@@ -72,6 +79,7 @@ if (MILVUS_SUPPORT_SPTAG)
 endif ()
 
 include(ThirdPartyPackagesCore)
+add_subdirectory(thirdparty)
 
 if (CMAKE_BUILD_TYPE STREQUAL "Release")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp")
@@ -93,7 +101,7 @@ endif ()
 
 set(INDEX_INCLUDE_DIRS ${INDEX_INCLUDE_DIRS} PARENT_SCOPE)
 
-if (BUILD_UNIT_TEST)
+if (KNOWHERE_BUILD_TESTS)
 	set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DELPP_DISABLE_LOGS")
     add_subdirectory(unittest)
 endif ()

internal/core/src/index/build.sh

@@ -21,8 +21,8 @@ do
                 ;;
              r)
                 if [[ -d cmake_build ]]; then
-                    rm ./cmake_build -r
-                    MAKE_CLEAN="ON"
+                    cd cmake_build
+                    make clean
                 fi
                 ;;
              g)
@@ -52,26 +52,24 @@ done
 
 if [[ ! -d cmake_build ]]; then
 	mkdir cmake_build
-	MAKE_CLEAN="ON"
 fi
 
 cd cmake_build
 
 CUDA_COMPILER=/usr/local/cuda/bin/nvcc
 
-if [[ ${MAKE_CLEAN} == "ON" ]]; then
-    CMAKE_CMD="cmake -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
-    -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}
-    -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
-    -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
-    -DMILVUS_ENABLE_PROFILING=${PROFILING} \
-    ../"
-    echo ${CMAKE_CMD}
+CMAKE_CMD="cmake -DKNOWHERE_BUILD_TESTS=${BUILD_UNITTEST} \
+-DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}
+-DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
+-DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
+-DMILVUS_ENABLE_PROFILING=${PROFILING} \
+../"
+echo ${CMAKE_CMD}
 
-    ${CMAKE_CMD}
-    make clean
-fi
+${CMAKE_CMD}
 
-make -j 8 || exit 1
+make -j 8 ||exit 1
 
 make install || exit 1
+
+

internal/core/src/index/cmake/DefineOptionsCore.cmake

@@ -55,15 +55,15 @@ endmacro()
 #----------------------------------------------------------------------
 set_option_category("Thirdparty")
 
-set(KNOWHERE_DEPENDENCY_SOURCE_DEFAULT "BUNDLED")
+set(KNOWHERE_DEPENDENCY_SOURCE "AUTO")
 
 define_option_string(KNOWHERE_DEPENDENCY_SOURCE
         "Method to use for acquiring KNOWHERE's build dependencies"
-        "${KNOWHERE_DEPENDENCY_SOURCE_DEFAULT}"
         "AUTO"
         "BUNDLED"
         "SYSTEM")
 
+
 define_option(KNOWHERE_USE_CCACHE "Use ccache when compiling (if available)" ON)
 
 define_option(KNOWHERE_VERBOSE_THIRDPARTY_BUILD

internal/core/src/index/cmake/FindArrow.cmake

@@ -1,431 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# - Find Arrow (arrow/api.h, libarrow.a, libarrow.so)
-# This module defines
-#  ARROW_FOUND, whether Arrow has been found
-#  ARROW_FULL_SO_VERSION, full shared object version of found Arrow "100.0.0"
-#  ARROW_IMPORT_LIB, path to libarrow's import library (Windows only)
-#  ARROW_INCLUDE_DIR, directory containing headers
-#  ARROW_LIBS, deprecated. Use ARROW_LIB_DIR instead
-#  ARROW_LIB_DIR, directory containing Arrow libraries
-#  ARROW_SHARED_IMP_LIB, deprecated. Use ARROW_IMPORT_LIB instead
-#  ARROW_SHARED_LIB, path to libarrow's shared library
-#  ARROW_SO_VERSION, shared object version of found Arrow such as "100"
-#  ARROW_STATIC_LIB, path to libarrow.a
-#  ARROW_VERSION, version of found Arrow
-#  ARROW_VERSION_MAJOR, major version of found Arrow
-#  ARROW_VERSION_MINOR, minor version of found Arrow
-#  ARROW_VERSION_PATCH, patch version of found Arrow
-
-include(FindPkgConfig)
-include(FindPackageHandleStandardArgs)
-
-set(ARROW_SEARCH_LIB_PATH_SUFFIXES)
-if(CMAKE_LIBRARY_ARCHITECTURE)
-  list(APPEND ARROW_SEARCH_LIB_PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}")
-endif()
-list(APPEND ARROW_SEARCH_LIB_PATH_SUFFIXES
-            "lib64"
-            "lib32"
-            "lib"
-            "bin")
-set(ARROW_CONFIG_SUFFIXES
-    "_RELEASE"
-    "_RELWITHDEBINFO"
-    "_MINSIZEREL"
-    "_DEBUG"
-    "")
-if(CMAKE_BUILD_TYPE)
-  string(TOUPPER ${CMAKE_BUILD_TYPE} ARROW_CONFIG_SUFFIX_PREFERRED)
-  set(ARROW_CONFIG_SUFFIX_PREFERRED "_${ARROW_CONFIG_SUFFIX_PREFERRED}")
-  list(INSERT ARROW_CONFIG_SUFFIXES 0 "${ARROW_CONFIG_SUFFIX_PREFERRED}")
-endif()
-
-if(NOT DEFINED ARROW_MSVC_STATIC_LIB_SUFFIX)
-  if(MSVC)
-    set(ARROW_MSVC_STATIC_LIB_SUFFIX "_static")
-  else()
-    set(ARROW_MSVC_STATIC_LIB_SUFFIX "")
-  endif()
-endif()
-
-# Internal function.
-#
-# Set shared library name for ${base_name} to ${output_variable}.
-#
-# Example:
-#   arrow_build_shared_library_name(ARROW_SHARED_LIBRARY_NAME arrow)
-#   # -> ARROW_SHARED_LIBRARY_NAME=libarrow.so on Linux
-#   # -> ARROW_SHARED_LIBRARY_NAME=libarrow.dylib on macOS
-#   # -> ARROW_SHARED_LIBRARY_NAME=arrow.dll with MSVC on Windows
-#   # -> ARROW_SHARED_LIBRARY_NAME=libarrow.dll with MinGW on Windows
-function(arrow_build_shared_library_name output_variable base_name)
-  set(${output_variable}
-      "${CMAKE_SHARED_LIBRARY_PREFIX}${base_name}${CMAKE_SHARED_LIBRARY_SUFFIX}"
-      PARENT_SCOPE)
-endfunction()
-
-# Internal function.
-#
-# Set import library name for ${base_name} to ${output_variable}.
-# This is useful only for MSVC build. Import library is used only
-# with MSVC build.
-#
-# Example:
-#   arrow_build_import_library_name(ARROW_IMPORT_LIBRARY_NAME arrow)
-#   # -> ARROW_IMPORT_LIBRARY_NAME=arrow on Linux (meaningless)
-#   # -> ARROW_IMPORT_LIBRARY_NAME=arrow on macOS (meaningless)
-#   # -> ARROW_IMPORT_LIBRARY_NAME=arrow.lib with MSVC on Windows
-#   # -> ARROW_IMPORT_LIBRARY_NAME=libarrow.dll.a with MinGW on Windows
-function(arrow_build_import_library_name output_variable base_name)
-  set(${output_variable}
-      "${CMAKE_IMPORT_LIBRARY_PREFIX}${base_name}${CMAKE_IMPORT_LIBRARY_SUFFIX}"
-      PARENT_SCOPE)
-endfunction()
-
-# Internal function.
-#
-# Set static library name for ${base_name} to ${output_variable}.
-#
-# Example:
-#   arrow_build_static_library_name(ARROW_STATIC_LIBRARY_NAME arrow)
-#   # -> ARROW_STATIC_LIBRARY_NAME=libarrow.a on Linux
-#   # -> ARROW_STATIC_LIBRARY_NAME=libarrow.a on macOS
-#   # -> ARROW_STATIC_LIBRARY_NAME=arrow.lib with MSVC on Windows
-#   # -> ARROW_STATIC_LIBRARY_NAME=libarrow.dll.a with MinGW on Windows
-function(arrow_build_static_library_name output_variable base_name)
-  set(
-    ${output_variable}
-    "${CMAKE_STATIC_LIBRARY_PREFIX}${base_name}${ARROW_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}"
-    PARENT_SCOPE)
-endfunction()
-
-# Internal function.
-#
-# Set macro value for ${macro_name} in ${header_content} to ${output_variable}.
-#
-# Example:
-#   arrow_extract_macro_value(version_major
-#                             "ARROW_VERSION_MAJOR"
-#                             "#define ARROW_VERSION_MAJOR 1.0.0")
-#   # -> version_major=1.0.0
-function(arrow_extract_macro_value output_variable macro_name header_content)
-  string(REGEX MATCH "#define +${macro_name} +[^\r\n]+" macro_definition
-               "${header_content}")
-  string(REGEX
-         REPLACE "^#define +${macro_name} +(.+)$" "\\1" macro_value "${macro_definition}")
-  set(${output_variable} "${macro_value}" PARENT_SCOPE)
-endfunction()
-
-# Internal macro only for arrow_find_package.
-#
-# Find package in HOME.
-macro(arrow_find_package_home)
-  find_path(${prefix}_include_dir "${header_path}"
-            PATHS "${home}"
-            PATH_SUFFIXES "include"
-            NO_DEFAULT_PATH)
-  set(include_dir "${${prefix}_include_dir}")
-  set(${prefix}_INCLUDE_DIR "${include_dir}" PARENT_SCOPE)
-
-  if(MSVC)
-    set(CMAKE_SHARED_LIBRARY_SUFFIXES_ORIGINAL ${CMAKE_FIND_LIBRARY_SUFFIXES})
-    # .dll isn't found by find_library with MSVC because .dll isn't included in
-    # CMAKE_FIND_LIBRARY_SUFFIXES.
-    list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES "${CMAKE_SHARED_LIBRARY_SUFFIX}")
-  endif()
-  find_library(${prefix}_shared_lib
-               NAMES "${shared_lib_name}"
-               PATHS "${home}"
-               PATH_SUFFIXES ${ARROW_SEARCH_LIB_PATH_SUFFIXES}
-               NO_DEFAULT_PATH)
-  if(MSVC)
-    set(CMAKE_SHARED_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_ORIGINAL})
-  endif()
-  set(shared_lib "${${prefix}_shared_lib}")
-  set(${prefix}_SHARED_LIB "${shared_lib}" PARENT_SCOPE)
-  if(shared_lib)
-    add_library(${target_shared} SHARED IMPORTED)
-    set_target_properties(${target_shared} PROPERTIES IMPORTED_LOCATION "${shared_lib}")
-    if(include_dir)
-      set_target_properties(${target_shared}
-                            PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${include_dir}")
-    endif()
-    find_library(${prefix}_import_lib
-                 NAMES "${import_lib_name}"
-                 PATHS "${home}"
-                 PATH_SUFFIXES ${ARROW_SEARCH_LIB_PATH_SUFFIXES}
-                 NO_DEFAULT_PATH)
-    set(import_lib "${${prefix}_import_lib}")
-    set(${prefix}_IMPORT_LIB "${import_lib}" PARENT_SCOPE)
-    if(import_lib)
-      set_target_properties(${target_shared} PROPERTIES IMPORTED_IMPLIB "${import_lib}")
-    endif()
-  endif()
-
-  find_library(${prefix}_static_lib
-               NAMES "${static_lib_name}"
-               PATHS "${home}"
-               PATH_SUFFIXES ${ARROW_SEARCH_LIB_PATH_SUFFIXES}
-               NO_DEFAULT_PATH)
-  set(static_lib "${${prefix}_static_lib}")
-  set(${prefix}_STATIC_LIB "${static_lib}" PARENT_SCOPE)
-  if(static_lib)
-    add_library(${target_static} STATIC IMPORTED)
-    set_target_properties(${target_static} PROPERTIES IMPORTED_LOCATION "${static_lib}")
-    if(include_dir)
-      set_target_properties(${target_static}
-                            PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${include_dir}")
-    endif()
-  endif()
-endmacro()
-
-# Internal macro only for arrow_find_package.
-#
-# Find package by CMake package configuration.
-macro(arrow_find_package_cmake_package_configuration)
-  # ARROW-5575: We need to split target files for each component
-  if(TARGET ${target_shared} OR TARGET ${target_static})
-    set(${cmake_package_name}_FOUND TRUE)
-  else()
-    find_package(${cmake_package_name} CONFIG)
-  endif()
-  if(${cmake_package_name}_FOUND)
-    set(${prefix}_USE_CMAKE_PACKAGE_CONFIG TRUE PARENT_SCOPE)
-    if(TARGET ${target_shared})
-      foreach(suffix ${ARROW_CONFIG_SUFFIXES})
-        get_target_property(shared_lib ${target_shared} IMPORTED_LOCATION${suffix})
-        if(shared_lib)
-          # Remove shared library version:
-          #   libarrow.so.100.0.0 -> libarrow.so
-          # Because ARROW_HOME and pkg-config approaches don't add
-          # shared library version.
-          string(REGEX
-                 REPLACE "(${CMAKE_SHARED_LIBRARY_SUFFIX})[.0-9]+$" "\\1" shared_lib
-                         "${shared_lib}")
-          set(${prefix}_SHARED_LIB "${shared_lib}" PARENT_SCOPE)
-          break()
-        endif()
-      endforeach()
-    endif()
-    if(TARGET ${target_static})
-      foreach(suffix ${ARROW_CONFIG_SUFFIXES})
-        get_target_property(static_lib ${target_static} IMPORTED_LOCATION${suffix})
-        if(static_lib)
-          set(${prefix}_STATIC_LIB "${static_lib}" PARENT_SCOPE)
-          break()
-        endif()
-      endforeach()
-    endif()
-  endif()
-endmacro()
-
-# Internal macro only for arrow_find_package.
-#
-# Find package by pkg-config.
-macro(arrow_find_package_pkg_config)
-  pkg_check_modules(${prefix}_PC ${pkg_config_name})
-  if(${prefix}_PC_FOUND)
-    set(${prefix}_USE_PKG_CONFIG TRUE PARENT_SCOPE)
-
-    set(include_dir "${${prefix}_PC_INCLUDEDIR}")
-    set(lib_dir "${${prefix}_PC_LIBDIR}")
-    set(shared_lib_paths "${${prefix}_PC_LINK_LIBRARIES}")
-    # Use the first shared library path as the IMPORTED_LOCATION
-    # for ${target_shared}. This assumes that the first shared library
-    # path is the shared library path for this module.
-    list(GET shared_lib_paths 0 first_shared_lib_path)
-    # Use the rest shared library paths as the INTERFACE_LINK_LIBRARIES
-    # for ${target_shared}. This assumes that the rest shared library
-    # paths are dependency library paths for this module.
-    list(LENGTH shared_lib_paths n_shared_lib_paths)
-    if(n_shared_lib_paths LESS_EQUAL 1)
-      set(rest_shared_lib_paths)
-    else()
-      list(SUBLIST
-           shared_lib_paths
-           1
-           -1
-           rest_shared_lib_paths)
-    endif()
-
-    set(${prefix}_VERSION "${${prefix}_PC_VERSION}" PARENT_SCOPE)
-    set(${prefix}_INCLUDE_DIR "${include_dir}" PARENT_SCOPE)
-    set(${prefix}_SHARED_LIB "${first_shared_lib_path}" PARENT_SCOPE)
-
-    add_library(${target_shared} SHARED IMPORTED)
-    set_target_properties(${target_shared}
-                          PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
-                                     "${include_dir}"
-                                     INTERFACE_LINK_LIBRARIES
-                                     "${rest_shared_lib_paths}"
-                                     IMPORTED_LOCATION
-                                     "${first_shared_lib_path}")
-
-    find_library(${prefix}_static_lib
-                 NAMES "${static_lib_name}"
-                 PATHS "${lib_dir}"
-                 NO_DEFAULT_PATH)
-    set(static_lib "${${prefix}_static_lib}")
-    set(${prefix}_STATIC_LIB "${static_lib}" PARENT_SCOPE)
-    if(static_lib)
-      add_library(${target_static} STATIC IMPORTED)
-      set_target_properties(${target_static}
-                            PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${include_dir}"
-                                       IMPORTED_LOCATION "${static_lib}")
-    endif()
-  endif()
-endmacro()
-
-function(arrow_find_package
-         prefix
-         home
-         base_name
-         header_path
-         cmake_package_name
-         pkg_config_name)
-  arrow_build_shared_library_name(shared_lib_name ${base_name})
-  arrow_build_import_library_name(import_lib_name ${base_name})
-  arrow_build_static_library_name(static_lib_name ${base_name})
-
-  set(target_shared ${base_name}_shared)
-  set(target_static ${base_name}_static)
-
-  if(home)
-    arrow_find_package_home()
-    set(${prefix}_FIND_APPROACH "HOME: ${home}" PARENT_SCOPE)
-  else()
-    arrow_find_package_cmake_package_configuration()
-    if(${cmake_package_name}_FOUND)
-      set(${prefix}_FIND_APPROACH
-          "CMake package configuration: ${cmake_package_name}"
-          PARENT_SCOPE)
-    else()
-      arrow_find_package_pkg_config()
-      set(${prefix}_FIND_APPROACH "pkg-config: ${pkg_config_name}" PARENT_SCOPE)
-    endif()
-  endif()
-
-  if(NOT include_dir)
-    if(TARGET ${target_shared})
-      get_target_property(include_dir ${target_shared} INTERFACE_INCLUDE_DIRECTORIES)
-    elseif(TARGET ${target_static})
-      get_target_property(include_dir ${target_static} INTERFACE_INCLUDE_DIRECTORIES)
-    endif()
-  endif()
-  if(include_dir)
-    set(${prefix}_INCLUDE_DIR "${include_dir}" PARENT_SCOPE)
-  endif()
-
-  if(shared_lib)
-    get_filename_component(lib_dir "${shared_lib}" DIRECTORY)
-  elseif(static_lib)
-    get_filename_component(lib_dir "${static_lib}" DIRECTORY)
-  else()
-    set(lib_dir NOTFOUND)
-  endif()
-  set(${prefix}_LIB_DIR "${lib_dir}" PARENT_SCOPE)
-  # For backward compatibility
-  set(${prefix}_LIBS "${lib_dir}" PARENT_SCOPE)
-endfunction()
-
-if(NOT "$ENV{ARROW_HOME}" STREQUAL "")
-  file(TO_CMAKE_PATH "$ENV{ARROW_HOME}" ARROW_HOME)
-endif()
-arrow_find_package(ARROW
-                   "${ARROW_HOME}"
-                   arrow
-                   arrow/api.h
-                   Arrow
-                   arrow)
-
-if(ARROW_HOME)
-  if(ARROW_INCLUDE_DIR)
-    file(READ "${ARROW_INCLUDE_DIR}/arrow/util/config.h" ARROW_CONFIG_H_CONTENT)
-    arrow_extract_macro_value(ARROW_VERSION_MAJOR "ARROW_VERSION_MAJOR"
-                              "${ARROW_CONFIG_H_CONTENT}")
-    arrow_extract_macro_value(ARROW_VERSION_MINOR "ARROW_VERSION_MINOR"
-                              "${ARROW_CONFIG_H_CONTENT}")
-    arrow_extract_macro_value(ARROW_VERSION_PATCH "ARROW_VERSION_PATCH"
-                              "${ARROW_CONFIG_H_CONTENT}")
-    if("${ARROW_VERSION_MAJOR}" STREQUAL ""
-       OR "${ARROW_VERSION_MINOR}" STREQUAL ""
-       OR "${ARROW_VERSION_PATCH}" STREQUAL "")
-      set(ARROW_VERSION "0.0.0")
-    else()
-      set(ARROW_VERSION
-          "${ARROW_VERSION_MAJOR}.${ARROW_VERSION_MINOR}.${ARROW_VERSION_PATCH}")
-    endif()
-
-    arrow_extract_macro_value(ARROW_SO_VERSION_QUOTED "ARROW_SO_VERSION"
-                              "${ARROW_CONFIG_H_CONTENT}")
-    string(REGEX REPLACE "^\"(.+)\"$" "\\1" ARROW_SO_VERSION "${ARROW_SO_VERSION_QUOTED}")
-    arrow_extract_macro_value(ARROW_FULL_SO_VERSION_QUOTED "ARROW_FULL_SO_VERSION"
-                              "${ARROW_CONFIG_H_CONTENT}")
-    string(REGEX
-           REPLACE "^\"(.+)\"$" "\\1" ARROW_FULL_SO_VERSION
-                   "${ARROW_FULL_SO_VERSION_QUOTED}")
-  endif()
-else()
-  if(ARROW_USE_CMAKE_PACKAGE_CONFIG)
-    find_package(Arrow CONFIG)
-  elseif(ARROW_USE_PKG_CONFIG)
-    pkg_get_variable(ARROW_SO_VERSION arrow so_version)
-    pkg_get_variable(ARROW_FULL_SO_VERSION arrow full_so_version)
-  endif()
-endif()
-
-set(ARROW_ABI_VERSION ${ARROW_SO_VERSION})
-
-mark_as_advanced(ARROW_ABI_VERSION
-                 ARROW_CONFIG_SUFFIXES
-                 ARROW_FULL_SO_VERSION
-                 ARROW_IMPORT_LIB
-                 ARROW_INCLUDE_DIR
-                 ARROW_LIBS
-                 ARROW_LIB_DIR
-                 ARROW_SEARCH_LIB_PATH_SUFFIXES
-                 ARROW_SHARED_IMP_LIB
-                 ARROW_SHARED_LIB
-                 ARROW_SO_VERSION
-                 ARROW_STATIC_LIB
-                 ARROW_VERSION
-                 ARROW_VERSION_MAJOR
-                 ARROW_VERSION_MINOR
-                 ARROW_VERSION_PATCH)
-
-find_package_handle_standard_args(Arrow REQUIRED_VARS
-                                  # The first required variable is shown
-                                  # in the found message. So this list is
-                                  # not sorted alphabetically.
-                                  ARROW_INCLUDE_DIR
-                                  ARROW_LIB_DIR
-                                  ARROW_FULL_SO_VERSION
-                                  ARROW_SO_VERSION
-                                  VERSION_VAR
-                                  ARROW_VERSION)
-set(ARROW_FOUND ${Arrow_FOUND})
-
-if(Arrow_FOUND AND NOT Arrow_FIND_QUIETLY)
-  message(STATUS "Arrow version: ${ARROW_VERSION} (${ARROW_FIND_APPROACH})")
-  message(STATUS "Arrow SO and ABI version: ${ARROW_SO_VERSION}")
-  message(STATUS "Arrow full SO version: ${ARROW_FULL_SO_VERSION}")
-  message(STATUS "Found the Arrow core shared library: ${ARROW_SHARED_LIB}")
-  message(STATUS "Found the Arrow core import library: ${ARROW_IMPORT_LIB}")
-  message(STATUS "Found the Arrow core static library: ${ARROW_STATIC_LIB}")
-endif()

internal/core/src/index/cmake/FindOpenBLAS.cmake

@@ -1,93 +0,0 @@
-
-if (OpenBLAS_FOUND) # the git version propose a OpenBLASConfig.cmake
-    message(STATUS "OpenBLASConfig found")
-    set(OpenBLAS_INCLUDE_DIR ${OpenBLAS_INCLUDE_DIRS})
-else()
-    message("OpenBLASConfig not found")
-    unset(OpenBLAS_DIR CACHE)
-    set(OpenBLAS_INCLUDE_SEARCH_PATHS
-            /usr/local/openblas/include
-            /usr/include
-            /usr/include/openblas
-            /usr/include/openblas-base
-            /usr/local/include
-            /usr/local/include/openblas
-            /usr/local/include/openblas-base
-            /opt/OpenBLAS/include
-            /usr/local/opt/openblas/include
-            $ENV{OpenBLAS_HOME}
-            $ENV{OpenBLAS_HOME}/include
-            )
-
-    set(OpenBLAS_LIB_SEARCH_PATHS
-            /usr/local/openblas/lib
-            /lib/
-            /lib/openblas-base
-            /lib64/
-            /usr/lib
-            /usr/lib/openblas-base
-            /usr/lib64
-            /usr/local/lib
-            /usr/local/lib64
-            /usr/local/opt/openblas/lib
-            /opt/OpenBLAS/lib
-            $ENV{OpenBLAS}
-            $ENV{OpenBLAS}/lib
-            $ENV{OpenBLAS_HOME}
-            $ENV{OpenBLAS_HOME}/lib
-            )
-    set(DEFAULT_OpenBLAS_LIB_PATH
-            /usr/local/openblas/lib
-            ${OPENBLAS_PREFIX}/lib)
-
-    message("DEFAULT_OpenBLAS_LIB_PATH: ${DEFAULT_OpenBLAS_LIB_PATH}")
-    find_path(OpenBLAS_INCLUDE_DIR NAMES openblas_config.h lapacke.h PATHS ${OpenBLAS_INCLUDE_SEARCH_PATHS})
-    find_library(OpenBLAS_LIB NAMES openblas PATHS ${DEFAULT_OpenBLAS_LIB_PATH} NO_DEFAULT_PATH)
-    find_library(OpenBLAS_LIB NAMES openblas PATHS ${OpenBLAS_LIB_SEARCH_PATHS})
-    # mostly for debian
-    find_library(Lapacke_LIB NAMES lapacke PATHS ${DEFAULT_OpenBLAS_LIB_PATH} NO_DEFAULT_PATH)
-    find_library(Lapacke_LIB NAMES lapacke PATHS ${OpenBLAS_LIB_SEARCH_PATHS})
-
-    set(OpenBLAS_FOUND ON)
-
-    #    Check include files
-    if(NOT OpenBLAS_INCLUDE_DIR)
-        set(OpenBLAS_FOUND OFF)
-        message(STATUS "Could not find OpenBLAS include. Turning OpenBLAS_FOUND off")
-    else()
-        message(STATUS "find OpenBLAS include:${OpenBLAS_INCLUDE_DIR} ")
-    endif()
-
-    #    Check libraries
-    if(NOT OpenBLAS_LIB)
-        set(OpenBLAS_FOUND OFF)
-        message(STATUS "Could not find OpenBLAS lib. Turning OpenBLAS_FOUND off")
-    else()
-        message(STATUS "find OpenBLAS lib:${OpenBLAS_LIB} ")
-    endif()
-
-    if (OpenBLAS_FOUND)
-        set(FOUND_OPENBLAS "true" PARENT_SCOPE)
-        set(OpenBLAS_LIBRARIES ${OpenBLAS_LIB})
-        STRING(REGEX REPLACE "/libopenblas.so" "" OpenBLAS_LIB_DIR ${OpenBLAS_LIBRARIES})
-        message(STATUS "find OpenBLAS libraries:${OpenBLAS_LIBRARIES} ")
-        if (Lapacke_LIB)
-            set(OpenBLAS_LIBRARIES ${OpenBLAS_LIBRARIES} ${Lapacke_LIB})
-        endif()
-        if (NOT OpenBLAS_FIND_QUIETLY)
-            message(STATUS "Found OpenBLAS libraries: ${OpenBLAS_LIBRARIES}")
-            message(STATUS "Found OpenBLAS include: ${OpenBLAS_INCLUDE_DIR}")
-        endif()
-    else()
-        set(FOUND_OPENBLAS "false" PARENT_SCOPE)
-        if (OpenBLAS_FIND_REQUIRED)
-            message(FATAL_ERROR "Could not find OpenBLAS")
-        endif()
-    endif()
-endif()
-
-mark_as_advanced(
-        OpenBLAS_INCLUDE_DIR
-        OpenBLAS_LIBRARIES
-        OpenBLAS_LIB_DIR
-)

internal/core/src/index/cmake/ThirdPartyPackagesCore.cmake

@@ -10,14 +10,10 @@
 # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 # or implied. See the License for the specific language governing permissions and limitations under the License.
 #-------------------------------------------------------------------------------
-
 set(KNOWHERE_THIRDPARTY_DEPENDENCIES
-        Arrow
-        FAISS
-        GTest
         OpenBLAS
-        MKL
-        )
+        FAISS
+)
 
 message(STATUS "Using ${KNOWHERE_DEPENDENCY_SOURCE} approach to find dependencies")
 
@@ -28,83 +24,52 @@ foreach (DEPENDENCY ${KNOWHERE_THIRDPARTY_DEPENDENCIES})
     endif ()
 endforeach ()
 
-macro(build_dependency DEPENDENCY_NAME)
-    if ("${DEPENDENCY_NAME}" STREQUAL "Arrow")
-        build_arrow()
-    elseif ("${DEPENDENCY_NAME}" STREQUAL "OpenBLAS")
-        build_openblas()
-    elseif ("${DEPENDENCY_NAME}" STREQUAL "FAISS")
-        build_faiss()
-    elseif ("${DEPENDENCY_NAME}" STREQUAL "MKL")
-        build_mkl()
-    else ()
-        message(FATAL_ERROR "Unknown thirdparty dependency to build: ${DEPENDENCY_NAME}")
-    endif ()
-endmacro()
-
-macro(resolve_dependency DEPENDENCY_NAME)
-    if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO")
-        find_package(${DEPENDENCY_NAME} MODULE)
-        if (NOT ${${DEPENDENCY_NAME}_FOUND})
-            build_dependency(${DEPENDENCY_NAME})
-        endif ()
-    elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED")
-        build_dependency(${DEPENDENCY_NAME})
-    elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM")
-        find_package(${DEPENDENCY_NAME} REQUIRED)
-    endif ()
-endmacro()
-
 # ----------------------------------------------------------------------
 # Identify OS
 if (UNIX)
-    if (APPLE)
-        set(CMAKE_OS_NAME "osx" CACHE STRING "Operating system name" FORCE)
-    else (APPLE)
-        ## Check for Debian GNU/Linux ________________
-        find_file(DEBIAN_FOUND debian_version debconf.conf
+    ## Check for Debian GNU/Linux ________________
+    find_file(DEBIAN_FOUND debian_version debconf.conf
+            PATHS /etc
+           )
+    if (DEBIAN_FOUND)
+        set(CMAKE_OS_NAME "debian" CACHE STRING "Operating system name" FORCE)
+    endif (DEBIAN_FOUND)
+    ##  Check for Fedora _________________________
+    find_file(FEDORA_FOUND fedora-release
+            PATHS /etc
+            )
+    if (FEDORA_FOUND)
+        set(CMAKE_OS_NAME "fedora" CACHE STRING "Operating system name" FORCE)
+    endif (FEDORA_FOUND)
+    ##  Check for RedHat _________________________
+    find_file(REDHAT_FOUND redhat-release inittab.RH
+            PATHS /etc
+            )
+    if (REDHAT_FOUND)
+        set(CMAKE_OS_NAME "redhat" CACHE STRING "Operating system name" FORCE)
+    endif (REDHAT_FOUND)
+    ## Extra check for Ubuntu ____________________
+    if (DEBIAN_FOUND)
+        ## At its core Ubuntu is a Debian system, with
+        ## a slightly altered configuration; hence from
+        ## a first superficial inspection a system will
+        ## be considered as Debian, which signifies an
+        ## extra check is required.
+        find_file(UBUNTU_EXTRA legal issue
                 PATHS /etc
                 )
-        if (DEBIAN_FOUND)
-            set(CMAKE_OS_NAME "debian" CACHE STRING "Operating system name" FORCE)
-        endif (DEBIAN_FOUND)
-        ##  Check for Fedora _________________________
-        find_file(FEDORA_FOUND fedora-release
-                PATHS /etc
-                )
-        if (FEDORA_FOUND)
-            set(CMAKE_OS_NAME "fedora" CACHE STRING "Operating system name" FORCE)
-        endif (FEDORA_FOUND)
-        ##  Check for RedHat _________________________
-        find_file(REDHAT_FOUND redhat-release inittab.RH
-                PATHS /etc
-                )
-        if (REDHAT_FOUND)
-            set(CMAKE_OS_NAME "redhat" CACHE STRING "Operating system name" FORCE)
-        endif (REDHAT_FOUND)
-        ## Extra check for Ubuntu ____________________
-        if (DEBIAN_FOUND)
-            ## At its core Ubuntu is a Debian system, with
-            ## a slightly altered configuration; hence from
-            ## a first superficial inspection a system will
-            ## be considered as Debian, which signifies an
-            ## extra check is required.
-            find_file(UBUNTU_EXTRA legal issue
-                    PATHS /etc
+           if (UBUNTU_EXTRA)
+            ## Scan contents of file
+            file(STRINGS ${UBUNTU_EXTRA} UBUNTU_FOUND
+                    REGEX Ubuntu
                     )
-            if (UBUNTU_EXTRA)
-                ## Scan contents of file
-                file(STRINGS ${UBUNTU_EXTRA} UBUNTU_FOUND
-                        REGEX Ubuntu
-                        )
-                ## Check result of string search
-                if (UBUNTU_FOUND)
-                    set(CMAKE_OS_NAME "ubuntu" CACHE STRING "Operating system name" FORCE)
-                    set(DEBIAN_FOUND FALSE)
-                endif (UBUNTU_FOUND)
-            endif (UBUNTU_EXTRA)
-        endif (DEBIAN_FOUND)
-    endif (APPLE)
+           ## Check result of string search
+            if (UBUNTU_FOUND)
+                set(CMAKE_OS_NAME "ubuntu" CACHE STRING "Operating system name" FORCE)
+                set(DEBIAN_FOUND FALSE)
+            endif (UBUNTU_FOUND)
+        endif (UBUNTU_EXTRA)
+    endif (DEBIAN_FOUND)
 endif (UNIX)
 
 
@@ -170,13 +135,6 @@ endif ()
 
 set(MAKE_BUILD_ARGS "-j6")
 
-
-# ----------------------------------------------------------------------
-# Find pthreads
-
-set(THREADS_PREFER_PTHREAD_FLAG ON)
-find_package(Threads REQUIRED)
-
 # ----------------------------------------------------------------------
 # Versions and URLs for toolchain builds, which also can be used to configure
 # offline builds
@@ -203,121 +161,19 @@ foreach (_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT})
     set(${_LIB_NAME} "${_LIB_VERSION}")
 endforeach ()
 
-set(FAISS_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/faiss)
-
-if (DEFINED ENV{KNOWHERE_ARROW_URL})
-    set(ARROW_SOURCE_URL "$ENV{KNOWHERE_ARROW_URL}")
-else ()
-    set(ARROW_SOURCE_URL
-            "https://github.com/apache/arrow.git"
-            )
-endif ()
-
-
 if (DEFINED ENV{KNOWHERE_OPENBLAS_URL})
     set(OPENBLAS_SOURCE_URL "$ENV{KNOWHERE_OPENBLAS_URL}")
 else ()
     set(OPENBLAS_SOURCE_URL
-            "https://github.com/xianyi/OpenBLAS/archive/v${OPENBLAS_VERSION}.tar.gz")
+            "https://github.com.cnpmjs.org/xianyi/OpenBLAS/archive/v${OPENBLAS_VERSION}.tar.gz")
 endif ()
 
 # ----------------------------------------------------------------------
-# ARROW
-set(ARROW_PREFIX "${INDEX_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp")
-
-macro(build_arrow)
-    message(STATUS "Building Apache ARROW-${ARROW_VERSION} from source")
-    set(ARROW_STATIC_LIB_NAME arrow)
-    set(ARROW_LIB_DIR "${ARROW_PREFIX}/lib")
-    set(ARROW_STATIC_LIB
-            "${ARROW_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}"
-            )
-    set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include")
-
-    set(ARROW_CMAKE_ARGS
-            ${EP_COMMON_CMAKE_ARGS}
-            -DARROW_BUILD_STATIC=ON
-            -DARROW_BUILD_SHARED=OFF
-            -DARROW_USE_GLOG=OFF
-            -DCMAKE_INSTALL_PREFIX=${ARROW_PREFIX}
-            -DCMAKE_INSTALL_LIBDIR=${ARROW_LIB_DIR}
-            -DARROW_CUDA=OFF
-            -DARROW_FLIGHT=OFF
-            -DARROW_GANDIVA=OFF
-            -DARROW_GANDIVA_JAVA=OFF
-            -DARROW_HDFS=OFF
-            -DARROW_HIVESERVER2=OFF
-            -DARROW_ORC=OFF
-            -DARROW_PARQUET=OFF
-            -DARROW_PLASMA=OFF
-            -DARROW_PLASMA_JAVA_CLIENT=OFF
-            -DARROW_PYTHON=OFF
-            -DARROW_WITH_BZ2=OFF
-            -DARROW_WITH_ZLIB=OFF
-            -DARROW_WITH_LZ4=OFF
-            -DARROW_WITH_SNAPPY=OFF
-            -DARROW_WITH_ZSTD=OFF
-            -DARROW_WITH_BROTLI=OFF
-            -DCMAKE_BUILD_TYPE=Release
-            -DARROW_DEPENDENCY_SOURCE=BUNDLED #Build all arrow dependencies from source instead of calling find_package first
-            -DBOOST_SOURCE=AUTO #try to find BOOST in the system default locations and build from source if not found
-            )
-
-    externalproject_add(arrow_ep
-            GIT_REPOSITORY
-            ${ARROW_SOURCE_URL}
-            GIT_TAG
-            ${ARROW_VERSION}
-            GIT_SHALLOW
-            TRUE
-            SOURCE_SUBDIR
-            cpp
-            ${EP_LOG_OPTIONS}
-            CMAKE_ARGS
-            ${ARROW_CMAKE_ARGS}
-            BUILD_COMMAND
-            ""
-            INSTALL_COMMAND
-            ${MAKE} ${MAKE_BUILD_ARGS} install
-            BUILD_BYPRODUCTS
-            "${ARROW_STATIC_LIB}"
-            )
-
-    file(MAKE_DIRECTORY "${ARROW_INCLUDE_DIR}")
-    add_library(arrow STATIC IMPORTED)
-    set_target_properties(arrow
-            PROPERTIES IMPORTED_LOCATION "${ARROW_STATIC_LIB}"
-            INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}")
-    add_dependencies(arrow arrow_ep)
-
-    set(JEMALLOC_PREFIX "${INDEX_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep-build/jemalloc_ep-prefix/src/jemalloc_ep")
-
-    add_custom_command(TARGET arrow_ep POST_BUILD
-            COMMAND ${CMAKE_COMMAND} -E make_directory ${ARROW_LIB_DIR}
-            COMMAND ${CMAKE_COMMAND} -E copy ${JEMALLOC_PREFIX}/lib/libjemalloc_pic.a ${ARROW_LIB_DIR}
-            DEPENDS ${JEMALLOC_PREFIX}/lib/libjemalloc_pic.a)
-
-endmacro()
-
-if (KNOWHERE_WITH_ARROW AND NOT TARGET arrow_ep)
-
-    resolve_dependency(Arrow)
-
-    link_directories(SYSTEM ${ARROW_LIB_DIR})
-    include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
-endif ()
-
-# ----------------------------------------------------------------------
-# OpenBLAS
-set(OPENBLAS_PREFIX "${INDEX_BINARY_DIR}/openblas_ep-prefix/src/openblas_ep")
+# Openblas
 macro(build_openblas)
     message(STATUS "Building OpenBLAS-${OPENBLAS_VERSION} from source")
-    set(OpenBLAS_INCLUDE_DIR "${OPENBLAS_PREFIX}/include")
-    set(OpenBLAS_LIB_DIR "${OPENBLAS_PREFIX}/lib")
-    set(OPENBLAS_SHARED_LIB
-            "${OPENBLAS_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas${CMAKE_SHARED_LIBRARY_SUFFIX}")
-    set(OPENBLAS_STATIC_LIB
-            "${OPENBLAS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}openblas${CMAKE_STATIC_LIBRARY_SUFFIX}")
+    set (KNOWHERE_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX})
+
     set(OPENBLAS_CMAKE_ARGS
             ${EP_COMMON_CMAKE_ARGS}
             -DCMAKE_BUILD_TYPE=Release
@@ -332,233 +188,57 @@ macro(build_openblas)
             -DCC=gcc
             -DINTERFACE64=0
             -DNUM_THREADS=128
-            -DNO_LAPACKE=1
+            -DNO_LAPACKE=0
             "-DVERSION=${OPENBLAS_VERSION}"
-            "-DCMAKE_INSTALL_PREFIX=${OPENBLAS_PREFIX}"
-            -DCMAKE_INSTALL_LIBDIR=lib)
+            "-DCMAKE_INSTALL_PREFIX=${KNOWHERE_INSTALL_PREFIX}"
+            )
 
     externalproject_add(openblas_ep
-            URL
-            ${OPENBLAS_SOURCE_URL}
-            ${EP_LOG_OPTIONS}
-            CMAKE_ARGS
-            ${OPENBLAS_CMAKE_ARGS}
-            BUILD_COMMAND
-            ${MAKE}
-            ${MAKE_BUILD_ARGS}
-            BUILD_IN_SOURCE
-            1
-            INSTALL_COMMAND
-            ${MAKE}
-            PREFIX=${OPENBLAS_PREFIX}
-            install
-            BUILD_BYPRODUCTS
-            ${OPENBLAS_SHARED_LIB}
-            ${OPENBLAS_STATIC_LIB})
+            URL ${OPENBLAS_SOURCE_URL}
+            URL_MD5 "28cc19a6acbf636f5aab5f10b9a0dfe1"
+            CMAKE_ARGS ${OPENBLAS_CMAKE_ARGS}
+            BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS}
+            PREFIX              ${CMAKE_BINARY_DIR}/3rdparty_download/openblas-subbuild
+            BINARY_DIR          openblas-bin
+            INSTALL_DIR         ${KNOWHERE_INSTALL_PREFIX}
+            )
+
+    ExternalProject_Get_Property(openblas_ep INSTALL_DIR)
+
+    if( NOT IS_DIRECTORY ${INSTALL_DIR}/include )
+        file( MAKE_DIRECTORY "${INSTALL_DIR}/include" )
+    endif()
+
 
-    file(MAKE_DIRECTORY "${OpenBLAS_INCLUDE_DIR}")
     add_library(openblas SHARED IMPORTED)
-    set_target_properties(
-            openblas
+    set_target_properties( openblas
             PROPERTIES
-            IMPORTED_LOCATION "${OPENBLAS_SHARED_LIB}"
-            LIBRARY_OUTPUT_NAME "openblas"
-            INTERFACE_INCLUDE_DIRECTORIES "${OpenBLAS_INCLUDE_DIR}")
+                IMPORTED_GLOBAL     TRUE
+                IMPORTED_LOCATION   ${INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/libopenblas.so
+                INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/${CMAKE_INSTALL_INCLUDEDIR})
+
     add_dependencies(openblas openblas_ep)
-    get_target_property(OpenBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES)
-    set(OpenBLAS_LIBRARIES "${OPENBLAS_SHARED_LIB}")
 endmacro()
 
 if (KNOWHERE_WITH_OPENBLAS)
-    resolve_dependency(OpenBLAS)
-    include_directories(SYSTEM "${OpenBLAS_INCLUDE_DIR}")
-    link_directories(SYSTEM "${OpenBLAS_LIB_DIR}")
+    if (OpenBLAS_SOURCE STREQUAL "AUTO")
+        set (BLA_VENDOR OpenBLAS)
+        find_package(BLAS)
+
+        message(STATUS "Knowhere openblas libraries: ${BLAS_LIBRARIES}")
+        message(STATUS "Knowhere openblas found: ${BLAS_FOUND}")
+
+        if (BLAS_FOUND)
+            add_library(openblas ALIAS BLAS::BLAS)
+        else()
+            build_openblas()
+        endif()
+
+    elseif (OpenBLAS_SOURCE STREQUAL "BUNDLED")
+        build_openblas()
+    elseif (OpenBLAS_SOURCE STREQUAL "SYSTEM")
+        set (BLA_VENDOR OpenBLAS)
+        find_package(BLAS REQUIRED)
+        add_library(openblas ALIAS BLAS::BLAS)
+    endif ()
 endif()
-
-
-# ----------------------------------------------------------------------
-# MKL
-
-macro(build_mkl)
-
-    if (FAISS_WITH_MKL)
-        if (EXISTS "/proc/cpuinfo")
-            FILE(READ /proc/cpuinfo PROC_CPUINFO)
-
-            SET(VENDOR_ID_RX "vendor_id[ \t]*:[ \t]*([a-zA-Z]+)\n")
-            STRING(REGEX MATCH "${VENDOR_ID_RX}" VENDOR_ID "${PROC_CPUINFO}")
-            STRING(REGEX REPLACE "${VENDOR_ID_RX}" "\\1" VENDOR_ID "${VENDOR_ID}")
-
-            if (NOT ${VENDOR_ID} STREQUAL "GenuineIntel")
-                set(FAISS_WITH_MKL OFF)
-            endif ()
-        endif ()
-
-        find_path(MKL_LIB_PATH
-                NAMES "libmkl_intel_ilp64.a" "libmkl_gnu_thread.a" "libmkl_core.a"
-                PATH_SUFFIXES "intel/compilers_and_libraries_${MKL_VERSION}/linux/mkl/lib/intel64/")
-        if (${MKL_LIB_PATH} STREQUAL "MKL_LIB_PATH-NOTFOUND")
-            message(FATAL_ERROR "Could not find MKL libraries")
-        endif ()
-        message(STATUS "MKL lib path = ${MKL_LIB_PATH}")
-
-        set(MKL_LIBS
-                ${MKL_LIB_PATH}/libmkl_intel_ilp64.a
-                ${MKL_LIB_PATH}/libmkl_gnu_thread.a
-                ${MKL_LIB_PATH}/libmkl_core.a
-                )
-    endif ()
-endmacro()
-
-# ----------------------------------------------------------------------
-# FAISS
-
-macro(build_faiss)
-    message(STATUS "Building FAISS-${FAISS_VERSION} from source")
-
-    set(FAISS_PREFIX "${INDEX_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep")
-    set(FAISS_INCLUDE_DIR "${FAISS_PREFIX}/include")
-    set(FAISS_STATIC_LIB
-            "${FAISS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX}")
-
-    if (CCACHE_FOUND)
-        set(FAISS_C_COMPILER "${CCACHE_FOUND} ${CMAKE_C_COMPILER}")
-        if (MILVUS_GPU_VERSION)
-            set(FAISS_CXX_COMPILER "${CMAKE_CXX_COMPILER}")
-            set(FAISS_CUDA_COMPILER "${CCACHE_FOUND} ${CMAKE_CUDA_COMPILER}")
-        else ()
-            set(FAISS_CXX_COMPILER "${CCACHE_FOUND} ${CMAKE_CXX_COMPILER}")
-        endif()
-    else ()
-        set(FAISS_C_COMPILER "${CMAKE_C_COMPILER}")
-        set(FAISS_CXX_COMPILER "${CMAKE_CXX_COMPILER}")
-    endif()
-
-    set(FAISS_CONFIGURE_ARGS
-            "--prefix=${FAISS_PREFIX}"
-            "CC=${FAISS_C_COMPILER}"
-            "CXX=${FAISS_CXX_COMPILER}"
-            "NVCC=${FAISS_CUDA_COMPILER}"
-            "CFLAGS=${EP_C_FLAGS}"
-            "CXXFLAGS=${EP_CXX_FLAGS} -mf16c -O3"
-            --without-python)
-
-    if (FAISS_WITH_MKL)
-        set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
-                "CPPFLAGS=-DFINTEGER=long -DMKL_ILP64 -m64 -I${MKL_LIB_PATH}/../../include"
-                "LDFLAGS=-L${MKL_LIB_PATH}"
-                )
-    else ()
-        message(STATUS "Build Faiss with OpenBlas/LAPACK")
-        if(OpenBLAS_FOUND)
-            set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
-                "LDFLAGS=-L${OpenBLAS_LIB_DIR}")
-        else()
-            set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
-                "LDFLAGS=-L${OPENBLAS_PREFIX}/lib")
-        endif()
-    endif ()
-
-    if (MILVUS_GPU_VERSION)
-        if (NOT MILVUS_CUDA_ARCH OR MILVUS_CUDA_ARCH STREQUAL "DEFAULT")
-            set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
-                "--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}"
-                "--with-cuda-arch=-gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75"
-                )
-        else()
-            STRING(REPLACE ";" " " MILVUS_CUDA_ARCH "${MILVUS_CUDA_ARCH}")
-            set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
-                "--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}"
-                "--with-cuda-arch=${MILVUS_CUDA_ARCH}"
-                )
-        endif ()
-    else ()
-        set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
-                "CPPFLAGS=-DUSE_CPU"
-                --without-cuda)
-    endif ()
-
-    message(STATUS "Building FAISS with configure args -${FAISS_CONFIGURE_ARGS}")
-
-    if (DEFINED ENV{FAISS_SOURCE_URL})
-        set(FAISS_SOURCE_URL "$ENV{FAISS_SOURCE_URL}")
-        externalproject_add(faiss_ep
-                URL
-                ${FAISS_SOURCE_URL}
-                ${EP_LOG_OPTIONS}
-                CONFIGURE_COMMAND
-                "./configure"
-                ${FAISS_CONFIGURE_ARGS}
-                BUILD_COMMAND
-                ${MAKE} ${MAKE_BUILD_ARGS} all
-                BUILD_IN_SOURCE
-                1
-                INSTALL_COMMAND
-                ${MAKE} install
-                BUILD_BYPRODUCTS
-                ${FAISS_STATIC_LIB})
-    else ()
-        externalproject_add(faiss_ep
-                DOWNLOAD_COMMAND
-                ""
-                SOURCE_DIR
-                ${FAISS_SOURCE_DIR}
-                ${EP_LOG_OPTIONS}
-                CONFIGURE_COMMAND
-                "./configure"
-                ${FAISS_CONFIGURE_ARGS}
-                BUILD_COMMAND
-                ${MAKE} ${MAKE_BUILD_ARGS} all
-                BUILD_IN_SOURCE
-                1
-                INSTALL_COMMAND
-                ${MAKE} install
-                BUILD_BYPRODUCTS
-                ${FAISS_STATIC_LIB})
-    endif ()
-
-    if(NOT OpenBLAS_FOUND)
-        message("add faiss dependencies: openblas_ep")
-        ExternalProject_Add_StepDependencies(faiss_ep configure openblas_ep)
-    endif()
-
-    file(MAKE_DIRECTORY "${FAISS_INCLUDE_DIR}")
-    add_library(faiss STATIC IMPORTED)
-
-    set_target_properties(
-            faiss
-            PROPERTIES
-            IMPORTED_LOCATION "${FAISS_STATIC_LIB}"
-            INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}"
-    )
-    if (FAISS_WITH_MKL)
-        set_target_properties(
-                faiss
-                PROPERTIES
-                INTERFACE_LINK_LIBRARIES "${MKL_LIBS}")
-    else ()
-        set_target_properties(
-                faiss
-                PROPERTIES
-                INTERFACE_LINK_LIBRARIES "${OpenBLAS_LIBRARIES}")
-    endif ()
-
-    add_dependencies(faiss faiss_ep)
-
-endmacro()
-
-if (KNOWHERE_WITH_FAISS AND NOT TARGET faiss_ep)
-
-    if (FAISS_WITH_MKL)
-        resolve_dependency(MKL)
-    else ()
-        message("faiss with no mkl")
-    endif ()
-
-    resolve_dependency(FAISS)
-    get_target_property(FAISS_INCLUDE_DIR faiss INTERFACE_INCLUDE_DIRECTORIES)
-    include_directories(SYSTEM "${FAISS_INCLUDE_DIR}")
-    link_directories(SYSTEM ${FAISS_PREFIX}/lib/)
-endif ()
-
-add_subdirectory(thirdparty/NGT)

internal/core/src/index/knowhere/CMakeLists.txt

@@ -15,6 +15,7 @@ include_directories(${INDEX_SOURCE_DIR}/knowhere)
 include_directories(${INDEX_SOURCE_DIR}/thirdparty)
 include_directories(${INDEX_SOURCE_DIR}/thirdparty/NGT/lib)
 
+set(KNOWHERE_THIRDPARTY_SRC ${INDEX_SOURCE_DIR}/thirdparty)
 if (MILVUS_SUPPORT_SPTAG)
     include_directories(${INDEX_SOURCE_DIR}/thirdparty/SPTAG/AnnService)
 
@@ -45,12 +46,11 @@ set(external_srcs
 
 set (LOG_SRC
         knowhere/common/Log.cpp
-        ${MILVUS_THIRDPARTY_SRC}/easyloggingpp/easylogging++.cc
+        ${KNOWHERE_THIRDPARTY_SRC}/easyloggingpp/easylogging++.cc
         )
 add_library(index_log STATIC ${LOG_SRC})
 set_target_properties(index_log PROPERTIES RULE_LAUNCH_COMPILE "")
 set_target_properties(index_log PROPERTIES RULE_LAUNCH_LINK "")
-include_directories(${MILVUS_THIRDPARTY_SRC})
 
 set(config_srcs
         knowhere/archive/KnowhereConfig.cpp

internal/core/src/index/knowhere/cache/Cache.h

@@ -0,0 +1,104 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#pragma once
+
+#include "LRU.h"
+#include "utils/Log.h"
+
+#include <atomic>
+#include <mutex>
+#include <set>
+#include <string>
+
+namespace milvus {
+namespace cache {
+
+template <typename ItemObj>
+class Cache {
+ public:
+    // mem_capacity, units:GB
+    Cache(int64_t capacity_gb, int64_t cache_max_count, const std::string& header = "");
+    ~Cache() = default;
+
+    int64_t
+    usage() const {
+        return usage_;
+    }
+
+    // unit: BYTE
+    int64_t
+    capacity() const {
+        return capacity_;
+    }
+
+    // unit: BYTE
+    void
+    set_capacity(int64_t capacity);
+
+    double
+    freemem_percent() const {
+        return freemem_percent_;
+    }
+
+    void
+    set_freemem_percent(double percent) {
+        freemem_percent_ = percent;
+    }
+
+    size_t
+    size() const;
+
+    bool
+    exists(const std::string& key);
+
+    ItemObj
+    get(const std::string& key);
+
+    void
+    insert(const std::string& key, const ItemObj& item);
+
+    void
+    erase(const std::string& key);
+
+    bool
+    reserve(const int64_t size);
+
+    void
+    print();
+
+    void
+    clear();
+
+ private:
+    void
+    insert_internal(const std::string& key, const ItemObj& item);
+
+    void
+    erase_internal(const std::string& key);
+
+    void
+    free_memory_internal(const int64_t target_size);
+
+ private:
+    std::string header_;
+    int64_t usage_;
+    int64_t capacity_;
+    double freemem_percent_;
+
+    LRU<std::string, ItemObj> lru_;
+    mutable std::mutex mutex_;
+};
+
+}  // namespace cache
+}  // namespace milvus
+
+#include "cache/Cache.inl"

internal/core/src/index/knowhere/cache/Cache.inl

@@ -0,0 +1,191 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+namespace milvus {
+namespace cache {
+
+constexpr double DEFAULT_THRESHOLD_PERCENT = 0.7;
+
+template <typename ItemObj>
+Cache<ItemObj>::Cache(int64_t capacity, int64_t cache_max_count, const std::string& header)
+    : header_(header),
+      usage_(0),
+      capacity_(capacity),
+      freemem_percent_(DEFAULT_THRESHOLD_PERCENT),
+      lru_(cache_max_count) {
+}
+
+template <typename ItemObj>
+void
+Cache<ItemObj>::set_capacity(int64_t capacity) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    if (capacity > 0) {
+        capacity_ = capacity;
+        free_memory_internal(capacity);
+    }
+}
+
+template <typename ItemObj>
+size_t
+Cache<ItemObj>::size() const {
+    std::lock_guard<std::mutex> lock(mutex_);
+    return lru_.size();
+}
+
+template <typename ItemObj>
+bool
+Cache<ItemObj>::exists(const std::string& key) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    return lru_.exists(key);
+}
+
+template <typename ItemObj>
+ItemObj
+Cache<ItemObj>::get(const std::string& key) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    if (!lru_.exists(key)) {
+        return nullptr;
+    }
+    return lru_.get(key);
+}
+
+template <typename ItemObj>
+void
+Cache<ItemObj>::insert(const std::string& key, const ItemObj& item) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    insert_internal(key, item);
+}
+
+template <typename ItemObj>
+void
+Cache<ItemObj>::erase(const std::string& key) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    erase_internal(key);
+}
+
+template <typename ItemObj>
+bool
+Cache<ItemObj>::reserve(const int64_t item_size) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    if (item_size > capacity_) {
+        LOG_SERVER_ERROR_ << header_ << " item size " << (item_size >> 20) << "MB too big to insert into cache capacity"
+                         << (capacity_ >> 20) << "MB";
+        return false;
+    }
+    if (item_size > capacity_ - usage_) {
+        free_memory_internal(capacity_ - item_size);
+    }
+    return true;
+}
+
+template <typename ItemObj>
+void
+Cache<ItemObj>::clear() {
+    std::lock_guard<std::mutex> lock(mutex_);
+    lru_.clear();
+    usage_ = 0;
+    LOG_SERVER_DEBUG_ << header_ << " Clear cache !";
+}
+
+
+template <typename ItemObj>
+void
+Cache<ItemObj>::print() {
+    std::lock_guard<std::mutex> lock(mutex_);
+    size_t cache_count = lru_.size();
+    // for (auto it = lru_.begin(); it != lru_.end(); ++it) {
+    //     LOG_SERVER_DEBUG_ << it->first;
+    // }
+    LOG_SERVER_DEBUG_ << header_ << " [item count]: " << cache_count << ", [usage] " << (usage_ >> 20)
+                     << "MB, [capacity] " << (capacity_ >> 20) << "MB";
+}
+
+template <typename ItemObj>
+void
+Cache<ItemObj>::insert_internal(const std::string& key, const ItemObj& item) {
+    if (item == nullptr) {
+        return;
+    }
+
+    size_t item_size = item->Size();
+
+    // if key already exist, subtract old item size
+    if (lru_.exists(key)) {
+        const ItemObj& old_item = lru_.get(key);
+        usage_ -= old_item->Size();
+    }
+
+    // plus new item size
+    usage_ += item_size;
+
+    // if usage exceed capacity, free some items
+    if (usage_ > capacity_) {
+        LOG_SERVER_DEBUG_ << header_ << " Current usage " << (usage_ >> 20) << "MB is too high for capacity "
+                         << (capacity_ >> 20) << "MB, start free memory";
+        free_memory_internal(capacity_);
+    }
+
+    // insert new item
+    lru_.put(key, item);
+    LOG_SERVER_DEBUG_ << header_ << " Insert " << key << " size: " << (item_size >> 20) << "MB into cache";
+    LOG_SERVER_DEBUG_ << header_ << " Count: " << lru_.size() << ", Usage: " << (usage_ >> 20) << "MB, Capacity: "
+                     << (capacity_ >> 20) << "MB";
+}
+
+template <typename ItemObj>
+void
+Cache<ItemObj>::erase_internal(const std::string& key) {
+    if (!lru_.exists(key)) {
+        return;
+    }
+
+    const ItemObj& item = lru_.get(key);
+    size_t item_size = item->Size();
+
+    lru_.erase(key);
+
+    usage_ -= item_size;
+    LOG_SERVER_DEBUG_ << header_ << " Erase " << key << " size: " << (item_size >> 20) << "MB from cache";
+    LOG_SERVER_DEBUG_ << header_ << " Count: " << lru_.size() << ", Usage: " << (usage_ >> 20) << "MB, Capacity: "
+                     << (capacity_ >> 20) << "MB";
+}
+
+template <typename ItemObj>
+void
+Cache<ItemObj>::free_memory_internal(const int64_t target_size) {
+    int64_t threshold = std::min((int64_t)(capacity_ * freemem_percent_), target_size);
+    int64_t delta_size = usage_ - threshold;
+    if (delta_size <= 0) {
+        delta_size = 1;  // ensure at least one item erased
+    }
+
+    std::set<std::string> key_array;
+    int64_t released_size = 0;
+
+    auto it = lru_.rbegin();
+    while (it != lru_.rend() && released_size < delta_size) {
+        auto& key = it->first;
+        auto& obj_ptr = it->second;
+
+        key_array.emplace(key);
+        released_size += obj_ptr->Size();
+        ++it;
+    }
+
+    LOG_SERVER_DEBUG_ << header_ << " To be released memory size: " << (released_size >> 20) << "MB";
+
+    for (auto& key : key_array) {
+        erase_internal(key);
+    }
+}
+
+}  // namespace cache
+}  // namespace milvus

internal/core/src/index/knowhere/cache/CacheMgr.h

@@ -0,0 +1,71 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#pragma once
+
+#include "Cache.h"
+#include "utils/Log.h"
+
+#include <memory>
+#include <string>
+
+namespace milvus {
+namespace cache {
+
+template <typename ItemObj>
+class CacheMgr {
+ public:
+    virtual uint64_t
+    ItemCount() const;
+
+    virtual bool
+    ItemExists(const std::string& key);
+
+    virtual ItemObj
+    GetItem(const std::string& key);
+
+    virtual void
+    InsertItem(const std::string& key, const ItemObj& data);
+
+    virtual void
+    EraseItem(const std::string& key);
+
+    virtual bool
+    Reserve(const int64_t size);
+
+    virtual void
+    PrintInfo();
+
+    virtual void
+    ClearCache();
+
+    int64_t
+    CacheUsage() const;
+
+    int64_t
+    CacheCapacity() const;
+
+    void
+    SetCapacity(int64_t capacity);
+
+ protected:
+    CacheMgr();
+
+    virtual ~CacheMgr();
+
+ protected:
+    std::shared_ptr<Cache<ItemObj>> cache_;
+};
+
+}  // namespace cache
+}  // namespace milvus
+
+#include "cache/CacheMgr.inl"

internal/core/src/index/knowhere/cache/CacheMgr.inl

@@ -0,0 +1,134 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+namespace milvus {
+namespace cache {
+
+template <typename ItemObj>
+CacheMgr<ItemObj>::CacheMgr() {
+}
+
+template <typename ItemObj>
+CacheMgr<ItemObj>::~CacheMgr() {
+}
+
+template <typename ItemObj>
+uint64_t
+CacheMgr<ItemObj>::ItemCount() const {
+    if (cache_ == nullptr) {
+        LOG_SERVER_ERROR_ << "Cache doesn't exist";
+        return 0;
+    }
+    return (uint64_t)(cache_->size());
+}
+
+template <typename ItemObj>
+bool
+CacheMgr<ItemObj>::ItemExists(const std::string& key) {
+    if (cache_ == nullptr) {
+        LOG_SERVER_ERROR_ << "Cache doesn't exist";
+        return false;
+    }
+    return cache_->exists(key);
+}
+
+template <typename ItemObj>
+ItemObj
+CacheMgr<ItemObj>::GetItem(const std::string& key) {
+    if (cache_ == nullptr) {
+        LOG_SERVER_ERROR_ << "Cache doesn't exist";
+        return nullptr;
+    }
+    return cache_->get(key);
+}
+
+template <typename ItemObj>
+void
+CacheMgr<ItemObj>::InsertItem(const std::string& key, const ItemObj& data) {
+    if (cache_ == nullptr) {
+        LOG_SERVER_ERROR_ << "Cache doesn't exist";
+        return;
+    }
+    cache_->insert(key, data);
+}
+
+template <typename ItemObj>
+void
+CacheMgr<ItemObj>::EraseItem(const std::string& key) {
+    if (cache_ == nullptr) {
+        LOG_SERVER_ERROR_ << "Cache doesn't exist";
+        return;
+    }
+    cache_->erase(key);
+}
+
+template <typename ItemObj>
+bool
+CacheMgr<ItemObj>::Reserve(const int64_t size) {
+    if (cache_ == nullptr) {
+        LOG_SERVER_ERROR_ << "Cache doesn't exist";
+        return false;
+    }
+    return cache_->reserve(size);
+}
+
+template <typename ItemObj>
+void
+CacheMgr<ItemObj>::PrintInfo() {
+    if (cache_ == nullptr) {
+        LOG_SERVER_ERROR_ << "Cache doesn't exist";
+        return;
+    }
+    cache_->print();
+}
+
+template <typename ItemObj>
+void
+CacheMgr<ItemObj>::ClearCache() {
+    if (cache_ == nullptr) {
+        LOG_SERVER_ERROR_ << "Cache doesn't exist";
+        return;
+    }
+    cache_->clear();
+}
+
+template <typename ItemObj>
+int64_t
+CacheMgr<ItemObj>::CacheUsage() const {
+    if (cache_ == nullptr) {
+        LOG_SERVER_ERROR_ << "Cache doesn't exist";
+        return 0;
+    }
+    return cache_->usage();
+}
+
+template <typename ItemObj>
+int64_t
+CacheMgr<ItemObj>::CacheCapacity() const {
+    if (cache_ == nullptr) {
+        LOG_SERVER_ERROR_ << "Cache doesn't exist";
+        return 0;
+    }
+    return cache_->capacity();
+}
+
+template <typename ItemObj>
+void
+CacheMgr<ItemObj>::SetCapacity(int64_t capacity) {
+    if (cache_ == nullptr) {
+        LOG_SERVER_ERROR_ << "Cache doesn't exist";
+        return;
+    }
+    cache_->set_capacity(capacity);
+}
+
+}  // namespace cache
+}  // namespace milvus

internal/core/src/index/knowhere/cache/CpuCacheMgr.cpp

@@ -0,0 +1,55 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#include "cache/CpuCacheMgr.h"
+
+#include <utility>
+
+#include <fiu/fiu-local.h>
+
+#include "utils/Log.h"
+#include "value/config/ServerConfig.h"
+
+namespace milvus {
+namespace cache {
+
+CpuCacheMgr&
+CpuCacheMgr::GetInstance() {
+    static CpuCacheMgr s_mgr;
+    return s_mgr;
+}
+
+CpuCacheMgr::CpuCacheMgr() {
+    cache_ = std::make_shared<Cache<DataObjPtr>>(config.cache.cache_size(), 1UL << 32, "[CACHE CPU]");
+
+    if (config.cache.cpu_cache_threshold() > 0.0) {
+        cache_->set_freemem_percent(config.cache.cpu_cache_threshold());
+    }
+    ConfigMgr::GetInstance().Attach("cache.cache_size", this);
+}
+
+CpuCacheMgr::~CpuCacheMgr() {
+    ConfigMgr::GetInstance().Detach("cache.cache_size", this);
+}
+
+DataObjPtr
+CpuCacheMgr::GetItem(const std::string& key) {
+    auto ret = CacheMgr<DataObjPtr>::GetItem(key);
+    return ret;
+}
+
+void
+CpuCacheMgr::ConfigUpdate(const std::string& name) {
+    SetCapacity(config.cache.cache_size());
+}
+
+}  // namespace cache
+}  // namespace milvus

internal/core/src/index/knowhere/cache/CpuCacheMgr.h

@@ -0,0 +1,44 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include "cache/CacheMgr.h"
+#include "cache/DataObj.h"
+#include "value/config/ConfigMgr.h"
+
+namespace milvus {
+namespace cache {
+
+class CpuCacheMgr : public CacheMgr<DataObjPtr>, public ConfigObserver {
+ public:
+    static CpuCacheMgr&
+    GetInstance();
+
+ private:
+    CpuCacheMgr();
+
+    ~CpuCacheMgr();
+
+ public:
+    DataObjPtr
+    GetItem(const std::string& key) override;
+
+ public:
+    void
+    ConfigUpdate(const std::string& name) override;
+};
+
+}  // namespace cache
+}  // namespace milvus

internal/core/src/index/knowhere/cache/DataObj.h

@@ -0,0 +1,29 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#pragma once
+
+#include <memory>
+
+namespace milvus {
+namespace cache {
+
+class DataObj {
+ public:
+    virtual int64_t
+    Size() = 0;
+    virtual ~DataObj() = default;
+};
+
+using DataObjPtr = std::shared_ptr<DataObj>;
+
+}  // namespace cache
+}  // namespace milvus

internal/core/src/index/knowhere/cache/GpuCacheMgr.cpp

@@ -0,0 +1,63 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#include "cache/GpuCacheMgr.h"
+#include "utils/Log.h"
+#include "value/config/ServerConfig.h"
+
+#include <fiu/fiu-local.h>
+#include <sstream>
+#include <utility>
+
+namespace milvus {
+namespace cache {
+
+#ifdef MILVUS_GPU_VERSION
+std::mutex GpuCacheMgr::global_mutex_;
+std::unordered_map<int64_t, GpuCacheMgrPtr> GpuCacheMgr::instance_;
+
+GpuCacheMgr::GpuCacheMgr(int64_t gpu_id) : gpu_id_(gpu_id) {
+    std::string header = "[CACHE GPU" + std::to_string(gpu_id) + "]";
+    cache_ = std::make_shared<Cache<DataObjPtr>>(config.gpu.cache_size(), 1UL << 32, header);
+
+    if (config.gpu.cache_threshold() > 0.0) {
+        cache_->set_freemem_percent(config.gpu.cache_threshold());
+    }
+    ConfigMgr::GetInstance().Attach("gpu.cache_threshold", this);
+}
+
+GpuCacheMgr::~GpuCacheMgr() {
+    ConfigMgr::GetInstance().Detach("gpu.cache_threshold", this);
+}
+
+GpuCacheMgrPtr
+GpuCacheMgr::GetInstance(int64_t gpu_id) {
+    if (instance_.find(gpu_id) == instance_.end()) {
+        std::lock_guard<std::mutex> lock(global_mutex_);
+        if (instance_.find(gpu_id) == instance_.end()) {
+            instance_[gpu_id] = std::make_shared<GpuCacheMgr>(gpu_id);
+        }
+    }
+    return instance_[gpu_id];
+}
+
+void
+GpuCacheMgr::ConfigUpdate(const std::string& name) {
+    std::lock_guard<std::mutex> lock(global_mutex_);
+    for (auto& it : instance_) {
+        it.second->SetCapacity(config.gpu.cache_size());
+    }
+}
+
+#endif
+
+}  // namespace cache
+}  // namespace milvus

internal/core/src/index/knowhere/cache/GpuCacheMgr.h

@@ -0,0 +1,51 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#include <memory>
+#include <mutex>
+#include <string>
+#include <unordered_map>
+#include <utility>
+
+#include "cache/CacheMgr.h"
+#include "cache/DataObj.h"
+#include "value/config/ConfigMgr.h"
+
+namespace milvus {
+namespace cache {
+
+#ifdef MILVUS_GPU_VERSION
+class GpuCacheMgr;
+using GpuCacheMgrPtr = std::shared_ptr<GpuCacheMgr>;
+using MutexPtr = std::shared_ptr<std::mutex>;
+
+class GpuCacheMgr : public CacheMgr<DataObjPtr>, public ConfigObserver {
+ public:
+    explicit GpuCacheMgr(int64_t gpu_id);
+
+    ~GpuCacheMgr();
+
+    static GpuCacheMgrPtr
+    GetInstance(int64_t gpu_id);
+
+ public:
+    void
+    ConfigUpdate(const std::string& name) override;
+
+ private:
+    int64_t gpu_id_;
+    static std::mutex global_mutex_;
+    static std::unordered_map<int64_t, GpuCacheMgrPtr> instance_;
+};
+#endif
+
+}  // namespace cache
+}  // namespace milvus

internal/core/src/index/knowhere/cache/LRU.h

@@ -0,0 +1,116 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#pragma once
+
+#include <cstddef>
+#include <list>
+#include <stdexcept>
+#include <unordered_map>
+#include <utility>
+
+namespace milvus {
+namespace cache {
+
+template <typename key_t, typename value_t>
+class LRU {
+ public:
+    typedef typename std::pair<key_t, value_t> key_value_pair_t;
+    typedef typename std::list<key_value_pair_t>::iterator list_iterator_t;
+    typedef typename std::list<key_value_pair_t>::reverse_iterator reverse_list_iterator_t;
+
+    explicit LRU(size_t max_size) : max_size_(max_size) {
+    }
+
+    void
+    put(const key_t& key, const value_t& value) {
+        auto it = cache_items_map_.find(key);
+        cache_items_list_.push_front(key_value_pair_t(key, value));
+        if (it != cache_items_map_.end()) {
+            cache_items_list_.erase(it->second);
+            cache_items_map_.erase(it);
+        }
+        cache_items_map_[key] = cache_items_list_.begin();
+
+        if (cache_items_map_.size() > max_size_) {
+            auto last = cache_items_list_.end();
+            last--;
+            cache_items_map_.erase(last->first);
+            cache_items_list_.pop_back();
+        }
+    }
+
+    const value_t&
+    get(const key_t& key) {
+        auto it = cache_items_map_.find(key);
+        if (it == cache_items_map_.end()) {
+            throw std::range_error("There is no such key in cache");
+        } else {
+            cache_items_list_.splice(cache_items_list_.begin(), cache_items_list_, it->second);
+            return it->second->second;
+        }
+    }
+
+    void
+    erase(const key_t& key) {
+        auto it = cache_items_map_.find(key);
+        if (it != cache_items_map_.end()) {
+            cache_items_list_.erase(it->second);
+            cache_items_map_.erase(it);
+        }
+    }
+
+    bool
+    exists(const key_t& key) const {
+        return cache_items_map_.find(key) != cache_items_map_.end();
+    }
+
+    size_t
+    size() const {
+        return cache_items_map_.size();
+    }
+
+    list_iterator_t
+    begin() {
+        iter_ = cache_items_list_.begin();
+        return iter_;
+    }
+
+    list_iterator_t
+    end() {
+        return cache_items_list_.end();
+    }
+
+    reverse_list_iterator_t
+    rbegin() {
+        return cache_items_list_.rbegin();
+    }
+
+    reverse_list_iterator_t
+    rend() {
+        return cache_items_list_.rend();
+    }
+
+    void
+    clear() {
+        cache_items_list_.clear();
+        cache_items_map_.clear();
+    }
+
+ private:
+    std::list<key_value_pair_t> cache_items_list_;
+    std::unordered_map<key_t, list_iterator_t> cache_items_map_;
+    size_t max_size_;
+    list_iterator_t iter_;
+};
+
+}  // namespace cache
+}  // namespace milvus

internal/core/src/index/knowhere/knowhere/archive/KnowhereConfig.cpp

@@ -27,7 +27,7 @@
 #include "utils/ConfigUtils.h"
 #include "utils/Error.h"
 #include "utils/Log.h"
-#include "index/knowhere/knowhere/common/Exception.h"
+#include "knowhere/common/Exception.h"
 
 #include <string>
 #include <vector>

internal/core/src/index/knowhere/log/DeprecatedLog.h

@@ -0,0 +1,119 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#pragma once
+
+/*
+ * Please use LOG_MODULE_LEVEL_C macro in member function of class
+ * and LOG_MODULE_LEVEL_ macro in other functions.
+ */
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+#define SERVER_MODULE_NAME "SERVER"
+#define SERVER_MODULE_CLASS_FUNCTION \
+    LogOut("[%s][%s::%s][%s] ", SERVER_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, GetThreadName().c_str())
+#define SERVER_MODULE_FUNCTION LogOut("[%s][%s][%s] ", SERVER_MODULE_NAME, __FUNCTION__, GetThreadName().c_str())
+
+#define LOG_SERVER_TRACE_C LOG(TRACE) << SERVER_MODULE_CLASS_FUNCTION
+#define LOG_SERVER_DEBUG_C LOG(DEBUG) << SERVER_MODULE_CLASS_FUNCTION
+#define LOG_SERVER_INFO_C LOG(INFO) << SERVER_MODULE_CLASS_FUNCTION
+#define LOG_SERVER_WARNING_C LOG(WARNING) << SERVER_MODULE_CLASS_FUNCTION
+#define LOG_SERVER_ERROR_C LOG(ERROR) << SERVER_MODULE_CLASS_FUNCTION
+#define LOG_SERVER_FATAL_C LOG(FATAL) << SERVER_MODULE_CLASS_FUNCTION
+
+#define LOG_SERVER_TRACE_ LOG(TRACE) << SERVER_MODULE_FUNCTION
+#define LOG_SERVER_DEBUG_ LOG(DEBUG) << SERVER_MODULE_FUNCTION
+#define LOG_SERVER_INFO_ LOG(INFO) << SERVER_MODULE_FUNCTION
+#define LOG_SERVER_WARNING_ LOG(WARNING) << SERVER_MODULE_FUNCTION
+#define LOG_SERVER_ERROR_ LOG(ERROR) << SERVER_MODULE_FUNCTION
+#define LOG_SERVER_FATAL_ LOG(FATAL) << SERVER_MODULE_FUNCTION
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+#define ENGINE_MODULE_NAME "ENGINE"
+#define ENGINE_MODULE_CLASS_FUNCTION \
+    LogOut("[%s][%s::%s][%s] ", ENGINE_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, GetThreadName().c_str())
+#define ENGINE_MODULE_FUNCTION LogOut("[%s][%s][%s] ", ENGINE_MODULE_NAME, __FUNCTION__, GetThreadName().c_str())
+
+#define LOG_ENGINE_TRACE_C LOG(TRACE) << ENGINE_MODULE_CLASS_FUNCTION
+#define LOG_ENGINE_DEBUG_C LOG(DEBUG) << ENGINE_MODULE_CLASS_FUNCTION
+#define LOG_ENGINE_INFO_C LOG(INFO) << ENGINE_MODULE_CLASS_FUNCTION
+#define LOG_ENGINE_WARNING_C LOG(WARNING) << ENGINE_MODULE_CLASS_FUNCTION
+#define LOG_ENGINE_ERROR_C LOG(ERROR) << ENGINE_MODULE_CLASS_FUNCTION
+#define LOG_ENGINE_FATAL_C LOG(FATAL) << ENGINE_MODULE_CLASS_FUNCTION
+
+#define LOG_ENGINE_TRACE_ LOG(TRACE) << ENGINE_MODULE_FUNCTION
+#define LOG_ENGINE_DEBUG_ LOG(DEBUG) << ENGINE_MODULE_FUNCTION
+#define LOG_ENGINE_INFO_ LOG(INFO) << ENGINE_MODULE_FUNCTION
+#define LOG_ENGINE_WARNING_ LOG(WARNING) << ENGINE_MODULE_FUNCTION
+#define LOG_ENGINE_ERROR_ LOG(ERROR) << ENGINE_MODULE_FUNCTION
+#define LOG_ENGINE_FATAL_ LOG(FATAL) << ENGINE_MODULE_FUNCTION
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+#define WRAPPER_MODULE_NAME "WRAPPER"
+#define WRAPPER_MODULE_CLASS_FUNCTION \
+    LogOut("[%s][%s::%s][%s] ", WRAPPER_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, GetThreadName().c_str())
+#define WRAPPER_MODULE_FUNCTION LogOut("[%s][%s][%s] ", WRAPPER_MODULE_NAME, __FUNCTION__, GetThreadName().c_str())
+
+#define LOG_WRAPPER_TRACE_C LOG(TRACE) << WRAPPER_MODULE_CLASS_FUNCTION
+#define LOG_WRAPPER_DEBUG_C LOG(DEBUG) << WRAPPER_MODULE_CLASS_FUNCTION
+#define LOG_WRAPPER_INFO_C LOG(INFO) << WRAPPER_MODULE_CLASS_FUNCTION
+#define LOG_WRAPPER_WARNING_C LOG(WARNING) << WRAPPER_MODULE_CLASS_FUNCTION
+#define LOG_WRAPPER_ERROR_C LOG(ERROR) << WRAPPER_MODULE_CLASS_FUNCTION
+#define LOG_WRAPPER_FATAL_C LOG(FATAL) << WRAPPER_MODULE_CLASS_FUNCTION
+
+#define LOG_WRAPPER_TRACE_ LOG(TRACE) << WRAPPER_MODULE_FUNCTION
+#define LOG_WRAPPER_DEBUG_ LOG(DEBUG) << WRAPPER_MODULE_FUNCTION
+#define LOG_WRAPPER_INFO_ LOG(INFO) << WRAPPER_MODULE_FUNCTION
+#define LOG_WRAPPER_WARNING_ LOG(WARNING) << WRAPPER_MODULE_FUNCTION
+#define LOG_WRAPPER_ERROR_ LOG(ERROR) << WRAPPER_MODULE_FUNCTION
+#define LOG_WRAPPER_FATAL_ LOG(FATAL) << WRAPPER_MODULE_FUNCTION
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+#define STORAGE_MODULE_NAME "STORAGE"
+#define STORAGE_MODULE_CLASS_FUNCTION \
+    LogOut("[%s][%s::%s][%s] ", STORAGE_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, GetThreadName().c_str())
+#define STORAGE_MODULE_FUNCTION LogOut("[%s][%s][%s] ", STORAGE_MODULE_NAME, __FUNCTION__, GetThreadName().c_str())
+
+#define LOG_STORAGE_TRACE_C LOG(TRACE) << STORAGE_MODULE_CLASS_FUNCTION
+#define LOG_STORAGE_DEBUG_C LOG(DEBUG) << STORAGE_MODULE_CLASS_FUNCTION
+#define LOG_STORAGE_INFO_C LOG(INFO) << STORAGE_MODULE_CLASS_FUNCTION
+#define LOG_STORAGE_WARNING_C LOG(WARNING) << STORAGE_MODULE_CLASS_FUNCTION
+#define LOG_STORAGE_ERROR_C LOG(ERROR) << STORAGE_MODULE_CLASS_FUNCTION
+#define LOG_STORAGE_FATAL_C LOG(FATAL) << STORAGE_MODULE_CLASS_FUNCTION
+
+#define LOG_STORAGE_TRACE_ LOG(TRACE) << STORAGE_MODULE_FUNCTION
+#define LOG_STORAGE_DEBUG_ LOG(DEBUG) << STORAGE_MODULE_FUNCTION
+#define LOG_STORAGE_INFO_ LOG(INFO) << STORAGE_MODULE_FUNCTION
+#define LOG_STORAGE_WARNING_ LOG(WARNING) << STORAGE_MODULE_FUNCTION
+#define LOG_STORAGE_ERROR_ LOG(ERROR) << STORAGE_MODULE_FUNCTION
+#define LOG_STORAGE_FATAL_ LOG(FATAL) << STORAGE_MODULE_FUNCTION
+
+/////////////////////////////////////////////////////////////////////////////////////////////////
+#define WAL_MODULE_NAME "WAL"
+#define WAL_MODULE_CLASS_FUNCTION \
+    LogOut("[%s][%s::%s][%s] ", WAL_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, GetThreadName().c_str())
+#define WAL_MODULE_FUNCTION LogOut("[%s][%s][%s] ", WAL_MODULE_NAME, __FUNCTION__, GetThreadName().c_str())
+
+#define LOG_WAL_TRACE_C LOG(TRACE) << WAL_MODULE_CLASS_FUNCTION
+#define LOG_WAL_DEBUG_C LOG(DEBUG) << WAL_MODULE_CLASS_FUNCTION
+#define LOG_WAL_INFO_C LOG(INFO) << WAL_MODULE_CLASS_FUNCTION
+#define LOG_WAL_WARNING_C LOG(WARNING) << WAL_MODULE_CLASS_FUNCTION
+#define LOG_WAL_ERROR_C LOG(ERROR) << WAL_MODULE_CLASS_FUNCTION
+#define LOG_WAL_FATAL_C LOG(FATAL) << WAL_MODULE_CLASS_FUNCTION
+
+#define LOG_WAL_TRACE_ LOG(TRACE) << WAL_MODULE_FUNCTION
+#define LOG_WAL_DEBUG_ LOG(DEBUG) << WAL_MODULE_FUNCTION
+#define LOG_WAL_INFO_ LOG(INFO) << WAL_MODULE_FUNCTION
+#define LOG_WAL_WARNING_ LOG(WARNING) << WAL_MODULE_FUNCTION
+#define LOG_WAL_ERROR_ LOG(ERROR) << WAL_MODULE_FUNCTION
+#define LOG_WAL_FATAL_ LOG(FATAL) << WAL_MODULE_FUNCTION
+
+/////////////////////////////////////////////////////////////////////////////////////////////////////

internal/core/src/index/knowhere/log/Log.cpp

@@ -0,0 +1,108 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#include "log/Log.h"
+INITIALIZE_EASYLOGGINGPP
+
+#include <chrono>
+#include <cstdarg>
+#include <cstdio>
+#include <memory>
+#include <stdexcept>
+#include <string>
+
+namespace milvus {
+
+std::string
+LogOut(const char* pattern, ...) {
+    size_t len = strnlen(pattern, 1024) + 256;
+    auto str_p = std::make_unique<char[]>(len);
+    memset(str_p.get(), 0, len);
+
+    va_list vl;
+    va_start(vl, pattern);
+    vsnprintf(str_p.get(), len, pattern, vl);  // NOLINT
+    va_end(vl);
+
+    return std::string(str_p.get());
+}
+
+void
+SetThreadName(const std::string& name) {
+    // Note: the name cannot exceed 16 bytes
+    pthread_setname_np(pthread_self(), name.c_str());
+}
+
+std::string
+GetThreadName() {
+    std::string thread_name = "unamed";
+    char name[16];
+    size_t len = 16;
+    auto err = pthread_getname_np(pthread_self(), name, len);
+    if (not err) {
+        thread_name = name;
+    }
+
+    return thread_name;
+}
+
+int64_t
+get_now_timestamp() {
+    auto now = std::chrono::system_clock::now().time_since_epoch();
+    return std::chrono::duration_cast<std::chrono::seconds>(now).count();
+}
+
+int64_t
+get_system_boottime() {
+    FILE* uptime = fopen("/proc/uptime", "r");
+    float since_sys_boot, _;
+    auto ret = fscanf(uptime, "%f %f", &since_sys_boot, &_);
+    fclose(uptime);
+    if (ret != 2) {
+        throw std::runtime_error("read /proc/uptime failed.");
+    }
+    return static_cast<int64_t>(since_sys_boot);
+}
+
+int64_t
+get_thread_starttime() {
+    int64_t tid = gettid();
+    int64_t pid = getpid();
+    char filename[256];
+    snprintf(filename, sizeof(filename), "/proc/%ld/task/%ld/stat", pid, tid);
+
+    int64_t val = 0;
+    char comm[16], state;
+    FILE* thread_stat = fopen(filename, "r");
+    auto ret = fscanf(thread_stat, "%ld %s %s ", &val, comm, &state);
+    for (auto i = 4; i < 23; i++) {
+        ret = fscanf(thread_stat, "%ld ", &val);
+        if (i == 22) {
+            break;
+        }
+    }
+    fclose(thread_stat);
+    if (ret != 1) {
+        throw std::runtime_error("read " + std::string(filename) + " failed.");
+    }
+    return val / sysconf(_SC_CLK_TCK);
+}
+
+int64_t
+get_thread_start_timestamp() {
+    try {
+        return get_now_timestamp() - get_system_boottime() + get_thread_starttime();
+    } catch (...) {
+        return 0;
+    }
+}
+
+}  // namespace milvus

internal/core/src/index/knowhere/log/Log.h

@@ -0,0 +1,77 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#pragma once
+
+#include <string>
+
+#include <sys/types.h>
+#include <unistd.h>
+#include "easyloggingpp/easylogging++.h"
+
+#include "log/DeprecatedLog.h"
+
+namespace milvus {
+
+#if __GLIBC__ == 2 && __GLIBC_MINOR__ < 30
+#include <sys/syscall.h>
+#define gettid() syscall(SYS_gettid)
+#endif
+
+// Log message format: %timestamp | %request_id | %level | %collection_name | %client_id | %client_tag | %client_ipport
+// | %thread_id | %thread_start_timestamp | %command_tag | %module | %error_code | %message
+
+#define VAR_REQUEST_ID (context->request_id())
+#define VAR_COLLECTION_NAME (context->collection_name())
+#define VAR_CLIENT_ID ("")
+#define VAR_CLIENT_TAG (context->client_tag())
+#define VAR_CLIENT_IPPORT (context->client_ipport())
+#define VAR_THREAD_ID (gettid())
+#define VAR_THREAD_START_TIMESTAMP (get_thread_start_timestamp())
+#define VAR_COMMAND_TAG (context->command_tag())
+
+// Use this macro whenever possible
+// Depends variables: context Context
+#define MLOG(level, module, error_code)                                                                                \
+    LOG(level) << " | " << VAR_REQUEST_ID << " | " << #level << " | " << VAR_COLLECTION_NAME << " | " << VAR_CLIENT_ID \
+               << " | " << VAR_CLIENT_TAG << " | " << VAR_CLIENT_IPPORT << " | " << VAR_THREAD_ID << " | "             \
+               << VAR_THREAD_START_TIMESTAMP << " | " << VAR_COMMAND_TAG << " | " << #module << " | " << error_code    \
+               << " | "
+
+// Use in some background process only
+#define MLOG_(level, module, error_code)                                                 \
+    LOG(level) << " | "                                                                  \
+               << ""                                                                     \
+               << " | " << #level << " | "                                               \
+               << ""                                                                     \
+               << " | "                                                                  \
+               << ""                                                                     \
+               << " | "                                                                  \
+               << ""                                                                     \
+               << " | "                                                                  \
+               << ""                                                                     \
+               << " | " << VAR_THREAD_ID << " | " << VAR_THREAD_START_TIMESTAMP << " | " \
+               << ""                                                                     \
+               << " | " << #module << " | " << error_code << " | "
+
+std::string
+LogOut(const char* pattern, ...);
+
+void
+SetThreadName(const std::string& name);
+
+std::string
+GetThreadName();
+
+int64_t
+get_thread_start_timestamp();
+
+}  // namespace milvus

internal/core/src/index/knowhere/log/LogMgr.cpp

@@ -0,0 +1,262 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#include <fiu/fiu-local.h>
+#include <libgen.h>
+#include <cctype>
+#include <string>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include <boost/filesystem.hpp>
+
+#include "log/LogMgr.h"
+#include "utils/Status.h"
+#include "value/config/ServerConfig.h"
+
+namespace milvus {
+
+int LogMgr::trace_idx = 0;
+int LogMgr::global_idx = 0;
+int LogMgr::debug_idx = 0;
+int LogMgr::info_idx = 0;
+int LogMgr::warning_idx = 0;
+int LogMgr::error_idx = 0;
+int LogMgr::fatal_idx = 0;
+int64_t LogMgr::logs_delete_exceeds = 1;
+bool LogMgr::enable_log_delete = false;
+
+Status
+LogMgr::InitLog(bool trace_enable,
+                const std::string& level,
+                const std::string& logs_path,
+                const std::string& filename,
+                int64_t max_log_file_size,
+                int64_t log_rotate_num,
+                bool log_to_stdout,
+                bool log_to_file) {
+    try {
+        auto enables = parse_level(level);
+        enables["trace"] = trace_enable;
+        LogMgr log_mgr(logs_path);
+        log_mgr.Default()
+            .Filename(filename)
+            .Level(enables)
+            .To(log_to_stdout, log_to_file)
+            .Rotate(max_log_file_size, log_rotate_num)
+            .Setup();
+    } catch (std::exception& ex) {
+        return Status(SERVER_UNEXPECTED_ERROR, ex.what());
+    }
+
+    return Status::OK();
+}
+
+// TODO(yzb) : change the easylogging library to get the log level from parameter rather than filename
+void
+LogMgr::RolloutHandler(const char* filename, std::size_t size, el::Level level) {
+    char* dirc = strdup(filename);
+    char* basec = strdup(filename);
+    char* dir = dirname(dirc);
+    char* base = basename(basec);
+
+    std::string s(base);
+    std::vector<std::string> list = {"\\", " ", "\'", "\"", "*", "\?", "{", "}", ";", "<",
+                                     ">",  "|", "^",  "&",  "$", "#",  "!", "`", "~"};
+    std::string::size_type position;
+    for (auto& substr : list) {
+        position = 0;
+        while ((position = s.find_first_of(substr, position)) != std::string::npos) {
+            s.insert(position, "\\");
+            position += 2;
+        }
+    }
+    std::string m(std::string(dir) + "/" + s);
+    try {
+        switch (level) {
+            case el::Level::Trace: {
+                rename_and_delete(m, ++trace_idx);
+                break;
+            }
+            case el::Level::Global: {
+                rename_and_delete(m, ++global_idx);
+                break;
+            }
+            case el::Level::Debug: {
+                rename_and_delete(m, ++debug_idx);
+                break;
+            }
+            case el::Level::Info: {
+                rename_and_delete(m, ++info_idx);
+                break;
+            }
+            case el::Level::Warning: {
+                rename_and_delete(m, ++warning_idx);
+                break;
+            }
+            case el::Level::Error: {
+                rename_and_delete(m, ++error_idx);
+                break;
+            }
+            case el::Level::Fatal: {
+                rename_and_delete(m, ++fatal_idx);
+                break;
+            }
+            default: {
+                break;
+            }
+        }
+    } catch (const std::exception& exc) {
+        std::cerr << exc.what() << ". Exception throws from RolloutHandler." << std::endl;
+    }
+}
+
+LogMgr::LogMgr(std::string log_path) : logs_path_(std::move(log_path)) {
+}
+
+LogMgr&
+LogMgr::Default() {
+    el_config_.setToDefault();
+    el_config_.setGlobally(el::ConfigurationType::Format, "[%datetime][%level]%msg");
+    el_config_.setGlobally(el::ConfigurationType::SubsecondPrecision, "3");
+    el_config_.setGlobally(el::ConfigurationType::PerformanceTracking, "false");
+
+    return *this;
+}
+
+LogMgr&
+LogMgr::Filename(const std::string& filename) {
+    std::string logs_reg_path = logs_path_.rfind('/') == logs_path_.length() - 1 ? logs_path_ : logs_path_ + "/";
+
+    std::string log_file = logs_reg_path + filename;
+
+    /* Set set log file at Global level to make all level log output to the same log file*/
+    el_config_.set(el::Level::Global, el::ConfigurationType::Filename, log_file.c_str());
+
+    return *this;
+}
+
+LogMgr&
+LogMgr::Level(std::unordered_map<std::string, bool>& enables) {
+    fiu_do_on("LogMgr.Level.trace_enable_to_false", enables["trace"] = false);
+    enable(el_config_, el::Level::Trace, enables["trace"]);
+
+    fiu_do_on("LogMgr.Level.info_enable_to_false", enables["info"] = false);
+    enable(el_config_, el::Level::Info, enables["info"]);
+
+    fiu_do_on("LogMgr.Level.debug_enable_to_false", enables["debug"] = false);
+    enable(el_config_, el::Level::Debug, enables["debug"]);
+
+    fiu_do_on("LogMgr.Level.warning_enable_to_false", enables["warning"] = false);
+    enable(el_config_, el::Level::Warning, enables["warning"]);
+
+    fiu_do_on("LogMgr.Level.error_enable_to_false", enables["error"] = false);
+    enable(el_config_, el::Level::Error, enables["error"]);
+
+    fiu_do_on("LogMgr.Level.fatal_enable_to_false", enables["fatal"] = false);
+    enable(el_config_, el::Level::Fatal, enables["fatal"]);
+
+    return *this;
+}
+
+LogMgr&
+LogMgr::To(bool log_to_stdout, bool log_to_file) {
+    el_config_.setGlobally(el::ConfigurationType::ToStandardOutput, (log_to_stdout ? "true" : "false"));
+    el_config_.setGlobally(el::ConfigurationType::ToFile, (log_to_file ? "true" : "false"));
+
+    return *this;
+}
+
+LogMgr&
+LogMgr::Rotate(int64_t max_log_file_size, int64_t log_rotate_num) {
+    fiu_do_on("LogMgr.Rotate.set_max_log_size_small_than_min", max_log_file_size = MAX_LOG_FILE_SIZE_MIN - 1);
+    if (max_log_file_size < MAX_LOG_FILE_SIZE_MIN || max_log_file_size > MAX_LOG_FILE_SIZE_MAX) {
+        std::string msg = "max_log_file_size must in range[" + std::to_string(MAX_LOG_FILE_SIZE_MIN) + ", " +
+                          std::to_string(MAX_LOG_FILE_SIZE_MAX) + "], now is " + std::to_string(max_log_file_size);
+        throw std::runtime_error(msg);
+    }
+
+    el_config_.setGlobally(el::ConfigurationType::MaxLogFileSize, std::to_string(max_log_file_size));
+    el::Loggers::addFlag(el::LoggingFlag::StrictLogFileSizeCheck);
+    el::Helpers::installPreRollOutCallback(LogMgr::RolloutHandler);
+    el::Loggers::addFlag(el::LoggingFlag::DisableApplicationAbortOnFatalLog);
+
+    // set delete_exceeds = 0 means disable throw away log file even they reach certain limit.
+    if (log_rotate_num != 0) {
+        fiu_do_on("LogMgr.Rotate.delete_exceeds_small_than_min", log_rotate_num = LOG_ROTATE_NUM_MIN - 1);
+        if (log_rotate_num < LOG_ROTATE_NUM_MIN || log_rotate_num > LOG_ROTATE_NUM_MAX) {
+            std::string msg = "log_rotate_num must in range[" + std::to_string(LOG_ROTATE_NUM_MIN) + ", " +
+                              std::to_string(LOG_ROTATE_NUM_MAX) + "], now is " + std::to_string(log_rotate_num);
+            throw std::runtime_error(msg);
+        }
+
+        /* global variable */
+        enable_log_delete = true;
+        logs_delete_exceeds = log_rotate_num;
+    }
+
+    return *this;
+}
+
+void
+LogMgr::Setup() {
+    el::Loggers::reconfigureLogger("default", el_config_);
+}
+
+void
+LogMgr::rename_and_delete(const std::string& filename, int64_t idx) {
+    std::string target_filename = filename + "." + std::to_string(idx);
+    rename(filename.c_str(), target_filename.c_str());
+    if (enable_log_delete && idx - logs_delete_exceeds > 0) {
+        std::string to_delete = filename + "." + std::to_string(trace_idx - logs_delete_exceeds);
+        boost::filesystem::remove(to_delete);
+    }
+}
+
+std::unordered_map<std::string, bool>
+LogMgr::parse_level(const std::string& level) {
+    std::unordered_map<std::string, bool> enables{
+        {"debug", false}, {"info", false}, {"warning", false}, {"error", false}, {"fatal", false},
+    };
+
+    std::unordered_map<std::string, int64_t> level_to_int{
+        {"debug", 5}, {"info", 4}, {"warning", 3}, {"error", 2}, {"fatal", 1},
+    };
+
+    switch (level_to_int[level]) {
+        case 5:
+            enables["debug"] = true;
+        case 4:
+            enables["info"] = true;
+        case 3:
+            enables["warning"] = true;
+        case 2:
+            enables["error"] = true;
+        case 1:
+            enables["fatal"] = true;
+            break;
+        default: {
+            std::string msg = "Cannot parse level " + level +
+                              ": invalid log level, must be one of debug, info, warning, error, fatal.";
+            throw std::runtime_error(msg);
+        }
+    }
+    return enables;
+}
+
+void
+LogMgr::enable(el::Configurations& default_conf, el::Level level, bool enable) {
+    std::string enable_str = enable ? "true" : "false";
+    default_conf.set(level, el::ConfigurationType::Enabled, enable_str);
+}
+
+}  // namespace milvus

internal/core/src/index/knowhere/log/LogMgr.h

@@ -0,0 +1,95 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#pragma once
+
+#include "easyloggingpp/easylogging++.h"
+#include "utils/Status.h"
+#include "value/config/ServerConfig.h"
+
+#include <sstream>
+#include <string>
+#include <unordered_map>
+
+namespace milvus {
+
+class LogMgr {
+ public:
+    static Status
+    InitLog(bool trace_enable,
+            const std::string& level,
+            const std::string& logs_path,
+            const std::string& filename,
+            int64_t max_log_file_size,
+            int64_t delete_exceeds,
+            bool log_to_stdout,
+            bool log_to_file);
+
+    static void
+    RolloutHandler(const char* filename, std::size_t size, el::Level level);
+
+ private:
+    explicit LogMgr(std::string log_path);
+
+    LogMgr&
+    Default();
+
+    LogMgr&
+    Filename(const std::string& filename);
+
+    /* Non-const for fiu to injecting error */
+    LogMgr&
+    Level(std::unordered_map<std::string, bool>& enables);
+
+    LogMgr&
+    To(bool log_to_stdout, bool log_to_file);
+
+    LogMgr&
+    Rotate(int64_t max_log_file_size, int64_t log_rotate_num);
+
+    void
+    Setup();
+
+ private:
+    static void
+    rename_and_delete(const std::string& filename, int64_t idx);
+
+    static std::unordered_map<std::string, bool>
+    parse_level(const std::string& level);
+
+    /**
+     * @brief Configures if output corresponding level log
+     */
+    static void
+    enable(el::Configurations& default_conf, el::Level level, bool enable);
+
+ private:
+    el::Configurations el_config_;
+    std::string logs_path_;
+
+ private:
+    static int trace_idx;
+    static int global_idx;
+    static int debug_idx;
+    static int info_idx;
+    static int warning_idx;
+    static int error_idx;
+    static int fatal_idx;
+    static int64_t logs_delete_exceeds;
+    static bool enable_log_delete;
+
+    const int64_t MAX_LOG_FILE_SIZE_MIN = 536870912;  /* 512 MB */
+    const int64_t MAX_LOG_FILE_SIZE_MAX = 4294967296; /* 4 GB */
+    const int64_t LOG_ROTATE_NUM_MIN = 0;
+    const int64_t LOG_ROTATE_NUM_MAX = 1024;
+};
+
+}  // namespace milvus

internal/core/src/index/knowhere/utils/BlockingQueue.h

@@ -0,0 +1,95 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#pragma once
+
+#include <assert.h>
+#include <condition_variable>
+#include <iostream>
+#include <queue>
+#include <vector>
+
+namespace milvus {
+
+template <typename T>
+class BlockingQueue {
+ public:
+    BlockingQueue() : mtx(), full_(), empty_() {
+    }
+
+    virtual ~BlockingQueue() {
+    }
+
+    BlockingQueue(const BlockingQueue& rhs) = delete;
+
+    BlockingQueue&
+    operator=(const BlockingQueue& rhs) = delete;
+
+    void
+    Put(const T& task) {
+        std::unique_lock<std::mutex> lock(mtx);
+        full_.wait(lock, [this] { return (queue_.size() < capacity_); });
+        queue_.push(task);
+        empty_.notify_all();
+    }
+
+    T
+    Take() {
+        std::unique_lock<std::mutex> lock(mtx);
+        empty_.wait(lock, [this] { return !queue_.empty(); });
+        T front(queue_.front());
+        queue_.pop();
+        full_.notify_all();
+        return front;
+    }
+
+    T
+    Front() {
+        std::unique_lock<std::mutex> lock(mtx);
+        empty_.wait(lock, [this] { return !queue_.empty(); });
+        T front(queue_.front());
+        return front;
+    }
+
+    T
+    Back() {
+        std::unique_lock<std::mutex> lock(mtx);
+        empty_.wait(lock, [this] { return !queue_.empty(); });
+        T back(queue_.back());
+        return back;
+    }
+
+    size_t
+    Size() const {
+        std::lock_guard<std::mutex> lock(mtx);
+        return queue_.size();
+    }
+
+    bool
+    Empty() const {
+        std::unique_lock<std::mutex> lock(mtx);
+        return queue_.empty();
+    }
+
+    void
+    SetCapacity(const size_t capacity) {
+        capacity_ = (capacity > 0 ? capacity : capacity_);
+    }
+
+ protected:
+    mutable std::mutex mtx;
+    std::condition_variable full_;
+    std::condition_variable empty_;
+    std::queue<T> queue_;
+    size_t capacity_ = 32;
+};
+
+}  // namespace milvus

internal/core/src/index/knowhere/utils/CommonUtil.cpp

@@ -0,0 +1,243 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#include "utils/CommonUtil.h"
+#include "utils/Log.h"
+
+#include <dirent.h>
+#include <fiu/fiu-local.h>
+#include <pwd.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <boost/filesystem.hpp>
+#include <iostream>
+#include <vector>
+
+namespace milvus {
+
+namespace fs = boost::filesystem;
+
+bool
+CommonUtil::IsDirectoryExist(const std::string& path) {
+    DIR* dp = nullptr;
+    if ((dp = opendir(path.c_str())) == nullptr) {
+        return false;
+    }
+
+    closedir(dp);
+    return true;
+}
+
+Status
+CommonUtil::CreateDirectory(const std::string& path) {
+    if (path.empty()) {
+        return Status::OK();
+    }
+
+    struct stat directory_stat;
+    int status = stat(path.c_str(), &directory_stat);
+    if (status == 0) {
+        return Status::OK();  // already exist
+    }
+
+    fs::path fs_path(path);
+    fs::path parent_path = fs_path.parent_path();
+    Status err_status = CreateDirectory(parent_path.string());
+    fiu_do_on("CommonUtil.CreateDirectory.create_parent_fail", err_status = Status(SERVER_INVALID_ARGUMENT, ""));
+    if (!err_status.ok()) {
+        return err_status;
+    }
+
+    status = stat(path.c_str(), &directory_stat);
+    if (status == 0) {
+        return Status::OK();  // already exist
+    }
+
+    int makeOK = mkdir(path.c_str(), S_IRWXU | S_IRGRP | S_IROTH);
+    fiu_do_on("CommonUtil.CreateDirectory.create_dir_fail", makeOK = 1);
+    if (makeOK != 0) {
+        return Status(SERVER_UNEXPECTED_ERROR, "failed to create directory: " + path);
+    }
+
+    return Status::OK();
+}
+
+namespace {
+void
+RemoveDirectory(const std::string& path) {
+    DIR* dir = nullptr;
+    const int32_t buf_size = 256;
+    char file_name[buf_size];
+
+    std::string folder_name = path + "/%s";
+    if ((dir = opendir(path.c_str())) != nullptr) {
+        struct dirent* dmsg;
+        while ((dmsg = readdir(dir)) != nullptr) {
+            if (strcmp(dmsg->d_name, ".") != 0 && strcmp(dmsg->d_name, "..") != 0) {
+                snprintf(file_name, buf_size, folder_name.c_str(), dmsg->d_name);
+                std::string tmp = file_name;
+                if (tmp.find('.') == std::string::npos) {
+                    RemoveDirectory(file_name);
+                }
+                remove(file_name);
+            }
+        }
+    }
+
+    if (dir != nullptr) {
+        closedir(dir);
+    }
+    remove(path.c_str());
+}
+}  // namespace
+
+Status
+CommonUtil::DeleteDirectory(const std::string& path) {
+    if (path.empty()) {
+        return Status::OK();
+    }
+
+    struct stat directory_stat;
+    int statOK = stat(path.c_str(), &directory_stat);
+    if (statOK != 0) {
+        return Status::OK();
+    }
+
+    RemoveDirectory(path);
+    return Status::OK();
+}
+
+bool
+CommonUtil::IsFileExist(const std::string& path) {
+    return (access(path.c_str(), F_OK) == 0);
+}
+
+uint64_t
+CommonUtil::GetFileSize(const std::string& path) {
+    struct stat file_info;
+    if (stat(path.c_str(), &file_info) < 0) {
+        return 0;
+    }
+
+    return static_cast<uint64_t>(file_info.st_size);
+}
+
+std::string
+CommonUtil::GetFileName(std::string filename) {
+    int pos = filename.find_last_of('/');
+    return filename.substr(pos + 1);
+}
+
+std::string
+CommonUtil::GetExePath() {
+    const int64_t buf_len = 1024;
+    char buf[buf_len];
+    int64_t cnt = readlink("/proc/self/exe", buf, buf_len);
+    fiu_do_on("CommonUtil.GetExePath.readlink_fail", cnt = -1);
+    if (cnt < 0 || cnt >= buf_len) {
+        return "";
+    }
+
+    buf[cnt] = '\0';
+
+    std::string exe_path = buf;
+    fiu_do_on("CommonUtil.GetExePath.exe_path_error", exe_path = "/");
+    if (exe_path.rfind('/') != exe_path.length() - 1) {
+        std::string sub_str = exe_path.substr(0, exe_path.rfind('/'));
+        return sub_str + "/";
+    }
+    return exe_path;
+}
+
+bool
+CommonUtil::TimeStrToTime(const std::string& time_str,
+                          time_t& time_integer,
+                          tm& time_struct,
+                          const std::string& format) {
+    time_integer = 0;
+    memset(&time_struct, 0, sizeof(tm));
+
+    int ret = sscanf(time_str.c_str(), format.c_str(), &(time_struct.tm_year), &(time_struct.tm_mon),
+                     &(time_struct.tm_mday), &(time_struct.tm_hour), &(time_struct.tm_min), &(time_struct.tm_sec));
+    if (ret <= 0) {
+        return false;
+    }
+
+    time_struct.tm_year -= 1900;
+    time_struct.tm_mon--;
+    time_integer = mktime(&time_struct);
+
+    return true;
+}
+
+void
+CommonUtil::GetCurrentTimeStr(std::string& time_str) {
+    auto t = std::time(nullptr);
+    struct tm ltm;
+    localtime_r(&t, &ltm);
+
+    time_str = "";
+    time_str += std::to_string(ltm.tm_year + 1900);
+    time_str += "-";
+    time_str += std::to_string(ltm.tm_mon + 1);
+    time_str += "-";
+    time_str += std::to_string(ltm.tm_mday);
+    time_str += "_";
+    time_str += std::to_string(ltm.tm_hour);
+    time_str += ":";
+    time_str += std::to_string(ltm.tm_min);
+    time_str += ":";
+    time_str += std::to_string(ltm.tm_sec);
+}
+
+void
+CommonUtil::ConvertTime(time_t time_integer, tm& time_struct) {
+    localtime_r(&time_integer, &time_struct);
+}
+
+void
+CommonUtil::ConvertTime(tm time_struct, time_t& time_integer) {
+    time_integer = mktime(&time_struct);
+}
+
+std::string
+CommonUtil::ConvertSize(int64_t size) {
+    const int64_t gb = 1024ll * 1024 * 1024;
+    const int64_t mb = 1024ll * 1024;
+    const int64_t kb = 1024ll;
+    if (size % gb == 0) {
+        return std::to_string(size / gb) + "GB";
+    } else if (size % mb == 0) {
+        return std::to_string(size / mb) + "MB";
+    } else if (size % kb == 0) {
+        return std::to_string(size / kb) + "KB";
+    } else {
+        return std::to_string(size);
+    }
+}
+
+#ifdef ENABLE_CPU_PROFILING
+std::string
+CommonUtil::GetCurrentTimeStr() {
+    time_t tt;
+    time(&tt);
+    tt = tt + 8 * 60;
+    tm t;
+    gmtime_r(&tt, &t);
+
+    std::string str = std::to_string(t.tm_year + 1900) + "_" + std::to_string(t.tm_mon + 1) + "_" +
+                      std::to_string(t.tm_mday) + "_" + std::to_string(t.tm_hour) + "_" + std::to_string(t.tm_min) +
+                      "_" + std::to_string(t.tm_sec);
+    return str;
+}
+#endif
+
+}  // namespace milvus

internal/core/src/index/knowhere/utils/CommonUtil.h

@@ -0,0 +1,62 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#pragma once
+
+#include "utils/Status.h"
+
+#include <time.h>
+#include <string>
+
+namespace milvus {
+
+class CommonUtil {
+ public:
+    static bool
+    IsFileExist(const std::string& path);
+    static uint64_t
+    GetFileSize(const std::string& path);
+    static bool
+    IsDirectoryExist(const std::string& path);
+    static Status
+    CreateDirectory(const std::string& path);
+    static Status
+    DeleteDirectory(const std::string& path);
+
+    static std::string
+    GetFileName(std::string filename);
+    static std::string
+    GetExePath();
+
+    static bool
+    TimeStrToTime(const std::string& time_str,
+                  time_t& time_integer,
+                  tm& time_struct,
+                  const std::string& format = "%d-%d-%d %d:%d:%d");
+
+    static void
+    GetCurrentTimeStr(std::string& time_str);
+
+    static void
+    ConvertTime(time_t time_integer, tm& time_struct);
+    static void
+    ConvertTime(tm time_struct, time_t& time_integer);
+
+    static std::string
+    ConvertSize(int64_t size);
+
+#ifdef ENABLE_CPU_PROFILING
+    static std::string
+    GetCurrentTimeStr();
+#endif
+};
+
+}  // namespace milvus

internal/core/src/index/knowhere/utils/ConfigUtils.cpp

@@ -0,0 +1,320 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#include "utils/ConfigUtils.h"
+#include "utils/Log.h"
+#include "utils/StringHelpFunctions.h"
+
+#include <arpa/inet.h>
+#include <algorithm>
+#include <cmath>
+#ifdef MILVUS_GPU_VERSION
+#include <cuda_runtime.h>
+#endif
+#include <fiu/fiu-local.h>
+#include <sys/stat.h>
+#include <sys/sysinfo.h>
+#include <unistd.h>
+#include <limits>
+#include <regex>
+#include <set>
+#include <unordered_map>
+
+#if defined(__x86_64__)
+#define THREAD_MULTIPLY_CPU 1
+#elif defined(__powerpc64__)
+#define THREAD_MULTIPLY_CPU 4
+#else
+#define THREAD_MULTIPLY_CPU 1
+#endif
+
+namespace milvus {
+namespace server {
+
+std::unordered_map<std::string, int64_t> BYTE_UNITS = {
+    {"b", 1},
+    {"k", 1024},
+    {"m", 1024 * 1024},
+    {"g", 1024 * 1024 * 1024},
+};
+
+bool
+is_number(const std::string& s) {
+    return !s.empty() && std::find_if(s.begin(), s.end(), [](unsigned char c) { return !std::isdigit(c); }) == s.end();
+}
+
+bool
+is_alpha(const std::string& s) {
+    return !s.empty() && std::find_if(s.begin(), s.end(), [](unsigned char c) { return !std::isalpha(c); }) == s.end();
+}
+
+std::string
+str_tolower(std::string s) {
+    std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return std::tolower(c); });
+    return s;
+}
+
+int64_t
+parse_bytes(const std::string& str, std::string& err) {
+    try {
+        std::string s = str;
+        if (is_number(s)) {
+            return std::stoll(s);
+        }
+        if (s.length() == 0) {
+            return 0;
+        }
+
+        auto last_two = s.substr(s.length() - 2, 2);
+        auto last_one = s.substr(s.length() - 1);
+        if (is_alpha(last_two) && is_alpha(last_one)) {
+            if (last_one == "b" or last_one == "B") {
+                s = s.substr(0, s.length() - 1);
+            }
+        }
+        auto& units = BYTE_UNITS;
+        auto suffix = str_tolower(s.substr(s.length() - 1));
+
+        std::string digits_part;
+        if (is_number(suffix)) {
+            digits_part = s;
+            suffix = 'b';
+        } else {
+            digits_part = s.substr(0, s.length() - 1);
+        }
+
+        if (units.find(suffix) != units.end() or is_number(suffix)) {
+            auto digits = std::stoll(digits_part);
+            return digits * units[suffix];
+        } else {
+            std::stringstream ss;
+            ss << "The specified value for memory (" << str << ") should specify the units."
+               << "The postfix should be one of the `b` `k` `m` `g` characters";
+            err = ss.str();
+        }
+    } catch (...) {
+        err = "Unknown error happened on parse bytes.";
+    }
+    return 0;
+}
+
+bool
+GetSystemMemInfo(int64_t& total_mem, int64_t& free_mem) {
+    struct sysinfo info;
+    int ret = sysinfo(&info);
+    total_mem = info.totalram;
+    free_mem = info.freeram;
+
+    return ret == 0;  // succeed 0, failed -1
+}
+
+bool
+GetSystemAvailableThreads(int64_t& thread_count) {
+    // threadCnt = std::thread::hardware_concurrency();
+    thread_count = sysconf(_SC_NPROCESSORS_CONF);
+    thread_count *= THREAD_MULTIPLY_CPU;
+    fiu_do_on("GetSystemAvailableThreads.zero_thread", thread_count = 0);
+
+    if (thread_count == 0) {
+        thread_count = 8;
+    }
+
+    return true;
+}
+
+Status
+ValidateGpuIndex(int32_t gpu_index) {
+#ifdef MILVUS_GPU_VERSION
+    int num_devices = 0;
+    auto cuda_err = cudaGetDeviceCount(&num_devices);
+    fiu_do_on("config.ValidateGpuIndex.get_device_count_fail", cuda_err = cudaError::cudaErrorUnknown);
+
+    if (cuda_err != cudaSuccess) {
+        std::string msg = "Failed to get gpu card number, cuda error:" + std::to_string(cuda_err);
+        LOG_SERVER_ERROR_ << msg;
+        return Status(SERVER_UNEXPECTED_ERROR, msg);
+    }
+
+    if (gpu_index >= num_devices) {
+        std::string msg = "Invalid gpu index: " + std::to_string(gpu_index);
+        LOG_SERVER_ERROR_ << msg;
+        return Status(SERVER_INVALID_ARGUMENT, msg);
+    }
+#endif
+
+    return Status::OK();
+}
+
+#ifdef MILVUS_GPU_VERSION
+Status
+GetGpuMemory(int32_t gpu_index, int64_t& memory) {
+    fiu_return_on("config.GetGpuMemory.return_error", Status(SERVER_UNEXPECTED_ERROR, ""));
+
+    cudaDeviceProp deviceProp;
+    auto cuda_err = cudaGetDeviceProperties(&deviceProp, gpu_index);
+    if (cuda_err) {
+        std::string msg = "Failed to get gpu properties for gpu" + std::to_string(gpu_index) +
+                          " , cuda error:" + std::to_string(cuda_err);
+        LOG_SERVER_ERROR_ << msg;
+        return Status(SERVER_UNEXPECTED_ERROR, msg);
+    }
+
+    memory = deviceProp.totalGlobalMem;
+    return Status::OK();
+}
+#endif
+
+Status
+ValidateIpAddress(const std::string& ip_address) {
+    struct in_addr address;
+
+    int result = inet_pton(AF_INET, ip_address.c_str(), &address);
+    fiu_do_on("config.ValidateIpAddress.error_ip_result", result = 2);
+
+    switch (result) {
+        case 1:
+            return Status::OK();
+        case 0: {
+            std::string msg = "Invalid IP address: " + ip_address;
+            LOG_SERVER_ERROR_ << msg;
+            return Status(SERVER_INVALID_ARGUMENT, msg);
+        }
+        default: {
+            std::string msg = "IP address conversion error: " + ip_address;
+            LOG_SERVER_ERROR_ << msg;
+            return Status(SERVER_UNEXPECTED_ERROR, msg);
+        }
+    }
+}
+
+Status
+ValidateStringIsNumber(const std::string& str) {
+    if (str.empty() || !std::all_of(str.begin(), str.end(), ::isdigit)) {
+        return Status(SERVER_INVALID_ARGUMENT, "Invalid number");
+    }
+    try {
+        int64_t value = std::stol(str);
+        fiu_do_on("config.ValidateStringIsNumber.throw_exception", throw std::exception());
+        if (value < 0) {
+            return Status(SERVER_INVALID_ARGUMENT, "Negative number");
+        }
+    } catch (...) {
+        return Status(SERVER_INVALID_ARGUMENT, "Invalid number");
+    }
+    return Status::OK();
+}
+
+Status
+ValidateStringIsBool(const std::string& str) {
+    fiu_return_on("ValidateStringNotBool", Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str));
+    std::string s = str;
+    std::transform(s.begin(), s.end(), s.begin(), ::tolower);
+    if (s == "true" || s == "on" || s == "yes" || s == "1" || s == "false" || s == "off" || s == "no" || s == "0" ||
+        s.empty()) {
+        return Status::OK();
+    }
+    return Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str);
+}
+
+Status
+ValidateStringIsFloat(const std::string& str) {
+    try {
+        float val = std::stof(str);
+        if (val < 0.0) {
+            return Status(SERVER_INVALID_ARGUMENT, "Negative float: " + str);
+        }
+    } catch (...) {
+        return Status(SERVER_INVALID_ARGUMENT, "Invalid float: " + str);
+    }
+    return Status::OK();
+}
+
+Status
+ValidateDbURI(const std::string& uri) {
+    std::string dialectRegex = "(.*)";
+    std::string usernameRegex = "(.*)";
+    std::string passwordRegex = "(.*)";
+    std::string hostRegex = "(.*)";
+    std::string portRegex = "(.*)";
+    std::string dbNameRegex = "(.*)";
+    std::string uriRegexStr = dialectRegex + R"(\:\/\/)" + usernameRegex + R"(\:)" + passwordRegex + R"(\@)" +
+                              hostRegex + R"(\:)" + portRegex + R"(\/)" + dbNameRegex;
+    std::regex uriRegex(uriRegexStr);
+    std::smatch pieces_match;
+
+    bool okay = true;
+
+    if (std::regex_match(uri, pieces_match, uriRegex)) {
+        std::string dialect = pieces_match[1].str();
+        std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower);
+        if (dialect.find("mysql") == std::string::npos && dialect.find("sqlite") == std::string::npos &&
+            dialect.find("mock") == std::string::npos) {
+            LOG_SERVER_ERROR_ << "Invalid dialect in URI: dialect = " << dialect;
+            okay = false;
+        }
+
+        /*
+         *      Could be DNS, skip checking
+         *
+                std::string host = pieces_match[4].str();
+                if (!host.empty() && host != "localhost") {
+                    if (ValidateIpAddress(host) != SERVER_SUCCESS) {
+                        LOG_SERVER_ERROR_ << "Invalid host ip address in uri = " << host;
+                        okay = false;
+                    }
+                }
+        */
+
+        std::string port = pieces_match[5].str();
+        if (!port.empty()) {
+            auto status = ValidateStringIsNumber(port);
+            if (!status.ok()) {
+                LOG_SERVER_ERROR_ << "Invalid port in uri = " << port;
+                okay = false;
+            }
+        }
+    } else {
+        LOG_SERVER_ERROR_ << "Wrong URI format: URI = " << uri;
+        okay = false;
+    }
+
+    return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Invalid db backend uri"));
+}
+
+Status
+ValidateStoragePath(const std::string& path) {
+    // Validate storage path if is valid, only correct absolute path will be validated pass
+    // Invalid path only contain character[a-zA-Z], number[0-9], '-', and '_',
+    // and path must start with '/'.
+    // examples below are invalid
+    // '/a//a', '/a--/a', '/-a/a', '/a@#/a', 'aaa/sfs'
+    std::string path_pattern = R"(^\/(\w+-?\/?)+$)";
+    std::regex regex(path_pattern);
+
+    return std::regex_match(path, regex) ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Invalid file path");
+}
+
+Status
+ValidateLogLevel(const std::string& level) {
+    std::set<std::string> supported_level{"debug", "info", "warning", "error", "fatal"};
+
+    return supported_level.find(level) != supported_level.end()
+               ? Status::OK()
+               : Status(SERVER_INVALID_ARGUMENT, "Log level must be one of debug, info, warning, error and fatal.");
+}
+
+bool
+IsNumber(const std::string& s) {
+    return !s.empty() && std::all_of(s.begin(), s.end(), ::isdigit);
+}
+
+}  // namespace server
+}  // namespace milvus

internal/core/src/index/knowhere/utils/ConfigUtils.h

@@ -0,0 +1,63 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "utils/Status.h"
+
+namespace milvus {
+namespace server {
+
+extern int64_t
+parse_bytes(const std::string& str, std::string& err);
+
+extern bool
+GetSystemMemInfo(int64_t& total_mem, int64_t& free_mem);
+
+extern bool
+GetSystemAvailableThreads(int64_t& thread_count);
+
+extern Status
+ValidateGpuIndex(int32_t gpu_index);
+
+#ifdef MILVUS_GPU_VERSION
+extern Status
+GetGpuMemory(int32_t gpu_index, int64_t& memory);
+#endif
+
+extern Status
+ValidateIpAddress(const std::string& ip_address);
+
+extern Status
+ValidateStringIsNumber(const std::string& str);
+
+extern Status
+ValidateStringIsBool(const std::string& str);
+
+extern Status
+ValidateStringIsFloat(const std::string& str);
+
+extern Status
+ValidateDbURI(const std::string& uri);
+
+extern Status
+ValidateStoragePath(const std::string& path);
+
+extern Status
+ValidateLogLevel(const std::string& level);
+
+extern bool
+IsNumber(const std::string& s);
+}  // namespace server
+}  // namespace milvus

internal/core/src/index/knowhere/utils/Error.h

@@ -0,0 +1,145 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#pragma once
+
+#include <cstdint>
+#include <exception>
+#include <string>
+
+namespace milvus {
+
+using ErrorCode = int32_t;
+
+constexpr ErrorCode SERVER_SUCCESS = 0;
+constexpr ErrorCode SERVER_ERROR_CODE_BASE = 30000;
+
+constexpr ErrorCode
+ToServerErrorCode(const ErrorCode error_code) {
+    return SERVER_ERROR_CODE_BASE + error_code;
+}
+
+constexpr ErrorCode DB_SUCCESS = 0;
+constexpr ErrorCode DB_ERROR_CODE_BASE = 40000;
+
+constexpr ErrorCode
+ToDbErrorCode(const ErrorCode error_code) {
+    return DB_ERROR_CODE_BASE + error_code;
+}
+
+constexpr ErrorCode KNOWHERE_SUCCESS = 0;
+constexpr ErrorCode KNOWHERE_ERROR_CODE_BASE = 50000;
+
+constexpr ErrorCode
+ToKnowhereErrorCode(const ErrorCode error_code) {
+    return KNOWHERE_ERROR_CODE_BASE + error_code;
+}
+
+constexpr ErrorCode WAL_SUCCESS = 0;
+constexpr ErrorCode WAL_ERROR_CODE_BASE = 60000;
+
+constexpr ErrorCode
+ToWalErrorCode(const ErrorCode error_code) {
+    return WAL_ERROR_CODE_BASE + error_code;
+}
+
+constexpr ErrorCode SS_SUCCESS = 0;
+constexpr ErrorCode SS_ERROR_CODE_BASE = 70000;
+
+constexpr ErrorCode
+ToSSErrorCode(const ErrorCode error_code) {
+    return SS_ERROR_CODE_BASE + error_code;
+}
+
+// server error code
+constexpr ErrorCode SERVER_UNEXPECTED_ERROR = ToServerErrorCode(1);
+constexpr ErrorCode SERVER_UNSUPPORTED_ERROR = ToServerErrorCode(2);
+constexpr ErrorCode SERVER_NULL_POINTER = ToServerErrorCode(3);
+constexpr ErrorCode SERVER_INVALID_ARGUMENT = ToServerErrorCode(4);
+constexpr ErrorCode SERVER_FILE_NOT_FOUND = ToServerErrorCode(5);
+constexpr ErrorCode SERVER_NOT_IMPLEMENT = ToServerErrorCode(6);
+constexpr ErrorCode SERVER_CANNOT_CREATE_FOLDER = ToServerErrorCode(8);
+constexpr ErrorCode SERVER_CANNOT_CREATE_FILE = ToServerErrorCode(9);
+constexpr ErrorCode SERVER_CANNOT_DELETE_FOLDER = ToServerErrorCode(10);
+constexpr ErrorCode SERVER_CANNOT_DELETE_FILE = ToServerErrorCode(11);
+constexpr ErrorCode SERVER_BUILD_INDEX_ERROR = ToServerErrorCode(12);
+constexpr ErrorCode SERVER_CANNOT_OPEN_FILE = ToServerErrorCode(13);
+constexpr ErrorCode SERVER_FILE_MAGIC_BYTES_ERROR = ToServerErrorCode(14);
+constexpr ErrorCode SERVER_FILE_SUM_BYTES_ERROR = ToServerErrorCode(15);
+constexpr ErrorCode SERVER_CANNOT_READ_FILE = ToServerErrorCode(16);
+
+constexpr ErrorCode SERVER_COLLECTION_NOT_EXIST = ToServerErrorCode(100);
+constexpr ErrorCode SERVER_INVALID_COLLECTION_NAME = ToServerErrorCode(101);
+constexpr ErrorCode SERVER_INVALID_COLLECTION_DIMENSION = ToServerErrorCode(102);
+constexpr ErrorCode SERVER_INVALID_VECTOR_DIMENSION = ToServerErrorCode(104);
+constexpr ErrorCode SERVER_INVALID_INDEX_TYPE = ToServerErrorCode(105);
+constexpr ErrorCode SERVER_INVALID_ROWRECORD = ToServerErrorCode(106);
+constexpr ErrorCode SERVER_INVALID_ROWRECORD_ARRAY = ToServerErrorCode(107);
+constexpr ErrorCode SERVER_INVALID_TOPK = ToServerErrorCode(108);
+constexpr ErrorCode SERVER_ILLEGAL_VECTOR_ID = ToServerErrorCode(109);
+constexpr ErrorCode SERVER_ILLEGAL_SEARCH_RESULT = ToServerErrorCode(110);
+constexpr ErrorCode SERVER_CACHE_FULL = ToServerErrorCode(111);
+constexpr ErrorCode SERVER_WRITE_ERROR = ToServerErrorCode(112);
+constexpr ErrorCode SERVER_INVALID_NPROBE = ToServerErrorCode(113);
+constexpr ErrorCode SERVER_INVALID_INDEX_NLIST = ToServerErrorCode(114);
+constexpr ErrorCode SERVER_INVALID_INDEX_METRIC_TYPE = ToServerErrorCode(115);
+constexpr ErrorCode SERVER_INVALID_SEGMENT_ROW_COUNT = ToServerErrorCode(116);
+constexpr ErrorCode SERVER_OUT_OF_MEMORY = ToServerErrorCode(117);
+constexpr ErrorCode SERVER_INVALID_PARTITION_TAG = ToServerErrorCode(118);
+constexpr ErrorCode SERVER_INVALID_BINARY_QUERY = ToServerErrorCode(119);
+constexpr ErrorCode SERVER_INVALID_DSL_PARAMETER = ToServerErrorCode(120);
+constexpr ErrorCode SERVER_INVALID_FIELD_NAME = ToServerErrorCode(121);
+constexpr ErrorCode SERVER_INVALID_FIELD_NUM = ToServerErrorCode(122);
+
+// db error code
+constexpr ErrorCode DB_META_TRANSACTION_FAILED = ToDbErrorCode(1);
+constexpr ErrorCode DB_ERROR = ToDbErrorCode(2);
+constexpr ErrorCode DB_NOT_FOUND = ToDbErrorCode(3);
+constexpr ErrorCode DB_ALREADY_EXIST = ToDbErrorCode(4);
+constexpr ErrorCode DB_INVALID_PATH = ToDbErrorCode(5);
+constexpr ErrorCode DB_INCOMPATIB_META = ToDbErrorCode(6);
+constexpr ErrorCode DB_INVALID_META_URI = ToDbErrorCode(7);
+constexpr ErrorCode DB_EMPTY_COLLECTION = ToDbErrorCode(8);
+constexpr ErrorCode DB_BLOOM_FILTER_ERROR = ToDbErrorCode(9);
+constexpr ErrorCode DB_PARTITION_NOT_FOUND = ToDbErrorCode(10);
+constexpr ErrorCode DB_OUT_OF_STORAGE = ToDbErrorCode(11);
+constexpr ErrorCode DB_META_QUERY_FAILED = ToDbErrorCode(12);
+constexpr ErrorCode DB_FILE_NOT_FOUND = ToDbErrorCode(13);
+constexpr ErrorCode DB_PERMISSION_ERROR = ToDbErrorCode(14);
+
+// knowhere error code
+constexpr ErrorCode KNOWHERE_ERROR = ToKnowhereErrorCode(1);
+constexpr ErrorCode KNOWHERE_INVALID_ARGUMENT = ToKnowhereErrorCode(2);
+constexpr ErrorCode KNOWHERE_UNEXPECTED_ERROR = ToKnowhereErrorCode(3);
+constexpr ErrorCode KNOWHERE_NO_SPACE = ToKnowhereErrorCode(4);
+
+// knowhere error code
+constexpr ErrorCode WAL_ERROR = ToWalErrorCode(1);
+constexpr ErrorCode WAL_META_ERROR = ToWalErrorCode(2);
+constexpr ErrorCode WAL_FILE_ERROR = ToWalErrorCode(3);
+constexpr ErrorCode WAL_PATH_ERROR = ToWalErrorCode(4);
+
+// Snapshot error code
+constexpr ErrorCode SS_ERROR = ToSSErrorCode(1);
+constexpr ErrorCode SS_STALE_ERROR = ToSSErrorCode(2);
+constexpr ErrorCode SS_NOT_FOUND_ERROR = ToSSErrorCode(3);
+constexpr ErrorCode SS_INVALID_CONTEX_ERROR = ToSSErrorCode(4);
+constexpr ErrorCode SS_DUPLICATED_ERROR = ToSSErrorCode(5);
+constexpr ErrorCode SS_NOT_ACTIVE_ERROR = ToSSErrorCode(6);
+constexpr ErrorCode SS_CONSTRAINT_CHECK_ERROR = ToSSErrorCode(7);
+constexpr ErrorCode SS_INVALID_ARGUMENT_ERROR = ToSSErrorCode(8);
+constexpr ErrorCode SS_OPERATION_PENDING = ToSSErrorCode(9);
+constexpr ErrorCode SS_TIMEOUT = ToSSErrorCode(10);
+constexpr ErrorCode SS_NOT_COMMITED = ToSSErrorCode(11);
+constexpr ErrorCode SS_COLLECTION_DROPPED = ToSSErrorCode(12);
+constexpr ErrorCode SS_EMPTY_HOLDER = ToSSErrorCode(13);
+
+}  // namespace milvus

internal/core/src/index/knowhere/utils/Exception.h

@@ -0,0 +1,61 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#pragma once
+
+#include "utils/Error.h"
+
+#include <exception>
+#include <string>
+#include <utility>
+
+namespace milvus {
+
+#define THROW_ERROR(err_code, err_msg) \
+    LOG_ENGINE_ERROR_ << err_msg;      \
+    throw Exception(err_code, err_msg);
+
+class Exception : public std::exception {
+ public:
+    Exception(ErrorCode code, std::string msg) : code_(code), message_(std::move(msg)) {
+    }
+
+    ErrorCode
+    code() const noexcept {
+        return code_;
+    }
+
+    const char*
+    what() const noexcept override {
+        if (message_.empty()) {
+            return "Default Exception.";
+        } else {
+            return message_.c_str();
+        }
+    }
+
+    ~Exception() noexcept override = default;
+
+ protected:
+    ErrorCode code_;
+    std::string message_;
+};
+
+class InvalidArgumentException : public Exception {
+ public:
+    InvalidArgumentException() : Exception(SERVER_INVALID_ARGUMENT, "Invalid Argument") {
+    }
+
+    explicit InvalidArgumentException(const std::string& message) : Exception(SERVER_INVALID_ARGUMENT, message) {
+    }
+};
+
+}  // namespace milvus

internal/core/src/index/knowhere/utils/Json.h

@@ -0,0 +1,34 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#pragma once
+
+#include "nlohmann/json.hpp"
+
+namespace milvus {
+
+using json = nlohmann::json;
+
+#define JSON_NULL_CHECK(json)                                       \
+    do {                                                            \
+        if (json.empty()) {                                         \
+            return Status{SERVER_INVALID_ARGUMENT, "Json is null"}; \
+        }                                                           \
+    } while (false)
+
+#define JSON_OBJECT_CHECK(json)                                                  \
+    do {                                                                         \
+        if (!json.is_object()) {                                                 \
+            return Status{SERVER_INVALID_ARGUMENT, "Json is not a json object"}; \
+        }                                                                        \
+    } while (false)
+
+}  // namespace milvus

internal/core/src/index/knowhere/utils/Log.h

@@ -0,0 +1,14 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#pragma once
+
+#include "log/Log.h"

internal/core/src/index/knowhere/utils/SignalHandler.cpp

@@ -0,0 +1,60 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#include "utils/SignalHandler.h"
+#include "utils/Log.h"
+
+#include <execinfo.h>
+#include <string>
+
+namespace milvus {
+
+signal_func_ptr signal_routine_func = nullptr;
+
+void
+HandleSignal(int signum) {
+    int32_t exit_code = 1; /* 0: normal exit; 1: exception */
+    switch (signum) {
+        case SIGINT:
+        case SIGUSR2:
+            exit_code = 0;
+            /* no break */
+        default: {
+            if (exit_code == 0) {
+                LOG_SERVER_INFO_ << "Server received signal: " << signum;
+            } else {
+                LOG_SERVER_INFO_ << "Server received critical signal: " << signum;
+                PrintStacktrace();
+            }
+            if (signal_routine_func != nullptr) {
+                (*signal_routine_func)(exit_code);
+            }
+        }
+    }
+}
+
+void
+PrintStacktrace() {
+    const int bt_depth = 128;
+    void* array[bt_depth];
+    int stack_num = backtrace(array, bt_depth);
+    char** stacktrace = backtrace_symbols(array, stack_num);
+
+    LOG_SERVER_INFO_ << "Call stack:";
+    for (int i = 0; i < stack_num; ++i) {
+        std::string info = stacktrace[i];
+        std::cout << "No." << i << ": " << info << std::endl;
+        LOG_SERVER_INFO_ << info;
+    }
+    free(stacktrace);
+}
+
+}  // namespace milvus

internal/core/src/index/knowhere/utils/SignalHandler.h

@@ -0,0 +1,28 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#pragma once
+
+#include <cstdint>
+
+namespace milvus {
+
+typedef void (*signal_func_ptr)(int32_t);
+
+extern signal_func_ptr signal_routine_func;
+
+extern void
+HandleSignal(int signum);
+
+extern void
+PrintStacktrace();
+
+}  // namespace milvus

internal/core/src/index/knowhere/utils/Status.cpp

@@ -0,0 +1,127 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#include "utils/Status.h"
+#include "memory"
+
+#include <cstring>
+
+namespace milvus {
+
+constexpr int CODE_WIDTH = sizeof(StatusCode);
+
+Status::Status(StatusCode code, const std::string& msg) {
+    // 4 bytes store code
+    // 4 bytes store message length
+    // the left bytes store message string
+    auto length = static_cast<uint32_t>(msg.size());
+    // auto result = new char[length + sizeof(length) + CODE_WIDTH];
+    state_.resize(length + sizeof(length) + CODE_WIDTH);
+    std::memcpy(state_.data(), &code, CODE_WIDTH);
+    std::memcpy(state_.data() + CODE_WIDTH, &length, sizeof(length));
+    memcpy(state_.data() + sizeof(length) + CODE_WIDTH, msg.data(), length);
+}
+
+Status::~Status() {
+}
+
+Status::Status(const Status& s) {
+    CopyFrom(s);
+}
+
+Status::Status(Status&& s) noexcept {
+    MoveFrom(s);
+}
+
+Status&
+Status::operator=(const Status& s) {
+    CopyFrom(s);
+    return *this;
+}
+
+Status&
+Status::operator=(Status&& s) noexcept {
+    MoveFrom(s);
+    return *this;
+}
+
+void
+Status::CopyFrom(const Status& s) {
+    state_.clear();
+    if (s.state_.empty()) {
+        return;
+    }
+
+    uint32_t length = 0;
+    memcpy(&length, s.state_.data() + CODE_WIDTH, sizeof(length));
+    int buff_len = length + sizeof(length) + CODE_WIDTH;
+    state_.resize(buff_len);
+    memcpy(state_.data(), s.state_.data(), buff_len);
+}
+
+void
+Status::MoveFrom(Status& s) {
+    state_ = s.state_;
+    s.state_.clear();
+}
+
+std::string
+Status::message() const {
+    if (state_.empty()) {
+        return "OK";
+    }
+
+    std::string msg;
+    uint32_t length = 0;
+    memcpy(&length, state_.data() + CODE_WIDTH, sizeof(length));
+    if (length > 0) {
+        msg.append(state_.data() + sizeof(length) + CODE_WIDTH, length);
+    }
+
+    return msg;
+}
+
+std::string
+Status::ToString() const {
+    if (state_.empty()) {
+        return "OK";
+    }
+
+    std::string result;
+    switch (code()) {
+        case DB_SUCCESS:
+            result = "OK ";
+            break;
+        case DB_ERROR:
+            result = "Error: ";
+            break;
+        case DB_META_TRANSACTION_FAILED:
+            result = "Database error: ";
+            break;
+        case DB_NOT_FOUND:
+            result = "Not found: ";
+            break;
+        case DB_ALREADY_EXIST:
+            result = "Already exist: ";
+            break;
+        case DB_INVALID_PATH:
+            result = "Invalid path: ";
+            break;
+        default:
+            result = "Error code(" + std::to_string(code()) + "): ";
+            break;
+    }
+
+    result += message();
+    return result;
+}
+
+}  // namespace milvus

internal/core/src/index/knowhere/utils/Status.h

@@ -0,0 +1,79 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#pragma once
+
+#include "utils/Error.h"
+
+#include <string>
+
+namespace milvus {
+
+class Status;
+#define STATUS_CHECK(func) \
+    do {                   \
+        Status s = func;   \
+        if (!s.ok()) {     \
+            return s;      \
+        }                  \
+    } while (false)
+
+using StatusCode = ErrorCode;
+
+class Status {
+ public:
+    Status(StatusCode code, const std::string& msg);
+    Status() = default;
+    virtual ~Status();
+
+    Status(const Status& s);
+
+    Status(Status&& s) noexcept;
+
+    Status&
+    operator=(const Status& s);
+
+    Status&
+    operator=(Status&& s) noexcept;
+
+    static Status
+    OK() {
+        return Status();
+    }
+
+    bool
+    ok() const {
+        return state_.empty() || code() == 0;
+    }
+
+    StatusCode
+    code() const {
+        return (state_.empty()) ? 0 : *(StatusCode*)(state_.data());
+    }
+
+    std::string
+    message() const;
+
+    std::string
+    ToString() const;
+
+ private:
+    inline void
+    CopyFrom(const Status& s);
+
+    inline void
+    MoveFrom(Status& s);
+
+ private:
+    std::string state_;
+};  // Status
+
+}  // namespace milvus

internal/core/src/index/knowhere/utils/StringHelpFunctions.cpp

@@ -0,0 +1,175 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#include "utils/StringHelpFunctions.h"
+#include "utils/Log.h"
+
+#include <fiu/fiu-local.h>
+#include <algorithm>
+#include <regex>
+#include <string>
+
+namespace milvus {
+
+void
+StringHelpFunctions::TrimStringBlank(std::string& string) {
+    if (!string.empty()) {
+        static std::string s_format(" \n\r\t");
+        string.erase(0, string.find_first_not_of(s_format));
+        string.erase(string.find_last_not_of(s_format) + 1);
+    }
+}
+
+void
+StringHelpFunctions::TrimStringQuote(std::string& string, const std::string& qoute) {
+    if (!string.empty()) {
+        string.erase(0, string.find_first_not_of(qoute));
+        string.erase(string.find_last_not_of(qoute) + 1);
+    }
+}
+
+void
+StringHelpFunctions::SplitStringByDelimeter(const std::string& str,
+                                            const std::string& delimeter,
+                                            std::vector<std::string>& result) {
+    if (str.empty()) {
+        return;
+    }
+
+    size_t prev = 0;
+    while (true) {
+        size_t pos = str.find_first_of(delimeter, prev);
+        if (pos == std::string::npos) {
+            result.emplace_back(str.substr(prev));
+            break;
+        } else {
+            result.emplace_back(str.substr(prev, pos - prev));
+            prev = pos + 1;
+        }
+    }
+}
+
+void
+StringHelpFunctions::MergeStringWithDelimeter(const std::vector<std::string>& strs,
+                                              const std::string& delimeter,
+                                              std::string& result) {
+    if (strs.empty()) {
+        result = "";
+        return;
+    }
+
+    result = strs[0];
+    for (size_t i = 1; i < strs.size(); i++) {
+        result = result + delimeter + strs[i];
+    }
+}
+
+Status
+StringHelpFunctions::SplitStringByQuote(const std::string& str,
+                                        const std::string& delimeter,
+                                        const std::string& quote,
+                                        std::vector<std::string>& result) {
+    if (quote.empty()) {
+        SplitStringByDelimeter(str, delimeter, result);
+        return Status::OK();
+    }
+
+    size_t last = 0;
+    size_t index = str.find_first_of(quote, last);
+    if (index == std::string::npos) {
+        SplitStringByDelimeter(str, delimeter, result);
+        return Status::OK();
+    }
+
+    std::string process_str = str;
+    while (index != std::string::npos) {
+        std::string prefix = process_str.substr(last, index - last);
+        std::string append_prefix;
+        if (!prefix.empty()) {
+            std::vector<std::string> prefix_split;
+            SplitStringByDelimeter(prefix, delimeter, prefix_split);
+            for (size_t i = 0; i < prefix_split.size() - 1; i++) {
+                result.push_back(prefix_split[i]);
+            }
+            append_prefix = prefix_split[prefix_split.size() - 1];
+        }
+        last = index + 1;
+        std::string postfix = process_str.substr(last);
+        index = postfix.find_first_of(quote, 0);
+        fiu_do_on("StringHelpFunctions.SplitStringByQuote.invalid_index", index = std::string::npos);
+
+        if (index == std::string::npos) {
+            return Status(SERVER_UNEXPECTED_ERROR, "");
+        }
+        std::string quoted_text = postfix.substr(0, index);
+        append_prefix += quoted_text;
+
+        last = index + 1;
+        index = postfix.find_first_of(delimeter, last);
+        fiu_do_on("StringHelpFunctions.SplitStringByQuote.index_gt_last", last = 0);
+        fiu_do_on("StringHelpFunctions.SplitStringByQuote.invalid_index2", index = std::string::npos);
+
+        if (index != std::string::npos) {
+            if (index > last) {
+                append_prefix += postfix.substr(last, index - last);
+            }
+        } else {
+            append_prefix += postfix.substr(last);
+        }
+        result.emplace_back(append_prefix);
+        fiu_do_on("StringHelpFunctions.SplitStringByQuote.last_is_end", last = postfix.length());
+
+        if (last == postfix.length()) {
+            return Status::OK();
+        }
+
+        process_str = postfix.substr(index + 1);
+        last = 0;
+        index = process_str.find_first_of(quote, last);
+    }
+
+    if (!process_str.empty()) {
+        SplitStringByDelimeter(process_str, delimeter, result);
+    }
+
+    return Status::OK();
+}
+
+bool
+StringHelpFunctions::IsRegexMatch(const std::string& target_str, const std::string& pattern_str) {
+    // if target_str equals pattern_str, return true
+    if (target_str == pattern_str) {
+        return true;
+    }
+
+    // regex match
+    // for illegal regex expression, the std::regex will throw exception, regard as unmatch
+    try {
+        std::regex pattern(pattern_str);
+        std::smatch results;
+        return std::regex_match(target_str, results, pattern);
+    } catch (std::exception& e) {
+        LOG_SERVER_ERROR_ << "Regex exception: " << e.what();
+    }
+
+    return false;
+}
+
+Status
+StringHelpFunctions::ConvertToBoolean(const std::string& str, bool& value) {
+    std::string s = str;
+    std::transform(s.begin(), s.end(), s.begin(), ::tolower);
+    value = s == "true" || s == "on" || s == "yes" || s == "1";
+
+    return Status::OK();
+}
+
+}  // namespace milvus

internal/core/src/index/knowhere/utils/StringHelpFunctions.h

@@ -0,0 +1,76 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#pragma once
+
+#include "utils/Status.h"
+
+#include <string>
+#include <vector>
+
+namespace milvus {
+
+class StringHelpFunctions {
+ private:
+    StringHelpFunctions() = default;
+
+ public:
+    // trim blanks from begin and end
+    // "  a b c  "  =>  "a b c"
+    static void
+    TrimStringBlank(std::string& string);
+
+    // trim quotes from begin and end
+    // "'abc'"  =>  "abc"
+    static void
+    TrimStringQuote(std::string& string, const std::string& qoute);
+
+    // split string by delimeter ','
+    // a,b,c            a | b | c
+    // a,b,             a | b |
+    // ,b,c               | b | c
+    // ,b,                | b |
+    // ,,                 |   |
+    // a                    a
+    static void
+    SplitStringByDelimeter(const std::string& str, const std::string& delimeter, std::vector<std::string>& result);
+
+    // merge strings with delimeter
+    // "a", "b", "c"  => "a,b,c"
+    static void
+    MergeStringWithDelimeter(const std::vector<std::string>& strs, const std::string& delimeter, std::string& result);
+
+    // assume the collection has two columns, quote='\"', delimeter=','
+    //  a,b             a | b
+    //  "aa,gg,yy",b    aa,gg,yy | b
+    //  aa"dd,rr"kk,pp  aadd,rrkk | pp
+    //  "aa,bb"         aa,bb
+    //  55,1122\"aa,bb\",yyy,\"kkk\"    55 | 1122aa,bb | yyy | kkk
+    //  "55,1122"aa,bb",yyy,"kkk"   illegal
+    static Status
+    SplitStringByQuote(const std::string& str,
+                       const std::string& delimeter,
+                       const std::string& quote,
+                       std::vector<std::string>& result);
+
+    // std regex match function
+    // regex grammar reference: http://www.cplusplus.com/reference/regex/ECMAScript/
+    static bool
+    IsRegexMatch(const std::string& target_str, const std::string& pattern);
+
+    // conversion rules refer to ValidationUtil::ValidateStringIsBool()
+    // "true", "on", "yes", "1" ==> true
+    // "false", "off", "no", "0", "" ==> false
+    static Status
+    ConvertToBoolean(const std::string& str, bool& value);
+};
+
+}  // namespace milvus

internal/core/src/index/knowhere/utils/ThreadPool.h

@@ -0,0 +1,128 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#pragma once
+
+#include <fiu/fiu-local.h>
+#include <atomic>
+#include <condition_variable>
+#include <functional>
+#include <future>
+#include <memory>
+#include <mutex>
+#include <queue>
+#include <stdexcept>
+#include <thread>
+#include <utility>
+#include <vector>
+
+#define MAX_THREADS_NUM 32
+
+namespace milvus {
+
+class ThreadPool {
+ public:
+    explicit ThreadPool(size_t threads, size_t queue_size = 1000);
+
+    template <class F, class... Args>
+    auto
+    enqueue(F&& f, Args&&... args) -> std::future<typename std::result_of<F(Args...)>::type>;
+
+    void
+    Stop();
+
+    ~ThreadPool();
+
+ private:
+    // need to keep track of threads so we can join them
+    std::vector<std::thread> workers_;
+
+    // the task queue
+    std::queue<std::function<void()>> tasks_;
+
+    size_t max_queue_size_;
+
+    // synchronization
+    std::mutex queue_mutex_;
+
+    std::condition_variable condition_;
+
+    std::atomic_bool stop_;
+};
+
+// the constructor just launches some amount of workers
+inline ThreadPool::ThreadPool(size_t threads, size_t queue_size) : max_queue_size_(queue_size), stop_(false) {
+    for (size_t i = 0; i < threads; ++i)
+        workers_.emplace_back([this] {
+            for (;;) {
+                std::function<void()> task;
+
+                {
+                    std::unique_lock<std::mutex> lock(this->queue_mutex_);
+                    this->condition_.wait(lock, [this] { return this->stop_ || !this->tasks_.empty(); });
+                    if (this->stop_ && this->tasks_.empty())
+                        return;
+                    task = std::move(this->tasks_.front());
+                    this->tasks_.pop();
+                }
+                this->condition_.notify_all();
+
+                task();
+            }
+        });
+}
+
+// add new work item to the pool
+template <class F, class... Args>
+auto
+ThreadPool::enqueue(F&& f, Args&&... args) -> std::future<typename std::result_of<F(Args...)>::type> {
+    using return_type = typename std::result_of<F(Args...)>::type;
+
+    auto task =
+        std::make_shared<std::packaged_task<return_type()>>(std::bind(std::forward<F>(f), std::forward<Args>(args)...));
+    fiu_do_on("ThreadPool.enqueue.stop_is_true", stop_ = true);
+    std::future<return_type> res = task->get_future();
+    {
+        std::unique_lock<std::mutex> lock(queue_mutex_);
+        this->condition_.wait(lock, [this] { return this->tasks_.size() < max_queue_size_; });
+        // don't allow enqueueing after stopping the pool
+        if (stop_)
+            throw std::runtime_error("enqueue on stopped ThreadPool");
+
+        tasks_.emplace([task]() { (*task)(); });
+    }
+    condition_.notify_all();
+    return res;
+}
+
+inline void
+ThreadPool::Stop() {
+    if (stop_) {
+        return;
+    }
+    {
+        std::unique_lock<std::mutex> lock(queue_mutex_);
+        stop_ = true;
+    }
+    condition_.notify_all();
+    for (std::thread& worker : workers_) {
+        worker.join();
+    }
+}
+
+// the destructor joins all threads
+inline ThreadPool::~ThreadPool() {
+    Stop();
+}
+
+using ThreadPoolPtr = std::shared_ptr<ThreadPool>;
+
+}  // namespace milvus

internal/core/src/index/knowhere/utils/TimeRecorder.cpp

@@ -0,0 +1,93 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#include <utility>
+
+#include "utils/Log.h"
+#include "utils/TimeRecorder.h"
+
+namespace milvus {
+
+TimeRecorder::TimeRecorder(std::string hdr, int64_t log_level) : header_(std::move(hdr)), log_level_(log_level) {
+    start_ = last_ = stdclock::now();
+}
+
+std::string
+TimeRecorder::GetTimeSpanStr(double span) {
+    std::string str_sec = std::to_string(span * 0.000001) + ((span > 1000000) ? " seconds" : " second");
+    std::string str_ms = std::to_string(span * 0.001) + " ms";
+
+    return str_sec + " [" + str_ms + "]";
+}
+
+void
+TimeRecorder::PrintTimeRecord(const std::string& msg, double span) {
+    std::string str_log;
+    if (!header_.empty()) {
+        str_log += header_ + ": ";
+    }
+    str_log += msg;
+    str_log += " (";
+    str_log += TimeRecorder::GetTimeSpanStr(span);
+    str_log += ")";
+
+    switch (log_level_) {
+        case 0:
+            LOG_SERVER_TRACE_ << str_log;
+            break;
+        case 1:
+            LOG_SERVER_DEBUG_ << str_log;
+            break;
+        case 2:
+            LOG_SERVER_INFO_ << str_log;
+            break;
+        case 3:
+            LOG_SERVER_WARNING_ << str_log;
+            break;
+        case 4:
+            LOG_SERVER_ERROR_ << str_log;
+            break;
+        case 5:
+            LOG_SERVER_FATAL_ << str_log;
+            break;
+        default:
+            LOG_SERVER_INFO_ << str_log;
+            break;
+    }
+}
+
+double
+TimeRecorder::RecordSection(const std::string& msg) {
+    stdclock::time_point curr = stdclock::now();
+    double span = (std::chrono::duration<double, std::micro>(curr - last_)).count();
+    last_ = curr;
+
+    PrintTimeRecord(msg, span);
+    return span;
+}
+
+double
+TimeRecorder::ElapseFromBegin(const std::string& msg) {
+    stdclock::time_point curr = stdclock::now();
+    double span = (std::chrono::duration<double, std::micro>(curr - start_)).count();
+
+    PrintTimeRecord(msg, span);
+    return span;
+}
+
+TimeRecorderAuto::TimeRecorderAuto(std::string hdr, int64_t log_level) : TimeRecorder(hdr, log_level) {
+}
+
+TimeRecorderAuto::~TimeRecorderAuto() {
+    ElapseFromBegin("totally cost");
+}
+
+}  // namespace milvus

internal/core/src/index/knowhere/utils/TimeRecorder.h

@@ -0,0 +1,65 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#pragma once
+
+#include <chrono>
+#include <string>
+#include "utils/Log.h"
+
+namespace milvus {
+
+inline void
+print_timestamp(const std::string& message) {
+    std::chrono::time_point<std::chrono::system_clock> now = std::chrono::system_clock::now();
+    auto duration = now.time_since_epoch();
+    auto micros = std::chrono::duration_cast<std::chrono::microseconds>(duration).count();
+    micros %= 1000000;
+    double millisecond = (double)micros / 1000.0;
+
+    LOG_SERVER_DEBUG_ << std::fixed << " " << millisecond << "(ms) [timestamp]" << message;
+}
+
+class TimeRecorder {
+    using stdclock = std::chrono::high_resolution_clock;
+
+ public:
+    // trace = 0, debug = 1, info = 2, warn = 3, error = 4, critical = 5
+    explicit TimeRecorder(std::string hdr, int64_t log_level = 1);
+    virtual ~TimeRecorder() = default;
+
+    double
+    RecordSection(const std::string& msg);
+
+    double
+    ElapseFromBegin(const std::string& msg);
+
+    static std::string
+    GetTimeSpanStr(double span);
+
+ private:
+    void
+    PrintTimeRecord(const std::string& msg, double span);
+
+ private:
+    std::string header_;
+    stdclock::time_point start_;
+    stdclock::time_point last_;
+    int64_t log_level_;
+};
+
+class TimeRecorderAuto : public TimeRecorder {
+ public:
+    explicit TimeRecorderAuto(std::string hdr, int64_t log_level = 1);
+    ~TimeRecorderAuto() override;
+};
+
+}  // namespace milvus

internal/core/src/index/knowhere/utils/TimerContext.h

@@ -0,0 +1,76 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#pragma once
+
+#include <boost/asio.hpp>
+#include <boost/date_time/posix_time/posix_time.hpp>
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "utils/Log.h"
+#include "utils/ThreadPool.h"
+
+namespace milvus {
+
+struct TimerContext {
+    using HandlerT = std::function<void(const boost::system::error_code&)>;
+    struct Context {
+        /* Context(int interval_us, HandlerT& handler, ThreadPoolPtr pool = nullptr) */
+        /*     : interval_(interval_us), handler_(handler), timer_(io, interval_), pool_(pool) { */
+        /* } */
+        int interval_us;
+        HandlerT handler;
+        ThreadPoolPtr pool = nullptr;
+    };
+
+    TimerContext(boost::asio::io_service& io, int interval_us, HandlerT& handler, ThreadPoolPtr pool)
+        : io_(io), interval_(interval_us), handler_(handler), timer_(io, interval_), pool_(pool) {
+    }
+    TimerContext(boost::asio::io_service& io, Context& context)
+        : io_(io),
+          interval_(context.interval_us),
+          handler_(context.handler),
+          timer_(io, interval_),
+          pool_(context.pool) {
+    }
+
+    void
+    Reschedule(const boost::system::error_code& ec);
+
+    boost::asio::io_service& io_;
+    boost::posix_time::microseconds interval_;
+    boost::asio::deadline_timer timer_;
+    HandlerT handler_;
+    ThreadPoolPtr pool_;
+};
+
+inline void
+TimerContext::Reschedule(const boost::system::error_code& ec) {
+    try {
+        pool_->enqueue(handler_, ec);
+    } catch (std::exception& ex) {
+        LOG_SERVER_ERROR_ << "Fail to enqueue handler: " << std::string(ex.what());
+    }
+    boost::system::error_code e;
+    auto new_expires = timer_.expires_at() + interval_;
+    timer_.expires_at(new_expires, e);
+    if (e) {
+        LOG_SERVER_ERROR_ << "Fail to Reschedule: " << e;
+    }
+    timer_.async_wait(std::bind(&TimerContext::Reschedule, this, std::placeholders::_1));
+}
+
+using TimerContextPtr = std::shared_ptr<TimerContext>;
+
+}  // namespace milvus

internal/core/src/index/knowhere/utils/TimerManager.cpp

@@ -0,0 +1,85 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#include <utility>
+
+#include "utils/Log.h"
+#include "utils/TimerManager.h"
+
+namespace milvus {
+
+TimerManager::TimerManager(unsigned int pool_size) : pool_size_(pool_size) {
+}
+
+TimerManager::~TimerManager() {
+}
+
+Status
+TimerManager::SetPoolSize(unsigned int pool_size) {
+    if (timer_exeutors_) {
+        return Status(SERVER_UNEXPECTED_ERROR, "Cannot set pool size since it has been set already");
+    }
+    pool_size_ = pool_size;
+    return Status::OK();
+}
+
+Status
+TimerManager::Run() {
+    boost::system::error_code ec;
+    io_.run(ec);
+    Status status;
+    if (ec) {
+        status = Status(SERVER_UNEXPECTED_ERROR, ec.message());
+    }
+    return status;
+}
+
+Status
+TimerManager::Start() {
+    for (auto& timer : timers_) {
+        timer->timer_.async_wait(std::bind(&TimerContext::Reschedule, timer, std::placeholders::_1));
+    }
+    return Status::OK();
+}
+
+void
+TimerManager::Stop() {
+    boost::system::error_code ec;
+    for (auto& timer : timers_) {
+        timer->timer_.cancel(ec);
+        if (ec) {
+            LOG_SERVER_ERROR_ << "Fail to cancel timer: " << ec;
+        }
+    }
+    if (timer_exeutors_) {
+        timer_exeutors_->Stop();
+    }
+}
+
+void
+TimerManager::AddTimer(int interval_us, TimerContext::HandlerT handler) {
+    if (!timer_exeutors_) {
+        timer_exeutors_ = std::make_shared<ThreadPool>(pool_size_);
+    }
+    timers_.emplace_back(std::make_shared<TimerContext>(io_, interval_us, handler, timer_exeutors_));
+}
+
+void
+TimerManager::AddTimer(const TimerContext::Context& ctx) {
+    if (!timer_exeutors_) {
+        timer_exeutors_ = std::make_shared<ThreadPool>(pool_size_);
+    }
+    TimerContext::Context context(ctx);
+    context.pool = timer_exeutors_;
+    timers_.emplace_back(std::make_shared<TimerContext>(io_, context));
+}
+
+}  // namespace milvus

internal/core/src/index/knowhere/utils/TimerManager.h

@@ -0,0 +1,57 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#pragma once
+
+#include <boost/asio.hpp>
+#include <functional>
+#include <memory>
+#include <vector>
+
+#include "utils/Status.h"
+#include "utils/ThreadPool.h"
+#include "utils/TimerContext.h"
+
+namespace milvus {
+
+class TimerManager {
+ public:
+    TimerManager() = default;
+    explicit TimerManager(unsigned int pool_size);
+
+    Status
+    SetPoolSize(unsigned int pool_size);
+
+    void
+    AddTimer(int interval_us, TimerContext::HandlerT handler);
+    void
+    AddTimer(const TimerContext::Context& ctx);
+
+    virtual Status
+    Run();
+
+    virtual Status
+    Start();
+    virtual void
+    Stop();
+
+    virtual ~TimerManager();
+
+ protected:
+    boost::asio::io_service io_;
+    ThreadPoolPtr timer_exeutors_;
+    unsigned int pool_size_;
+    std::vector<TimerContextPtr> timers_;
+};
+
+using TimerManagerPtr = std::shared_ptr<TimerManager>;
+
+}  // namespace milvus

internal/core/src/index/knowhere/utils/Types.cpp

@@ -0,0 +1,12 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License
+
+#include "Types.h"

internal/core/src/index/knowhere/utils/Types.h

@@ -0,0 +1,51 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#pragma once
+
+// #include <faiss/Index.h>
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+namespace milvus::engine {
+
+using idx_t = int64_t;
+using offset_t = int32_t;
+using date_t = int32_t;
+using distance_t = float;
+
+using IDNumbers = std::vector<idx_t>;
+
+enum class DataType {
+    NONE = 0,
+    BOOL = 1,
+    INT8 = 2,
+    INT16 = 3,
+    INT32 = 4,
+    INT64 = 5,
+
+    FLOAT = 10,
+    DOUBLE = 11,
+
+    STRING = 20,
+
+    VECTOR_BINARY = 100,
+    VECTOR_FLOAT = 101,
+};
+
+}  // namespace milvus::engine

internal/core/src/index/knowhere/utils/tools.h

@@ -0,0 +1,30 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License
+
+#pragma once
+#include <cstdint>
+#include "exceptions/EasyAssert.h"
+namespace milvus {
+inline int64_t
+upper_align(int64_t value, int64_t align) {
+    Assert(align > 0);
+    auto groups = (value + align - 1) / align;
+    return groups * align;
+}
+
+inline int64_t
+upper_div(int64_t value, int64_t align) {
+    Assert(align > 0);
+    auto groups = (value + align - 1) / align;
+    return groups;
+}
+
+}  // namespace milvus

internal/core/src/index/thirdparty/CMakeLists.txt

@@ -0,0 +1,179 @@
+add_compile_options( -O3 -fPIC -Wno-error -fopenmp )
+
+if ( NOT KNOWHERE_VERBOSE_THIRDPARTY_BUILD )
+    set( EP_LOG_OPTIONS LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1 LOG_DOWNLOAD 1 )
+else ()
+    set( EP_LOG_OPTIONS )
+endif ()
+
+set( MAKE_BUILD_ARGS "-j6" )
+
+include( FetchContent )
+set( FETCHCONTENT_BASE_DIR  ${INDEX_BINARY_DIR}/3rdparty_download )
+set( FETCHCONTENT_QUIET     OFF )
+
+if( CUSTOM_THIRDPARTY_DOWNLOAD_PATH )
+    set( THIRDPARTY_DOWNLOAD_PATH ${CUSTOM_THIRDPARTY_DOWNLOAD_PATH} )
+else()
+    set( THIRDPARTY_DOWNLOAD_PATH ${CMAKE_BINARY_DIR}/3rdparty_download/download )
+endif()
+message( STATUS "Thirdparty downloaded file path: ${THIRDPARTY_DOWNLOAD_PATH}" )
+# ----------------------------------------------------------------------
+# Find pthreads
+
+set( THREADS_PREFER_PTHREAD_FLAG ON )
+find_package( Threads REQUIRED )
+
+
+# ****************************** Thirdparty googletest ***************************************
+if ( KNOWHERE_BUILD_TESTS )
+    if(NOT TARGET gtest::gtest_main)
+        add_subdirectory( gtest )
+    endif()
+
+    if(NOT TARGET benchmark::benchmark_main)
+        add_subdirectory( google_benchmark )
+    endif()
+endif()
+
+
+if(NOT TARGET fiu)
+    add_subdirectory(fiu)
+endif()
+
+# ****************************** Thirdparty NGT ***************************************
+add_subdirectory(NGT)
+
+# ****************************** Thirdparty Faiss ***************************************
+set(FAISS_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/faiss)
+
+macro(build_faiss)
+    message(STATUS "Building FAISS-${FAISS_VERSION} from source")
+
+    set(FAISS_PREFIX "${INDEX_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep")
+    set(FAISS_INCLUDE_DIR "${FAISS_PREFIX}/include")
+    set(FAISS_STATIC_LIB
+            "${FAISS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX}")
+
+    if (CCACHE_FOUND)
+        set(FAISS_C_COMPILER "${CCACHE_FOUND} ${CMAKE_C_COMPILER}")
+        if (MILVUS_GPU_VERSION)
+            set(FAISS_CXX_COMPILER "${CMAKE_CXX_COMPILER}")
+            set(FAISS_CUDA_COMPILER "${CCACHE_FOUND} ${CMAKE_CUDA_COMPILER}")
+        else ()
+            set(FAISS_CXX_COMPILER "${CCACHE_FOUND} ${CMAKE_CXX_COMPILER}")
+        endif()
+    else ()
+        set(FAISS_C_COMPILER "${CMAKE_C_COMPILER}")
+        set(FAISS_CXX_COMPILER "${CMAKE_CXX_COMPILER}")
+    endif()
+
+    set(FAISS_CONFIGURE_ARGS
+            "--prefix=${FAISS_PREFIX}"
+            "CC=${FAISS_C_COMPILER}"
+            "CXX=${FAISS_CXX_COMPILER}"
+            "NVCC=${FAISS_CUDA_COMPILER}"
+            "CFLAGS=${EP_C_FLAGS}"
+            "CXXFLAGS=${EP_CXX_FLAGS} -mf16c -O3"
+            --without-python)
+
+    message(STATUS "${FAISS_CONFIGURE_ARGS}")
+
+    message(STATUS "Build Faiss with OpenBlas/LAPACK")
+    set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
+        "LDFLAGS=-L${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}"
+        )
+
+    if (MILVUS_GPU_VERSION)
+        if (NOT MILVUS_CUDA_ARCH OR MILVUS_CUDA_ARCH STREQUAL "DEFAULT")
+            set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
+                "--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}"
+                "--with-cuda-arch=-gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75"
+                )
+        else()
+            STRING(REPLACE ";" " " MILVUS_CUDA_ARCH "${MILVUS_CUDA_ARCH}")
+            set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
+                "--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}"
+                "--with-cuda-arch=${MILVUS_CUDA_ARCH}"
+                )
+        endif ()
+    else ()
+        set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
+                "CPPFLAGS=-DUSE_CPU"
+                --without-cuda)
+    endif ()
+
+    message(STATUS "Building FAISS with configure args -${FAISS_CONFIGURE_ARGS}")
+
+    if (DEFINED ENV{FAISS_SOURCE_URL})
+        set(FAISS_SOURCE_URL "$ENV{FAISS_SOURCE_URL}")
+        externalproject_add(faiss_ep
+                URL
+                ${FAISS_SOURCE_URL}
+                ${EP_LOG_OPTIONS}
+                CONFIGURE_COMMAND
+                "./configure"
+                ${FAISS_CONFIGURE_ARGS}
+                BUILD_COMMAND
+                ${MAKE} ${MAKE_BUILD_ARGS} all
+                BUILD_IN_SOURCE
+                1
+                INSTALL_COMMAND
+                ${MAKE} install
+                BUILD_BYPRODUCTS
+                ${FAISS_STATIC_LIB})
+    else ()
+        externalproject_add(faiss_ep
+                DOWNLOAD_COMMAND
+                ""
+                SOURCE_DIR
+                ${FAISS_SOURCE_DIR}
+                ${EP_LOG_OPTIONS}
+                CONFIGURE_COMMAND
+                "./configure"
+                ${FAISS_CONFIGURE_ARGS}
+                BUILD_COMMAND
+                ${MAKE} ${MAKE_BUILD_ARGS} all
+                BUILD_IN_SOURCE
+                1
+                INSTALL_COMMAND
+                ${MAKE} install
+                BUILD_BYPRODUCTS
+                ${FAISS_STATIC_LIB})
+    endif ()
+
+    if(NOT BLAS_FOUND)
+        message(STATUS "OpenBLAS BOUNDED")
+        ExternalProject_Add_StepDependencies(faiss_ep configure openblas_ep)
+    endif()
+
+    file(MAKE_DIRECTORY "${FAISS_INCLUDE_DIR}")
+    add_library(faiss STATIC IMPORTED)
+
+    set_target_properties( faiss
+            PROPERTIES
+                IMPORTED_GLOBAL                 TRUE
+                IMPORTED_LOCATION               "${FAISS_STATIC_LIB}"
+                INTERFACE_INCLUDE_DIRECTORIES   "${FAISS_INCLUDE_DIR}")
+
+    target_link_libraries( faiss INTERFACE openblas )
+    add_dependencies(faiss faiss_ep)
+
+endmacro()
+
+if (KNOWHERE_WITH_FAISS AND NOT TARGET faiss_ep)
+    if (FAISS_SOURCE STREQUAL "AUTO")
+        find_package(FAISS MODULE)
+        if (NOT FAISS_FOUND})
+            build_faiss()
+        endif ()
+    elseif (FAISS_SOURCE STREQUAL "BUNDLED")
+        build_faiss()
+    elseif (FAISS_SOURCE STREQUAL "SYSTEM")
+        find_package(FAISS REQUIRED)
+    endif ()
+
+    get_target_property(FAISS_INCLUDE_DIR faiss INTERFACE_INCLUDE_DIRECTORIES)
+    include_directories(SYSTEM "${FAISS_INCLUDE_DIR}")
+    link_directories(SYSTEM ${FAISS_PREFIX}/lib/)
+endif ()

internal/core/src/index/thirdparty/OpenBLAS/CMakeLists.txt

@@ -0,0 +1,86 @@
+if (DEFINED ENV{KNOWHERE_OPENBLAS_URL})
+    set(OPENBLAS_SOURCE_URL "$ENV{KNOWHERE_OPENBLAS_URL}")
+else ()
+    set(OPENBLAS_SOURCE_URL
+            "https://github.com.cnpmjs.org/xianyi/OpenBLAS/archive/v${OPENBLAS_VERSION}.tar.gz")
+endif ()
+
+set(OPENBLAS_PREFIX "${INDEX_BINARY_DIR}/openblas_ep-prefix/src/openblas_ep")
+
+macro(build_openblas)
+    message(STATUS "Building OpenBLAS-${OPENBLAS_VERSION} from source")
+    set(OpenBLAS_INCLUDE_DIR "${OPENBLAS_PREFIX}/include")
+    set(OpenBLAS_LIB_DIR "${OPENBLAS_PREFIX}/lib")
+    set(OPENBLAS_SHARED_LIB
+            "${OPENBLAS_PREFIX}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}openblas${CMAKE_SHARED_LIBRARY_SUFFIX}")
+    set(OPENBLAS_STATIC_LIB
+            "${OPENBLAS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}openblas${CMAKE_STATIC_LIBRARY_SUFFIX}")
+    set(OPENBLAS_CMAKE_ARGS
+            ${EP_COMMON_CMAKE_ARGS}
+            -DCMAKE_BUILD_TYPE=Release
+            -DBUILD_SHARED_LIBS=ON
+            -DBUILD_STATIC_LIBS=ON
+            -DTARGET=CORE2
+            -DDYNAMIC_ARCH=1
+            -DDYNAMIC_OLDER=1
+            -DUSE_THREAD=0
+            -DUSE_OPENMP=OPENBLAS_SOURCE_URL
+            -DFC=gfortran
+            -DCC=gcc
+            -DINTERFACE64=0
+            -DNUM_THREADS=128
+            -DNO_LAPACKE=1
+            "-DVERSION=${OPENBLAS_VERSION}"
+            "-DCMAKE_INSTALL_PREFIX=${OPENBLAS_PREFIX}"
+            -DCMAKE_INSTALL_LIBDIR=lib)
+
+    externalproject_add(openblas_ep
+	            URL
+		    ${OPENBLAS_SOURCE_URL}
+		    URL_MD5
+                    "28cc19a6acbf636f5aab5f10b9a0dfe1" 
+		    ${EP_LOG_OPTIONS}
+		    CMAKE_ARGS
+		    ${OPENBLAS_CMAKE_ARGS}
+		    BUILD_COMMAND
+		    ${MAKE}
+		    ${MAKE_BUILD_ARGS}
+		    BUILD_IN_SOURCE
+		    1
+		    INSTALL_COMMAND
+		    ${MAKE}
+		    PREFIX=${OPENBLAS_PREFIX}
+		    install
+		    BUILD_BYPRODUCTS
+		    ${OPENBLAS_SHARED_LIB}
+		    ${OPENBLAS_STATIC_LIB})
+
+    file(MAKE_DIRECTORY "${OpenBLAS_INCLUDE_DIR}")
+    add_library(openblas SHARED IMPORTED)
+
+    set_target_properties(
+            openblas
+            PROPERTIES
+            IMPORTED_LOCATION "${OPENBLAS_SHARED_LIB}"
+            LIBRARY_OUTPUT_NAME "openblas"
+            INTERFACE_INCLUDE_DIRECTORIES "${OpenBLAS_INCLUDE_DIR}")
+    add_dependencies(openblas openblas_ep)
+    get_target_property(OpenBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES)
+    set(OpenBLAS_LIBRARIES "${OPENBLAS_SHARED_LIB}")
+endmacro()
+
+
+if (KNOWHERE_WITH_OPENBLAS)
+    if (OpenBLAS_SOURCE STREQUAL "AUTO")
+        find_package(OpenBLAS MODULE)
+        if (NOT ${OpenBLAS_FOUND})
+            build_openblas()
+        endif ()
+    elseif (OpenBLAS_SOURCE STREQUAL "BUNDLED")
+        build_openblas()
+    elseif (OpenBLAS_SOURCE STREQUAL "SYSTEM")
+        find_package(OpenBLAS REQUIRED)
+    endif ()    
+    include_directories(SYSTEM "${OpenBLAS_INCLUDE_DIR}")
+    link_directories(SYSTEM "${OpenBLAS_LIB_DIR}")
+endif()

internal/core/src/index/thirdparty/fiu/CMakeLists.txt

@@ -0,0 +1,61 @@
+#-------------------------------------------------------------------------------
+# Copyright (C) 2019-2020 Zilliz. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied. See the License for the specific language governing permissions and limitations under the License.
+#-------------------------------------------------------------------------------
+if(NOT DEFINED FIU_VERSION)
+   set(FIU_VERSION 1.00)
+endif()
+
+if ( DEFINED ENV{Knowhere_FIU_URL} )
+    set( FIU_SOURCE_URL "$ENV{MILVUS_FIU_URL}" )
+else ()
+    set( FIU_SOURCE_URL "https://github.com.cnpmjs.org/albertito/libfiu/archive/${FIU_VERSION}.tar.gz" )
+endif ()
+
+macro( build_fiu )
+    message( STATUS "Building FIU-${FIU_VERSION} from source" )
+    ExternalProject_Add(
+            fiu_ep
+            PREFIX              ${CMAKE_BINARY_DIR}/3rdparty_download/fiu-subbuild
+            DOWNLOAD_DIR        ${THIRDPARTY_DOWNLOAD_PATH}
+            INSTALL_DIR         ${CMAKE_CURRENT_BINARY_DIR}
+            URL                 ${FIU_SOURCE_URL}
+            URL_MD5             "75f9d076daf964c9410611701f07c61b"
+            CONFIGURE_COMMAND   ""
+            BUILD_IN_SOURCE     1
+            BUILD_COMMAND       ${MAKE}
+            INSTALL_COMMAND     ${MAKE} "PREFIX=<INSTALL_DIR>" install
+            ${EP_LOG_OPTIONS}
+            )
+
+    ExternalProject_Get_Property( fiu_ep INSTALL_DIR )
+    if( NOT IS_DIRECTORY ${INSTALL_DIR}/include )
+        file( MAKE_DIRECTORY "${INSTALL_DIR}/include" )
+    endif()
+    add_library( fiu SHARED IMPORTED )
+    set_target_properties( fiu
+            PROPERTIES
+                IMPORTED_GLOBAL                 TRUE
+                IMPORTED_LOCATION               ${INSTALL_DIR}/lib/libfiu.so
+                INTERFACE_INCLUDE_DIRECTORIES   ${INSTALL_DIR}/include )
+    add_dependencies(fiu fiu_ep)
+endmacro()
+
+build_fiu()
+
+install( FILES  ${INSTALL_DIR}/lib/libfiu.so
+                ${INSTALL_DIR}/lib/libfiu.so.0
+                ${INSTALL_DIR}/lib/libfiu.so.1.00
+        DESTINATION lib )
+
+get_target_property( var fiu INTERFACE_INCLUDE_DIRECTORIES )
+message( STATUS ${var} )
+set_directory_properties( PROPERTY INCLUDE_DIRECTORIES ${var} )

internal/core/src/index/thirdparty/fiu/fiu-local.h

@@ -0,0 +1,37 @@
+
+/* libfiu - Fault Injection in Userspace
+ *
+ * This header, part of libfiu, is meant to be included in your project to
+ * avoid having libfiu as a mandatory build-time dependency.
+ *
+ * You can add it to your project, and #include it instead of fiu.h.
+ * The real fiu.h will be used only when FIU_ENABLE is defined.
+ *
+ * This header, as the rest of libfiu, is in the public domain.
+ *
+ * You can find more information about libfiu at
+ * http://blitiri.com.ar/p/libfiu.
+ */
+
+#ifndef _FIU_LOCAL_H
+#define _FIU_LOCAL_H
+
+/* Only define the stubs when fiu is disabled, otherwise use the real fiu.h
+ * header */
+#ifndef FIU_ENABLE
+
+#define fiu_init(flags) 0
+#define fiu_fail(name) 0
+#define fiu_failinfo() NULL
+#define fiu_do_on(name, action)
+#define fiu_exit_on(name)
+#define fiu_return_on(name, retval)
+
+#else
+
+#include <fiu.h>
+
+#endif /* FIU_ENABLE */
+
+#endif /* _FIU_LOCAL_H */
+

internal/core/src/index/thirdparty/google_benchmark/CMakeLists.txt

@@ -0,0 +1,20 @@
+FetchContent_Declare(google_benchmark
+    URL             https://github.com/google/benchmark/archive/v1.5.2.tar.gz
+    URL_MD5         084b34aceaeac11a6607d35220ca2efa
+    DOWNLOAD_DIR    ${THIRDPARTY_DOWNLOAD_PATH}
+    SOURCE_DIR      ${CMAKE_CURRENT_BINARY_DIR}/google_benchmark
+    BINARY_DIR      ${CMAKE_CURRENT_BINARY_DIR}/google_benchmark
+        )
+
+FetchContent_GetProperties( google_benchmark )
+if ( NOT google_benchmark_POPULATED )
+
+    FetchContent_Populate( google_benchmark )
+
+    # Adding the following targets:
+    # gtest, gtest_main, gmock, gmock_main
+    message("gb=${google_benchmark_SOURCE_DIR}")
+    add_subdirectory( ${google_benchmark_SOURCE_DIR}
+                      ${google_benchmark_BINARY_DIR}
+                      EXCLUDE_FROM_ALL )
+endif()

internal/core/src/index/thirdparty/gtest/CMakeLists.txt

@@ -0,0 +1,66 @@
+#-------------------------------------------------------------------------------
+# Copyright (C) 2019-2020 Zilliz. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied. See the License for the specific language governing permissions and limitations under the License.
+#-------------------------------------------------------------------------------
+set( GTEST_VERSION 1.8.1 )
+
+if ( DEFINED ENV{MILVUS_GTEST_URL} )
+    set( GTEST_SOURCE_URL "$ENV{MILVUS_GTEST_URL}" )
+else()
+    set( GTEST_SOURCE_URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.zip" )
+endif()
+
+message( STATUS "Building gtest-${GTEST_VERSION} from source" )
+
+set( CMAKE_POLICY_DEFAULT_CMP0022 NEW ) # for googletest only
+
+FetchContent_Declare(
+        googletest
+        URL             ${GTEST_SOURCE_URL} 
+        URL_MD5         "ad6868782b5952b7476a7c1c72d5a714"
+        SOURCE_DIR      ${CMAKE_CURRENT_BINARY_DIR}/googletest-src
+        BINARY_DIR      ${CMAKE_CURRENT_BINARY_DIR}/googletest-build
+        DOWNLOAD_DIR    ${THIRDPARTY_DOWNLOAD_PATH} )
+
+FetchContent_GetProperties( googletest )
+if ( NOT googletest_POPULATED )
+
+    FetchContent_Populate( googletest )
+
+    # Adding the following targets:
+    # gtest, gtest_main, gmock, gmock_main
+    add_subdirectory( ${googletest_SOURCE_DIR}
+                      ${googletest_BINARY_DIR}
+                      EXCLUDE_FROM_ALL )
+endif()
+# include(GoogleTest)
+
+# ****************************************************************
+# Create ALIAS Target
+# ****************************************************************
+# if (NOT TARGET GTest:gtest)
+#     add_library( GTest::gtest ALIAS gtest )
+# endif()
+# if (NOT TARGET GTest:main)
+#     add_library( GTest::main ALIAS gtest_main )
+# endif()
+# if (NOT TARGET GMock:gmock)
+#     target_link_libraries( gmock INTERFACE GTest::gtest )
+#     add_library( GMock::gmock ALIAS gmock )
+# endif()
+# if (NOT TARGET GMock:main)
+#     target_link_libraries( gmock_main INTERFACE GTest::gtest )
+#     add_library( GMock::main ALIAS gmock_main )
+# endif()
+
+
+get_property( var DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/googletest-src" PROPERTY COMPILE_OPTIONS )
+message( STATUS "gtest compile options: ${var}" )

internal/core/src/index/unittest/CMakeLists.txt

@@ -15,32 +15,21 @@ include_directories(${INDEX_SOURCE_DIR}/thirdparty/NGT/lib)
 include_directories(${INDEX_SOURCE_DIR}/knowhere)
 include_directories(${INDEX_SOURCE_DIR})
 
-set(depend_libs
-        gtest gmock gtest_main gmock_main
-        faiss fiu
+set(DEPEND_LIBS
+        gtest
+        gmock
+        gtest_main
+        gmock_main
+        faiss
+        fiu
         ngt
         index_log
-        )
-if (FAISS_WITH_MKL)
-    set(depend_libs ${depend_libs}
-            "-Wl,--start-group \
-            ${MKL_LIB_PATH}/libmkl_intel_ilp64.a \
-            ${MKL_LIB_PATH}/libmkl_gnu_thread.a \
-            ${MKL_LIB_PATH}/libmkl_core.a \
-            -Wl,--end-group -lgomp -lpthread -lm -ldl"
-            )
-else ()
-    set(depend_libs ${depend_libs}
-            ${OpenBLAS_LIBRARIES}
-            ${LAPACK_LIBRARIES}
-            )
-endif ()
-
-set(basic_libs
-        gomp gfortran pthread
+        openblas
         )
 
-set(util_srcs
+set(BASIC_LIBS gomp gfortran pthread)
+
+set(UTIL_SRCS
         ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/adapter/VectorAdapter.cpp
         ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/FaissIO.cpp
         ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/IndexParameter.cpp
@@ -56,19 +45,17 @@ set(util_srcs
 if (MILVUS_GPU_VERSION)
     include_directories(${CUDA_INCLUDE_DIRS})
     link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
-    set(cuda_lib
+    set(CUDA_LIB
             cudart
             cublas
             )
-    set(basic_libs ${basic_libs}
-            ${cuda_lib}
-            )
-    set(util_srcs ${util_srcs}
+    set(BASIC_LIBS ${basic_libs} ${cuda_lib})
+    set(UTIL_SRCS ${util_srcs}
             ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/FaissGpuResourceMgr.cpp
             )
 endif ()
 
-set(faiss_srcs
+set(FAISS_SRCS
         ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/FaissBaseIndex.cpp
         ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/FaissBaseBinaryIndex.cpp
         ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexBinaryIDMAP.cpp
@@ -81,8 +68,9 @@ set(faiss_srcs
         ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_offset_index/OffsetBaseIndex.cpp
         ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_offset_index/IndexIVF_NM.cpp
         )
+
 if (MILVUS_GPU_VERSION)
-    set(faiss_srcs ${faiss_srcs}
+    set(FAISS_SRCS ${faiss_srcs}
             ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/ConfAdapter.cpp
             ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp
             ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexGPUIDMAP.cpp
@@ -106,8 +94,8 @@ else ()
     message(FATAL_ERROR "no OpenMP supprot")
 endif ()
 
-set(all_indexing_srcs
-        ${util_srcs}
+set(ALL_INDEXING_SRCS
+        ${UTIL_SRCS}
         test_instructionset.cpp
         test_common.cpp
         test_customized_index.cpp
@@ -127,21 +115,22 @@ set(all_indexing_srcs
         test_ngtpanng.cpp
         test_ngtonng.cpp
         )
+
 if (MILVUS_GPU_VERSION)
-    set(all_indexing_srcs
-            ${all_indexing_srcs}
+    set(ALL_INDEXING_SRCS
+            ${ALL_INDEXING_SRCS}
             test_ivf_gpu_nm.cpp
             test_gpuresource.cpp
             )
 endif ()
 
 if (MILVUS_SUPPORT_SPTAG)
-    set(all_indexing_srcs
+    set(ALL_INDEXING_SRCS
             test_sptag.cpp
             )
 endif()
 
-add_executable(test_knowhere ${all_indexing_srcs})
-target_link_libraries(test_knowhere knowhere ${depend_libs})
+add_executable(test_knowhere ${ALL_INDEXING_SRCS})
+target_link_libraries(test_knowhere knowhere ${DEPEND_LIBS})
 
-install(TARGETS test_knowhere DESTINATION unittest)
+install(TARGETS test_knowhere)

internal/core/src/index/unittest/test_common.cpp

@@ -14,8 +14,8 @@
 #include "knowhere/common/Timer.h"
 #include "knowhere/knowhere/common/Exception.h"
 #include "unittest/utils.h"
-#include "index/thirdparty/faiss/utils/BitsetView.h"
-#include "index/thirdparty/faiss/utils/ConcurrentBitset.h"
+#include "faiss/utils/BitsetView.h"
+#include "faiss/utils/ConcurrentBitset.h"
 #include <boost/dynamic_bitset.hpp>
 
 /*Some unittest for knowhere/common, mainly for improve code coverage.*/

internal/core/thirdparty/CMakeLists.txt

@@ -41,7 +41,7 @@ find_package( Threads REQUIRED )
 # ****************************** Thirdparty googletest ***************************************
 if ( MILVUS_BUILD_TESTS )
     add_subdirectory( gtest )
-    add_subdirectory( google_benchmark)
+    add_subdirectory( google_benchmark )
 endif()
 
 

internal/core/thirdparty/google_benchmark/CMakeLists.txt

@@ -12,7 +12,7 @@ if ( NOT google_benchmark_POPULATED )
     FetchContent_Populate( google_benchmark )
 
     # Adding the following targets:
-    # gtest, gtest_main, gmock, gmock_main
+    # benchmark::benchmark_main
     message("gb=${google_benchmark_SOURCE_DIR}")
     add_subdirectory( ${google_benchmark_SOURCE_DIR}
                       ${google_benchmark_BINARY_DIR}