test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-02 20:09:57 +08:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

Merge knowhere unittests (#5139)

Browse Source 
Re-enable knowhere unittests and merge them all together. 
Signed-off-by: fluorinedog <fluorinedog@gmail.com>
This commit is contained in:
FluorineDog 2021-05-07 19:50:13 +08:00 committed by GitHub
parent b7977698a8
commit b1974aede2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 52 additions and 236 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
internal/core/src/index/CMakeLists.txt
View File

@ -94,8 +94,8 @@ endif ()
set(INDEX_INCLUDE_DIRS ${INDEX_INCLUDE_DIRS} PARENT_SCOPE) set(INDEX_INCLUDE_DIRS ${INDEX_INCLUDE_DIRS} PARENT_SCOPE)
if (KNOWHERE_BUILD_TESTS) if (KNOWHERE_BUILD_TESTS)
#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DELPP_DISABLE_LOGS") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DELPP_DISABLE_LOGS")
#add_subdirectory(unittest) add_subdirectory(unittest)
endif () endif ()
config_summary() config_summary()

271
internal/core/src/index/unittest/CMakeLists.txt
View File

@ -82,105 +82,18 @@ set(faiss_srcs
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_offset_index/IndexIVF_NM.cpp ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_offset_index/IndexIVF_NM.cpp
) )
if (MILVUS_GPU_VERSION) if (MILVUS_GPU_VERSION)
set(faiss_srcs ${faiss_srcs} set(faiss_srcs ${faiss_srcs}
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/ConfAdapter.cpp ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/ConfAdapter.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexGPUIDMAP.cpp ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexGPUIDMAP.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVF.cpp ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVF.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFSQ.cpp ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFSQ.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFPQ.cpp ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFPQ.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexIVFSQHybrid.cpp ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexIVFSQHybrid.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_offset_index/gpu/IndexGPUIVF_NM.cpp ${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_offset_index/gpu/IndexGPUIVF_NM.cpp
) )
endif () endif ()
################################################################################
#<INSTRUCTIONSET-TEST>
if (NOT TARGET test_instructionset)
add_executable(test_instructionset test_instructionset.cpp)
endif ()
target_link_libraries(test_instructionset ${depend_libs} ${unittest_libs})
install(TARGETS test_instructionset DESTINATION unittest)
################################################################################
#<KNOWHERE-COMMON-TEST>
if (NOT TARGET test_knowhere_common)
add_executable(test_knowhere_common test_common.cpp ${util_srcs})
endif ()
target_link_libraries(test_knowhere_common ${depend_libs} ${unittest_libs} ${basic_libs})
install(TARGETS test_knowhere_common DESTINATION unittest)
if (MILVUS_GPU_VERSION)
################################################################################
#<GPU-RESOURCE-TEST>
add_executable(test_gpuresource test_gpuresource.cpp ${util_srcs} ${faiss_srcs})
target_link_libraries(test_gpuresource ${depend_libs} ${unittest_libs} ${basic_libs})
install(TARGETS test_gpuresource DESTINATION unittest)
################################################################################
#<CUSTOMIZED-INDEX-TEST>
add_executable(test_customized_index test_customized_index.cpp ${util_srcs} ${faiss_srcs})
target_link_libraries(test_customized_index ${depend_libs} ${unittest_libs} ${basic_libs})
install(TARGETS test_customized_index DESTINATION unittest)
endif ()
################################################################################
#<IDMAP-TEST>
if (NOT TARGET test_idmap)
add_executable(test_idmap test_idmap.cpp ${faiss_srcs} ${util_srcs})
endif ()
target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs})
install(TARGETS test_idmap DESTINATION unittest)
################################################################################
#<IVF-TEST>
if (NOT TARGET test_ivf)
add_executable(test_ivf test_ivf.cpp ${faiss_srcs} ${util_srcs})
endif ()
target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs})
install(TARGETS test_ivf DESTINATION unittest)
################################################################################
#<IVFHNSW-TEST>
if (NOT TARGET test_ivf_hnsw)
add_executable(test_ivf_hnsw test_ivf_hnsw.cpp ${faiss_srcs} ${util_srcs})
endif ()
target_link_libraries(test_ivf_hnsw ${depend_libs} ${unittest_libs} ${basic_libs})
install(TARGETS test_ivf_hnsw DESTINATION unittest)
################################################################################
#<IVFNM-TEST-CPU>
if (NOT TARGET test_ivf_cpu_nm)
add_executable(test_ivf_cpu_nm test_ivf_cpu_nm.cpp ${faiss_srcs} ${util_srcs})
endif ()
target_link_libraries(test_ivf_cpu_nm ${depend_libs} ${unittest_libs} ${basic_libs})
install(TARGETS test_ivf_cpu_nm DESTINATION unittest)
################################################################################
#<IVFNM-TEST-GPU>
if (NOT TARGET test_ivf_gpu_nm)
add_executable(test_ivf_gpu_nm test_ivf_gpu_nm.cpp ${faiss_srcs} ${util_srcs})
endif ()
target_link_libraries(test_ivf_gpu_nm ${depend_libs} ${unittest_libs} ${basic_libs})
install(TARGETS test_ivf_gpu_nm DESTINATION unittest)
################################################################################
#<BinaryIDMAP-TEST>
if (NOT TARGET test_binaryidmap)
add_executable(test_binaryidmap test_binaryidmap.cpp ${faiss_srcs} ${util_srcs})
endif ()
target_link_libraries(test_binaryidmap ${depend_libs} ${unittest_libs} ${basic_libs})
install(TARGETS test_binaryidmap DESTINATION unittest)
################################################################################
#<BinaryIVF-TEST>
if (NOT TARGET test_binaryivf)
add_executable(test_binaryivf test_binaryivf.cpp ${faiss_srcs} ${util_srcs})
endif ()
target_link_libraries(test_binaryivf ${depend_libs} ${unittest_libs} ${basic_libs})
install(TARGETS test_binaryivf DESTINATION unittest)
################################################################################ ################################################################################
#<NSG-TEST> #<NSG-TEST>
add_definitions(-std=c++11 -O3 -march=native -Werror -DINFO) add_definitions(-std=c++11 -O3 -march=native -Werror -DINFO)
@ -193,140 +106,42 @@ else ()
message(FATAL_ERROR "no OpenMP supprot") message(FATAL_ERROR "no OpenMP supprot")
endif () endif ()
include_directories(${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/impl/nsg) set(all_indexing_srcs
aux_source_directory(${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/impl/nsg nsg_src) ${util_srcs}
set(interface_src test_instructionset.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_offset_index/IndexNSG_NM.cpp test_common.cpp
test_customized_index.cpp
test_idmap.cpp
test_ivf.cpp
test_ivf_hnsw.cpp
test_ivf_cpu_nm.cpp
test_binaryidmap.cpp
test_binaryivf.cpp
test_nsg.cpp
test_hnsw.cpp
test_rhnsw_flat.cpp
test_rhnsw_pq.cpp
test_rhnsw_sq8.cpp
test_structured_index_sort.cpp
test_structured_index_flat.cpp
test_ngtpanng.cpp
test_ngtonng.cpp
) )
if (NOT TARGET test_nsg) if (MILVUS_GPU_VERSION)
add_executable(test_nsg test_nsg.cpp ${interface_src} ${nsg_src} ${util_srcs} ${faiss_srcs}) set(all_indexing_srcs
endif () ${all_indexing_srcs}
target_link_libraries(test_nsg ${depend_libs} ${unittest_libs} ${basic_libs}) test_ivf_gpu_nm.cpp
install(TARGETS test_nsg DESTINATION unittest) test_gpuresource.cpp
################################################################################
#<HNSW-TEST>
set(hnsw_srcs
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexHNSW.cpp
)
if (NOT TARGET test_hnsw)
add_executable(test_hnsw test_hnsw.cpp ${hnsw_srcs} ${util_srcs})
endif ()
target_link_libraries(test_hnsw ${depend_libs} ${unittest_libs} ${basic_libs})
install(TARGETS test_hnsw DESTINATION unittest)
################################################################################
#<RHNSW_FLAT-TEST>
set(rhnsw_flat_srcs
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexRHNSW.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexRHNSWFlat.cpp
)
if (NOT TARGET test_rhnsw_flat)
add_executable(test_rhnsw_flat test_rhnsw_flat.cpp ${rhnsw_flat_srcs} ${util_srcs} ${faiss_srcs})
endif ()
target_link_libraries(test_rhnsw_flat ${depend_libs} ${unittest_libs} ${basic_libs})
install(TARGETS test_rhnsw_flat DESTINATION unittest)
################################################################################
#<RHNSW_PQ-TEST>
set(rhnsw_pq_srcs
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexRHNSW.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexRHNSWPQ.cpp
)
if (NOT TARGET test_rhnsw_pq)
add_executable(test_rhnsw_pq test_rhnsw_pq.cpp ${rhnsw_pq_srcs} ${util_srcs} ${faiss_srcs})
endif ()
target_link_libraries(test_rhnsw_pq ${depend_libs} ${unittest_libs} ${basic_libs})
install(TARGETS test_rhnsw_pq DESTINATION unittest)
################################################################################
#<RHNSW_SQ8-TEST>
set(rhnsw_sq8_srcs
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexRHNSW.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexRHNSWSQ.cpp
)
if (NOT TARGET test_rhnsw_sq8)
add_executable(test_rhnsw_sq8 test_rhnsw_sq8.cpp ${rhnsw_sq8_srcs} ${util_srcs} ${faiss_srcs})
endif ()
target_link_libraries(test_rhnsw_sq8 ${depend_libs} ${unittest_libs} ${basic_libs})
install(TARGETS test_rhnsw_sq8 DESTINATION unittest)
################################################################################
#<SPTAG-TEST>
if (MILVUS_SUPPORT_SPTAG)
set(sptag_srcs
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/adapter/SptagAdapter.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp
) )
if (NOT TARGET test_sptag)
add_executable(test_sptag test_sptag.cpp ${sptag_srcs} ${util_srcs})
endif ()
target_link_libraries(test_sptag
SPTAGLibStatic
${depend_libs} ${unittest_libs} ${basic_libs})
install(TARGETS test_sptag DESTINATION unittest)
endif () endif ()
################################################################################ if (MILVUS_SUPPORT_SPTAG)
#<ANNOY-TEST> set(all_indexing_srcs
set(annoy_srcs test_sptag.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexAnnoy.cpp )
) endif()
if (NOT TARGET test_annoy)
add_executable(test_annoy test_annoy.cpp ${annoy_srcs} ${util_srcs})
endif ()
target_link_libraries(test_annoy ${depend_libs} ${unittest_libs} ${basic_libs})
install(TARGETS test_annoy DESTINATION unittest)
################################################################################ add_executable(test_knowhere ${all_indexing_srcs})
#<STRUCTURED-INDEX-SORT-TEST> target_link_libraries(test_knowhere knowhere ${depend_libs})
set(structured_index_sort_srcs
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/structured_index/StructuredIndexSort-inl.h
)
if (NOT TARGET test_structured_index_sort)
add_executable(test_structured_index_sort test_structured_index_sort.cpp ${structured_index_sort_srcs} ${util_srcs})
endif ()
target_link_libraries(test_structured_index_sort ${depend_libs} ${unittest_libs} ${basic_libs})
install(TARGETS test_structured_index_sort DESTINATION unittest)
#add_subdirectory(faiss_benchmark) install(TARGETS test_knowhere DESTINATION unittest)
#add_subdirectory(metric_alg_benchmark)
################################################################################
#<STRUCTURED-INDEX-FLAT-TEST>
set(structured_index_flat_srcs
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/structured_index/StructuredIndexFlat-inl.h
)
if (NOT TARGET test_structured_index_flat)
add_executable(test_structured_index_flat test_structured_index_flat.cpp ${structured_index_flat_srcs} ${util_srcs})
endif ()
target_link_libraries(test_structured_index_flat ${depend_libs} ${unittest_libs} ${basic_libs})
install(TARGETS test_structured_index_flat DESTINATION unittest)
################################################################################
#<NGTPANNG-TEST>
set(ngtpanng_srcs
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexNGTPANNG.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexNGT.cpp
)
if (NOT TARGET test_ngtpanng)
add_executable(test_ngtpanng test_ngtpanng.cpp ${ngtpanng_srcs} ${util_srcs})
endif ()
target_link_libraries(test_ngtpanng ${depend_libs} ${unittest_libs} ${basic_libs})
install(TARGETS test_ngtpanng DESTINATION unittest)
################################################################################
#<NGTPANNG-TEST>
set(ngtonng_srcs
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexNGTONNG.cpp
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexNGT.cpp
)
if (NOT TARGET test_ngtonng)
add_executable(test_ngtonng test_ngtonng.cpp ${ngtonng_srcs} ${util_srcs})
endif ()
target_link_libraries(test_ngtonng ${depend_libs} ${unittest_libs} ${basic_libs})
install(TARGETS test_ngtonng DESTINATION unittest)
################################################################################

7
internal/core/src/index/unittest/Helper.h
View File

@ -8,6 +8,7 @@
// Unless required by applicable law or agreed to in writing, software distributed under the License // Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License. // or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <memory> #include <memory>
#include <string> #include <string>
@ -28,7 +29,7 @@
#include "knowhere/index/vector_offset_index/gpu/IndexGPUIVF_NM.h" #include "knowhere/index/vector_offset_index/gpu/IndexGPUIVF_NM.h"
#endif #endif
int DEVICEID = 0; constexpr int DEVICEID = 0;
constexpr int64_t DIM = 128; constexpr int64_t DIM = 128;
constexpr int64_t NB = 10000; constexpr int64_t NB = 10000;
constexpr int64_t NQ = 10; constexpr int64_t NQ = 10;
@ -37,7 +38,7 @@ constexpr int64_t PINMEM = 1024 * 1024 * 200;
constexpr int64_t TEMPMEM = 1024 * 1024 * 300; constexpr int64_t TEMPMEM = 1024 * 1024 * 300;
constexpr int64_t RESNUM = 2; constexpr int64_t RESNUM = 2;
milvus::knowhere::IVFPtr inline milvus::knowhere::IVFPtr
IndexFactory(const milvus::knowhere::IndexType& type, const milvus::knowhere::IndexMode mode) { IndexFactory(const milvus::knowhere::IndexType& type, const milvus::knowhere::IndexMode mode) {
if (mode == milvus::knowhere::IndexMode::MODE_CPU) { if (mode == milvus::knowhere::IndexMode::MODE_CPU) {
if (type == milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT) { if (type == milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT) {
@ -71,7 +72,7 @@ IndexFactory(const milvus::knowhere::IndexType& type, const milvus::knowhere::In
return nullptr; return nullptr;
} }
milvus::knowhere::IVFNMPtr inline milvus::knowhere::IVFNMPtr
IndexFactoryNM(const milvus::knowhere::IndexType& type, const milvus::knowhere::IndexMode mode) { IndexFactoryNM(const milvus::knowhere::IndexType& type, const milvus::knowhere::IndexMode mode) {
if (mode == milvus::knowhere::IndexMode::MODE_CPU) { if (mode == milvus::knowhere::IndexMode::MODE_CPU) {
if (type == milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT) { if (type == milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT) {

6
internal/core/src/index/unittest/test_structured_index_flat.cpp
View File

@ -20,7 +20,7 @@
#include "unittest/utils.h" #include "unittest/utils.h"
void static void
gen_rand_data(int range, int n, int*& p) { gen_rand_data(int range, int n, int*& p) {
srand((unsigned int)time(nullptr)); srand((unsigned int)time(nullptr));
p = (int*)malloc(n * sizeof(int)); p = (int*)malloc(n * sizeof(int));
@ -30,7 +30,7 @@ gen_rand_data(int range, int n, int*& p) {
} }
} }
void static void
gen_rand_int64_data(int64_t range, int64_t n, int64_t*& p) { gen_rand_int64_data(int64_t range, int64_t n, int64_t*& p) {
srand((int64_t)time(nullptr)); srand((int64_t)time(nullptr));
p = (int64_t*)malloc(n * sizeof(int64_t)); p = (int64_t*)malloc(n * sizeof(int64_t));
@ -40,7 +40,7 @@ gen_rand_int64_data(int64_t range, int64_t n, int64_t*& p) {
} }
} }
void static void
gen_rand_double_data(double range, int64_t n, double*& p) { gen_rand_double_data(double range, int64_t n, double*& p) {
std::uniform_real_distribution<double> unif(0, range); std::uniform_real_distribution<double> unif(0, range);
std::default_random_engine re; std::default_random_engine re;