mirror of
https://gitee.com/milvus-io/milvus.git
synced 2024-11-30 10:59:32 +08:00
Merge knowhere unittests (#5139)
Re-enable knowhere unittests and merge them all together. Signed-off-by: fluorinedog <fluorinedog@gmail.com>
This commit is contained in:
FluorineDog
GitHub
parent
b7977698a8
commit
b1974aede2
@ -94,8 +94,8 @@ endif ()
|
||||
set(INDEX_INCLUDE_DIRS ${INDEX_INCLUDE_DIRS} PARENT_SCOPE)
|
||||
|
||||
if (KNOWHERE_BUILD_TESTS)
|
||||
#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DELPP_DISABLE_LOGS")
|
||||
#add_subdirectory(unittest)
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DELPP_DISABLE_LOGS")
|
||||
add_subdirectory(unittest)
|
||||
endif ()
|
||||
|
||||
config_summary()
|
||||
|
||||
@ -82,105 +82,18 @@ set(faiss_srcs
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_offset_index/IndexIVF_NM.cpp
|
||||
)
|
||||
if (MILVUS_GPU_VERSION)
|
||||
set(faiss_srcs ${faiss_srcs}
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/ConfAdapter.cpp
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexGPUIDMAP.cpp
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVF.cpp
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFSQ.cpp
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFPQ.cpp
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexIVFSQHybrid.cpp
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_offset_index/gpu/IndexGPUIVF_NM.cpp
|
||||
)
|
||||
set(faiss_srcs ${faiss_srcs}
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/ConfAdapter.cpp
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexGPUIDMAP.cpp
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVF.cpp
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFSQ.cpp
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexGPUIVFPQ.cpp
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu/IndexIVFSQHybrid.cpp
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_offset_index/gpu/IndexGPUIVF_NM.cpp
|
||||
)
|
||||
endif ()
|
||||
|
||||
################################################################################
|
||||
#<INSTRUCTIONSET-TEST>
|
||||
if (NOT TARGET test_instructionset)
|
||||
add_executable(test_instructionset test_instructionset.cpp)
|
||||
endif ()
|
||||
target_link_libraries(test_instructionset ${depend_libs} ${unittest_libs})
|
||||
install(TARGETS test_instructionset DESTINATION unittest)
|
||||
|
||||
################################################################################
|
||||
#<KNOWHERE-COMMON-TEST>
|
||||
if (NOT TARGET test_knowhere_common)
|
||||
add_executable(test_knowhere_common test_common.cpp ${util_srcs})
|
||||
endif ()
|
||||
target_link_libraries(test_knowhere_common ${depend_libs} ${unittest_libs} ${basic_libs})
|
||||
install(TARGETS test_knowhere_common DESTINATION unittest)
|
||||
|
||||
if (MILVUS_GPU_VERSION)
|
||||
################################################################################
|
||||
#<GPU-RESOURCE-TEST>
|
||||
add_executable(test_gpuresource test_gpuresource.cpp ${util_srcs} ${faiss_srcs})
|
||||
target_link_libraries(test_gpuresource ${depend_libs} ${unittest_libs} ${basic_libs})
|
||||
install(TARGETS test_gpuresource DESTINATION unittest)
|
||||
|
||||
################################################################################
|
||||
#<CUSTOMIZED-INDEX-TEST>
|
||||
add_executable(test_customized_index test_customized_index.cpp ${util_srcs} ${faiss_srcs})
|
||||
target_link_libraries(test_customized_index ${depend_libs} ${unittest_libs} ${basic_libs})
|
||||
install(TARGETS test_customized_index DESTINATION unittest)
|
||||
endif ()
|
||||
|
||||
################################################################################
|
||||
#<IDMAP-TEST>
|
||||
if (NOT TARGET test_idmap)
|
||||
add_executable(test_idmap test_idmap.cpp ${faiss_srcs} ${util_srcs})
|
||||
endif ()
|
||||
target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs})
|
||||
install(TARGETS test_idmap DESTINATION unittest)
|
||||
|
||||
################################################################################
|
||||
#<IVF-TEST>
|
||||
if (NOT TARGET test_ivf)
|
||||
add_executable(test_ivf test_ivf.cpp ${faiss_srcs} ${util_srcs})
|
||||
endif ()
|
||||
target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs})
|
||||
install(TARGETS test_ivf DESTINATION unittest)
|
||||
|
||||
################################################################################
|
||||
#<IVFHNSW-TEST>
|
||||
if (NOT TARGET test_ivf_hnsw)
|
||||
add_executable(test_ivf_hnsw test_ivf_hnsw.cpp ${faiss_srcs} ${util_srcs})
|
||||
endif ()
|
||||
target_link_libraries(test_ivf_hnsw ${depend_libs} ${unittest_libs} ${basic_libs})
|
||||
install(TARGETS test_ivf_hnsw DESTINATION unittest)
|
||||
|
||||
################################################################################
|
||||
#<IVFNM-TEST-CPU>
|
||||
if (NOT TARGET test_ivf_cpu_nm)
|
||||
add_executable(test_ivf_cpu_nm test_ivf_cpu_nm.cpp ${faiss_srcs} ${util_srcs})
|
||||
endif ()
|
||||
target_link_libraries(test_ivf_cpu_nm ${depend_libs} ${unittest_libs} ${basic_libs})
|
||||
install(TARGETS test_ivf_cpu_nm DESTINATION unittest)
|
||||
|
||||
################################################################################
|
||||
#<IVFNM-TEST-GPU>
|
||||
if (NOT TARGET test_ivf_gpu_nm)
|
||||
add_executable(test_ivf_gpu_nm test_ivf_gpu_nm.cpp ${faiss_srcs} ${util_srcs})
|
||||
endif ()
|
||||
target_link_libraries(test_ivf_gpu_nm ${depend_libs} ${unittest_libs} ${basic_libs})
|
||||
install(TARGETS test_ivf_gpu_nm DESTINATION unittest)
|
||||
|
||||
################################################################################
|
||||
#<BinaryIDMAP-TEST>
|
||||
if (NOT TARGET test_binaryidmap)
|
||||
add_executable(test_binaryidmap test_binaryidmap.cpp ${faiss_srcs} ${util_srcs})
|
||||
endif ()
|
||||
target_link_libraries(test_binaryidmap ${depend_libs} ${unittest_libs} ${basic_libs})
|
||||
install(TARGETS test_binaryidmap DESTINATION unittest)
|
||||
|
||||
################################################################################
|
||||
#<BinaryIVF-TEST>
|
||||
if (NOT TARGET test_binaryivf)
|
||||
add_executable(test_binaryivf test_binaryivf.cpp ${faiss_srcs} ${util_srcs})
|
||||
endif ()
|
||||
target_link_libraries(test_binaryivf ${depend_libs} ${unittest_libs} ${basic_libs})
|
||||
install(TARGETS test_binaryivf DESTINATION unittest)
|
||||
|
||||
|
||||
################################################################################
|
||||
#<NSG-TEST>
|
||||
add_definitions(-std=c++11 -O3 -march=native -Werror -DINFO)
|
||||
@ -193,140 +106,42 @@ else ()
|
||||
message(FATAL_ERROR "no OpenMP supprot")
|
||||
endif ()
|
||||
|
||||
include_directories(${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/impl/nsg)
|
||||
aux_source_directory(${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/impl/nsg nsg_src)
|
||||
set(interface_src
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_offset_index/IndexNSG_NM.cpp
|
||||
set(all_indexing_srcs
|
||||
${util_srcs}
|
||||
test_instructionset.cpp
|
||||
test_common.cpp
|
||||
test_customized_index.cpp
|
||||
test_idmap.cpp
|
||||
test_ivf.cpp
|
||||
test_ivf_hnsw.cpp
|
||||
test_ivf_cpu_nm.cpp
|
||||
test_binaryidmap.cpp
|
||||
test_binaryivf.cpp
|
||||
test_nsg.cpp
|
||||
test_hnsw.cpp
|
||||
test_rhnsw_flat.cpp
|
||||
test_rhnsw_pq.cpp
|
||||
test_rhnsw_sq8.cpp
|
||||
test_structured_index_sort.cpp
|
||||
test_structured_index_flat.cpp
|
||||
test_ngtpanng.cpp
|
||||
test_ngtonng.cpp
|
||||
)
|
||||
if (NOT TARGET test_nsg)
|
||||
add_executable(test_nsg test_nsg.cpp ${interface_src} ${nsg_src} ${util_srcs} ${faiss_srcs})
|
||||
endif ()
|
||||
target_link_libraries(test_nsg ${depend_libs} ${unittest_libs} ${basic_libs})
|
||||
install(TARGETS test_nsg DESTINATION unittest)
|
||||
|
||||
################################################################################
|
||||
#<HNSW-TEST>
|
||||
set(hnsw_srcs
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexHNSW.cpp
|
||||
)
|
||||
if (NOT TARGET test_hnsw)
|
||||
add_executable(test_hnsw test_hnsw.cpp ${hnsw_srcs} ${util_srcs})
|
||||
endif ()
|
||||
target_link_libraries(test_hnsw ${depend_libs} ${unittest_libs} ${basic_libs})
|
||||
install(TARGETS test_hnsw DESTINATION unittest)
|
||||
|
||||
################################################################################
|
||||
#<RHNSW_FLAT-TEST>
|
||||
set(rhnsw_flat_srcs
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexRHNSW.cpp
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexRHNSWFlat.cpp
|
||||
)
|
||||
if (NOT TARGET test_rhnsw_flat)
|
||||
add_executable(test_rhnsw_flat test_rhnsw_flat.cpp ${rhnsw_flat_srcs} ${util_srcs} ${faiss_srcs})
|
||||
endif ()
|
||||
target_link_libraries(test_rhnsw_flat ${depend_libs} ${unittest_libs} ${basic_libs})
|
||||
install(TARGETS test_rhnsw_flat DESTINATION unittest)
|
||||
|
||||
################################################################################
|
||||
#<RHNSW_PQ-TEST>
|
||||
set(rhnsw_pq_srcs
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexRHNSW.cpp
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexRHNSWPQ.cpp
|
||||
)
|
||||
if (NOT TARGET test_rhnsw_pq)
|
||||
add_executable(test_rhnsw_pq test_rhnsw_pq.cpp ${rhnsw_pq_srcs} ${util_srcs} ${faiss_srcs})
|
||||
endif ()
|
||||
target_link_libraries(test_rhnsw_pq ${depend_libs} ${unittest_libs} ${basic_libs})
|
||||
install(TARGETS test_rhnsw_pq DESTINATION unittest)
|
||||
|
||||
################################################################################
|
||||
#<RHNSW_SQ8-TEST>
|
||||
set(rhnsw_sq8_srcs
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexRHNSW.cpp
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexRHNSWSQ.cpp
|
||||
)
|
||||
if (NOT TARGET test_rhnsw_sq8)
|
||||
add_executable(test_rhnsw_sq8 test_rhnsw_sq8.cpp ${rhnsw_sq8_srcs} ${util_srcs} ${faiss_srcs})
|
||||
endif ()
|
||||
target_link_libraries(test_rhnsw_sq8 ${depend_libs} ${unittest_libs} ${basic_libs})
|
||||
install(TARGETS test_rhnsw_sq8 DESTINATION unittest)
|
||||
|
||||
################################################################################
|
||||
#<SPTAG-TEST>
|
||||
if (MILVUS_SUPPORT_SPTAG)
|
||||
set(sptag_srcs
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/adapter/SptagAdapter.cpp
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/helpers/SPTAGParameterMgr.cpp
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexSPTAG.cpp
|
||||
if (MILVUS_GPU_VERSION)
|
||||
set(all_indexing_srcs
|
||||
${all_indexing_srcs}
|
||||
test_ivf_gpu_nm.cpp
|
||||
test_gpuresource.cpp
|
||||
)
|
||||
if (NOT TARGET test_sptag)
|
||||
add_executable(test_sptag test_sptag.cpp ${sptag_srcs} ${util_srcs})
|
||||
endif ()
|
||||
target_link_libraries(test_sptag
|
||||
SPTAGLibStatic
|
||||
${depend_libs} ${unittest_libs} ${basic_libs})
|
||||
install(TARGETS test_sptag DESTINATION unittest)
|
||||
endif ()
|
||||
|
||||
################################################################################
|
||||
#<ANNOY-TEST>
|
||||
set(annoy_srcs
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexAnnoy.cpp
|
||||
)
|
||||
if (NOT TARGET test_annoy)
|
||||
add_executable(test_annoy test_annoy.cpp ${annoy_srcs} ${util_srcs})
|
||||
endif ()
|
||||
target_link_libraries(test_annoy ${depend_libs} ${unittest_libs} ${basic_libs})
|
||||
install(TARGETS test_annoy DESTINATION unittest)
|
||||
if (MILVUS_SUPPORT_SPTAG)
|
||||
set(all_indexing_srcs
|
||||
test_sptag.cpp
|
||||
)
|
||||
endif()
|
||||
|
||||
################################################################################
|
||||
#<STRUCTURED-INDEX-SORT-TEST>
|
||||
set(structured_index_sort_srcs
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/structured_index/StructuredIndexSort-inl.h
|
||||
)
|
||||
if (NOT TARGET test_structured_index_sort)
|
||||
add_executable(test_structured_index_sort test_structured_index_sort.cpp ${structured_index_sort_srcs} ${util_srcs})
|
||||
endif ()
|
||||
target_link_libraries(test_structured_index_sort ${depend_libs} ${unittest_libs} ${basic_libs})
|
||||
install(TARGETS test_structured_index_sort DESTINATION unittest)
|
||||
add_executable(test_knowhere ${all_indexing_srcs})
|
||||
target_link_libraries(test_knowhere knowhere ${depend_libs})
|
||||
|
||||
#add_subdirectory(faiss_benchmark)
|
||||
#add_subdirectory(metric_alg_benchmark)
|
||||
################################################################################
|
||||
|
||||
#<STRUCTURED-INDEX-FLAT-TEST>
|
||||
set(structured_index_flat_srcs
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/structured_index/StructuredIndexFlat-inl.h
|
||||
)
|
||||
if (NOT TARGET test_structured_index_flat)
|
||||
add_executable(test_structured_index_flat test_structured_index_flat.cpp ${structured_index_flat_srcs} ${util_srcs})
|
||||
endif ()
|
||||
target_link_libraries(test_structured_index_flat ${depend_libs} ${unittest_libs} ${basic_libs})
|
||||
install(TARGETS test_structured_index_flat DESTINATION unittest)
|
||||
|
||||
################################################################################
|
||||
|
||||
#<NGTPANNG-TEST>
|
||||
set(ngtpanng_srcs
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexNGTPANNG.cpp
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexNGT.cpp
|
||||
)
|
||||
if (NOT TARGET test_ngtpanng)
|
||||
add_executable(test_ngtpanng test_ngtpanng.cpp ${ngtpanng_srcs} ${util_srcs})
|
||||
endif ()
|
||||
target_link_libraries(test_ngtpanng ${depend_libs} ${unittest_libs} ${basic_libs})
|
||||
install(TARGETS test_ngtpanng DESTINATION unittest)
|
||||
|
||||
################################################################################
|
||||
#<NGTPANNG-TEST>
|
||||
set(ngtonng_srcs
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexNGTONNG.cpp
|
||||
${INDEX_SOURCE_DIR}/knowhere/knowhere/index/vector_index/IndexNGT.cpp
|
||||
)
|
||||
if (NOT TARGET test_ngtonng)
|
||||
add_executable(test_ngtonng test_ngtonng.cpp ${ngtonng_srcs} ${util_srcs})
|
||||
endif ()
|
||||
target_link_libraries(test_ngtonng ${depend_libs} ${unittest_libs} ${basic_libs})
|
||||
install(TARGETS test_ngtonng DESTINATION unittest)
|
||||
|
||||
################################################################################
|
||||
install(TARGETS test_knowhere DESTINATION unittest)
|
||||
|
||||
@ -8,6 +8,7 @@
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
@ -28,7 +29,7 @@
|
||||
#include "knowhere/index/vector_offset_index/gpu/IndexGPUIVF_NM.h"
|
||||
#endif
|
||||
|
||||
int DEVICEID = 0;
|
||||
constexpr int DEVICEID = 0;
|
||||
constexpr int64_t DIM = 128;
|
||||
constexpr int64_t NB = 10000;
|
||||
constexpr int64_t NQ = 10;
|
||||
@ -37,7 +38,7 @@ constexpr int64_t PINMEM = 1024 * 1024 * 200;
|
||||
constexpr int64_t TEMPMEM = 1024 * 1024 * 300;
|
||||
constexpr int64_t RESNUM = 2;
|
||||
|
||||
milvus::knowhere::IVFPtr
|
||||
inline milvus::knowhere::IVFPtr
|
||||
IndexFactory(const milvus::knowhere::IndexType& type, const milvus::knowhere::IndexMode mode) {
|
||||
if (mode == milvus::knowhere::IndexMode::MODE_CPU) {
|
||||
if (type == milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT) {
|
||||
@ -71,7 +72,7 @@ IndexFactory(const milvus::knowhere::IndexType& type, const milvus::knowhere::In
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
milvus::knowhere::IVFNMPtr
|
||||
inline milvus::knowhere::IVFNMPtr
|
||||
IndexFactoryNM(const milvus::knowhere::IndexType& type, const milvus::knowhere::IndexMode mode) {
|
||||
if (mode == milvus::knowhere::IndexMode::MODE_CPU) {
|
||||
if (type == milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT) {
|
||||
|
||||
@ -20,7 +20,7 @@
|
||||
|
||||
#include "unittest/utils.h"
|
||||
|
||||
void
|
||||
static void
|
||||
gen_rand_data(int range, int n, int*& p) {
|
||||
srand((unsigned int)time(nullptr));
|
||||
p = (int*)malloc(n * sizeof(int));
|
||||
@ -30,7 +30,7 @@ gen_rand_data(int range, int n, int*& p) {
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
static void
|
||||
gen_rand_int64_data(int64_t range, int64_t n, int64_t*& p) {
|
||||
srand((int64_t)time(nullptr));
|
||||
p = (int64_t*)malloc(n * sizeof(int64_t));
|
||||
@ -40,7 +40,7 @@ gen_rand_int64_data(int64_t range, int64_t n, int64_t*& p) {
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
static void
|
||||
gen_rand_double_data(double range, int64_t n, double*& p) {
|
||||
std::uniform_real_distribution<double> unif(0, range);
|
||||
std::default_random_engine re;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user