mirror of
https://gitee.com/milvus-io/milvus.git
synced 2024-12-03 20:39:36 +08:00
c4b37fb285
Issue: #31285 Basically, I've replaced `FixedVector<bool>` and `boost::dynamic_bitset` with custom bitset and bitsetview in order to reduce the memory bandwidth & increase performance for the filtering. This PR is for internal use only. Current progress (numbers are for GCC 9.5.0 on Ubuntu 22.04 LTS; clang-17 produces better performance numbers): Baseline: ``` [ RUN ] CApiTest.AssembeChunkPerfTest start test cost: 17903us [ OK ] CApiTest.AssembeChunkPerfTest (183 ms) [ RUN ] Expr.TestMultiLogicalExprsOptimization cost: 1391us cost: 5us cost: 4us cost: 4us cost: 6us cost: 4us cost: 4us cost: 4us cost: 4us cost: 4us 143 cost: 10us cost: 8us cost: 10us cost: 8us cost: 8us cost: 8us cost: 8us cost: 8us cost: 8us cost: 9us 8 /home/ubuntu/zilliz/milvus4/milvus/internal/core/unittest/test_expr.cpp:1561: Failure Expected: (cost_op) < (cost_no_op), actual: 143 vs 8 [ FAILED ] Expr.TestMultiLogicalExprsOptimization (7 ms) [ RUN ] Expr.TestExprs start test 3cost: 889us start test 10cost: 2us start test 20cost: 2us start test 30cost: 2us start test 50cost: 3us start test 100cost: 7us start test 200cost: 16us [ OK ] Expr.TestExprs (9 ms) [ RUN ] Expr.TestUnaryBenchTest start test type:2 cost: 124.8us start test type:3 cost: 163.1us start test type:4 cost: 275.9us start test type:5 cost: 590.9us start test type:10 cost: 62.7us start test type:11 cost: 65.9us [ OK ] Expr.TestUnaryBenchTest (1153 ms) [ RUN ] Expr.TestBinaryRangeBenchTest start test type:2 cost: 151.4us start test type:3 cost: 198.4us start test type:4 cost: 361.9us start test type:5 cost: 753.9us start test type:10 cost: 64.6us start test type:11 cost: 62.2us [ OK ] Expr.TestBinaryRangeBenchTest (1151 ms) [ RUN ] Expr.TestLogicalUnaryBenchTest start test type:2 cost: 121.14us start test type:3 cost: 156.84us start test type:4 cost: 249.76us start test type:5 cost: 534.44us start test type:10 cost: 82.2us start test type:11 cost: 83.52us [ OK ] Expr.TestLogicalUnaryBenchTest (1202 ms) [ RUN ] Expr.TestBinaryLogicalBenchTest start test type:2 cost: 80.64us start test type:3 cost: 78.22us start test type:4 cost: 255.76us start test type:5 cost: 532.04us start test type:10 cost: 89.26us start test type:11 cost: 90us [ OK ] Expr.TestBinaryLogicalBenchTest (1198 ms) [ RUN ] Expr.TestBinaryArithOpEvalRangeBenchExpr start test type:2 cost: 401.7us start test type:3 cost: 420.96us start test type:4 cost: 418.04us start test type:5 cost: 470.54us start test type:10 cost: 250.32us start test type:11 cost: 850.08us [ OK ] Expr.TestBinaryArithOpEvalRangeBenchExpr (1273 ms) [ RUN ] Expr.TestCompareExprBenchTest start test type:2 cost: 162us start test type:3 cost: 142us start test type:4 cost: 374us start test type:5 cost: 674us start test type:10 cost: 366us start test type:11 cost: 645us [ OK ] Expr.TestCompareExprBenchTest (1214 ms) [ RUN ] Expr.TestRefactorExprs start test 3cost: 1253us start test 10cost: 1060us start test 20cost: 681us start test 30cost: 522us start test 50cost: 511us start test 100cost: 506us start test 200cost: 497us [ OK ] Expr.TestRefactorExprs (1142 ms) ``` Candidate: ``` [ RUN ] CApiTest.AssembeChunkPerfTest start test cost: 6099us [ OK ] CApiTest.AssembeChunkPerfTest (153 ms) [ RUN ] Expr.TestMultiLogicalExprsOptimization cost: 42us cost: 15us cost: 15us cost: 14us cost: 15us cost: 15us cost: 15us cost: 15us cost: 15us cost: 15us 17 cost: 41us cost: 39us cost: 33us cost: 33us cost: 33us cost: 33us cost: 34us cost: 41us cost: 34us cost: 34us 35 [ OK ] Expr.TestMultiLogicalExprsOptimization (6 ms) [ RUN ] Expr.TestExprs start test 3cost: 20us start test 10cost: 2us start test 20cost: 2us start test 30cost: 2us start test 50cost: 4us start test 100cost: 8us start test 200cost: 15us [ OK ] Expr.TestExprs (8 ms) [ RUN ] Expr.TestUnaryBenchTest start test type:2 cost: 55.7us start test type:3 cost: 79.8us start test type:4 cost: 177.6us start test type:5 cost: 337.2us start test type:10 cost: 16.9us start test type:11 cost: 15.7us [ OK ] Expr.TestUnaryBenchTest (1140 ms) [ RUN ] Expr.TestBinaryRangeBenchTest start test type:2 cost: 57.1us start test type:3 cost: 87us start test type:4 cost: 177.5us start test type:5 cost: 342.7us start test type:10 cost: 17.9us start test type:11 cost: 16.7us [ OK ] Expr.TestBinaryRangeBenchTest (1152 ms) [ RUN ] Expr.TestLogicalUnaryBenchTest start test type:2 cost: 34.58us start test type:3 cost: 68.86us start test type:4 cost: 151.38us start test type:5 cost: 286.8us start test type:10 cost: 16.54us start test type:11 cost: 16.7us [ OK ] Expr.TestLogicalUnaryBenchTest (1165 ms) [ RUN ] Expr.TestBinaryLogicalBenchTest start test type:2 cost: 20us start test type:3 cost: 17.1us start test type:4 cost: 154.12us start test type:5 cost: 286.1us start test type:10 cost: 19.6us start test type:11 cost: 19.24us [ OK ] Expr.TestBinaryLogicalBenchTest (1188 ms) [ RUN ] Expr.TestBinaryArithOpEvalRangeBenchExpr start test type:2 cost: 125.7us start test type:3 cost: 111.34us start test type:4 cost: 148.02us start test type:5 cost: 306.7us start test type:10 cost: 149.3us start test type:11 cost: 282.94us [ OK ] Expr.TestBinaryArithOpEvalRangeBenchExpr (1221 ms) [ RUN ] Expr.TestCompareExprBenchTest start test type:2 cost: 89us start test type:3 cost: 79us start test type:4 cost: 323us start test type:5 cost: 629us start test type:10 cost: 313us start test type:11 cost: 591us [ OK ] Expr.TestCompareExprBenchTest (1228 ms) [ RUN ] Expr.TestRefactorExprs start test 3cost: 874us start test 10cost: 611us start test 20cost: 290us start test 30cost: 294us start test 50cost: 272us start test 100cost: 278us start test 200cost: 279us [ OK ] Expr.TestRefactorExprs (1149 ms) ``` Signed-off-by: Alexandr Guzhva <alexanderguzhva@gmail.com>
170 lines
5.0 KiB
CMake
170 lines
5.0 KiB
CMake
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
|
# with the License. You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
|
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
|
# or implied. See the License for the specific language governing permissions and limitations under the License
|
|
|
|
include_directories(${CMAKE_HOME_DIRECTORY}/src)
|
|
include_directories(${CMAKE_HOME_DIRECTORY}/src/thirdparty)
|
|
|
|
add_definitions(-DMILVUS_TEST_SEGCORE_YAML_PATH="${CMAKE_SOURCE_DIR}/unittest/test_utils/test_segcore.yaml")
|
|
|
|
# TODO: better to use ls/find pattern
|
|
set(MILVUS_TEST_FILES
|
|
init_gtest.cpp
|
|
test_bf.cpp
|
|
test_bf_sparse.cpp
|
|
test_binary.cpp
|
|
test_bitmap.cpp
|
|
test_bool_index.cpp
|
|
test_common.cpp
|
|
test_concurrent_vector.cpp
|
|
test_c_api.cpp
|
|
test_expr_materialized_view.cpp
|
|
test_expr.cpp
|
|
test_float16.cpp
|
|
test_growing.cpp
|
|
test_growing_index.cpp
|
|
test_indexing.cpp
|
|
test_index_c_api.cpp
|
|
test_index_wrapper.cpp
|
|
test_init.cpp
|
|
test_parquet_c.cpp
|
|
test_query.cpp
|
|
test_reduce.cpp
|
|
test_reduce_c.cpp
|
|
test_relational.cpp
|
|
test_retrieve.cpp
|
|
test_scalar_index.cpp
|
|
test_sealed.cpp
|
|
test_segcore.cpp
|
|
test_similarity_corelation.cpp
|
|
test_span.cpp
|
|
test_string_expr.cpp
|
|
test_timestamp_index.cpp
|
|
test_utils.cpp
|
|
test_data_codec.cpp
|
|
test_range_search_sort.cpp
|
|
test_tracer.cpp
|
|
test_local_chunk_manager.cpp
|
|
test_disk_file_manager_test.cpp
|
|
test_integer_overflow.cpp
|
|
test_offset_ordered_map.cpp
|
|
test_offset_ordered_array.cpp
|
|
test_always_true_expr.cpp
|
|
test_plan_proto.cpp
|
|
test_chunk_cache.cpp
|
|
test_binlog_index.cpp
|
|
test_storage.cpp
|
|
test_exec.cpp
|
|
test_inverted_index.cpp
|
|
test_group_by.cpp
|
|
test_regex_query_util.cpp
|
|
test_regex_query.cpp
|
|
)
|
|
|
|
if ( BUILD_DISK_ANN STREQUAL "ON" )
|
|
set(MILVUS_TEST_FILES
|
|
${MILVUS_TEST_FILES}
|
|
#need update aws-sdk-cpp, see more from https://github.com/aws/aws-sdk-cpp/issues/1757
|
|
#test_minio_chunk_manager.cpp
|
|
)
|
|
endif()
|
|
|
|
if (LINUX OR APPLE)
|
|
set(MILVUS_TEST_FILES
|
|
${MILVUS_TEST_FILES}
|
|
test_scalar_index_creator.cpp
|
|
test_string_index.cpp
|
|
test_array.cpp test_array_expr.cpp)
|
|
endif()
|
|
|
|
if (DEFINED AZURE_BUILD_DIR)
|
|
set(MILVUS_TEST_FILES
|
|
${MILVUS_TEST_FILES}
|
|
test_azure_chunk_manager.cpp
|
|
#need update aws-sdk-cpp, see more from https://github.com/aws/aws-sdk-cpp/issues/2119
|
|
#test_remote_chunk_manager.cpp
|
|
)
|
|
include_directories("${AZURE_BUILD_DIR}/vcpkg_installed/${VCPKG_TARGET_TRIPLET}/include")
|
|
endif()
|
|
|
|
if (LINUX)
|
|
message( STATUS "Building Milvus Unit Test on Linux")
|
|
option(USE_ASAN "Whether to use AddressSanitizer" OFF)
|
|
if ( USE_ASAN AND false )
|
|
message( STATUS "Building Milvus using AddressSanitizer")
|
|
add_compile_options(-fno-stack-protector -fno-omit-frame-pointer -fno-var-tracking -fsanitize=address)
|
|
add_link_options(-fno-stack-protector -fno-omit-frame-pointer -fno-var-tracking -fsanitize=address)
|
|
endif()
|
|
|
|
# check if memory leak exists in index builder
|
|
set(INDEX_BUILDER_TEST_FILES
|
|
test_index_wrapper.cpp
|
|
test_scalar_index_creator.cpp
|
|
test_index_c_api.cpp
|
|
)
|
|
|
|
add_executable(index_builder_test
|
|
${INDEX_BUILDER_TEST_FILES}
|
|
)
|
|
|
|
target_link_libraries(index_builder_test
|
|
gtest
|
|
gtest_main
|
|
milvus_segcore
|
|
milvus_storage
|
|
milvus_indexbuilder
|
|
milvus_common
|
|
)
|
|
install(TARGETS index_builder_test DESTINATION unittest)
|
|
endif()
|
|
|
|
add_executable(all_tests
|
|
${MILVUS_TEST_FILES}
|
|
)
|
|
|
|
target_link_libraries(all_tests
|
|
gtest
|
|
milvus_segcore
|
|
milvus_storage
|
|
milvus_indexbuilder
|
|
pthread
|
|
milvus_common
|
|
milvus_exec
|
|
)
|
|
|
|
install(TARGETS all_tests DESTINATION unittest)
|
|
|
|
if (LINUX)
|
|
add_subdirectory(bench)
|
|
endif ()
|
|
|
|
# if (USE_DYNAMIC_SIMD)
|
|
# add_executable(dynamic_simd_test
|
|
# test_simd.cpp)
|
|
#
|
|
# target_link_libraries(dynamic_simd_test
|
|
# milvus_simd
|
|
# milvus_log
|
|
# gtest
|
|
# ${CONAN_LIBS})
|
|
#
|
|
# install(TARGETS dynamic_simd_test DESTINATION unittest)
|
|
# endif()
|
|
|
|
add_executable(bitset_test
|
|
test_bitset.cpp
|
|
)
|
|
target_link_libraries(bitset_test
|
|
milvus_bitset
|
|
gtest
|
|
${CONAN_LIBS}
|
|
)
|
|
install(TARGETS bitset_test DESTINATION unittest)
|