From a773836b8982cba13eb5aaf385664e7d7365030c Mon Sep 17 00:00:00 2001 From: Zhen Ye Date: Fri, 23 Aug 2024 12:35:02 +0800 Subject: [PATCH] enhance: optimize milvus core building (#35610) issue: #35549,#35611,#35633 - remove milvus_segcore milvus_indexbuilder..., add libmilvus_core - core building only link once - move opendal compilation into cmake - fix odr --------- Signed-off-by: chyezh --- build/rpm/milvus.spec | 8 +- internal/core/CMakeLists.txt | 1 - internal/core/cmake/Utils.cmake | 12 ++- internal/core/src/CMakeLists.txt | 50 +++++++++++++ internal/core/src/bitset/CMakeLists.txt | 4 +- internal/core/src/clustering/CMakeLists.txt | 15 +--- .../src/clustering/milvus_clustering.pc.in | 9 --- internal/core/src/common/CMakeLists.txt | 37 +-------- internal/core/src/common/ChunkFileWriter.cpp | 35 --------- internal/core/src/common/ChunkFileWriter.h | 44 ----------- internal/core/src/common/milvus_common.pc.in | 9 --- internal/core/src/config/CMakeLists.txt | 13 +--- internal/core/src/exec/CMakeLists.txt | 24 +----- internal/core/src/futures/CMakeLists.txt | 15 +--- .../core/src/futures/milvus_futures.pc.in | 9 --- internal/core/src/index/CMakeLists.txt | 22 +----- internal/core/src/indexbuilder/CMakeLists.txt | 17 +---- .../indexbuilder/milvus_indexbuilder.pc.in | 9 --- internal/core/src/log/CMakeLists.txt | 10 +-- .../milvus_index.pc.in => milvus_core.pc.in} | 6 +- internal/core/src/monitor/CMakeLists.txt | 14 +--- .../core/src/monitor/milvus_monitor.pc.in | 8 -- internal/core/src/pb/CMakeLists.txt | 10 +-- internal/core/src/query/CMakeLists.txt | 25 +------ internal/core/src/segcore/CMakeLists.txt | 38 +--------- .../core/src/segcore/milvus_segcore.pc.in | 9 --- internal/core/src/storage/CMakeLists.txt | 75 ++++--------------- internal/core/src/storage/Util.cpp | 4 +- .../storage/azure-blob-storage/CMakeLists.txt | 4 +- .../storage/{ => azure}/AzureChunkManager.cpp | 2 +- .../storage/{ => azure}/AzureChunkManager.h | 0 .../core/src/storage/milvus_storage.pc.in | 10 --- .../{ => opendal}/OpenDALChunkManager.cpp | 2 +- .../{ => opendal}/OpenDALChunkManager.h | 0 .../core/thirdparty/knowhere/CMakeLists.txt | 2 + .../core/thirdparty/opendal/CMakeLists.txt | 42 +++++++++-- .../core/thirdparty/simdjson/CMakeLists.txt | 2 + .../core/thirdparty/tantivy/CMakeLists.txt | 2 + internal/core/unittest/CMakeLists.txt | 24 +++--- internal/core/unittest/bench/CMakeLists.txt | 11 +-- .../unittest/test_azure_chunk_manager.cpp | 2 +- internal/indexnode/indexnode.go | 2 +- internal/proxy/cgo_util.go | 2 +- internal/querynodev2/segments/cgo_util.go | 2 +- internal/querynodev2/segments/collection.go | 2 +- .../querynodev2/segments/index_attr_cache.go | 2 +- .../segments/load_field_data_info.go | 2 +- .../querynodev2/segments/load_index_info.go | 2 +- internal/querynodev2/segments/manager.go | 2 +- internal/querynodev2/segments/plan.go | 2 +- internal/querynodev2/segments/reduce.go | 2 +- internal/querynodev2/segments/segment.go | 2 +- .../querynodev2/segments/segment_loader.go | 2 +- internal/querynodev2/segments/trace.go | 2 +- internal/querynodev2/segments/utils.go | 2 +- internal/querynodev2/server.go | 2 +- internal/util/analyzecgowrapper/analyze.go | 2 +- internal/util/analyzecgowrapper/helper.go | 2 +- internal/util/cgo/errors.go | 2 +- internal/util/cgo/executor.go | 2 +- internal/util/cgo/futures.go | 2 +- internal/util/cgo/futures_test_case.go | 2 +- .../util/indexcgowrapper/build_index_info.go | 2 +- internal/util/indexcgowrapper/helper.go | 2 +- internal/util/indexcgowrapper/index.go | 2 +- internal/util/initcore/init_core.go | 2 +- internal/util/metrics/c_registry.go | 2 +- scripts/3rdparty_build.sh | 12 --- 68 files changed, 198 insertions(+), 501 deletions(-) delete mode 100644 internal/core/src/clustering/milvus_clustering.pc.in delete mode 100644 internal/core/src/common/ChunkFileWriter.cpp delete mode 100644 internal/core/src/common/ChunkFileWriter.h delete mode 100644 internal/core/src/common/milvus_common.pc.in delete mode 100644 internal/core/src/futures/milvus_futures.pc.in delete mode 100644 internal/core/src/indexbuilder/milvus_indexbuilder.pc.in rename internal/core/src/{index/milvus_index.pc.in => milvus_core.pc.in} (59%) delete mode 100644 internal/core/src/monitor/milvus_monitor.pc.in delete mode 100644 internal/core/src/segcore/milvus_segcore.pc.in rename internal/core/src/storage/{ => azure}/AzureChunkManager.cpp (99%) rename internal/core/src/storage/{ => azure}/AzureChunkManager.h (100%) delete mode 100644 internal/core/src/storage/milvus_storage.pc.in rename internal/core/src/storage/{ => opendal}/OpenDALChunkManager.cpp (99%) rename internal/core/src/storage/{ => opendal}/OpenDALChunkManager.h (100%) diff --git a/build/rpm/milvus.spec b/build/rpm/milvus.spec index eb5b9cd1f1..7643a283fe 100644 --- a/build/rpm/milvus.spec +++ b/build/rpm/milvus.spec @@ -38,9 +38,7 @@ install -m 755 bin/minio %{buildroot}/usr/bin/milvus-minio # lib install -m 755 lib/libknowhere.so %{buildroot}/lib64/milvus/libknowhere.so -install -m 755 lib/libmilvus_common.so %{buildroot}/lib64/milvus/libmilvus_common.so -install -m 755 lib/libmilvus_indexbuilder.so %{buildroot}/lib64/milvus/libmilvus_indexbuilder.so -install -m 755 lib/libmilvus_segcore.so %{buildroot}/lib64/milvus/libmilvus_segcore.so +install -m 755 lib/libmivlus_core.so %{buildroot}/lib64/milvus/libmilvus_core.so install -m 755 /usr/lib/libopenblas-r0.3.9.so %{buildroot}/lib64/milvus/libopenblas.so.0 install -m 755 lib/libngt.so.1.12.0 %{buildroot}/lib64/milvus/libngt.so.1 install -m 755 /usr/lib64/libgfortran.so.4.0.0 %{buildroot}/lib64/milvus/libgfortran.so.4 @@ -82,9 +80,7 @@ systemctl daemon-reload /usr/bin/milvus-minio /lib64/milvus/libknowhere.so -/lib64/milvus/libmilvus_common.so -/lib64/milvus/libmilvus_indexbuilder.so -/lib64/milvus/libmilvus_segcore.so +/lib64/milvus/libmilvus_core.so /lib64/milvus/libopenblas.so.0 /lib64/milvus/libngt.so.1 /lib64/milvus/libgfortran.so.4 diff --git a/internal/core/CMakeLists.txt b/internal/core/CMakeLists.txt index fd43ede085..ccc4111b01 100644 --- a/internal/core/CMakeLists.txt +++ b/internal/core/CMakeLists.txt @@ -319,4 +319,3 @@ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/futures/ install(DIRECTORY ${CMAKE_BINARY_DIR}/lib/ DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR} ) - diff --git a/internal/core/cmake/Utils.cmake b/internal/core/cmake/Utils.cmake index f057a48457..8facf7cccd 100644 --- a/internal/core/cmake/Utils.cmake +++ b/internal/core/cmake/Utils.cmake @@ -99,4 +99,14 @@ macro(create_library) add_library(${L_TARGET} ${L_SRCS}) target_link_libraries(${L_TARGET} PRIVATE ${L_LIBS}) target_compile_definitions(${L_TARGET} PRIVATE ${L_DEFS}) -endmacro() \ No newline at end of file +endmacro() + +macro(add_source_at_current_directory_recursively) + file(GLOB_RECURSE SOURCE_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc" "*.cpp" "*.c" "*.cxx") + message(STATUS "${CMAKE_CURRENT_SOURCE_DIR} add new source files at current directory recursively: ${SOURCE_FILES}") +endmacro() + +macro(add_source_at_current_directory) + file(GLOB SOURCE_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc" "*.cpp" "*.c" "*.cxx") + message(STATUS "${CMAKE_CURRENT_SOURCE_DIR} add new source files at current directory: ${SOURCE_FILES}") +endmacro() diff --git a/internal/core/src/CMakeLists.txt b/internal/core/src/CMakeLists.txt index 7cc3961fad..e53cb09c50 100644 --- a/internal/core/src/CMakeLists.txt +++ b/internal/core/src/CMakeLists.txt @@ -20,8 +20,19 @@ else() project(core CXX C) endif() +option( EMBEDDED_MILVUS "Enable embedded Milvus" OFF ) +if ( EMBEDDED_MILVUS ) + add_compile_definitions( EMBEDDED_MILVUS ) +endif() + include_directories(${MILVUS_ENGINE_SRC}) include_directories(${MILVUS_THIRDPARTY_SRC}) +include_directories( + ${KNOWHERE_INCLUDE_DIR} + ${SIMDJSON_INCLUDE_DIR} + ${TANTIVY_INCLUDE_DIR} + ${CONAN_INCLUDE_DIRS} +) add_subdirectory( pb ) add_subdirectory( log ) @@ -37,3 +48,42 @@ add_subdirectory( clustering ) add_subdirectory( exec ) add_subdirectory( bitset ) add_subdirectory( futures ) + +milvus_add_pkg_config("milvus_core") + +add_library(milvus_core SHARED + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ +) + +set(LINK_TARGETS + boost_bitset_ext + simdjson + tantivy_binding + knowhere + ${OpenMP_CXX_FLAGS} + ${CONAN_LIBS}) + +if(USE_OPENDAL) + set(LINK_TARGETS ${LINK_TARGETS} opendal) +endif() + +if(DEFINED AZURE_BUILD_DIR) + set(LINK_TARGETS ${LINK_TARGETS} azure_blob_chunk_manager) +endif() + +target_link_libraries(milvus_core ${LINK_TARGETS}) + +install(TARGETS milvus_core DESTINATION "${CMAKE_INSTALL_LIBDIR}") diff --git a/internal/core/src/bitset/CMakeLists.txt b/internal/core/src/bitset/CMakeLists.txt index 8b2137ca25..3f7c6ae24d 100644 --- a/internal/core/src/bitset/CMakeLists.txt +++ b/internal/core/src/bitset/CMakeLists.txt @@ -13,6 +13,8 @@ set(BITSET_SRCS detail/platform/dynamic.cpp ) + + if (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86_64") list(APPEND BITSET_SRCS detail/platform/x86/avx2-inst.cpp @@ -38,4 +40,4 @@ elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm*") #set_source_files_properties(detail/platform/arm/sve-inst.cpp PROPERTIES COMPILE_FLAGS "-mcpu=neoverse-v1") endif() -add_library(milvus_bitset ${BITSET_SRCS}) +add_library(milvus_bitset OBJECT ${BITSET_SRCS}) diff --git a/internal/core/src/clustering/CMakeLists.txt b/internal/core/src/clustering/CMakeLists.txt index 40833d9ef2..7331281499 100644 --- a/internal/core/src/clustering/CMakeLists.txt +++ b/internal/core/src/clustering/CMakeLists.txt @@ -9,16 +9,5 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing permissions and limitations under the License - -set(CLUSTERING_FILES - analyze_c.cpp - KmeansClustering.cpp - ) - -milvus_add_pkg_config("milvus_clustering") -add_library(milvus_clustering SHARED ${CLUSTERING_FILES}) - -# link order matters -target_link_libraries(milvus_clustering milvus_index) - -install(TARGETS milvus_clustering DESTINATION "${CMAKE_INSTALL_LIBDIR}") +add_source_at_current_directory_recursively() +add_library(milvus_clustering OBJECT ${SOURCE_FILES}) diff --git a/internal/core/src/clustering/milvus_clustering.pc.in b/internal/core/src/clustering/milvus_clustering.pc.in deleted file mode 100644 index d1bbb3d3ba..0000000000 --- a/internal/core/src/clustering/milvus_clustering.pc.in +++ /dev/null @@ -1,9 +0,0 @@ -libdir=@CMAKE_INSTALL_FULL_LIBDIR@ -includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ - -Name: Milvus Clustering -Description: Clustering modules for Milvus -Version: @MILVUS_VERSION@ - -Libs: -L${libdir} -lmilvus_clustering -Cflags: -I${includedir} diff --git a/internal/core/src/common/CMakeLists.txt b/internal/core/src/common/CMakeLists.txt index 53eb53787a..e1239c4d49 100644 --- a/internal/core/src/common/CMakeLists.txt +++ b/internal/core/src/common/CMakeLists.txt @@ -9,38 +9,5 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing permissions and limitations under the License -milvus_add_pkg_config("milvus_common") - -set(COMMON_SRC - Schema.cpp - SystemProperty.cpp - Slice.cpp - binary_set_c.cpp - init_c.cpp - Common.cpp - RangeSearchHelper.cpp - Tracer.cpp - IndexMeta.cpp - EasyAssert.cpp - FieldData.cpp - RegexQuery.cpp - ChunkTarget.cpp - Chunk.cpp - ChunkWriter.cpp - ) - -add_library(milvus_common SHARED ${COMMON_SRC}) - -target_link_libraries(milvus_common - milvus_bitset - milvus_config - milvus_log - milvus_proto - yaml-cpp - boost_bitset_ext - simdjson - ${CONAN_LIBS} - re2 - ) - -install(TARGETS milvus_common DESTINATION "${CMAKE_INSTALL_LIBDIR}") +add_source_at_current_directory_recursively() +add_library(milvus_common OBJECT ${SOURCE_FILES}) diff --git a/internal/core/src/common/ChunkFileWriter.cpp b/internal/core/src/common/ChunkFileWriter.cpp deleted file mode 100644 index 29eda3e525..0000000000 --- a/internal/core/src/common/ChunkFileWriter.cpp +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License - -#include "common/ChunkFileWriter.h" -#include "common/ChunkWriter.h" -namespace milvus { -ChunkFileWriter::ChunkFileWriter(std::string& file_path) - : file_(File::Open(file_path, O_RDWR | O_CREAT | O_TRUNC)) { -} - -void -ChunkFileWriter::write_chunk(std::shared_ptr r) { - // FIXME - size_t file_offset = file_.Seek(0, SEEK_END); - auto chunk = create_chunk(field_meta_, dim_, file_, file_offset, r); - // TODO: stat_writer_.write(chunk); - rep_.chunks.push_back(*chunk); -} - -FileRep -ChunkFileWriter::finish() { - // TODO: stat_writer_.finish(); - // rep_.stat_chunk = stat_writer_.get(); - return rep_; -} - -} // namespace milvus \ No newline at end of file diff --git a/internal/core/src/common/ChunkFileWriter.h b/internal/core/src/common/ChunkFileWriter.h deleted file mode 100644 index 028fb32e20..0000000000 --- a/internal/core/src/common/ChunkFileWriter.h +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License - -#pragma once -#include -#include "arrow/record_batch.h" -#include "arrow/table_builder.h" -#include "common/Chunk.h" -#include "common/ChunkTarget.h" -#include "common/FieldMeta.h" -namespace milvus { -class StatisticsChunkWriter; -class StatisticsChunk; - -class ChunkFileWriter { - public: - ChunkFileWriter() = default; - ChunkFileWriter(std::string& file_path); - struct FileRep { - std::vector chunks; - }; - void - write_chunk(std::shared_ptr r); - - FileRep - finish(); - - private: - FieldMeta& field_meta_; - int dim_; - StatisticsChunkWriter stat_writer_; - File file_; - FileRep rep_; -}; - -} // namespace milvus \ No newline at end of file diff --git a/internal/core/src/common/milvus_common.pc.in b/internal/core/src/common/milvus_common.pc.in deleted file mode 100644 index d1ae3f504f..0000000000 --- a/internal/core/src/common/milvus_common.pc.in +++ /dev/null @@ -1,9 +0,0 @@ -libdir=@CMAKE_INSTALL_FULL_LIBDIR@ -includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ - -Name: Milvus Common -Description: Common modules for Milvus -Version: @MILVUS_VERSION@ - -Libs: -L${libdir} -lmilvus_common -Cflags: -I${includedir} diff --git a/internal/core/src/config/CMakeLists.txt b/internal/core/src/config/CMakeLists.txt index 36f3ccc6a7..0fed02c95d 100644 --- a/internal/core/src/config/CMakeLists.txt +++ b/internal/core/src/config/CMakeLists.txt @@ -14,14 +14,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -option( EMBEDDED_MILVUS "Enable embedded Milvus" OFF ) - -if ( EMBEDDED_MILVUS ) - add_compile_definitions( EMBEDDED_MILVUS ) -endif() - -set(CONFIG_SRC ConfigKnowhere.cpp) - -add_library(milvus_config STATIC ${CONFIG_SRC}) - -target_link_libraries(milvus_config knowhere) +add_source_at_current_directory_recursively() +add_library(milvus_config OBJECT ${SOURCE_FILES}) \ No newline at end of file diff --git a/internal/core/src/exec/CMakeLists.txt b/internal/core/src/exec/CMakeLists.txt index 8e134f5128..ec4ca32314 100644 --- a/internal/core/src/exec/CMakeLists.txt +++ b/internal/core/src/exec/CMakeLists.txt @@ -9,25 +9,5 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing permissions and limitations under the License -set(MILVUS_EXEC_SRCS - expression/Expr.cpp - expression/UnaryExpr.cpp - expression/ConjunctExpr.cpp - expression/LogicalUnaryExpr.cpp - expression/LogicalBinaryExpr.cpp - expression/TermExpr.cpp - expression/BinaryArithOpEvalRangeExpr.cpp - expression/BinaryRangeExpr.cpp - expression/AlwaysTrueExpr.cpp - expression/CompareExpr.cpp - expression/JsonContainsExpr.cpp - expression/ExistsExpr.cpp - operator/FilterBits.cpp - operator/Operator.cpp - Driver.cpp - Task.cpp - ) - -add_library(milvus_exec STATIC ${MILVUS_EXEC_SRCS}) - -target_link_libraries(milvus_exec milvus_common ${CONAN_LIBS}) +add_source_at_current_directory_recursively() +add_library(milvus_exec OBJECT ${SOURCE_FILES}) \ No newline at end of file diff --git a/internal/core/src/futures/CMakeLists.txt b/internal/core/src/futures/CMakeLists.txt index 59d4bdd9f2..9b948ac70e 100644 --- a/internal/core/src/futures/CMakeLists.txt +++ b/internal/core/src/futures/CMakeLists.txt @@ -9,16 +9,5 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing permissions and limitations under the License -milvus_add_pkg_config("milvus_futures") - -set(FUTURES_SRC - Executor.cpp - future_c.cpp - future_test_case_c.cpp - ) - -add_library(milvus_futures SHARED ${FUTURES_SRC}) - -target_link_libraries(milvus_futures milvus_common) - -install(TARGETS milvus_futures DESTINATION "${CMAKE_INSTALL_LIBDIR}") +add_source_at_current_directory_recursively() +add_library(milvus_futures OBJECT ${SOURCE_FILES}) \ No newline at end of file diff --git a/internal/core/src/futures/milvus_futures.pc.in b/internal/core/src/futures/milvus_futures.pc.in deleted file mode 100644 index dc75e325e8..0000000000 --- a/internal/core/src/futures/milvus_futures.pc.in +++ /dev/null @@ -1,9 +0,0 @@ -libdir=@CMAKE_INSTALL_FULL_LIBDIR@ -includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ - -Name: Milvus Futures -Description: Futures modules for Milvus -Version: @MILVUS_VERSION@ - -Libs: -L${libdir} -lmilvus_futures -Cflags: -I${includedir} diff --git a/internal/core/src/index/CMakeLists.txt b/internal/core/src/index/CMakeLists.txt index c4f1012e51..688711808b 100644 --- a/internal/core/src/index/CMakeLists.txt +++ b/internal/core/src/index/CMakeLists.txt @@ -9,23 +9,5 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing permissions and limitations under the License -set(INDEX_FILES - StringIndexMarisa.cpp - Utils.cpp - VectorMemIndex.cpp - IndexFactory.cpp - VectorDiskIndex.cpp - ScalarIndex.cpp - ScalarIndexSort.cpp - SkipIndex.cpp - InvertedIndexTantivy.cpp - BitmapIndex.cpp - HybridScalarIndex.cpp - ) - -milvus_add_pkg_config("milvus_index") -add_library(milvus_index SHARED ${INDEX_FILES}) - -target_link_libraries(milvus_index milvus_storage tantivy_binding) - -install(TARGETS milvus_index DESTINATION "${CMAKE_INSTALL_LIBDIR}") +add_source_at_current_directory_recursively() +add_library(milvus_index OBJECT ${SOURCE_FILES}) \ No newline at end of file diff --git a/internal/core/src/indexbuilder/CMakeLists.txt b/internal/core/src/indexbuilder/CMakeLists.txt index a6fdea6456..da97de8037 100644 --- a/internal/core/src/indexbuilder/CMakeLists.txt +++ b/internal/core/src/indexbuilder/CMakeLists.txt @@ -9,18 +9,5 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing permissions and limitations under the License - -set(INDEXBUILDER_FILES - VecIndexCreator.cpp - index_c.cpp - init_c.cpp - ScalarIndexCreator.cpp - ) - -milvus_add_pkg_config("milvus_indexbuilder") -add_library(milvus_indexbuilder SHARED ${INDEXBUILDER_FILES}) - -# link order matters -target_link_libraries(milvus_indexbuilder milvus_index) - -install(TARGETS milvus_indexbuilder DESTINATION "${CMAKE_INSTALL_LIBDIR}") +add_source_at_current_directory_recursively() +add_library(milvus_indexbuilder OBJECT ${SOURCE_FILES}) \ No newline at end of file diff --git a/internal/core/src/indexbuilder/milvus_indexbuilder.pc.in b/internal/core/src/indexbuilder/milvus_indexbuilder.pc.in deleted file mode 100644 index f155bf1ca8..0000000000 --- a/internal/core/src/indexbuilder/milvus_indexbuilder.pc.in +++ /dev/null @@ -1,9 +0,0 @@ -libdir=@CMAKE_INSTALL_FULL_LIBDIR@ -includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ - -Name: Milvus IndexBuilder -Description: IndexBuilder modules for Milvus -Version: @MILVUS_VERSION@ - -Libs: -L${libdir} -lmilvus_indexbuilder -Cflags: -I${includedir} diff --git a/internal/core/src/log/CMakeLists.txt b/internal/core/src/log/CMakeLists.txt index 4332e43e97..ac62a32e82 100644 --- a/internal/core/src/log/CMakeLists.txt +++ b/internal/core/src/log/CMakeLists.txt @@ -10,12 +10,6 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing permissions and limitations under the License. #------------------------------------------------------------------------------- -set(LOG_FILES ${MILVUS_ENGINE_SRC}/log/Log.cpp - ${MILVUS_ENGINE_SRC}/log/Log.h - #${MILVUS_THIRDPARTY_SRC}/easyloggingpp/easylogging++.cc - #${MILVUS_THIRDPARTY_SRC}/easyloggingpp/easylogging++.h - ) -add_library(milvus_log STATIC ${LOG_FILES}) -set_target_properties(milvus_log PROPERTIES RULE_LAUNCH_COMPILE "") -set_target_properties(milvus_log PROPERTIES RULE_LAUNCH_LINK "") +add_source_at_current_directory_recursively() +add_library(milvus_log OBJECT ${SOURCE_FILES}) \ No newline at end of file diff --git a/internal/core/src/index/milvus_index.pc.in b/internal/core/src/milvus_core.pc.in similarity index 59% rename from internal/core/src/index/milvus_index.pc.in rename to internal/core/src/milvus_core.pc.in index ffc4559bdf..eac5fd6c11 100644 --- a/internal/core/src/index/milvus_index.pc.in +++ b/internal/core/src/milvus_core.pc.in @@ -1,9 +1,9 @@ libdir=@CMAKE_INSTALL_FULL_LIBDIR@ includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ -Name: Milvus index -Description: index modules for Milvus +Name: Milvus Core +Description: Core modules for Milvus Version: @MILVUS_VERSION@ -Libs: -L${libdir} -lmilvus_index +Libs: -L${libdir} -lmilvus_core Cflags: -I${includedir} diff --git a/internal/core/src/monitor/CMakeLists.txt b/internal/core/src/monitor/CMakeLists.txt index 435dad13d9..01a8d6d9ea 100644 --- a/internal/core/src/monitor/CMakeLists.txt +++ b/internal/core/src/monitor/CMakeLists.txt @@ -9,15 +9,5 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing permissions and limitations under the License -milvus_add_pkg_config("milvus_monitor") - -set(MONITOR_SRC - monitor_c.cpp - prometheus_client.cpp - ) - -add_library(milvus_monitor SHARED ${MONITOR_SRC}) - -target_link_libraries(milvus_monitor PUBLIC ${CONAN_LIBS}) - -install(TARGETS milvus_monitor DESTINATION "${CMAKE_INSTALL_LIBDIR}") \ No newline at end of file +add_source_at_current_directory_recursively() +add_library(milvus_monitor OBJECT ${SOURCE_FILES}) \ No newline at end of file diff --git a/internal/core/src/monitor/milvus_monitor.pc.in b/internal/core/src/monitor/milvus_monitor.pc.in deleted file mode 100644 index 64d1b137b0..0000000000 --- a/internal/core/src/monitor/milvus_monitor.pc.in +++ /dev/null @@ -1,8 +0,0 @@ -libdir=@CMAKE_INSTALL_FULL_LIBDIR@ -includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ - -Name: Milvus Monitor -Description: Monitor modules for Milvus -Version: @MILVUS_VERSION@ - -Libs: -L${libdir} -lmilvus_monitor diff --git a/internal/core/src/pb/CMakeLists.txt b/internal/core/src/pb/CMakeLists.txt index d49637702d..899eaa2f18 100644 --- a/internal/core/src/pb/CMakeLists.txt +++ b/internal/core/src/pb/CMakeLists.txt @@ -11,11 +11,5 @@ find_package(Protobuf REQUIRED) -file(GLOB_RECURSE milvus_proto_srcs - "${CMAKE_CURRENT_SOURCE_DIR}/*.cc") -add_library(milvus_proto STATIC - ${milvus_proto_srcs} -) -message(STATUS "milvus proto sources: " ${milvus_proto_srcs}) - -target_link_libraries( milvus_proto PUBLIC ${CONAN_LIBS} ) +add_source_at_current_directory_recursively() +add_library(milvus_pb OBJECT ${SOURCE_FILES}) diff --git a/internal/core/src/query/CMakeLists.txt b/internal/core/src/query/CMakeLists.txt index 6bbc488c3b..57ceea8423 100644 --- a/internal/core/src/query/CMakeLists.txt +++ b/internal/core/src/query/CMakeLists.txt @@ -9,26 +9,5 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing permissions and limitations under the License -set(MILVUS_QUERY_SRCS - generated/PlanNode.cpp - generated/Expr.cpp - visitors/ShowPlanNodeVisitor.cpp - visitors/ShowExprVisitor.cpp - visitors/ExecPlanNodeVisitor.cpp - visitors/ExecExprVisitor.cpp - visitors/VerifyPlanNodeVisitor.cpp - visitors/VerifyExprVisitor.cpp - visitors/ExtractInfoPlanNodeVisitor.cpp - visitors/ExtractInfoExprVisitor.cpp - Plan.cpp - SearchOnGrowing.cpp - SearchOnSealed.cpp - SearchOnIndex.cpp - SearchBruteForce.cpp - SubSearchResult.cpp - groupby/SearchGroupByOperator.cpp - PlanProto.cpp - ) -add_library(milvus_query ${MILVUS_QUERY_SRCS}) - -target_link_libraries(milvus_query milvus_index milvus_bitset milvus_monitor) +add_source_at_current_directory_recursively() +add_library(milvus_query OBJECT ${SOURCE_FILES}) \ No newline at end of file diff --git a/internal/core/src/segcore/CMakeLists.txt b/internal/core/src/segcore/CMakeLists.txt index 378240aee9..618396132b 100644 --- a/internal/core/src/segcore/CMakeLists.txt +++ b/internal/core/src/segcore/CMakeLists.txt @@ -9,40 +9,6 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing permissions and limitations under the License -option( EMBEDDED_MILVUS "Enable embedded Milvus" OFF ) -if ( EMBEDDED_MILVUS ) - add_compile_definitions( EMBEDDED_MILVUS ) -endif() - -milvus_add_pkg_config("milvus_segcore") - -set(SEGCORE_FILES - Collection.cpp - collection_c.cpp - segment_c.cpp - SegmentGrowingImpl.cpp - SegmentSealedImpl.cpp - FieldIndexing.cpp - metrics_c.cpp - plan_c.cpp - reduce_c.cpp - load_index_c.cpp - load_field_data_c.cpp - SegmentInterface.cpp - SegcoreConfig.cpp - IndexConfigGenerator.cpp - segcore_init_c.cpp - TimestampIndex.cpp - Utils.cpp - ConcurrentVector.cpp - ReduceUtils.cpp - check_vec_index_c.cpp - reduce/Reduce.cpp - reduce/StreamReduce.cpp - reduce/GroupReduce.cpp) -add_library(milvus_segcore SHARED ${SEGCORE_FILES}) - -target_link_libraries(milvus_segcore milvus_query milvus_bitset milvus_exec ${OpenMP_CXX_FLAGS} milvus_futures) - -install(TARGETS milvus_segcore DESTINATION "${CMAKE_INSTALL_LIBDIR}") +add_source_at_current_directory_recursively() +add_library(milvus_segcore OBJECT ${SOURCE_FILES}) \ No newline at end of file diff --git a/internal/core/src/segcore/milvus_segcore.pc.in b/internal/core/src/segcore/milvus_segcore.pc.in deleted file mode 100644 index 7b367dbb37..0000000000 --- a/internal/core/src/segcore/milvus_segcore.pc.in +++ /dev/null @@ -1,9 +0,0 @@ -libdir=@CMAKE_INSTALL_FULL_LIBDIR@ -includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ - -Name: Milvus Segcore -Description: Segcore modules for Milvus -Version: @MILVUS_VERSION@ - -Libs: -L${libdir} -lmilvus_segcore -Cflags: -I${includedir} diff --git a/internal/core/src/storage/CMakeLists.txt b/internal/core/src/storage/CMakeLists.txt index 11cd21c5d9..0d69386e08 100644 --- a/internal/core/src/storage/CMakeLists.txt +++ b/internal/core/src/storage/CMakeLists.txt @@ -14,72 +14,27 @@ # See the License for the specific language governing permissions and # limitations under the License. -option(EMBEDDED_MILVUS "Enable embedded Milvus" OFF) - -if(EMBEDDED_MILVUS) - add_compile_definitions(EMBEDDED_MILVUS) -endif() - -milvus_add_pkg_config("milvus_storage") - +add_source_at_current_directory() if (DEFINED AZURE_BUILD_DIR) add_definitions(-DAZURE_BUILD_DIR) include_directories(azure-blob-storage) include_directories("${AZURE_BUILD_DIR}/vcpkg_installed/${VCPKG_TARGET_TRIPLET}/include") - set(STORAGE_FILES - ${STORAGE_FILES} - AzureChunkManager.cpp - ) + set(SOURCE_FILES ${SOURCE_FILES} azure/AzureChunkManager.cpp) + add_library(azure_blob_chunk_manager SHARED IMPORTED) + set_target_properties(azure_blob_chunk_manager + PROPERTIES + IMPORTED_GLOBAL TRUE + IMPORTED_LOCATION "${AZURE_BUILD_DIR}/libblob-chunk-manager${CMAKE_SHARED_LIBRARY_SUFFIX}" + ) + get_target_property(AZURE_IMPORTED_LOCATION azure_blob_chunk_manager IMPORTED_LOCATION) + get_target_property(AZURE_INTERFACE_INCLUDE_DIRECTORIES azure_blob_chunk_manager INTERFACE_INCLUDE_DIRECTORIES) + message("AZURE_IMPORTED_LOCATION: ${AZURE_IMPORTED_LOCATION}") + message("AZURE_INTERFACE_INCLUDE_DIRECTORIES: ${AZURE_INTERFACE_INCLUDE_DIRECTORIES}") endif() -set(STORAGE_FILES - ${STORAGE_FILES} - PayloadStream.cpp - DataCodec.cpp - Util.cpp - PayloadReader.cpp - PayloadWriter.cpp - BinlogReader.cpp - IndexData.cpp - InsertData.cpp - Event.cpp - ThreadPool.cpp - storage_c.cpp - ChunkManager.cpp - MinioChunkManager.cpp - AliyunSTSClient.cpp - AliyunCredentialsProvider.cpp - MemFileManagerImpl.cpp - LocalChunkManager.cpp - DiskFileManagerImpl.cpp - ThreadPools.cpp - ChunkCache.cpp - TencentCloudCredentialsProvider.cpp - TencentCloudSTSClient.cpp - MmapChunkManager.cpp) - if(USE_OPENDAL) - list(APPEND STORAGE_FILES OpenDALChunkManager.cpp) + include_directories(${OPENDAL_INCLUDE_DIR}) + set(SOURCE_FILES ${SOURCE_FILES} opendal/OpenDALChunkManager.cpp) endif() -add_library(milvus_storage SHARED ${STORAGE_FILES}) - -if (DEFINED AZURE_BUILD_DIR) - target_link_libraries(milvus_storage PUBLIC - "-L${AZURE_BUILD_DIR} -lblob-chunk-manager" - blob-chunk-manager - milvus_common - milvus_monitor - pthread - ${CONAN_LIBS} - ) -else () - target_link_libraries(milvus_storage PUBLIC - milvus_common - milvus_monitor - pthread - ${CONAN_LIBS} - ) -endif() - -install(TARGETS milvus_storage DESTINATION "${CMAKE_INSTALL_LIBDIR}") +add_library(milvus_storage OBJECT ${SOURCE_FILES}) diff --git a/internal/core/src/storage/Util.cpp b/internal/core/src/storage/Util.cpp index 95dc7ba131..84f43eb36e 100644 --- a/internal/core/src/storage/Util.cpp +++ b/internal/core/src/storage/Util.cpp @@ -27,7 +27,7 @@ #include "common/FieldData.h" #include "common/FieldDataInterface.h" #ifdef AZURE_BUILD_DIR -#include "storage/AzureChunkManager.h" +#include "storage/azure/AzureChunkManager.h" #endif #include "storage/ChunkManager.h" #include "storage/DiskFileManagerImpl.h" @@ -36,7 +36,7 @@ #include "storage/MemFileManagerImpl.h" #include "storage/MinioChunkManager.h" #ifdef USE_OPENDAL -#include "storage/OpenDALChunkManager.h" +#include "storage/opendal/OpenDALChunkManager.h" #endif #include "storage/Types.h" #include "storage/Util.h" diff --git a/internal/core/src/storage/azure-blob-storage/CMakeLists.txt b/internal/core/src/storage/azure-blob-storage/CMakeLists.txt index 62b2e971c8..cf6ffa0f72 100644 --- a/internal/core/src/storage/azure-blob-storage/CMakeLists.txt +++ b/internal/core/src/storage/azure-blob-storage/CMakeLists.txt @@ -32,10 +32,10 @@ endif() find_package(azure-storage-blobs-cpp CONFIG REQUIRED) find_package(azure-identity-cpp CONFIG REQUIRED) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter -Wno-return-type -Wno-pedantic") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter -Wno-return-type -Wno-pedantic -fPIC") add_library(blob-chunk-manager SHARED AzureBlobChunkManager.cpp) target_link_libraries(blob-chunk-manager PUBLIC Azure::azure-identity Azure::azure-storage-blobs) - +# should be link directly into libmilvus_core in future. install(TARGETS blob-chunk-manager DESTINATION "${CMAKE_INSTALL_LIBDIR}") if ( BUILD_UNIT_TEST STREQUAL "ON" ) diff --git a/internal/core/src/storage/AzureChunkManager.cpp b/internal/core/src/storage/azure/AzureChunkManager.cpp similarity index 99% rename from internal/core/src/storage/AzureChunkManager.cpp rename to internal/core/src/storage/azure/AzureChunkManager.cpp index 99a29a8142..854504e28a 100644 --- a/internal/core/src/storage/AzureChunkManager.cpp +++ b/internal/core/src/storage/azure/AzureChunkManager.cpp @@ -21,7 +21,7 @@ #include "common/EasyAssert.h" #include "log/Log.h" #include "monitor/prometheus_client.h" -#include "storage/AzureChunkManager.h" +#include "storage/azure/AzureChunkManager.h" namespace milvus { namespace storage { diff --git a/internal/core/src/storage/AzureChunkManager.h b/internal/core/src/storage/azure/AzureChunkManager.h similarity index 100% rename from internal/core/src/storage/AzureChunkManager.h rename to internal/core/src/storage/azure/AzureChunkManager.h diff --git a/internal/core/src/storage/milvus_storage.pc.in b/internal/core/src/storage/milvus_storage.pc.in deleted file mode 100644 index 6ad79e91ac..0000000000 --- a/internal/core/src/storage/milvus_storage.pc.in +++ /dev/null @@ -1,10 +0,0 @@ - -libdir=@CMAKE_INSTALL_FULL_LIBDIR@ -includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ - -Name: Milvus Storage -Description: Storage wrapper for Milvus -Version: @MILVUS_VERSION@ - -Libs: -L${libdir} -lmilvus_storage -Cflags: -I${includedir} diff --git a/internal/core/src/storage/OpenDALChunkManager.cpp b/internal/core/src/storage/opendal/OpenDALChunkManager.cpp similarity index 99% rename from internal/core/src/storage/OpenDALChunkManager.cpp rename to internal/core/src/storage/opendal/OpenDALChunkManager.cpp index 3affe3ae07..465c0a939a 100644 --- a/internal/core/src/storage/OpenDALChunkManager.cpp +++ b/internal/core/src/storage/opendal/OpenDALChunkManager.cpp @@ -22,7 +22,7 @@ #include "opendal.h" #include "common/EasyAssert.h" #include "storage/Util.h" -#include "storage/OpenDALChunkManager.h" +#include "storage/opendal/OpenDALChunkManager.h" namespace milvus::storage { diff --git a/internal/core/src/storage/OpenDALChunkManager.h b/internal/core/src/storage/opendal/OpenDALChunkManager.h similarity index 100% rename from internal/core/src/storage/OpenDALChunkManager.h rename to internal/core/src/storage/opendal/OpenDALChunkManager.h diff --git a/internal/core/thirdparty/knowhere/CMakeLists.txt b/internal/core/thirdparty/knowhere/CMakeLists.txt index 64910c9927..4aa95bee06 100644 --- a/internal/core/thirdparty/knowhere/CMakeLists.txt +++ b/internal/core/thirdparty/knowhere/CMakeLists.txt @@ -60,3 +60,5 @@ endif() # get prometheus COMPILE_OPTIONS get_property( var DIRECTORY "${knowhere_SOURCE_DIR}" PROPERTY COMPILE_OPTIONS ) message( STATUS "knowhere src compile options: ${var}" ) + +set( KNOWHERE_INCLUDE_DIR ${knowhere_SOURCE_DIR}/include CACHE INTERNAL "Path to knowhere include directory" ) diff --git a/internal/core/thirdparty/opendal/CMakeLists.txt b/internal/core/thirdparty/opendal/CMakeLists.txt index 25fd05e83c..a01f153621 100644 --- a/internal/core/thirdparty/opendal/CMakeLists.txt +++ b/internal/core/thirdparty/opendal/CMakeLists.txt @@ -14,16 +14,48 @@ # ---------------------------------------------------------------------- message(STATUS "Building (vendored) opendal from source") -set(OPENDAL_LIB_DIR "${CMAKE_INSTALL_PREFIX}/lib") -set(OPENDAL_INCLUDE_DIR "${CMAKE_INSTALL_PREFIX}/include") -set(OPENDAL_NAME "libopendal_c${CMAKE_STATIC_LIBRARY_SUFFIX}") +set(GIT_REPOSITORY "https://github.com/apache/opendal.git") +set(GIT_TAG "v0.43.0-rc.2") + +if (CMAKE_BUILD_TYPE STREQUAL "Debug") + set(CARGO_CMD cargo +1.73 build --verbose) + set(TARGET_DIR "debug") +else () + set(CARGO_CMD cargo +1.73 build --release --verbose) + set(TARGET_DIR "release") +endif () + +set(SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/src") + +FetchContent_Declare( + opendal + GIT_REPOSITORY ${GIT_REPOSITORY} + GIT_TAG ${GIT_TAG} + GIT_SHALLOW TRUE + SOURCE_DIR ${SOURCE_DIR} + DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH}) + +FetchContent_GetProperties(opendal) +if ( NOT opendal_POPULATED ) + FetchContent_Populate(opendal) +endif() + +set(OPENDAL_LIB_DIR "${SOURCE_DIR}/target/${TARGET_DIR}" CACHE INTERNAL "opendal lib dir") +set(OPENDAL_INCLUDE_DIR "${SOURCE_DIR}/bindings/c/include" CACHE INTERNAL "opendal include dir") +set(OPENDAL_LIB "libopendal_c${CMAKE_STATIC_LIBRARY_SUFFIX}" CACHE INTERNAL "opendal lib") + +add_custom_target(build_opendal + COMMAND ${CARGO_CMD} + WORKING_DIRECTORY ${SOURCE_DIR}/bindings/c +) add_library(opendal STATIC IMPORTED) +add_dependencies(opendal build_opendal) set_target_properties(opendal PROPERTIES IMPORTED_GLOBAL TRUE - IMPORTED_LOCATION "${OPENDAL_LIB_DIR}/${OPENDAL_NAME}" - INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_INSTALL_PREFIX}/include") + IMPORTED_LOCATION "${OPENDAL_LIB_DIR}/${OPENDAL_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${OPENDAL_INCLUDE_DIR}") get_target_property(OPENDAL_IMPORTED_LOCATION opendal IMPORTED_LOCATION) get_target_property(OPENDAL_INTERFACE_INCLUDE_DIRECTORIES opendal INTERFACE_INCLUDE_DIRECTORIES) diff --git a/internal/core/thirdparty/simdjson/CMakeLists.txt b/internal/core/thirdparty/simdjson/CMakeLists.txt index 1000c2b3cc..ddf2af9c2b 100644 --- a/internal/core/thirdparty/simdjson/CMakeLists.txt +++ b/internal/core/thirdparty/simdjson/CMakeLists.txt @@ -17,3 +17,5 @@ FetchContent_Declare( URL_HASH MD5=1b0d75ad32179c77f84f4a09d4214057 ) FetchContent_MakeAvailable(simdjson) + +set( SIMDJSON_INCLUDE_DIR ${simdjson_SOURCE_DIR}/include CACHE INTERNAL "Path to simdjson include directory" ) diff --git a/internal/core/thirdparty/tantivy/CMakeLists.txt b/internal/core/thirdparty/tantivy/CMakeLists.txt index c1435a032a..3963647eee 100644 --- a/internal/core/thirdparty/tantivy/CMakeLists.txt +++ b/internal/core/thirdparty/tantivy/CMakeLists.txt @@ -77,3 +77,5 @@ target_link_libraries(ffi_demo tantivy_binding dl ) + +set( TANTIVY_INCLUDE_DIR ${LIB_HEADER_FOLDER};${CMAKE_CURRENT_SOURCE_DIR} CACHE INTERNAL "Path to tantivy include directory" ) diff --git a/internal/core/unittest/CMakeLists.txt b/internal/core/unittest/CMakeLists.txt index cfe2ad00b7..d442fa3f0f 100644 --- a/internal/core/unittest/CMakeLists.txt +++ b/internal/core/unittest/CMakeLists.txt @@ -11,6 +11,12 @@ include_directories(${CMAKE_HOME_DIRECTORY}/src) include_directories(${CMAKE_HOME_DIRECTORY}/src/thirdparty) +include_directories( + ${KNOWHERE_INCLUDE_DIR} + ${SIMDJSON_INCLUDE_DIR} + ${TANTIVY_INCLUDE_DIR} + ${CONAN_INCLUDE_DIRS} +) add_definitions(-DMILVUS_TEST_SEGCORE_YAML_PATH="${CMAKE_SOURCE_DIR}/unittest/test_utils/test_segcore.yaml") @@ -130,13 +136,8 @@ if (LINUX) target_link_libraries(index_builder_test gtest - gtest_main - milvus_monitor - milvus_segcore - milvus_storage - milvus_indexbuilder - milvus_clustering - milvus_common + milvus_core + knowhere ) install(TARGETS index_builder_test DESTINATION unittest) endif() @@ -147,13 +148,8 @@ add_executable(all_tests target_link_libraries(all_tests gtest - milvus_segcore - milvus_storage - milvus_indexbuilder - milvus_clustering - pthread - milvus_common - milvus_exec + milvus_core + knowhere ) install(TARGETS all_tests DESTINATION unittest) diff --git a/internal/core/unittest/bench/CMakeLists.txt b/internal/core/unittest/bench/CMakeLists.txt index 6147e37332..68bed1644c 100644 --- a/internal/core/unittest/bench/CMakeLists.txt +++ b/internal/core/unittest/bench/CMakeLists.txt @@ -23,8 +23,8 @@ set(indexbuilder_bench_srcs add_executable(all_bench ${bench_srcs}) target_link_libraries(all_bench - milvus_segcore - milvus_log + milvus_core + knowhere pthread ) @@ -32,12 +32,9 @@ target_link_libraries(all_bench benchmark_main) add_executable(indexbuilder_bench ${indexbuilder_bench_srcs}) target_link_libraries(indexbuilder_bench - milvus_segcore - milvus_indexbuilder - milvus_index - milvus_log - pthread + milvus_core knowhere + pthread ) target_link_libraries(indexbuilder_bench benchmark_main) diff --git a/internal/core/unittest/test_azure_chunk_manager.cpp b/internal/core/unittest/test_azure_chunk_manager.cpp index ed9665e2cf..e1ac91a8ab 100644 --- a/internal/core/unittest/test_azure_chunk_manager.cpp +++ b/internal/core/unittest/test_azure_chunk_manager.cpp @@ -14,7 +14,7 @@ #include #include "common/EasyAssert.h" -#include "storage/AzureChunkManager.h" +#include "storage/azure/AzureChunkManager.h" #include "storage/Util.h" using namespace std; diff --git a/internal/indexnode/indexnode.go b/internal/indexnode/indexnode.go index f5ef580872..ed9b4ea19f 100644 --- a/internal/indexnode/indexnode.go +++ b/internal/indexnode/indexnode.go @@ -17,7 +17,7 @@ package indexnode /* -#cgo pkg-config: milvus_common milvus_indexbuilder milvus_clustering milvus_segcore +#cgo pkg-config: milvus_core #include #include diff --git a/internal/proxy/cgo_util.go b/internal/proxy/cgo_util.go index ca1336d048..1f929d29b4 100644 --- a/internal/proxy/cgo_util.go +++ b/internal/proxy/cgo_util.go @@ -17,7 +17,7 @@ package proxy /* -#cgo pkg-config: milvus_segcore +#cgo pkg-config: milvus_core #include "segcore/check_vec_index_c.h" #include */ diff --git a/internal/querynodev2/segments/cgo_util.go b/internal/querynodev2/segments/cgo_util.go index 11c9957e7d..b127a19909 100644 --- a/internal/querynodev2/segments/cgo_util.go +++ b/internal/querynodev2/segments/cgo_util.go @@ -17,7 +17,7 @@ package segments /* -#cgo pkg-config: milvus_segcore milvus_storage +#cgo pkg-config: milvus_core #include "segcore/collection_c.h" #include "common/type_c.h" diff --git a/internal/querynodev2/segments/collection.go b/internal/querynodev2/segments/collection.go index 86c85e0f80..5a5679c0c9 100644 --- a/internal/querynodev2/segments/collection.go +++ b/internal/querynodev2/segments/collection.go @@ -17,7 +17,7 @@ package segments /* -#cgo pkg-config: milvus_segcore +#cgo pkg-config: milvus_core #include "segcore/collection_c.h" #include "segcore/segment_c.h" diff --git a/internal/querynodev2/segments/index_attr_cache.go b/internal/querynodev2/segments/index_attr_cache.go index 73f1cfbe9f..d4b8fbed63 100644 --- a/internal/querynodev2/segments/index_attr_cache.go +++ b/internal/querynodev2/segments/index_attr_cache.go @@ -17,7 +17,7 @@ package segments /* -#cgo pkg-config: milvus_segcore +#cgo pkg-config: milvus_core #include "segcore/load_index_c.h" */ diff --git a/internal/querynodev2/segments/load_field_data_info.go b/internal/querynodev2/segments/load_field_data_info.go index fdca37fe86..3625c06fc2 100644 --- a/internal/querynodev2/segments/load_field_data_info.go +++ b/internal/querynodev2/segments/load_field_data_info.go @@ -17,7 +17,7 @@ package segments /* -#cgo pkg-config: milvus_segcore +#cgo pkg-config: milvus_core #include "segcore/load_field_data_c.h" */ import "C" diff --git a/internal/querynodev2/segments/load_index_info.go b/internal/querynodev2/segments/load_index_info.go index e0117f8272..554b3cc41d 100644 --- a/internal/querynodev2/segments/load_index_info.go +++ b/internal/querynodev2/segments/load_index_info.go @@ -17,7 +17,7 @@ package segments /* -#cgo pkg-config: milvus_common milvus_segcore +#cgo pkg-config: milvus_core #include "segcore/load_index_c.h" #include "common/binary_set_c.h" diff --git a/internal/querynodev2/segments/manager.go b/internal/querynodev2/segments/manager.go index 268e9be73f..0a881a6dcf 100644 --- a/internal/querynodev2/segments/manager.go +++ b/internal/querynodev2/segments/manager.go @@ -17,7 +17,7 @@ package segments /* -#cgo pkg-config: milvus_segcore +#cgo pkg-config: milvus_core #include "segcore/collection_c.h" #include "segcore/segment_c.h" diff --git a/internal/querynodev2/segments/plan.go b/internal/querynodev2/segments/plan.go index c18a04792a..ff94ac63c9 100644 --- a/internal/querynodev2/segments/plan.go +++ b/internal/querynodev2/segments/plan.go @@ -17,7 +17,7 @@ package segments /* -#cgo pkg-config: milvus_segcore +#cgo pkg-config: milvus_core #include "segcore/collection_c.h" #include "segcore/segment_c.h" diff --git a/internal/querynodev2/segments/reduce.go b/internal/querynodev2/segments/reduce.go index 7cbaccc37b..6fcee69956 100644 --- a/internal/querynodev2/segments/reduce.go +++ b/internal/querynodev2/segments/reduce.go @@ -17,7 +17,7 @@ package segments /* -#cgo pkg-config: milvus_segcore +#cgo pkg-config: milvus_core #include "segcore/plan_c.h" #include "segcore/reduce_c.h" diff --git a/internal/querynodev2/segments/segment.go b/internal/querynodev2/segments/segment.go index 3dbb26e3b0..ee2c3bfc64 100644 --- a/internal/querynodev2/segments/segment.go +++ b/internal/querynodev2/segments/segment.go @@ -17,7 +17,7 @@ package segments /* -#cgo pkg-config: milvus_segcore milvus_futures +#cgo pkg-config: milvus_core #include "futures/future_c.h" #include "segcore/collection_c.h" diff --git a/internal/querynodev2/segments/segment_loader.go b/internal/querynodev2/segments/segment_loader.go index c956f08d07..242990f855 100644 --- a/internal/querynodev2/segments/segment_loader.go +++ b/internal/querynodev2/segments/segment_loader.go @@ -17,7 +17,7 @@ package segments /* -#cgo pkg-config: milvus_segcore +#cgo pkg-config: milvus_core #include "segcore/load_index_c.h" */ diff --git a/internal/querynodev2/segments/trace.go b/internal/querynodev2/segments/trace.go index 7fb9c565bf..44ed576e2a 100644 --- a/internal/querynodev2/segments/trace.go +++ b/internal/querynodev2/segments/trace.go @@ -17,7 +17,7 @@ package segments /* -#cgo pkg-config: milvus_segcore +#cgo pkg-config: milvus_core #include "segcore/segment_c.h" */ diff --git a/internal/querynodev2/segments/utils.go b/internal/querynodev2/segments/utils.go index 8eb1eec424..14ddedd2d8 100644 --- a/internal/querynodev2/segments/utils.go +++ b/internal/querynodev2/segments/utils.go @@ -1,7 +1,7 @@ package segments /* -#cgo pkg-config: milvus_segcore milvus_common +#cgo pkg-config: milvus_core #include "segcore/collection_c.h" #include "segcore/segment_c.h" diff --git a/internal/querynodev2/server.go b/internal/querynodev2/server.go index 9b6d26ad68..fbcd32b6a3 100644 --- a/internal/querynodev2/server.go +++ b/internal/querynodev2/server.go @@ -17,7 +17,7 @@ package querynodev2 /* -#cgo pkg-config: milvus_segcore milvus_common +#cgo pkg-config: milvus_core #include "segcore/collection_c.h" #include "segcore/segment_c.h" diff --git a/internal/util/analyzecgowrapper/analyze.go b/internal/util/analyzecgowrapper/analyze.go index 3bfe9d3ff0..ad57fc3f71 100644 --- a/internal/util/analyzecgowrapper/analyze.go +++ b/internal/util/analyzecgowrapper/analyze.go @@ -17,7 +17,7 @@ package analyzecgowrapper /* -#cgo pkg-config: milvus_clustering +#cgo pkg-config: milvus_core #include // free #include "clustering/analyze_c.h" diff --git a/internal/util/analyzecgowrapper/helper.go b/internal/util/analyzecgowrapper/helper.go index 5b2f0b8fcc..7b8bbd564b 100644 --- a/internal/util/analyzecgowrapper/helper.go +++ b/internal/util/analyzecgowrapper/helper.go @@ -18,7 +18,7 @@ package analyzecgowrapper /* -#cgo pkg-config: milvus_common +#cgo pkg-config: milvus_core #include // free #include "common/type_c.h" diff --git a/internal/util/cgo/errors.go b/internal/util/cgo/errors.go index c0bb6e482f..11e703499c 100644 --- a/internal/util/cgo/errors.go +++ b/internal/util/cgo/errors.go @@ -1,7 +1,7 @@ package cgo /* -#cgo pkg-config: milvus_common +#cgo pkg-config: milvus_core #include "common/type_c.h" #include diff --git a/internal/util/cgo/executor.go b/internal/util/cgo/executor.go index a589513469..0c68b4d660 100644 --- a/internal/util/cgo/executor.go +++ b/internal/util/cgo/executor.go @@ -1,7 +1,7 @@ package cgo /* -#cgo pkg-config: milvus_futures +#cgo pkg-config: milvus_core #include "futures/future_c.h" */ diff --git a/internal/util/cgo/futures.go b/internal/util/cgo/futures.go index 3b6aadf454..c0a3b9885e 100644 --- a/internal/util/cgo/futures.go +++ b/internal/util/cgo/futures.go @@ -1,7 +1,7 @@ package cgo /* -#cgo pkg-config: milvus_futures +#cgo pkg-config: milvus_core #include "futures/future_c.h" #include diff --git a/internal/util/cgo/futures_test_case.go b/internal/util/cgo/futures_test_case.go index 3cc933c095..b77d0230d0 100644 --- a/internal/util/cgo/futures_test_case.go +++ b/internal/util/cgo/futures_test_case.go @@ -4,7 +4,7 @@ package cgo /* -#cgo pkg-config: milvus_futures +#cgo pkg-config: milvus_core #include "futures/future_c.h" #include diff --git a/internal/util/indexcgowrapper/build_index_info.go b/internal/util/indexcgowrapper/build_index_info.go index efef8020d0..72523a840e 100644 --- a/internal/util/indexcgowrapper/build_index_info.go +++ b/internal/util/indexcgowrapper/build_index_info.go @@ -17,7 +17,7 @@ package indexcgowrapper /* -#cgo pkg-config: milvus_indexbuilder +#cgo pkg-config: milvus_core #include // free #include "indexbuilder/index_c.h" */ diff --git a/internal/util/indexcgowrapper/helper.go b/internal/util/indexcgowrapper/helper.go index 32f79a17ba..6ebbd9009e 100644 --- a/internal/util/indexcgowrapper/helper.go +++ b/internal/util/indexcgowrapper/helper.go @@ -1,7 +1,7 @@ package indexcgowrapper /* -#cgo pkg-config: milvus_common milvus_storage +#cgo pkg-config: milvus_core #include // free #include "common/binary_set_c.h" diff --git a/internal/util/indexcgowrapper/index.go b/internal/util/indexcgowrapper/index.go index f2458e1c86..e60051bb87 100644 --- a/internal/util/indexcgowrapper/index.go +++ b/internal/util/indexcgowrapper/index.go @@ -1,7 +1,7 @@ package indexcgowrapper /* -#cgo pkg-config: milvus_indexbuilder +#cgo pkg-config: milvus_core #include // free #include "indexbuilder/index_c.h" diff --git a/internal/util/initcore/init_core.go b/internal/util/initcore/init_core.go index 0651ef9d13..332e559bc7 100644 --- a/internal/util/initcore/init_core.go +++ b/internal/util/initcore/init_core.go @@ -17,7 +17,7 @@ package initcore /* -#cgo pkg-config: milvus_common milvus_storage milvus_segcore +#cgo pkg-config: milvus_core #include #include diff --git a/internal/util/metrics/c_registry.go b/internal/util/metrics/c_registry.go index 5505f31341..9d4963f623 100644 --- a/internal/util/metrics/c_registry.go +++ b/internal/util/metrics/c_registry.go @@ -19,7 +19,7 @@ package metrics /* -#cgo pkg-config: milvus_segcore milvus_storage milvus_common milvus_monitor +#cgo pkg-config: milvus_core #include #include "segcore/metrics_c.h" diff --git a/scripts/3rdparty_build.sh b/scripts/3rdparty_build.sh index 5a9b4ce39d..2e3e35c0e2 100644 --- a/scripts/3rdparty_build.sh +++ b/scripts/3rdparty_build.sh @@ -92,15 +92,3 @@ else bash -c "curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain=1.73 -y" || { echo 'rustup install failed'; exit 1;} source $HOME/.cargo/env fi - -echo "BUILD_OPENDAL: ${BUILD_OPENDAL}" -if [ "${BUILD_OPENDAL}" = "ON" ]; then - git clone --depth=1 --branch v0.43.0-rc.2 https://github.com/apache/opendal.git opendal - cd opendal - pushd bindings/c - cargo +1.73 build --release --verbose || { echo 'opendal_c build failed'; exit 1; } - popd - cp target/release/libopendal_c.a ${ROOT_DIR}/internal/core/output/lib/libopendal_c.a - cp bindings/c/include/opendal.h ${ROOT_DIR}/internal/core/output/include/opendal.h -fi -popd