enhance: optimize milvus core building (#35610)

issue: #35549,#35611,#35633

- remove milvus_segcore milvus_indexbuilder..., add libmilvus_core
- core building only link once
- move opendal compilation into cmake
- fix odr

---------

Signed-off-by: chyezh <chyezh@outlook.com>

build/rpm/milvus.spec

@@ -38,9 +38,7 @@ install -m 755 bin/minio %{buildroot}/usr/bin/milvus-minio
 
 # lib
 install -m 755 lib/libknowhere.so %{buildroot}/lib64/milvus/libknowhere.so
-install -m 755 lib/libmilvus_common.so %{buildroot}/lib64/milvus/libmilvus_common.so
+install -m 755 lib/libmivlus_core.so %{buildroot}/lib64/milvus/libmilvus_core.so
-install -m 755 lib/libmilvus_indexbuilder.so %{buildroot}/lib64/milvus/libmilvus_indexbuilder.so
-install -m 755 lib/libmilvus_segcore.so %{buildroot}/lib64/milvus/libmilvus_segcore.so
 install -m 755 /usr/lib/libopenblas-r0.3.9.so %{buildroot}/lib64/milvus/libopenblas.so.0
 install -m 755 lib/libngt.so.1.12.0 %{buildroot}/lib64/milvus/libngt.so.1
 install -m 755 /usr/lib64/libgfortran.so.4.0.0 %{buildroot}/lib64/milvus/libgfortran.so.4
@@ -82,9 +80,7 @@ systemctl daemon-reload
 /usr/bin/milvus-minio
 
 /lib64/milvus/libknowhere.so
-/lib64/milvus/libmilvus_common.so
+/lib64/milvus/libmilvus_core.so
-/lib64/milvus/libmilvus_indexbuilder.so
-/lib64/milvus/libmilvus_segcore.so
 /lib64/milvus/libopenblas.so.0
 /lib64/milvus/libngt.so.1
 /lib64/milvus/libgfortran.so.4

internal/core/CMakeLists.txt

@@ -319,4 +319,3 @@ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/futures/
 install(DIRECTORY ${CMAKE_BINARY_DIR}/lib/
         DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}
 )
-

internal/core/cmake/Utils.cmake

@@ -99,4 +99,14 @@ macro(create_library)
     add_library(${L_TARGET} ${L_SRCS})
     target_link_libraries(${L_TARGET} PRIVATE ${L_LIBS})
     target_compile_definitions(${L_TARGET} PRIVATE ${L_DEFS})
-endmacro()
+endmacro()
+
+macro(add_source_at_current_directory_recursively)
+    file(GLOB_RECURSE SOURCE_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc" "*.cpp" "*.c" "*.cxx")
+    message(STATUS "${CMAKE_CURRENT_SOURCE_DIR}  add new source files at current directory recursively: ${SOURCE_FILES}")
+endmacro()
+
+macro(add_source_at_current_directory)
+    file(GLOB SOURCE_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc" "*.cpp" "*.c" "*.cxx")
+    message(STATUS "${CMAKE_CURRENT_SOURCE_DIR}  add new source files at current directory: ${SOURCE_FILES}")
+endmacro()

internal/core/src/CMakeLists.txt

@@ -20,8 +20,19 @@ else()
     project(core CXX C)
 endif()
 
+option( EMBEDDED_MILVUS "Enable embedded Milvus" OFF )
+if ( EMBEDDED_MILVUS )
+    add_compile_definitions( EMBEDDED_MILVUS )
+endif()
+
 include_directories(${MILVUS_ENGINE_SRC})
 include_directories(${MILVUS_THIRDPARTY_SRC})
+include_directories(
+    ${KNOWHERE_INCLUDE_DIR}
+    ${SIMDJSON_INCLUDE_DIR}
+    ${TANTIVY_INCLUDE_DIR}
+    ${CONAN_INCLUDE_DIRS}
+)
 
 add_subdirectory( pb )
 add_subdirectory( log )
@@ -37,3 +48,42 @@ add_subdirectory( clustering )
 add_subdirectory( exec )
 add_subdirectory( bitset )
 add_subdirectory( futures )
+
+milvus_add_pkg_config("milvus_core")
+
+add_library(milvus_core SHARED
+    $<TARGET_OBJECTS:milvus_pb>
+    $<TARGET_OBJECTS:milvus_log>
+    $<TARGET_OBJECTS:milvus_config>
+    $<TARGET_OBJECTS:milvus_common>
+    $<TARGET_OBJECTS:milvus_monitor>
+    $<TARGET_OBJECTS:milvus_storage>
+    $<TARGET_OBJECTS:milvus_index>
+    $<TARGET_OBJECTS:milvus_query>
+    $<TARGET_OBJECTS:milvus_segcore>
+    $<TARGET_OBJECTS:milvus_indexbuilder>
+    $<TARGET_OBJECTS:milvus_clustering>
+    $<TARGET_OBJECTS:milvus_exec>
+    $<TARGET_OBJECTS:milvus_bitset>
+    $<TARGET_OBJECTS:milvus_futures>
+)
+
+set(LINK_TARGETS 
+    boost_bitset_ext
+    simdjson
+    tantivy_binding
+    knowhere
+    ${OpenMP_CXX_FLAGS}
+    ${CONAN_LIBS})
+
+if(USE_OPENDAL)
+    set(LINK_TARGETS ${LINK_TARGETS} opendal)
+endif()
+
+if(DEFINED AZURE_BUILD_DIR)
+    set(LINK_TARGETS ${LINK_TARGETS} azure_blob_chunk_manager)
+endif()
+
+target_link_libraries(milvus_core ${LINK_TARGETS})
+
+install(TARGETS milvus_core DESTINATION "${CMAKE_INSTALL_LIBDIR}")

internal/core/src/bitset/CMakeLists.txt

@@ -13,6 +13,8 @@ set(BITSET_SRCS
     detail/platform/dynamic.cpp
 )
 
+
+
 if (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86_64")
     list(APPEND BITSET_SRCS
         detail/platform/x86/avx2-inst.cpp
@@ -38,4 +40,4 @@ elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm*")
     #set_source_files_properties(detail/platform/arm/sve-inst.cpp PROPERTIES COMPILE_FLAGS "-mcpu=neoverse-v1")
 endif()
 
-add_library(milvus_bitset ${BITSET_SRCS})
+add_library(milvus_bitset OBJECT ${BITSET_SRCS})

internal/core/src/clustering/CMakeLists.txt

@@ -9,16 +9,5 @@
 # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 # or implied. See the License for the specific language governing permissions and limitations under the License
 
-
+add_source_at_current_directory_recursively()
-set(CLUSTERING_FILES
+add_library(milvus_clustering OBJECT ${SOURCE_FILES})
-        analyze_c.cpp
-        KmeansClustering.cpp
-        )
-
-milvus_add_pkg_config("milvus_clustering")
-add_library(milvus_clustering SHARED ${CLUSTERING_FILES})
-
-# link order matters
-target_link_libraries(milvus_clustering milvus_index)
-
-install(TARGETS milvus_clustering DESTINATION "${CMAKE_INSTALL_LIBDIR}")

internal/core/src/clustering/milvus_clustering.pc.in

@@ -1,9 +0,0 @@
-libdir=@CMAKE_INSTALL_FULL_LIBDIR@
-includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@
-
-Name: Milvus Clustering
-Description: Clustering modules for Milvus
-Version: @MILVUS_VERSION@
-
-Libs: -L${libdir} -lmilvus_clustering
-Cflags: -I${includedir}

internal/core/src/common/CMakeLists.txt

@@ -9,38 +9,5 @@
 # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 # or implied. See the License for the specific language governing permissions and limitations under the License
 
-milvus_add_pkg_config("milvus_common")
+add_source_at_current_directory_recursively()
-
+add_library(milvus_common OBJECT ${SOURCE_FILES})
-set(COMMON_SRC
-        Schema.cpp
-        SystemProperty.cpp
-        Slice.cpp
-        binary_set_c.cpp
-        init_c.cpp
-        Common.cpp
-        RangeSearchHelper.cpp
-        Tracer.cpp
-        IndexMeta.cpp
-        EasyAssert.cpp
-        FieldData.cpp
-        RegexQuery.cpp
-        ChunkTarget.cpp
-        Chunk.cpp
-        ChunkWriter.cpp
-        )
-
-add_library(milvus_common SHARED ${COMMON_SRC})
-
-target_link_libraries(milvus_common
-        milvus_bitset
-        milvus_config
-        milvus_log
-        milvus_proto
-        yaml-cpp
-        boost_bitset_ext
-        simdjson
-        ${CONAN_LIBS}
-        re2
-        )
-
-install(TARGETS milvus_common DESTINATION "${CMAKE_INSTALL_LIBDIR}")

internal/core/src/common/ChunkFileWriter.cpp

@@ -1,35 +0,0 @@
-// Copyright (C) 2019-2020 Zilliz. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software distributed under the License
-// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-// or implied. See the License for the specific language governing permissions and limitations under the License
-
-#include "common/ChunkFileWriter.h"
-#include "common/ChunkWriter.h"
-namespace milvus {
-ChunkFileWriter::ChunkFileWriter(std::string& file_path)
-    : file_(File::Open(file_path, O_RDWR | O_CREAT | O_TRUNC)) {
-}
-
-void
-ChunkFileWriter::write_chunk(std::shared_ptr<arrow::RecordBatchReader> r) {
-    // FIXME
-    size_t file_offset = file_.Seek(0, SEEK_END);
-    auto chunk = create_chunk(field_meta_, dim_, file_, file_offset, r);
-    // TODO: stat_writer_.write(chunk);
-    rep_.chunks.push_back(*chunk);
-}
-
-FileRep
-ChunkFileWriter::finish() {
-    // TODO: stat_writer_.finish();
-    // rep_.stat_chunk = stat_writer_.get();
-    return rep_;
-}
-
-}  // namespace milvus

internal/core/src/common/ChunkFileWriter.h

@@ -1,44 +0,0 @@
-// Copyright (C) 2019-2020 Zilliz. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software distributed under the License
-// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-// or implied. See the License for the specific language governing permissions and limitations under the License
-
-#pragma once
-#include <memory>
-#include "arrow/record_batch.h"
-#include "arrow/table_builder.h"
-#include "common/Chunk.h"
-#include "common/ChunkTarget.h"
-#include "common/FieldMeta.h"
-namespace milvus {
-class StatisticsChunkWriter;
-class StatisticsChunk;
-
-class ChunkFileWriter {
- public:
-    ChunkFileWriter() = default;
-    ChunkFileWriter(std::string& file_path);
-    struct FileRep {
-        std::vector<Chunk> chunks;
-    };
-    void
-    write_chunk(std::shared_ptr<arrow::RecordBatchReader> r);
-
-    FileRep
-    finish();
-
- private:
-    FieldMeta& field_meta_;
-    int dim_;
-    StatisticsChunkWriter stat_writer_;
-    File file_;
-    FileRep rep_;
-};
-
-}  // namespace milvus

internal/core/src/common/milvus_common.pc.in

@@ -1,9 +0,0 @@
-libdir=@CMAKE_INSTALL_FULL_LIBDIR@
-includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@
-
-Name: Milvus Common
-Description: Common modules for Milvus
-Version: @MILVUS_VERSION@
-
-Libs: -L${libdir} -lmilvus_common
-Cflags: -I${includedir}

internal/core/src/config/CMakeLists.txt

@@ -14,14 +14,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-option( EMBEDDED_MILVUS "Enable embedded Milvus" OFF )
+add_source_at_current_directory_recursively()
-
+add_library(milvus_config OBJECT ${SOURCE_FILES})
-if ( EMBEDDED_MILVUS )
-    add_compile_definitions( EMBEDDED_MILVUS )
-endif()
-
-set(CONFIG_SRC ConfigKnowhere.cpp)
-
-add_library(milvus_config STATIC ${CONFIG_SRC})
-
-target_link_libraries(milvus_config knowhere)

internal/core/src/exec/CMakeLists.txt

@@ -9,25 +9,5 @@
 # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 # or implied. See the License for the specific language governing permissions and limitations under the License
 
-set(MILVUS_EXEC_SRCS
+add_source_at_current_directory_recursively()
-        expression/Expr.cpp
+add_library(milvus_exec OBJECT ${SOURCE_FILES})
-        expression/UnaryExpr.cpp
-        expression/ConjunctExpr.cpp
-        expression/LogicalUnaryExpr.cpp
-        expression/LogicalBinaryExpr.cpp
-        expression/TermExpr.cpp
-        expression/BinaryArithOpEvalRangeExpr.cpp
-        expression/BinaryRangeExpr.cpp
-        expression/AlwaysTrueExpr.cpp
-        expression/CompareExpr.cpp
-        expression/JsonContainsExpr.cpp
-        expression/ExistsExpr.cpp
-        operator/FilterBits.cpp
-        operator/Operator.cpp
-        Driver.cpp
-        Task.cpp
-        )
-
-add_library(milvus_exec STATIC ${MILVUS_EXEC_SRCS})
-
-target_link_libraries(milvus_exec milvus_common ${CONAN_LIBS})

internal/core/src/futures/CMakeLists.txt

@@ -9,16 +9,5 @@
 # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 # or implied. See the License for the specific language governing permissions and limitations under the License
 
-milvus_add_pkg_config("milvus_futures")
+add_source_at_current_directory_recursively()
-
+add_library(milvus_futures OBJECT ${SOURCE_FILES})
-set(FUTURES_SRC
-        Executor.cpp
-        future_c.cpp
-        future_test_case_c.cpp
-        )
-
-add_library(milvus_futures SHARED ${FUTURES_SRC})
-
-target_link_libraries(milvus_futures milvus_common)
-
-install(TARGETS milvus_futures DESTINATION "${CMAKE_INSTALL_LIBDIR}")

internal/core/src/futures/milvus_futures.pc.in

@@ -1,9 +0,0 @@
-libdir=@CMAKE_INSTALL_FULL_LIBDIR@
-includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@
-
-Name: Milvus Futures
-Description: Futures modules for Milvus
-Version: @MILVUS_VERSION@
-
-Libs: -L${libdir} -lmilvus_futures
-Cflags: -I${includedir}

internal/core/src/index/CMakeLists.txt

@@ -9,23 +9,5 @@
 # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 # or implied. See the License for the specific language governing permissions and limitations under the License
 
-set(INDEX_FILES
+add_source_at_current_directory_recursively()
-        StringIndexMarisa.cpp
+add_library(milvus_index OBJECT ${SOURCE_FILES})
-        Utils.cpp
-        VectorMemIndex.cpp
-        IndexFactory.cpp
-        VectorDiskIndex.cpp
-        ScalarIndex.cpp
-        ScalarIndexSort.cpp
-        SkipIndex.cpp
-        InvertedIndexTantivy.cpp
-        BitmapIndex.cpp
-        HybridScalarIndex.cpp
-        )
-
-milvus_add_pkg_config("milvus_index")
-add_library(milvus_index SHARED ${INDEX_FILES})
-
-target_link_libraries(milvus_index milvus_storage tantivy_binding)
-
-install(TARGETS milvus_index DESTINATION "${CMAKE_INSTALL_LIBDIR}")

internal/core/src/indexbuilder/CMakeLists.txt

@@ -9,18 +9,5 @@
 # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 # or implied. See the License for the specific language governing permissions and limitations under the License
 
-
+add_source_at_current_directory_recursively()
-set(INDEXBUILDER_FILES
+add_library(milvus_indexbuilder OBJECT ${SOURCE_FILES})
-        VecIndexCreator.cpp
-        index_c.cpp
-        init_c.cpp
-        ScalarIndexCreator.cpp
-        )
-
-milvus_add_pkg_config("milvus_indexbuilder")
-add_library(milvus_indexbuilder SHARED ${INDEXBUILDER_FILES})
-
-# link order matters
-target_link_libraries(milvus_indexbuilder milvus_index)
-
-install(TARGETS milvus_indexbuilder DESTINATION "${CMAKE_INSTALL_LIBDIR}")

internal/core/src/indexbuilder/milvus_indexbuilder.pc.in

@@ -1,9 +0,0 @@
-libdir=@CMAKE_INSTALL_FULL_LIBDIR@
-includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@
-
-Name: Milvus IndexBuilder
-Description: IndexBuilder modules for Milvus
-Version: @MILVUS_VERSION@
-
-Libs: -L${libdir} -lmilvus_indexbuilder
-Cflags: -I${includedir}

internal/core/src/log/CMakeLists.txt

@@ -10,12 +10,6 @@
 # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 # or implied. See the License for the specific language governing permissions and limitations under the License.
 #-------------------------------------------------------------------------------
-set(LOG_FILES   ${MILVUS_ENGINE_SRC}/log/Log.cpp
-                ${MILVUS_ENGINE_SRC}/log/Log.h
-                #${MILVUS_THIRDPARTY_SRC}/easyloggingpp/easylogging++.cc
-                #${MILVUS_THIRDPARTY_SRC}/easyloggingpp/easylogging++.h
-                )
 
-add_library(milvus_log STATIC ${LOG_FILES})
+add_source_at_current_directory_recursively()
-set_target_properties(milvus_log PROPERTIES RULE_LAUNCH_COMPILE "")
+add_library(milvus_log OBJECT ${SOURCE_FILES})
-set_target_properties(milvus_log PROPERTIES RULE_LAUNCH_LINK "")

internal/core/src/milvus_core.pc.in

@@ -1,9 +1,9 @@
 libdir=@CMAKE_INSTALL_FULL_LIBDIR@
 includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@
 
-Name: Milvus index
+Name: Milvus Core
-Description: index modules for Milvus
+Description: Core modules for Milvus
 Version: @MILVUS_VERSION@
 
-Libs: -L${libdir} -lmilvus_index
+Libs: -L${libdir} -lmilvus_core
 Cflags: -I${includedir}

internal/core/src/monitor/CMakeLists.txt

@@ -9,15 +9,5 @@
 # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 # or implied. See the License for the specific language governing permissions and limitations under the License
 
-milvus_add_pkg_config("milvus_monitor")
+add_source_at_current_directory_recursively()
-
+add_library(milvus_monitor OBJECT ${SOURCE_FILES})
-set(MONITOR_SRC
-        monitor_c.cpp
-        prometheus_client.cpp
-        )
-
-add_library(milvus_monitor SHARED ${MONITOR_SRC})
-
-target_link_libraries(milvus_monitor PUBLIC ${CONAN_LIBS})
-
-install(TARGETS milvus_monitor DESTINATION "${CMAKE_INSTALL_LIBDIR}")

internal/core/src/monitor/milvus_monitor.pc.in

@@ -1,8 +0,0 @@
-libdir=@CMAKE_INSTALL_FULL_LIBDIR@
-includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@
-
-Name: Milvus Monitor
-Description: Monitor modules for Milvus
-Version: @MILVUS_VERSION@
-
-Libs: -L${libdir} -lmilvus_monitor

internal/core/src/pb/CMakeLists.txt

@@ -11,11 +11,5 @@
 
 find_package(Protobuf REQUIRED)
 
-file(GLOB_RECURSE milvus_proto_srcs
+add_source_at_current_directory_recursively()
-        "${CMAKE_CURRENT_SOURCE_DIR}/*.cc")
+add_library(milvus_pb OBJECT ${SOURCE_FILES})
-add_library(milvus_proto STATIC
-    ${milvus_proto_srcs}
-)
-message(STATUS "milvus proto sources: " ${milvus_proto_srcs})
-
-target_link_libraries( milvus_proto PUBLIC ${CONAN_LIBS} )

internal/core/src/query/CMakeLists.txt

@@ -9,26 +9,5 @@
 # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 # or implied. See the License for the specific language governing permissions and limitations under the License
 
-set(MILVUS_QUERY_SRCS
+add_source_at_current_directory_recursively()
-        generated/PlanNode.cpp
+add_library(milvus_query OBJECT ${SOURCE_FILES})
-        generated/Expr.cpp
-        visitors/ShowPlanNodeVisitor.cpp
-        visitors/ShowExprVisitor.cpp
-        visitors/ExecPlanNodeVisitor.cpp
-        visitors/ExecExprVisitor.cpp
-        visitors/VerifyPlanNodeVisitor.cpp
-        visitors/VerifyExprVisitor.cpp
-        visitors/ExtractInfoPlanNodeVisitor.cpp
-        visitors/ExtractInfoExprVisitor.cpp
-        Plan.cpp
-        SearchOnGrowing.cpp
-        SearchOnSealed.cpp
-        SearchOnIndex.cpp
-        SearchBruteForce.cpp
-        SubSearchResult.cpp
-        groupby/SearchGroupByOperator.cpp
-        PlanProto.cpp
-        )
-add_library(milvus_query ${MILVUS_QUERY_SRCS})
-
-target_link_libraries(milvus_query milvus_index milvus_bitset milvus_monitor)

internal/core/src/segcore/CMakeLists.txt

@@ -9,40 +9,6 @@
 # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 # or implied. See the License for the specific language governing permissions and limitations under the License
 
-option( EMBEDDED_MILVUS "Enable embedded Milvus" OFF )
 
-if ( EMBEDDED_MILVUS )
+add_source_at_current_directory_recursively()
-    add_compile_definitions( EMBEDDED_MILVUS )
+add_library(milvus_segcore OBJECT ${SOURCE_FILES})
-endif()
-
-milvus_add_pkg_config("milvus_segcore")
-
-set(SEGCORE_FILES
-        Collection.cpp
-        collection_c.cpp
-        segment_c.cpp
-        SegmentGrowingImpl.cpp
-        SegmentSealedImpl.cpp
-        FieldIndexing.cpp
-        metrics_c.cpp
-        plan_c.cpp
-        reduce_c.cpp
-        load_index_c.cpp
-        load_field_data_c.cpp
-        SegmentInterface.cpp
-        SegcoreConfig.cpp
-        IndexConfigGenerator.cpp
-        segcore_init_c.cpp
-        TimestampIndex.cpp
-        Utils.cpp
-        ConcurrentVector.cpp
-        ReduceUtils.cpp
-        check_vec_index_c.cpp
-        reduce/Reduce.cpp
-        reduce/StreamReduce.cpp
-        reduce/GroupReduce.cpp)
-add_library(milvus_segcore SHARED ${SEGCORE_FILES})
-
-target_link_libraries(milvus_segcore milvus_query milvus_bitset milvus_exec ${OpenMP_CXX_FLAGS} milvus_futures)
-
-install(TARGETS milvus_segcore DESTINATION "${CMAKE_INSTALL_LIBDIR}")

internal/core/src/segcore/milvus_segcore.pc.in

@@ -1,9 +0,0 @@
-libdir=@CMAKE_INSTALL_FULL_LIBDIR@
-includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@
-
-Name: Milvus Segcore
-Description: Segcore modules for Milvus
-Version: @MILVUS_VERSION@
-
-Libs: -L${libdir} -lmilvus_segcore
-Cflags: -I${includedir}

internal/core/src/storage/CMakeLists.txt

@@ -14,72 +14,27 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-option(EMBEDDED_MILVUS "Enable embedded Milvus" OFF)
+add_source_at_current_directory()
-
-if(EMBEDDED_MILVUS)
-    add_compile_definitions(EMBEDDED_MILVUS)
-endif()
-
-milvus_add_pkg_config("milvus_storage")
-
 if (DEFINED AZURE_BUILD_DIR)
     add_definitions(-DAZURE_BUILD_DIR)
     include_directories(azure-blob-storage)
     include_directories("${AZURE_BUILD_DIR}/vcpkg_installed/${VCPKG_TARGET_TRIPLET}/include")
-    set(STORAGE_FILES
+    set(SOURCE_FILES ${SOURCE_FILES} azure/AzureChunkManager.cpp)
-        ${STORAGE_FILES}
+    add_library(azure_blob_chunk_manager SHARED IMPORTED)
-        AzureChunkManager.cpp
+    set_target_properties(azure_blob_chunk_manager
-        )
+        PROPERTIES
+        IMPORTED_GLOBAL TRUE
+        IMPORTED_LOCATION "${AZURE_BUILD_DIR}/libblob-chunk-manager${CMAKE_SHARED_LIBRARY_SUFFIX}"
+    )
+    get_target_property(AZURE_IMPORTED_LOCATION azure_blob_chunk_manager IMPORTED_LOCATION)
+    get_target_property(AZURE_INTERFACE_INCLUDE_DIRECTORIES azure_blob_chunk_manager INTERFACE_INCLUDE_DIRECTORIES)
+    message("AZURE_IMPORTED_LOCATION: ${AZURE_IMPORTED_LOCATION}")
+    message("AZURE_INTERFACE_INCLUDE_DIRECTORIES: ${AZURE_INTERFACE_INCLUDE_DIRECTORIES}")
 endif()
 
-set(STORAGE_FILES
-    ${STORAGE_FILES}
-    PayloadStream.cpp
-    DataCodec.cpp
-    Util.cpp
-    PayloadReader.cpp
-    PayloadWriter.cpp
-    BinlogReader.cpp
-    IndexData.cpp
-    InsertData.cpp
-    Event.cpp
-    ThreadPool.cpp
-    storage_c.cpp
-    ChunkManager.cpp
-    MinioChunkManager.cpp
-    AliyunSTSClient.cpp
-    AliyunCredentialsProvider.cpp
-    MemFileManagerImpl.cpp
-    LocalChunkManager.cpp
-    DiskFileManagerImpl.cpp
-    ThreadPools.cpp
-    ChunkCache.cpp
-    TencentCloudCredentialsProvider.cpp
-    TencentCloudSTSClient.cpp
-    MmapChunkManager.cpp)
-
 if(USE_OPENDAL)
-    list(APPEND STORAGE_FILES OpenDALChunkManager.cpp)
+    include_directories(${OPENDAL_INCLUDE_DIR})
+    set(SOURCE_FILES ${SOURCE_FILES} opendal/OpenDALChunkManager.cpp)
 endif()
 
-add_library(milvus_storage SHARED ${STORAGE_FILES})
+add_library(milvus_storage OBJECT ${SOURCE_FILES})
-
-if (DEFINED AZURE_BUILD_DIR)
-    target_link_libraries(milvus_storage PUBLIC
-            "-L${AZURE_BUILD_DIR} -lblob-chunk-manager"
-            blob-chunk-manager
-            milvus_common
-            milvus_monitor
-            pthread
-            ${CONAN_LIBS}
-            )
-else ()
-    target_link_libraries(milvus_storage PUBLIC
-            milvus_common
-            milvus_monitor
-            pthread
-            ${CONAN_LIBS}
-            )
-endif()
-
-install(TARGETS milvus_storage DESTINATION "${CMAKE_INSTALL_LIBDIR}")

internal/core/src/storage/Util.cpp

@@ -27,7 +27,7 @@
 #include "common/FieldData.h"
 #include "common/FieldDataInterface.h"
 #ifdef AZURE_BUILD_DIR
-#include "storage/AzureChunkManager.h"
+#include "storage/azure/AzureChunkManager.h"
 #endif
 #include "storage/ChunkManager.h"
 #include "storage/DiskFileManagerImpl.h"
@@ -36,7 +36,7 @@
 #include "storage/MemFileManagerImpl.h"
 #include "storage/MinioChunkManager.h"
 #ifdef USE_OPENDAL
-#include "storage/OpenDALChunkManager.h"
+#include "storage/opendal/OpenDALChunkManager.h"
 #endif
 #include "storage/Types.h"
 #include "storage/Util.h"

internal/core/src/storage/azure-blob-storage/CMakeLists.txt

@@ -32,10 +32,10 @@ endif()
 
 find_package(azure-storage-blobs-cpp CONFIG REQUIRED)
 find_package(azure-identity-cpp CONFIG REQUIRED)
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter -Wno-return-type -Wno-pedantic")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter -Wno-return-type -Wno-pedantic -fPIC")
 add_library(blob-chunk-manager SHARED AzureBlobChunkManager.cpp)
 target_link_libraries(blob-chunk-manager PUBLIC Azure::azure-identity Azure::azure-storage-blobs)
-
+# should be link directly into libmilvus_core in future.
 install(TARGETS blob-chunk-manager DESTINATION "${CMAKE_INSTALL_LIBDIR}")
 
 if ( BUILD_UNIT_TEST STREQUAL "ON" )

internal/core/src/storage/azure/AzureChunkManager.cpp

@@ -21,7 +21,7 @@
 #include "common/EasyAssert.h"
 #include "log/Log.h"
 #include "monitor/prometheus_client.h"
-#include "storage/AzureChunkManager.h"
+#include "storage/azure/AzureChunkManager.h"
 
 namespace milvus {
 namespace storage {

internal/core/src/storage/milvus_storage.pc.in

@@ -1,10 +0,0 @@
-
-libdir=@CMAKE_INSTALL_FULL_LIBDIR@
-includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@
-
-Name: Milvus Storage
-Description: Storage wrapper for Milvus
-Version: @MILVUS_VERSION@
-
-Libs: -L${libdir} -lmilvus_storage
-Cflags: -I${includedir}

internal/core/src/storage/opendal/OpenDALChunkManager.cpp

@@ -22,7 +22,7 @@
 #include "opendal.h"
 #include "common/EasyAssert.h"
 #include "storage/Util.h"
-#include "storage/OpenDALChunkManager.h"
+#include "storage/opendal/OpenDALChunkManager.h"
 
 namespace milvus::storage {
 

internal/core/thirdparty/knowhere/CMakeLists.txt

@@ -60,3 +60,5 @@ endif()
 # get prometheus COMPILE_OPTIONS
 get_property( var DIRECTORY "${knowhere_SOURCE_DIR}" PROPERTY COMPILE_OPTIONS )
 message( STATUS "knowhere src compile options: ${var}" )
+
+set( KNOWHERE_INCLUDE_DIR ${knowhere_SOURCE_DIR}/include CACHE INTERNAL "Path to knowhere include directory" )

internal/core/thirdparty/opendal/CMakeLists.txt

@@ -14,16 +14,48 @@
 # ----------------------------------------------------------------------
 message(STATUS "Building (vendored) opendal from source")
 
-set(OPENDAL_LIB_DIR "${CMAKE_INSTALL_PREFIX}/lib")
+set(GIT_REPOSITORY  "https://github.com/apache/opendal.git")
-set(OPENDAL_INCLUDE_DIR "${CMAKE_INSTALL_PREFIX}/include")
+set(GIT_TAG "v0.43.0-rc.2")
-set(OPENDAL_NAME "libopendal_c${CMAKE_STATIC_LIBRARY_SUFFIX}")
+
+if (CMAKE_BUILD_TYPE STREQUAL "Debug")
+    set(CARGO_CMD cargo +1.73 build --verbose)
+    set(TARGET_DIR "debug")
+else ()
+    set(CARGO_CMD cargo +1.73 build --release --verbose)
+    set(TARGET_DIR "release")
+endif ()
+
+set(SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/src")
+
+FetchContent_Declare(
+        opendal
+        GIT_REPOSITORY  ${GIT_REPOSITORY}
+        GIT_TAG         ${GIT_TAG}
+        GIT_SHALLOW     TRUE
+        SOURCE_DIR      ${SOURCE_DIR}
+        DOWNLOAD_DIR    ${THIRDPARTY_DOWNLOAD_PATH})
+
+FetchContent_GetProperties(opendal)
+if ( NOT opendal_POPULATED )
+    FetchContent_Populate(opendal)
+endif()
+
+set(OPENDAL_LIB_DIR "${SOURCE_DIR}/target/${TARGET_DIR}" CACHE INTERNAL "opendal lib dir")
+set(OPENDAL_INCLUDE_DIR "${SOURCE_DIR}/bindings/c/include" CACHE INTERNAL "opendal include dir")
+set(OPENDAL_LIB "libopendal_c${CMAKE_STATIC_LIBRARY_SUFFIX}" CACHE INTERNAL "opendal lib")
+
+add_custom_target(build_opendal
+    COMMAND ${CARGO_CMD}
+    WORKING_DIRECTORY ${SOURCE_DIR}/bindings/c
+)
 
 add_library(opendal STATIC IMPORTED)
+add_dependencies(opendal build_opendal)
 set_target_properties(opendal
     PROPERTIES
     IMPORTED_GLOBAL TRUE
-    IMPORTED_LOCATION "${OPENDAL_LIB_DIR}/${OPENDAL_NAME}"
+    IMPORTED_LOCATION "${OPENDAL_LIB_DIR}/${OPENDAL_LIB}"
-    INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_INSTALL_PREFIX}/include")
+    INTERFACE_INCLUDE_DIRECTORIES "${OPENDAL_INCLUDE_DIR}")
 
 get_target_property(OPENDAL_IMPORTED_LOCATION opendal IMPORTED_LOCATION)
 get_target_property(OPENDAL_INTERFACE_INCLUDE_DIRECTORIES opendal INTERFACE_INCLUDE_DIRECTORIES)

internal/core/thirdparty/simdjson/CMakeLists.txt

@@ -17,3 +17,5 @@ FetchContent_Declare(
     URL_HASH MD5=1b0d75ad32179c77f84f4a09d4214057
 )
 FetchContent_MakeAvailable(simdjson)
+
+set( SIMDJSON_INCLUDE_DIR ${simdjson_SOURCE_DIR}/include CACHE INTERNAL "Path to simdjson include directory" )

internal/core/thirdparty/tantivy/CMakeLists.txt

@@ -77,3 +77,5 @@ target_link_libraries(ffi_demo
         tantivy_binding
         dl
         )
+
+set( TANTIVY_INCLUDE_DIR ${LIB_HEADER_FOLDER};${CMAKE_CURRENT_SOURCE_DIR} CACHE INTERNAL "Path to tantivy include directory" )

internal/core/unittest/CMakeLists.txt

@@ -11,6 +11,12 @@
 
 include_directories(${CMAKE_HOME_DIRECTORY}/src)
 include_directories(${CMAKE_HOME_DIRECTORY}/src/thirdparty)
+include_directories(
+    ${KNOWHERE_INCLUDE_DIR}
+    ${SIMDJSON_INCLUDE_DIR}
+    ${TANTIVY_INCLUDE_DIR}
+    ${CONAN_INCLUDE_DIRS}
+)
 
 add_definitions(-DMILVUS_TEST_SEGCORE_YAML_PATH="${CMAKE_SOURCE_DIR}/unittest/test_utils/test_segcore.yaml")
 
@@ -130,13 +136,8 @@ if (LINUX)
 
     target_link_libraries(index_builder_test
             gtest
-            gtest_main
+            milvus_core
-            milvus_monitor
+            knowhere
-            milvus_segcore
-            milvus_storage
-            milvus_indexbuilder
-            milvus_clustering
-            milvus_common
             )
     install(TARGETS index_builder_test DESTINATION unittest)
 endif()
@@ -147,13 +148,8 @@ add_executable(all_tests
 
 target_link_libraries(all_tests
         gtest
-        milvus_segcore
+        milvus_core
-        milvus_storage
+        knowhere
-        milvus_indexbuilder
-        milvus_clustering
-        pthread
-        milvus_common
-        milvus_exec
         )
 
 install(TARGETS all_tests DESTINATION unittest)

internal/core/unittest/bench/CMakeLists.txt

@@ -23,8 +23,8 @@ set(indexbuilder_bench_srcs
 
 add_executable(all_bench ${bench_srcs})
 target_link_libraries(all_bench
-        milvus_segcore
+        milvus_core
-        milvus_log
+        knowhere
         pthread
         )
 
@@ -32,12 +32,9 @@ target_link_libraries(all_bench benchmark_main)
 
 add_executable(indexbuilder_bench ${indexbuilder_bench_srcs})
 target_link_libraries(indexbuilder_bench
-        milvus_segcore
+        milvus_core
-        milvus_indexbuilder
-        milvus_index
-        milvus_log
-        pthread
         knowhere
+        pthread
         )
 
 target_link_libraries(indexbuilder_bench benchmark_main)

internal/core/unittest/test_azure_chunk_manager.cpp

@@ -14,7 +14,7 @@
 #include <vector>
 
 #include "common/EasyAssert.h"
-#include "storage/AzureChunkManager.h"
+#include "storage/azure/AzureChunkManager.h"
 #include "storage/Util.h"
 
 using namespace std;

internal/indexnode/indexnode.go

@@ -17,7 +17,7 @@
 package indexnode
 
 /*
-#cgo pkg-config: milvus_common milvus_indexbuilder milvus_clustering milvus_segcore
+#cgo pkg-config: milvus_core
 
 #include <stdlib.h>
 #include <stdint.h>

internal/proxy/cgo_util.go

@@ -17,7 +17,7 @@
 package proxy
 
 /*
-#cgo pkg-config: milvus_segcore
+#cgo pkg-config: milvus_core
 #include "segcore/check_vec_index_c.h"
 #include <stdlib.h>
 */

internal/querynodev2/segments/cgo_util.go

@@ -17,7 +17,7 @@
 package segments
 
 /*
-#cgo pkg-config: milvus_segcore milvus_storage
+#cgo pkg-config: milvus_core
 
 #include "segcore/collection_c.h"
 #include "common/type_c.h"

internal/querynodev2/segments/collection.go

@@ -17,7 +17,7 @@
 package segments
 
 /*
-#cgo pkg-config: milvus_segcore
+#cgo pkg-config: milvus_core
 
 #include "segcore/collection_c.h"
 #include "segcore/segment_c.h"

internal/querynodev2/segments/index_attr_cache.go

@@ -17,7 +17,7 @@
 package segments
 
 /*
-#cgo pkg-config: milvus_segcore
+#cgo pkg-config: milvus_core
 
 #include "segcore/load_index_c.h"
 */

internal/querynodev2/segments/load_field_data_info.go

@@ -17,7 +17,7 @@
 package segments
 
 /*
-#cgo pkg-config: milvus_segcore
+#cgo pkg-config: milvus_core
 #include "segcore/load_field_data_c.h"
 */
 import "C"

internal/querynodev2/segments/load_index_info.go

@@ -17,7 +17,7 @@
 package segments
 
 /*
-#cgo pkg-config: milvus_common milvus_segcore
+#cgo pkg-config: milvus_core
 
 #include "segcore/load_index_c.h"
 #include "common/binary_set_c.h"

internal/querynodev2/segments/manager.go

@@ -17,7 +17,7 @@
 package segments
 
 /*
-#cgo pkg-config: milvus_segcore
+#cgo pkg-config: milvus_core
 
 #include "segcore/collection_c.h"
 #include "segcore/segment_c.h"

internal/querynodev2/segments/plan.go

@@ -17,7 +17,7 @@
 package segments
 
 /*
-#cgo pkg-config: milvus_segcore
+#cgo pkg-config: milvus_core
 
 #include "segcore/collection_c.h"
 #include "segcore/segment_c.h"

internal/querynodev2/segments/reduce.go

@@ -17,7 +17,7 @@
 package segments
 
 /*
-#cgo pkg-config: milvus_segcore
+#cgo pkg-config: milvus_core
 
 #include "segcore/plan_c.h"
 #include "segcore/reduce_c.h"

internal/querynodev2/segments/segment.go

@@ -17,7 +17,7 @@
 package segments
 
 /*
-#cgo pkg-config: milvus_segcore milvus_futures
+#cgo pkg-config: milvus_core
 
 #include "futures/future_c.h"
 #include "segcore/collection_c.h"

internal/querynodev2/segments/segment_loader.go

@@ -17,7 +17,7 @@
 package segments
 
 /*
-#cgo pkg-config: milvus_segcore
+#cgo pkg-config: milvus_core
 
 #include "segcore/load_index_c.h"
 */

internal/querynodev2/segments/trace.go

@@ -17,7 +17,7 @@
 package segments
 
 /*
-#cgo pkg-config: milvus_segcore
+#cgo pkg-config: milvus_core
 
 #include "segcore/segment_c.h"
 */

internal/querynodev2/segments/utils.go

@@ -1,7 +1,7 @@
 package segments
 
 /*
-#cgo pkg-config: milvus_segcore milvus_common
+#cgo pkg-config: milvus_core
 
 #include "segcore/collection_c.h"
 #include "segcore/segment_c.h"

internal/querynodev2/server.go

@@ -17,7 +17,7 @@
 package querynodev2
 
 /*
-#cgo pkg-config: milvus_segcore milvus_common
+#cgo pkg-config: milvus_core
 
 #include "segcore/collection_c.h"
 #include "segcore/segment_c.h"

internal/util/analyzecgowrapper/analyze.go

@@ -17,7 +17,7 @@
 package analyzecgowrapper
 
 /*
-#cgo pkg-config: milvus_clustering
+#cgo pkg-config: milvus_core
 
 #include <stdlib.h>	// free
 #include "clustering/analyze_c.h"

internal/util/analyzecgowrapper/helper.go

@@ -18,7 +18,7 @@ package analyzecgowrapper
 
 /*
 
-#cgo pkg-config: milvus_common
+#cgo pkg-config: milvus_core
 
 #include <stdlib.h>	// free
 #include "common/type_c.h"

internal/util/cgo/errors.go

@@ -1,7 +1,7 @@
 package cgo
 
 /*
-#cgo pkg-config: milvus_common
+#cgo pkg-config: milvus_core
 
 #include "common/type_c.h"
 #include <stdlib.h>

internal/util/cgo/executor.go

@@ -1,7 +1,7 @@
 package cgo
 
 /*
-#cgo pkg-config: milvus_futures
+#cgo pkg-config: milvus_core
 
 #include "futures/future_c.h"
 */

internal/util/cgo/futures.go

@@ -1,7 +1,7 @@
 package cgo
 
 /*
-#cgo pkg-config: milvus_futures
+#cgo pkg-config: milvus_core
 
 #include "futures/future_c.h"
 #include <stdlib.h>

internal/util/cgo/futures_test_case.go

@@ -4,7 +4,7 @@
 package cgo
 
 /*
-#cgo pkg-config: milvus_futures
+#cgo pkg-config: milvus_core
 
 #include "futures/future_c.h"
 #include <stdlib.h>

internal/util/indexcgowrapper/build_index_info.go

@@ -17,7 +17,7 @@
 package indexcgowrapper
 
 /*
-#cgo pkg-config: milvus_indexbuilder
+#cgo pkg-config: milvus_core
 #include <stdlib.h>	// free
 #include "indexbuilder/index_c.h"
 */

internal/util/indexcgowrapper/helper.go

@@ -1,7 +1,7 @@
 package indexcgowrapper
 
 /*
-#cgo pkg-config: milvus_common milvus_storage
+#cgo pkg-config: milvus_core
 
 #include <stdlib.h>	// free
 #include "common/binary_set_c.h"

internal/util/indexcgowrapper/index.go

@@ -1,7 +1,7 @@
 package indexcgowrapper
 
 /*
-#cgo pkg-config: milvus_indexbuilder
+#cgo pkg-config: milvus_core
 
 #include <stdlib.h>	// free
 #include "indexbuilder/index_c.h"

internal/util/initcore/init_core.go

@@ -17,7 +17,7 @@
 package initcore
 
 /*
-#cgo pkg-config: milvus_common milvus_storage milvus_segcore
+#cgo pkg-config: milvus_core
 
 #include <stdlib.h>
 #include <stdint.h>

internal/util/metrics/c_registry.go

@@ -19,7 +19,7 @@
 package metrics
 
 /*
-#cgo pkg-config: milvus_segcore milvus_storage milvus_common milvus_monitor
+#cgo pkg-config: milvus_core
 
 #include <stdlib.h>
 #include "segcore/metrics_c.h"

scripts/3rdparty_build.sh

@@ -92,15 +92,3 @@ else
     bash -c "curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain=1.73 -y" || { echo 'rustup install failed'; exit 1;}
     source $HOME/.cargo/env
 fi
-
-echo "BUILD_OPENDAL: ${BUILD_OPENDAL}"
-if [ "${BUILD_OPENDAL}" = "ON" ]; then
-    git clone --depth=1 --branch v0.43.0-rc.2 https://github.com/apache/opendal.git opendal
-    cd opendal
-    pushd bindings/c
-    cargo +1.73 build --release --verbose || { echo 'opendal_c build failed'; exit 1; }
-    popd
-    cp target/release/libopendal_c.a ${ROOT_DIR}/internal/core/output/lib/libopendal_c.a
-    cp bindings/c/include/opendal.h ${ROOT_DIR}/internal/core/output/include/opendal.h
-fi
-popd