Use librdkafka for all platform (#25538)

Signed-off-by: Enwei Jiao <enwei.jiao@zilliz.com>
2024-11-29 18:38:44 +08:00 · 2023-07-13 15:34:33 +08:00 · 2023-07-13 15:34:33 +08:00 · 4aed32ff61
commit 4aed32ff61
parent 50454bec4c
11 changed files with 59 additions and 77 deletions
--- a/.github/workflows/code-checker.yaml
+++ b/.github/workflows/code-checker.yaml
@ -31,7 +31,7 @@ jobs:
  ubuntu:
    name: ${{ matrix.name }}
    runs-on: ${{ matrix.os }}
-    timeout-minutes: 120
+    timeout-minutes: 180
    strategy:
      fail-fast: false
      matrix:
@ -78,7 +78,7 @@ jobs:
    name: Code Checker CentOS 7
    # Run in centos docker
    runs-on: ubuntu-20.04
-    timeout-minutes: 120
+    timeout-minutes: 180
    steps:
      - name: Checkout
        uses: actions/checkout@v2
--- a/19
+++ b/19
@ -18,7 +18,6 @@ OBJPREFIX := "github.com/milvus-io/milvus/cmd/milvus"
 INSTALL_PATH := $(PWD)/bin
 LIBRARY_PATH := $(PWD)/lib
 OS := $(shell uname -s)
-ARCH := $(shell arch)
 mode = Release
 disk_index = OFF
 useasan = false
@ -34,14 +33,14 @@ milvus: build-cpp print-build-info
 	@source $(PWD)/scripts/setenv.sh && \
 		mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \
 		GO111MODULE=on $(GO) build -ldflags="-r $${RPATH} -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \
-		${AARCH64_FLAG} -o $(INSTALL_PATH)/milvus $(PWD)/cmd/main.go 1>/dev/null
+		-tags dynamic -o $(INSTALL_PATH)/milvus $(PWD)/cmd/main.go 1>/dev/null

 milvus-gpu: build-cpp-gpu print-gpu-build-info
 	@echo "Building Milvus-gpu ..."
 	@source $(PWD)/scripts/setenv.sh && \
 		mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \
 		GO111MODULE=on $(GO) build -ldflags="-r $${RPATH} -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS_GPU)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \
-		${AARCH64_FLAG} -o $(INSTALL_PATH)/milvus $(PWD)/cmd/main.go 1>/dev/null
+		-tags dynamic -o $(INSTALL_PATH)/milvus $(PWD)/cmd/main.go 1>/dev/null

 get-build-deps:
 	@(env bash $(PWD)/scripts/install_deps.sh)
@ -100,7 +99,7 @@ meta-migration:
 	@source $(PWD)/scripts/setenv.sh && \
    		mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \
    		GO111MODULE=on $(GO) build -ldflags="-r $${RPATH} -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \
-    		${AARCH64_FLAG} -o $(INSTALL_PATH)/meta-migration $(MIGRATION_PATH)/main.go 1>/dev/null
+    		-tags dynamic -o $(INSTALL_PATH)/meta-migration $(MIGRATION_PATH)/main.go 1>/dev/null

 INTERATION_PATH = $(PWD)/tests/integration
 integration-test:
@ -108,7 +107,7 @@ integration-test:
 	@source $(PWD)/scripts/setenv.sh && \
    		mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \
    		GO111MODULE=on $(GO) build -ldflags="-r $${RPATH} -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \
-    		${APPLE_SILICON_FLAG} -o $(INSTALL_PATH)/integration-test $(INTERATION_PATH)/ 1>/dev/null
+    		-tags dynamic -o $(INSTALL_PATH)/integration-test $(INTERATION_PATH)/ 1>/dev/null

 BUILD_TAGS = $(shell git describe --tags --always --dirty="-dev")
 BUILD_TAGS_GPU = ${BUILD_TAGS}-gpu
@ -116,14 +115,6 @@ BUILD_TIME = $(shell date -u)
 GIT_COMMIT = $(shell git rev-parse --short HEAD)
 GO_VERSION = $(shell go version)

-ifeq ($(ARCH),arm64)
-	AARCH64_FLAG = -tags dynamic
-endif
-
-ifeq ($(ARCH),aarch64)
-	AARCH64_FLAG = -tags dynamic
-endif
-
 print-build-info:
 	$(shell git config --global --add safe.directory '*')
 	@echo "Build Tag: $(BUILD_TAGS)"
@ -363,4 +354,4 @@ generate-mockery: getdeps
 	#internal/proxy
 	$(PWD)/bin/mockery --name=LBPolicy --dir=$(PWD)/internal/proxy --output=$(PWD)/internal/proxy --filename=mock_lb_policy.go --structname=MockLBPolicy --with-expecter --outpkg=proxy --inpackage
 	$(PWD)/bin/mockery --name=LBBalancer --dir=$(PWD)/internal/proxy --output=$(PWD)/internal/proxy --filename=mock_lb_balancer.go --structname=MockLBBalancer --with-expecter --outpkg=proxy  --inpackage
-	$(PWD)/bin/mockery --name=shardClientMgr --dir=$(PWD)/internal/proxy --output=$(PWD)/internal/proxy --filename=mock_shardclient_manager.go --structname=MockShardClientManager --with-expecter --outpkg=proxy  --inpackage
+	$(PWD)/bin/mockery --name=shardClientMgr --dir=$(PWD)/internal/proxy --output=$(PWD)/internal/proxy --filename=mock_shardclient_manager.go --structname=MockShardClientManager --with-expecter --outpkg=proxy  --inpackage
--- a/internal/core/conanfile.py
+++ b/internal/core/conanfile.py
@ -51,9 +51,11 @@ class MilvusConan(ConanFile):
        "folly/2023.05.22.02@milvus/dev",
        "google-cloud-cpp/2.5.0@milvus/dev",
        "opentelemetry-cpp/1.8.1.1@milvus/dev",
+        "librdkafka/1.9.1",
    )
    generators = ("cmake", "cmake_find_package")
    default_options = {
+        "librdkafka:shared": True,
        "rocksdb:shared": True,
        "rocksdb:with_zstd": True,
        "arrow:parquet": True,
--- a/internal/core/thirdparty/CMakeLists.txt
+++ b/internal/core/thirdparty/CMakeLists.txt
@ -42,6 +42,7 @@ add_subdirectory(knowhere)

 add_subdirectory(boost_ext)
 add_subdirectory(rocksdb)
+add_subdirectory(rdkafka)
 add_subdirectory(simdjson)

 if (LINUX)
--- a/internal/core/thirdparty/rdkafka/CMakeLists.txt
+++ b/internal/core/thirdparty/rdkafka/CMakeLists.txt
@ -0,0 +1,3 @@
+milvus_add_pkg_config("rdkafka")
+
+install(DIRECTORY "${CONAN_INCLUDE_DIRS_LIBRDKAFKA}/librdkafka/" DESTINATION "${CMAKE_INSTALL_PREFIX}/include/librdkafka/")
--- a/internal/core/thirdparty/rdkafka/rdkafka.pc.in
+++ b/internal/core/thirdparty/rdkafka/rdkafka.pc.in
@ -0,0 +1,9 @@
+libdir=@CMAKE_INSTALL_FULL_LIBDIR@
+includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@
+Name: Rdkafka
+Description: RdKafka
+Version: @RDKAFKA_VERSION@
+
+Libs: -L${libdir} -lrdkafka
+# Libs.private: -lz -lbz2
+Cflags: -I${includedir}
--- a/internal/core/thirdparty/rocksdb/CMakeLists.txt
+++ b/internal/core/thirdparty/rocksdb/CMakeLists.txt
@ -1,8 +1,3 @@
-if (MSYS)
-  configure_file(rocksdb-win.pc.in "${CMAKE_CURRENT_BINARY_DIR}/rocksdb.pc" @ONLY)
-  install(FILES "${CMAKE_CURRENT_BINARY_DIR}/rocksdb.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
-else()
-  milvus_add_pkg_config("rocksdb")
-endif()
+milvus_add_pkg_config("rocksdb")

 install(DIRECTORY "${CONAN_INCLUDE_DIRS_ROCKSDB}/rocksdb/" DESTINATION "${CMAKE_INSTALL_PREFIX}/include/rocksdb/")
--- a/internal/core/thirdparty/rocksdb/rocksdb-win.pc.in
+++ b/internal/core/thirdparty/rocksdb/rocksdb-win.pc.in
@ -1,9 +0,0 @@
-libdir=@CMAKE_INSTALL_FULL_LIBDIR@
-includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@
-Name: Rocksdb
-Description: Rocksdb
-Version: @ROCKSDB_VERSION@
-
-Libs: -L${libdir} -lrocksdb
-Libs.private: -lz -lbz2 
-Cflags: -I${includedir}
--- a/scripts/run_go_codecov.sh
+++ b/scripts/run_go_codecov.sh
@ -30,18 +30,15 @@ echo "Running unittest under ./internal & ./pkg"

 # starting the timer
 beginTime=`date +%s`
-if [[ $(uname -s) == "Darwin" && "$(uname -m)" == "arm64" ]]; then
-    APPLE_SILICON_FLAG="-tags dynamic"
-fi
 for d in $(go list ./internal/... | grep -v -e vendor -e kafka -e planparserv2/generated -e mocks); do
-    go test -race ${APPLE_SILICON_FLAG} -v -coverpkg=./... -coverprofile=profile.out -covermode=atomic "$d"
+    go test -race -tags dynamic -v -coverpkg=./... -coverprofile=profile.out -covermode=atomic "$d"
    if [ -f profile.out ]; then
        grep -v kafka profile.out | grep -v planparserv2/generated | grep -v mocks | sed '1d' >> ${FILE_COVERAGE_INFO}
        rm profile.out
    fi
 done
 for d in $(go list ./pkg/... | grep -v -e vendor -e kafka -e planparserv2/generated -e mocks); do
-    go test -race ${APPLE_SILICON_FLAG} -v -coverpkg=./... -coverprofile=profile.out -covermode=atomic "$d"
+    go test -race -tags dynamic -v -coverpkg=./... -coverprofile=profile.out -covermode=atomic "$d"
    if [ -f profile.out ]; then
        grep -v kafka profile.out | grep -v planparserv2/generated | grep -v mocks | sed '1d' >> ${FILE_COVERAGE_INFO}
        rm profile.out
--- a/scripts/run_go_unittest.sh
+++ b/scripts/run_go_unittest.sh
@ -26,10 +26,6 @@ if [[ $(uname -s) == "Darwin" ]]; then
    export MallocNanoZone=0
 fi

-if [[ "$(uname -m)" == "arm64" ]]; then
-  APPLE_SILICON_FLAG="-tags dynamic"
-fi
-
 # ignore MinIO,S3 unittests
 MILVUS_DIR="${ROOT_DIR}/internal/"
 PKG_DIR="${ROOT_DIR}/pkg/"
@ -64,106 +60,106 @@ done

 function test_proxy()
 {
-go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/proxy/..." -failfast -count=1 
-go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/proxy/..." -failfast -count=1  
+go test -race -cover -tags dynamic "${MILVUS_DIR}/proxy/..." -failfast -count=1
+go test -race -cover -tags dynamic "${MILVUS_DIR}/distributed/proxy/..." -failfast -count=1
 }

 function test_querynode()
 {
-go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/querynodev2/..." -failfast -count=1
-go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/querynode/..." -failfast -count=1
+go test -race -cover -tags dynamic "${MILVUS_DIR}/querynodev2/..." -failfast -count=1
+go test -race -cover -tags dynamic "${MILVUS_DIR}/distributed/querynode/..." -failfast -count=1
 }


 function test_kv()
 {
-go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/kv/..." -failfast -count=1
+go test -race -cover -tags dynamic "${MILVUS_DIR}/kv/..." -failfast -count=1
 }

 function test_mq()
 {
-go test -race -cover ${APPLE_SILICON_FLAG} $(go list "${MILVUS_DIR}/mq/..." | grep -v kafka)  -failfast -count=1
+go test -race -cover -tags dynamic $(go list "${MILVUS_DIR}/mq/..." | grep -v kafka)  -failfast -count=1
 }

 function test_storage()
 {
-go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/storage" -failfast -count=1
+go test -race -cover -tags dynamic "${MILVUS_DIR}/storage" -failfast -count=1
 }

 function test_allocator()
 {
-go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/allocator/..." -failfast -count=1
+go test -race -cover -tags dynamic "${MILVUS_DIR}/allocator/..." -failfast -count=1
 }

 function test_tso()
 {
-go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/tso/..." -failfast -count=1
+go test -race -cover -tags dynamic "${MILVUS_DIR}/tso/..." -failfast -count=1
 }

 function test_config()
 {
-go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/config/..." -failfast -count=1
+go test -race -cover -tags dynamic "${MILVUS_DIR}/config/..." -failfast -count=1
 }

 function test_util()
 {
-go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/util/funcutil/..." -failfast -count=1
-go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/util/paramtable/..." -failfast -count=1
-go test -race -cover ${APPLE_SILICON_FLAG} "${PKG_DIR}/util/retry/..." -failfast -count=1
-go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/util/sessionutil/..." -failfast -count=1
-go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/util/typeutil/..." -failfast -count=1
-go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/util/importutil/..." -failfast -count=1
+go test -race -cover -tags dynamic "${MILVUS_DIR}/util/funcutil/..." -failfast -count=1
+go test -race -cover -tags dynamic "${MILVUS_DIR}/util/paramtable/..." -failfast -count=1
+go test -race -cover -tags dynamic "${PKG_DIR}/util/retry/..." -failfast -count=1
+go test -race -cover -tags dynamic "${MILVUS_DIR}/util/sessionutil/..." -failfast -count=1
+go test -race -cover -tags dynamic "${MILVUS_DIR}/util/typeutil/..." -failfast -count=1
+go test -race -cover -tags dynamic "${MILVUS_DIR}/util/importutil/..." -failfast -count=1
 }

 function test_pkg()
 {
-go test -race -cover ${APPLE_SILICON_FLAG} "${PKG_DIR}/common/..." -failfast -count=1
-go test -race -cover ${APPLE_SILICON_FLAG} "${PKG_DIR}/config/..." -failfast -count=1
-go test -race -cover ${APPLE_SILICON_FLAG} "${PKG_DIR}/log/..." -failfast -count=1
-go test -race -cover ${APPLE_SILICON_FLAG} "${PKG_DIR}/mq/..." -failfast -count=1
-go test -race -cover ${APPLE_SILICON_FLAG} "${PKG_DIR}/tracer/..." -failfast -count=1
-go test -race -cover ${APPLE_SILICON_FLAG} "${PKG_DIR}/util/..." -failfast -count=1
+go test -race -cover -tags dynamic "${PKG_DIR}/common/..." -failfast -count=1
+go test -race -cover -tags dynamic "${PKG_DIR}/config/..." -failfast -count=1
+go test -race -cover -tags dynamic "${PKG_DIR}/log/..." -failfast -count=1
+go test -race -cover -tags dynamic "${PKG_DIR}/mq/..." -failfast -count=1
+go test -race -cover -tags dynamic "${PKG_DIR}/tracer/..." -failfast -count=1
+go test -race -cover -tags dynamic "${PKG_DIR}/util/..." -failfast -count=1
 }

 function test_datanode
 {

-go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/datanode/..." -failfast -count=1
-go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/datanode/..." -failfast -count=1
+go test -race -cover -tags dynamic "${MILVUS_DIR}/datanode/..." -failfast -count=1
+go test -race -cover -tags dynamic "${MILVUS_DIR}/distributed/datanode/..." -failfast -count=1

 }

 function test_indexnode()
 {
-go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/indexnode/..." -failfast -count=1
+go test -race -cover -tags dynamic "${MILVUS_DIR}/indexnode/..." -failfast -count=1
 }

 function test_rootcoord()
 {
-go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/rootcoord/..." -failfast -count=1
-go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/rootcoord" -failfast
+go test -race -cover -tags dynamic "${MILVUS_DIR}/distributed/rootcoord/..." -failfast -count=1
+go test -race -cover -tags dynamic "${MILVUS_DIR}/rootcoord" -failfast
 }

 function test_datacoord()
 {
-go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/datacoord/..." -failfast -count=1
-go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/datacoord/..." -failfast -count=1
+go test -race -cover -tags dynamic "${MILVUS_DIR}/distributed/datacoord/..." -failfast -count=1
+go test -race -cover -tags dynamic "${MILVUS_DIR}/datacoord/..." -failfast -count=1
 }

 function test_querycoord()
 {
-go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/querycoord/..." -failfast -count=1
-go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/querycoordv2/..." -failfast -count=1
+go test -race -cover -tags dynamic "${MILVUS_DIR}/distributed/querycoord/..." -failfast -count=1
+go test -race -cover -tags dynamic "${MILVUS_DIR}/querycoordv2/..." -failfast -count=1
 }

 #function test_indexcoord()
 #{
-#go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/indexcoord/..." -failfast
+#go test -race -cover -tags dynamic "${MILVUS_DIR}/indexcoord/..." -failfast
 #}

 function test_metastore()
 {
-go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/metastore/..." -failfast -count=1
+go test -race -cover -tags dynamic "${MILVUS_DIR}/metastore/..." -failfast -count=1
 }

 function test_all()
--- a/scripts/run_intergration_test.sh
+++ b/scripts/run_intergration_test.sh
@ -28,13 +28,10 @@ echo "mode: atomic" > ${FILE_COVERAGE_INFO}

 # starting the timer
 beginTime=`date +%s`
-if [[ $(uname -s) == "Darwin" && "$(uname -m)" == "arm64" ]]; then
-    APPLE_SILICON_FLAG="-tags dynamic"
-fi

 for d in $(go list ./tests/integration/...); do
    echo "$d"
-    go test -race ${APPLE_SILICON_FLAG} -v -coverpkg=./... -coverprofile=profile.out -covermode=atomic "$d" -timeout=20m
+    go test -race -tags dynamic -v -coverpkg=./... -coverprofile=profile.out -covermode=atomic "$d" -timeout=20m
    if [ -f profile.out ]; then
        grep -v kafka profile.out | grep -v planparserv2/generated | grep -v mocks | sed '1d' >> ${FILE_COVERAGE_INFO}
        rm profile.out