From 4aed32ff617a87c7a19b5d4cf5d218c11fdae959 Mon Sep 17 00:00:00 2001 From: Enwei Jiao Date: Thu, 13 Jul 2023 15:34:33 +0800 Subject: [PATCH] Use librdkafka for all platform (#25538) Signed-off-by: Enwei Jiao --- .github/workflows/code-checker.yaml | 4 +- Makefile | 19 ++--- internal/core/conanfile.py | 2 + internal/core/thirdparty/CMakeLists.txt | 1 + .../core/thirdparty/rdkafka/CMakeLists.txt | 3 + .../core/thirdparty/rdkafka/rdkafka.pc.in | 9 +++ .../core/thirdparty/rocksdb/CMakeLists.txt | 7 +- .../core/thirdparty/rocksdb/rocksdb-win.pc.in | 9 --- scripts/run_go_codecov.sh | 7 +- scripts/run_go_unittest.sh | 70 +++++++++---------- scripts/run_intergration_test.sh | 5 +- 11 files changed, 59 insertions(+), 77 deletions(-) create mode 100644 internal/core/thirdparty/rdkafka/CMakeLists.txt create mode 100644 internal/core/thirdparty/rdkafka/rdkafka.pc.in delete mode 100644 internal/core/thirdparty/rocksdb/rocksdb-win.pc.in diff --git a/.github/workflows/code-checker.yaml b/.github/workflows/code-checker.yaml index 0fb49a590d..329cc0d779 100644 --- a/.github/workflows/code-checker.yaml +++ b/.github/workflows/code-checker.yaml @@ -31,7 +31,7 @@ jobs: ubuntu: name: ${{ matrix.name }} runs-on: ${{ matrix.os }} - timeout-minutes: 120 + timeout-minutes: 180 strategy: fail-fast: false matrix: @@ -78,7 +78,7 @@ jobs: name: Code Checker CentOS 7 # Run in centos docker runs-on: ubuntu-20.04 - timeout-minutes: 120 + timeout-minutes: 180 steps: - name: Checkout uses: actions/checkout@v2 diff --git a/Makefile b/Makefile index 4cda9a40ef..52e98dd21b 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,6 @@ OBJPREFIX := "github.com/milvus-io/milvus/cmd/milvus" INSTALL_PATH := $(PWD)/bin LIBRARY_PATH := $(PWD)/lib OS := $(shell uname -s) -ARCH := $(shell arch) mode = Release disk_index = OFF useasan = false @@ -34,14 +33,14 @@ milvus: build-cpp print-build-info @source $(PWD)/scripts/setenv.sh && \ mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \ GO111MODULE=on $(GO) build -ldflags="-r $${RPATH} -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \ - ${AARCH64_FLAG} -o $(INSTALL_PATH)/milvus $(PWD)/cmd/main.go 1>/dev/null + -tags dynamic -o $(INSTALL_PATH)/milvus $(PWD)/cmd/main.go 1>/dev/null milvus-gpu: build-cpp-gpu print-gpu-build-info @echo "Building Milvus-gpu ..." @source $(PWD)/scripts/setenv.sh && \ mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \ GO111MODULE=on $(GO) build -ldflags="-r $${RPATH} -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS_GPU)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \ - ${AARCH64_FLAG} -o $(INSTALL_PATH)/milvus $(PWD)/cmd/main.go 1>/dev/null + -tags dynamic -o $(INSTALL_PATH)/milvus $(PWD)/cmd/main.go 1>/dev/null get-build-deps: @(env bash $(PWD)/scripts/install_deps.sh) @@ -100,7 +99,7 @@ meta-migration: @source $(PWD)/scripts/setenv.sh && \ mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \ GO111MODULE=on $(GO) build -ldflags="-r $${RPATH} -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \ - ${AARCH64_FLAG} -o $(INSTALL_PATH)/meta-migration $(MIGRATION_PATH)/main.go 1>/dev/null + -tags dynamic -o $(INSTALL_PATH)/meta-migration $(MIGRATION_PATH)/main.go 1>/dev/null INTERATION_PATH = $(PWD)/tests/integration integration-test: @@ -108,7 +107,7 @@ integration-test: @source $(PWD)/scripts/setenv.sh && \ mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \ GO111MODULE=on $(GO) build -ldflags="-r $${RPATH} -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \ - ${APPLE_SILICON_FLAG} -o $(INSTALL_PATH)/integration-test $(INTERATION_PATH)/ 1>/dev/null + -tags dynamic -o $(INSTALL_PATH)/integration-test $(INTERATION_PATH)/ 1>/dev/null BUILD_TAGS = $(shell git describe --tags --always --dirty="-dev") BUILD_TAGS_GPU = ${BUILD_TAGS}-gpu @@ -116,14 +115,6 @@ BUILD_TIME = $(shell date -u) GIT_COMMIT = $(shell git rev-parse --short HEAD) GO_VERSION = $(shell go version) -ifeq ($(ARCH),arm64) - AARCH64_FLAG = -tags dynamic -endif - -ifeq ($(ARCH),aarch64) - AARCH64_FLAG = -tags dynamic -endif - print-build-info: $(shell git config --global --add safe.directory '*') @echo "Build Tag: $(BUILD_TAGS)" @@ -363,4 +354,4 @@ generate-mockery: getdeps #internal/proxy $(PWD)/bin/mockery --name=LBPolicy --dir=$(PWD)/internal/proxy --output=$(PWD)/internal/proxy --filename=mock_lb_policy.go --structname=MockLBPolicy --with-expecter --outpkg=proxy --inpackage $(PWD)/bin/mockery --name=LBBalancer --dir=$(PWD)/internal/proxy --output=$(PWD)/internal/proxy --filename=mock_lb_balancer.go --structname=MockLBBalancer --with-expecter --outpkg=proxy --inpackage - $(PWD)/bin/mockery --name=shardClientMgr --dir=$(PWD)/internal/proxy --output=$(PWD)/internal/proxy --filename=mock_shardclient_manager.go --structname=MockShardClientManager --with-expecter --outpkg=proxy --inpackage \ No newline at end of file + $(PWD)/bin/mockery --name=shardClientMgr --dir=$(PWD)/internal/proxy --output=$(PWD)/internal/proxy --filename=mock_shardclient_manager.go --structname=MockShardClientManager --with-expecter --outpkg=proxy --inpackage diff --git a/internal/core/conanfile.py b/internal/core/conanfile.py index 396e0c64b0..6264ab4834 100644 --- a/internal/core/conanfile.py +++ b/internal/core/conanfile.py @@ -51,9 +51,11 @@ class MilvusConan(ConanFile): "folly/2023.05.22.02@milvus/dev", "google-cloud-cpp/2.5.0@milvus/dev", "opentelemetry-cpp/1.8.1.1@milvus/dev", + "librdkafka/1.9.1", ) generators = ("cmake", "cmake_find_package") default_options = { + "librdkafka:shared": True, "rocksdb:shared": True, "rocksdb:with_zstd": True, "arrow:parquet": True, diff --git a/internal/core/thirdparty/CMakeLists.txt b/internal/core/thirdparty/CMakeLists.txt index ac516d3659..49e2064aaf 100644 --- a/internal/core/thirdparty/CMakeLists.txt +++ b/internal/core/thirdparty/CMakeLists.txt @@ -42,6 +42,7 @@ add_subdirectory(knowhere) add_subdirectory(boost_ext) add_subdirectory(rocksdb) +add_subdirectory(rdkafka) add_subdirectory(simdjson) if (LINUX) diff --git a/internal/core/thirdparty/rdkafka/CMakeLists.txt b/internal/core/thirdparty/rdkafka/CMakeLists.txt new file mode 100644 index 0000000000..fbec27a0a9 --- /dev/null +++ b/internal/core/thirdparty/rdkafka/CMakeLists.txt @@ -0,0 +1,3 @@ +milvus_add_pkg_config("rdkafka") + +install(DIRECTORY "${CONAN_INCLUDE_DIRS_LIBRDKAFKA}/librdkafka/" DESTINATION "${CMAKE_INSTALL_PREFIX}/include/librdkafka/") diff --git a/internal/core/thirdparty/rdkafka/rdkafka.pc.in b/internal/core/thirdparty/rdkafka/rdkafka.pc.in new file mode 100644 index 0000000000..fffbf6187f --- /dev/null +++ b/internal/core/thirdparty/rdkafka/rdkafka.pc.in @@ -0,0 +1,9 @@ +libdir=@CMAKE_INSTALL_FULL_LIBDIR@ +includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ +Name: Rdkafka +Description: RdKafka +Version: @RDKAFKA_VERSION@ + +Libs: -L${libdir} -lrdkafka +# Libs.private: -lz -lbz2 +Cflags: -I${includedir} diff --git a/internal/core/thirdparty/rocksdb/CMakeLists.txt b/internal/core/thirdparty/rocksdb/CMakeLists.txt index ca1295b79c..d7fd4b03d8 100644 --- a/internal/core/thirdparty/rocksdb/CMakeLists.txt +++ b/internal/core/thirdparty/rocksdb/CMakeLists.txt @@ -1,8 +1,3 @@ -if (MSYS) - configure_file(rocksdb-win.pc.in "${CMAKE_CURRENT_BINARY_DIR}/rocksdb.pc" @ONLY) - install(FILES "${CMAKE_CURRENT_BINARY_DIR}/rocksdb.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") -else() - milvus_add_pkg_config("rocksdb") -endif() +milvus_add_pkg_config("rocksdb") install(DIRECTORY "${CONAN_INCLUDE_DIRS_ROCKSDB}/rocksdb/" DESTINATION "${CMAKE_INSTALL_PREFIX}/include/rocksdb/") diff --git a/internal/core/thirdparty/rocksdb/rocksdb-win.pc.in b/internal/core/thirdparty/rocksdb/rocksdb-win.pc.in deleted file mode 100644 index 5b423589e8..0000000000 --- a/internal/core/thirdparty/rocksdb/rocksdb-win.pc.in +++ /dev/null @@ -1,9 +0,0 @@ -libdir=@CMAKE_INSTALL_FULL_LIBDIR@ -includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ -Name: Rocksdb -Description: Rocksdb -Version: @ROCKSDB_VERSION@ - -Libs: -L${libdir} -lrocksdb -Libs.private: -lz -lbz2 -Cflags: -I${includedir} diff --git a/scripts/run_go_codecov.sh b/scripts/run_go_codecov.sh index 136fb57da8..f6b30d26ff 100755 --- a/scripts/run_go_codecov.sh +++ b/scripts/run_go_codecov.sh @@ -30,18 +30,15 @@ echo "Running unittest under ./internal & ./pkg" # starting the timer beginTime=`date +%s` -if [[ $(uname -s) == "Darwin" && "$(uname -m)" == "arm64" ]]; then - APPLE_SILICON_FLAG="-tags dynamic" -fi for d in $(go list ./internal/... | grep -v -e vendor -e kafka -e planparserv2/generated -e mocks); do - go test -race ${APPLE_SILICON_FLAG} -v -coverpkg=./... -coverprofile=profile.out -covermode=atomic "$d" + go test -race -tags dynamic -v -coverpkg=./... -coverprofile=profile.out -covermode=atomic "$d" if [ -f profile.out ]; then grep -v kafka profile.out | grep -v planparserv2/generated | grep -v mocks | sed '1d' >> ${FILE_COVERAGE_INFO} rm profile.out fi done for d in $(go list ./pkg/... | grep -v -e vendor -e kafka -e planparserv2/generated -e mocks); do - go test -race ${APPLE_SILICON_FLAG} -v -coverpkg=./... -coverprofile=profile.out -covermode=atomic "$d" + go test -race -tags dynamic -v -coverpkg=./... -coverprofile=profile.out -covermode=atomic "$d" if [ -f profile.out ]; then grep -v kafka profile.out | grep -v planparserv2/generated | grep -v mocks | sed '1d' >> ${FILE_COVERAGE_INFO} rm profile.out diff --git a/scripts/run_go_unittest.sh b/scripts/run_go_unittest.sh index b4f879f0c8..cd8bdba946 100755 --- a/scripts/run_go_unittest.sh +++ b/scripts/run_go_unittest.sh @@ -26,10 +26,6 @@ if [[ $(uname -s) == "Darwin" ]]; then export MallocNanoZone=0 fi -if [[ "$(uname -m)" == "arm64" ]]; then - APPLE_SILICON_FLAG="-tags dynamic" -fi - # ignore MinIO,S3 unittests MILVUS_DIR="${ROOT_DIR}/internal/" PKG_DIR="${ROOT_DIR}/pkg/" @@ -64,106 +60,106 @@ done function test_proxy() { -go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/proxy/..." -failfast -count=1 -go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/proxy/..." -failfast -count=1 +go test -race -cover -tags dynamic "${MILVUS_DIR}/proxy/..." -failfast -count=1 +go test -race -cover -tags dynamic "${MILVUS_DIR}/distributed/proxy/..." -failfast -count=1 } function test_querynode() { -go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/querynodev2/..." -failfast -count=1 -go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/querynode/..." -failfast -count=1 +go test -race -cover -tags dynamic "${MILVUS_DIR}/querynodev2/..." -failfast -count=1 +go test -race -cover -tags dynamic "${MILVUS_DIR}/distributed/querynode/..." -failfast -count=1 } function test_kv() { -go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/kv/..." -failfast -count=1 +go test -race -cover -tags dynamic "${MILVUS_DIR}/kv/..." -failfast -count=1 } function test_mq() { -go test -race -cover ${APPLE_SILICON_FLAG} $(go list "${MILVUS_DIR}/mq/..." | grep -v kafka) -failfast -count=1 +go test -race -cover -tags dynamic $(go list "${MILVUS_DIR}/mq/..." | grep -v kafka) -failfast -count=1 } function test_storage() { -go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/storage" -failfast -count=1 +go test -race -cover -tags dynamic "${MILVUS_DIR}/storage" -failfast -count=1 } function test_allocator() { -go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/allocator/..." -failfast -count=1 +go test -race -cover -tags dynamic "${MILVUS_DIR}/allocator/..." -failfast -count=1 } function test_tso() { -go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/tso/..." -failfast -count=1 +go test -race -cover -tags dynamic "${MILVUS_DIR}/tso/..." -failfast -count=1 } function test_config() { -go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/config/..." -failfast -count=1 +go test -race -cover -tags dynamic "${MILVUS_DIR}/config/..." -failfast -count=1 } function test_util() { -go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/util/funcutil/..." -failfast -count=1 -go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/util/paramtable/..." -failfast -count=1 -go test -race -cover ${APPLE_SILICON_FLAG} "${PKG_DIR}/util/retry/..." -failfast -count=1 -go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/util/sessionutil/..." -failfast -count=1 -go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/util/typeutil/..." -failfast -count=1 -go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/util/importutil/..." -failfast -count=1 +go test -race -cover -tags dynamic "${MILVUS_DIR}/util/funcutil/..." -failfast -count=1 +go test -race -cover -tags dynamic "${MILVUS_DIR}/util/paramtable/..." -failfast -count=1 +go test -race -cover -tags dynamic "${PKG_DIR}/util/retry/..." -failfast -count=1 +go test -race -cover -tags dynamic "${MILVUS_DIR}/util/sessionutil/..." -failfast -count=1 +go test -race -cover -tags dynamic "${MILVUS_DIR}/util/typeutil/..." -failfast -count=1 +go test -race -cover -tags dynamic "${MILVUS_DIR}/util/importutil/..." -failfast -count=1 } function test_pkg() { -go test -race -cover ${APPLE_SILICON_FLAG} "${PKG_DIR}/common/..." -failfast -count=1 -go test -race -cover ${APPLE_SILICON_FLAG} "${PKG_DIR}/config/..." -failfast -count=1 -go test -race -cover ${APPLE_SILICON_FLAG} "${PKG_DIR}/log/..." -failfast -count=1 -go test -race -cover ${APPLE_SILICON_FLAG} "${PKG_DIR}/mq/..." -failfast -count=1 -go test -race -cover ${APPLE_SILICON_FLAG} "${PKG_DIR}/tracer/..." -failfast -count=1 -go test -race -cover ${APPLE_SILICON_FLAG} "${PKG_DIR}/util/..." -failfast -count=1 +go test -race -cover -tags dynamic "${PKG_DIR}/common/..." -failfast -count=1 +go test -race -cover -tags dynamic "${PKG_DIR}/config/..." -failfast -count=1 +go test -race -cover -tags dynamic "${PKG_DIR}/log/..." -failfast -count=1 +go test -race -cover -tags dynamic "${PKG_DIR}/mq/..." -failfast -count=1 +go test -race -cover -tags dynamic "${PKG_DIR}/tracer/..." -failfast -count=1 +go test -race -cover -tags dynamic "${PKG_DIR}/util/..." -failfast -count=1 } function test_datanode { -go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/datanode/..." -failfast -count=1 -go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/datanode/..." -failfast -count=1 +go test -race -cover -tags dynamic "${MILVUS_DIR}/datanode/..." -failfast -count=1 +go test -race -cover -tags dynamic "${MILVUS_DIR}/distributed/datanode/..." -failfast -count=1 } function test_indexnode() { -go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/indexnode/..." -failfast -count=1 +go test -race -cover -tags dynamic "${MILVUS_DIR}/indexnode/..." -failfast -count=1 } function test_rootcoord() { -go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/rootcoord/..." -failfast -count=1 -go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/rootcoord" -failfast +go test -race -cover -tags dynamic "${MILVUS_DIR}/distributed/rootcoord/..." -failfast -count=1 +go test -race -cover -tags dynamic "${MILVUS_DIR}/rootcoord" -failfast } function test_datacoord() { -go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/datacoord/..." -failfast -count=1 -go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/datacoord/..." -failfast -count=1 +go test -race -cover -tags dynamic "${MILVUS_DIR}/distributed/datacoord/..." -failfast -count=1 +go test -race -cover -tags dynamic "${MILVUS_DIR}/datacoord/..." -failfast -count=1 } function test_querycoord() { -go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/querycoord/..." -failfast -count=1 -go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/querycoordv2/..." -failfast -count=1 +go test -race -cover -tags dynamic "${MILVUS_DIR}/distributed/querycoord/..." -failfast -count=1 +go test -race -cover -tags dynamic "${MILVUS_DIR}/querycoordv2/..." -failfast -count=1 } #function test_indexcoord() #{ -#go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/indexcoord/..." -failfast +#go test -race -cover -tags dynamic "${MILVUS_DIR}/indexcoord/..." -failfast #} function test_metastore() { -go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/metastore/..." -failfast -count=1 +go test -race -cover -tags dynamic "${MILVUS_DIR}/metastore/..." -failfast -count=1 } function test_all() diff --git a/scripts/run_intergration_test.sh b/scripts/run_intergration_test.sh index a8472c566d..704b41cf4c 100755 --- a/scripts/run_intergration_test.sh +++ b/scripts/run_intergration_test.sh @@ -28,13 +28,10 @@ echo "mode: atomic" > ${FILE_COVERAGE_INFO} # starting the timer beginTime=`date +%s` -if [[ $(uname -s) == "Darwin" && "$(uname -m)" == "arm64" ]]; then - APPLE_SILICON_FLAG="-tags dynamic" -fi for d in $(go list ./tests/integration/...); do echo "$d" - go test -race ${APPLE_SILICON_FLAG} -v -coverpkg=./... -coverprofile=profile.out -covermode=atomic "$d" -timeout=20m + go test -race -tags dynamic -v -coverpkg=./... -coverprofile=profile.out -covermode=atomic "$d" -timeout=20m if [ -f profile.out ]; then grep -v kafka profile.out | grep -v planparserv2/generated | grep -v mocks | sed '1d' >> ${FILE_COVERAGE_INFO} rm profile.out