Simplify makefile (#23583)

Signed-off-by: Enwei Jiao <enwei.jiao@zilliz.com>

.github/workflows/main.yaml

@@ -88,7 +88,7 @@ jobs:
           restore-keys: ubuntu${{ matrix.ubuntu }}-conan-
       - name: Build
         run: |
-          ./build/builder.sh /bin/bash -c "make USE_ASAN=${{env.useasan}} build-cpp-with-coverage generated-proto-go-without-cpp"
+          ./build/builder.sh /bin/bash -c "make USE_ASAN=${{env.useasan}} build-cpp-with-coverage"
       - run: |
           zip -r code.zip . -x "./.docker/*"
       - name: Archive code

Makefile

@@ -21,9 +21,9 @@ OS := $(shell uname -s)
 ARCH := $(shell arch)
 mode = Release
 disk_index = OFF
-useasan = false 
+useasan = false
 ifeq (${USE_ASAN}, true)
-useasan = true 
+useasan = true
 endif
 
 export GIT_BRANCH=master
@@ -58,19 +58,6 @@ tools/bin/revive: tools/check/go.mod
 cppcheck:
 	@(env bash ${PWD}/scripts/core_build.sh -l)
 
-# put generate proto as a separated target because build cpp have different cases like with unittest.
-generated-proto-go-without-cpp: export protoc:=${PWD}/cmake_build/bin/protoc
-generated-proto-go-without-cpp: 
-	@mkdir -p ${GOPATH}/bin
-	@which protoc-gen-go 1>/dev/null || (echo "Installing protoc-gen-go" && cd /tmp && go install github.com/golang/protobuf/protoc-gen-go@v1.3.2)
-	@(env bash $(PWD)/scripts/proto_gen_go.sh)
-
-generated-proto-go: build-cpp generated-proto-go-without-cpp
-
-check-proto-product-only:
-	 @(env bash $(PWD)/scripts/check_proto_product.sh)
-check-proto-product: generated-proto-go check-proto-product-only
-	
 
 fmt:
 ifdef GO_DIFF_FILES
@@ -150,13 +137,6 @@ print-gpu-build-info:
 	@echo "Git Commit: $(GIT_COMMIT)"
 	@echo "Go Version: $(GO_VERSION)"
 
-embd-milvus: build-cpp-embd print-build-info
-	@echo "Building **Embedded** Milvus ..."
-	@source $(PWD)/scripts/setenv.sh && \
-		mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \
-		GO111MODULE=on $(GO) build -ldflags="-r /tmp/milvus/lib/ -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \
-		${AARCH64_FLAG} -buildmode=c-shared -o $(INSTALL_PATH)/embd-milvus.so $(PWD)/pkg/embedded/embedded.go 1>/dev/null
-
 update-milvus-api: download-milvus-proto
 	@echo "Update milvus/api version ..."
 	@(env bash $(PWD)/scripts/update-api-version.sh)
@@ -165,26 +145,35 @@ download-milvus-proto:
 	@echo "Download milvus-proto repo ..."
 	@(env bash $(PWD)/scripts/download_milvus_proto.sh)
 
-build-cpp: download-milvus-proto
+build-3rdparty:
+	@echo "Build 3rdparty ..."
+	@(env bash $(PWD)/scripts/3rdparty_build.sh)
+
+generated-proto: download-milvus-proto build-3rdparty
+	@echo "Generate proto ..."
+	@mkdir -p ${GOPATH}/bin
+	@which protoc-gen-go 1>/dev/null || (echo "Installing protoc-gen-go" && cd /tmp && go install github.com/golang/protobuf/protoc-gen-go@v1.3.2)
+	@(env bash $(PWD)/scripts/generate_proto.sh)
+
+build-cpp: generated-proto
 	@echo "Building Milvus cpp library ..."
 	@(env bash $(PWD)/scripts/core_build.sh -t ${mode} -f "$(CUSTOM_THIRDPARTY_PATH)" -n ${disk_index})
 
-build-cpp-gpu: download-milvus-proto
+build-cpp-gpu: generated-proto
 	@echo "Building Milvus cpp gpu library ..."
 	@(env bash $(PWD)/scripts/core_build.sh -t ${mode} -g -f "$(CUSTOM_THIRDPARTY_PATH)" -n ${disk_index})
 
-build-cpp-embd: download-milvus-proto
+build-cpp-with-unittest: generated-proto
-	@echo "Building **Embedded** Milvus cpp library ..."
-	@(env bash $(PWD)/scripts/core_build.sh -b -t ${mode} -f "$(CUSTOM_THIRDPARTY_PATH)" -n ${disk_index})
-
-build-cpp-with-unittest: download-milvus-proto
 	@echo "Building Milvus cpp library with unittest ..."
 	@(env bash $(PWD)/scripts/core_build.sh -t ${mode} -u -f "$(CUSTOM_THIRDPARTY_PATH)" -n ${disk_index})
 
-build-cpp-with-coverage: download-milvus-proto
+build-cpp-with-coverage: generated-proto
 	@echo "Building Milvus cpp library with coverage and unittest ..."
 	@(env bash $(PWD)/scripts/core_build.sh -t ${mode} -u -a ${useasan} -c -f "$(CUSTOM_THIRDPARTY_PATH)" -n ${disk_index})
 
+check-proto-product: generated-proto
+	 @(env bash $(PWD)/scripts/check_proto_product.sh)
+
 
 # Run the tests.
 unittest: test-cpp test-go
@@ -274,11 +263,6 @@ codecov-cpp: build-cpp-with-coverage
 	@echo "Running cpp coverage..."
 	@(env bash $(PWD)/scripts/run_cpp_codecov.sh)
 
-# Package docker image locally.
-# TODO: fix error occur at starting up
-docker: install
-	./build/build_image.sh
-
 # Build each component and install binary to $GOPATH/bin.
 install: milvus
 	@echo "Installing binary to './bin'"
@@ -298,7 +282,6 @@ gpu-install: milvus-gpu
 	-cp -r -P $(PWD)/internal/core/output/lib64/*.so* $(LIBRARY_PATH) 2>/dev/null
 	@echo "Installation successful."
 
-
 clean:
 	@echo "Cleaning up all the generated files"
 	@find . -name '*.test' | xargs rm -fv
@@ -351,10 +334,10 @@ generate-mockery: getdeps
 	# internal/datanode
 	$(PWD)/bin/mockery --name=Allocator --dir=$(PWD)/internal/datanode/allocator/ --output=$(PWD)/internal/datanode/allocator --filename=mock_allocator.go --with-expecter --structname=MockAllocator  --outpkg=allocator --inpackage
 	# internal/querycoordv2
-	$(PWD)/bin/mockery --name=QueryNodeServer --dir=$(PWD)/internal/proto/querypb/ --output=$(PWD)/internal/querycoordv2/mocks --filename=mock_querynode.go --with-expecter --structname=MockQueryNodeServer 
+	$(PWD)/bin/mockery --name=QueryNodeServer --dir=$(PWD)/internal/proto/querypb/ --output=$(PWD)/internal/querycoordv2/mocks --filename=mock_querynode.go --with-expecter --structname=MockQueryNodeServer
-	$(PWD)/bin/mockery --name=Broker --dir=$(PWD)/internal/querycoordv2/meta --output=$(PWD)/internal/querycoordv2/meta --filename=mock_broker.go --with-expecter --structname=MockBroker --outpkg=meta 
+	$(PWD)/bin/mockery --name=Broker --dir=$(PWD)/internal/querycoordv2/meta --output=$(PWD)/internal/querycoordv2/meta --filename=mock_broker.go --with-expecter --structname=MockBroker --outpkg=meta
-	$(PWD)/bin/mockery --name=Scheduler --dir=$(PWD)/internal/querycoordv2/task --output=$(PWD)/internal/querycoordv2/task --filename=mock_scheduler.go --with-expecter --structname=MockScheduler --outpkg=task --inpackage 
+	$(PWD)/bin/mockery --name=Scheduler --dir=$(PWD)/internal/querycoordv2/task --output=$(PWD)/internal/querycoordv2/task --filename=mock_scheduler.go --with-expecter --structname=MockScheduler --outpkg=task --inpackage
-	$(PWD)/bin/mockery --name=Cluster --dir=$(PWD)/internal/querycoordv2/session --output=$(PWD)/internal/querycoordv2/session --filename=mock_cluster.go --with-expecter --structname=MockCluster --outpkg=session --inpackage 
+	$(PWD)/bin/mockery --name=Cluster --dir=$(PWD)/internal/querycoordv2/session --output=$(PWD)/internal/querycoordv2/session --filename=mock_cluster.go --with-expecter --structname=MockCluster --outpkg=session --inpackage
 	$(PWD)/bin/mockery --name=Store --dir=$(PWD)/internal/querycoordv2/meta --output=$(PWD)/internal/querycoordv2/meta --filename=mock_store.go --with-expecter --structname=MockStore --outpkg=meta --inpackage
 	$(PWD)/bin/mockery --name=Balance --dir=$(PWD)/internal/querycoordv2/balance --output=$(PWD)/internal/querycoordv2/balance --filename=mock_balancer.go --with-expecter --structname=MockBalancer --outpkg=balance --inpackage
 	$(PWD)/bin/mockery --name=Controller --dir=$(PWD)/internal/querycoordv2/dist --output=$(PWD)/internal/querycoordv2/dist --filename=mock_controller.go --with-expecter --structname=MockController --outpkg=dist --inpackage
@@ -370,7 +353,5 @@ generate-mockery: getdeps
 	$(PWD)/bin/mockery --name=Manager --dir=$(PWD)/internal/querynodev2/cluster --output=$(PWD)/internal/querynodev2/cluster --filename=mock_manager.go --with-expecter --outpkg=cluster --structname=MockManager --inpackage
 	$(PWD)/bin/mockery --name=Loader --dir=$(PWD)/internal/querynodev2/segments --output=$(PWD)/internal/querynodev2/segments --filename=mock_loader.go --with-expecter --outpkg=segments --structname=MockLoader --inpackage
 	$(PWD)/bin/mockery --name=Worker --dir=$(PWD)/internal/querynodev2/cluster --output=$(PWD)/internal/querynodev2/cluster --filename=mock_worker.go --with-expecter --outpkg=worker --structname=MockWorker --inpackage
-	$(PWD)/bin/mockery --name=ShardDelegator --dir=$(PWD)/internal/querynodev2/delegator/ --output=$(PWD)/internal/querynodev2/delegator/ --filename=mock_delegator.go --with-expecter --outpkg=delegator --structname=MockShardDelegator --inpackage        
+	$(PWD)/bin/mockery --name=ShardDelegator --dir=$(PWD)/internal/querynodev2/delegator/ --output=$(PWD)/internal/querynodev2/delegator/ --filename=mock_delegator.go --with-expecter --outpkg=delegator --structname=MockShardDelegator --inpackage
-
-ci-ut: build-cpp-with-coverage generated-proto-go-without-cpp codecov-cpp codecov-go
 

internal/core/src/pb/CMakeLists.txt

@@ -10,51 +10,14 @@
 # or implied. See the License for the specific language governing permissions and limitations under the License
 
 find_package(Protobuf REQUIRED)
-set( proto_file_names
+
-    schema.proto
+add_library(milvus_proto STATIC
-    common.proto
+    common.pb.cc
+    index_cgo_msg.pb.cc
+    plan.pb.cc
+    schema.pb.cc
+    segcore.pb.cc
     )
-
-set( PROTO_PATH  "${MILVUS_SOURCE_DIR}/../../cmake_build/thirdparty/milvus-proto/proto/")
-
-FOREACH(proto_file ${proto_file_names})
-    STRING( REGEX REPLACE "[^/]proto" "" proto_src_file_name ${proto_file} )
-    LIST( APPEND milvus_proto_srcs "${proto_src_file_name}.pb.cc" )
-
-    ADD_CUSTOM_COMMAND(
-        OUTPUT  "${CMAKE_CURRENT_SOURCE_DIR}/${proto_src_file_name}.pb.h"
-                "${CMAKE_CURRENT_SOURCE_DIR}/${proto_src_file_name}.pb.cc"
-        COMMAND protobuf::protoc --proto_path=${PROTO_PATH}/
-                --cpp_out=${CMAKE_CURRENT_SOURCE_DIR}/
-                ${proto_file}
-        DEPENDS ${PROTO_PATH}/${proto_file} )
-ENDFOREACH(proto_file)
-
-set( proto_file_names
-    segcore.proto
-    index_cgo_msg.proto
-    plan.proto
-    )
-
-set( PROTO_PATH  "${MILVUS_SOURCE_DIR}/../proto/")
-set( PROTO_API_PATH  "${MILVUS_SOURCE_DIR}/../../cmake_build/thirdparty/milvus-proto/proto/" )
-
-FOREACH(proto_file ${proto_file_names})
-    STRING( REGEX REPLACE "[^/]proto" "" proto_src_file_name ${proto_file} )
-    LIST( APPEND milvus_proto_srcs "${proto_src_file_name}.pb.cc" )
-
-    ADD_CUSTOM_COMMAND(
-        OUTPUT  "${CMAKE_CURRENT_SOURCE_DIR}/${proto_src_file_name}.pb.h"
-                "${CMAKE_CURRENT_SOURCE_DIR}/${proto_src_file_name}.pb.cc"
-        COMMAND protobuf::protoc --proto_path=${PROTO_PATH}/
-                --proto_path=${PROTO_API_PATH}/
-                --cpp_out=${CMAKE_CURRENT_SOURCE_DIR}/
-                ${proto_file}
-        DEPENDS ${PROTO_PATH}/${proto_file} )
-
-ENDFOREACH(proto_file)
-
-add_library(milvus_proto STATIC ${milvus_proto_srcs})
 message(STATUS "milvus proto sources: " ${milvus_proto_srcs})
 
 target_link_libraries( milvus_proto PUBLIC ${CONAN_LIBS} )

scripts/3rdparty_build.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+# Licensed to the LF AI & Data foundation under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+ROOT_DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
+CPP_SRC_DIR="${ROOT_DIR}/internal/core"
+BUILD_OUTPUT_DIR="${ROOT_DIR}/cmake_build"
+
+if [[ ! -d ${BUILD_OUTPUT_DIR} ]]; then
+  mkdir ${BUILD_OUTPUT_DIR}
+fi
+
+source ${ROOT_DIR}/scripts/setenv.sh
+pushd ${BUILD_OUTPUT_DIR}
+
+export CONAN_REVISIONS_ENABLED=1
+if [[ ! `conan remote list` == *default-conan-local* ]]; then
+    conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local
+fi
+unameOut="$(uname -s)"
+case "${unameOut}" in
+  Darwin*)
+    conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler=clang -s compiler.version=${llvm_version} -s compiler.libcxx=libc++ -s compiler.cppstd=17 || { echo 'conan install failed'; exit 1; }
+    ;;
+  Linux*)
+    echo "Running on ${OS_NAME}"
+    export CPU_TARGET=avx
+    GCC_VERSION=`gcc -dumpversion`
+    if [[ `gcc -v 2>&1 | sed -n 's/.*\(--with-default-libstdcxx-abi\)=\(\w*\).*/\2/p'` == "gcc4" ]]; then
+      conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler.version=${GCC_VERSION} || { echo 'conan install failed'; exit 1; }
+    else
+      conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler.version=${GCC_VERSION} -s compiler.libcxx=libstdc++11 || { echo 'conan install failed'; exit 1; }
+    fi
+    ;;
+  *)
+    echo "Cannot build on windows"
+    ;;
+esac
+
+popd

scripts/core_build.sh

@@ -149,6 +149,7 @@ done
 if [[ ! -d ${BUILD_OUTPUT_DIR} ]]; then
   mkdir ${BUILD_OUTPUT_DIR}
 fi
+source ${ROOT_DIR}/scripts/setenv.sh
 
 CMAKE_GENERATOR="Unix Makefiles"
 
@@ -188,47 +189,6 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then
   exit 0
 fi
 
-export CONAN_REVISIONS_ENABLED=1
-conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local
-unameOut="$(uname -s)"
-case "${unameOut}" in
-  Darwin*)
-    # detect llvm version by valid list
-    for llvm_version in 15 14 NOT_FOUND ; do
-      if brew ls --versions llvm@${llvm_version} > /dev/null; then
-        break
-      fi
-    done
-    if [ "${llvm_version}" = "NOT_FOUND" ] ; then
-      echo "valid llvm(14 or 15) not installed"
-      exit 1
-    fi
-    llvm_prefix="$(brew --prefix llvm@${llvm_version})"
-    export CLANG_TOOLS_PATH="${llvm_prefix}/bin"
-    export CC="ccache ${llvm_prefix}/bin/clang"
-    export CXX="ccache ${llvm_prefix}/bin/clang++"
-    export ASM="${llvm_prefix}/bin/clang"
-    export CFLAGS="-Wno-deprecated-declarations -I$(brew --prefix libomp)/include"
-    export CXXFLAGS=${CFLAGS}
-    export LDFLAGS="-L$(brew --prefix libomp)/lib"
-
-    conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler=clang -s compiler.version=${llvm_version} -s compiler.libcxx=libc++ -s compiler.cppstd=17 || { echo 'conan install failed'; exit 1; }
-    ;;
-  Linux*)
-    echo "Running on ${OS_NAME}"
-    export CPU_TARGET=avx
-    GCC_VERSION=`gcc -dumpversion`
-    if [[ `gcc -v 2>&1 | sed -n 's/.*\(--with-default-libstdcxx-abi\)=\(\w*\).*/\2/p'` == "gcc4" ]]; then
-      conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler.version=${GCC_VERSION} || { echo 'conan install failed'; exit 1; }
-    else
-      conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler.version=${GCC_VERSION} -s compiler.libcxx=libstdc++11 || { echo 'conan install failed'; exit 1; }
-    fi
-    ;;
-  *)
-    echo "Cannot build on windows"
-    ;;
-esac
-
 arch=$(uname -m)
 CMAKE_CMD="cmake \
 ${CMAKE_EXTRA_ARGS} \
@@ -250,6 +210,7 @@ ${CMAKE_EXTRA_ARGS} \
 -DUSE_ASAN=${USE_ASAN} \
 ${CPP_SRC_DIR}"
 
+echo "CC $CC"
 echo ${CMAKE_CMD}
 ${CMAKE_CMD} -G "${CMAKE_GENERATOR}"
 

scripts/generate_proto.sh

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+
+# Licensed to the LF AI & Data foundation under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+ROOT_DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
+
+PROTO_DIR=$ROOT_DIR/internal/proto/
+API_PROTO_DIR=$ROOT_DIR/cmake_build/thirdparty/milvus-proto/proto
+CPP_SRC_DIR=$ROOT_DIR/internal/core
+PROTOC_BIN=$ROOT_DIR/cmake_build/bin/protoc
+
+PROGRAM=$(basename "$0")
+GOPATH=$(go env GOPATH)
+
+if [ -z $GOPATH ]; then
+    printf "Error: the environment variable GOPATH is not set, please set it before running %s\n" $PROGRAM > /dev/stderr
+    exit 1
+fi
+
+export PATH=${GOPATH}/bin:$PATH
+
+# official go code ship with the crate, so we need to generate it manually.
+pushd ${PROTO_DIR}
+
+mkdir -p etcdpb
+mkdir -p indexcgopb
+
+mkdir -p internalpb
+mkdir -p rootcoordpb
+
+mkdir -p segcorepb
+mkdir -p proxypb
+
+mkdir -p indexpb
+mkdir -p datapb
+mkdir -p querypb
+mkdir -p planpb
+
+mkdir -p $ROOT_DIR/cmd/tools/migration/legacy/legacypb
+
+protoc_opt="${PROTOC_BIN} --proto_path=${API_PROTO_DIR} --proto_path=."
+
+${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./etcdpb etcd_meta.proto || { echo 'generate etcd_meta.proto failed'; exit 1; }
+${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./indexcgopb index_cgo_msg.proto || { echo 'generate index_cgo_msg failed '; exit 1; }
+${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./rootcoordpb root_coord.proto || { echo 'generate root_coord.proto failed'; exit 1; }
+${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./internalpb internal.proto || { echo 'generate internal.proto failed'; exit 1; }
+${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./proxypb proxy.proto|| { echo 'generate proxy.proto failed'; exit 1; }
+${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./indexpb index_coord.proto|| { echo 'generate index_coord.proto failed'; exit 1; }
+${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./datapb data_coord.proto|| { echo 'generate data_coord.proto failed'; exit 1; }
+${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./querypb query_coord.proto|| { echo 'generate query_coord.proto failed'; exit 1; }
+${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./planpb plan.proto|| { echo 'generate plan.proto failed'; exit 1; }
+${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./segcorepb segcore.proto|| { echo 'generate segcore.proto failed'; exit 1; }
+
+${protoc_opt} --proto_path=$ROOT_DIR/cmd/tools/migration/legacy/ \
+  --go_out=plugins=grpc,paths=source_relative:../../cmd/tools/migration/legacy/legacypb legacy.proto || { echo 'generate legacy.proto failed'; exit 1; }
+
+${protoc_opt} --cpp_out=$CPP_SRC_DIR/src/pb schema.proto|| { echo 'generate schema.proto failed'; exit 1; }
+${protoc_opt} --cpp_out=$CPP_SRC_DIR/src/pb common.proto|| { echo 'generate common.proto failed'; exit 1; }
+${protoc_opt} --cpp_out=$CPP_SRC_DIR/src/pb segcore.proto|| { echo 'generate segcore.proto failed'; exit 1; }
+${protoc_opt} --cpp_out=$CPP_SRC_DIR/src/pb index_cgo_msg.proto|| { echo 'generate index_cgo_msg.proto failed'; exit 1; }
+${protoc_opt} --cpp_out=$CPP_SRC_DIR/src/pb plan.proto|| { echo 'generate plan.proto failed'; exit 1; }
+
+popd
+
+

scripts/proto_gen_go.sh

@@ -1,69 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the LF AI & Data foundation under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-SCRIPTS_DIR=$(dirname "$0")
-
-PROTO_DIR=$SCRIPTS_DIR/../internal/proto/
-API_PROTO_DIR=$SCRIPTS_DIR/../cmake_build/thirdparty/milvus-proto/proto/
-
-PROGRAM=$(basename "$0")
-GOPATH=$(go env GOPATH)
-
-if [ -z $GOPATH ]; then
-    printf "Error: the environment variable GOPATH is not set, please set it before running %s\n" $PROGRAM > /dev/stderr
-    exit 1
-fi
-
-export PATH=${GOPATH}/bin:$PATH
-
-# official go code ship with the crate, so we need to generate it manually.
-pushd ${PROTO_DIR}
-
-mkdir -p etcdpb
-mkdir -p indexcgopb
-
-mkdir -p internalpb
-mkdir -p rootcoordpb
-
-mkdir -p segcorepb
-mkdir -p proxypb
-
-mkdir -p indexpb
-mkdir -p datapb
-mkdir -p querypb
-mkdir -p planpb
-
-mkdir -p ../../cmd/tools/migration/legacy/legacypb
-
-protoc_opt="${protoc} --proto_path=${API_PROTO_DIR} --proto_path=/usr/local/include --proto_path=/usr/include --proto_path=."
-
-${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./etcdpb etcd_meta.proto
-${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./indexcgopb index_cgo_msg.proto
-${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./rootcoordpb root_coord.proto
-${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./internalpb internal.proto
-${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./proxypb proxy.proto
-${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./indexpb index_coord.proto
-${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./datapb data_coord.proto
-${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./querypb query_coord.proto
-${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./planpb plan.proto
-${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./segcorepb segcore.proto
-
-${protoc_opt} --proto_path=../../cmd/tools/migration/legacy/ \
-  --go_out=plugins=grpc,paths=source_relative:../../cmd/tools/migration/legacy/legacypb legacy.proto
-
-popd

scripts/setenv.sh

@@ -17,11 +17,11 @@
 # limitations under the License.
 
 # Exit immediately for non zero status
-set -e
+set +e
 
 SOURCE="${BASH_SOURCE[0]}"
 # fix on zsh environment
-if [[ "$SOURCE" == "" ]]; then 
+if [[ "$SOURCE" == "" ]]; then
   SOURCE="$0"
 fi
 
@@ -35,7 +35,7 @@ ROOT_DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
 unameOut="$(uname -s)"
 
 case "${unameOut}" in
-    Linux*)     
+    Linux*)
       LIBJEMALLOC=$PWD/internal/core/output/lib/libjemalloc.so
       if test -f "$LIBJEMALLOC"; then
         export LD_PRELOAD="$LIBJEMALLOC"
@@ -45,11 +45,30 @@ case "${unameOut}" in
       export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:$ROOT_DIR/internal/core/output/lib/pkgconfig:$ROOT_DIR/internal/core/output/lib64/pkgconfig"
       export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:$ROOT_DIR/internal/core/output/lib:$ROOT_DIR/internal/core/output/lib64"
       export RPATH=$LD_LIBRARY_PATH;;
-    Darwin*)    
+    Darwin*)
+      # detect llvm version by valid list
+      for llvm_version in 15 14 NOT_FOUND ; do
+        if brew ls --versions llvm@${llvm_version} > /dev/null; then
+          break
+        fi
+      done
+      if [ "${llvm_version}" = "NOT_FOUND" ] ; then
+        echo "valid llvm(14 or 15) not installed"
+        exit 1
+      fi
+      llvm_prefix="$(brew --prefix llvm@${llvm_version})"
+      export CLANG_TOOLS_PATH="${llvm_prefix}/bin"
+      export CC="ccache ${llvm_prefix}/bin/clang"
+      export CXX="ccache ${llvm_prefix}/bin/clang++"
+      export ASM="${llvm_prefix}/bin/clang"
+      export CFLAGS="-Wno-deprecated-declarations -I$(brew --prefix libomp)/include"
+      export CXXFLAGS=${CFLAGS}
+      export LDFLAGS="-L$(brew --prefix libomp)/lib"
+
       export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:$ROOT_DIR/internal/core/output/lib/pkgconfig"
       export DYLD_LIBRARY_PATH=$ROOT_DIR/internal/core/output/lib
       export RPATH=$DYLD_LIBRARY_PATH;;
-    MINGW*)          
+    MINGW*)
       extra_path=$(cygpath -w "$ROOT_DIR/internal/core/output/lib")
       export PKG_CONFIG_PATH="${PKG_CONFIG_PATH};${extra_path}\pkgconfig"
       export LD_LIBRARY_PATH=$extra_path