fix windows ldflags (#17807)

Signed-off-by: Enwei Jiao <enwei.jiao@zilliz.com>
This commit is contained in:
Enwei Jiao 2022-06-28 19:58:25 +08:00 committed by GitHub
parent 3904a78211
commit ea901393ef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 129 additions and 71 deletions

View File

@ -11,7 +11,8 @@
GO ?= go
PWD := $(shell pwd)
GOPATH := $(shell $(GO) env GOPATH)
GOPATH := $(shell $(GO) env GOPATH)
SHELL := /bin/bash
OBJPREFIX := "github.com/milvus-io/milvus/cmd/milvus"
INSTALL_PATH := $(PWD)/bin
@ -19,7 +20,6 @@ LIBRARY_PATH := $(PWD)/lib
OS := $(shell uname -s)
ARCH := $(shell arch)
mode = Release
PKG_CONFIG = $(shell echo "${PKG_CONFIG_PATH}:`pwd`/internal/core/output/lib/pkgconfig:`pwd`/internal/core/output/lib64/pkgconfig")
all: build-cpp build-go
@ -66,8 +66,8 @@ lint: tools/bin/revive
static-check:
@echo "Running $@ check"
@GO111MODULE=on ${GOPATH}/bin/golangci-lint cache clean
@GO111MODULE=on PKG_CONFIG_PATH="$(PKG_CONFIG)" ${GOPATH}/bin/golangci-lint run --timeout=30m --config ./.golangci.yml ./internal/...
@GO111MODULE=on PKG_CONFIG_PATH="$(PKG_CONFIG)" ${GOPATH}/bin/golangci-lint run --timeout=30m --config ./.golangci.yml ./cmd/...
@source $(PWD)/scripts/setenv.sh && GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=30m --config ./.golangci.yml ./internal/...
@source $(PWD)/scripts/setenv.sh && GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=30m --config ./.golangci.yml ./cmd/...
# @GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=30m --config ./.golangci.yml ./tests/go_client/...
verifiers: build-cpp getdeps cppcheck fmt static-check
@ -96,18 +96,17 @@ print-build-info:
milvus: build-cpp print-build-info
@echo "Building Milvus ..."
@echo "if build fails on Mac M1 machines, you probably need to rerun scripts/install_deps.sh and then run: \`export PKG_CONFIG_PATH=\"/opt/homebrew/opt/openssl@3/lib/pkgconfig\"\`"
@mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \
export PKG_CONFIG_PATH="$(PKG_CONFIG)" && \
export RPATH=$$(pkg-config --libs-only-L milvus_common | cut -c 3-) && \
GO111MODULE=on $(GO) build -ldflags="-r $${RPATH} -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \
@source $(PWD)/scripts/setenv.sh && \
mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \
GO111MODULE=on $(GO) build -ldflags="-r $${RPATH} -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \
${APPLE_SILICON_FLAG} -o $(INSTALL_PATH)/milvus $(PWD)/cmd/main.go 1>/dev/null
embd-milvus: build-cpp-embd print-build-info
@echo "Building **Embedded** Milvus ..."
@echo "if build fails on Mac M1 machines, rerun scripts/install_deps.sh and then run: \`export PKG_CONFIG_PATH=\"/opt/homebrew/opt/openssl@3/lib/pkgconfig\"\`"
@mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \
export PKG_CONFIG_PATH="$(PKG_CONFIG)" && GO111MODULE=on $(GO) build \
-ldflags="-r /tmp/milvus/lib/ -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \
@source $(PWD)/scripts/setenv.sh && \
mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \
GO111MODULE=on $(GO) build -ldflags="-r /tmp/milvus/lib/ -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \
${APPLE_SILICON_FLAG} -buildmode=c-shared -o $(INSTALL_PATH)/embd-milvus.so $(PWD)/pkg/embedded/embedded.go 1>/dev/null
build-go: milvus

View File

@ -14,24 +14,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# if ( APPLE )
set( ROCKSDB_VERSION "6.27.3" )
set( ROCKSDB_SOURCE_URL
"https://github.com/facebook/rocksdb/archive/v${ROCKSDB_VERSION}.tar.gz")
# else ()
# set( ROCKSDB_VERSION "6.15.2" )
# set( ROCKSDB_SOURCE_URL
# "https://github.com/facebook/rocksdb/archive/v${ROCKSDB_VERSION}.tar.gz")
# endif ()
macro( build_rocksdb )
message( STATUS "Building ROCKSDB-${ROCKSDB_VERSION} from source" )
# if ( APPLE )
set ( ROCKSDB_MD5 "e4a0625f0cec82060e62c81b787a1124" )
# else ()
# set ( ROCKSDB_MD5 "67f9e04fda62af551dd039c37b1808ac" )
# endif ()
if ( EMBEDDED_MILVUS )
message ( STATUS "Turning on fPIC while building embedded Milvus" )
@ -80,4 +70,10 @@ macro( build_rocksdb )
endmacro()
build_rocksdb()
if (MSYS)
message(STATUS "Using rocksdb form system")
configure_file(rocksdb-win.pc.in "${CMAKE_CURRENT_BINARY_DIR}/rocksdb.pc" @ONLY)
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/rocksdb.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
else (MSYS)
build_rocksdb()
endif (MSYS)

View File

@ -0,0 +1,10 @@
libdir=/mingw64/lib
includedir=/mingw64/include
Name: Rocksdb
Description: Rocksdb
Version: @ROCKSDB_VERSION@
Libs: -L${libdir} -lrocksdb
Libs.private: -lz -lbz2
Cflags: -I${includedir}

View File

@ -7,7 +7,9 @@ if [[ "${MSYSTEM}" != "MINGW64" ]] ; then
exit 1
fi
pacman -S --noconfirm --needed \
pacmanInstall()
{
pacman -S --noconfirm --needed \
git make tar dos2unix zip unzip patch \
mingw-w64-x86_64-toolchain \
mingw-w64-x86_64-make \
@ -21,8 +23,20 @@ pacman -S --noconfirm --needed \
mingw-w64-x86_64-python2 \
mingw-w64-x86_64-diffutils \
mingw-w64-x86_64-arrow \
mingw-w64-x86_64-go
# mingw-w64-x86_64-rocksdb
mingw-w64-x86_64-go \
mingw-w64-x86_64-rocksdb
}
updateKey()
{
pacman-key --refresh-keys
}
pacmanInstall || {
updateKey
pacmanInstall
}
# dummy empty dl, TODO: remove later

View File

@ -22,16 +22,8 @@ FILE_COVERAGE_HTML="go_coverage.html"
set -ex
echo "mode: atomic" > ${FILE_COVERAGE_INFO}
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
SOURCE="$(readlink "$SOURCE")"
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
done
ROOT_DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:$ROOT_DIR/internal/core/output/lib/pkgconfig:$ROOT_DIR/internal/core/output/lib64/pkgconfig"
export RPATH=$(pkg-config --libs-only-L milvus_common | cut -c 3-)
BASEDIR=$(dirname "$0")
source $BASEDIR/setenv.sh
# run unittest
echo "Running unittest under ./internal"
@ -39,7 +31,7 @@ if [[ $(uname -s) == "Darwin" && "$(uname -m)" == "arm64" ]]; then
APPLE_SILICON_FLAG="-tags dynamic"
fi
for d in $(go list ./internal/... | grep -v -e vendor -e kafka -e planparserv2/generated); do
go test -race ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" -v -coverpkg=./... -coverprofile=profile.out -covermode=atomic "$d"
go test -race ${APPLE_SILICON_FLAG} -v -coverpkg=./... -coverprofile=profile.out -covermode=atomic "$d"
if [ -f profile.out ]; then
grep -v kafka profile.out | grep -v planparserv2/generated | sed '1d' >> ${FILE_COVERAGE_INFO}
rm profile.out

View File

@ -19,16 +19,8 @@
# Exit immediately for non zero status
set -e
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
SOURCE="$(readlink "$SOURCE")"
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
done
ROOT_DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:$ROOT_DIR/internal/core/output/lib/pkgconfig:$ROOT_DIR/internal/core/output/lib64/pkgconfig"
export RPATH=$(pkg-config --libs-only-L milvus_common | cut -c 3-)
BASEDIR=$(dirname "$0")
source $BASEDIR/setenv.sh
if [[ $(uname -s) == "Darwin" ]]; then
export MallocNanoZone=0
@ -42,30 +34,30 @@ fi
MILVUS_DIR="${ROOT_DIR}/internal/"
echo "Running go unittest under $MILVUS_DIR"
go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/allocator/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/kv/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" $(go list "${MILVUS_DIR}/mq/..." | grep -v kafka) -failfast
go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/storage" -failfast
go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/tso/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/util/funcutil/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/util/paramtable/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/util/retry/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/util/sessionutil/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/util/trace/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/util/typeutil/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/util/importutil/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/proxy/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/datanode/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/indexnode/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/querynode/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/distributed/rootcoord/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/distributed/datacoord/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/distributed/querycoord/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/distributed/proxy/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/distributed/datanode/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/distributed/querynode/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/rootcoord" -failfast
go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/datacoord/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/indexcoord/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/allocator/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/kv/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} $(go list "${MILVUS_DIR}/mq/..." | grep -v kafka) -failfast
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/storage" -failfast
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/tso/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/util/funcutil/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/util/paramtable/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/util/retry/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/util/sessionutil/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/util/trace/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/util/typeutil/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/util/importutil/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/proxy/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/datanode/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/indexnode/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/querynode/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/rootcoord/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/datacoord/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/querycoord/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/proxy/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/datanode/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/querynode/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/rootcoord" -failfast
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/datacoord/..." -failfast
go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/indexcoord/..." -failfast
echo " Go unittest finished"

55
scripts/setenv.sh Normal file
View File

@ -0,0 +1,55 @@
#!/bin/bash
# Licensed to the LF AI & Data foundation under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Exit immediately for non zero status
set -e
SOURCE="${BASH_SOURCE[0]}"
# fix on zsh environment
if [[ "$SOURCE" == "" ]]; then
SOURCE="$0"
fi
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
SOURCE="$(readlink "$SOURCE")"
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
done
ROOT_DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
unameOut="$(uname -s)"
case "${unameOut}" in
Linux*)
export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:$ROOT_DIR/internal/core/output/lib/pkgconfig:$ROOT_DIR/internal/core/output/lib64/pkgconfig"
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:$ROOT_DIR/internal/core/output/lib:$ROOT_DIR/internal/core/output/lib64"
export RPATH=$LD_LIBRARY_PATH;;
Darwin*)
export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:$ROOT_DIR/internal/core/output/lib/pkgconfig"
export DYLD_LIBRARY_PATH=$ROOT_DIR/internal/core/output/lib
export RPATH=$DYLD_LIBRARY_PATH;;
MINGW*)
extra_path=$(cygpath -w "$ROOT_DIR/internal/core/output/lib")
export PKG_CONFIG_PATH="${PKG_CONFIG_PATH};${extra_path}\pkgconfig"
export LD_LIBRARY_PATH=$extra_path
export RPATH=$LD_LIBRARY_PATH;;
*)
echo "does not supported"
esac