test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-01 19:39:21 +08:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

Combine all service/node binaries into one binary with named 'milvus'

Browse Source 
Signed-off-by: Cai Yudong <yudong.cai@zilliz.com>
This commit is contained in:
Cai Yudong 2021-04-15 09:28:46 +08:00 committed by yefu.chen
parent 0a6b0f3bf7
commit e4256a4400
5 changed files with 146 additions and 209 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
Makefile
View File

@ -84,46 +84,50 @@ binlog:
@mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/binlog $(PWD)/cmd/binlog/main.go 1>/dev/null
master:
@echo "Building master service ..."
@echo "Building master ..."
@mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/masterservice $(PWD)/cmd/masterservice/main.go 1>/dev/null
proxyservice:
@echo "Building proxy service ..."
@echo "Building proxyservice ..."
@mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/proxyservice $(PWD)/cmd/proxy/service/proxy_service.go 1>/dev/null
proxynode:
@echo "Building proxy node ..."
@echo "Building proxynode ..."
@mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/proxynode $(PWD)/cmd/proxy/node/proxy_node.go 1>/dev/null
queryservice:
@echo "Building query service ..."
@echo "Building queryservice ..."
@mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/queryservice $(PWD)/cmd/queryservice/queryservice.go 1>/dev/null
querynode:
@echo "Building query node ..."
@echo "Building querynode ..."
@mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/querynode $(PWD)/cmd/querynode/querynode.go 1>/dev/null
dataservice:
@echo "Building data service ..."
@echo "Building dataservice ..."
@mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/dataservice $(PWD)/cmd/dataservice/main.go 1>/dev/null
datanode:
@echo "Building data node ..."
@echo "Building datanode ..."
@mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/datanode $(PWD)/cmd/datanode/main.go 1>/dev/null
indexservice: build-cpp
@echo "Building index service ..."
@echo "Building indexservice ..."
@mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/indexservice $(PWD)/cmd/indexservice/main.go 1>/dev/null
indexnode: build-cpp
@echo "Building index node ..."
@echo "Building indexnode ..."
@mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/indexnode $(PWD)/cmd/indexnode/main.go 1>/dev/null
singlenode: build-cpp
@echo "Building single node ..."
@echo "Building Milvus singlenode ..."
@mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/singlenode $(PWD)/cmd/singlenode/main.go 1>/dev/null
build-go: binlog master proxyservice proxynode queryservice querynode indexservice indexnode dataservice datanode singlenode
milvus: build-cpp
@echo "Building Milvus distributed ..."
@mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/milvus $(PWD)/cmd/distributed/main.go 1>/dev/null
build-go: singlenode milvus
build-cpp:
@(env bash $(PWD)/scripts/core_build.sh -f "$(CUSTOM_THIRDPARTY_PATH)")
@ -156,16 +160,8 @@ docker: verifiers
# Builds each component and installs it to $GOPATH/bin.
install: all
@echo "Installing binary to './bin'"
@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/masterservice $(GOPATH)/bin/masterservice
@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/queryservice $(GOPATH)/bin/queryservice
@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/querynode $(GOPATH)/bin/querynode
@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/proxynode $(GOPATH)/bin/proxynode
@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/proxyservice $(GOPATH)/bin/proxyservice
@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/singlenode $(GOPATH)/bin/singlenode
@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/indexservice $(GOPATH)/bin/indexservice
@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/indexnode $(GOPATH)/bin/indexnode
@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/dataservice $(GOPATH)/bin/dataservice
@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/datanode $(GOPATH)/bin/datanode
@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/milvus $(GOPATH)/bin/milvus
@mkdir -p $(LIBRARY_PATH) && cp -f $(PWD)/internal/core/output/lib/* $(LIBRARY_PATH)
@echo "Installation successful."
@ -175,13 +171,4 @@ clean:
@find . -name '*~' | xargs rm -fv
@rm -rf bin/
@rm -rf lib/
@rm -rf $(GOPATH)/bin/masterservice
@rm -rf $(GOPATH)/bin/proxynode
@rm -rf $(GOPATH)/bin/proxyservice
@rm -rf $(GOPATH)/bin/queryservice
@rm -rf $(GOPATH)/bin/querynode
@rm -rf $(GOPATH)/bin/singlenode
@rm -rf $(GOPATH)/bin/indexservice
@rm -rf $(GOPATH)/bin/indexnode
@rm -rf $(GOPATH)/bin/dataservice
@rm -rf $(GOPATH)/bin/datanode
@rm -rf $(GOPATH)/bin/*

2
build/ci/jenkins/Jenkinsfile vendored
View File

@ -84,7 +84,7 @@ pipeline {
}
}
environment {
HELM_BRANCH = "main"
HELM_BRANCH = "2.0"
HELM_RELEASE_NAMESPACE = "dev"
HELM_RELEASE_NAME = "${PROJECT_NAME}-${SEMVER}-${env.BUILD_NUMBER}-${REGRESSION_SERVICE_TYPE}".replaceAll("\\.", "-").replaceAll("_", "-").toLowerCase()
DEV_TEST_ARTIFACTS_PATH = "artifacts"

2
cmd/distributed/main.go
View File

@ -65,8 +65,6 @@ func run(serverType, runtTimeDir string) error {
role.EnableIndexService = true
case "indexnode":
role.EnableIndexNode = true
case "standalone":
role.EnableStandalone = true
default:
return fmt.Errorf("unknown server type = %s", serverType)
}

243
cmd/distributed/roles/roles.go
View File

@ -8,11 +8,20 @@ import (
"strings"
"syscall"
"github.com/zilliztech/milvus-distributed/internal/datanode"
"github.com/zilliztech/milvus-distributed/internal/dataservice"
"github.com/zilliztech/milvus-distributed/internal/indexnode"
"github.com/zilliztech/milvus-distributed/internal/indexservice"
"github.com/zilliztech/milvus-distributed/internal/log"
"github.com/zilliztech/milvus-distributed/internal/masterservice"
"github.com/zilliztech/milvus-distributed/internal/proxynode"
"github.com/zilliztech/milvus-distributed/internal/proxyservice"
"github.com/zilliztech/milvus-distributed/internal/querynode"
"github.com/zilliztech/milvus-distributed/internal/queryservice"
"github.com/zilliztech/milvus-distributed/cmd/distributed/components"
ds "github.com/zilliztech/milvus-distributed/internal/dataservice"
"github.com/zilliztech/milvus-distributed/internal/logutil"
"github.com/zilliztech/milvus-distributed/internal/msgstream"
"github.com/zilliztech/milvus-distributed/internal/util/rocksmq/server/rocksmq"
"github.com/zilliztech/milvus-distributed/internal/util/trace"
)
@ -34,168 +43,234 @@ type MilvusRoles struct {
EnableIndexService bool `env:"ENABLE_INDEX_SERVICE"`
EnableIndexNode bool `env:"ENABLE_INDEX_NODE"`
EnableMsgStreamService bool `env:"ENABLE_MSGSTREAM_SERVICE"`
EnableStandalone bool `env:"ENABLE_STANDALONE"`
}
func (mr *MilvusRoles) HasAnyRole() bool {
return mr.EnableMaster || mr.EnableMsgStreamService ||
mr.EnableProxyService || mr.EnableProxyNode ||
mr.EnableQueryService || mr.EnableQueryNode ||
mr.EnableDataService || mr.EnableDataNode ||
mr.EnableIndexService || mr.EnableIndexNode || mr.EnableStandalone
}
func (mr *MilvusRoles) EnvValue(env string) bool {
env = strings.ToLower(env)
env = strings.Trim(env, " ")
if env == "1" || env == "true" {
return true
}
return false
return env == "1" || env == "true"
}
func (mr *MilvusRoles) Run(localMsg bool) {
closer := trace.InitTracing("singleNode")
if closer != nil {
defer closer.Close()
}
if !mr.HasAnyRole() {
return
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var masterService *components.MasterService
if mr.EnableMaster {
var ms *components.MasterService
go func() {
masterservice.Params.Init()
logutil.SetupLogger(&masterservice.Params.Log)
defer log.Sync()
factory := newMsgFactory(localMsg)
var err error
masterService, err = components.NewMasterService(ctx, factory)
ms, err = components.NewMasterService(ctx, factory)
if err != nil {
panic(err)
}
_ = masterService.Run()
_ = ms.Run()
}()
if ms != nil {
defer ms.Stop()
}
}
var proxyService *components.ProxyService
if mr.EnableProxyService {
var ps *components.ProxyService
go func() {
proxyservice.Params.Init()
logutil.SetupLogger(&proxyservice.Params.Log)
defer log.Sync()
factory := newMsgFactory(localMsg)
var err error
proxyService, err = components.NewProxyService(ctx, factory)
ps, err = components.NewProxyService(ctx, factory)
if err != nil {
panic(err)
}
_ = proxyService.Run()
_ = ps.Run()
}()
if ps != nil {
defer ps.Stop()
}
}
var proxyNode *components.ProxyNode
if mr.EnableProxyNode {
var pn *components.ProxyNode
go func() {
proxynode.Params.Init()
logutil.SetupLogger(&proxynode.Params.Log)
defer log.Sync()
factory := newMsgFactory(localMsg)
var err error
proxyNode, err = components.NewProxyNode(ctx, factory)
pn, err = components.NewProxyNode(ctx, factory)
if err != nil {
panic(err)
}
_ = proxyNode.Run()
_ = pn.Run()
}()
if pn != nil {
defer pn.Stop()
}
}
var queryService *components.QueryService
if mr.EnableQueryService {
var qs *components.QueryService
go func() {
queryservice.Params.Init()
logutil.SetupLogger(&queryservice.Params.Log)
defer log.Sync()
factory := newMsgFactory(localMsg)
var err error
queryService, err = components.NewQueryService(ctx, factory)
qs, err = components.NewQueryService(ctx, factory)
if err != nil {
panic(err)
}
_ = queryService.Run()
_ = qs.Run()
}()
if qs != nil {
defer qs.Stop()
}
}
var queryNode *components.QueryNode
if mr.EnableQueryNode {
var qn *components.QueryNode
go func() {
querynode.Params.Init()
logutil.SetupLogger(&querynode.Params.Log)
defer log.Sync()
factory := newMsgFactory(localMsg)
var err error
queryNode, err = components.NewQueryNode(ctx, factory)
qn, err = components.NewQueryNode(ctx, factory)
if err != nil {
panic(err)
}
_ = queryNode.Run()
_ = qn.Run()
}()
if qn != nil {
defer qn.Stop()
}
}
var dataService *components.DataService
if mr.EnableDataService {
var ds *components.DataService
go func() {
dataservice.Params.Init()
logutil.SetupLogger(&dataservice.Params.Log)
defer log.Sync()
factory := newMsgFactory(localMsg)
var err error
// Init data service params
ds.Params.Init()
logutil.SetupLogger(&ds.Params.Log)
dataService, err = components.NewDataService(ctx, factory)
ds, err = components.NewDataService(ctx, factory)
if err != nil {
panic(err)
}
_ = dataService.Run()
_ = ds.Run()
}()
if ds != nil {
defer ds.Stop()
}
}
var dataNode *components.DataNode
if mr.EnableDataNode {
var dn *components.DataNode
go func() {
datanode.Params.Init()
logutil.SetupLogger(&datanode.Params.Log)
defer log.Sync()
factory := newMsgFactory(localMsg)
var err error
dataNode, err = components.NewDataNode(ctx, factory)
dn, err = components.NewDataNode(ctx, factory)
if err != nil {
panic(err)
}
_ = dataNode.Run()
_ = dn.Run()
}()
if dn != nil {
defer dn.Stop()
}
}
var indexService *components.IndexService
if mr.EnableIndexService {
var is *components.IndexService
go func() {
indexservice.Params.Init()
logutil.SetupLogger(&indexservice.Params.Log)
defer log.Sync()
var err error
indexService, err = components.NewIndexService(ctx)
is, err = components.NewIndexService(ctx)
if err != nil {
panic(err)
}
_ = indexService.Run()
_ = is.Run()
}()
if is != nil {
defer is.Stop()
}
}
var indexNode *components.IndexNode
if mr.EnableIndexNode {
var in *components.IndexNode
go func() {
indexnode.Params.Init()
logutil.SetupLogger(&indexnode.Params.Log)
defer log.Sync()
var err error
indexNode, err = components.NewIndexNode(ctx)
in, err = components.NewIndexNode(ctx)
if err != nil {
panic(err)
}
_ = indexNode.Run()
_ = in.Run()
}()
if in != nil {
in.Stop()
}
}
var msgStream *components.MsgStream
if mr.EnableMsgStreamService {
var mss *components.MsgStream
go func() {
var err error
msgStream, err = components.NewMsgStreamService(ctx)
mss, err = components.NewMsgStreamService(ctx)
if err != nil {
panic(err)
}
_ = msgStream.Run()
_ = mss.Run()
}()
if mss != nil {
defer mss.Stop()
}
}
sc := make(chan os.Signal, 1)
@ -206,66 +281,4 @@ func (mr *MilvusRoles) Run(localMsg bool) {
syscall.SIGQUIT)
sig := <-sc
fmt.Printf("Get %s signal to exit", sig.String())
if mr.EnableMaster {
if masterService != nil {
_ = masterService.Stop()
}
}
if mr.EnableProxyService {
if proxyService != nil {
_ = proxyService.Stop()
}
}
if mr.EnableProxyNode {
if proxyNode != nil {
_ = proxyNode.Stop()
}
}
if mr.EnableQueryService {
if queryService != nil {
_ = queryService.Stop()
}
}
if mr.EnableQueryNode {
if queryNode != nil {
_ = queryNode.Stop()
}
}
if mr.EnableDataService {
if dataService != nil {
_ = dataService.Stop()
}
}
if mr.EnableDataNode {
if dataNode != nil {
_ = dataNode.Stop()
}
}
if mr.EnableIndexService {
if indexService != nil {
_ = indexService.Stop()
}
}
if mr.EnableIndexNode {
if indexNode != nil {
_ = indexNode.Stop()
}
}
if mr.EnableMsgStreamService {
if msgStream != nil {
_ = msgStream.Stop()
}
}
defer rocksmq.CloseRocksMQ()
}

61
cmd/distributed/roles/roles_test.go
View File

@ -9,7 +9,6 @@ import (
func TestRoles(t *testing.T) {
r := MilvusRoles{}
assert.False(t, r.HasAnyRole())
assert.True(t, r.EnvValue("1"))
assert.True(t, r.EnvValue(" 1 "))
@ -26,64 +25,4 @@ func TestRoles(t *testing.T) {
assert.Equal(t, len(ss), 1)
ss = strings.SplitN("adb=def", "=", 2)
assert.Equal(t, len(ss), 2)
{
var roles MilvusRoles
roles.EnableMaster = true
assert.True(t, roles.HasAnyRole())
}
{
var roles MilvusRoles
roles.EnableProxyService = true
assert.True(t, roles.HasAnyRole())
}
{
var roles MilvusRoles
roles.EnableProxyNode = true
assert.True(t, roles.HasAnyRole())
}
{
var roles MilvusRoles
roles.EnableQueryService = true
assert.True(t, roles.HasAnyRole())
}
{
var roles MilvusRoles
roles.EnableQueryNode = true
assert.True(t, roles.HasAnyRole())
}
{
var roles MilvusRoles
roles.EnableDataService = true
assert.True(t, roles.HasAnyRole())
}
{
var roles MilvusRoles
roles.EnableDataNode = true
assert.True(t, roles.HasAnyRole())
}
{
var roles MilvusRoles
roles.EnableIndexService = true
assert.True(t, roles.HasAnyRole())
}
{
var roles MilvusRoles
roles.EnableIndexNode = true
assert.True(t, roles.HasAnyRole())
}
{
var roles MilvusRoles
roles.EnableMsgStreamService = true
assert.True(t, roles.HasAnyRole())
}
}