Support launch multiple services in one process (#5446)

Resolves: #5450 

Signed-off-by: yudong.cai <yudong.cai@zilliz.com>
This commit is contained in:
Cai Yudong 2021-05-27 22:02:58 +08:00 committed by GitHub
parent 6e3a0d2a8e
commit dde74e2165
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 109 additions and 672 deletions

View File

@ -1,62 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
package main
import (
"context"
"os"
"os/signal"
"syscall"
"github.com/milvus-io/milvus/internal/logutil"
"go.uber.org/zap"
dn "github.com/milvus-io/milvus/internal/datanode"
distributed "github.com/milvus-io/milvus/cmd/distributed/components"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/msgstream"
)
func main() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
msFactory := msgstream.NewPmsFactory()
dn.Params.Init()
logutil.SetupLogger(&dn.Params.Log)
dn, err := distributed.NewDataNode(ctx, msFactory)
if err != nil {
panic(err)
}
if err = dn.Run(); err != nil {
panic(err)
}
sc := make(chan os.Signal, 1)
signal.Notify(sc,
syscall.SIGHUP,
syscall.SIGINT,
syscall.SIGTERM,
syscall.SIGQUIT)
sig := <-sc
log.Debug("Got signal to exit signal", zap.String("signal", sig.String()))
err = dn.Stop()
if err != nil {
panic(err)
}
}

View File

@ -1,58 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
package main
import (
"context"
"os"
"os/signal"
"syscall"
"github.com/milvus-io/milvus/internal/logutil"
"github.com/milvus-io/milvus/internal/dataservice"
"github.com/milvus-io/milvus/cmd/distributed/components"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/msgstream"
)
func main() {
ctx, cancel := context.WithCancel(context.Background())
defer logutil.LogPanic()
dataservice.Params.Init()
logutil.SetupLogger(&dataservice.Params.Log)
defer log.Sync()
msFactory := msgstream.NewPmsFactory()
svr, err := components.NewDataService(ctx, msFactory)
if err != nil {
panic(err)
}
if err = svr.Run(); err != nil {
panic(err)
}
sc := make(chan os.Signal)
signal.Notify(sc,
syscall.SIGHUP,
syscall.SIGINT,
syscall.SIGTERM,
syscall.SIGQUIT)
<-sc
cancel()
if err := svr.Stop(); err != nil {
panic(err)
}
log.Debug("shut down data service")
}

View File

@ -22,85 +22,80 @@ import (
"github.com/milvus-io/milvus/cmd/distributed/roles"
)
func run(serverType, runtTimeDir, svrAlias string) error {
var fileName string
if len(svrAlias) != 0 {
fileName = fmt.Sprintf("%s-%s.pid", serverType, svrAlias)
const (
roleMaster = "master"
roleProxyService = "proxyservice"
roleQueryService = "queryservice"
roleIndexService = "indexservice"
roleDataService = "dataservice"
roleProxyNode = "proxynode"
roleQueryNode = "querynode"
roleIndexNode = "indexnode"
roleDataNode = "datanode"
roleMixture = "mixture"
)
func getPidFileName(service string, alias string) string {
var filename string
if len(alias) != 0 {
filename = fmt.Sprintf("%s-%s.pid", service, alias)
} else {
fileName = serverType + ".pid"
filename = service + ".pid"
}
var fd *os.File
var err error
if fd, err = os.OpenFile(path.Join(runtTimeDir, fileName), os.O_CREATE|os.O_RDWR, 0664); err != nil {
return fmt.Errorf("service %s is running, error = %w", serverType, err)
}
fmt.Println("open pid file:", path.Join(runtTimeDir, fileName))
if err := syscall.Flock(int(fd.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
return fmt.Errorf("service %s is running, error = %w", serverType, err)
}
fmt.Println("lock pid file")
defer func() {
_ = syscall.Close(int(fd.Fd()))
_ = os.Remove(path.Join(runtTimeDir, fileName))
}()
fd.Truncate(0)
_, _ = fd.WriteString(fmt.Sprintf("%d", os.Getpid()))
role := roles.MilvusRoles{}
switch serverType {
case "master":
role.EnableMaster = true
case "msgstream":
role.EnableMsgStreamService = true
case "proxyservice":
role.EnableProxyService = true
case "proxynode":
role.EnableProxyNode = true
case "queryservice":
role.EnableQueryService = true
case "querynode":
role.EnableQueryNode = true
case "dataservice":
role.EnableDataService = true
case "datanode":
role.EnableDataNode = true
case "indexservice":
role.EnableIndexService = true
case "indexnode":
role.EnableIndexNode = true
default:
return fmt.Errorf("unknown server type = %s", serverType)
}
role.Run(false)
return nil
return filename
}
func stop(serverType, runtimeDir, svrAlias string) error {
var fileName string
if len(svrAlias) != 0 {
fileName = fmt.Sprintf("%s-%s.pid", serverType, svrAlias)
} else {
fileName = serverType + ".pid"
func createPidFile(filename string, runtimeDir string) (*os.File, error) {
fileFullName := path.Join(runtimeDir, filename)
fd, err := os.OpenFile(fileFullName, os.O_CREATE|os.O_RDWR, 0664)
if err != nil {
return nil, fmt.Errorf("file %s is locked, error = %w", filename, err)
}
var err error
var fd *os.File
if fd, err = os.OpenFile(path.Join(runtimeDir, fileName), os.O_RDONLY, 0664); err != nil {
fmt.Println("open pid file:", fileFullName)
err = syscall.Flock(int(fd.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
if err != nil {
return nil, fmt.Errorf("file %s is locked, error = %w", filename, err)
}
fmt.Println("lock pid file:", fileFullName)
fd.Truncate(0)
_, err = fd.WriteString(fmt.Sprintf("%d", os.Getpid()))
if err != nil {
return nil, fmt.Errorf("file %s write fail, error = %w", filename, err)
}
return fd, nil
}
func closePidFile(fd *os.File) {
fd.Close()
}
func removePidFile(fd *os.File) {
syscall.Close(int(fd.Fd()))
os.Remove(fd.Name())
}
func stopPid(filename string, runtimeDir string) error {
var pid int
fd, err := os.OpenFile(path.Join(runtimeDir, filename), os.O_RDONLY, 0664)
if err != nil {
return err
}
defer func() {
_ = fd.Close()
}()
var pid int
defer closePidFile(fd)
_, err = fmt.Fscanf(fd, "%d", &pid)
if err != nil {
return err
}
p, err := os.FindProcess(pid)
process, err := os.FindProcess(pid)
if err != nil {
return err
}
err = p.Signal(syscall.SIGTERM)
err = process.Signal(syscall.SIGTERM)
if err != nil {
return err
}
@ -117,7 +112,7 @@ func makeRuntimeDir(dir string) error {
return nil
}
if !st.IsDir() {
return fmt.Errorf("%s is exist, but is not directory", dir)
return fmt.Errorf("%s is not directory", dir)
}
tmpFile, err := ioutil.TempFile(dir, "tmp")
if err != nil {
@ -139,32 +134,74 @@ func main() {
flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
var svrAlias string
flags.StringVar(&svrAlias, "alias", "", "set aliase")
flags.StringVar(&svrAlias, "alias", "", "set alias")
var enableMaster, enableProxyService, enableQueryService, enableIndexService, enableDataService bool
flags.BoolVar(&enableMaster, roleMaster, false, "enable master")
flags.BoolVar(&enableProxyService, roleProxyService, false, "enable proxy service")
flags.BoolVar(&enableQueryService, roleQueryService, false, "enable query service")
flags.BoolVar(&enableIndexService, roleIndexService, false, "enable index service")
flags.BoolVar(&enableDataService, roleDataService, false, "enable data service")
if err := flags.Parse(os.Args[3:]); err != nil {
os.Exit(-1)
}
role := roles.MilvusRoles{}
switch serverType {
case roleMaster:
role.EnableMaster = true
case roleProxyService:
role.EnableProxyService = true
case roleProxyNode:
role.EnableProxyNode = true
case roleQueryService:
role.EnableQueryService = true
case roleQueryNode:
role.EnableQueryNode = true
case roleDataService:
role.EnableDataService = true
case roleDataNode:
role.EnableDataNode = true
case roleIndexService:
role.EnableIndexService = true
case roleIndexNode:
role.EnableIndexNode = true
case roleMixture:
role.EnableMaster = enableMaster
role.EnableProxyService = enableProxyService
role.EnableQueryService = enableQueryService
role.EnableDataService = enableDataService
role.EnableIndexService = enableIndexService
default:
fmt.Fprintf(os.Stderr, "Unknown server type = %s\n", serverType)
os.Exit(-1)
}
runtimeDir := "/run/milvus"
if err := makeRuntimeDir(runtimeDir); err != nil {
_, _ = fmt.Fprintf(os.Stderr, "set runtime dir at : %s failed, set it to /tmp/milvus directory\n", runtimeDir)
fmt.Fprintf(os.Stderr, "Set runtime dir at %s failed, set it to /tmp/milvus directory\n", runtimeDir)
runtimeDir = "/tmp/milvus"
if err = makeRuntimeDir(runtimeDir); err != nil {
_, _ = fmt.Fprintf(os.Stderr, "create runtime director at : %s failed\n", runtimeDir)
fmt.Fprintf(os.Stderr, "Create runtime directory at %s failed\n", runtimeDir)
os.Exit(-1)
}
}
filename := getPidFileName(serverType, svrAlias)
switch command {
case "run":
if err := run(serverType, runtimeDir, svrAlias); err != nil {
fd, err := createPidFile(filename, runtimeDir)
if err != nil {
panic(err)
}
defer removePidFile(fd)
role.Run(false)
case "stop":
if err := stop(serverType, runtimeDir, svrAlias); err != nil {
if err := stopPid(filename, runtimeDir); err != nil {
panic(err)
}
default:
_, _ = fmt.Fprintf(os.Stderr, "unknown command : %s", command)
fmt.Fprintf(os.Stderr, "unknown command : %s", command)
}
}

View File

@ -1,70 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
package main
import (
"context"
"os"
"os/signal"
"syscall"
"go.uber.org/zap"
"github.com/milvus-io/milvus/cmd/distributed/components"
"github.com/milvus-io/milvus/internal/indexnode"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/logutil"
)
func main() {
ctx, cancel := context.WithCancel(context.Background())
indexnode.Params.Init()
logutil.SetupLogger(&indexnode.Params.Log)
n, err := components.NewIndexNode(ctx)
if err != nil {
log.Error("create server failed", zap.Error(err))
}
sc := make(chan os.Signal, 1)
signal.Notify(sc,
syscall.SIGHUP,
syscall.SIGINT,
syscall.SIGTERM,
syscall.SIGQUIT)
var sig os.Signal
go func() {
sig = <-sc
cancel()
}()
if err := n.Run(); err != nil {
log.Error("run builder server failed", zap.Error(err))
}
<-ctx.Done()
log.Debug("Got signal to exit", zap.String("signal", sig.String()))
if err := n.Stop(); err != nil {
log.Fatal("stop builder server failed", zap.Error(err))
}
switch sig {
case syscall.SIGTERM:
exit(0)
default:
exit(1)
}
}
func exit(code int) {
os.Exit(code)
}

View File

@ -1,71 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
package main
import (
"context"
"os"
"os/signal"
"syscall"
"go.uber.org/zap"
"github.com/milvus-io/milvus/cmd/distributed/components"
"github.com/milvus-io/milvus/internal/indexservice"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/logutil"
)
func main() {
ctx, cancel := context.WithCancel(context.Background())
indexservice.Params.Init()
logutil.SetupLogger(&indexservice.Params.Log)
s, err := components.NewIndexService(ctx)
if err != nil {
log.Debug("create server failed", zap.Error(err))
}
sc := make(chan os.Signal, 1)
signal.Notify(sc,
syscall.SIGHUP,
syscall.SIGINT,
syscall.SIGTERM,
syscall.SIGQUIT)
var sig os.Signal
go func() {
sig = <-sc
cancel()
}()
if err := s.Run(); err != nil {
log.Fatal("run builder server failed", zap.Error(err))
}
<-ctx.Done()
log.Debug("Got signal to exit", zap.String("signal", sig.String()))
if err := s.Stop(); err != nil {
log.Fatal("stop server failed", zap.Error(err))
}
switch sig {
case syscall.SIGTERM:
exit(0)
default:
exit(1)
}
}
func exit(code int) {
os.Exit(code)
}

View File

@ -1,62 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
package main
import (
"context"
"os"
"os/signal"
"syscall"
"github.com/milvus-io/milvus/internal/logutil"
distributed "github.com/milvus-io/milvus/cmd/distributed/components"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/masterservice"
"github.com/milvus-io/milvus/internal/msgstream"
"go.uber.org/zap"
)
func main() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
masterservice.Params.Init()
logutil.SetupLogger(&masterservice.Params.Log)
defer func() {
if err := log.Sync(); err != nil {
panic(err)
}
}()
msFactory := msgstream.NewPmsFactory()
ms, err := distributed.NewMasterService(ctx, msFactory)
if err != nil {
panic(err)
}
if err = ms.Run(); err != nil {
panic(err)
}
sc := make(chan os.Signal, 1)
signal.Notify(sc,
syscall.SIGHUP,
syscall.SIGINT,
syscall.SIGTERM,
syscall.SIGQUIT)
sig := <-sc
log.Debug("Get signal to exit", zap.String("signal", sig.String()))
err = ms.Stop()
if err != nil {
panic(err)
}
}

View File

@ -1,74 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
package main
import (
"context"
"os"
"os/signal"
"syscall"
"go.uber.org/zap"
"github.com/milvus-io/milvus/cmd/distributed/components"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/logutil"
"github.com/milvus-io/milvus/internal/msgstream"
"github.com/milvus-io/milvus/internal/proxynode"
)
func main() {
os.Setenv("DEPLOY_MODE", "DISTRIBUTED")
ctx, cancel := context.WithCancel(context.Background())
msFactory := msgstream.NewPmsFactory()
proxynode.Params.Init()
logutil.SetupLogger(&proxynode.Params.Log)
n, err := components.NewProxyNode(ctx, msFactory)
if err != nil {
log.Error("create server failed", zap.Error(err))
}
sc := make(chan os.Signal, 1)
signal.Notify(sc,
syscall.SIGHUP,
syscall.SIGINT,
syscall.SIGTERM,
syscall.SIGQUIT)
var sig os.Signal
go func() {
sig = <-sc
log.Debug("receive stop signal ...")
cancel()
}()
if err := n.Run(); err != nil {
log.Fatal("Init server failed", zap.Error(err))
}
<-ctx.Done()
log.Debug("Got signal to exit", zap.String("signal", sig.String()))
if err := n.Stop(); err != nil {
log.Fatal("stop server failed", zap.Error(err))
}
switch sig {
case syscall.SIGTERM:
exit(0)
default:
exit(1)
}
}
func exit(code int) {
os.Exit(code)
}

View File

@ -1,73 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
package main
import (
"context"
"os"
"os/signal"
"syscall"
"go.uber.org/zap"
"github.com/milvus-io/milvus/cmd/distributed/components"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/logutil"
"github.com/milvus-io/milvus/internal/msgstream"
"github.com/milvus-io/milvus/internal/proxyservice"
)
func main() {
ctx, cancel := context.WithCancel(context.Background())
msFactory := msgstream.NewPmsFactory()
proxyservice.Params.Init()
logutil.SetupLogger(&proxyservice.Params.Log)
s, err := components.NewProxyService(ctx, msFactory)
if err != nil {
log.Fatal("create proxy service error: " + err.Error())
}
sc := make(chan os.Signal, 1)
signal.Notify(sc,
syscall.SIGHUP,
syscall.SIGINT,
syscall.SIGTERM,
syscall.SIGQUIT)
var sig os.Signal
go func() {
sig = <-sc
log.Debug("receive stop signal")
cancel()
}()
if err := s.Run(); err != nil {
log.Fatal("init server failed", zap.Error(err))
}
<-ctx.Done()
log.Debug("Got signal to exit", zap.String("signal", sig.String()))
if err := s.Stop(); err != nil {
log.Fatal("stop server failed", zap.Error(err))
}
switch sig {
case syscall.SIGTERM:
exit(0)
default:
exit(1)
}
}
func exit(code int) {
os.Exit(code)
}

View File

@ -1,65 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
package main
import (
"context"
"os"
"os/signal"
"syscall"
"go.uber.org/zap"
distributed "github.com/milvus-io/milvus/cmd/distributed/components"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/logutil"
"github.com/milvus-io/milvus/internal/msgstream"
"github.com/milvus-io/milvus/internal/querynode"
)
func main() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
querynode.Params.Init()
logutil.SetupLogger(&querynode.Params.Log)
defer func() {
if err := log.Sync(); err != nil {
panic(err)
}
}()
msFactory := msgstream.NewPmsFactory()
svr, err := distributed.NewQueryNode(ctx, msFactory)
if err != nil {
panic(err)
}
if err = svr.Run(); err != nil {
panic(err)
}
sc := make(chan os.Signal, 1)
signal.Notify(sc,
syscall.SIGHUP,
syscall.SIGINT,
syscall.SIGTERM,
syscall.SIGQUIT)
sig := <-sc
log.Debug("Get signal to exit", zap.String("signal", sig.String()))
if err := svr.Stop(); err != nil {
panic(err)
}
}

View File

@ -1,65 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
package main
import (
"context"
"os"
"os/signal"
"syscall"
"github.com/milvus-io/milvus/internal/logutil"
"go.uber.org/zap"
distributed "github.com/milvus-io/milvus/cmd/distributed/components"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/msgstream"
"github.com/milvus-io/milvus/internal/queryservice"
)
func main() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
queryservice.Params.Init()
logutil.SetupLogger(&queryservice.Params.Log)
defer func() {
if err := log.Sync(); err != nil {
panic(err)
}
}()
msFactory := msgstream.NewPmsFactory()
svr, err := distributed.NewQueryService(ctx, msFactory)
if err != nil {
panic(err)
}
if err := svr.Run(); err != nil {
panic(err)
}
sc := make(chan os.Signal, 1)
signal.Notify(sc,
syscall.SIGHUP,
syscall.SIGINT,
syscall.SIGTERM,
syscall.SIGQUIT)
sig := <-sc
log.Debug("Get signal to exit", zap.String("signal", sig.String()))
if err := svr.Stop(); err != nil {
panic(err)
}
}