2021-10-09 18:03:34 +08:00
|
|
|
// Licensed to the LF AI & Data foundation under one
|
|
|
|
// or more contributor license agreements. See the NOTICE file
|
|
|
|
// distributed with this work for additional information
|
|
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
|
|
// to you under the Apache License, Version 2.0 (the
|
|
|
|
// "License"); you may not use this file except in compliance
|
2021-06-18 15:20:08 +08:00
|
|
|
// with the License. You may obtain a copy of the License at
|
|
|
|
//
|
2021-10-09 18:03:34 +08:00
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
2021-06-18 15:20:08 +08:00
|
|
|
//
|
2021-10-09 18:03:34 +08:00
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
2021-06-18 15:20:08 +08:00
|
|
|
|
|
|
|
package roles
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
2022-11-14 14:23:07 +08:00
|
|
|
"fmt"
|
2021-06-18 15:20:08 +08:00
|
|
|
"os"
|
|
|
|
"os/signal"
|
2023-05-26 15:07:26 +08:00
|
|
|
"path/filepath"
|
2021-06-18 15:20:08 +08:00
|
|
|
"strings"
|
|
|
|
"sync"
|
|
|
|
"syscall"
|
2022-12-07 18:01:19 +08:00
|
|
|
"time"
|
2021-06-18 15:20:08 +08:00
|
|
|
|
2023-09-21 09:45:27 +08:00
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
|
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
|
|
|
"go.uber.org/zap"
|
|
|
|
|
2021-06-18 15:20:08 +08:00
|
|
|
"github.com/milvus-io/milvus/cmd/components"
|
2023-04-06 19:14:32 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/http"
|
|
|
|
"github.com/milvus-io/milvus/internal/http/healthz"
|
2023-01-06 14:21:37 +08:00
|
|
|
rocksmqimpl "github.com/milvus-io/milvus/internal/mq/mqimpl/rocksmq/server"
|
2022-10-10 15:55:22 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/util/dependency"
|
2023-10-27 07:36:12 +08:00
|
|
|
kvfactory "github.com/milvus-io/milvus/internal/util/dependency/kv"
|
2023-08-16 14:49:32 +08:00
|
|
|
internalmetrics "github.com/milvus-io/milvus/internal/util/metrics"
|
2023-04-06 19:14:32 +08:00
|
|
|
"github.com/milvus-io/milvus/pkg/log"
|
|
|
|
"github.com/milvus-io/milvus/pkg/metrics"
|
2024-02-26 11:38:55 +08:00
|
|
|
"github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper/nmq"
|
2023-04-06 19:14:32 +08:00
|
|
|
"github.com/milvus-io/milvus/pkg/tracer"
|
|
|
|
"github.com/milvus-io/milvus/pkg/util/etcd"
|
2023-12-22 18:36:44 +08:00
|
|
|
"github.com/milvus-io/milvus/pkg/util/expr"
|
2023-09-14 13:37:18 +08:00
|
|
|
"github.com/milvus-io/milvus/pkg/util/generic"
|
2023-04-06 19:14:32 +08:00
|
|
|
"github.com/milvus-io/milvus/pkg/util/logutil"
|
|
|
|
"github.com/milvus-io/milvus/pkg/util/metricsinfo"
|
|
|
|
"github.com/milvus-io/milvus/pkg/util/paramtable"
|
|
|
|
_ "github.com/milvus-io/milvus/pkg/util/symbolizer" // support symbolizer and crash dump
|
|
|
|
"github.com/milvus-io/milvus/pkg/util/typeutil"
|
2021-06-18 15:20:08 +08:00
|
|
|
)
|
|
|
|
|
2022-03-18 15:51:22 +08:00
|
|
|
// all milvus related metrics is in a separate registry
|
2023-08-16 14:49:32 +08:00
|
|
|
var Registry *internalmetrics.MilvusRegistry
|
2022-03-18 15:51:22 +08:00
|
|
|
|
|
|
|
func init() {
|
2023-08-16 14:49:32 +08:00
|
|
|
Registry = internalmetrics.NewMilvusRegistry()
|
2023-06-26 17:52:44 +08:00
|
|
|
metrics.Register(Registry.GoRegistry)
|
|
|
|
metrics.RegisterMetaMetrics(Registry.GoRegistry)
|
|
|
|
metrics.RegisterMsgStreamMetrics(Registry.GoRegistry)
|
|
|
|
metrics.RegisterStorageMetrics(Registry.GoRegistry)
|
2022-03-18 15:51:22 +08:00
|
|
|
}
|
|
|
|
|
2021-12-02 10:27:32 +08:00
|
|
|
func stopRocksmq() {
|
2022-03-03 21:57:56 +08:00
|
|
|
rocksmqimpl.CloseRocksMQ()
|
2021-12-02 10:27:32 +08:00
|
|
|
}
|
|
|
|
|
2022-10-14 14:19:24 +08:00
|
|
|
type component interface {
|
|
|
|
healthz.Indicator
|
|
|
|
Run() error
|
|
|
|
Stop() error
|
|
|
|
}
|
|
|
|
|
2023-05-26 15:07:26 +08:00
|
|
|
func cleanLocalDir(path string) {
|
|
|
|
_, statErr := os.Stat(path)
|
|
|
|
// path exist, but stat error
|
|
|
|
if statErr != nil && !os.IsNotExist(statErr) {
|
|
|
|
log.Warn("Check if path exists failed when clean local data cache", zap.Error(statErr))
|
|
|
|
panic(statErr)
|
|
|
|
}
|
|
|
|
// path exist, remove all
|
|
|
|
if statErr == nil {
|
|
|
|
err := os.RemoveAll(path)
|
|
|
|
if err != nil {
|
|
|
|
log.Warn("Clean local data cache failed", zap.Error(err))
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
log.Info("Clean local data cache", zap.String("path", path))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-10-14 14:19:24 +08:00
|
|
|
func runComponent[T component](ctx context.Context,
|
|
|
|
localMsg bool,
|
2023-01-12 16:09:39 +08:00
|
|
|
runWg *sync.WaitGroup,
|
2022-10-14 14:19:24 +08:00
|
|
|
creator func(context.Context, dependency.Factory) (T, error),
|
2023-06-07 10:00:37 +08:00
|
|
|
metricRegister func(*prometheus.Registry),
|
2023-09-14 13:37:18 +08:00
|
|
|
) component {
|
2022-10-14 14:19:24 +08:00
|
|
|
var role T
|
|
|
|
|
2023-08-29 19:28:28 +08:00
|
|
|
sign := make(chan struct{})
|
2022-10-14 14:19:24 +08:00
|
|
|
go func() {
|
|
|
|
factory := dependency.NewFactory(localMsg)
|
|
|
|
var err error
|
|
|
|
role, err = creator(ctx, factory)
|
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
2023-08-29 19:28:28 +08:00
|
|
|
close(sign)
|
2023-08-09 14:33:15 +08:00
|
|
|
if err := role.Run(); err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
2023-01-12 16:09:39 +08:00
|
|
|
runWg.Done()
|
2022-10-14 14:19:24 +08:00
|
|
|
}()
|
2023-08-29 19:28:28 +08:00
|
|
|
|
|
|
|
<-sign
|
2022-10-14 14:19:24 +08:00
|
|
|
|
|
|
|
healthz.Register(role)
|
2023-04-07 15:34:29 +08:00
|
|
|
metricRegister(Registry.GoRegistry)
|
2023-09-14 13:37:18 +08:00
|
|
|
if generic.IsZero(role) {
|
|
|
|
return nil
|
|
|
|
}
|
2022-10-14 14:19:24 +08:00
|
|
|
return role
|
|
|
|
}
|
|
|
|
|
2022-02-24 14:41:52 +08:00
|
|
|
// MilvusRoles decides which components are brought up with Milvus.
|
2021-06-18 15:20:08 +08:00
|
|
|
type MilvusRoles struct {
|
2021-12-06 18:20:31 +08:00
|
|
|
EnableRootCoord bool `env:"ENABLE_ROOT_COORD"`
|
|
|
|
EnableProxy bool `env:"ENABLE_PROXY"`
|
|
|
|
EnableQueryCoord bool `env:"ENABLE_QUERY_COORD"`
|
|
|
|
EnableQueryNode bool `env:"ENABLE_QUERY_NODE"`
|
|
|
|
EnableDataCoord bool `env:"ENABLE_DATA_COORD"`
|
|
|
|
EnableDataNode bool `env:"ENABLE_DATA_NODE"`
|
|
|
|
EnableIndexCoord bool `env:"ENABLE_INDEX_COORD"`
|
|
|
|
EnableIndexNode bool `env:"ENABLE_INDEX_NODE"`
|
2023-08-29 19:28:28 +08:00
|
|
|
|
2023-09-05 10:31:48 +08:00
|
|
|
Local bool
|
2023-10-16 10:24:10 +08:00
|
|
|
Alias string
|
2023-09-05 10:31:48 +08:00
|
|
|
Embedded bool
|
2024-01-26 14:05:01 +08:00
|
|
|
|
|
|
|
ServerType string
|
2023-10-16 10:24:10 +08:00
|
|
|
|
|
|
|
closed chan struct{}
|
|
|
|
once sync.Once
|
2023-08-29 19:28:28 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
// NewMilvusRoles creates a new MilvusRoles with private fields initialized.
|
|
|
|
func NewMilvusRoles() *MilvusRoles {
|
|
|
|
mr := &MilvusRoles{
|
|
|
|
closed: make(chan struct{}),
|
|
|
|
}
|
|
|
|
return mr
|
2021-06-18 15:20:08 +08:00
|
|
|
}
|
|
|
|
|
2021-11-01 22:35:18 +08:00
|
|
|
// EnvValue not used now.
|
2021-06-18 15:20:08 +08:00
|
|
|
func (mr *MilvusRoles) EnvValue(env string) bool {
|
|
|
|
env = strings.ToLower(env)
|
|
|
|
env = strings.Trim(env, " ")
|
|
|
|
return env == "1" || env == "true"
|
|
|
|
}
|
|
|
|
|
2022-06-10 21:58:08 +08:00
|
|
|
func (mr *MilvusRoles) printLDPreLoad() {
|
|
|
|
const LDPreLoad = "LD_PRELOAD"
|
|
|
|
val, ok := os.LookupEnv(LDPreLoad)
|
|
|
|
if ok {
|
|
|
|
log.Info("Enable Jemalloc", zap.String("Jemalloc Path", val))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-09-14 13:37:18 +08:00
|
|
|
func (mr *MilvusRoles) runRootCoord(ctx context.Context, localMsg bool, wg *sync.WaitGroup) component {
|
2023-01-12 16:09:39 +08:00
|
|
|
wg.Add(1)
|
|
|
|
return runComponent(ctx, localMsg, wg, components.NewRootCoord, metrics.RegisterRootCoord)
|
2021-06-18 15:20:08 +08:00
|
|
|
}
|
|
|
|
|
2023-09-14 13:37:18 +08:00
|
|
|
func (mr *MilvusRoles) runProxy(ctx context.Context, localMsg bool, wg *sync.WaitGroup) component {
|
2023-01-12 16:09:39 +08:00
|
|
|
wg.Add(1)
|
|
|
|
return runComponent(ctx, localMsg, wg, components.NewProxy, metrics.RegisterProxy)
|
2021-06-18 15:20:08 +08:00
|
|
|
}
|
|
|
|
|
2023-09-14 13:37:18 +08:00
|
|
|
func (mr *MilvusRoles) runQueryCoord(ctx context.Context, localMsg bool, wg *sync.WaitGroup) component {
|
2023-01-12 16:09:39 +08:00
|
|
|
wg.Add(1)
|
|
|
|
return runComponent(ctx, localMsg, wg, components.NewQueryCoord, metrics.RegisterQueryCoord)
|
2021-06-18 15:20:08 +08:00
|
|
|
}
|
|
|
|
|
2023-09-14 13:37:18 +08:00
|
|
|
func (mr *MilvusRoles) runQueryNode(ctx context.Context, localMsg bool, wg *sync.WaitGroup) component {
|
2023-01-12 16:09:39 +08:00
|
|
|
wg.Add(1)
|
2023-09-15 10:21:20 +08:00
|
|
|
// clear local storage
|
2023-05-26 15:07:26 +08:00
|
|
|
rootPath := paramtable.Get().LocalStorageCfg.Path.GetValue()
|
|
|
|
queryDataLocalPath := filepath.Join(rootPath, typeutil.QueryNodeRole)
|
|
|
|
cleanLocalDir(queryDataLocalPath)
|
2023-09-15 10:21:20 +08:00
|
|
|
// clear mmap dir
|
|
|
|
mmapDir := paramtable.Get().QueryNodeCfg.MmapDirPath.GetValue()
|
|
|
|
if len(mmapDir) > 0 {
|
|
|
|
cleanLocalDir(mmapDir)
|
|
|
|
}
|
2023-05-26 15:07:26 +08:00
|
|
|
|
2023-01-12 16:09:39 +08:00
|
|
|
return runComponent(ctx, localMsg, wg, components.NewQueryNode, metrics.RegisterQueryNode)
|
2021-06-18 15:20:08 +08:00
|
|
|
}
|
|
|
|
|
2023-09-14 13:37:18 +08:00
|
|
|
func (mr *MilvusRoles) runDataCoord(ctx context.Context, localMsg bool, wg *sync.WaitGroup) component {
|
2023-01-12 16:09:39 +08:00
|
|
|
wg.Add(1)
|
|
|
|
return runComponent(ctx, localMsg, wg, components.NewDataCoord, metrics.RegisterDataCoord)
|
2021-06-18 15:20:08 +08:00
|
|
|
}
|
|
|
|
|
2023-09-14 13:37:18 +08:00
|
|
|
func (mr *MilvusRoles) runDataNode(ctx context.Context, localMsg bool, wg *sync.WaitGroup) component {
|
2023-01-12 16:09:39 +08:00
|
|
|
wg.Add(1)
|
|
|
|
return runComponent(ctx, localMsg, wg, components.NewDataNode, metrics.RegisterDataNode)
|
2021-06-18 15:20:08 +08:00
|
|
|
}
|
|
|
|
|
2023-09-14 13:37:18 +08:00
|
|
|
func (mr *MilvusRoles) runIndexCoord(ctx context.Context, localMsg bool, wg *sync.WaitGroup) component {
|
2023-01-12 16:09:39 +08:00
|
|
|
wg.Add(1)
|
|
|
|
return runComponent(ctx, localMsg, wg, components.NewIndexCoord, func(registry *prometheus.Registry) {})
|
2021-06-18 15:20:08 +08:00
|
|
|
}
|
|
|
|
|
2023-09-14 13:37:18 +08:00
|
|
|
func (mr *MilvusRoles) runIndexNode(ctx context.Context, localMsg bool, wg *sync.WaitGroup) component {
|
2023-01-12 16:09:39 +08:00
|
|
|
wg.Add(1)
|
2023-05-26 15:07:26 +08:00
|
|
|
rootPath := paramtable.Get().LocalStorageCfg.Path.GetValue()
|
|
|
|
indexDataLocalPath := filepath.Join(rootPath, typeutil.IndexNodeRole)
|
|
|
|
cleanLocalDir(indexDataLocalPath)
|
|
|
|
|
2023-01-12 16:09:39 +08:00
|
|
|
return runComponent(ctx, localMsg, wg, components.NewIndexNode, metrics.RegisterIndexNode)
|
2021-06-18 15:20:08 +08:00
|
|
|
}
|
|
|
|
|
2022-11-08 14:33:03 +08:00
|
|
|
func (mr *MilvusRoles) setupLogger() {
|
2023-02-23 11:37:46 +08:00
|
|
|
params := paramtable.Get()
|
|
|
|
logConfig := log.Config{
|
|
|
|
Level: params.LogCfg.Level.GetValue(),
|
|
|
|
GrpcLevel: params.LogCfg.GrpcLogLevel.GetValue(),
|
|
|
|
Format: params.LogCfg.Format.GetValue(),
|
|
|
|
Stdout: params.LogCfg.Stdout.GetAsBool(),
|
|
|
|
File: log.FileLogConfig{
|
|
|
|
RootPath: params.LogCfg.RootPath.GetValue(),
|
|
|
|
MaxSize: params.LogCfg.MaxSize.GetAsInt(),
|
|
|
|
MaxDays: params.LogCfg.MaxAge.GetAsInt(),
|
|
|
|
MaxBackups: params.LogCfg.MaxBackups.GetAsInt(),
|
|
|
|
},
|
|
|
|
}
|
2022-11-08 14:33:03 +08:00
|
|
|
id := paramtable.GetNodeID()
|
|
|
|
roleName := paramtable.GetRole()
|
|
|
|
rootPath := logConfig.File.RootPath
|
|
|
|
if rootPath != "" {
|
2022-11-14 14:23:07 +08:00
|
|
|
logConfig.File.Filename = fmt.Sprintf("%s-%d.log", roleName, id)
|
2022-11-08 14:33:03 +08:00
|
|
|
} else {
|
|
|
|
logConfig.File.Filename = ""
|
|
|
|
}
|
|
|
|
|
|
|
|
logutil.SetupLogger(&logConfig)
|
|
|
|
}
|
|
|
|
|
2023-04-06 19:14:32 +08:00
|
|
|
// Register serves prometheus http service
|
2023-08-16 14:49:32 +08:00
|
|
|
func setupPrometheusHTTPServer(r *internalmetrics.MilvusRegistry) {
|
2023-10-19 17:14:10 +08:00
|
|
|
log.Info("setupPrometheusHTTPServer")
|
2023-04-06 19:14:32 +08:00
|
|
|
http.Register(&http.Handler{
|
|
|
|
Path: "/metrics",
|
|
|
|
Handler: promhttp.HandlerFor(r, promhttp.HandlerOpts{}),
|
|
|
|
})
|
|
|
|
http.Register(&http.Handler{
|
|
|
|
Path: "/metrics_default",
|
|
|
|
Handler: promhttp.Handler(),
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2023-08-29 19:28:28 +08:00
|
|
|
func (mr *MilvusRoles) handleSignals() func() {
|
|
|
|
sign := make(chan struct{})
|
|
|
|
done := make(chan struct{})
|
|
|
|
|
|
|
|
sc := make(chan os.Signal, 1)
|
|
|
|
signal.Notify(sc,
|
|
|
|
syscall.SIGHUP,
|
|
|
|
syscall.SIGINT,
|
|
|
|
syscall.SIGTERM,
|
|
|
|
syscall.SIGQUIT)
|
|
|
|
|
|
|
|
go func() {
|
|
|
|
defer close(done)
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-sign:
|
|
|
|
log.Info("All cleanup done, handleSignals goroutine quit")
|
|
|
|
return
|
|
|
|
case sig := <-sc:
|
|
|
|
log.Warn("Get signal to exit", zap.String("signal", sig.String()))
|
|
|
|
mr.once.Do(func() {
|
|
|
|
close(mr.closed)
|
2023-08-31 16:51:02 +08:00
|
|
|
// reset other signals, only handle SIGINT from now
|
|
|
|
signal.Reset(syscall.SIGQUIT, syscall.SIGHUP, syscall.SIGTERM)
|
2023-08-29 19:28:28 +08:00
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
return func() {
|
|
|
|
close(sign)
|
|
|
|
<-done
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-12-13 09:29:50 +08:00
|
|
|
// Run Milvus components.
|
2023-10-16 10:24:10 +08:00
|
|
|
func (mr *MilvusRoles) Run() {
|
2023-08-29 19:28:28 +08:00
|
|
|
// start signal handler, defer close func
|
|
|
|
closeFn := mr.handleSignals()
|
|
|
|
defer closeFn()
|
|
|
|
|
2022-04-14 20:01:34 +08:00
|
|
|
log.Info("starting running Milvus components")
|
2021-06-18 15:20:08 +08:00
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
2023-08-29 19:28:28 +08:00
|
|
|
defer cancel()
|
|
|
|
|
2022-06-10 21:58:08 +08:00
|
|
|
mr.printLDPreLoad()
|
2021-06-18 15:20:08 +08:00
|
|
|
|
|
|
|
// only standalone enable localMsg
|
2023-09-05 10:31:48 +08:00
|
|
|
if mr.Local {
|
2021-12-14 18:31:25 +08:00
|
|
|
if err := os.Setenv(metricsinfo.DeployModeEnvKey, metricsinfo.StandaloneDeployMode); err != nil {
|
|
|
|
log.Error("Failed to set deploy mode: ", zap.Error(err))
|
|
|
|
}
|
2022-12-16 15:59:23 +08:00
|
|
|
|
2023-09-05 10:31:48 +08:00
|
|
|
if mr.Embedded {
|
|
|
|
// setup config for embedded milvus
|
|
|
|
paramtable.InitWithBaseTable(paramtable.NewBaseTable(paramtable.Files([]string{"embedded-milvus.yaml"})))
|
|
|
|
} else {
|
|
|
|
paramtable.Init()
|
|
|
|
}
|
2024-02-26 11:38:55 +08:00
|
|
|
|
2022-11-04 14:25:38 +08:00
|
|
|
params := paramtable.Get()
|
2024-02-26 11:38:55 +08:00
|
|
|
if paramtable.Get().RocksmqEnable() {
|
|
|
|
defer stopRocksmq()
|
|
|
|
} else if paramtable.Get().NatsmqEnable() {
|
|
|
|
defer nmq.CloseNatsMQ()
|
|
|
|
} else {
|
|
|
|
panic("only support Rocksmq and Natsmq in standalone mode")
|
|
|
|
}
|
2022-11-17 18:59:09 +08:00
|
|
|
if params.EtcdCfg.UseEmbedEtcd.GetAsBool() {
|
2022-02-24 14:41:52 +08:00
|
|
|
// Start etcd server.
|
2022-11-30 18:23:15 +08:00
|
|
|
etcd.InitEtcdServer(
|
|
|
|
params.EtcdCfg.UseEmbedEtcd.GetAsBool(),
|
|
|
|
params.EtcdCfg.ConfigPath.GetValue(),
|
|
|
|
params.EtcdCfg.DataDir.GetValue(),
|
|
|
|
params.EtcdCfg.EtcdLogPath.GetValue(),
|
|
|
|
params.EtcdCfg.EtcdLogLevel.GetValue())
|
2021-12-29 14:35:21 +08:00
|
|
|
defer etcd.StopEtcdServer()
|
|
|
|
}
|
2023-01-12 16:09:39 +08:00
|
|
|
paramtable.SetRole(typeutil.StandaloneRole)
|
2021-09-03 19:35:47 +08:00
|
|
|
} else {
|
2021-12-14 18:31:25 +08:00
|
|
|
if err := os.Setenv(metricsinfo.DeployModeEnvKey, metricsinfo.ClusterDeployMode); err != nil {
|
2021-10-28 18:02:30 +08:00
|
|
|
log.Error("Failed to set deploy mode: ", zap.Error(err))
|
2021-09-03 19:35:47 +08:00
|
|
|
}
|
2022-11-04 14:25:38 +08:00
|
|
|
paramtable.Init()
|
2024-01-26 14:05:01 +08:00
|
|
|
paramtable.SetRole(mr.ServerType)
|
2021-06-18 15:20:08 +08:00
|
|
|
}
|
|
|
|
|
2023-12-22 18:36:44 +08:00
|
|
|
expr.Init()
|
|
|
|
expr.Register("param", paramtable.Get())
|
2023-04-06 19:14:32 +08:00
|
|
|
http.ServeHTTP()
|
2023-10-19 17:14:10 +08:00
|
|
|
setupPrometheusHTTPServer(Registry)
|
2023-02-02 12:03:58 +08:00
|
|
|
|
2023-01-12 16:09:39 +08:00
|
|
|
var wg sync.WaitGroup
|
2023-09-05 10:31:48 +08:00
|
|
|
local := mr.Local
|
2023-09-14 13:37:18 +08:00
|
|
|
|
|
|
|
var rootCoord, queryCoord, indexCoord, dataCoord component
|
|
|
|
var proxy, dataNode, indexNode, queryNode component
|
2021-06-18 15:20:08 +08:00
|
|
|
if mr.EnableRootCoord {
|
2023-09-14 13:37:18 +08:00
|
|
|
rootCoord = mr.runRootCoord(ctx, local, &wg)
|
2021-06-18 15:20:08 +08:00
|
|
|
}
|
|
|
|
|
2023-11-01 11:40:15 +08:00
|
|
|
if mr.EnableDataCoord {
|
|
|
|
dataCoord = mr.runDataCoord(ctx, local, &wg)
|
|
|
|
}
|
|
|
|
|
|
|
|
if mr.EnableIndexCoord {
|
|
|
|
indexCoord = mr.runIndexCoord(ctx, local, &wg)
|
2021-06-18 15:20:08 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
if mr.EnableQueryCoord {
|
2023-09-14 13:37:18 +08:00
|
|
|
queryCoord = mr.runQueryCoord(ctx, local, &wg)
|
2021-06-18 15:20:08 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
if mr.EnableQueryNode {
|
2023-09-14 13:37:18 +08:00
|
|
|
queryNode = mr.runQueryNode(ctx, local, &wg)
|
2021-06-18 15:20:08 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
if mr.EnableDataNode {
|
2023-09-14 13:37:18 +08:00
|
|
|
dataNode = mr.runDataNode(ctx, local, &wg)
|
2021-06-18 15:20:08 +08:00
|
|
|
}
|
|
|
|
if mr.EnableIndexNode {
|
2023-09-14 13:37:18 +08:00
|
|
|
indexNode = mr.runIndexNode(ctx, local, &wg)
|
2021-06-18 15:20:08 +08:00
|
|
|
}
|
|
|
|
|
2023-11-01 11:40:15 +08:00
|
|
|
if mr.EnableProxy {
|
|
|
|
proxy = mr.runProxy(ctx, local, &wg)
|
|
|
|
}
|
|
|
|
|
2023-01-12 16:09:39 +08:00
|
|
|
wg.Wait()
|
2022-11-14 14:23:07 +08:00
|
|
|
|
2023-01-12 16:09:39 +08:00
|
|
|
mr.setupLogger()
|
|
|
|
tracer.Init()
|
2022-12-07 18:01:19 +08:00
|
|
|
|
|
|
|
paramtable.SetCreateTime(time.Now())
|
|
|
|
paramtable.SetUpdateTime(time.Now())
|
|
|
|
|
2023-08-29 19:28:28 +08:00
|
|
|
<-mr.closed
|
2023-08-31 16:51:02 +08:00
|
|
|
|
|
|
|
// stop coordinators first
|
2023-11-01 11:40:15 +08:00
|
|
|
coordinators := []component{rootCoord, dataCoord, indexCoord, queryCoord}
|
2023-09-14 13:37:18 +08:00
|
|
|
for idx, coord := range coordinators {
|
|
|
|
log.Warn("stop processing")
|
2023-08-31 16:51:02 +08:00
|
|
|
if coord != nil {
|
2023-11-01 11:40:15 +08:00
|
|
|
log.Info("stop coord", zap.Int("idx", idx), zap.Any("coord", coord))
|
|
|
|
coord.Stop()
|
2023-08-31 16:51:02 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
log.Info("All coordinators have stopped")
|
|
|
|
|
|
|
|
// stop nodes
|
2023-09-14 13:37:18 +08:00
|
|
|
nodes := []component{queryNode, indexNode, dataNode}
|
2023-11-01 11:40:15 +08:00
|
|
|
for idx, node := range nodes {
|
2023-08-31 16:51:02 +08:00
|
|
|
if node != nil {
|
2023-11-01 11:40:15 +08:00
|
|
|
log.Info("stop node", zap.Int("idx", idx), zap.Any("node", node))
|
|
|
|
node.Stop()
|
2023-08-31 16:51:02 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
log.Info("All nodes have stopped")
|
|
|
|
|
2023-09-14 13:37:18 +08:00
|
|
|
if proxy != nil {
|
|
|
|
proxy.Stop()
|
2023-11-01 11:40:15 +08:00
|
|
|
log.Info("proxy stopped!")
|
2023-08-31 16:51:02 +08:00
|
|
|
}
|
|
|
|
|
2023-10-27 07:36:12 +08:00
|
|
|
// close reused etcd client
|
|
|
|
kvfactory.CloseEtcdClient()
|
|
|
|
|
2023-08-31 16:51:02 +08:00
|
|
|
log.Info("Milvus components graceful stop done")
|
2021-06-18 15:20:08 +08:00
|
|
|
}
|
2023-10-16 10:24:10 +08:00
|
|
|
|
|
|
|
func (mr *MilvusRoles) GetRoles() []string {
|
|
|
|
roles := make([]string, 0)
|
|
|
|
if mr.EnableRootCoord {
|
|
|
|
roles = append(roles, typeutil.RootCoordRole)
|
|
|
|
}
|
|
|
|
if mr.EnableProxy {
|
|
|
|
roles = append(roles, typeutil.ProxyRole)
|
|
|
|
}
|
|
|
|
if mr.EnableQueryCoord {
|
|
|
|
roles = append(roles, typeutil.QueryCoordRole)
|
|
|
|
}
|
|
|
|
if mr.EnableQueryNode {
|
|
|
|
roles = append(roles, typeutil.QueryNodeRole)
|
|
|
|
}
|
|
|
|
if mr.EnableDataCoord {
|
|
|
|
roles = append(roles, typeutil.DataCoordRole)
|
|
|
|
}
|
|
|
|
if mr.EnableDataNode {
|
|
|
|
roles = append(roles, typeutil.DataNodeRole)
|
|
|
|
}
|
|
|
|
if mr.EnableIndexCoord {
|
|
|
|
roles = append(roles, typeutil.IndexCoordRole)
|
|
|
|
}
|
|
|
|
if mr.EnableIndexNode {
|
|
|
|
roles = append(roles, typeutil.IndexNodeRole)
|
|
|
|
}
|
|
|
|
return roles
|
|
|
|
}
|