// Copyright (C) 2019-2020 Zilliz. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software distributed under the License // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // or implied. See the License for the specific language governing permissions and limitations under the License. package querynode import ( "fmt" "path" "strconv" "strings" "sync" "github.com/milvus-io/milvus/internal/log" "github.com/milvus-io/milvus/internal/util/paramtable" ) type ParamTable struct { paramtable.BaseTable PulsarAddress string RocksmqPath string EtcdEndpoints []string MetaRootPath string Alias string QueryNodeIP string QueryNodePort int64 QueryNodeID UniqueID // channel prefix ClusterChannelPrefix string QueryTimeTickChannelName string StatsChannelName string MsgChannelSubName string FlowGraphMaxQueueLength int32 FlowGraphMaxParallelism int32 // minio MinioEndPoint string MinioAccessKeyID string MinioSecretAccessKey string MinioUseSSLStr bool MinioBucketName string // search SearchChannelNames []string SearchResultChannelNames []string SearchReceiveBufSize int64 SearchPulsarBufSize int64 SearchResultReceiveBufSize int64 // Retrieve RetrieveChannelNames []string RetrieveResultChannelNames []string RetrieveReceiveBufSize int64 RetrievePulsarBufSize int64 RetrieveResultReceiveBufSize int64 // stats StatsPublishInterval int GracefulTime int64 SliceIndex int // segcore ChunkRows int64 SimdType string Log log.Config } var Params ParamTable var once sync.Once func (p *ParamTable) InitAlias(alias string) { p.Alias = alias } func (p *ParamTable) InitOnce() { once.Do(func() { p.Init() }) } func (p *ParamTable) Init() { p.BaseTable.Init() if err := p.LoadYaml("advanced/query_node.yaml"); err != nil { panic(err) } if err := p.LoadYaml("advanced/knowhere.yaml"); err != nil { panic(err) } p.initMinioEndPoint() p.initMinioAccessKeyID() p.initMinioSecretAccessKey() p.initMinioUseSSLStr() p.initMinioBucketName() p.initPulsarAddress() p.initRocksmqPath() p.initEtcdEndpoints() p.initMetaRootPath() p.initGracefulTime() p.initFlowGraphMaxQueueLength() p.initFlowGraphMaxParallelism() p.initSearchReceiveBufSize() p.initSearchPulsarBufSize() p.initSearchResultReceiveBufSize() // Has to init global msgchannel prefix before other channel names p.initClusterMsgChannelPrefix() p.initQueryTimeTickChannelName() p.initStatsChannelName() p.initMsgChannelSubName() p.initStatsPublishInterval() p.initSegcoreChunkRows() p.initKnowhereSimdType() p.initLogCfg() } // ---------------------------------------------------------- minio func (p *ParamTable) initMinioEndPoint() { url, err := p.Load("_MinioAddress") if err != nil { panic(err) } p.MinioEndPoint = url } func (p *ParamTable) initMinioAccessKeyID() { id, err := p.Load("minio.accessKeyID") if err != nil { panic(err) } p.MinioAccessKeyID = id } func (p *ParamTable) initMinioSecretAccessKey() { key, err := p.Load("minio.secretAccessKey") if err != nil { panic(err) } p.MinioSecretAccessKey = key } func (p *ParamTable) initMinioUseSSLStr() { ssl, err := p.Load("minio.useSSL") if err != nil { panic(err) } sslBoolean, err := strconv.ParseBool(ssl) if err != nil { panic(err) } p.MinioUseSSLStr = sslBoolean } func (p *ParamTable) initMinioBucketName() { bucketName, err := p.Load("minio.bucketName") if err != nil { panic(err) } p.MinioBucketName = bucketName } func (p *ParamTable) initPulsarAddress() { url, err := p.Load("_PulsarAddress") if err != nil { panic(err) } p.PulsarAddress = url } func (p *ParamTable) initRocksmqPath() { path, err := p.Load("_RocksmqPath") if err != nil { panic(err) } p.RocksmqPath = path } // advanced params // stats func (p *ParamTable) initStatsPublishInterval() { p.StatsPublishInterval = p.ParseInt("queryNode.stats.publishInterval") } // dataSync: func (p *ParamTable) initFlowGraphMaxQueueLength() { p.FlowGraphMaxQueueLength = p.ParseInt32("queryNode.dataSync.flowGraph.maxQueueLength") } func (p *ParamTable) initFlowGraphMaxParallelism() { p.FlowGraphMaxParallelism = p.ParseInt32("queryNode.dataSync.flowGraph.maxParallelism") } // msgStream func (p *ParamTable) initSearchReceiveBufSize() { p.SearchReceiveBufSize = p.ParseInt64("queryNode.msgStream.search.recvBufSize") } func (p *ParamTable) initSearchPulsarBufSize() { p.SearchPulsarBufSize = p.ParseInt64("queryNode.msgStream.search.pulsarBufSize") } func (p *ParamTable) initSearchResultReceiveBufSize() { p.SearchResultReceiveBufSize = p.ParseInt64("queryNode.msgStream.searchResult.recvBufSize") } // ------------------------ channel names func (p *ParamTable) initClusterMsgChannelPrefix() { name, err := p.Load("msgChannel.chanNamePrefix.cluster") if err != nil { panic(err) } p.ClusterChannelPrefix = name } func (p *ParamTable) initQueryTimeTickChannelName() { config, err := p.Load("msgChannel.chanNamePrefix.queryTimeTick") if err != nil { log.Warn(err.Error()) } s := []string{p.ClusterChannelPrefix, config} p.QueryTimeTickChannelName = strings.Join(s, "-") } func (p *ParamTable) initMsgChannelSubName() { namePrefix, err := p.Load("msgChannel.subNamePrefix.queryNodeSubNamePrefix") if err != nil { log.Warn(err.Error()) } s := []string{p.ClusterChannelPrefix, namePrefix, strconv.FormatInt(p.QueryNodeID, 10)} p.MsgChannelSubName = strings.Join(s, "-") } func (p *ParamTable) initStatsChannelName() { config, err := p.Load("msgChannel.chanNamePrefix.queryNodeStats") if err != nil { panic(err) } s := []string{p.ClusterChannelPrefix, config} p.StatsChannelName = strings.Join(s, "-") } // ETCD configs func (p *ParamTable) initEtcdEndpoints() { endpoints, err := p.Load("_EtcdEndpoints") if err != nil { panic(err) } p.EtcdEndpoints = strings.Split(endpoints, ",") } func (p *ParamTable) initMetaRootPath() { rootPath, err := p.Load("etcd.rootPath") if err != nil { panic(err) } subPath, err := p.Load("etcd.metaSubPath") if err != nil { panic(err) } p.MetaRootPath = rootPath + "/" + subPath } func (p *ParamTable) initGracefulTime() { p.GracefulTime = p.ParseInt64("queryNode.gracefulTime") } func (p *ParamTable) initSegcoreChunkRows() { p.ChunkRows = p.ParseInt64("queryNode.segcore.chunkRows") } func (p *ParamTable) initKnowhereSimdType() { simdType, err := p.LoadWithDefault("knowhere.simdType", "auto") if err != nil { panic(err) } p.SimdType = simdType } func (p *ParamTable) initLogCfg() { p.Log = log.Config{} format, err := p.Load("log.format") if err != nil { panic(err) } p.Log.Format = format level, err := p.Load("log.level") if err != nil { panic(err) } p.Log.Level = level p.Log.File.MaxSize = p.ParseInt("log.file.maxSize") p.Log.File.MaxBackups = p.ParseInt("log.file.maxBackups") p.Log.File.MaxDays = p.ParseInt("log.file.maxAge") rootPath, err := p.Load("log.file.rootPath") if err != nil { panic(err) } if len(rootPath) != 0 { p.Log.File.Filename = path.Join(rootPath, fmt.Sprintf("querynode-%s.log", p.Alias)) } else { p.Log.File.Filename = "" } }