2021-06-07 09:47:36 +08:00
|
|
|
// Copyright (C) 2019-2020 Zilliz. All rights reserved.//
|
2021-04-19 11:35:38 +08:00
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
|
|
|
// with the License. You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
|
|
|
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
|
|
|
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
2021-06-22 10:42:07 +08:00
|
|
|
package datacoord
|
2021-01-15 17:09:41 +08:00
|
|
|
|
|
|
|
import (
|
2021-02-19 15:37:04 +08:00
|
|
|
"path"
|
2021-01-26 09:43:41 +08:00
|
|
|
"strconv"
|
2021-06-11 22:04:41 +08:00
|
|
|
"strings"
|
2021-02-06 13:39:15 +08:00
|
|
|
"sync"
|
2021-01-26 09:43:41 +08:00
|
|
|
|
2021-04-22 14:45:57 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/log"
|
2021-02-19 15:37:04 +08:00
|
|
|
|
2021-04-22 14:45:57 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/util/paramtable"
|
2021-01-15 17:09:41 +08:00
|
|
|
)
|
|
|
|
|
|
|
|
type ParamTable struct {
|
|
|
|
paramtable.BaseTable
|
|
|
|
|
2021-02-23 11:40:30 +08:00
|
|
|
NodeID int64
|
2021-01-22 19:43:27 +08:00
|
|
|
|
2021-05-21 19:28:52 +08:00
|
|
|
IP string
|
|
|
|
Port int
|
|
|
|
|
2021-05-20 11:34:45 +08:00
|
|
|
// --- ETCD ---
|
2021-06-11 22:04:41 +08:00
|
|
|
EtcdEndpoints []string
|
2021-05-20 11:34:45 +08:00
|
|
|
MetaRootPath string
|
|
|
|
KvRootPath string
|
|
|
|
SegmentBinlogSubPath string
|
|
|
|
CollectionBinlogSubPath string
|
|
|
|
|
|
|
|
// --- Pulsar ---
|
2021-01-22 11:07:07 +08:00
|
|
|
PulsarAddress string
|
2021-01-15 17:09:41 +08:00
|
|
|
|
2021-05-19 14:13:53 +08:00
|
|
|
FlushStreamPosSubPath string
|
|
|
|
StatsStreamPosSubPath string
|
|
|
|
|
2021-01-15 17:09:41 +08:00
|
|
|
// segment
|
2021-06-22 16:18:21 +08:00
|
|
|
SegmentMaxSize float64
|
|
|
|
SegmentSealProportion float64
|
|
|
|
SegAssignmentExpiration int64
|
2021-01-15 17:09:41 +08:00
|
|
|
|
2021-06-21 11:40:15 +08:00
|
|
|
InsertChannelPrefixName string
|
|
|
|
StatisticsChannelName string
|
|
|
|
TimeTickChannelName string
|
|
|
|
SegmentInfoChannelName string
|
|
|
|
DataCoordSubscriptionName string
|
2021-01-26 15:14:49 +08:00
|
|
|
|
2021-06-08 19:25:37 +08:00
|
|
|
Log log.Config
|
2021-01-15 17:09:41 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
var Params ParamTable
|
2021-02-06 13:39:15 +08:00
|
|
|
var once sync.Once
|
2021-01-15 17:09:41 +08:00
|
|
|
|
|
|
|
func (p *ParamTable) Init() {
|
2021-02-06 13:39:15 +08:00
|
|
|
once.Do(func() {
|
|
|
|
// load yaml
|
|
|
|
p.BaseTable.Init()
|
|
|
|
|
2021-06-22 16:18:21 +08:00
|
|
|
if err := p.LoadYaml("advanced/data_coord.yaml"); err != nil {
|
2021-02-06 13:39:15 +08:00
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// set members
|
2021-06-11 22:04:41 +08:00
|
|
|
p.initEtcdEndpoints()
|
2021-02-06 13:39:15 +08:00
|
|
|
p.initMetaRootPath()
|
|
|
|
p.initKvRootPath()
|
2021-05-20 11:34:45 +08:00
|
|
|
p.initSegmentBinlogSubPath()
|
|
|
|
p.initCollectionBinlogSubPath()
|
|
|
|
|
2021-02-06 13:39:15 +08:00
|
|
|
p.initPulsarAddress()
|
|
|
|
|
2021-06-22 16:18:21 +08:00
|
|
|
p.initSegmentMaxSize()
|
|
|
|
p.initSegmentSealProportion()
|
|
|
|
p.initSegAssignmentExpiration()
|
2021-02-06 13:39:15 +08:00
|
|
|
p.initInsertChannelPrefixName()
|
|
|
|
p.initStatisticsChannelName()
|
|
|
|
p.initTimeTickChannelName()
|
|
|
|
p.initSegmentInfoChannelName()
|
2021-06-21 11:40:15 +08:00
|
|
|
p.initDataCoordSubscriptionName()
|
2021-02-19 15:37:04 +08:00
|
|
|
p.initLogCfg()
|
2021-05-19 14:13:53 +08:00
|
|
|
|
|
|
|
p.initFlushStreamPosSubPath()
|
|
|
|
p.initStatsStreamPosSubPath()
|
2021-02-06 13:39:15 +08:00
|
|
|
})
|
2021-01-15 17:09:41 +08:00
|
|
|
}
|
|
|
|
|
2021-06-11 22:04:41 +08:00
|
|
|
func (p *ParamTable) initEtcdEndpoints() {
|
|
|
|
endpoints, err := p.Load("_EtcdEndpoints")
|
2021-01-15 17:09:41 +08:00
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
2021-06-11 22:04:41 +08:00
|
|
|
p.EtcdEndpoints = strings.Split(endpoints, ",")
|
2021-01-15 17:09:41 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
func (p *ParamTable) initPulsarAddress() {
|
|
|
|
addr, err := p.Load("_PulsarAddress")
|
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
p.PulsarAddress = addr
|
|
|
|
}
|
|
|
|
|
|
|
|
func (p *ParamTable) initMetaRootPath() {
|
|
|
|
rootPath, err := p.Load("etcd.rootPath")
|
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
subPath, err := p.Load("etcd.metaSubPath")
|
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
p.MetaRootPath = rootPath + "/" + subPath
|
|
|
|
}
|
|
|
|
|
|
|
|
func (p *ParamTable) initKvRootPath() {
|
|
|
|
rootPath, err := p.Load("etcd.rootPath")
|
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
subPath, err := p.Load("etcd.kvSubPath")
|
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
p.KvRootPath = rootPath + "/" + subPath
|
|
|
|
}
|
2021-05-20 11:34:45 +08:00
|
|
|
|
|
|
|
func (p *ParamTable) initSegmentBinlogSubPath() {
|
|
|
|
subPath, err := p.Load("etcd.segmentBinlogSubPath")
|
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
p.SegmentBinlogSubPath = subPath
|
|
|
|
}
|
|
|
|
|
|
|
|
func (p *ParamTable) initCollectionBinlogSubPath() {
|
|
|
|
subPath, err := p.Load("etcd.collectionBinlogSubPath")
|
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
p.CollectionBinlogSubPath = subPath
|
|
|
|
}
|
|
|
|
|
2021-06-22 16:18:21 +08:00
|
|
|
func (p *ParamTable) initSegmentMaxSize() {
|
|
|
|
p.SegmentMaxSize = p.ParseFloat("datacoord.segment.maxSize")
|
2021-01-15 17:09:41 +08:00
|
|
|
}
|
|
|
|
|
2021-06-22 16:18:21 +08:00
|
|
|
func (p *ParamTable) initSegmentSealProportion() {
|
|
|
|
p.SegmentSealProportion = p.ParseFloat("datacoord.segment.sealProportion")
|
2021-01-15 17:09:41 +08:00
|
|
|
}
|
|
|
|
|
2021-06-22 16:18:21 +08:00
|
|
|
func (p *ParamTable) initSegAssignmentExpiration() {
|
|
|
|
p.SegAssignmentExpiration = p.ParseInt64("datacoord.segment.assignmentExpiration")
|
2021-01-22 19:43:27 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
func (p *ParamTable) initInsertChannelPrefixName() {
|
2021-01-26 09:43:41 +08:00
|
|
|
var err error
|
2021-06-21 11:40:15 +08:00
|
|
|
p.InsertChannelPrefixName, err = p.Load("msgChannel.chanNamePrefix.dataCoordInsertChannel")
|
2021-01-26 09:43:41 +08:00
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
2021-01-22 19:43:27 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
func (p *ParamTable) initStatisticsChannelName() {
|
2021-01-26 09:43:41 +08:00
|
|
|
var err error
|
2021-06-21 11:40:15 +08:00
|
|
|
p.StatisticsChannelName, err = p.Load("msgChannel.chanNamePrefix.dataCoordStatistic")
|
2021-01-26 09:43:41 +08:00
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
2021-01-22 19:43:27 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
func (p *ParamTable) initTimeTickChannelName() {
|
2021-01-26 09:43:41 +08:00
|
|
|
var err error
|
2021-06-21 11:40:15 +08:00
|
|
|
p.TimeTickChannelName, err = p.Load("msgChannel.chanNamePrefix.dataCoordTimeTick")
|
2021-01-26 09:43:41 +08:00
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
2021-01-22 19:43:27 +08:00
|
|
|
}
|
|
|
|
|
2021-01-25 15:17:17 +08:00
|
|
|
func (p *ParamTable) initSegmentInfoChannelName() {
|
2021-01-26 09:43:41 +08:00
|
|
|
var err error
|
2021-06-21 11:40:15 +08:00
|
|
|
p.SegmentInfoChannelName, err = p.Load("msgChannel.chanNamePrefix.dataCoordSegmentInfo")
|
2021-01-26 09:43:41 +08:00
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
2021-01-25 15:17:17 +08:00
|
|
|
}
|
|
|
|
|
2021-06-21 11:40:15 +08:00
|
|
|
func (p *ParamTable) initDataCoordSubscriptionName() {
|
2021-01-26 09:43:41 +08:00
|
|
|
var err error
|
2021-06-21 11:40:15 +08:00
|
|
|
p.DataCoordSubscriptionName, err = p.Load("msgChannel.subNamePrefix.dataCoordSubNamePrefix")
|
2021-01-26 09:43:41 +08:00
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-02-19 15:37:04 +08:00
|
|
|
func (p *ParamTable) initLogCfg() {
|
|
|
|
p.Log = log.Config{}
|
|
|
|
format, err := p.Load("log.format")
|
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
p.Log.Format = format
|
|
|
|
level, err := p.Load("log.level")
|
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
p.Log.Level = level
|
|
|
|
p.Log.File.MaxSize = p.ParseInt("log.file.maxSize")
|
|
|
|
p.Log.File.MaxBackups = p.ParseInt("log.file.maxBackups")
|
|
|
|
p.Log.File.MaxDays = p.ParseInt("log.file.maxAge")
|
|
|
|
rootPath, err := p.Load("log.file.rootPath")
|
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
2021-03-08 16:24:55 +08:00
|
|
|
if len(rootPath) != 0 {
|
2021-06-21 11:40:15 +08:00
|
|
|
p.Log.File.Filename = path.Join(rootPath, "datacoord-"+strconv.FormatInt(p.NodeID, 10)+".log")
|
2021-03-08 16:24:55 +08:00
|
|
|
} else {
|
|
|
|
p.Log.File.Filename = ""
|
|
|
|
}
|
2021-02-19 15:37:04 +08:00
|
|
|
}
|
2021-03-27 14:01:52 +08:00
|
|
|
|
2021-05-19 14:13:53 +08:00
|
|
|
func (p *ParamTable) initFlushStreamPosSubPath() {
|
|
|
|
subPath, err := p.Load("etcd.flushStreamPosSubPath")
|
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
p.FlushStreamPosSubPath = subPath
|
|
|
|
}
|
|
|
|
|
|
|
|
func (p *ParamTable) initStatsStreamPosSubPath() {
|
|
|
|
subPath, err := p.Load("etcd.statsStreamPosSubPath")
|
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
p.StatsStreamPosSubPath = subPath
|
|
|
|
}
|