test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-11-30 02:48:45 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

Remove useless configs in config files

Browse Source 
Signed-off-by: XuanYang-cn <xuan.yang@zilliz.com>
This commit is contained in:
XuanYang-cn 2021-04-13 10:04:39 +08:00 committed by yefu.chen
parent a7dac818ee
commit e281f6a4a1
15 changed files with 24 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
configs/advanced/channel.yaml
View File

@ -22,11 +22,9 @@ msgChannel:
k2s: "k2s"
proxyTimeTick: "proxyTimeTick"
proxyServiceTimeTick: "proxyServiceTimeTick"
writeNodeTimeTick: "writeNodeTimeTick" # GOOSE TODO: remove this
dataNodeTimeTick: "dataNodeTimeTick"
queryTimeTick: "queryTimeTick"
dataNodeSegStatistics: "dataNodeSegStatistics"
# old name: statsChannels: "statistic"
queryNodeStats: "query-node-stats"
# cmd for loadIndex, flush, etc...
cmd: "cmd"
@ -40,7 +38,6 @@ msgChannel:
masterSubNamePrefix: "master"
proxySubNamePrefix: "proxy"
queryNodeSubNamePrefix: "queryNode"
writeNodeSubNamePrefix: "writeNode" # GOOSE TODO: remove this
dataNodeSubNamePrefix: "dataNode"
dataServiceSubNamePrefix: "dataService"

1
configs/advanced/common.yaml
View File

@ -10,6 +10,5 @@
# or implied. See the License for the specific language governing permissions and limitations under the License.
common:
defaultPartitionTag: _default #TODO, remove
defaultPartitionName: "_default"
defaultIndexName: "_default_idx"

4
configs/advanced/data_node.yaml
View File

@ -23,12 +23,10 @@ dataNode:
pulsarBufSize: 64 # pulsar chan buffer size
insert:
#streamBufSize: 1024 # msgPack chan buffer size
recvBufSize: 1024 # msgPack chan buffer size
pulsarBufSize: 1024 # pulsar chan buffer size
delete:
#streamBufSize: 1024 # msgPack chan buffer size
recvBufSize: 1024 # msgPack chan buffer size
pulsarBufSize: 1024 # pulsar chan buffer size
@ -38,5 +36,5 @@ dataNode:
flush:
# max buffer size to flush
insertBufSize: 32000
insertBufSize: 32000 # number of rows
ddBufSize: 20

4
configs/advanced/data_service.yaml
View File

@ -3,11 +3,9 @@ dataservice:
address: "127.0.0.1"
port: 13333
segment:
# old name: segmentThreshold: 536870912
size: 512 # MB
sizeFactor: 0.75
defaultSizePerRecord: 1024
# old name: segmentExpireDuration: 2000
IDAssignExpiration: 2000 # ms
insertChannelNum: 2
dataNodeNum: 1
dataNodeNum: 1

12
configs/advanced/master.yaml
View File

@ -10,18 +10,6 @@
# or implied. See the License for the specific language governing permissions and limitations under the License.
master:
timeSync:
softTimeTickBarrierInterval: 500 # ms
segment:
# old name: segmentThreshold: 536870912
size: 512 # MB
sizeFactor: 0.75
defaultSizePerRecord: 1024
minIDAssignCnt: 1024
maxIDAssignCnt: 16384
# old name: segmentExpireDuration: 2000
IDAssignExpiration: 2000 # ms
maxPartitionNum: 4096
minSegmentSizeToEnableIndex: 1024
nodeID: 100

1
configs/advanced/proxy_node.yaml
View File

@ -14,7 +14,6 @@ proxyNode:
msgStream:
insert:
#streamBufSize: 1024 # msgPack chan buffer size
bufSize: 1024 # msgPack chan buffer size
search:

2
configs/advanced/query_node.yaml
View File

@ -20,7 +20,6 @@ queryNode:
msgStream:
insert:
#streamBufSize: 1024 # msgPack chan buffer size
recvBufSize: 1024 # msgPack chan buffer size
pulsarBufSize: 1024 # pulsar chan buffer size
@ -29,7 +28,6 @@ queryNode:
pulsarBufSize: 64 # pulsar chan buffer size
delete:
#streamBufSize: 1024 # msgPack chan buffer size
recvBufSize: 1024 # msgPack chan buffer size
pulsarBufSize: 1024 # pulsar chan buffer size

24
configs/milvus.yaml
View File

@ -10,10 +10,9 @@
# or implied. See the License for the specific language governing permissions and limitations under the License.
nodeID: # will be deprecated after v0.2
proxyIDList: [0]
nodeID: # will be deprecated later
queryNodeIDList: [1]
dataNodeIDList: [3]
dataNodeIDList: [3]
etcd:
address: localhost
@ -22,10 +21,7 @@ etcd:
metaSubPath: meta # metaRootPath = rootPath + '/' + metaSubPath
kvSubPath: kv # kvRootPath = rootPath + '/' + kvSubPath
segFlushMetaSubPath: datanode/segment # Full Path = rootPath/metaSubPath/segFlushMetaSubPath
ddlFlushMetaSubPath: datanode/ddl # Full Path = rootPath/metaSubPaht/ddlFlushMetaSubPath
writeNodeSegKvSubPath: writer/segment # GOOSE TODO: remove this
writeNodeDDLKvSubPath: writer/ddl # GOOSE TODO: remove this
segThreshold: 10000
ddlFlushMetaSubPath: datanode/ddl # Full Path = rootPath/metaSubPath/ddlFlushMetaSubPath
minio:
address: localhost
@ -38,11 +34,7 @@ minio:
pulsar:
address: localhost
port: 6650
rest-port: 18080 # keep same with pulsar container
maxMessageSize: 5242880 # 5 * 1024 * 1024
authentication: false
user: user-default
token: eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJKb2UifQ.ipevRNuRP6HflG8cFKnmUPtypruRC4fb1DWtoLL62SY
maxMessageSize: 5242880 # 5 * 1024 * 1024 Bytes
master:
address: localhost
@ -60,7 +52,7 @@ proxyService:
port: 21122
queryNode:
gracefulTime: 5000 #ms
gracefulTime: 5000 # ms, for search
port: 21123
indexService:
@ -78,11 +70,11 @@ dataNode:
port: 21124
log:
level: debug
level: debug # info, warn, error, panic, fatal, dpanic,
file:
rootPath: ""
rootPath: "" # default to stdout, stderr
maxSize: 300 # MB
maxAge: 10 # day
maxBackups: 20
dev: true
dev: true # false, change behaviour of dpaniclevel
format: text # text/json

2
docs/developer_guides/chap05_proxy.md
View File

@ -403,7 +403,7 @@ type GlobalParamsTable struct {
MaxNameLength int64
MaxFieldNum int64
MaxDimension int64
DefaultPartitionTag string
DefaultPartitionName string
DefaultIndexName string
}

2
internal/proxynode/impl.go
View File

@ -946,7 +946,7 @@ func (node *ProxyNode) Insert(ctx context.Context, request *milvuspb.InsertReque
rowIDAllocator: node.idAllocator,
}
if len(it.PartitionName) <= 0 {
it.PartitionName = Params.DefaultPartitionTag
it.PartitionName = Params.DefaultPartitionName
}
err := node.sched.DmQueue.Enqueue(it)

28
internal/proxynode/paramtable.go
View File

@ -54,7 +54,7 @@ type ParamTable struct {
MaxNameLength int64
MaxFieldNum int64
MaxDimension int64
DefaultPartitionTag string
DefaultPartitionName string
DefaultIndexName string
PulsarMaxMessageSize int
@ -136,7 +136,6 @@ func (pt *ParamTable) initParams() {
pt.initPulsarAddress()
pt.initQueryNodeIDList()
pt.initQueryNodeNum()
// pt.initProxyID()
pt.initTimeTickInterval()
pt.initInsertChannelNames()
pt.initDeleteChannelNames()
@ -154,7 +153,7 @@ func (pt *ParamTable) initParams() {
pt.initMaxNameLength()
pt.initMaxFieldNum()
pt.initMaxDimension()
pt.initDefaultPartitionTag()
pt.initDefaultPartitionName()
pt.initDefaultIndexName()
pt.initPulsarMaxMessageSize()
@ -191,18 +190,6 @@ func (pt *ParamTable) initQueryNodeIDList() []UniqueID {
return ret
}
func (pt *ParamTable) initProxyID() {
proxyID, err := pt.Load("_proxyID")
if err != nil {
panic(err)
}
ID, err := strconv.Atoi(proxyID)
if err != nil {
panic(err)
}
pt.ProxyID = UniqueID(ID)
}
func (pt *ParamTable) initTimeTickInterval() {
intervalStr, err := pt.Load("proxyNode.timeTickInterval")
if err != nil {
@ -311,11 +298,6 @@ func (pt *ParamTable) initProxySubName() {
if err != nil {
panic(err)
}
pt.ProxySubName = prefix
// proxyIDStr, err := pt.Load("_proxyID")
// if err != nil {
// panic(err)
// }
pt.ProxySubName = prefix + "-" + strconv.Itoa(int(pt.ProxyID))
}
@ -393,12 +375,12 @@ func (pt *ParamTable) initMaxDimension() {
pt.MaxDimension = maxDimension
}
func (pt *ParamTable) initDefaultPartitionTag() {
tag, err := pt.Load("common.defaultPartitionTag")
func (pt *ParamTable) initDefaultPartitionName() {
name, err := pt.Load("common.defaultPartitionName")
if err != nil {
panic(err)
}
pt.DefaultPartitionTag = tag
pt.DefaultPartitionName = name
}
func (pt *ParamTable) initDefaultIndexName() {

2
internal/proxynode/task.go
View File

@ -159,7 +159,7 @@ func (it *InsertTask) Execute(ctx context.Context) error {
return err
}
} else {
partitionID, err = globalMetaCache.GetPartitionID(ctx, collectionName, Params.DefaultPartitionTag)
partitionID, err = globalMetaCache.GetPartitionID(ctx, collectionName, Params.DefaultPartitionName)
if err != nil {
return err
}

1
internal/querynode/load_service_test.go
View File

@ -1122,7 +1122,6 @@ func TestSegmentLoad_Search_Vector(t *testing.T) {
// mock write insert bin log
keyPrefix := path.Join("query-node-seg-manager-test-minio-prefix", strconv.FormatInt(collectionID, 10), strconv.FormatInt(partitionID, 10))
Params.WriteNodeSegKvSubPath = keyPrefix
node := newQueryNodeMock()
defer node.Stop()

16
internal/querynode/param_table.go
View File

@ -14,10 +14,9 @@ import (
type ParamTable struct {
paramtable.BaseTable
PulsarAddress string
ETCDAddress string
MetaRootPath string
WriteNodeSegKvSubPath string
PulsarAddress string
ETCDAddress string
MetaRootPath string
QueryNodeIP string
QueryNodePort int64
@ -106,7 +105,6 @@ func (p *ParamTable) Init() {
p.initPulsarAddress()
p.initETCDAddress()
p.initMetaRootPath()
p.initWriteNodeSegKvSubPath()
p.initGracefulTime()
p.initMsgChannelSubName()
@ -312,14 +310,6 @@ func (p *ParamTable) initMetaRootPath() {
p.MetaRootPath = rootPath + "/" + subPath
}
func (p *ParamTable) initWriteNodeSegKvSubPath() {
subPath, err := p.Load("etcd.writeNodeSegKvSubPath")
if err != nil {
panic(err)
}
p.WriteNodeSegKvSubPath = subPath + "/"
}
func (p *ParamTable) initGracefulTime() {
p.GracefulTime = p.ParseInt64("queryNode.gracefulTime")
}

41
internal/util/paramtable/paramtable.go
View File

@ -329,32 +329,14 @@ func (gp *BaseTable) ParseInt(key string) int {
return value
}
// GOOSE TODO: remove writenode
func (gp *BaseTable) WriteNodeIDList() []UniqueID {
proxyIDStr, err := gp.Load("nodeID.dataNodeIDList")
if err != nil {
panic(err)
}
var ret []UniqueID
proxyIDs := strings.Split(proxyIDStr, ",")
for _, i := range proxyIDs {
v, err := strconv.Atoi(i)
if err != nil {
log.Panicf("load write node id list error, %s", err.Error())
}
ret = append(ret, UniqueID(v))
}
return ret
}
func (gp *BaseTable) DataNodeIDList() []UniqueID {
proxyIDStr, err := gp.Load("nodeID.dataNodeIDList")
datanodeIDStr, err := gp.Load("nodeID.dataNodeIDList")
if err != nil {
panic(err)
}
var ret []UniqueID
proxyIDs := strings.Split(proxyIDStr, ",")
for _, i := range proxyIDs {
datanodeIDs := strings.Split(datanodeIDStr, ",")
for _, i := range datanodeIDs {
v, err := strconv.Atoi(i)
if err != nil {
log.Panicf("load write node id list error, %s", err.Error())
@ -364,23 +346,6 @@ func (gp *BaseTable) DataNodeIDList() []UniqueID {
return ret
}
func (gp *BaseTable) ProxyIDList() []UniqueID {
proxyIDStr, err := gp.Load("nodeID.proxyIDList")
if err != nil {
panic(err)
}
var ret []UniqueID
proxyIDs := strings.Split(proxyIDStr, ",")
for _, i := range proxyIDs {
v, err := strconv.Atoi(i)
if err != nil {
log.Panicf("load proxy id list error, %s", err.Error())
}
ret = append(ret, UniqueID(v))
}
return ret
}
func (gp *BaseTable) QueryNodeIDList() []UniqueID {
queryNodeIDStr, err := gp.Load("nodeID.queryNodeIDList")
if err != nil {