2022-03-10 10:35:59 +08:00
# Licensed to the LF AI & Data foundation under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Related configuration of etcd, used to store Milvus metadata.
etcd :
endpoints :
- localhost:2379
rootPath : by-dev # The root path where data is stored in etcd
metaSubPath : meta # metaRootPath = rootPath + '/' + metaSubPath
kvSubPath : kv # kvRootPath = rootPath + '/' + kvSubPath
log :
# path is one of:
# - "default" as os.Stderr,
# - "stderr" as os.Stderr,
# - "stdout" as os.Stdout,
# - file path to append server logs to.
2022-04-25 14:17:45 +08:00
# please adjust in embedded Milvus: /tmp/milvus/logs/etcd.log
2022-03-10 10:35:59 +08:00
path : /tmp/milvus/logs/etcd.log
level : info # Only supports debug, info, warn, error, panic, or fatal. Default 'info'.
use :
2022-04-25 14:17:45 +08:00
# please adjust in embedded Milvus: true
2022-03-10 10:35:59 +08:00
embed : true # Whether to enable embedded Etcd (an in-process EtcdServer).
2022-04-26 14:21:46 +08:00
data :
# Embedded Etcd only.
# please adjust in embedded Milvus: /tmp/milvus/etcdData/
dir : /tmp/milvus/etcdData/
2022-03-10 10:35:59 +08:00
2022-04-25 14:17:45 +08:00
# please adjust in embedded Milvus: /tmp/milvus/data/
2022-04-14 19:57:34 +08:00
localStorage :
path : /tmp/milvus/data/
2022-03-10 10:35:59 +08:00
# Related configuration of minio, which is responsible for data persistence for Milvus.
minio :
address : localhost # Address of MinIO/S3
port : 9000 # Port of MinIO/S3
accessKeyID : minioadmin # accessKeyID of MinIO/S3
secretAccessKey : minioadmin # MinIO/S3 encryption string
useSSL : false # Access to MinIO/S3 with SSL
bucketName : "a-bucket" # Bucket name in MinIO/S3
rootPath : files # The root path where the message is stored in MinIO/S3
2022-04-25 14:17:45 +08:00
# Milvus supports three MQ: rocksmq(based on RockDB), Pulsar and Kafka, which should be reserved in config what you use.
# There is a note about enabling priority if we config multiple mq in this file
# 1. standalone(local) mode: rockskmq(default) > Pulsar > Kafka
# 2. cluster mode: Pulsar(default) > Kafka (rocksmq is unsupported)
2022-03-10 10:35:59 +08:00
# Related configuration of pulsar, used to manage Milvus logs of recent mutation operations, output streaming log, and provide log publish-subscribe services.
pulsar :
address : localhost # Address of pulsar
port : 6650 # Port of pulsar
maxMessageSize : 5242880 # 5 * 1024 * 1024 Bytes, Maximum size of each message in pulsar.
2022-04-25 14:17:45 +08:00
# If you want to enable kafka, needs to comment the pulsar configs
#kafka:
# brokerList: localhost1:9092,localhost2:9092,localhost3:9092
2022-03-10 10:35:59 +08:00
rocksmq :
2022-04-25 14:17:45 +08:00
# please adjust in embedded Milvus: /tmp/milvus/rdb_data
2022-03-16 16:51:22 +08:00
path : /tmp/milvus/rdb_data # The path where the message is stored in rocksmq
2022-03-10 10:35:59 +08:00
rocksmqPageSize : 2147483648 # 2 GB, 2 * 1024 * 1024 * 1024 bytes, The size of each page of messages in rocksmq
retentionTimeInMinutes : 10080 # 7 days, 7 * 24 * 60 minutes, The retention time of the message in rocksmq.
retentionSizeInMB : 8192 # 8 GB, 8 * 1024 MB, The retention size of the message in rocksmq.
# Related configuration of rootCoord, used to handle data definition language (DDL) and data control language (DCL) requests
rootCoord :
address : localhost
port : 53100
grpc :
serverMaxRecvSize : 2147483647 # math.MaxInt32, Maximum data size received by the server
serverMaxSendSize : 2147483647 # math.MaxInt32, Maximum data size sent by the server
clientMaxRecvSize : 104857600 # 100 MB, Maximum data size received by the client
clientMaxSendSize : 104857600 # 100 MB, Maximum data size sent by the client
dmlChannelNum : 256 # The number of dml channels created at system startup
maxPartitionNum : 4096 # Maximum number of partitions in a collection
minSegmentSizeToEnableIndex : 1024 # It's a threshold. When the segment size is less than this value, the segment will not be indexed
2022-04-14 19:57:34 +08:00
# (in seconds) Duration after which an import task will expire (be killed). Default 3600 seconds (1 hour).
# Note: If default value is to be changed, change also the default in: internal/util/paramtable/component_param.go
importTaskExpiration : 3600
# (in seconds) Milvus will keep the record of import tasks for at least `importTaskRetention` seconds. Default 86400
# seconds (24 hours).
# Note: If default value is to be changed, change also the default in: internal/util/paramtable/component_param.go
importTaskRetention : 86400
# (in seconds) During index building phase of an import task, Milvus will check the building status of a task's
# segments' indices every `importIndexCheckInterval` seconds. Default 300 seconds (5 minutes).
# Note: If default value is to be changed, change also the default in: internal/util/paramtable/component_param.go
importIndexCheckInterval : 300
# (in seconds) Maximum time to wait before pushing flushed segments online (make them searchable) during importing.
# Default 1200 seconds (20 minutes).
# Note: If default value is to be changed, change also the default in: internal/util/paramtable/component_param.go
importIndexWaitLimit : 1200
2022-03-10 10:35:59 +08:00
# Related configuration of proxy, used to validate client requests and reduce the returned results.
proxy :
port : 19530
http :
enabled : true # Whether to enable the http server
port : 8080 # Whether to enable the http server
readTimeout : 30000 # 30000 ms http read timeout
writeTimeout : 30000 # 30000 ms http write timeout
grpc :
serverMaxRecvSize : 536870912 # 512 MB, 512 * 1024 * 1024 Bytes
serverMaxSendSize : 536870912 # 512 MB, 512 * 1024 * 1024 Bytes
clientMaxRecvSize : 104857600 # 100 MB, 100 * 1024 * 1024
clientMaxSendSize : 104857600 # 100 MB, 100 * 1024 * 1024
timeTickInterval : 200 # ms, the interval that proxy synchronize the time tick
msgStream :
timeTick :
bufSize : 512
maxNameLength : 255 # Maximum length of name for a collection or alias
maxFieldNum : 256 # Maximum number of fields in a collection
maxDimension : 32768 # Maximum dimension of a vector
maxShardNum : 256 # Maximum number of shards in a collection
maxTaskNum : 1024 # max task number of proxy task queue
2022-04-25 14:17:45 +08:00
# please adjust in embedded Milvus: false
2022-03-10 10:35:59 +08:00
ginLogging : false # Whether to produce gin logs.
# Related configuration of queryCoord, used to manage topology and load balancing for the query nodes, and handoff from growing segments to sealed segments.
queryCoord :
address : localhost
port : 19531
autoHandoff : true # Enable auto handoff
autoBalance : true # Enable auto balance
overloadedMemoryThresholdPercentage : 90 # The threshold percentage that memory overload
balanceIntervalSeconds : 60
memoryUsageMaxDifferencePercentage : 30
grpc :
serverMaxRecvSize : 2147483647 # math.MaxInt32
serverMaxSendSize : 2147483647 # math.MaxInt32
clientMaxRecvSize : 104857600 # 100 MB, 100 * 1024 * 1024
clientMaxSendSize : 104857600 # 100 MB, 100 * 1024 * 1024
# Related configuration of queryNode, used to run hybrid search between vector and scalar data.
queryNode :
cacheSize : 32 # GB, default 32 GB, `cacheSize` is the memory used for caching data for faster query. The `cacheSize` must be less than system memory size.
gracefulTime : 0 # Minimum time before the newly inserted data can be searched (in ms)
port : 21123
grpc :
serverMaxRecvSize : 2147483647 # math.MaxInt32
serverMaxSendSize : 2147483647 # math.MaxInt32
clientMaxRecvSize : 104857600 # 100 MB, 100 * 1024 * 1024
clientMaxSendSize : 104857600 # 100 MB, 100 * 1024 * 1024
2022-04-25 14:17:45 +08:00
2022-03-10 10:35:59 +08:00
stats :
publishInterval : 1000 # Interval for querynode to report node information (milliseconds)
dataSync :
flowGraph :
maxQueueLength : 1024 # Maximum length of task queue in flowgraph
maxParallelism : 1024 # Maximum number of tasks executed in parallel in the flowgraph
msgStream :
search :
recvBufSize : 512 # msgPack channel buffer size
pulsarBufSize : 512 # pulsar channel buffer size
searchResult :
recvBufSize : 64 # msgPack channel buffer size
# Segcore will divide a segment into multiple chunks.
segcore :
chunkRows : 32768 # The number of vectors in a chunk.
2022-04-25 14:17:45 +08:00
2022-04-14 19:57:34 +08:00
cache :
enabled : true
memoryLimit : 2147483648 # 2 GB, 2 * 1024 *1024 *1024
2022-03-10 10:35:59 +08:00
indexCoord :
address : localhost
port : 31000
grpc :
serverMaxRecvSize : 2147483647 # math.MaxInt32
serverMaxSendSize : 2147483647 # math.MaxInt32
clientMaxRecvSize : 104857600 # 100 MB, 100 * 1024 * 1024
clientMaxSendSize : 104857600 # 100 MB, 100 * 1024 * 1024
indexNode :
port : 21121
grpc :
serverMaxRecvSize : 2147483647 # math.MaxInt32
serverMaxSendSize : 2147483647 # math.MaxInt32
clientMaxRecvSize : 104857600 # 100 MB, 100 * 1024 * 1024
clientMaxSendSize : 104857600 # 100 MB, 100 * 1024 * 1024
dataCoord :
address : localhost
port : 13333
grpc :
serverMaxRecvSize : 2147483647 # math.MaxInt32
serverMaxSendSize : 2147483647 # math.MaxInt32
clientMaxRecvSize : 104857600 # 100 MB, 100 * 1024 * 1024
clientMaxSendSize : 104857600 # 100 MB, 100 * 1024 * 1024
enableCompaction : true # Enable data segment compression
enableGarbageCollection : false
segment :
maxSize : 512 # Maximum size of a segment in MB
sealProportion : 0.75 # It's the minimum proportion for a segment which can be sealed
assignmentExpiration : 2000 # The time of the assignment expiration in ms
compaction :
enableAutoCompaction : true
entityExpiration : 9223372037 # Entity expiration in seconds, CAUTION make sure entityExpiration >= retentionDuration and 9223372037 is the maximum value of entityExpiration
gc :
interval : 3600 # gc interval in seconds
missingTolerance : 86400 # file meta missing tolerance duration in seconds, 60*24
dropTolerance : 86400 # file belongs to dropped entity tolerance duration in seconds, 60*24
dataNode :
port : 21124
grpc :
serverMaxRecvSize : 2147483647 # math.MaxInt32
serverMaxSendSize : 2147483647 # math.MaxInt32
clientMaxRecvSize : 104857600 # 100 MB, 100 * 1024 * 1024
clientMaxSendSize : 104857600 # 100 MB, 100 * 1024 * 1024
dataSync :
flowGraph :
maxQueueLength : 1024 # Maximum length of task queue in flowgraph
maxParallelism : 1024 # Maximum number of tasks executed in parallel in the flowgraph
flush :
# Max buffer size to flush for a single segment.
insertBufSize : 16777216 # Bytes, 16 MB
# Configures the system log output.
log :
level : info # Only supports debug, info, warn, error, panic, or fatal. Default 'info'.
file :
2022-04-25 14:17:45 +08:00
# please adjust in embedded Milvus: /tmp/milvus/logs
2022-03-10 10:35:59 +08:00
rootPath : /tmp/milvus/logs # default to stdout, stderr
maxSize : 300 # MB
maxAge : 10 # Maximum time for log retention in day.
maxBackups : 20
format : text # text/json
common :
# Channel name generation rule: ${namePrefix}-${ChannelIdx}
chanNamePrefix :
cluster : "by-dev"
rootCoordTimeTick : "rootcoord-timetick"
rootCoordStatistics : "rootcoord-statistics"
rootCoordDml : "rootcoord-dml"
rootCoordDelta : "rootcoord-delta"
search : "search"
searchResult : "searchResult"
queryTimeTick : "queryTimeTick"
queryNodeStats : "query-node-stats"
# Cmd for loadIndex, flush, etc...
cmd : "cmd"
dataCoordStatistic : "datacoord-statistics-channel"
dataCoordTimeTick : "datacoord-timetick-channel"
dataCoordSegmentInfo : "segment-info-channel"
# Sub name generation rule: ${subNamePrefix}-${NodeID}
subNamePrefix :
rootCoordSubNamePrefix : "rootCoord"
proxySubNamePrefix : "proxy"
queryNodeSubNamePrefix : "queryNode"
dataNodeSubNamePrefix : "dataNode"
dataCoordSubNamePrefix : "dataCoord"
defaultPartitionName : "_default" # default partition name for a collection
defaultIndexName : "_default_idx" # default index name
retentionDuration : 432000 # 5 days in seconds
# Default value: auto
# Valid values: [auto, avx512, avx2, avx, sse4_2]
# This configuration is only used by querynode and indexnode, it selects CPU instruction set for Searching and Index-building.
simdType : auto
2022-04-14 19:57:34 +08:00
indexSliceSize : 4 # MB
2022-04-25 14:17:45 +08:00
# please adjust in embedded Milvus: local
storageType : local
security :
authorizationEnabled : false