2021-10-15 18:05:16 +08:00
|
|
|
// Licensed to the LF AI & Data foundation under one
|
|
|
|
// or more contributor license agreements. See the NOTICE file
|
|
|
|
// distributed with this work for additional information
|
|
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
|
|
// to you under the Apache License, Version 2.0 (the
|
|
|
|
// "License"); you may not use this file except in compliance
|
2021-09-13 17:12:19 +08:00
|
|
|
// with the License. You may obtain a copy of the License at
|
|
|
|
//
|
2021-10-15 18:05:16 +08:00
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
2021-09-13 17:12:19 +08:00
|
|
|
//
|
2021-10-15 18:05:16 +08:00
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
2021-09-13 17:12:19 +08:00
|
|
|
|
|
|
|
package common
|
|
|
|
|
2023-11-02 23:52:16 +08:00
|
|
|
import (
|
|
|
|
"encoding/binary"
|
|
|
|
|
|
|
|
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
|
|
|
|
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
|
|
|
|
)
|
2021-11-02 18:16:32 +08:00
|
|
|
|
2023-06-21 14:00:42 +08:00
|
|
|
// system field id:
|
2021-09-13 17:12:19 +08:00
|
|
|
// 0: unique row id
|
|
|
|
// 1: timestamp
|
|
|
|
// 100: first user field id
|
|
|
|
// 101: second user field id
|
|
|
|
// 102: ...
|
|
|
|
|
|
|
|
const (
|
2021-10-03 08:45:55 +08:00
|
|
|
// StartOfUserFieldID represents the starting ID of the user-defined field
|
2021-09-13 17:12:19 +08:00
|
|
|
StartOfUserFieldID = 100
|
2021-10-03 08:45:55 +08:00
|
|
|
|
|
|
|
// RowIDField is the ID of the RowID field reserved by the system
|
2021-10-03 10:48:02 +08:00
|
|
|
RowIDField = 0
|
2021-10-03 08:45:55 +08:00
|
|
|
|
|
|
|
// TimeStampField is the ID of the Timestamp field reserved by the system
|
2021-10-03 10:48:02 +08:00
|
|
|
TimeStampField = 1
|
2021-10-03 08:45:55 +08:00
|
|
|
|
|
|
|
// RowIDFieldName defines the name of the RowID field
|
2021-10-03 10:48:02 +08:00
|
|
|
RowIDFieldName = "RowID"
|
2021-10-03 08:45:55 +08:00
|
|
|
|
|
|
|
// TimeStampFieldName defines the name of the Timestamp field
|
2021-09-13 17:12:19 +08:00
|
|
|
TimeStampFieldName = "Timestamp"
|
2021-10-03 08:45:55 +08:00
|
|
|
|
2023-05-18 09:33:24 +08:00
|
|
|
// MetaFieldName is the field name of dynamic schema
|
|
|
|
MetaFieldName = "$meta"
|
|
|
|
|
2021-10-03 08:45:55 +08:00
|
|
|
// DefaultShardsNum defines the default number of shards when creating a collection
|
2023-04-21 07:08:32 +08:00
|
|
|
DefaultShardsNum = int32(1)
|
2021-10-18 20:08:42 +08:00
|
|
|
|
2023-06-06 10:24:34 +08:00
|
|
|
// DefaultPartitionsWithPartitionKey defines the default number of partitions when use partition key
|
|
|
|
DefaultPartitionsWithPartitionKey = int64(64)
|
|
|
|
|
2021-10-18 20:08:42 +08:00
|
|
|
// InvalidPartitionID indicates that the partition is not specified. It will be set when the partitionName is empty
|
|
|
|
InvalidPartitionID = int64(-1)
|
2021-10-22 14:31:13 +08:00
|
|
|
|
2021-10-28 20:17:28 +08:00
|
|
|
// InvalidFieldID indicates that the field does not exist . It will be set when the field is not found.
|
2021-10-22 14:31:13 +08:00
|
|
|
InvalidFieldID = int64(-1)
|
2021-11-19 13:57:12 +08:00
|
|
|
|
|
|
|
// NotRegisteredID means node is not registered into etcd.
|
|
|
|
NotRegisteredID = int64(-1)
|
2022-09-02 10:42:59 +08:00
|
|
|
|
|
|
|
// InvalidNodeID indicates that node is not valid in querycoord replica or shard cluster.
|
|
|
|
InvalidNodeID = int64(-1)
|
2021-09-13 17:12:19 +08:00
|
|
|
)
|
2021-11-02 18:16:32 +08:00
|
|
|
|
2021-11-05 22:46:56 +08:00
|
|
|
// Endian is type alias of binary.LittleEndian.
|
|
|
|
// Milvus uses little endian by default.
|
2021-11-02 18:16:32 +08:00
|
|
|
var Endian = binary.LittleEndian
|
2022-08-25 19:32:53 +08:00
|
|
|
|
|
|
|
const (
|
|
|
|
// SegmentInsertLogPath storage path const for segment insert binlog.
|
|
|
|
SegmentInsertLogPath = `insert_log`
|
|
|
|
|
|
|
|
// SegmentDeltaLogPath storage path const for segment delta log.
|
|
|
|
SegmentDeltaLogPath = `delta_log`
|
|
|
|
|
|
|
|
// SegmentStatslogPath storage path const for segment stats log.
|
|
|
|
SegmentStatslogPath = `stats_log`
|
|
|
|
|
|
|
|
// SegmentIndexPath storage path const for segment index files.
|
|
|
|
SegmentIndexPath = `index_files`
|
|
|
|
)
|
2022-10-08 15:38:58 +08:00
|
|
|
|
2023-04-06 14:32:29 +08:00
|
|
|
// Search, Index parameter keys
|
2022-10-08 15:38:58 +08:00
|
|
|
const (
|
2023-12-28 10:04:46 +08:00
|
|
|
TopKKey = "topk"
|
|
|
|
SearchParamKey = "search_param"
|
|
|
|
SegmentNumKey = "segment_num"
|
|
|
|
WithFilterKey = "with_filter"
|
|
|
|
WithOptimizeKey = "with_optimize"
|
|
|
|
CollectionKey = "collection"
|
2023-04-06 14:32:29 +08:00
|
|
|
|
2022-10-08 15:38:58 +08:00
|
|
|
IndexParamsKey = "params"
|
|
|
|
IndexTypeKey = "index_type"
|
|
|
|
MetricTypeKey = "metric_type"
|
|
|
|
DimKey = "dim"
|
2023-05-16 17:41:22 +08:00
|
|
|
MaxLengthKey = "max_length"
|
2023-09-19 14:23:23 +08:00
|
|
|
MaxCapacityKey = "max_capacity"
|
2022-10-08 15:38:58 +08:00
|
|
|
)
|
2022-10-10 20:31:22 +08:00
|
|
|
|
|
|
|
// Collection properties key
|
|
|
|
|
|
|
|
const (
|
2023-05-10 17:45:20 +08:00
|
|
|
CollectionTTLConfigKey = "collection.ttl.seconds"
|
|
|
|
CollectionAutoCompactionKey = "collection.autocompaction.enabled"
|
2023-06-21 15:02:42 +08:00
|
|
|
|
|
|
|
// rate limit
|
|
|
|
CollectionInsertRateMaxKey = "collection.insertRate.max.mb"
|
|
|
|
CollectionInsertRateMinKey = "collection.insertRate.min.mb"
|
2023-07-11 11:20:34 +08:00
|
|
|
CollectionUpsertRateMaxKey = "collection.upsertRate.max.mb"
|
|
|
|
CollectionUpsertRateMinKey = "collection.upsertRate.min.mb"
|
2023-06-21 15:02:42 +08:00
|
|
|
CollectionDeleteRateMaxKey = "collection.deleteRate.max.mb"
|
|
|
|
CollectionDeleteRateMinKey = "collection.deleteRate.min.mb"
|
|
|
|
CollectionBulkLoadRateMaxKey = "collection.bulkLoadRate.max.mb"
|
|
|
|
CollectionBulkLoadRateMinKey = "collection.bulkLoadRate.min.mb"
|
|
|
|
CollectionQueryRateMaxKey = "collection.queryRate.max.qps"
|
|
|
|
CollectionQueryRateMinKey = "collection.queryRate.min.qps"
|
|
|
|
CollectionSearchRateMaxKey = "collection.searchRate.max.vps"
|
|
|
|
CollectionSearchRateMinKey = "collection.searchRate.min.vps"
|
|
|
|
CollectionDiskQuotaKey = "collection.diskProtection.diskQuota.mb"
|
2022-10-10 20:31:22 +08:00
|
|
|
)
|
2022-11-25 23:11:12 +08:00
|
|
|
|
2023-11-02 23:52:16 +08:00
|
|
|
// common properties
|
|
|
|
const (
|
2024-03-01 18:15:29 +08:00
|
|
|
MmapEnabledKey = "mmap.enabled"
|
|
|
|
LazyLoadEnableKey = "lazyload.enabled"
|
2023-11-02 23:52:16 +08:00
|
|
|
)
|
|
|
|
|
2022-11-25 23:11:12 +08:00
|
|
|
const (
|
|
|
|
PropertiesKey string = "properties"
|
|
|
|
TraceIDKey string = "uber-trace-id"
|
|
|
|
)
|
2022-12-08 10:15:19 +08:00
|
|
|
|
|
|
|
func IsSystemField(fieldID int64) bool {
|
|
|
|
return fieldID < StartOfUserFieldID
|
|
|
|
}
|
2023-04-14 09:54:28 +08:00
|
|
|
|
2023-11-02 23:52:16 +08:00
|
|
|
func IsMmapEnabled(kvs ...*commonpb.KeyValuePair) bool {
|
|
|
|
for _, kv := range kvs {
|
|
|
|
if kv.Key == MmapEnabledKey && kv.Value == "true" {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
func IsFieldMmapEnabled(schema *schemapb.CollectionSchema, fieldID int64) bool {
|
|
|
|
for _, field := range schema.GetFields() {
|
|
|
|
if field.GetFieldID() == fieldID {
|
|
|
|
return IsMmapEnabled(field.GetTypeParams()...)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2024-03-01 18:15:29 +08:00
|
|
|
func IsCollectionLazyLoadEnabled(kvs ...*commonpb.KeyValuePair) bool {
|
|
|
|
for _, kv := range kvs {
|
|
|
|
if kv.Key == LazyLoadEnableKey && kv.Value == "true" {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2023-04-14 09:54:28 +08:00
|
|
|
const (
|
|
|
|
// LatestVerision is the magic number for watch latest revision
|
|
|
|
LatestRevision = int64(-1)
|
|
|
|
)
|