diff --git a/configs/milvus.yaml b/configs/milvus.yaml index 2f50e545a5..d4eabe7a51 100644 --- a/configs/milvus.yaml +++ b/configs/milvus.yaml @@ -768,7 +768,6 @@ common: BeamWidthRatio: 4 gracefulTime: 5000 # milliseconds. it represents the interval (in ms) by which the request arrival time needs to be subtracted in the case of Bounded Consistency. gracefulStopTimeout: 1800 # seconds. it will force quit the server if the graceful stop process is not completed during this time. - bitmapIndexCardinalityBound: 500 storageType: remote # please adjust in embedded Milvus: local, available values are [local, remote, opendal], value minio is deprecated, use remote instead # Default value: auto # Valid values: [auto, avx512, avx2, avx, sse4_2] diff --git a/internal/core/src/common/Consts.h b/internal/core/src/common/Consts.h index 662a2d28cd..1f9d51e447 100644 --- a/internal/core/src/common/Consts.h +++ b/internal/core/src/common/Consts.h @@ -70,6 +70,8 @@ const int64_t DEFAULT_MAX_OUTPUT_SIZE = 67108864; // bytes, 64MB const int64_t DEFAULT_CHUNK_MANAGER_REQUEST_TIMEOUT_MS = 10000; -const int64_t DEFAULT_BITMAP_INDEX_CARDINALITY_BOUND = 500; +const int64_t DEFAULT_BITMAP_INDEX_BUILD_MODE_BOUND = 500; + +const int64_t DEFAULT_HYBRID_INDEX_BITMAP_CARDINALITY_LIMIT = 100; const size_t MARISA_NULL_KEY_ID = -1; diff --git a/internal/core/src/index/BitmapIndex.cpp b/internal/core/src/index/BitmapIndex.cpp index 7f05ca5262..cc4de8e3bf 100644 --- a/internal/core/src/index/BitmapIndex.cpp +++ b/internal/core/src/index/BitmapIndex.cpp @@ -97,7 +97,7 @@ BitmapIndex::Build(size_t n, const T* data) { valid_bitset.set(i); } - if (data_.size() < DEFAULT_BITMAP_INDEX_CARDINALITY_BOUND) { + if (data_.size() < DEFAULT_BITMAP_INDEX_BUILD_MODE_BOUND) { for (auto it = data_.begin(); it != data_.end(); ++it) { bitsets_[it->first] = ConvertRoaringToBitset(it->second); } @@ -330,7 +330,7 @@ BitmapIndex::DeserializeIndexMeta(const uint8_t* data_ptr, template void BitmapIndex::ChooseIndexLoadMode(int64_t index_length) { - if (index_length <= DEFAULT_BITMAP_INDEX_CARDINALITY_BOUND) { + if (index_length <= DEFAULT_BITMAP_INDEX_BUILD_MODE_BOUND) { LOG_DEBUG("load bitmap index with bitset mode"); build_mode_ = BitmapIndexBuildMode::BITSET; } else { diff --git a/internal/core/src/index/HybridScalarIndex.cpp b/internal/core/src/index/HybridScalarIndex.cpp index b220883724..a70a70ed6f 100644 --- a/internal/core/src/index/HybridScalarIndex.cpp +++ b/internal/core/src/index/HybridScalarIndex.cpp @@ -32,7 +32,8 @@ HybridScalarIndex::HybridScalarIndex( const storage::FileManagerContext& file_manager_context) : ScalarIndex(HYBRID_INDEX_TYPE), is_built_(false), - bitmap_index_cardinality_limit_(DEFAULT_BITMAP_INDEX_CARDINALITY_BOUND), + bitmap_index_cardinality_limit_( + DEFAULT_HYBRID_INDEX_BITMAP_CARDINALITY_LIMIT), file_manager_context_(file_manager_context) { if (file_manager_context.Valid()) { mem_file_manager_ = diff --git a/internal/proxy/task_index.go b/internal/proxy/task_index.go index a9cee3a377..ff68a38e42 100644 --- a/internal/proxy/task_index.go +++ b/internal/proxy/task_index.go @@ -454,7 +454,7 @@ func checkTrain(field *schemapb.FieldSchema, indexParams map[string]string) erro if indexType == indexparamcheck.IndexHybrid { _, exist := indexParams[common.BitmapCardinalityLimitKey] if !exist { - indexParams[common.BitmapCardinalityLimitKey] = paramtable.Get().CommonCfg.BitmapIndexCardinalityBound.GetValue() + indexParams[common.BitmapCardinalityLimitKey] = paramtable.Get().AutoIndexConfig.BitmapCardinalityLimit.GetValue() } } checker, err := indexparamcheck.GetIndexCheckerMgrInstance().GetChecker(indexType) diff --git a/internal/proxy/task_index_test.go b/internal/proxy/task_index_test.go index 8c89d403de..55db6e2b59 100644 --- a/internal/proxy/task_index_test.go +++ b/internal/proxy/task_index_test.go @@ -664,7 +664,7 @@ func Test_parseIndexParams(t *testing.T) { sortKeyValuePairs(cit.newIndexParams) assert.Equal(t, cit.newIndexParams, []*commonpb.KeyValuePair{ {Key: common.IndexTypeKey, Value: indexparamcheck.IndexHybrid}, - {Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapIndexCardinalityBound)}, + {Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapCardinalityLimit)}, }) }) @@ -708,7 +708,7 @@ func Test_parseIndexParams(t *testing.T) { sortKeyValuePairs(cit.newIndexParams) assert.Equal(t, cit.newIndexParams, []*commonpb.KeyValuePair{ {Key: common.IndexTypeKey, Value: indexparamcheck.IndexHybrid}, - {Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapIndexCardinalityBound)}, + {Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapCardinalityLimit)}, }) }) @@ -937,7 +937,7 @@ func Test_parseIndexParams(t *testing.T) { sortKeyValuePairs(cit.newIndexParams) assert.Equal(t, cit.newIndexParams, []*commonpb.KeyValuePair{ {Key: common.IndexTypeKey, Value: indexparamcheck.IndexHybrid}, - {Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapIndexCardinalityBound)}, + {Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapCardinalityLimit)}, }) }) @@ -967,7 +967,7 @@ func Test_parseIndexParams(t *testing.T) { sortKeyValuePairs(cit.newIndexParams) assert.Equal(t, cit.newIndexParams, []*commonpb.KeyValuePair{ {Key: common.IndexTypeKey, Value: indexparamcheck.IndexHybrid}, - {Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapIndexCardinalityBound)}, + {Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapCardinalityLimit)}, }) }) @@ -997,7 +997,7 @@ func Test_parseIndexParams(t *testing.T) { sortKeyValuePairs(cit.newIndexParams) assert.Equal(t, cit.newIndexParams, []*commonpb.KeyValuePair{ {Key: common.IndexTypeKey, Value: indexparamcheck.IndexHybrid}, - {Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapIndexCardinalityBound)}, + {Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapCardinalityLimit)}, }) }) diff --git a/pkg/util/paramtable/autoindex_param.go b/pkg/util/paramtable/autoindex_param.go index 31df71a4a3..d9d783e81b 100644 --- a/pkg/util/paramtable/autoindex_param.go +++ b/pkg/util/paramtable/autoindex_param.go @@ -18,6 +18,7 @@ package paramtable import ( "fmt" + "strconv" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/pkg/common" @@ -50,8 +51,14 @@ type autoIndexConfig struct { ScalarVarcharIndexType ParamItem `refreshable:"true"` ScalarBoolIndexType ParamItem `refreshable:"true"` ScalarFloatIndexType ParamItem `refreshable:"true"` + + BitmapCardinalityLimit ParamItem `refreshable:"true"` } +const ( + DefaultBitmapCardinalityLimit = 100 +) + func (p *autoIndexConfig) init(base *BaseTable) { p.Enable = ParamItem{ Key: "autoIndex.enable", @@ -195,6 +202,14 @@ func (p *autoIndexConfig) init(base *BaseTable) { } p.ScalarFloatIndexType.Init(base.mgr) + p.BitmapCardinalityLimit = ParamItem{ + Key: "scalarAutoIndex.params.bitmapCardinalityLimit", + Version: "2.5.0", + DefaultValue: strconv.Itoa(DefaultBitmapCardinalityLimit), + Export: true, + } + p.BitmapCardinalityLimit.Init(base.mgr) + p.ScalarVarcharIndexType = ParamItem{ Version: "2.4.0", Formatter: func(v string) string { diff --git a/pkg/util/paramtable/component_param.go b/pkg/util/paramtable/component_param.go index e813964f8b..816dd6cace 100644 --- a/pkg/util/paramtable/component_param.go +++ b/pkg/util/paramtable/component_param.go @@ -48,14 +48,13 @@ const ( DefaultSessionTTL = 30 // s DefaultSessionRetryTimes = 30 - DefaultMaxDegree = 56 - DefaultSearchListSize = 100 - DefaultPQCodeBudgetGBRatio = 0.125 - DefaultBuildNumThreadsRatio = 1.0 - DefaultSearchCacheBudgetGBRatio = 0.10 - DefaultLoadNumThreadRatio = 8.0 - DefaultBeamWidthRatio = 4.0 - DefaultBitmapIndexCardinalityBound = 500 + DefaultMaxDegree = 56 + DefaultSearchListSize = 100 + DefaultPQCodeBudgetGBRatio = 0.125 + DefaultBuildNumThreadsRatio = 1.0 + DefaultSearchCacheBudgetGBRatio = 0.10 + DefaultLoadNumThreadRatio = 8.0 + DefaultBeamWidthRatio = 4.0 ) // ComponentParam is used to quickly and easily access all components' configurations. @@ -229,7 +228,6 @@ type commonConfig struct { BeamWidthRatio ParamItem `refreshable:"true"` GracefulTime ParamItem `refreshable:"true"` GracefulStopTimeout ParamItem `refreshable:"true"` - BitmapIndexCardinalityBound ParamItem `refreshable:"false"` StorageType ParamItem `refreshable:"false"` SimdType ParamItem `refreshable:"false"` @@ -502,14 +500,6 @@ This configuration is only used by querynode and indexnode, it selects CPU instr } p.IndexSliceSize.Init(base.mgr) - p.BitmapIndexCardinalityBound = ParamItem{ - Key: "common.bitmapIndexCardinalityBound", - Version: "2.5.0", - DefaultValue: strconv.Itoa(DefaultBitmapIndexCardinalityBound), - Export: true, - } - p.BitmapIndexCardinalityBound.Init(base.mgr) - p.EnableMaterializedView = ParamItem{ Key: "common.materializedView.enabled", Version: "2.4.6",