test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-02 03:48:37 +08:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

enhance: rename some params and reduce default bitmapCardinalityLimit… (#36138)

Browse Source 
#32900

Signed-off-by: luzhang <luzhang@zilliz.com>
Co-authored-by: luzhang <luzhang@zilliz.com>
This commit is contained in:
zhagnlu 2024-09-12 12:09:08 +08:00 committed by GitHub
parent 11dbe1e755
commit 5e5e87cc2f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 35 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
configs/milvus.yaml
View File

@ -768,7 +768,6 @@ common:
BeamWidthRatio: 4
gracefulTime: 5000 # milliseconds. it represents the interval (in ms) by which the request arrival time needs to be subtracted in the case of Bounded Consistency.
gracefulStopTimeout: 1800 # seconds. it will force quit the server if the graceful stop process is not completed during this time.
bitmapIndexCardinalityBound: 500
storageType: remote # please adjust in embedded Milvus: local, available values are [local, remote, opendal], value minio is deprecated, use remote instead
# Default value: auto
# Valid values: [auto, avx512, avx2, avx, sse4_2]

4
internal/core/src/common/Consts.h
View File

@ -70,6 +70,8 @@ const int64_t DEFAULT_MAX_OUTPUT_SIZE = 67108864; // bytes, 64MB
const int64_t DEFAULT_CHUNK_MANAGER_REQUEST_TIMEOUT_MS = 10000;
const int64_t DEFAULT_BITMAP_INDEX_CARDINALITY_BOUND = 500;
const int64_t DEFAULT_BITMAP_INDEX_BUILD_MODE_BOUND = 500;
const int64_t DEFAULT_HYBRID_INDEX_BITMAP_CARDINALITY_LIMIT = 100;
const size_t MARISA_NULL_KEY_ID = -1;

4
internal/core/src/index/BitmapIndex.cpp
View File

@ -97,7 +97,7 @@ BitmapIndex<T>::Build(size_t n, const T* data) {
valid_bitset.set(i);
}
if (data_.size() < DEFAULT_BITMAP_INDEX_CARDINALITY_BOUND) {
if (data_.size() < DEFAULT_BITMAP_INDEX_BUILD_MODE_BOUND) {
for (auto it = data_.begin(); it != data_.end(); ++it) {
bitsets_[it->first] = ConvertRoaringToBitset(it->second);
}
@ -330,7 +330,7 @@ BitmapIndex<T>::DeserializeIndexMeta(const uint8_t* data_ptr,
template <typename T>
void
BitmapIndex<T>::ChooseIndexLoadMode(int64_t index_length) {
if (index_length <= DEFAULT_BITMAP_INDEX_CARDINALITY_BOUND) {
if (index_length <= DEFAULT_BITMAP_INDEX_BUILD_MODE_BOUND) {
LOG_DEBUG("load bitmap index with bitset mode");
build_mode_ = BitmapIndexBuildMode::BITSET;
} else {

3
internal/core/src/index/HybridScalarIndex.cpp
View File

@ -32,7 +32,8 @@ HybridScalarIndex<T>::HybridScalarIndex(
const storage::FileManagerContext& file_manager_context)
: ScalarIndex<T>(HYBRID_INDEX_TYPE),
is_built_(false),
bitmap_index_cardinality_limit_(DEFAULT_BITMAP_INDEX_CARDINALITY_BOUND),
bitmap_index_cardinality_limit_(
DEFAULT_HYBRID_INDEX_BITMAP_CARDINALITY_LIMIT),
file_manager_context_(file_manager_context) {
if (file_manager_context.Valid()) {
mem_file_manager_ =

2
internal/proxy/task_index.go
View File

@ -454,7 +454,7 @@ func checkTrain(field *schemapb.FieldSchema, indexParams map[string]string) erro
if indexType == indexparamcheck.IndexHybrid {
_, exist := indexParams[common.BitmapCardinalityLimitKey]
if !exist {
indexParams[common.BitmapCardinalityLimitKey] = paramtable.Get().CommonCfg.BitmapIndexCardinalityBound.GetValue()
indexParams[common.BitmapCardinalityLimitKey] = paramtable.Get().AutoIndexConfig.BitmapCardinalityLimit.GetValue()
}
}
checker, err := indexparamcheck.GetIndexCheckerMgrInstance().GetChecker(indexType)

10
internal/proxy/task_index_test.go
View File

@ -664,7 +664,7 @@ func Test_parseIndexParams(t *testing.T) {
sortKeyValuePairs(cit.newIndexParams)
assert.Equal(t, cit.newIndexParams, []*commonpb.KeyValuePair{
{Key: common.IndexTypeKey, Value: indexparamcheck.IndexHybrid},
{Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapIndexCardinalityBound)},
{Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapCardinalityLimit)},
})
})
@ -708,7 +708,7 @@ func Test_parseIndexParams(t *testing.T) {
sortKeyValuePairs(cit.newIndexParams)
assert.Equal(t, cit.newIndexParams, []*commonpb.KeyValuePair{
{Key: common.IndexTypeKey, Value: indexparamcheck.IndexHybrid},
{Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapIndexCardinalityBound)},
{Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapCardinalityLimit)},
})
})
@ -937,7 +937,7 @@ func Test_parseIndexParams(t *testing.T) {
sortKeyValuePairs(cit.newIndexParams)
assert.Equal(t, cit.newIndexParams, []*commonpb.KeyValuePair{
{Key: common.IndexTypeKey, Value: indexparamcheck.IndexHybrid},
{Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapIndexCardinalityBound)},
{Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapCardinalityLimit)},
})
})
@ -967,7 +967,7 @@ func Test_parseIndexParams(t *testing.T) {
sortKeyValuePairs(cit.newIndexParams)
assert.Equal(t, cit.newIndexParams, []*commonpb.KeyValuePair{
{Key: common.IndexTypeKey, Value: indexparamcheck.IndexHybrid},
{Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapIndexCardinalityBound)},
{Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapCardinalityLimit)},
})
})
@ -997,7 +997,7 @@ func Test_parseIndexParams(t *testing.T) {
sortKeyValuePairs(cit.newIndexParams)
assert.Equal(t, cit.newIndexParams, []*commonpb.KeyValuePair{
{Key: common.IndexTypeKey, Value: indexparamcheck.IndexHybrid},
{Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapIndexCardinalityBound)},
{Key: common.BitmapCardinalityLimitKey, Value: strconv.Itoa(paramtable.DefaultBitmapCardinalityLimit)},
})
})

15
pkg/util/paramtable/autoindex_param.go
View File

@ -18,6 +18,7 @@ package paramtable
import (
"fmt"
"strconv"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/pkg/common"
@ -50,8 +51,14 @@ type autoIndexConfig struct {
ScalarVarcharIndexType ParamItem `refreshable:"true"`
ScalarBoolIndexType ParamItem `refreshable:"true"`
ScalarFloatIndexType ParamItem `refreshable:"true"`
BitmapCardinalityLimit ParamItem `refreshable:"true"`
}
const (
DefaultBitmapCardinalityLimit = 100
)
func (p *autoIndexConfig) init(base *BaseTable) {
p.Enable = ParamItem{
Key: "autoIndex.enable",
@ -195,6 +202,14 @@ func (p *autoIndexConfig) init(base *BaseTable) {
}
p.ScalarFloatIndexType.Init(base.mgr)
p.BitmapCardinalityLimit = ParamItem{
Key: "scalarAutoIndex.params.bitmapCardinalityLimit",
Version: "2.5.0",
DefaultValue: strconv.Itoa(DefaultBitmapCardinalityLimit),
Export: true,
}
p.BitmapCardinalityLimit.Init(base.mgr)
p.ScalarVarcharIndexType = ParamItem{
Version: "2.4.0",
Formatter: func(v string) string {

24
pkg/util/paramtable/component_param.go
View File

@ -48,14 +48,13 @@ const (
DefaultSessionTTL = 30 // s
DefaultSessionRetryTimes = 30
DefaultMaxDegree = 56
DefaultSearchListSize = 100
DefaultPQCodeBudgetGBRatio = 0.125
DefaultBuildNumThreadsRatio = 1.0
DefaultSearchCacheBudgetGBRatio = 0.10
DefaultLoadNumThreadRatio = 8.0
DefaultBeamWidthRatio = 4.0
DefaultBitmapIndexCardinalityBound = 500
DefaultMaxDegree = 56
DefaultSearchListSize = 100
DefaultPQCodeBudgetGBRatio = 0.125
DefaultBuildNumThreadsRatio = 1.0
DefaultSearchCacheBudgetGBRatio = 0.10
DefaultLoadNumThreadRatio = 8.0
DefaultBeamWidthRatio = 4.0
)
// ComponentParam is used to quickly and easily access all components' configurations.
@ -229,7 +228,6 @@ type commonConfig struct {
BeamWidthRatio ParamItem `refreshable:"true"`
GracefulTime ParamItem `refreshable:"true"`
GracefulStopTimeout ParamItem `refreshable:"true"`
BitmapIndexCardinalityBound ParamItem `refreshable:"false"`
StorageType ParamItem `refreshable:"false"`
SimdType ParamItem `refreshable:"false"`
@ -502,14 +500,6 @@ This configuration is only used by querynode and indexnode, it selects CPU instr
}
p.IndexSliceSize.Init(base.mgr)
p.BitmapIndexCardinalityBound = ParamItem{
Key: "common.bitmapIndexCardinalityBound",
Version: "2.5.0",
DefaultValue: strconv.Itoa(DefaultBitmapIndexCardinalityBound),
Export: true,
}
p.BitmapIndexCardinalityBound.Init(base.mgr)
p.EnableMaterializedView = ParamItem{
Key: "common.materializedView.enabled",
Version: "2.4.6",