test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-11-29 18:38:44 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

fix: Sparse to use CC index as growing/temp index (#37591)

Browse Source 
relate: https://github.com/milvus-io/milvus/issues/35853

Signed-off-by: Buqian Zheng <zhengbuqian@gmail.com>
This commit is contained in:
Buqian Zheng 2024-11-14 10:54:31 +08:00 committed by GitHub
parent 1c5b5e1e3d
commit 0565300b7f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 9 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
internal/core/src/segcore/IndexConfigGenerator.cpp
View File

@ -24,26 +24,18 @@ VecIndexConfig::VecIndexConfig(const int64_t max_index_row_cout,
is_sparse_(is_sparse) { is_sparse_(is_sparse) {
origin_index_type_ = index_meta_.GetIndexType(); origin_index_type_ = index_meta_.GetIndexType();
metric_type_ = index_meta_.GeMetricType(); metric_type_ = index_meta_.GeMetricType();
// Currently for dense vector index, if the segment is growing, we use IVFCC // For Dense vector, use IVFFLAT_CC as the growing and temp index type.
// as the index type; if the segment is sealed but its index has not been //
// built by the index node, we use IVFFLAT as the temp index type and // For Sparse vector, use SPARSE_WAND_CC for INDEX_SPARSE_WAND index, or use
// release it once the index node has finished building the index and query // SPARSE_INVERTED_INDEX_CC for INDEX_SPARSE_INVERTED_INDEX/other sparse
// node has loaded it. // index types as the growing and temp index type.
// But for sparse vector index(INDEX_SPARSE_INVERTED_INDEX and if (origin_index_type_ == knowhere::IndexEnum::INDEX_SPARSE_WAND) {
// INDEX_SPARSE_WAND), those index themselves can be used as the temp index index_type_ = knowhere::IndexEnum::INDEX_SPARSE_WAND_CC;
// type, so we can avoid the extra step of "releast temp and load".
// When using HNSW(cardinal) for sparse, we use INDEX_SPARSE_INVERTED_INDEX
// as the growing index.
if (origin_index_type_ ==
knowhere::IndexEnum::INDEX_SPARSE_INVERTED_INDEX ||
origin_index_type_ == knowhere::IndexEnum::INDEX_SPARSE_WAND) {
index_type_ = origin_index_type_;
} else if (is_sparse_) { } else if (is_sparse_) {
index_type_ = knowhere::IndexEnum::INDEX_SPARSE_INVERTED_INDEX; index_type_ = knowhere::IndexEnum::INDEX_SPARSE_INVERTED_INDEX_CC;
} else { } else {
index_type_ = support_index_types.at(segment_type); index_type_ = knowhere::IndexEnum::INDEX_FAISS_IVFFLAT_CC;
} }
build_params_[knowhere::meta::METRIC_TYPE] = metric_type_; build_params_[knowhere::meta::METRIC_TYPE] = metric_type_;
build_params_[knowhere::indexparam::NLIST] = build_params_[knowhere::indexparam::NLIST] =

4
internal/core/src/segcore/IndexConfigGenerator.h
View File

@ -30,10 +30,6 @@ enum class IndexConfigLevel {
// this is the config used for generating growing index or the temp sealed index // this is the config used for generating growing index or the temp sealed index
// when the segment is sealed before the index is built. // when the segment is sealed before the index is built.
class VecIndexConfig { class VecIndexConfig {
inline static const std::map<SegmentType, std::string> support_index_types =
{{SegmentType::Growing, knowhere::IndexEnum::INDEX_FAISS_IVFFLAT_CC},
{SegmentType::Sealed, knowhere::IndexEnum::INDEX_FAISS_IVFFLAT_CC}};
inline static const std::map<std::string, double> index_build_ratio = { inline static const std::map<std::string, double> index_build_ratio = {
{knowhere::IndexEnum::INDEX_FAISS_IVFFLAT_CC, 0.1}}; {knowhere::IndexEnum::INDEX_FAISS_IVFFLAT_CC, 0.1}};