diff --git a/CHANGELOG.md b/CHANGELOG.md index 6aa63d30ab..9fbb6d7a0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ Please mark all change in change log and use the issue from GitHub - \#1521 Make cache_insert_data take effect in-service - \#1525 Add setter API for config preload_table - \#1530 Set table file with correct engine type in meta +- \#1532 Search with ivf_flat failed with open-dataset: sift-256-hamming - \#1535 Degradation searching performance with metric_type: binary_idmap ## Feature diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 96f04a758c..29ed17308a 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -812,7 +812,7 @@ DBImpl::CompactFile(const std::string& table_id, const meta::TableFileSchema& fi // Update table files state // if index type isn't IDMAP, set file type to TO_INDEX if file size exceed index_file_size // else set file type to RAW, no need to build index - if (compacted_file.engine_type_ != (int)EngineType::FAISS_IDMAP) { + if (!utils::IsRawIndexType(compacted_file.engine_type_)) { compacted_file.file_type_ = (segment_writer_ptr->Size() >= compacted_file.index_file_size_) ? meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; @@ -1468,7 +1468,7 @@ DBImpl::MergeFiles(const std::string& table_id, const meta::TableFilesSchema& fi // step 4: update table files state // if index type isn't IDMAP, set file type to TO_INDEX if file size exceed index_file_size // else set file type to RAW, no need to build index - if (table_file.engine_type_ != (int)EngineType::FAISS_IDMAP) { + if (!utils::IsRawIndexType(table_file.engine_type_)) { table_file.file_type_ = (segment_writer_ptr->Size() >= table_file.index_file_size_) ? meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; @@ -1770,7 +1770,7 @@ DBImpl::BuildTableIndexRecursively(const std::string& table_id, const TableIndex // for IDMAP type, only wait all NEW file converted to RAW file // for other type, wait NEW/RAW/NEW_MERGE/NEW_INDEX/TO_INDEX files converted to INDEX files std::vector file_types; - if (index.engine_type_ == static_cast(EngineType::FAISS_IDMAP)) { + if (utils::IsRawIndexType(index.engine_type_)) { file_types = { static_cast(meta::TableFileSchema::NEW), static_cast(meta::TableFileSchema::NEW_MERGE), @@ -1792,7 +1792,7 @@ DBImpl::BuildTableIndexRecursively(const std::string& table_id, const TableIndex while (!table_files.empty()) { ENGINE_LOG_DEBUG << "Non index files detected! Will build index " << times; - if (index.engine_type_ != (int)EngineType::FAISS_IDMAP) { + if (!utils::IsRawIndexType(index.engine_type_)) { status = meta_ptr_->UpdateTableFilesToIndex(table_id); } diff --git a/core/src/db/Utils.cpp b/core/src/db/Utils.cpp index 3fa69c083a..f9ac5dfe40 100644 --- a/core/src/db/Utils.cpp +++ b/core/src/db/Utils.cpp @@ -215,6 +215,11 @@ IsSameIndex(const TableIndex& index1, const TableIndex& index2) { index1.metric_type_ == index2.metric_type_; } +bool +IsRawIndexType(int32_t type) { + return (type == (int32_t)EngineType::FAISS_IDMAP) || (type == (int32_t)EngineType::FAISS_BIN_IDMAP); +} + meta::DateT GetDate(const std::time_t& t, int day_delta) { struct tm ltm; diff --git a/core/src/db/Utils.h b/core/src/db/Utils.h index 810a0b5292..88197d7d5c 100644 --- a/core/src/db/Utils.h +++ b/core/src/db/Utils.h @@ -45,6 +45,9 @@ GetParentPath(const std::string& path, std::string& parent_path); bool IsSameIndex(const TableIndex& index1, const TableIndex& index2); +bool +IsRawIndexType(int32_t type); + meta::DateT GetDate(const std::time_t& t, int day_delta = 0); meta::DateT diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 8bd16fdc34..83d5f4f396 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -406,7 +406,7 @@ ExecutionEngineImpl::Load(bool to_cache) { utils::GetParentPath(location_, segment_dir); auto segment_reader_ptr = std::make_shared(segment_dir); - if (index_type_ == EngineType::FAISS_IDMAP || index_type_ == EngineType::FAISS_BIN_IDMAP) { + if (utils::IsRawIndexType((int32_t)index_type_)) { index_ = index_type_ == EngineType::FAISS_IDMAP ? GetVecIndexFactory(IndexType::FAISS_IDMAP) : GetVecIndexFactory(IndexType::FAISS_BIN_IDMAP); milvus::json conf{{knowhere::meta::DEVICEID, gpu_num_}, {knowhere::meta::DIM, dim_}}; diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index 8862ab5ee8..6f6e8cdcd3 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -2086,8 +2086,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/) // If we are deleting a raw table file, it means it's okay to delete the entire segment directory. // Else, we can only delete the single file // TODO(zhiru): We determine whether a table file is raw by its engine type. This is a bit hacky - if (table_file.engine_type_ == (int32_t)EngineType::FAISS_IDMAP || - table_file.engine_type_ == (int32_t)EngineType::FAISS_BIN_IDMAP) { + if (utils::IsRawIndexType(table_file.engine_type_)) { utils::DeleteSegment(options_, table_file); std::string segment_dir; utils::GetParentPath(table_file.location_, segment_dir); diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index 0521680946..031a473a9e 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -1425,8 +1425,7 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/ // If we are deleting a raw table file, it means it's okay to delete the entire segment directory. // Else, we can only delete the single file // TODO(zhiru): We determine whether a table file is raw by its engine type. This is a bit hacky - if (table_file.engine_type_ == (int32_t)EngineType::FAISS_IDMAP || - table_file.engine_type_ == (int32_t)EngineType::FAISS_BIN_IDMAP) { + if (utils::IsRawIndexType(table_file.engine_type_)) { utils::DeleteSegment(options_, table_file); std::string segment_dir; utils::GetParentPath(table_file.location_, segment_dir);