diff --git a/CHANGELOG.md b/CHANGELOG.md index f86f15976a..70ac6ced65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ Please mark all change in change log and use the issue from GitHub - \#1549 Fix server/wal config setting bug - \#1556 Index file not created after table and index created - \#1560 Search crashed with Super-high dimensional binary vector +- \#1574 Set all existing bitset in cache when applying deletes ## Feature - \#216 Add CLI to get server info diff --git a/core/src/db/insert/MemTable.cpp b/core/src/db/insert/MemTable.cpp index f0bdb7dedf..f91efbabe3 100644 --- a/core/src/db/insert/MemTable.cpp +++ b/core/src/db/insert/MemTable.cpp @@ -236,11 +236,27 @@ MemTable::ApplyDeletes() { utils::GetParentPath(table_file.location_, segment_dir); segment::SegmentReader segment_reader(segment_dir); - auto index = - std::static_pointer_cast(cache::CpuCacheMgr::GetInstance()->GetIndex(table_file.location_)); - faiss::ConcurrentBitsetPtr blacklist = nullptr; - if (index != nullptr) { - status = index->GetBlacklist(blacklist); + auto& segment_id = table_file.segment_id_; + meta::TableFilesSchema segment_files; + status = meta_->GetTableFilesBySegmentId(segment_id, segment_files); + if (!status.ok()) { + break; + } + + // Get all index that contains blacklist in cache + std::vector indexes; + std::vector blacklists; + for (auto& file : segment_files) { + auto index = + std::static_pointer_cast(cache::CpuCacheMgr::GetInstance()->GetIndex(file.location_)); + faiss::ConcurrentBitsetPtr blacklist = nullptr; + if (index != nullptr) { + index->GetBlacklist(blacklist); + if (blacklist != nullptr) { + indexes.emplace_back(index); + blacklists.emplace_back(blacklist); + } + } } std::vector uids; @@ -293,7 +309,7 @@ MemTable::ApplyDeletes() { id_bloom_filter_ptr->Remove(uids[i]); } - if (blacklist != nullptr) { + for (auto& blacklist : blacklists) { if (!blacklist->test(i)) { blacklist->set(i); } @@ -308,8 +324,8 @@ MemTable::ApplyDeletes() { << find_diff.count() << " s in total"; ENGINE_LOG_DEBUG << "Setting deleted docs and bloom filter took " << set_diff.count() << " s in total"; - if (index != nullptr) { - index->SetBlacklist(blacklist); + for (auto i = 0; i < indexes.size(); ++i) { + indexes[i]->SetBlacklist(blacklists[i]); } start = std::chrono::high_resolution_clock::now(); @@ -339,12 +355,6 @@ MemTable::ApplyDeletes() { << " s"; // Update table file row count - auto& segment_id = table_file.segment_id_; - meta::TableFilesSchema segment_files; - status = meta_->GetTableFilesBySegmentId(segment_id, segment_files); - if (!status.ok()) { - break; - } for (auto& file : segment_files) { if (file.file_type_ == meta::TableFileSchema::RAW || file.file_type_ == meta::TableFileSchema::TO_INDEX || file.file_type_ == meta::TableFileSchema::INDEX || file.file_type_ == meta::TableFileSchema::BACKUP) { diff --git a/core/src/wrapper/VecIndex.h b/core/src/wrapper/VecIndex.h index f7cf50b5bc..7a98e971fe 100644 --- a/core/src/wrapper/VecIndex.h +++ b/core/src/wrapper/VecIndex.h @@ -12,10 +12,10 @@ #pragma once #include -#include #include #include +#include #include #include @@ -180,13 +180,14 @@ class VecIndex : public cache::DataObj { virtual Status SetBlacklist(faiss::ConcurrentBitsetPtr list) { - ENGINE_LOG_ERROR << "SetBlacklist not support"; + // ENGINE_LOG_ERROR << "SetBlacklist not support"; return Status::OK(); } virtual Status GetBlacklist(faiss::ConcurrentBitsetPtr& list) { - ENGINE_LOG_ERROR << "GetBlacklist not support"; + // ENGINE_LOG_ERROR << "GetBlacklist not support"; + ENGINE_LOG_WARNING << "Deletion on unsupported index type"; return Status::OK(); } diff --git a/core/unittest/db/test_delete.cpp b/core/unittest/db/test_delete.cpp index 4d6051455d..15618a950e 100644 --- a/core/unittest/db/test_delete.cpp +++ b/core/unittest/db/test_delete.cpp @@ -68,7 +68,7 @@ TEST_F(DeleteTest, delete_in_mem) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -81,7 +81,7 @@ TEST_F(DeleteTest, delete_in_mem) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -105,7 +105,7 @@ TEST_F(DeleteTest, delete_in_mem) { ids_to_delete.emplace_back(kv.first); } - stat = db_->DeleteVectors(GetTableName(), ids_to_delete); + stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete); ASSERT_TRUE(stat.ok()); // std::this_thread::sleep_for(std::chrono::seconds(3)); // ensure raw data write to disk @@ -113,7 +113,7 @@ TEST_F(DeleteTest, delete_in_mem) { ASSERT_TRUE(stat.ok()); uint64_t row_count; - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb - search_vectors.size()); @@ -124,7 +124,8 @@ TEST_F(DeleteTest, delete_in_mem) { std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, GetTableName(), tags, topk, nprobe, search, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids, + result_distances); ASSERT_NE(result_ids[0], pair.first); // ASSERT_LT(result_distances[0], 1e-4); ASSERT_GT(result_distances[0], 1); @@ -136,7 +137,7 @@ TEST_F(DeleteTest, delete_on_disk) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -149,7 +150,7 @@ TEST_F(DeleteTest, delete_on_disk) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -173,7 +174,7 @@ TEST_F(DeleteTest, delete_on_disk) { ASSERT_TRUE(stat.ok()); for (auto& kv : search_vectors) { - stat = db_->DeleteVector(GetTableName(), kv.first); + stat = db_->DeleteVector(table_info.table_id_, kv.first); ASSERT_TRUE(stat.ok()); } @@ -181,7 +182,7 @@ TEST_F(DeleteTest, delete_on_disk) { ASSERT_TRUE(stat.ok()); uint64_t row_count; - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb - search_vectors.size()); @@ -192,7 +193,8 @@ TEST_F(DeleteTest, delete_on_disk) { std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, GetTableName(), tags, topk, nprobe, search, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids, + result_distances); ASSERT_NE(result_ids[0], pair.first); // ASSERT_LT(result_distances[0], 1e-4); ASSERT_GT(result_distances[0], 1); @@ -204,7 +206,7 @@ TEST_F(DeleteTest, delete_multiple_times) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -217,7 +219,7 @@ TEST_F(DeleteTest, delete_multiple_times) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -243,7 +245,7 @@ TEST_F(DeleteTest, delete_multiple_times) { int topk = 10, nprobe = 10; for (auto& pair : search_vectors) { std::vector to_delete{pair.first}; - stat = db_->DeleteVectors(GetTableName(), to_delete); + stat = db_->DeleteVectors(table_info.table_id_, to_delete); ASSERT_TRUE(stat.ok()); stat = db_->Flush(); @@ -254,7 +256,8 @@ TEST_F(DeleteTest, delete_multiple_times) { std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, GetTableName(), tags, topk, nprobe, search, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids, + result_distances); ASSERT_NE(result_ids[0], pair.first); // ASSERT_LT(result_distances[0], 1e-4); ASSERT_GT(result_distances[0], 1); @@ -267,7 +270,7 @@ TEST_F(DeleteTest, delete_with_index) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -280,7 +283,7 @@ TEST_F(DeleteTest, delete_with_index) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -302,7 +305,7 @@ TEST_F(DeleteTest, delete_with_index) { milvus::engine::TableIndex index; index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8; index.extra_params_ = {{"nlist", 100}}; - stat = db_->CreateIndex(GetTableName(), index); + stat = db_->CreateIndex(table_info.table_id_, index); ASSERT_TRUE(stat.ok()); // std::this_thread::sleep_for(std::chrono::seconds(3)); // ensure raw data write to disk @@ -313,13 +316,13 @@ TEST_F(DeleteTest, delete_with_index) { for (auto& kv : search_vectors) { ids_to_delete.emplace_back(kv.first); } - stat = db_->DeleteVectors(GetTableName(), ids_to_delete); + stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete); stat = db_->Flush(); ASSERT_TRUE(stat.ok()); uint64_t row_count; - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb - ids_to_delete.size()); @@ -330,7 +333,86 @@ TEST_F(DeleteTest, delete_with_index) { std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, GetTableName(), tags, topk, nprobe, search, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids, + result_distances); + ASSERT_NE(result_ids[0], pair.first); + // ASSERT_LT(result_distances[0], 1e-4); + ASSERT_GT(result_distances[0], 1); + } +} + +TEST_F(DeleteTest, delete_multiple_times_with_index) { + milvus::engine::meta::TableSchema table_info = BuildTableSchema(); + auto stat = db_->CreateTable(table_info); + + milvus::engine::meta::TableSchema table_info_get; + table_info_get.table_id_ = table_info.table_id_; + stat = db_->DescribeTable(table_info_get); + ASSERT_TRUE(stat.ok()); + ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); + + int64_t nb = 100000; + milvus::engine::VectorsData xb; + BuildVectors(nb, xb); + + for (int64_t i = 0; i < nb; i++) { + xb.id_array_.push_back(i); + } + + stat = db_->InsertVectors(table_info.table_id_, "", xb); + ASSERT_TRUE(stat.ok()); + + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution dis(0, nb - 1); + + int64_t num_query = 10; + std::map search_vectors; + for (int64_t i = 0; i < num_query; ++i) { + int64_t index = dis(gen); + milvus::engine::VectorsData search; + search.vector_count_ = 1; + for (int64_t j = 0; j < TABLE_DIM; j++) { + search.float_data_.push_back(xb.float_data_[index * TABLE_DIM + j]); + } + search_vectors.insert(std::make_pair(xb.id_array_[index], search)); + } + + // std::this_thread::sleep_for(std::chrono::seconds(3)); // ensure raw data write to disk + stat = db_->Flush(); + ASSERT_TRUE(stat.ok()); + + milvus::engine::TableIndex index; + index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFFLAT; + index.extra_params_ = {{"nlist", 1}}; + stat = db_->CreateIndex(table_info.table_id_, index); + ASSERT_TRUE(stat.ok()); + + int topk = 10, nprobe = 10; + int deleted = 0; + for (auto& pair : search_vectors) { + std::vector to_delete{pair.first}; + stat = db_->DeleteVectors(table_info.table_id_, to_delete); + ASSERT_TRUE(stat.ok()); + + stat = db_->Flush(); + ASSERT_TRUE(stat.ok()); + + ++deleted; + + uint64_t row_count; + stat = db_->GetTableRowCount(table_info.table_id_, row_count); + ASSERT_TRUE(stat.ok()); + ASSERT_EQ(row_count, nb - deleted); + + auto& search = pair.second; + + std::vector tags; + milvus::engine::ResultIds result_ids; + milvus::engine::ResultDistances result_distances; + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids, + result_distances); + ASSERT_TRUE(stat.ok()); ASSERT_NE(result_ids[0], pair.first); // ASSERT_LT(result_distances[0], 1e-4); ASSERT_GT(result_distances[0], 1); @@ -342,7 +424,7 @@ TEST_F(DeleteTest, delete_single_vector) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -351,21 +433,21 @@ TEST_F(DeleteTest, delete_single_vector) { milvus::engine::VectorsData xb; BuildVectors(nb, xb); - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); // std::this_thread::sleep_for(std::chrono::seconds(3)); // ensure raw data write to disk stat = db_->Flush(); ASSERT_TRUE(stat.ok()); - stat = db_->DeleteVectors(GetTableName(), xb.id_array_); + stat = db_->DeleteVectors(table_info.table_id_, xb.id_array_); ASSERT_TRUE(stat.ok()); stat = db_->Flush(); ASSERT_TRUE(stat.ok()); uint64_t row_count; - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, 0); @@ -375,7 +457,7 @@ TEST_F(DeleteTest, delete_single_vector) { std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, xb, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, xb, result_ids, result_distances); ASSERT_TRUE(result_ids.empty()); ASSERT_TRUE(result_distances.empty()); // ASSERT_EQ(result_ids[0], -1); @@ -388,7 +470,7 @@ TEST_F(DeleteTest, delete_add_create_index) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -397,7 +479,7 @@ TEST_F(DeleteTest, delete_add_create_index) { milvus::engine::VectorsData xb; BuildVectors(nb, xb); - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); // stat = db_->Flush(); @@ -405,27 +487,27 @@ TEST_F(DeleteTest, delete_add_create_index) { milvus::engine::TableIndex index; index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8; index.extra_params_ = {{"nlist", 100}}; - stat = db_->CreateIndex(GetTableName(), index); + stat = db_->CreateIndex(table_info.table_id_, index); ASSERT_TRUE(stat.ok()); std::vector ids_to_delete; ids_to_delete.emplace_back(xb.id_array_.front()); - stat = db_->DeleteVectors(GetTableName(), ids_to_delete); + stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete); ASSERT_TRUE(stat.ok()); milvus::engine::VectorsData xb2 = xb; xb2.id_array_.clear(); // same vector, different id - stat = db_->InsertVectors(GetTableName(), "", xb2); + stat = db_->InsertVectors(table_info.table_id_, "", xb2); ASSERT_TRUE(stat.ok()); // stat = db_->Flush(); // ASSERT_TRUE(stat.ok()); - stat = db_->CreateIndex(GetTableName(), index); + stat = db_->CreateIndex(table_info.table_id_, index); ASSERT_TRUE(stat.ok()); uint64_t row_count; - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb * 2 - 1); @@ -439,15 +521,15 @@ TEST_F(DeleteTest, delete_add_create_index) { qb.float_data_.resize(TABLE_DIM); qb.vector_count_ = 1; qb.id_array_.clear(); - stat = db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, qb, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, qb, result_ids, result_distances); ASSERT_EQ(result_ids[0], xb2.id_array_.front()); ASSERT_LT(result_distances[0], 1e-4); result_ids.clear(); result_distances.clear(); - stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, ids_to_delete.front(), result_ids, - result_distances); + stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, ids_to_delete.front(), + result_ids, result_distances); ASSERT_EQ(result_ids[0], -1); ASSERT_EQ(result_distances[0], std::numeric_limits::max()); } @@ -457,7 +539,7 @@ TEST_F(DeleteTest, delete_add_auto_flush) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -466,7 +548,7 @@ TEST_F(DeleteTest, delete_add_auto_flush) { milvus::engine::VectorsData xb; BuildVectors(nb, xb); - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::this_thread::sleep_for(std::chrono::seconds(2)); @@ -475,28 +557,28 @@ TEST_F(DeleteTest, delete_add_auto_flush) { // ASSERT_TRUE(stat.ok()); // milvus::engine::TableIndex index; // index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8; - // stat = db_->CreateIndex(GetTableName(), index); + // stat = db_->CreateIndex(table_info.table_id_, index); // ASSERT_TRUE(stat.ok()); std::vector ids_to_delete; ids_to_delete.emplace_back(xb.id_array_.front()); - stat = db_->DeleteVectors(GetTableName(), ids_to_delete); + stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete); ASSERT_TRUE(stat.ok()); milvus::engine::VectorsData xb2 = xb; xb2.id_array_.clear(); // same vector, different id - stat = db_->InsertVectors(GetTableName(), "", xb2); + stat = db_->InsertVectors(table_info.table_id_, "", xb2); ASSERT_TRUE(stat.ok()); std::this_thread::sleep_for(std::chrono::seconds(2)); // stat = db_->Flush(); // ASSERT_TRUE(stat.ok()); - // stat = db_->CreateIndex(GetTableName(), index); + // stat = db_->CreateIndex(table_info.table_id_, index); // ASSERT_TRUE(stat.ok()); uint64_t row_count; - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb * 2 - 1); @@ -510,15 +592,15 @@ TEST_F(DeleteTest, delete_add_auto_flush) { qb.float_data_.resize(TABLE_DIM); qb.vector_count_ = 1; qb.id_array_.clear(); - stat = db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, qb, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, qb, result_ids, result_distances); ASSERT_EQ(result_ids[0], xb2.id_array_.front()); ASSERT_LT(result_distances[0], 1e-4); result_ids.clear(); result_distances.clear(); - stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, nprobe, ids_to_delete.front(), result_ids, - result_distances); + stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, ids_to_delete.front(), + result_ids, result_distances); ASSERT_EQ(result_ids[0], -1); ASSERT_EQ(result_distances[0], std::numeric_limits::max()); } @@ -528,7 +610,7 @@ TEST_F(CompactTest, compact_basic) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -537,7 +619,7 @@ TEST_F(CompactTest, compact_basic) { milvus::engine::VectorsData xb; BuildVectors(nb, xb); - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); stat = db_->Flush(); @@ -546,18 +628,18 @@ TEST_F(CompactTest, compact_basic) { std::vector ids_to_delete; ids_to_delete.emplace_back(xb.id_array_.front()); ids_to_delete.emplace_back(xb.id_array_.back()); - stat = db_->DeleteVectors(GetTableName(), ids_to_delete); + stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete); ASSERT_TRUE(stat.ok()); stat = db_->Flush(); ASSERT_TRUE(stat.ok()); uint64_t row_count; - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb - 2); - stat = db_->Compact(GetTableName()); + stat = db_->Compact(table_info.table_id_); ASSERT_TRUE(stat.ok()); const int topk = 1, nprobe = 1; @@ -569,8 +651,8 @@ TEST_F(CompactTest, compact_basic) { milvus::engine::VectorsData qb = xb; for (auto& id : ids_to_delete) { - stat = - db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, id, result_ids, result_distances); + stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, id, result_ids, + result_distances); ASSERT_EQ(result_ids[0], -1); ASSERT_EQ(result_distances[0], std::numeric_limits::max()); } @@ -583,7 +665,7 @@ TEST_F(CompactTest, compact_with_index) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -597,7 +679,7 @@ TEST_F(CompactTest, compact_with_index) { xb.id_array_.emplace_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -618,7 +700,7 @@ TEST_F(CompactTest, compact_with_index) { milvus::engine::TableIndex index; index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8; - stat = db_->CreateIndex(GetTableName(), index); + stat = db_->CreateIndex(table_info.table_id_, index); ASSERT_TRUE(stat.ok()); stat = db_->Flush(); @@ -628,25 +710,25 @@ TEST_F(CompactTest, compact_with_index) { for (auto& kv : search_vectors) { ids_to_delete.emplace_back(kv.first); } - stat = db_->DeleteVectors(GetTableName(), ids_to_delete); + stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete); stat = db_->Flush(); ASSERT_TRUE(stat.ok()); uint64_t row_count; - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb - ids_to_delete.size()); - stat = db_->Compact(GetTableName()); + stat = db_->Compact(table_info.table_id_); ASSERT_TRUE(stat.ok()); - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb - ids_to_delete.size()); milvus::engine::TableIndex table_index; - stat = db_->DescribeIndex(GetTableName(), table_index); + stat = db_->DescribeIndex(table_info.table_id_, table_index); ASSERT_TRUE(stat.ok()); ASSERT_FLOAT_EQ(table_index.engine_type_, index.engine_type_); @@ -659,8 +741,8 @@ TEST_F(CompactTest, compact_with_index) { std::vector tags; milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = - db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, search, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, search, result_ids, + result_distances); ASSERT_NE(result_ids[0], pair.first); // ASSERT_LT(result_distances[0], 1e-4); ASSERT_GT(result_distances[0], 1); diff --git a/core/unittest/db/test_search_by_id.cpp b/core/unittest/db/test_search_by_id.cpp index a4edb301b0..568626296b 100644 --- a/core/unittest/db/test_search_by_id.cpp +++ b/core/unittest/db/test_search_by_id.cpp @@ -68,7 +68,7 @@ TEST_F(SearchByIdTest, basic) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -81,7 +81,7 @@ TEST_F(SearchByIdTest, basic) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -108,7 +108,8 @@ TEST_F(SearchByIdTest, basic) { milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, i, result_ids, result_distances); + stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, i, result_ids, + result_distances); ASSERT_EQ(result_ids[0], i); ASSERT_LT(result_distances[0], 1e-4); } @@ -119,7 +120,7 @@ TEST_F(SearchByIdTest, with_index) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -132,7 +133,7 @@ TEST_F(SearchByIdTest, with_index) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -153,7 +154,7 @@ TEST_F(SearchByIdTest, with_index) { milvus::engine::TableIndex index; index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8; index.extra_params_ = {{"nlist", 10}}; - stat = db_->CreateIndex(GetTableName(), index); + stat = db_->CreateIndex(table_info.table_id_, index); ASSERT_TRUE(stat.ok()); const int topk = 10, nprobe = 10; @@ -165,7 +166,8 @@ TEST_F(SearchByIdTest, with_index) { milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, i, result_ids, result_distances); + stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, i, result_ids, + result_distances); ASSERT_EQ(result_ids[0], i); ASSERT_LT(result_distances[0], 1e-3); } @@ -176,7 +178,7 @@ TEST_F(SearchByIdTest, with_delete) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -189,7 +191,7 @@ TEST_F(SearchByIdTest, with_delete) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -211,7 +213,7 @@ TEST_F(SearchByIdTest, with_delete) { for (auto& id : ids_to_search) { ids_to_delete.emplace_back(id); } - stat = db_->DeleteVectors(GetTableName(), ids_to_delete); + stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete); stat = db_->Flush(); ASSERT_TRUE(stat.ok()); @@ -225,7 +227,8 @@ TEST_F(SearchByIdTest, with_delete) { milvus::engine::ResultIds result_ids; milvus::engine::ResultDistances result_distances; - stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, i, result_ids, result_distances); + stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, i, result_ids, + result_distances); ASSERT_EQ(result_ids[0], -1); ASSERT_EQ(result_distances[0], std::numeric_limits::max()); } @@ -236,7 +239,7 @@ TEST_F(GetVectorByIdTest, basic) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -249,7 +252,7 @@ TEST_F(GetVectorByIdTest, basic) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -277,11 +280,11 @@ TEST_F(GetVectorByIdTest, basic) { milvus::engine::ResultDistances result_distances; milvus::engine::VectorsData vector; - stat = db_->GetVectorByID(GetTableName(), id, vector); + stat = db_->GetVectorByID(table_info.table_id_, id, vector); ASSERT_TRUE(stat.ok()); - stat = - db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, vector, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, vector, result_ids, + result_distances); ASSERT_TRUE(stat.ok()); ASSERT_EQ(result_ids[0], id); ASSERT_LT(result_distances[0], 1e-4); @@ -293,7 +296,7 @@ TEST_F(GetVectorByIdTest, with_index) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -306,7 +309,7 @@ TEST_F(GetVectorByIdTest, with_index) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -327,7 +330,7 @@ TEST_F(GetVectorByIdTest, with_index) { milvus::engine::TableIndex index; index.extra_params_ = {{"nlist", 10}}; index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8; - stat = db_->CreateIndex(GetTableName(), index); + stat = db_->CreateIndex(table_info.table_id_, index); ASSERT_TRUE(stat.ok()); const int topk = 10, nprobe = 10; @@ -340,11 +343,11 @@ TEST_F(GetVectorByIdTest, with_index) { milvus::engine::ResultDistances result_distances; milvus::engine::VectorsData vector; - stat = db_->GetVectorByID(GetTableName(), id, vector); + stat = db_->GetVectorByID(table_info.table_id_, id, vector); ASSERT_TRUE(stat.ok()); - stat = - db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, vector, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, vector, result_ids, + result_distances); ASSERT_EQ(result_ids[0], id); ASSERT_LT(result_distances[0], 1e-3); } @@ -355,7 +358,7 @@ TEST_F(GetVectorByIdTest, with_delete) { auto stat = db_->CreateTable(table_info); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -368,7 +371,7 @@ TEST_F(GetVectorByIdTest, with_delete) { xb.id_array_.push_back(i); } - stat = db_->InsertVectors(GetTableName(), "", xb); + stat = db_->InsertVectors(table_info.table_id_, "", xb); ASSERT_TRUE(stat.ok()); std::random_device rd; @@ -390,7 +393,7 @@ TEST_F(GetVectorByIdTest, with_delete) { for (auto& id : ids_to_search) { ids_to_delete.emplace_back(id); } - stat = db_->DeleteVectors(GetTableName(), ids_to_delete); + stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete); stat = db_->Flush(); ASSERT_TRUE(stat.ok()); @@ -402,7 +405,7 @@ TEST_F(GetVectorByIdTest, with_delete) { milvus::engine::ResultDistances result_distances; milvus::engine::VectorsData vector; - stat = db_->GetVectorByID(GetTableName(), id, vector); + stat = db_->GetVectorByID(table_info.table_id_, id, vector); ASSERT_TRUE(stat.ok()); ASSERT_TRUE(vector.float_data_.empty()); ASSERT_EQ(vector.vector_count_, 0); @@ -419,7 +422,7 @@ TEST_F(SearchByIdTest, BINARY) { ASSERT_TRUE(stat.ok()); milvus::engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = GetTableName(); + table_info_get.table_id_ = table_info.table_id_; stat = db_->DescribeTable(table_info_get); ASSERT_TRUE(stat.ok()); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); @@ -445,7 +448,7 @@ TEST_F(SearchByIdTest, BINARY) { vectors.id_array_.emplace_back(k * nb + i); } - stat = db_->InsertVectors(GetTableName(), "", vectors); + stat = db_->InsertVectors(table_info.table_id_, "", vectors); ASSERT_TRUE(stat.ok()); } @@ -465,7 +468,7 @@ TEST_F(SearchByIdTest, BINARY) { ASSERT_TRUE(stat.ok()); uint64_t row_count; - stat = db_->GetTableRowCount(GetTableName(), row_count); + stat = db_->GetTableRowCount(table_info.table_id_, row_count); ASSERT_TRUE(stat.ok()); ASSERT_EQ(row_count, nb * insert_loop); @@ -479,12 +482,12 @@ TEST_F(SearchByIdTest, BINARY) { milvus::engine::ResultDistances result_distances; milvus::engine::VectorsData vector; - stat = db_->GetVectorByID(GetTableName(), id, vector); + stat = db_->GetVectorByID(table_info.table_id_, id, vector); ASSERT_TRUE(stat.ok()); ASSERT_EQ(vector.vector_count_, 1); - stat = - db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, vector, result_ids, result_distances); + stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, vector, result_ids, + result_distances); ASSERT_TRUE(stat.ok()); ASSERT_EQ(result_ids[0], id); ASSERT_LT(result_distances[0], 1e-4); @@ -493,8 +496,8 @@ TEST_F(SearchByIdTest, BINARY) { result_ids.clear(); result_distances.clear(); - stat = - db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, id, result_ids, result_distances); + stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, id, result_ids, + result_distances); ASSERT_TRUE(stat.ok()); ASSERT_EQ(result_ids[0], id); ASSERT_LT(result_distances[0], 1e-4);