test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-03 12:29:36 +08:00
Code Issues Actions 13 Packages Projects Releases Wiki Activity

Merge pull request #1575 from youny626/fix#1574

Browse Source 
Fix#1574
This commit is contained in:
Jin Hai 2020-03-09 15:10:59 +08:00 committed by GitHub
commit 7cd727aaea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 210 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -46,6 +46,7 @@ Please mark all change in change log and use the issue from GitHub
- \#1549 Fix server/wal config setting bug
- \#1556 Index file not created after table and index created
- \#1560 Search crashed with Super-high dimensional binary vector
- \#1574 Set all existing bitset in cache when applying deletes
## Feature
- \#216 Add CLI to get server info

38
core/src/db/insert/MemTable.cpp
View File

@ -236,11 +236,27 @@ MemTable::ApplyDeletes() {
utils::GetParentPath(table_file.location_, segment_dir);
segment::SegmentReader segment_reader(segment_dir);
auto index =
std::static_pointer_cast<VecIndex>(cache::CpuCacheMgr::GetInstance()->GetIndex(table_file.location_));
faiss::ConcurrentBitsetPtr blacklist = nullptr;
if (index != nullptr) {
status = index->GetBlacklist(blacklist);
auto& segment_id = table_file.segment_id_;
meta::TableFilesSchema segment_files;
status = meta_->GetTableFilesBySegmentId(segment_id, segment_files);
if (!status.ok()) {
break;
}
// Get all index that contains blacklist in cache
std::vector<VecIndexPtr> indexes;
std::vector<faiss::ConcurrentBitsetPtr> blacklists;
for (auto& file : segment_files) {
auto index =
std::static_pointer_cast<VecIndex>(cache::CpuCacheMgr::GetInstance()->GetIndex(file.location_));
faiss::ConcurrentBitsetPtr blacklist = nullptr;
if (index != nullptr) {
index->GetBlacklist(blacklist);
if (blacklist != nullptr) {
indexes.emplace_back(index);
blacklists.emplace_back(blacklist);
}
}
}
std::vector<segment::doc_id_t> uids;
@ -293,7 +309,7 @@ MemTable::ApplyDeletes() {
id_bloom_filter_ptr->Remove(uids[i]);
}
if (blacklist != nullptr) {
for (auto& blacklist : blacklists) {
if (!blacklist->test(i)) {
blacklist->set(i);
}
@ -308,8 +324,8 @@ MemTable::ApplyDeletes() {
<< find_diff.count() << " s in total";
ENGINE_LOG_DEBUG << "Setting deleted docs and bloom filter took " << set_diff.count() << " s in total";
if (index != nullptr) {
index->SetBlacklist(blacklist);
for (auto i = 0; i < indexes.size(); ++i) {
indexes[i]->SetBlacklist(blacklists[i]);
}
start = std::chrono::high_resolution_clock::now();
@ -339,12 +355,6 @@ MemTable::ApplyDeletes() {
<< " s";
// Update table file row count
auto& segment_id = table_file.segment_id_;
meta::TableFilesSchema segment_files;
status = meta_->GetTableFilesBySegmentId(segment_id, segment_files);
if (!status.ok()) {
break;
}
for (auto& file : segment_files) {
if (file.file_type_ == meta::TableFileSchema::RAW || file.file_type_ == meta::TableFileSchema::TO_INDEX ||
file.file_type_ == meta::TableFileSchema::INDEX || file.file_type_ == meta::TableFileSchema::BACKUP) {

7
core/src/wrapper/VecIndex.h
View File

@ -12,10 +12,10 @@
#pragma once
#include <faiss/utils/ConcurrentBitset.h>
#include <thirdparty/nlohmann/json.hpp>
#include <memory>
#include <string>
#include <thirdparty/nlohmann/json.hpp>
#include <utility>
#include <vector>
@ -180,13 +180,14 @@ class VecIndex : public cache::DataObj {
virtual Status
SetBlacklist(faiss::ConcurrentBitsetPtr list) {
ENGINE_LOG_ERROR << "SetBlacklist not support";
// ENGINE_LOG_ERROR << "SetBlacklist not support";
return Status::OK();
}
virtual Status
GetBlacklist(faiss::ConcurrentBitsetPtr& list) {
ENGINE_LOG_ERROR << "GetBlacklist not support";
// ENGINE_LOG_ERROR << "GetBlacklist not support";
ENGINE_LOG_WARNING << "Deletion on unsupported index type";
return Status::OK();
}

206
core/unittest/db/test_delete.cpp
View File

@ -68,7 +68,7 @@ TEST_F(DeleteTest, delete_in_mem) {
auto stat = db_->CreateTable(table_info);
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = GetTableName();
table_info_get.table_id_ = table_info.table_id_;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
@ -81,7 +81,7 @@ TEST_F(DeleteTest, delete_in_mem) {
xb.id_array_.push_back(i);
}
stat = db_->InsertVectors(GetTableName(), "", xb);
stat = db_->InsertVectors(table_info.table_id_, "", xb);
ASSERT_TRUE(stat.ok());
std::random_device rd;
@ -105,7 +105,7 @@ TEST_F(DeleteTest, delete_in_mem) {
ids_to_delete.emplace_back(kv.first);
}
stat = db_->DeleteVectors(GetTableName(), ids_to_delete);
stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete);
ASSERT_TRUE(stat.ok());
// std::this_thread::sleep_for(std::chrono::seconds(3)); // ensure raw data write to disk
@ -113,7 +113,7 @@ TEST_F(DeleteTest, delete_in_mem) {
ASSERT_TRUE(stat.ok());
uint64_t row_count;
stat = db_->GetTableRowCount(GetTableName(), row_count);
stat = db_->GetTableRowCount(table_info.table_id_, row_count);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(row_count, nb - search_vectors.size());
@ -124,7 +124,8 @@ TEST_F(DeleteTest, delete_in_mem) {
std::vector<std::string> tags;
milvus::engine::ResultIds result_ids;
milvus::engine::ResultDistances result_distances;
stat = db_->Query(dummy_context_, GetTableName(), tags, topk, nprobe, search, result_ids, result_distances);
stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids,
result_distances);
ASSERT_NE(result_ids[0], pair.first);
// ASSERT_LT(result_distances[0], 1e-4);
ASSERT_GT(result_distances[0], 1);
@ -136,7 +137,7 @@ TEST_F(DeleteTest, delete_on_disk) {
auto stat = db_->CreateTable(table_info);
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = GetTableName();
table_info_get.table_id_ = table_info.table_id_;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
@ -149,7 +150,7 @@ TEST_F(DeleteTest, delete_on_disk) {
xb.id_array_.push_back(i);
}
stat = db_->InsertVectors(GetTableName(), "", xb);
stat = db_->InsertVectors(table_info.table_id_, "", xb);
ASSERT_TRUE(stat.ok());
std::random_device rd;
@ -173,7 +174,7 @@ TEST_F(DeleteTest, delete_on_disk) {
ASSERT_TRUE(stat.ok());
for (auto& kv : search_vectors) {
stat = db_->DeleteVector(GetTableName(), kv.first);
stat = db_->DeleteVector(table_info.table_id_, kv.first);
ASSERT_TRUE(stat.ok());
}
@ -181,7 +182,7 @@ TEST_F(DeleteTest, delete_on_disk) {
ASSERT_TRUE(stat.ok());
uint64_t row_count;
stat = db_->GetTableRowCount(GetTableName(), row_count);
stat = db_->GetTableRowCount(table_info.table_id_, row_count);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(row_count, nb - search_vectors.size());
@ -192,7 +193,8 @@ TEST_F(DeleteTest, delete_on_disk) {
std::vector<std::string> tags;
milvus::engine::ResultIds result_ids;
milvus::engine::ResultDistances result_distances;
stat = db_->Query(dummy_context_, GetTableName(), tags, topk, nprobe, search, result_ids, result_distances);
stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids,
result_distances);
ASSERT_NE(result_ids[0], pair.first);
// ASSERT_LT(result_distances[0], 1e-4);
ASSERT_GT(result_distances[0], 1);
@ -204,7 +206,7 @@ TEST_F(DeleteTest, delete_multiple_times) {
auto stat = db_->CreateTable(table_info);
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = GetTableName();
table_info_get.table_id_ = table_info.table_id_;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
@ -217,7 +219,7 @@ TEST_F(DeleteTest, delete_multiple_times) {
xb.id_array_.push_back(i);
}
stat = db_->InsertVectors(GetTableName(), "", xb);
stat = db_->InsertVectors(table_info.table_id_, "", xb);
ASSERT_TRUE(stat.ok());
std::random_device rd;
@ -243,7 +245,7 @@ TEST_F(DeleteTest, delete_multiple_times) {
int topk = 10, nprobe = 10;
for (auto& pair : search_vectors) {
std::vector<int64_t> to_delete{pair.first};
stat = db_->DeleteVectors(GetTableName(), to_delete);
stat = db_->DeleteVectors(table_info.table_id_, to_delete);
ASSERT_TRUE(stat.ok());
stat = db_->Flush();
@ -254,7 +256,8 @@ TEST_F(DeleteTest, delete_multiple_times) {
std::vector<std::string> tags;
milvus::engine::ResultIds result_ids;
milvus::engine::ResultDistances result_distances;
stat = db_->Query(dummy_context_, GetTableName(), tags, topk, nprobe, search, result_ids, result_distances);
stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids,
result_distances);
ASSERT_NE(result_ids[0], pair.first);
// ASSERT_LT(result_distances[0], 1e-4);
ASSERT_GT(result_distances[0], 1);
@ -267,7 +270,7 @@ TEST_F(DeleteTest, delete_with_index) {
auto stat = db_->CreateTable(table_info);
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = GetTableName();
table_info_get.table_id_ = table_info.table_id_;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
@ -280,7 +283,7 @@ TEST_F(DeleteTest, delete_with_index) {
xb.id_array_.push_back(i);
}
stat = db_->InsertVectors(GetTableName(), "", xb);
stat = db_->InsertVectors(table_info.table_id_, "", xb);
ASSERT_TRUE(stat.ok());
std::random_device rd;
@ -302,7 +305,7 @@ TEST_F(DeleteTest, delete_with_index) {
milvus::engine::TableIndex index;
index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8;
index.extra_params_ = {{"nlist", 100}};
stat = db_->CreateIndex(GetTableName(), index);
stat = db_->CreateIndex(table_info.table_id_, index);
ASSERT_TRUE(stat.ok());
// std::this_thread::sleep_for(std::chrono::seconds(3)); // ensure raw data write to disk
@ -313,13 +316,13 @@ TEST_F(DeleteTest, delete_with_index) {
for (auto& kv : search_vectors) {
ids_to_delete.emplace_back(kv.first);
}
stat = db_->DeleteVectors(GetTableName(), ids_to_delete);
stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete);
stat = db_->Flush();
ASSERT_TRUE(stat.ok());
uint64_t row_count;
stat = db_->GetTableRowCount(GetTableName(), row_count);
stat = db_->GetTableRowCount(table_info.table_id_, row_count);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(row_count, nb - ids_to_delete.size());
@ -330,7 +333,86 @@ TEST_F(DeleteTest, delete_with_index) {
std::vector<std::string> tags;
milvus::engine::ResultIds result_ids;
milvus::engine::ResultDistances result_distances;
stat = db_->Query(dummy_context_, GetTableName(), tags, topk, nprobe, search, result_ids, result_distances);
stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids,
result_distances);
ASSERT_NE(result_ids[0], pair.first);
// ASSERT_LT(result_distances[0], 1e-4);
ASSERT_GT(result_distances[0], 1);
}
}
TEST_F(DeleteTest, delete_multiple_times_with_index) {
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = table_info.table_id_;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
int64_t nb = 100000;
milvus::engine::VectorsData xb;
BuildVectors(nb, xb);
for (int64_t i = 0; i < nb; i++) {
xb.id_array_.push_back(i);
}
stat = db_->InsertVectors(table_info.table_id_, "", xb);
ASSERT_TRUE(stat.ok());
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_int_distribution<int64_t> dis(0, nb - 1);
int64_t num_query = 10;
std::map<int64_t, milvus::engine::VectorsData> search_vectors;
for (int64_t i = 0; i < num_query; ++i) {
int64_t index = dis(gen);
milvus::engine::VectorsData search;
search.vector_count_ = 1;
for (int64_t j = 0; j < TABLE_DIM; j++) {
search.float_data_.push_back(xb.float_data_[index * TABLE_DIM + j]);
}
search_vectors.insert(std::make_pair(xb.id_array_[index], search));
}
// std::this_thread::sleep_for(std::chrono::seconds(3)); // ensure raw data write to disk
stat = db_->Flush();
ASSERT_TRUE(stat.ok());
milvus::engine::TableIndex index;
index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFFLAT;
index.extra_params_ = {{"nlist", 1}};
stat = db_->CreateIndex(table_info.table_id_, index);
ASSERT_TRUE(stat.ok());
int topk = 10, nprobe = 10;
int deleted = 0;
for (auto& pair : search_vectors) {
std::vector<int64_t> to_delete{pair.first};
stat = db_->DeleteVectors(table_info.table_id_, to_delete);
ASSERT_TRUE(stat.ok());
stat = db_->Flush();
ASSERT_TRUE(stat.ok());
++deleted;
uint64_t row_count;
stat = db_->GetTableRowCount(table_info.table_id_, row_count);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(row_count, nb - deleted);
auto& search = pair.second;
std::vector<std::string> tags;
milvus::engine::ResultIds result_ids;
milvus::engine::ResultDistances result_distances;
stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, search, result_ids,
result_distances);
ASSERT_TRUE(stat.ok());
ASSERT_NE(result_ids[0], pair.first);
// ASSERT_LT(result_distances[0], 1e-4);
ASSERT_GT(result_distances[0], 1);
@ -342,7 +424,7 @@ TEST_F(DeleteTest, delete_single_vector) {
auto stat = db_->CreateTable(table_info);
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = GetTableName();
table_info_get.table_id_ = table_info.table_id_;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
@ -351,21 +433,21 @@ TEST_F(DeleteTest, delete_single_vector) {
milvus::engine::VectorsData xb;
BuildVectors(nb, xb);
stat = db_->InsertVectors(GetTableName(), "", xb);
stat = db_->InsertVectors(table_info.table_id_, "", xb);
ASSERT_TRUE(stat.ok());
// std::this_thread::sleep_for(std::chrono::seconds(3)); // ensure raw data write to disk
stat = db_->Flush();
ASSERT_TRUE(stat.ok());
stat = db_->DeleteVectors(GetTableName(), xb.id_array_);
stat = db_->DeleteVectors(table_info.table_id_, xb.id_array_);
ASSERT_TRUE(stat.ok());
stat = db_->Flush();
ASSERT_TRUE(stat.ok());
uint64_t row_count;
stat = db_->GetTableRowCount(GetTableName(), row_count);
stat = db_->GetTableRowCount(table_info.table_id_, row_count);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(row_count, 0);
@ -375,7 +457,7 @@ TEST_F(DeleteTest, delete_single_vector) {
std::vector<std::string> tags;
milvus::engine::ResultIds result_ids;
milvus::engine::ResultDistances result_distances;
stat = db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, xb, result_ids, result_distances);
stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, xb, result_ids, result_distances);
ASSERT_TRUE(result_ids.empty());
ASSERT_TRUE(result_distances.empty());
// ASSERT_EQ(result_ids[0], -1);
@ -388,7 +470,7 @@ TEST_F(DeleteTest, delete_add_create_index) {
auto stat = db_->CreateTable(table_info);
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = GetTableName();
table_info_get.table_id_ = table_info.table_id_;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
@ -397,7 +479,7 @@ TEST_F(DeleteTest, delete_add_create_index) {
milvus::engine::VectorsData xb;
BuildVectors(nb, xb);
stat = db_->InsertVectors(GetTableName(), "", xb);
stat = db_->InsertVectors(table_info.table_id_, "", xb);
ASSERT_TRUE(stat.ok());
// stat = db_->Flush();
@ -405,27 +487,27 @@ TEST_F(DeleteTest, delete_add_create_index) {
milvus::engine::TableIndex index;
index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8;
index.extra_params_ = {{"nlist", 100}};
stat = db_->CreateIndex(GetTableName(), index);
stat = db_->CreateIndex(table_info.table_id_, index);
ASSERT_TRUE(stat.ok());
std::vector<milvus::engine::IDNumber> ids_to_delete;
ids_to_delete.emplace_back(xb.id_array_.front());
stat = db_->DeleteVectors(GetTableName(), ids_to_delete);
stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete);
ASSERT_TRUE(stat.ok());
milvus::engine::VectorsData xb2 = xb;
xb2.id_array_.clear(); // same vector, different id
stat = db_->InsertVectors(GetTableName(), "", xb2);
stat = db_->InsertVectors(table_info.table_id_, "", xb2);
ASSERT_TRUE(stat.ok());
// stat = db_->Flush();
// ASSERT_TRUE(stat.ok());
stat = db_->CreateIndex(GetTableName(), index);
stat = db_->CreateIndex(table_info.table_id_, index);
ASSERT_TRUE(stat.ok());
uint64_t row_count;
stat = db_->GetTableRowCount(GetTableName(), row_count);
stat = db_->GetTableRowCount(table_info.table_id_, row_count);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(row_count, nb * 2 - 1);
@ -439,15 +521,15 @@ TEST_F(DeleteTest, delete_add_create_index) {
qb.float_data_.resize(TABLE_DIM);
qb.vector_count_ = 1;
qb.id_array_.clear();
stat = db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, qb, result_ids, result_distances);
stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, qb, result_ids, result_distances);
ASSERT_EQ(result_ids[0], xb2.id_array_.front());
ASSERT_LT(result_distances[0], 1e-4);
result_ids.clear();
result_distances.clear();
stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, ids_to_delete.front(), result_ids,
result_distances);
stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, ids_to_delete.front(),
result_ids, result_distances);
ASSERT_EQ(result_ids[0], -1);
ASSERT_EQ(result_distances[0], std::numeric_limits<float>::max());
}
@ -457,7 +539,7 @@ TEST_F(DeleteTest, delete_add_auto_flush) {
auto stat = db_->CreateTable(table_info);
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = GetTableName();
table_info_get.table_id_ = table_info.table_id_;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
@ -466,7 +548,7 @@ TEST_F(DeleteTest, delete_add_auto_flush) {
milvus::engine::VectorsData xb;
BuildVectors(nb, xb);
stat = db_->InsertVectors(GetTableName(), "", xb);
stat = db_->InsertVectors(table_info.table_id_, "", xb);
ASSERT_TRUE(stat.ok());
std::this_thread::sleep_for(std::chrono::seconds(2));
@ -475,28 +557,28 @@ TEST_F(DeleteTest, delete_add_auto_flush) {
// ASSERT_TRUE(stat.ok());
// milvus::engine::TableIndex index;
// index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8;
// stat = db_->CreateIndex(GetTableName(), index);
// stat = db_->CreateIndex(table_info.table_id_, index);
// ASSERT_TRUE(stat.ok());
std::vector<milvus::engine::IDNumber> ids_to_delete;
ids_to_delete.emplace_back(xb.id_array_.front());
stat = db_->DeleteVectors(GetTableName(), ids_to_delete);
stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete);
ASSERT_TRUE(stat.ok());
milvus::engine::VectorsData xb2 = xb;
xb2.id_array_.clear(); // same vector, different id
stat = db_->InsertVectors(GetTableName(), "", xb2);
stat = db_->InsertVectors(table_info.table_id_, "", xb2);
ASSERT_TRUE(stat.ok());
std::this_thread::sleep_for(std::chrono::seconds(2));
// stat = db_->Flush();
// ASSERT_TRUE(stat.ok());
// stat = db_->CreateIndex(GetTableName(), index);
// stat = db_->CreateIndex(table_info.table_id_, index);
// ASSERT_TRUE(stat.ok());
uint64_t row_count;
stat = db_->GetTableRowCount(GetTableName(), row_count);
stat = db_->GetTableRowCount(table_info.table_id_, row_count);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(row_count, nb * 2 - 1);
@ -510,15 +592,15 @@ TEST_F(DeleteTest, delete_add_auto_flush) {
qb.float_data_.resize(TABLE_DIM);
qb.vector_count_ = 1;
qb.id_array_.clear();
stat = db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, qb, result_ids, result_distances);
stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, qb, result_ids, result_distances);
ASSERT_EQ(result_ids[0], xb2.id_array_.front());
ASSERT_LT(result_distances[0], 1e-4);
result_ids.clear();
result_distances.clear();
stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, nprobe, ids_to_delete.front(), result_ids,
result_distances);
stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, {{"nprobe", nprobe}}, ids_to_delete.front(),
result_ids, result_distances);
ASSERT_EQ(result_ids[0], -1);
ASSERT_EQ(result_distances[0], std::numeric_limits<float>::max());
}
@ -528,7 +610,7 @@ TEST_F(CompactTest, compact_basic) {
auto stat = db_->CreateTable(table_info);
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = GetTableName();
table_info_get.table_id_ = table_info.table_id_;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
@ -537,7 +619,7 @@ TEST_F(CompactTest, compact_basic) {
milvus::engine::VectorsData xb;
BuildVectors(nb, xb);
stat = db_->InsertVectors(GetTableName(), "", xb);
stat = db_->InsertVectors(table_info.table_id_, "", xb);
ASSERT_TRUE(stat.ok());
stat = db_->Flush();
@ -546,18 +628,18 @@ TEST_F(CompactTest, compact_basic) {
std::vector<milvus::engine::IDNumber> ids_to_delete;
ids_to_delete.emplace_back(xb.id_array_.front());
ids_to_delete.emplace_back(xb.id_array_.back());
stat = db_->DeleteVectors(GetTableName(), ids_to_delete);
stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete);
ASSERT_TRUE(stat.ok());
stat = db_->Flush();
ASSERT_TRUE(stat.ok());
uint64_t row_count;
stat = db_->GetTableRowCount(GetTableName(), row_count);
stat = db_->GetTableRowCount(table_info.table_id_, row_count);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(row_count, nb - 2);
stat = db_->Compact(GetTableName());
stat = db_->Compact(table_info.table_id_);
ASSERT_TRUE(stat.ok());
const int topk = 1, nprobe = 1;
@ -569,8 +651,8 @@ TEST_F(CompactTest, compact_basic) {
milvus::engine::VectorsData qb = xb;
for (auto& id : ids_to_delete) {
stat =
db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, id, result_ids, result_distances);
stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, id, result_ids,
result_distances);
ASSERT_EQ(result_ids[0], -1);
ASSERT_EQ(result_distances[0], std::numeric_limits<float>::max());
}
@ -583,7 +665,7 @@ TEST_F(CompactTest, compact_with_index) {
auto stat = db_->CreateTable(table_info);
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = GetTableName();
table_info_get.table_id_ = table_info.table_id_;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
@ -597,7 +679,7 @@ TEST_F(CompactTest, compact_with_index) {
xb.id_array_.emplace_back(i);
}
stat = db_->InsertVectors(GetTableName(), "", xb);
stat = db_->InsertVectors(table_info.table_id_, "", xb);
ASSERT_TRUE(stat.ok());
std::random_device rd;
@ -618,7 +700,7 @@ TEST_F(CompactTest, compact_with_index) {
milvus::engine::TableIndex index;
index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8;
stat = db_->CreateIndex(GetTableName(), index);
stat = db_->CreateIndex(table_info.table_id_, index);
ASSERT_TRUE(stat.ok());
stat = db_->Flush();
@ -628,25 +710,25 @@ TEST_F(CompactTest, compact_with_index) {
for (auto& kv : search_vectors) {
ids_to_delete.emplace_back(kv.first);
}
stat = db_->DeleteVectors(GetTableName(), ids_to_delete);
stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete);
stat = db_->Flush();
ASSERT_TRUE(stat.ok());
uint64_t row_count;
stat = db_->GetTableRowCount(GetTableName(), row_count);
stat = db_->GetTableRowCount(table_info.table_id_, row_count);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(row_count, nb - ids_to_delete.size());
stat = db_->Compact(GetTableName());
stat = db_->Compact(table_info.table_id_);
ASSERT_TRUE(stat.ok());
stat = db_->GetTableRowCount(GetTableName(), row_count);
stat = db_->GetTableRowCount(table_info.table_id_, row_count);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(row_count, nb - ids_to_delete.size());
milvus::engine::TableIndex table_index;
stat = db_->DescribeIndex(GetTableName(), table_index);
stat = db_->DescribeIndex(table_info.table_id_, table_index);
ASSERT_TRUE(stat.ok());
ASSERT_FLOAT_EQ(table_index.engine_type_, index.engine_type_);
@ -659,8 +741,8 @@ TEST_F(CompactTest, compact_with_index) {
std::vector<std::string> tags;
milvus::engine::ResultIds result_ids;
milvus::engine::ResultDistances result_distances;
stat =
db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, search, result_ids, result_distances);
stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, search, result_ids,
result_distances);
ASSERT_NE(result_ids[0], pair.first);
// ASSERT_LT(result_distances[0], 1e-4);
ASSERT_GT(result_distances[0], 1);

71
core/unittest/db/test_search_by_id.cpp
View File

@ -68,7 +68,7 @@ TEST_F(SearchByIdTest, basic) {
auto stat = db_->CreateTable(table_info);
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = GetTableName();
table_info_get.table_id_ = table_info.table_id_;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
@ -81,7 +81,7 @@ TEST_F(SearchByIdTest, basic) {
xb.id_array_.push_back(i);
}
stat = db_->InsertVectors(GetTableName(), "", xb);
stat = db_->InsertVectors(table_info.table_id_, "", xb);
ASSERT_TRUE(stat.ok());
std::random_device rd;
@ -108,7 +108,8 @@ TEST_F(SearchByIdTest, basic) {
milvus::engine::ResultIds result_ids;
milvus::engine::ResultDistances result_distances;
stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, i, result_ids, result_distances);
stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, i, result_ids,
result_distances);
ASSERT_EQ(result_ids[0], i);
ASSERT_LT(result_distances[0], 1e-4);
}
@ -119,7 +120,7 @@ TEST_F(SearchByIdTest, with_index) {
auto stat = db_->CreateTable(table_info);
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = GetTableName();
table_info_get.table_id_ = table_info.table_id_;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
@ -132,7 +133,7 @@ TEST_F(SearchByIdTest, with_index) {
xb.id_array_.push_back(i);
}
stat = db_->InsertVectors(GetTableName(), "", xb);
stat = db_->InsertVectors(table_info.table_id_, "", xb);
ASSERT_TRUE(stat.ok());
std::random_device rd;
@ -153,7 +154,7 @@ TEST_F(SearchByIdTest, with_index) {
milvus::engine::TableIndex index;
index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8;
index.extra_params_ = {{"nlist", 10}};
stat = db_->CreateIndex(GetTableName(), index);
stat = db_->CreateIndex(table_info.table_id_, index);
ASSERT_TRUE(stat.ok());
const int topk = 10, nprobe = 10;
@ -165,7 +166,8 @@ TEST_F(SearchByIdTest, with_index) {
milvus::engine::ResultIds result_ids;
milvus::engine::ResultDistances result_distances;
stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, i, result_ids, result_distances);
stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, i, result_ids,
result_distances);
ASSERT_EQ(result_ids[0], i);
ASSERT_LT(result_distances[0], 1e-3);
}
@ -176,7 +178,7 @@ TEST_F(SearchByIdTest, with_delete) {
auto stat = db_->CreateTable(table_info);
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = GetTableName();
table_info_get.table_id_ = table_info.table_id_;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
@ -189,7 +191,7 @@ TEST_F(SearchByIdTest, with_delete) {
xb.id_array_.push_back(i);
}
stat = db_->InsertVectors(GetTableName(), "", xb);
stat = db_->InsertVectors(table_info.table_id_, "", xb);
ASSERT_TRUE(stat.ok());
std::random_device rd;
@ -211,7 +213,7 @@ TEST_F(SearchByIdTest, with_delete) {
for (auto& id : ids_to_search) {
ids_to_delete.emplace_back(id);
}
stat = db_->DeleteVectors(GetTableName(), ids_to_delete);
stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete);
stat = db_->Flush();
ASSERT_TRUE(stat.ok());
@ -225,7 +227,8 @@ TEST_F(SearchByIdTest, with_delete) {
milvus::engine::ResultIds result_ids;
milvus::engine::ResultDistances result_distances;
stat = db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, i, result_ids, result_distances);
stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, i, result_ids,
result_distances);
ASSERT_EQ(result_ids[0], -1);
ASSERT_EQ(result_distances[0], std::numeric_limits<float>::max());
}
@ -236,7 +239,7 @@ TEST_F(GetVectorByIdTest, basic) {
auto stat = db_->CreateTable(table_info);
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = GetTableName();
table_info_get.table_id_ = table_info.table_id_;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
@ -249,7 +252,7 @@ TEST_F(GetVectorByIdTest, basic) {
xb.id_array_.push_back(i);
}
stat = db_->InsertVectors(GetTableName(), "", xb);
stat = db_->InsertVectors(table_info.table_id_, "", xb);
ASSERT_TRUE(stat.ok());
std::random_device rd;
@ -277,11 +280,11 @@ TEST_F(GetVectorByIdTest, basic) {
milvus::engine::ResultDistances result_distances;
milvus::engine::VectorsData vector;
stat = db_->GetVectorByID(GetTableName(), id, vector);
stat = db_->GetVectorByID(table_info.table_id_, id, vector);
ASSERT_TRUE(stat.ok());
stat =
db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, vector, result_ids, result_distances);
stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, vector, result_ids,
result_distances);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(result_ids[0], id);
ASSERT_LT(result_distances[0], 1e-4);
@ -293,7 +296,7 @@ TEST_F(GetVectorByIdTest, with_index) {
auto stat = db_->CreateTable(table_info);
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = GetTableName();
table_info_get.table_id_ = table_info.table_id_;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
@ -306,7 +309,7 @@ TEST_F(GetVectorByIdTest, with_index) {
xb.id_array_.push_back(i);
}
stat = db_->InsertVectors(GetTableName(), "", xb);
stat = db_->InsertVectors(table_info.table_id_, "", xb);
ASSERT_TRUE(stat.ok());
std::random_device rd;
@ -327,7 +330,7 @@ TEST_F(GetVectorByIdTest, with_index) {
milvus::engine::TableIndex index;
index.extra_params_ = {{"nlist", 10}};
index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8;
stat = db_->CreateIndex(GetTableName(), index);
stat = db_->CreateIndex(table_info.table_id_, index);
ASSERT_TRUE(stat.ok());
const int topk = 10, nprobe = 10;
@ -340,11 +343,11 @@ TEST_F(GetVectorByIdTest, with_index) {
milvus::engine::ResultDistances result_distances;
milvus::engine::VectorsData vector;
stat = db_->GetVectorByID(GetTableName(), id, vector);
stat = db_->GetVectorByID(table_info.table_id_, id, vector);
ASSERT_TRUE(stat.ok());
stat =
db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, vector, result_ids, result_distances);
stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, vector, result_ids,
result_distances);
ASSERT_EQ(result_ids[0], id);
ASSERT_LT(result_distances[0], 1e-3);
}
@ -355,7 +358,7 @@ TEST_F(GetVectorByIdTest, with_delete) {
auto stat = db_->CreateTable(table_info);
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = GetTableName();
table_info_get.table_id_ = table_info.table_id_;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
@ -368,7 +371,7 @@ TEST_F(GetVectorByIdTest, with_delete) {
xb.id_array_.push_back(i);
}
stat = db_->InsertVectors(GetTableName(), "", xb);
stat = db_->InsertVectors(table_info.table_id_, "", xb);
ASSERT_TRUE(stat.ok());
std::random_device rd;
@ -390,7 +393,7 @@ TEST_F(GetVectorByIdTest, with_delete) {
for (auto& id : ids_to_search) {
ids_to_delete.emplace_back(id);
}
stat = db_->DeleteVectors(GetTableName(), ids_to_delete);
stat = db_->DeleteVectors(table_info.table_id_, ids_to_delete);
stat = db_->Flush();
ASSERT_TRUE(stat.ok());
@ -402,7 +405,7 @@ TEST_F(GetVectorByIdTest, with_delete) {
milvus::engine::ResultDistances result_distances;
milvus::engine::VectorsData vector;
stat = db_->GetVectorByID(GetTableName(), id, vector);
stat = db_->GetVectorByID(table_info.table_id_, id, vector);
ASSERT_TRUE(stat.ok());
ASSERT_TRUE(vector.float_data_.empty());
ASSERT_EQ(vector.vector_count_, 0);
@ -419,7 +422,7 @@ TEST_F(SearchByIdTest, BINARY) {
ASSERT_TRUE(stat.ok());
milvus::engine::meta::TableSchema table_info_get;
table_info_get.table_id_ = GetTableName();
table_info_get.table_id_ = table_info.table_id_;
stat = db_->DescribeTable(table_info_get);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(table_info_get.dimension_, TABLE_DIM);
@ -445,7 +448,7 @@ TEST_F(SearchByIdTest, BINARY) {
vectors.id_array_.emplace_back(k * nb + i);
}
stat = db_->InsertVectors(GetTableName(), "", vectors);
stat = db_->InsertVectors(table_info.table_id_, "", vectors);
ASSERT_TRUE(stat.ok());
}
@ -465,7 +468,7 @@ TEST_F(SearchByIdTest, BINARY) {
ASSERT_TRUE(stat.ok());
uint64_t row_count;
stat = db_->GetTableRowCount(GetTableName(), row_count);
stat = db_->GetTableRowCount(table_info.table_id_, row_count);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(row_count, nb * insert_loop);
@ -479,12 +482,12 @@ TEST_F(SearchByIdTest, BINARY) {
milvus::engine::ResultDistances result_distances;
milvus::engine::VectorsData vector;
stat = db_->GetVectorByID(GetTableName(), id, vector);
stat = db_->GetVectorByID(table_info.table_id_, id, vector);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(vector.vector_count_, 1);
stat =
db_->Query(dummy_context_, GetTableName(), tags, topk, json_params, vector, result_ids, result_distances);
stat = db_->Query(dummy_context_, table_info.table_id_, tags, topk, json_params, vector, result_ids,
result_distances);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(result_ids[0], id);
ASSERT_LT(result_distances[0], 1e-4);
@ -493,8 +496,8 @@ TEST_F(SearchByIdTest, BINARY) {
result_ids.clear();
result_distances.clear();
stat =
db_->QueryByID(dummy_context_, GetTableName(), tags, topk, json_params, id, result_ids, result_distances);
stat = db_->QueryByID(dummy_context_, table_info.table_id_, tags, topk, json_params, id, result_ids,
result_distances);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(result_ids[0], id);
ASSERT_LT(result_distances[0], 1e-4);