test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-05 05:18:52 +08:00
Code Issues Actions 14 Packages Projects Releases Wiki Activity

feat(db): cache for mem merge

Browse Source 
Former-commit-id: bff342710683954dcd4b72b890fff483708ceba0
This commit is contained in:
Xu Peng 2019-04-24 16:04:14 +08:00
parent 22281e8ea5
commit 4e59f966c3
1 changed files with 5 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cpp/src/db/DBImpl.cpp
View File

@ -214,7 +214,11 @@ Status DBImpl::merge_files(const std::string& group_id, const meta::DateT& date,
meta::GroupFilesSchema updated; meta::GroupFilesSchema updated;
for (auto& file : files) { for (auto& file : files) {
auto file_index = dynamic_cast<faiss::IndexIDMap*>(faiss::read_index(file.location.c_str())); auto to_merge = zilliz::vecwise::cache::CpuCacheMgr::GetInstance()->GetIndex(file.location);
if (!to_merge) {
to_merge = read_index(file.location.c_str());
}
auto file_index = dynamic_cast<faiss::IndexIDMap*>(to_merge->data().get());
index->add_with_ids(file_index->ntotal, dynamic_cast<faiss::IndexFlat*>(file_index->index)->xb.data(), index->add_with_ids(file_index->ntotal, dynamic_cast<faiss::IndexFlat*>(file_index->index)->xb.data(),
file_index->id_map.data()); file_index->id_map.data());
auto file_schema = file; auto file_schema = file;
@ -222,7 +226,6 @@ Status DBImpl::merge_files(const std::string& group_id, const meta::DateT& date,
updated.push_back(file_schema); updated.push_back(file_schema);
LOG(DEBUG) << "About to merge file " << file_schema.file_id << LOG(DEBUG) << "About to merge file " << file_schema.file_id <<
" of size=" << file_schema.rows; " of size=" << file_schema.rows;
delete file_index;
} }
auto index_size = group_file.dimension * index->ntotal; auto index_size = group_file.dimension * index->ntotal;