test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-05 05:18:52 +08:00
Code Issues Actions 14 Packages Projects Releases Wiki Activity

Merge branch 'branch-0.4.0' into 'branch-0.4.0'

Browse Source 
MS-443 Create index hang again

See merge request megasearch/milvus!452

Former-commit-id: da8b457ad8225365711a84642eec059866a930e5
This commit is contained in:
peng.xu 2019-08-30 11:22:59 +08:00
commit 4a8e87e5dd
3 changed files with 42 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cpp/CHANGELOG.md
View File

@ -15,6 +15,7 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-431 - Search vectors params nprobe: 0/-1, expected result: raise exception
- MS-331 - Crate Table : when table exists, error code is META_FAILED(code=15) rather than ILLEGAL TABLE NAME(code=9))
- MS-430 - Search no result if index created with FLAT
- MS-443 - Create index hang again
## Improvement
- MS-327 - Clean code for milvus

3
cpp/src/db/DBImpl.cpp
View File

@ -581,6 +581,9 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index)
while (!file_ids.empty()) {
ENGINE_LOG_DEBUG << "Non index files detected! Will build index " << times;
if(index.engine_type_ != (int)EngineType::FAISS_IDMAP) {
status = meta_ptr_->UpdateTableFilesToIndex(table_id);
}
std::this_thread::sleep_for(std::chrono::milliseconds(std::min(10*1000, times*100)));
status = meta_ptr_->FilesByType(table_id, file_types, file_ids);

42
cpp/src/db/engine/ExecutionEngineImpl.cpp
View File

@ -93,7 +93,7 @@ Status ExecutionEngineImpl::AddWithIds(long n, const float *xdata, const long *x
size_t ExecutionEngineImpl::Count() const {
if(index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl::index is null";
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, return count 0";
return 0;
}
return index_->Count();
@ -105,7 +105,7 @@ size_t ExecutionEngineImpl::Size() const {
size_t ExecutionEngineImpl::Dimension() const {
if(index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl::index is null";
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, return dimension " << dim_;
return dim_;
}
return index_->Dimension();
@ -126,12 +126,16 @@ Status ExecutionEngineImpl::Serialize() {
Status ExecutionEngineImpl::Load(bool to_cache) {
index_ = zilliz::milvus::cache::CpuCacheMgr::GetInstance()->GetIndex(location_);
bool already_in_cache = (index_ != nullptr);
if (!index_) {
if (!already_in_cache) {
try {
double physical_size = PhysicalSize();
server::CollectExecutionEngineMetrics metrics(physical_size);
index_ = read_index(location_);
ENGINE_LOG_DEBUG << "Disk io from: " << location_;
if(index_ == nullptr) {
ENGINE_LOG_ERROR << "Failed to load index from " << location_;
} else {
ENGINE_LOG_DEBUG << "Disk io from: " << location_;
}
} catch (knowhere::KnowhereException &e) {
ENGINE_LOG_ERROR << e.what();
return Status::Error(e.what());
@ -152,6 +156,11 @@ Status ExecutionEngineImpl::CopyToGpu(uint64_t device_id) {
if (already_in_cache) {
index_ = index;
} else {
if(index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to copy to gpu";
return Status::Error("index is null");
}
try {
index_ = index_->CopyToGpu(device_id);
ENGINE_LOG_DEBUG << "CPU to GPU" << device_id;
@ -176,6 +185,11 @@ Status ExecutionEngineImpl::CopyToCpu() {
if (already_in_cache) {
index_ = index;
} else {
if(index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to copy to cpu";
return Status::Error("index is null");
}
try {
index_ = index_->CopyToCpu();
ENGINE_LOG_DEBUG << "GPU to CPU";
@ -194,6 +208,11 @@ Status ExecutionEngineImpl::CopyToCpu() {
}
ExecutionEnginePtr ExecutionEngineImpl::Clone() {
if(index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to clone";
return nullptr;
}
auto ret = std::make_shared<ExecutionEngineImpl>(dim_, location_, index_type_, metric_type_, nlist_);
ret->Init();
ret->index_ = index_->Clone();
@ -220,6 +239,11 @@ Status ExecutionEngineImpl::Merge(const std::string &location) {
}
}
if(index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to merge";
return Status::Error("index is null");
}
if (auto file_index = std::dynamic_pointer_cast<BFIndex>(to_merge)) {
auto ec = index_->Add(file_index->Count(), file_index->GetRawVectors(), file_index->GetRawIds());
if (ec != server::KNOWHERE_SUCCESS) {
@ -237,6 +261,11 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t
ENGINE_LOG_DEBUG << "Build index file: " << location << " from: " << location_;
auto from_index = std::dynamic_pointer_cast<BFIndex>(index_);
if(from_index == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: from_index is null, failed to build index";
return nullptr;
}
auto to_index = CreatetVecIndex(engine_type);
if (!to_index) {
throw Exception("Create Empty VecIndex");
@ -264,6 +293,11 @@ Status ExecutionEngineImpl::Search(long n,
long nprobe,
float *distances,
long *labels) const {
if(index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to search";
return Status::Error("index is null");
}
ENGINE_LOG_DEBUG << "Search Params: [k] " << k << " [nprobe] " << nprobe;
auto ec = index_->Search(n, data, distances, labels, Config::object{{"k", k}, {"nprobe", nprobe}});
if (ec != server::KNOWHERE_SUCCESS) {