From 20e07106617ba436fb7d8d9cdae9177df7e13b56 Mon Sep 17 00:00:00 2001 From: groot Date: Wed, 4 Dec 2019 02:19:52 -0600 Subject: [PATCH 1/6] Add important log (#694) * #670 Random failure of unittest db_test::SEARCH_TEST * add an important log * add an important log * add an important log --- core/src/db/engine/ExecutionEngineImpl.cpp | 2 +- core/src/scheduler/task/BuildIndexTask.cpp | 2 +- core/src/scheduler/task/SearchTask.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 4267c2e903..322509b866 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -603,7 +603,7 @@ ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, int64_t npr } if (!status.ok()) { - ENGINE_LOG_ERROR << "Search error"; + ENGINE_LOG_ERROR << "Search error:" << status.message(); } return status; } diff --git a/core/src/scheduler/task/BuildIndexTask.cpp b/core/src/scheduler/task/BuildIndexTask.cpp index b60de0a1a3..d0751c1b45 100644 --- a/core/src/scheduler/task/BuildIndexTask.cpp +++ b/core/src/scheduler/task/BuildIndexTask.cpp @@ -86,7 +86,7 @@ XBuildIndexTask::Load(milvus::scheduler::LoadType type, uint8_t device_id) { size_t file_size = to_index_engine_->PhysicalSize(); - std::string info = "Load file id:" + std::to_string(file_->id_) + " " + type_str + + std::string info = "Build index task load file id:" + std::to_string(file_->id_) + " " + type_str + " file type:" + std::to_string(file_->file_type_) + " size:" + std::to_string(file_size) + " bytes from location: " + file_->location_ + " totally cost"; double span = rc.ElapseFromBegin(info); diff --git a/core/src/scheduler/task/SearchTask.cpp b/core/src/scheduler/task/SearchTask.cpp index 34c8fc4b4b..275a21cd36 100644 --- a/core/src/scheduler/task/SearchTask.cpp +++ b/core/src/scheduler/task/SearchTask.cpp @@ -160,7 +160,7 @@ XSearchTask::Load(LoadType type, uint8_t device_id) { size_t file_size = index_engine_->PhysicalSize(); - std::string info = "Load file id:" + std::to_string(file_->id_) + + std::string info = "Search task load file id:" + std::to_string(file_->id_) + " " + type_str + " file type:" + std::to_string(file_->file_type_) + " size:" + std::to_string(file_size) + " bytes from location: " + file_->location_ + " totally cost"; double span = rc.ElapseFromBegin(info); From d82c7da6295bbda674de1c83f88ce3352a93235c Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Wed, 4 Dec 2019 16:27:44 +0800 Subject: [PATCH 2/6] [skip ci] update install.md (#688) * [skip ci] update install.md * Update install.md --- install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.md b/install.md index e95153a3a1..4051e4deb4 100644 --- a/install.md +++ b/install.md @@ -103,7 +103,7 @@ $ docker run -it -p 19530:19530 -d milvusdb/milvus-cpu-build-env:v0.6.0-ubuntu18 Start a GPU container: ```shell -$ docker run — runtime=nvidia -it -p 19530:19530 -d milvusdb/milvus-gpu-build-env:v0.6.0-ubuntu18.04 +$ docker run --runtime=nvidia -it -p 19530:19530 -d milvusdb/milvus-gpu-build-env:v0.6.0-ubuntu18.04 ``` To enter the container: From 8574b8b93f474d4e74433be8b7485b9eb89c7b4e Mon Sep 17 00:00:00 2001 From: Yukikaze-CZR <48198922+Yukikaze-CZR@users.noreply.github.com> Date: Wed, 4 Dec 2019 19:37:13 +0800 Subject: [PATCH 3/6] add code coverage rate and fix #670 (#693) --- .../index/vector_index/IndexIVFSQHybrid.cpp | 56 ------------------- .../index/vector_index/IndexIVFSQHybrid.h | 2 +- .../index/vector_index/helpers/Cloner.cpp | 2 + core/src/index/unittest/Helper.h | 2 + core/src/wrapper/gpu/GPUVecImpl.cpp | 2 + core/unittest/db/test_db.cpp | 21 ++++++- 6 files changed, 27 insertions(+), 58 deletions(-) diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp index f2a963eef2..4fecfb9dc5 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp @@ -295,61 +295,5 @@ FaissIVFQuantizer::~FaissIVFQuantizer() { // else do nothing } -#else - -QuantizerPtr -IVFSQHybrid::LoadQuantizer(const Config& conf) { - return knowhere::QuantizerPtr(); -} - -void -IVFSQHybrid::SetQuantizer(const QuantizerPtr& q) { -} - -void -IVFSQHybrid::UnsetQuantizer() { -} - -VectorIndexPtr -IVFSQHybrid::LoadData(const knowhere::QuantizerPtr& q, const Config& conf) { - return nullptr; -} - -std::pair -IVFSQHybrid::CopyCpuToGpuWithQuantizer(const int64_t& device_id, const Config& config) { - KNOWHERE_THROW_MSG("Not yet implemented"); -} - -IndexModelPtr -IVFSQHybrid::Train(const DatasetPtr& dataset, const Config& config) { - return GPUIVFSQ::Train(dataset, config); -} - -VectorIndexPtr -IVFSQHybrid::CopyGpuToCpu(const Config& config) { - return GPUIVFSQ::CopyGpuToCpu(config); -} - -VectorIndexPtr -IVFSQHybrid::CopyCpuToGpu(const int64_t& device_id, const Config& config) { - return IVF::CopyCpuToGpu(device_id, config); -} - -void -IVFSQHybrid::search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels, - const Config& cfg) { - GPUIVF::search_impl(n, data, k, distances, labels, cfg); -} - -void -IVFSQHybrid::LoadImpl(const BinarySet& index_binary) { - GPUIVF::LoadImpl(index_binary); -} - -void -IVFSQHybrid::set_index_model(IndexModelPtr model) { - GPUIVF::set_index_model(model); -} - #endif } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h index d2a3be6c39..5846d3e5ce 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h @@ -36,7 +36,6 @@ struct FaissIVFQuantizer : public Quantizer { ~FaissIVFQuantizer() override; }; using FaissIVFQuantizerPtr = std::shared_ptr; -#endif class IVFSQHybrid : public GPUIVFSQ { public: @@ -93,5 +92,6 @@ class IVFSQHybrid : public GPUIVFSQ { int64_t gpu_mode = 0; // 0,1,2 int64_t quantizer_gpu_id_ = -1; }; +#endif } // namespace knowhere diff --git a/core/src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp b/core/src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp index 363e629eca..4c5a7ec917 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp @@ -38,9 +38,11 @@ CopyGpuToCpu(const VectorIndexPtr& index, const Config& config) { VectorIndexPtr CopyCpuToGpu(const VectorIndexPtr& index, const int64_t& device_id, const Config& config) { +#ifdef CUSTOMIZATION if (auto device_index = std::dynamic_pointer_cast(index)) { return device_index->CopyCpuToGpu(device_id, config); } +#endif if (auto device_index = std::dynamic_pointer_cast(index)) { return device_index->CopyGpuToGpu(device_id, config); diff --git a/core/src/index/unittest/Helper.h b/core/src/index/unittest/Helper.h index 5d7df41b69..9095cd6f6a 100644 --- a/core/src/index/unittest/Helper.h +++ b/core/src/index/unittest/Helper.h @@ -53,8 +53,10 @@ IndexFactory(const std::string& type) { return std::make_shared(DEVICEID); } else if (type == "GPUIVFSQ") { return std::make_shared(DEVICEID); +#ifdef CUSTOMIZATION } else if (type == "IVFSQHybrid") { return std::make_shared(DEVICEID); +#endif #endif } } diff --git a/core/src/wrapper/gpu/GPUVecImpl.cpp b/core/src/wrapper/gpu/GPUVecImpl.cpp index e40f5038ea..8fca6949b2 100644 --- a/core/src/wrapper/gpu/GPUVecImpl.cpp +++ b/core/src/wrapper/gpu/GPUVecImpl.cpp @@ -73,6 +73,7 @@ IVFMixIndex::Load(const knowhere::BinarySet& index_binary) { return Status::OK(); } +#ifdef CUSTOMIZATION knowhere::QuantizerPtr IVFHybridIndex::LoadQuantizer(const Config& conf) { // TODO(linxj): Hardcode here @@ -158,6 +159,7 @@ IVFHybridIndex::CopyToGpuWithQuantizer(const int64_t& device_id, const Config& c } return std::make_pair(nullptr, nullptr); } +#endif } // namespace engine } // namespace milvus diff --git a/core/unittest/db/test_db.cpp b/core/unittest/db/test_db.cpp index dbed92b355..8db7d3f9bd 100644 --- a/core/unittest/db/test_db.cpp +++ b/core/unittest/db/test_db.cpp @@ -351,6 +351,19 @@ TEST_F(DBTest, SEARCH_TEST) { ASSERT_TRUE(stat.ok()); } + index.engine_type_ = (int)milvus::engine::EngineType::FAISS_PQ; + db_->CreateIndex(TABLE_NAME, index); // wait until build index finish + + { + std::vector tags; + milvus::engine::ResultIds result_ids; + milvus::engine::ResultDistances result_distances; + stat = db_->Query(TABLE_NAME, tags, k, nq, 10, xq.data(), result_ids, result_distances); + ASSERT_TRUE(stat.ok()); + stat = db_->Query(TABLE_NAME, tags, k, 1100, 10, xq.data(), result_ids, result_distances); + ASSERT_TRUE(stat.ok()); + } + #ifdef CUSTOMIZATION // test FAISS_IVFSQ8H optimizer index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8H; @@ -375,7 +388,13 @@ TEST_F(DBTest, SEARCH_TEST) { { // search by specify index file milvus::engine::meta::DatesT dates; - std::vector file_ids = {"1", "2", "3", "4", "5", "6"}; + std::vector file_ids; + // sometimes this case run fast to merge file and build index, old file will be deleted immediately, + // so the QueryByFileID cannot get files to search + // input 100 files ids to avoid random failure of this case + for (int i = 0; i < 100; i++) { + file_ids.push_back(std::to_string(i)); + } result_ids.clear(); result_dists.clear(); stat = db_->QueryByFileID(TABLE_NAME, file_ids, k, nq, 10, xq.data(), dates, result_ids, result_dists); From 5970450d48f9a695dbea75e2763644ddfbc07d50 Mon Sep 17 00:00:00 2001 From: Cai Yudong Date: Thu, 5 Dec 2019 11:42:09 +0800 Subject: [PATCH 4/6] #705 fix search SQ8H crash without GPU resource (#706) --- CHANGELOG.md | 1 + core/src/db/engine/ExecutionEngineImpl.cpp | 1 + core/src/wrapper/VecIndex.cpp | 24 +++++++++++++++------- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de4226c21a..792c7f8f35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#658 - Milvus error out when building SQ8H index without GPU resources - \#668 - Update badge of README - \#670 - Random failure of unittest db_test::SEARCH_TEST +- \#705 - Fix search SQ8H crash without GPU resource ## Feature - \#12 - Pure CPU version for Milvus diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 322509b866..e10497f058 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -634,6 +634,7 @@ ExecutionEngineImpl::Init() { Status s = config.GetGpuResourceConfigBuildIndexResources(gpu_ids); if (!s.ok()) { gpu_num_ = knowhere::INVALID_VALUE; + return s; } for (auto id : gpu_ids) { if (gpu_num_ == id) { diff --git a/core/src/wrapper/VecIndex.cpp b/core/src/wrapper/VecIndex.cpp index 75f75fb983..824e5be13d 100644 --- a/core/src/wrapper/VecIndex.cpp +++ b/core/src/wrapper/VecIndex.cpp @@ -24,6 +24,8 @@ #include "knowhere/index/vector_index/IndexIVFSQ.h" #include "knowhere/index/vector_index/IndexNSG.h" #include "knowhere/index/vector_index/IndexSPTAG.h" +#include "server/Config.h" +#include "utils/Exception.h" #include "utils/Log.h" #ifdef MILVUS_GPU_VERSION @@ -145,6 +147,10 @@ GetVecIndexFactory(const IndexType& type, const Config& cfg) { index = std::make_shared(gpu_device); break; } + case IndexType::FAISS_IVFFLAT_MIX: { + index = std::make_shared(gpu_device); + return std::make_shared(index, IndexType::FAISS_IVFFLAT_MIX); + } case IndexType::FAISS_IVFPQ_GPU: { index = std::make_shared(gpu_device); break; @@ -161,16 +167,20 @@ GetVecIndexFactory(const IndexType& type, const Config& cfg) { index = std::make_shared(gpu_device); break; } - case IndexType::FAISS_IVFFLAT_MIX: { - index = std::make_shared(gpu_device); - return std::make_shared(index, IndexType::FAISS_IVFFLAT_MIX); - } + +#endif #ifdef CUSTOMIZATION case IndexType::FAISS_IVFSQ8_HYBRID: { - index = std::make_shared(gpu_device); - return std::make_shared(index, IndexType::FAISS_IVFSQ8_HYBRID); + server::Config& config = server::Config::GetInstance(); + bool gpu_resource_enable = true; + config.GetGpuResourceConfigEnable(gpu_resource_enable); + if (gpu_resource_enable) { + index = std::make_shared(gpu_device); + return std::make_shared(index, IndexType::FAISS_IVFSQ8_HYBRID); + } else { + throw Exception(DB_ERROR, "No GPU resources for IndexType::FAISS_IVFSQ8_HYBRID"); + } } -#endif #endif case IndexType::NSG_MIX: { // TODO(linxj): bug. index = std::make_shared(gpu_device); From 564095fd8df3bfdd2391ded00335c9d456b21adc Mon Sep 17 00:00:00 2001 From: groot Date: Thu, 5 Dec 2019 02:50:54 -0600 Subject: [PATCH 5/6] #674 server down in stability test (#708) * #670 Random failure of unittest db_test::SEARCH_TEST * add an important log * add an important log * add an important log * #674 server down in stability test * add more log * remove log --- CHANGELOG.md | 1 + core/src/db/engine/ExecutionEngineImpl.cpp | 3 ++- core/src/scheduler/task/BuildIndexTask.cpp | 3 ++- core/src/scheduler/task/SearchTask.cpp | 2 +- core/src/wrapper/VecImpl.h | 2 +- 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 792c7f8f35..bc9322e7c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#658 - Milvus error out when building SQ8H index without GPU resources - \#668 - Update badge of README - \#670 - Random failure of unittest db_test::SEARCH_TEST +- \#674 - Server down in stability test - \#705 - Fix search SQ8H crash without GPU resource ## Feature diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index e10497f058..838346695d 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -400,10 +400,11 @@ ExecutionEngineImpl::CopyToGpu(uint64_t device_id, bool hybrid) { Status ExecutionEngineImpl::CopyToIndexFileToGpu(uint64_t device_id) { #ifdef MILVUS_GPU_VERSION + // the ToIndexData is only a placeholder, cpu-copy-to-gpu action is performed in gpu_num_ = device_id; auto to_index_data = std::make_shared(PhysicalSize()); cache::DataObjPtr obj = std::static_pointer_cast(to_index_data); - milvus::cache::GpuCacheMgr::GetInstance(device_id)->InsertItem(location_, obj); + milvus::cache::GpuCacheMgr::GetInstance(device_id)->InsertItem(location_ + "_placeholder", obj); #endif return Status::OK(); } diff --git a/core/src/scheduler/task/BuildIndexTask.cpp b/core/src/scheduler/task/BuildIndexTask.cpp index d0751c1b45..2621c202ff 100644 --- a/core/src/scheduler/task/BuildIndexTask.cpp +++ b/core/src/scheduler/task/BuildIndexTask.cpp @@ -55,7 +55,7 @@ XBuildIndexTask::Load(milvus::scheduler::LoadType type, uint8_t device_id) { type_str = "DISK2CPU"; } else if (type == LoadType::CPU2GPU) { stat = to_index_engine_->CopyToIndexFileToGpu(device_id); - type_str = "CPU2GPU"; + type_str = "CPU2GPU:" + std::to_string(device_id); } else { error_msg = "Wrong load type"; stat = Status(SERVER_UNEXPECTED_ERROR, error_msg); @@ -128,6 +128,7 @@ XBuildIndexTask::Execute() { // step 3: build index try { + ENGINE_LOG_DEBUG << "Begin build index for file:" + table_file.location_; index = to_index_engine_->BuildIndex(table_file.location_, (EngineType)table_file.engine_type_); if (index == nullptr) { throw Exception(DB_ERROR, "index NULL"); diff --git a/core/src/scheduler/task/SearchTask.cpp b/core/src/scheduler/task/SearchTask.cpp index 275a21cd36..a23af87d77 100644 --- a/core/src/scheduler/task/SearchTask.cpp +++ b/core/src/scheduler/task/SearchTask.cpp @@ -125,7 +125,7 @@ XSearchTask::Load(LoadType type, uint8_t device_id) { hybrid = true; } stat = index_engine_->CopyToGpu(device_id, hybrid); - type_str = "CPU2GPU"; + type_str = "CPU2GPU:" + std::to_string(device_id); } else if (type == LoadType::GPU2CPU) { stat = index_engine_->CopyToCpu(); type_str = "GPU2CPU"; diff --git a/core/src/wrapper/VecImpl.h b/core/src/wrapper/VecImpl.h index e8dee45d42..a6d81cba9f 100644 --- a/core/src/wrapper/VecImpl.h +++ b/core/src/wrapper/VecImpl.h @@ -108,7 +108,7 @@ class ToIndexData : public cache::DataObj { } private: - int64_t size_; + int64_t size_ = 0; }; } // namespace engine From ebb4cedeb794932e0bc7e8506a0b4553500e3d79 Mon Sep 17 00:00:00 2001 From: groot Date: Thu, 5 Dec 2019 03:32:42 -0600 Subject: [PATCH 6/6] #696 metric_type changed from IP to L2 (#710) * #670 Random failure of unittest db_test::SEARCH_TEST * add an important log * add an important log * add an important log * #674 server down in stability test * add more log * remove log * #696 metric_type changed from IP to L2 --- CHANGELOG.md | 1 + core/src/db/meta/MySQLMetaImpl.cpp | 2 -- core/src/db/meta/SqliteMetaImpl.cpp | 4 +--- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc9322e7c0..850a271008 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#668 - Update badge of README - \#670 - Random failure of unittest db_test::SEARCH_TEST - \#674 - Server down in stability test +- \#696 - Metric_type changed from IP to L2 - \#705 - Fix search SQ8H crash without GPU resource ## Feature diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index 5072df9cad..51a642aa11 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -1138,8 +1138,6 @@ MySQLMetaImpl::DropTableIndex(const std::string& table_id) { // set table index type to raw dropTableIndexQuery << "UPDATE " << META_TABLES << " SET engine_type = " << std::to_string(DEFAULT_ENGINE_TYPE) - << " ,nlist = " << std::to_string(DEFAULT_NLIST) - << " ,metric_type = " << std::to_string(DEFAULT_METRIC_TYPE) << " WHERE table_id = " << mysqlpp::quote << table_id << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropTableIndex: " << dropTableIndexQuery.str(); diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index 6a525d27d2..edd8187ba9 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -733,9 +733,7 @@ SqliteMetaImpl::DropTableIndex(const std::string& table_id) { //set table index type to raw ConnectorPtr->update_all( set( - c(&TableSchema::engine_type_) = DEFAULT_ENGINE_TYPE, - c(&TableSchema::nlist_) = DEFAULT_NLIST, - c(&TableSchema::metric_type_) = DEFAULT_METRIC_TYPE), + c(&TableSchema::engine_type_) = DEFAULT_ENGINE_TYPE), where( c(&TableSchema::table_id_) == table_id));