test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-11-30 19:08:30 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

Merge pull request #711 from milvus-io/0.6.0

Browse Source 
Merge from 0.6.0
This commit is contained in:
Jin Hai 2019-12-05 17:34:32 +08:00 committed by GitHub
commit fae3220cd9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 59 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@ -52,6 +52,9 @@ Please mark all change in change log and use the ticket from JIRA.
- \#658 - Milvus error out when building SQ8H index without GPU resources
- \#668 - Update badge of README
- \#670 - Random failure of unittest db_test::SEARCH_TEST
- \#674 - Server down in stability test
- \#696 - Metric_type changed from IP to L2
- \#705 - Fix search SQ8H crash without GPU resource
## Feature
- \#12 - Pure CPU version for Milvus

6
core/src/db/engine/ExecutionEngineImpl.cpp
View File

@ -400,10 +400,11 @@ ExecutionEngineImpl::CopyToGpu(uint64_t device_id, bool hybrid) {
Status
ExecutionEngineImpl::CopyToIndexFileToGpu(uint64_t device_id) {
#ifdef MILVUS_GPU_VERSION
// the ToIndexData is only a placeholder, cpu-copy-to-gpu action is performed in
gpu_num_ = device_id;
auto to_index_data = std::make_shared<ToIndexData>(PhysicalSize());
cache::DataObjPtr obj = std::static_pointer_cast<cache::DataObj>(to_index_data);
milvus::cache::GpuCacheMgr::GetInstance(device_id)->InsertItem(location_, obj);
milvus::cache::GpuCacheMgr::GetInstance(device_id)->InsertItem(location_ + "_placeholder", obj);
#endif
return Status::OK();
}
@ -603,7 +604,7 @@ ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, int64_t npr
}
if (!status.ok()) {
ENGINE_LOG_ERROR << "Search error";
ENGINE_LOG_ERROR << "Search error:" << status.message();
}
return status;
}
@ -634,6 +635,7 @@ ExecutionEngineImpl::Init() {
Status s = config.GetGpuResourceConfigBuildIndexResources(gpu_ids);
if (!s.ok()) {
gpu_num_ = knowhere::INVALID_VALUE;
return s;
}
for (auto id : gpu_ids) {
if (gpu_num_ == id) {

2
core/src/db/meta/MySQLMetaImpl.cpp
View File

@ -1138,8 +1138,6 @@ MySQLMetaImpl::DropTableIndex(const std::string& table_id) {
// set table index type to raw
dropTableIndexQuery << "UPDATE " << META_TABLES
<< " SET engine_type = " << std::to_string(DEFAULT_ENGINE_TYPE)
<< " ,nlist = " << std::to_string(DEFAULT_NLIST)
<< " ,metric_type = " << std::to_string(DEFAULT_METRIC_TYPE)
<< " WHERE table_id = " << mysqlpp::quote << table_id << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropTableIndex: " << dropTableIndexQuery.str();

4
core/src/db/meta/SqliteMetaImpl.cpp
View File

@ -733,9 +733,7 @@ SqliteMetaImpl::DropTableIndex(const std::string& table_id) {
//set table index type to raw
ConnectorPtr->update_all(
set(
c(&TableSchema::engine_type_) = DEFAULT_ENGINE_TYPE,
c(&TableSchema::nlist_) = DEFAULT_NLIST,
c(&TableSchema::metric_type_) = DEFAULT_METRIC_TYPE),
c(&TableSchema::engine_type_) = DEFAULT_ENGINE_TYPE),
where(
c(&TableSchema::table_id_) == table_id));

56
core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.cpp
View File

@ -295,61 +295,5 @@ FaissIVFQuantizer::~FaissIVFQuantizer() {
// else do nothing
}
#else
QuantizerPtr
IVFSQHybrid::LoadQuantizer(const Config& conf) {
return knowhere::QuantizerPtr();
}
void
IVFSQHybrid::SetQuantizer(const QuantizerPtr& q) {
}
void
IVFSQHybrid::UnsetQuantizer() {
}
VectorIndexPtr
IVFSQHybrid::LoadData(const knowhere::QuantizerPtr& q, const Config& conf) {
return nullptr;
}
std::pair<VectorIndexPtr, QuantizerPtr>
IVFSQHybrid::CopyCpuToGpuWithQuantizer(const int64_t& device_id, const Config& config) {
KNOWHERE_THROW_MSG("Not yet implemented");
}
IndexModelPtr
IVFSQHybrid::Train(const DatasetPtr& dataset, const Config& config) {
return GPUIVFSQ::Train(dataset, config);
}
VectorIndexPtr
IVFSQHybrid::CopyGpuToCpu(const Config& config) {
return GPUIVFSQ::CopyGpuToCpu(config);
}
VectorIndexPtr
IVFSQHybrid::CopyCpuToGpu(const int64_t& device_id, const Config& config) {
return IVF::CopyCpuToGpu(device_id, config);
}
void
IVFSQHybrid::search_impl(int64_t n, const float* data, int64_t k, float* distances, int64_t* labels,
const Config& cfg) {
GPUIVF::search_impl(n, data, k, distances, labels, cfg);
}
void
IVFSQHybrid::LoadImpl(const BinarySet& index_binary) {
GPUIVF::LoadImpl(index_binary);
}
void
IVFSQHybrid::set_index_model(IndexModelPtr model) {
GPUIVF::set_index_model(model);
}
#endif
} // namespace knowhere

2
core/src/index/knowhere/knowhere/index/vector_index/IndexIVFSQHybrid.h
View File

@ -36,7 +36,6 @@ struct FaissIVFQuantizer : public Quantizer {
~FaissIVFQuantizer() override;
};
using FaissIVFQuantizerPtr = std::shared_ptr<FaissIVFQuantizer>;
#endif
class IVFSQHybrid : public GPUIVFSQ {
public:
@ -93,5 +92,6 @@ class IVFSQHybrid : public GPUIVFSQ {
int64_t gpu_mode = 0; // 0,1,2
int64_t quantizer_gpu_id_ = -1;
};
#endif
} // namespace knowhere

2
core/src/index/knowhere/knowhere/index/vector_index/helpers/Cloner.cpp
View File

@ -38,9 +38,11 @@ CopyGpuToCpu(const VectorIndexPtr& index, const Config& config) {
VectorIndexPtr
CopyCpuToGpu(const VectorIndexPtr& index, const int64_t& device_id, const Config& config) {
#ifdef CUSTOMIZATION
if (auto device_index = std::dynamic_pointer_cast<IVFSQHybrid>(index)) {
return device_index->CopyCpuToGpu(device_id, config);
}
#endif
if (auto device_index = std::dynamic_pointer_cast<GPUIndex>(index)) {
return device_index->CopyGpuToGpu(device_id, config);

2
core/src/index/unittest/Helper.h
View File

@ -53,8 +53,10 @@ IndexFactory(const std::string& type) {
return std::make_shared<knowhere::GPUIVFPQ>(DEVICEID);
} else if (type == "GPUIVFSQ") {
return std::make_shared<knowhere::GPUIVFSQ>(DEVICEID);
#ifdef CUSTOMIZATION
} else if (type == "IVFSQHybrid") {
return std::make_shared<knowhere::IVFSQHybrid>(DEVICEID);
#endif
#endif
}
}

5
core/src/scheduler/task/BuildIndexTask.cpp
View File

@ -55,7 +55,7 @@ XBuildIndexTask::Load(milvus::scheduler::LoadType type, uint8_t device_id) {
type_str = "DISK2CPU";
} else if (type == LoadType::CPU2GPU) {
stat = to_index_engine_->CopyToIndexFileToGpu(device_id);
type_str = "CPU2GPU";
type_str = "CPU2GPU:" + std::to_string(device_id);
} else {
error_msg = "Wrong load type";
stat = Status(SERVER_UNEXPECTED_ERROR, error_msg);
@ -86,7 +86,7 @@ XBuildIndexTask::Load(milvus::scheduler::LoadType type, uint8_t device_id) {
size_t file_size = to_index_engine_->PhysicalSize();
std::string info = "Load file id:" + std::to_string(file_->id_) + " " + type_str +
std::string info = "Build index task load file id:" + std::to_string(file_->id_) + " " + type_str +
" file type:" + std::to_string(file_->file_type_) + " size:" + std::to_string(file_size) +
" bytes from location: " + file_->location_ + " totally cost";
double span = rc.ElapseFromBegin(info);
@ -128,6 +128,7 @@ XBuildIndexTask::Execute() {
// step 3: build index
try {
ENGINE_LOG_DEBUG << "Begin build index for file:" + table_file.location_;
index = to_index_engine_->BuildIndex(table_file.location_, (EngineType)table_file.engine_type_);
if (index == nullptr) {
throw Exception(DB_ERROR, "index NULL");

4
core/src/scheduler/task/SearchTask.cpp
View File

@ -125,7 +125,7 @@ XSearchTask::Load(LoadType type, uint8_t device_id) {
hybrid = true;
}
stat = index_engine_->CopyToGpu(device_id, hybrid);
type_str = "CPU2GPU";
type_str = "CPU2GPU:" + std::to_string(device_id);
} else if (type == LoadType::GPU2CPU) {
stat = index_engine_->CopyToCpu();
type_str = "GPU2CPU";
@ -160,7 +160,7 @@ XSearchTask::Load(LoadType type, uint8_t device_id) {
size_t file_size = index_engine_->PhysicalSize();
std::string info = "Load file id:" + std::to_string(file_->id_) +
std::string info = "Search task load file id:" + std::to_string(file_->id_) + " " + type_str +
" file type:" + std::to_string(file_->file_type_) + " size:" + std::to_string(file_size) +
" bytes from location: " + file_->location_ + " totally cost";
double span = rc.ElapseFromBegin(info);

2
core/src/wrapper/VecImpl.h
View File

@ -108,7 +108,7 @@ class ToIndexData : public cache::DataObj {
}
private:
int64_t size_;
int64_t size_ = 0;
};
} // namespace engine

20
core/src/wrapper/VecIndex.cpp
View File

@ -24,6 +24,8 @@
#include "knowhere/index/vector_index/IndexIVFSQ.h"
#include "knowhere/index/vector_index/IndexNSG.h"
#include "knowhere/index/vector_index/IndexSPTAG.h"
#include "server/Config.h"
#include "utils/Exception.h"
#include "utils/Log.h"
#ifdef MILVUS_GPU_VERSION
@ -145,6 +147,10 @@ GetVecIndexFactory(const IndexType& type, const Config& cfg) {
index = std::make_shared<knowhere::GPUIVF>(gpu_device);
break;
}
case IndexType::FAISS_IVFFLAT_MIX: {
index = std::make_shared<knowhere::GPUIVF>(gpu_device);
return std::make_shared<IVFMixIndex>(index, IndexType::FAISS_IVFFLAT_MIX);
}
case IndexType::FAISS_IVFPQ_GPU: {
index = std::make_shared<knowhere::GPUIVFPQ>(gpu_device);
break;
@ -161,16 +167,20 @@ GetVecIndexFactory(const IndexType& type, const Config& cfg) {
index = std::make_shared<knowhere::GPUIVFSQ>(gpu_device);
break;
}
case IndexType::FAISS_IVFFLAT_MIX: {
index = std::make_shared<knowhere::GPUIVF>(gpu_device);
return std::make_shared<IVFMixIndex>(index, IndexType::FAISS_IVFFLAT_MIX);
}
#endif
#ifdef CUSTOMIZATION
case IndexType::FAISS_IVFSQ8_HYBRID: {
server::Config& config = server::Config::GetInstance();
bool gpu_resource_enable = true;
config.GetGpuResourceConfigEnable(gpu_resource_enable);
if (gpu_resource_enable) {
index = std::make_shared<knowhere::IVFSQHybrid>(gpu_device);
return std::make_shared<IVFHybridIndex>(index, IndexType::FAISS_IVFSQ8_HYBRID);
} else {
throw Exception(DB_ERROR, "No GPU resources for IndexType::FAISS_IVFSQ8_HYBRID");
}
}
#endif
#endif
case IndexType::NSG_MIX: { // TODO(linxj): bug.
index = std::make_shared<knowhere::NSG>(gpu_device);

2
core/src/wrapper/gpu/GPUVecImpl.cpp
View File

@ -73,6 +73,7 @@ IVFMixIndex::Load(const knowhere::BinarySet& index_binary) {
return Status::OK();
}
#ifdef CUSTOMIZATION
knowhere::QuantizerPtr
IVFHybridIndex::LoadQuantizer(const Config& conf) {
// TODO(linxj): Hardcode here
@ -158,6 +159,7 @@ IVFHybridIndex::CopyToGpuWithQuantizer(const int64_t& device_id, const Config& c
}
return std::make_pair(nullptr, nullptr);
}
#endif
} // namespace engine
} // namespace milvus

21
core/unittest/db/test_db.cpp
View File

@ -351,6 +351,19 @@ TEST_F(DBTest, SEARCH_TEST) {
ASSERT_TRUE(stat.ok());
}
index.engine_type_ = (int)milvus::engine::EngineType::FAISS_PQ;
db_->CreateIndex(TABLE_NAME, index); // wait until build index finish
{
std::vector<std::string> tags;
milvus::engine::ResultIds result_ids;
milvus::engine::ResultDistances result_distances;
stat = db_->Query(TABLE_NAME, tags, k, nq, 10, xq.data(), result_ids, result_distances);
ASSERT_TRUE(stat.ok());
stat = db_->Query(TABLE_NAME, tags, k, 1100, 10, xq.data(), result_ids, result_distances);
ASSERT_TRUE(stat.ok());
}
#ifdef CUSTOMIZATION
// test FAISS_IVFSQ8H optimizer
index.engine_type_ = (int)milvus::engine::EngineType::FAISS_IVFSQ8H;
@ -375,7 +388,13 @@ TEST_F(DBTest, SEARCH_TEST) {
{ // search by specify index file
milvus::engine::meta::DatesT dates;
std::vector<std::string> file_ids = {"1", "2", "3", "4", "5", "6"};
std::vector<std::string> file_ids;
// sometimes this case run fast to merge file and build index, old file will be deleted immediately,
// so the QueryByFileID cannot get files to search
// input 100 files ids to avoid random failure of this case
for (int i = 0; i < 100; i++) {
file_ids.push_back(std::to_string(i));
}
result_ids.clear();
result_dists.clear();
stat = db_->QueryByFileID(TABLE_NAME, file_ids, k, nq, 10, xq.data(), dates, result_ids, result_dists);

2
install.md
View File

@ -103,7 +103,7 @@ $ docker run -it -p 19530:19530 -d milvusdb/milvus-cpu-build-env:v0.6.0-ubuntu18
Start a GPU container:
```shell
$ docker run — runtime=nvidia -it -p 19530:19530 -d milvusdb/milvus-gpu-build-env:v0.6.0-ubuntu18.04
$ docker run --runtime=nvidia -it -p 19530:19530 -d milvusdb/milvus-gpu-build-env:v0.6.0-ubuntu18.04
```
To enter the container: