test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-01 11:29:48 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

MS-510 unittest out of memory and crashed

Browse Source 
Former-commit-id: 17e5dd44414299840ccba10ae93ff67d10c80405
This commit is contained in:
starlord 2019-09-07 18:26:13 +08:00
parent 419bd23db4
commit 3610c91130
7 changed files with 47 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cpp/CHANGELOG.md
View File

@ -29,6 +29,7 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-492 - Drop index failed if index have been created with index_type: FLAT
- MS-493 - Knowhere unittest crash
- MS-453 - GPU search error when nprobe set more than 1024
- MS-510 - unittest out of memory and crashed
## Improvement
- MS-327 - Clean code for milvus

2
cpp/src/db/meta/SqliteMetaImpl.cpp
View File

@ -192,8 +192,6 @@ Status SqliteMetaImpl::CreateTable(TableSchema &table_schema) {
} catch (std::exception &e) {
return HandleException("Encounter exception when create table", e.what());
}
return Status::OK();
}
Status SqliteMetaImpl::DeleteTable(const std::string& table_id) {

12
cpp/src/db/scheduler/task/IndexLoadTask.cpp
View File

@ -50,7 +50,17 @@ std::shared_ptr<IScheduleTask> IndexLoadTask::Execute() {
file_->nlist_);
try {
index_ptr->Load();
auto stat = index_ptr->Load();
if(!stat.ok()) {
//typical error: file not available
ENGINE_LOG_ERROR << "Failed to load index file: file not available";
for(auto& context : search_contexts_) {
context->IndexSearchDone(file_->id_);//mark as done avoid dead lock, even failed
}
return nullptr;
}
} catch (std::exception& ex) {
//typical error: out of disk space or permition denied
std::string msg = "Failed to load index file: " + std::string(ex.what());

12
cpp/src/scheduler/task/SearchTask.cpp
View File

@ -99,7 +99,17 @@ XSearchTask::Load(LoadType type, uint8_t device_id) {
try {
if (type == LoadType::DISK2CPU) {
index_engine_->Load();
auto stat = index_engine_->Load();
if(!stat.ok()) {
//typical error: file not available
ENGINE_LOG_ERROR << "Failed to load index file: file not available";
for(auto& context : search_contexts_) {
context->IndexSearchDone(file_->id_);//mark as done avoid dead lock, even failed
}
return;
}
} else if (type == LoadType::CPU2GPU) {
index_engine_->CopyToGpu(device_id);
} else if (type == LoadType::GPU2CPU) {

14
cpp/src/utils/ValidationUtil.cpp
View File

@ -8,9 +8,9 @@ namespace zilliz {
namespace milvus {
namespace server {
constexpr size_t table_name_size_limit = 255;
constexpr int64_t table_dimension_limit = 16384;
constexpr int32_t index_file_size_limit = 4096; //index trigger size max = 4096 MB
constexpr size_t TABLE_NAME_SIZE_LIMIT = 255;
constexpr int64_t TABLE_DIMENSION_LIMIT = 16384;
constexpr int32_t INDEX_FILE_SIZE_LIMIT = 4096; //index trigger size max = 4096 MB
ErrorCode
ValidationUtil::ValidateTableName(const std::string &table_name) {
@ -22,7 +22,7 @@ ValidationUtil::ValidateTableName(const std::string &table_name) {
}
// Table name size shouldn't exceed 16384.
if (table_name.size() > table_name_size_limit) {
if (table_name.size() > TABLE_NAME_SIZE_LIMIT) {
SERVER_LOG_ERROR << "Table name size exceed the limitation";
return SERVER_INVALID_TABLE_NAME;
}
@ -48,8 +48,8 @@ ValidationUtil::ValidateTableName(const std::string &table_name) {
ErrorCode
ValidationUtil::ValidateTableDimension(int64_t dimension) {
if (dimension <= 0 || dimension > table_dimension_limit) {
SERVER_LOG_ERROR << "Table dimension excceed the limitation: " << table_dimension_limit;
if (dimension <= 0 || dimension > TABLE_DIMENSION_LIMIT) {
SERVER_LOG_ERROR << "Table dimension excceed the limitation: " << TABLE_DIMENSION_LIMIT;
return SERVER_INVALID_VECTOR_DIMENSION;
} else {
return SERVER_SUCCESS;
@ -77,7 +77,7 @@ ValidationUtil::ValidateTableIndexNlist(int32_t nlist) {
ErrorCode
ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) {
if(index_file_size <= 0 || index_file_size > index_file_size_limit) {
if(index_file_size <= 0 || index_file_size > INDEX_FILE_SIZE_LIMIT) {
return SERVER_INVALID_INDEX_FILE_SIZE;
}

6
cpp/src/wrapper/knowhere/vec_index.cpp
View File

@ -139,7 +139,11 @@ VecIndexPtr read_index(const std::string &location) {
knowhere::BinarySet load_data_list;
FileIOReader reader(location);
reader.fs.seekg(0, reader.fs.end);
size_t length = reader.fs.tellg();
int64_t length = reader.fs.tellg();
if(length <= 0) {
return nullptr;
}
reader.fs.seekg(0);
size_t rp = 0;

17
cpp/unittest/db/utils.cpp
View File

@ -94,7 +94,8 @@ void DBTest::TearDown() {
engine::ResMgrInst::GetInstance()->Stop();
engine::SchedInst::GetInstance()->Stop();
boost::filesystem::remove_all("/tmp/milvus_test");
auto options = GetOptions();
boost::filesystem::remove_all(options.meta.path);
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@ -110,11 +111,15 @@ engine::Options DBTest2::GetOptions() {
void MetaTest::SetUp() {
BaseTest::SetUp();
impl_ = engine::DBMetaImplFactory::Build();
auto options = GetOptions();
impl_ = std::make_shared<engine::meta::SqliteMetaImpl>(options.meta);
}
void MetaTest::TearDown() {
impl_->DropAll();
auto options = GetOptions();
boost::filesystem::remove_all(options.meta.path);
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@ -134,13 +139,15 @@ engine::Options MySqlDBTest::GetOptions() {
void MySqlMetaTest::SetUp() {
BaseTest::SetUp();
engine::DBMetaOptions options = GetOptions().meta;
int mode = engine::Options::MODE::SINGLE;
impl_ = std::make_shared<engine::meta::MySQLMetaImpl>(options, mode);
auto options = GetOptions();
impl_ = std::make_shared<engine::meta::MySQLMetaImpl>(options.meta, options.mode);
}
void MySqlMetaTest::TearDown() {
impl_->DropAll();
auto options = GetOptions();
boost::filesystem::remove_all(options.meta.path);
}
zilliz::milvus::engine::Options MySqlMetaTest::GetOptions() {