test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-05 05:18:52 +08:00
Code Issues Actions 14 Packages Projects Releases Wiki Activity

refactor(db): DropAll -Count CleanUp CleanUpFilesWithTTL

Browse Source 
Former-commit-id: b718bb89945651de19a1263eaf64eab22d11eb6c
This commit is contained in:
Xu Peng 2019-05-27 19:22:58 +08:00
parent 874eeb8f7a
commit ca33fbbe14
9 changed files with 45 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cpp/src/db/DB.h
View File

@ -38,8 +38,6 @@ public:
virtual Status drop_all() = 0;
virtual Status count(const std::string& table_id, long& result) = 0;
DB() = default;
DB(const DB&) = delete;
DB& operator=(const DB&) = delete;

9
cpp/src/db/DBImpl.cpp
View File

@ -316,7 +316,7 @@ Status DBImpl<EngineT>::background_merge_files(const std::string& table_id) {
try_build_index();
_pMeta->cleanup_ttl_files(1);
_pMeta->CleanUpFilesWithTTL(1);
return Status::OK();
}
@ -403,12 +403,7 @@ void DBImpl<EngineT>::background_compaction() {
template<typename EngineT>
Status DBImpl<EngineT>::drop_all() {
return _pMeta->drop_all();
}
template<typename EngineT>
Status DBImpl<EngineT>::count(const std::string& table_id, long& result) {
return _pMeta->count(table_id, result);
return _pMeta->DropAll();
}
template<typename EngineT>

2
cpp/src/db/DBImpl.h
View File

@ -48,8 +48,6 @@ public:
virtual Status drop_all() override;
virtual Status count(const std::string& table_id, long& result) override;
virtual Status size(long& result) override;
virtual ~DBImpl();

60
cpp/src/db/DBMetaImpl.cpp
View File

@ -105,7 +105,7 @@ Status DBMetaImpl::initialize() {
ConnectorPtr->open_forever(); // thread safe option
ConnectorPtr->pragma.journal_mode(journal_mode::WAL); // WAL => write ahead log
cleanup();
CleanUp();
return Status::OK();
}
@ -570,7 +570,7 @@ Status DBMetaImpl::UpdateTableFiles(TableFilesSchema& files) {
return Status::OK();
}
Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) {
Status DBMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
auto now = utils::GetMicroSecTimeStamp();
try {
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
@ -583,21 +583,21 @@ Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) {
c(&TableFileSchema::updated_time) > now - seconds*US_PS));
TableFilesSchema updated;
TableFileSchema table_file;
for (auto& file : selected) {
TableFileSchema group_file;
group_file.id = std::get<0>(file);
group_file.table_id = std::get<1>(file);
group_file.file_id = std::get<2>(file);
group_file.file_type = std::get<3>(file);
group_file.size = std::get<4>(file);
group_file.date = std::get<5>(file);
GetGroupFilePath(group_file);
if (group_file.file_type == TableFileSchema::TO_DELETE) {
boost::filesystem::remove(group_file.location);
table_file.id = std::get<0>(file);
table_file.table_id = std::get<1>(file);
table_file.file_id = std::get<2>(file);
table_file.file_type = std::get<3>(file);
table_file.size = std::get<4>(file);
table_file.date = std::get<5>(file);
GetGroupFilePath(table_file);
if (table_file.file_type == TableFileSchema::TO_DELETE) {
boost::filesystem::remove(table_file.location);
}
ConnectorPtr->remove<TableFileSchema>(group_file.id);
/* LOG(DEBUG) << "Removing deleted id=" << group_file.id << " location=" << group_file.location << std::endl; */
ConnectorPtr->remove<TableFileSchema>(table_file.id);
/* LOG(DEBUG) << "Removing deleted id=" << table_file.id << " location=" << table_file.location << std::endl; */
}
} catch (std::exception & e) {
LOG(DEBUG) << e.what();
@ -607,7 +607,7 @@ Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) {
return Status::OK();
}
Status DBMetaImpl::cleanup() {
Status DBMetaImpl::CleanUp() {
try {
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::table_id,
@ -619,21 +619,21 @@ Status DBMetaImpl::cleanup() {
c(&TableFileSchema::file_type) == (int)TableFileSchema::NEW));
TableFilesSchema updated;
TableFileSchema table_file;
for (auto& file : selected) {
TableFileSchema group_file;
group_file.id = std::get<0>(file);
group_file.table_id = std::get<1>(file);
group_file.file_id = std::get<2>(file);
group_file.file_type = std::get<3>(file);
group_file.size = std::get<4>(file);
group_file.date = std::get<5>(file);
GetGroupFilePath(group_file);
if (group_file.file_type == TableFileSchema::TO_DELETE) {
boost::filesystem::remove(group_file.location);
table_file.id = std::get<0>(file);
table_file.table_id = std::get<1>(file);
table_file.file_id = std::get<2>(file);
table_file.file_type = std::get<3>(file);
table_file.size = std::get<4>(file);
table_file.date = std::get<5>(file);
GetGroupFilePath(table_file);
if (table_file.file_type == TableFileSchema::TO_DELETE) {
boost::filesystem::remove(table_file.location);
}
ConnectorPtr->remove<TableFileSchema>(group_file.id);
/* LOG(DEBUG) << "Removing id=" << group_file.id << " location=" << group_file.location << std::endl; */
ConnectorPtr->remove<TableFileSchema>(table_file.id);
/* LOG(DEBUG) << "Removing id=" << table_file.id << " location=" << table_file.location << std::endl; */
}
} catch (std::exception & e) {
LOG(DEBUG) << e.what();
@ -643,7 +643,7 @@ Status DBMetaImpl::cleanup() {
return Status::OK();
}
Status DBMetaImpl::count(const std::string& table_id, long& result) {
Status DBMetaImpl::Count(const std::string& table_id, long& result) {
try {
auto selected = ConnectorPtr->select(columns(&TableFileSchema::size,
@ -674,7 +674,7 @@ Status DBMetaImpl::count(const std::string& table_id, long& result) {
return Status::OK();
}
Status DBMetaImpl::drop_all() {
Status DBMetaImpl::DropAll() {
if (boost::filesystem::is_directory(_options.path)) {
boost::filesystem::remove_all(_options.path);
}
@ -682,7 +682,7 @@ Status DBMetaImpl::drop_all() {
}
DBMetaImpl::~DBMetaImpl() {
cleanup();
CleanUp();
}
} // namespace meta

8
cpp/src/db/DBMetaImpl.h
View File

@ -46,13 +46,13 @@ public:
virtual Status Size(long& result) override;
virtual Status cleanup() override;
virtual Status CleanUp() override;
virtual Status cleanup_ttl_files(uint16_t seconds) override;
virtual Status CleanUpFilesWithTTL(uint16_t seconds) override;
virtual Status drop_all() override;
virtual Status DropAll() override;
virtual Status count(const std::string& table_id, long& result) override;
virtual Status Count(const std::string& table_id, long& result) override;
virtual ~DBMetaImpl();

8
cpp/src/db/Meta.h
View File

@ -48,12 +48,12 @@ public:
virtual Status FilesToIndex(TableFilesSchema&) = 0;
virtual Status cleanup() = 0;
virtual Status cleanup_ttl_files(uint16_t) = 0;
virtual Status CleanUp() = 0;
virtual Status CleanUpFilesWithTTL(uint16_t) = 0;
virtual Status drop_all() = 0;
virtual Status DropAll() = 0;
virtual Status count(const std::string& table_id, long& result) = 0;
virtual Status Count(const std::string& table_id, long& result) = 0;
static DateT GetDate(const std::time_t& t, int day_delta = 0);
static DateT GetDate();

4
cpp/unittest/db/db_tests.cpp
View File

@ -152,12 +152,12 @@ TEST_F(DBTest, DB_TEST) {
for (auto j=0; j<10; ++j) {
ss.str("");
db_->count(group_name, count);
db_->size(count);
prev_count = count;
START_TIMER;
stat = db_->search(group_name, k, qb, qxb, results);
ss << "Search " << j << " With Size " << (float)(count*group_dim*sizeof(float))/engine::meta::M << " M";
ss << "Search " << j << " With Size " << count/engine::meta::M << " M";
STOP_TIMER(ss.str());
ASSERT_STATS(stat);

4
cpp/unittest/db/meta_tests.cpp
View File

@ -137,7 +137,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
i++;
}
impl.drop_all();
impl.DropAll();
}
TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
@ -180,7 +180,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
++i;
}
impl.drop_all();
impl.DropAll();
}
TEST_F(MetaTest, TABLE_FILES_TEST) {

2
cpp/unittest/db/utils.cpp
View File

@ -59,5 +59,5 @@ void MetaTest::SetUp() {
}
void MetaTest::TearDown() {
impl_->drop_all();
impl_->DropAll();
}