test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-04 21:09:06 +08:00
Code Issues Actions 14 Packages Projects Releases Wiki Activity

code optimize (#1986)

Browse Source 
Signed-off-by: bigbraver <bigbraver@qq.com>
This commit is contained in:
bigbraver 2020-04-19 11:54:23 +08:00 committed by GitHub
parent 25b6efc8e8
commit e8f759c22b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 20 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
core/src/db/Constants.h
View File

@ -16,18 +16,14 @@
namespace milvus {
namespace engine {
constexpr uint64_t K = 1024UL;
constexpr uint64_t M = K * K;
constexpr uint64_t G = K * M;
constexpr uint64_t T = K * G;
constexpr int64_t KB = 1LL << 10;
constexpr int64_t MB = 1LL << 20;
constexpr int64_t GB = 1LL << 30;
constexpr int64_t TB = 1LL << 40;
constexpr uint64_t MAX_TABLE_FILE_MEM = 128 * M;
constexpr int64_t MAX_TABLE_FILE_MEM = 128 * MB;
constexpr int FLOAT_TYPE_SIZE = sizeof(float);
static constexpr uint64_t ONE_KB = K;
static constexpr uint64_t ONE_MB = ONE_KB * ONE_KB;
static constexpr uint64_t ONE_GB = ONE_KB * ONE_MB;
} // namespace engine
} // namespace milvus

4
core/src/db/DBImpl.cpp
View File

@ -204,7 +204,7 @@ DBImpl::CreateCollection(meta::CollectionSchema& collection_schema) {
}
meta::CollectionSchema temp_schema = collection_schema;
temp_schema.index_file_size_ *= ONE_MB; // store as MB
temp_schema.index_file_size_ *= MB; // store as MB
if (options_.wal_enable_) {
temp_schema.flush_lsn_ = wal_mgr_->CreateCollection(collection_schema.collection_id_);
}
@ -257,7 +257,7 @@ DBImpl::DescribeCollection(meta::CollectionSchema& collection_schema) {
}
auto stat = meta_ptr_->DescribeCollection(collection_schema);
collection_schema.index_file_size_ /= ONE_MB; // return as MB
collection_schema.index_file_size_ /= MB; // return as MB
return stat;
}

2
core/src/db/Options.h
View File

@ -68,7 +68,7 @@ struct DBOptions {
DBMetaOptions meta_;
int mode_ = MODE::SINGLE;
size_t insert_buffer_size_ = 4 * ONE_GB;
size_t insert_buffer_size_ = 4 * GB;
bool insert_cache_immediately_ = false;
int64_t auto_flush_interval_ = 1;

2
core/src/db/insert/MemManagerImpl.cpp
View File

@ -319,7 +319,7 @@ MemManagerImpl::GetMaxLSN(const MemList& tables) {
void
MemManagerImpl::OnInsertBufferSizeChanged(int64_t value) {
options_.insert_buffer_size_ = value * ONE_GB;
options_.insert_buffer_size_ = value * GB;
}
} // namespace engine

2
core/src/db/meta/MetaTypes.h
View File

@ -27,7 +27,7 @@ namespace meta {
constexpr int32_t DEFAULT_ENGINE_TYPE = (int)EngineType::FAISS_IDMAP;
constexpr int32_t DEFAULT_NLIST = 16384;
constexpr int32_t DEFAULT_METRIC_TYPE = (int)MetricType::L2;
constexpr int32_t DEFAULT_INDEX_FILE_SIZE = ONE_GB;
constexpr int32_t DEFAULT_INDEX_FILE_SIZE = GB;
constexpr char CURRENT_VERSION[] = MILVUS_VERSION;
constexpr int64_t FLAG_MASK_NO_USERID = 0x1;

2
core/src/db/meta/MySQLMetaImpl.cpp
View File

@ -2073,7 +2073,7 @@ MySQLMetaImpl::Archive() {
uint64_t sum = 0;
Size(sum);
auto to_delete = (sum - limit * G);
auto to_delete = (sum - limit * GB);
DiscardFiles(to_delete);
LOG_ENGINE_DEBUG_ << "Archive files to free disk";

2
core/src/db/meta/SqliteMetaImpl.cpp
View File

@ -1461,7 +1461,7 @@ SqliteMetaImpl::Archive() {
uint64_t sum = 0;
Size(sum);
int64_t to_delete = (int64_t)sum - limit * G;
int64_t to_delete = (int64_t)sum - limit * GB;
DiscardFiles(to_delete);
LOG_ENGINE_DEBUG_ << "Archive files to free disk";

4
core/unittest/db/test_db.cpp
View File

@ -200,7 +200,7 @@ TEST_F(DBTest, DB_TEST) {
std::vector<std::string> tags;
stat = db_->Query(dummy_context_, COLLECTION_NAME, tags, k, json_params, qxb, result_ids, result_distances);
ss << "Search " << j << " With Size " << count / milvus::engine::M << " M";
ss << "Search " << j << " With Size " << count / milvus::engine::MB << " MB";
STOP_TIMER(ss.str());
ASSERT_TRUE(stat.ok());
@ -946,7 +946,7 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) {
db_->Size(size);
LOG(DEBUG) << "size=" << size;
ASSERT_LE(size, 1 * milvus::engine::G);
ASSERT_LE(size, 1 * milvus::engine::GB);
}
TEST_F(DBTest2, DELETE_TEST) {

4
core/unittest/db/test_db_mysql.cpp
View File

@ -92,7 +92,7 @@ TEST_F(MySqlDBTest, DB_TEST) {
std::vector<std::string> tags;
stat = db_->Query(dummy_context_, COLLECTION_NAME, tags, k, json_params, qxb, result_ids, result_distances);
ss << "Search " << j << " With Size " << count / milvus::engine::M << " M";
ss << "Search " << j << " With Size " << count / milvus::engine::MB << " MB";
STOP_TIMER(ss.str());
ASSERT_TRUE(stat.ok());
@ -250,7 +250,7 @@ TEST_F(MySqlDBTest, ARHIVE_DISK_CHECK) {
db_->Size(size);
LOG(DEBUG) << "size=" << size;
ASSERT_LE(size, 1 * milvus::engine::G);
ASSERT_LE(size, 1 * milvus::engine::GB);
FIU_ENABLE_FIU("MySQLMetaImpl.Size.null_connection");
stat = db_->Size(size);

2
core/unittest/db/test_delete.cpp
View File

@ -740,7 +740,7 @@ TEST_F(CompactTest, compact_basic) {
TEST_F(CompactTest, compact_with_index) {
milvus::engine::meta::CollectionSchema collection_info = BuildCollectionSchema();
collection_info.index_file_size_ = milvus::engine::ONE_KB;
collection_info.index_file_size_ = milvus::engine::KB;
collection_info.engine_type_ = (int32_t)milvus::engine::EngineType::FAISS_IVFSQ8;
auto stat = db_->CreateCollection(collection_info);

2
core/unittest/db/test_meta.cpp
View File

@ -506,7 +506,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
for (auto i = 0; i < cnt; ++i) {
status = impl.CreateCollectionFile(table_file);
table_file.file_type_ = milvus::engine::meta::SegmentSchema::NEW;
table_file.file_size_ = each_size * milvus::engine::G;
table_file.file_size_ = each_size * milvus::engine::GB;
status = impl.UpdateCollectionFile(table_file);
files.push_back(table_file);
ids.push_back(table_file.id_);

4
core/unittest/db/test_meta_mysql.cpp
View File

@ -446,7 +446,7 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) {
for (auto i = 0; i < cnt; ++i) {
status = impl.CreateCollectionFile(table_file);
table_file.file_type_ = milvus::engine::meta::SegmentSchema::NEW;
table_file.file_size_ = each_size * milvus::engine::G;
table_file.file_size_ = each_size * milvus::engine::GB;
status = impl.UpdateCollectionFile(table_file);
files.push_back(table_file);
ids.push_back(table_file.id_);
@ -618,7 +618,7 @@ TEST_F(MySqlMetaTest, COLLECTION_FILES_TEST) {
ASSERT_EQ(status.code(), milvus::DB_NOT_FOUND);
table_file.file_type_ = milvus::engine::meta::SegmentSchema::RAW;
table_file.file_size_ = milvus::engine::ONE_GB + 1;
table_file.file_size_ = milvus::engine::GB + 1;
status = impl_->UpdateCollectionFile(table_file);
ASSERT_TRUE(status.ok());
#if 0