test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-04 12:59:23 +08:00
Code Issues Actions 14 Packages Projects Releases Wiki Activity

Merge branch 'mysql-0.3.0' into 'branch-0.3.1'

Browse Source 
MS-105 - Add MySQL

See merge request megasearch/milvus!145

Former-commit-id: 07b1d5f82454a6b07e6951abccf3d74b3c75cff3
This commit is contained in:
jinhai 2019-07-01 20:10:45 +08:00
commit 6c88dc1ed8
8 changed files with 153 additions and 357 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
cpp/coverage.sh
View File

@ -14,6 +14,26 @@ DIR_LCOV_OUTPUT="lcov_out"
DIR_GCNO="cmake_build"
DIR_UNITTEST="milvus/bin"
MYSQL_USER_NAME=root
MYSQL_PASSWORD=Fantast1c
MYSQL_HOST='192.168.1.194'
MYSQL_PORT='3306'
MYSQL_DB_NAME=milvus_`date +%s%N`
function mysql_exc()
{
cmd=$1
mysql -h${MYSQL_HOST} -u${MYSQL_USER_NAME} -p${MYSQL_PASSWORD} -e "${cmd}"
if [ $? -ne 0 ]; then
echo "mysql $cmd run failed"
fi
}
mysql_exc "CREATE DATABASE IF NOT EXISTS ${MYSQL_DB_NAME};"
mysql_exc "GRANT ALL PRIVILEGES ON ${MYSQL_DB_NAME}.* TO '${MYSQL_USER_NAME}'@'%';"
mysql_exc "FLUSH PRIVILEGES;"
# get baseline
${LCOV_CMD} -c -i -d ${DIR_GCNO} -o "${FILE_INFO_BASE}"
if [ $? -ne 0 ]; then
@ -22,10 +42,11 @@ if [ $? -ne 0 ]; then
fi
for test in `ls ${DIR_UNITTEST}`; do
echo $test
case ${test} in
db_test)
# set run args for db_test
args="mysql://root:Fantast1c@192.168.1.194:3306/test"
args="mysql://${MYSQL_USER_NAME}:${MYSQL_PASSWORD}@${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB_NAME}"
;;
*_test)
args=""
@ -38,6 +59,8 @@ for test in `ls ${DIR_UNITTEST}`; do
fi
done
mysql_exc "DROP DATABASE IF EXISTS ${MYSQL_DB_NAME};"
# gen test converage
${LCOV_CMD} -d ${DIR_GCNO} -o "${FILE_INFO_MILVUS}" -c
# merge coverage
@ -50,4 +73,4 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \
"*/cmake_build/*_ep-prefix/*" \
# gen html report
${LCOV_GEN_CMD} "${FILE_INFO_OUTPUT_NEW}" --output-directory ${DIR_LCOV_OUTPUT}/
${LCOV_GEN_CMD} "${FILE_INFO_OUTPUT_NEW}" --output-directory ${DIR_LCOV_OUTPUT}/

4
cpp/src/db/MySQLConnectionPool.h
View File

@ -69,6 +69,10 @@ public:
max_idle_time_ = max_idle;
}
std::string getDB() {
return db_;
}
protected:
// Superclass overrides

143
cpp/src/db/MySQLMetaImpl.cpp
View File

@ -193,9 +193,7 @@ namespace meta {
"engine_type INT DEFAULT 1 NOT NULL, " <<
"store_raw_data BOOL DEFAULT false NOT NULL);";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str();
if (!InitializeQuery.exec()) {
return Status::DBTransactionError("Initialization Error", InitializeQuery.error());
@ -212,9 +210,7 @@ namespace meta {
"created_on BIGINT NOT NULL, " <<
"date INT DEFAULT -1 NOT NULL);";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str();
if (!InitializeQuery.exec()) {
return Status::DBTransactionError("Initialization Error", InitializeQuery.error());
@ -305,9 +301,7 @@ namespace meta {
"WHERE table_id = " << quote << table_id << " AND " <<
"date in (" << dateListStr << ");";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropPartitionsByDates: " << dropPartitionsByDatesQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropPartitionsByDates: " << dropPartitionsByDatesQuery.str();
if (!dropPartitionsByDatesQuery.exec()) {
ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING PARTITIONS BY DATES";
@ -352,9 +346,7 @@ namespace meta {
"WHERE table_id = " << quote << table_schema.table_id_ << ";";
// ENGINE_LOG_DEBUG << "Create Table : " << createTableQuery.str();
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str();
StoreQueryResult res = createTableQuery.store();
assert(res && res.num_rows() <= 1);
@ -390,9 +382,7 @@ namespace meta {
created_on << ", " << files_cnt << ", " << engine_type << ", " << store_raw_data << ");";
// ENGINE_LOG_DEBUG << "Create Table : " << createTableQuery.str();
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str();
if (SimpleResult res = createTableQuery.execute()) {
table_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()?
@ -457,9 +447,7 @@ namespace meta {
"SET state = " << std::to_string(TableSchema::TO_DELETE) << " " <<
"WHERE table_id = " << quote << table_id << ";";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTable: " << deleteTableQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTable: " << deleteTableQuery.str();
if (!deleteTableQuery.exec()) {
ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE";
@ -506,9 +494,7 @@ namespace meta {
"WHERE table_id = " << quote << table_id << " AND " <<
"file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTableFiles: " << deleteTableFilesQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTableFiles: " << deleteTableFilesQuery.str();
if (!deleteTableFilesQuery.exec()) {
ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE FILES";
@ -551,9 +537,7 @@ namespace meta {
"WHERE table_id = " << quote << table_schema.table_id_ << " " <<
"AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::DescribeTable: " << describeTableQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::DescribeTable: " << describeTableQuery.str();
res = describeTableQuery.store();
} //Scoped Connection
@ -618,9 +602,7 @@ namespace meta {
"AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " <<
"AS " << quote << "check" << ";";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::HasTable: " << hasTableQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::HasTable: " << hasTableQuery.str();
res = hasTableQuery.store();
} //Scoped Connection
@ -664,9 +646,7 @@ namespace meta {
"FROM Tables " <<
"WHERE state <> " << std::to_string(TableSchema::TO_DELETE) << ";";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::AllTables: " << allTablesQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::AllTables: " << allTablesQuery.str();
res = allTablesQuery.store();
} //Scoped Connection
@ -755,9 +735,7 @@ namespace meta {
quote << file_id << ", " << file_type << ", " << size << ", " <<
updated_time << ", " << created_on << ", " << date << ");";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTableFile: " << createTableFileQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTableFile: " << createTableFileQuery.str();
if (SimpleResult res = createTableFileQuery.execute()) {
file_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()?
@ -821,9 +799,7 @@ namespace meta {
"FROM TableFiles " <<
"WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToIndex: " << filesToIndexQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToIndex: " << filesToIndexQuery.str();
res = filesToIndexQuery.store();
} //Scoped Connection
@ -911,9 +887,7 @@ namespace meta {
"file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " <<
"file_type = " << std::to_string(TableFileSchema::INDEX) << ");";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str();
res = filesToSearchQuery.store();
@ -936,9 +910,7 @@ namespace meta {
"file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " <<
"file_type = " << std::to_string(TableFileSchema::INDEX) << ");";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str();
res = filesToSearchQuery.store();
@ -1023,9 +995,7 @@ namespace meta {
"file_type = " << std::to_string(TableFileSchema::RAW) << " " <<
"ORDER BY size DESC" << ";";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToMerge: " << filesToMergeQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToMerge: " << filesToMergeQuery.str();
res = filesToMergeQuery.store();
} //Scoped Connection
@ -1116,9 +1086,7 @@ namespace meta {
"WHERE table_id = " << quote << table_id << " AND " <<
"(" << idStr << ");";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::GetTableFiles: " << getTableFileQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::GetTableFiles: " << getTableFileQuery.str();
res = getTableFileQuery.store();
} //Scoped Connection
@ -1200,9 +1168,7 @@ namespace meta {
"WHERE created_on < " << std::to_string(now - usecs) << " AND " <<
"file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::Archive: " << archiveQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::Archive: " << archiveQuery.str();
if (!archiveQuery.exec()) {
return Status::DBTransactionError("QUERY ERROR DURING ARCHIVE", archiveQuery.error());
@ -1251,9 +1217,7 @@ namespace meta {
"FROM TableFiles " <<
"WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::Size: " << getSizeQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::Size: " << getSizeQuery.str();
res = getSizeQuery.store();
} //Scoped Connection
@ -1315,9 +1279,7 @@ namespace meta {
"ORDER BY id ASC " <<
"LIMIT 10;";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str();
// std::cout << discardFilesQuery.str() << std::endl;
StoreQueryResult res = discardFilesQuery.store();
@ -1349,9 +1311,7 @@ namespace meta {
"updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " <<
"WHERE " << idsToDiscardStr << ";";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str();
status = discardFilesQuery.exec();
if (!status) {
@ -1397,9 +1357,7 @@ namespace meta {
updateTableFileQuery << "SELECT state FROM Tables " <<
"WHERE table_id = " << quote << file_schema.table_id_ << ";";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str();
StoreQueryResult res = updateTableFileQuery.store();
@ -1434,9 +1392,7 @@ namespace meta {
"date = " << date << " " <<
"WHERE id = " << id << ";";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str();
// std::cout << updateTableFileQuery.str() << std::endl;
@ -1491,9 +1447,7 @@ namespace meta {
"AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " <<
"AS " << quote << "check" << ";";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str();
StoreQueryResult res = updateTableFilesQuery.store();
@ -1530,9 +1484,7 @@ namespace meta {
"date = " << date << " " <<
"WHERE id = " << id << ";";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str();
if (!updateTableFilesQuery.exec()) {
ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILES";
@ -1582,9 +1534,7 @@ namespace meta {
"WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " AND " <<
"updated_time < " << std::to_string(now - seconds * US_PS) << ";";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str();
StoreQueryResult res = cleanUpFilesWithTTLQuery.store();
@ -1628,9 +1578,7 @@ namespace meta {
cleanUpFilesWithTTLQuery << "DELETE FROM TableFiles WHERE " <<
idsToDeleteStr << ";";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str();
if (!cleanUpFilesWithTTLQuery.exec()) {
ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL";
@ -1669,9 +1617,7 @@ namespace meta {
"FROM Tables " <<
"WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str();
StoreQueryResult res = cleanUpFilesWithTTLQuery.store();
assert(res);
@ -1697,9 +1643,7 @@ namespace meta {
cleanUpFilesWithTTLQuery << "DELETE FROM Tables WHERE " <<
idsToDeleteStr << ";";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str();
if (!cleanUpFilesWithTTLQuery.exec()) {
ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL";
@ -1733,17 +1677,26 @@ namespace meta {
// ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUp: connection in use = " << mysql_connection_pool_->getConnectionsInUse();
// }
ENGINE_LOG_DEBUG << "Remove table file type as NEW";
Query cleanUpQuery = connectionPtr->query();
cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";";
cleanUpQuery << "SELECT table_name " <<
"FROM information_schema.tables " <<
"WHERE table_schema = " << quote << mysql_connection_pool_->getDB() << quote << " " <<
"AND table_name = " << quote << "TableFiles" << quote << ";";
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str();
StoreQueryResult res = cleanUpQuery.store();
assert(res);
if (!res.empty()) {
ENGINE_LOG_DEBUG << "Remove table file type as NEW";
cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str();
}
if (!cleanUpQuery.exec()) {
ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES";
return Status::DBTransactionError("Clean up Error", cleanUpQuery.error());
if (!cleanUpQuery.exec()) {
ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES";
return Status::DBTransactionError("Clean up Error", cleanUpQuery.error());
}
}
} catch (const BadQuery& er) {
@ -1791,9 +1744,7 @@ namespace meta {
"file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " <<
"file_type = " << std::to_string(TableFileSchema::INDEX) << ");";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::Count: " << countQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::Count: " << countQuery.str();
res = countQuery.store();
} //Scoped Connection
@ -1838,9 +1789,7 @@ namespace meta {
Query dropTableQuery = connectionPtr->query();
dropTableQuery << "DROP TABLE IF EXISTS Tables, TableFiles;";
if (options_.sql_echo) {
ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropAll: " << dropTableQuery.str();
}
ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropAll: " << dropTableQuery.str();
if (dropTableQuery.exec()) {
return Status::OK();

1
cpp/src/db/Options.h
View File

@ -45,7 +45,6 @@ struct DBMetaOptions {
std::string path;
std::string backend_uri;
ArchiveConf archive_conf = ArchiveConf("delete");
bool sql_echo = false;
}; // DBMetaOptions
struct Options {

11
cpp/src/server/DBWrapper.cpp
View File

@ -23,17 +23,6 @@ DBWrapper::DBWrapper() {
if(index_size > 0) {//ensure larger than zero, unit is MB
opt.index_trigger_size = (size_t)index_size * engine::ONE_MB;
}
std::string sql_echo = config.GetValue(CONFIG_DB_SQL_ECHO, "off");
if (sql_echo == "on") {
opt.meta.sql_echo = true;
}
else if (sql_echo == "off") {
opt.meta.sql_echo = false;
}
else {
std::cout << "ERROR: sql_echo specified in db_config is not one of ['on', 'off']" << std::endl;
kill(0, SIGUSR1);
}
ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER);
std::string mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single");

1
cpp/src/server/ServerConfig.h
View File

@ -27,7 +27,6 @@ static const std::string CONFIG_DB_PATH = "db_path";
static const std::string CONFIG_DB_INDEX_TRIGGER_SIZE = "index_building_threshold";
static const std::string CONFIG_DB_ARCHIVE_DISK = "archive_disk_threshold";
static const std::string CONFIG_DB_ARCHIVE_DAYS = "archive_days_threshold";
static const std::string CONFIG_DB_SQL_ECHO = "sql_echo";
static const std::string CONFIG_LOG = "log_config";

70
cpp/unittest/db/mysql_db_test.cpp
View File

@ -18,28 +18,28 @@ using namespace zilliz::milvus;
namespace {
static const std::string TABLE_NAME = "test_group";
static constexpr int64_t TABLE_DIM = 256;
static constexpr int64_t VECTOR_COUNT = 250000;
static constexpr int64_t INSERT_LOOP = 100000;
static const std::string TABLE_NAME = "test_group";
static constexpr int64_t TABLE_DIM = 256;
static constexpr int64_t VECTOR_COUNT = 250000;
static constexpr int64_t INSERT_LOOP = 100000;
engine::meta::TableSchema BuildTableSchema() {
engine::meta::TableSchema table_info;
table_info.dimension_ = TABLE_DIM;
table_info.table_id_ = TABLE_NAME;
table_info.engine_type_ = (int)engine::EngineType::FAISS_IDMAP;
return table_info;
}
void BuildVectors(int64_t n, std::vector<float>& vectors) {
vectors.clear();
vectors.resize(n*TABLE_DIM);
float* data = vectors.data();
for(int i = 0; i < n; i++) {
for(int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48();
data[TABLE_DIM * i] += i / 2000.;
engine::meta::TableSchema BuildTableSchema() {
engine::meta::TableSchema table_info;
table_info.dimension_ = TABLE_DIM;
table_info.table_id_ = TABLE_NAME;
table_info.engine_type_ = (int)engine::EngineType::FAISS_IDMAP;
return table_info;
}
void BuildVectors(int64_t n, std::vector<float>& vectors) {
vectors.clear();
vectors.resize(n*TABLE_DIM);
float* data = vectors.data();
for(int i = 0; i < n; i++) {
for(int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48();
data[TABLE_DIM * i] += i / 2000.;
}
}
}
}
@ -69,10 +69,13 @@ TEST_F(MySQLDBTest, DB_TEST) {
std::vector<float> qxb;
BuildVectors(qb, qxb);
db_->InsertVectors(TABLE_NAME, qb, qxb.data(), target_ids);
ASSERT_EQ(target_ids.size(), qb);
std::thread search([&]() {
engine::QueryResults results;
int k = 10;
std::this_thread::sleep_for(std::chrono::seconds(2));
std::this_thread::sleep_for(std::chrono::seconds(5));
INIT_TIMER;
std::stringstream ss;
@ -91,7 +94,15 @@ TEST_F(MySQLDBTest, DB_TEST) {
ASSERT_STATS(stat);
for (auto k=0; k<qb; ++k) {
ASSERT_EQ(results[k][0].first, target_ids[k]);
// std::cout << results[k][0].first << " " << target_ids[k] << std::endl;
// ASSERT_EQ(results[k][0].first, target_ids[k]);
bool exists = false;
for (auto& result : results[k]) {
if (result.first == target_ids[k]) {
exists = true;
}
}
ASSERT_TRUE(exists);
ss.str("");
ss << "Result [" << k << "]:";
for (auto result : results[k]) {
@ -100,19 +111,20 @@ TEST_F(MySQLDBTest, DB_TEST) {
/* LOG(DEBUG) << ss.str(); */
}
ASSERT_TRUE(count >= prev_count);
std::this_thread::sleep_for(std::chrono::seconds(1));
std::this_thread::sleep_for(std::chrono::seconds(3));
}
});
int loop = INSERT_LOOP;
for (auto i=0; i<loop; ++i) {
if (i==40) {
db_->InsertVectors(TABLE_NAME, qb, qxb.data(), target_ids);
ASSERT_EQ(target_ids.size(), qb);
} else {
db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
}
// if (i==10) {
// db_->InsertVectors(TABLE_NAME, qb, qxb.data(), target_ids);
// ASSERT_EQ(target_ids.size(), qb);
// } else {
// db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
// }
db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids);
std::this_thread::sleep_for(std::chrono::microseconds(1));
}

253
cpp/unittest/db/mysql_meta_test.cpp
View File

@ -21,184 +21,7 @@
using namespace zilliz::milvus::engine;
//TEST_F(MySQLTest, InitializeTest) {
// DBMetaOptions options;
// //dialect+driver://username:password@host:port/database
// options.backend_uri = "mysql://root:1234@:/test";
// meta::MySQLMetaImpl impl(options);
// auto status = impl.Initialize();
// std::cout << status.ToString() << std::endl;
// ASSERT_TRUE(status.ok());
//}
TEST_F(MySQLTest, core) {
DBMetaOptions options;
// //dialect+driver://username:password@host:port/database
// options.backend_uri = "mysql://root:1234@:/test";
// options.path = "/tmp/vecwise_test";
try {
options = getDBMetaOptions();
} catch(std::exception& ex) {
ASSERT_TRUE(false);
return;
}
int mode = Options::MODE::SINGLE;
meta::MySQLMetaImpl impl(options, mode);
// auto status = impl.Initialize();
// ASSERT_TRUE(status.ok());
meta::TableSchema schema1;
schema1.table_id_ = "test1";
schema1.dimension_ = 123;
auto status = impl.CreateTable(schema1);
// std::cout << status.ToString() << std::endl;
ASSERT_TRUE(status.ok());
meta::TableSchema schema2;
schema2.table_id_ = "test2";
schema2.dimension_ = 321;
status = impl.CreateTable(schema2);
// std::cout << status.ToString() << std::endl;
ASSERT_TRUE(status.ok());
status = impl.CreateTable(schema2);
// std::cout << status.ToString() << std::endl;
// ASSERT_THROW(impl.CreateTable(schema), mysqlpp::BadQuery);
ASSERT_TRUE(status.ok());
status = impl.DeleteTable(schema2.table_id_);
// std::cout << status.ToString() << std::endl;
ASSERT_TRUE(status.ok());
size_t id1 = schema1.id_;
long created_on1 = schema1.created_on_;
status = impl.DescribeTable(schema1);
ASSERT_TRUE(status.ok());
ASSERT_EQ(schema1.id_, id1);
ASSERT_EQ(schema1.table_id_, "test1");
ASSERT_EQ(schema1.created_on_, created_on1);
ASSERT_EQ(schema1.files_cnt_, 0);
ASSERT_EQ(schema1.engine_type_, 1);
ASSERT_EQ(schema1.store_raw_data_, false);
bool check;
status = impl.HasTable("test1", check);
ASSERT_TRUE(status.ok());
ASSERT_EQ(check, true);
std::vector<meta::TableSchema> table_schema_array;
status = impl.AllTables(table_schema_array);
ASSERT_TRUE(status.ok());
ASSERT_EQ(table_schema_array.size(), 1);
meta::TableSchema resultSchema = table_schema_array[0];
ASSERT_EQ(resultSchema.id_, id1);
ASSERT_EQ(resultSchema.table_id_, "test1");
ASSERT_EQ(resultSchema.dimension_, 123);
ASSERT_EQ(resultSchema.files_cnt_, 0);
ASSERT_EQ(resultSchema.engine_type_, 1);
ASSERT_EQ(resultSchema.store_raw_data_, false);
meta::TableFileSchema tableFileSchema;
tableFileSchema.table_id_ = "test1";
status = impl.CreateTableFile(tableFileSchema);
// std::cout << status.ToString() << std::endl;
ASSERT_TRUE(status.ok());
tableFileSchema.file_type_ = meta::TableFileSchema::TO_INDEX;
status = impl.UpdateTableFile(tableFileSchema);
// std::cout << status.ToString() << std::endl;
ASSERT_TRUE(status.ok());
meta::TableFilesSchema filesToIndex;
status = impl.FilesToIndex(filesToIndex);
ASSERT_TRUE(status.ok());
ASSERT_EQ(filesToIndex.size(), 1);
meta::TableFileSchema fileToIndex = filesToIndex[0];
ASSERT_EQ(fileToIndex.table_id_, "test1");
ASSERT_EQ(fileToIndex.dimension_, 123);
// meta::TableFilesSchema filesToIndex;
// status = impl.FilesToIndex(filesToIndex);
// ASSERT_TRUE(status.ok());
// ASSERT_EQ(filesToIndex.size(), 0);
meta::DatesT partition;
partition.push_back(tableFileSchema.date_);
meta::DatePartionedTableFilesSchema filesToSearch;
status = impl.FilesToSearch(tableFileSchema.table_id_, partition, filesToSearch);
ASSERT_TRUE(status.ok());
ASSERT_EQ(filesToSearch.size(), 1);
ASSERT_EQ(filesToSearch[tableFileSchema.date_].size(), 1);
meta::TableFileSchema fileToSearch = filesToSearch[tableFileSchema.date_][0];
ASSERT_EQ(fileToSearch.table_id_, "test1");
ASSERT_EQ(fileToSearch.dimension_, 123);
tableFileSchema.file_type_ = meta::TableFileSchema::RAW;
status = impl.UpdateTableFile(tableFileSchema);
ASSERT_TRUE(status.ok());
meta::DatePartionedTableFilesSchema filesToMerge;
status = impl.FilesToMerge(tableFileSchema.table_id_, filesToMerge);
// std::cout << status.ToString() << std::endl;
ASSERT_TRUE(status.ok());
ASSERT_EQ(filesToMerge.size(), 1);
ASSERT_EQ(filesToMerge[tableFileSchema.date_].size(), 1);
meta::TableFileSchema fileToMerge = filesToMerge[tableFileSchema.date_][0];
ASSERT_EQ(fileToMerge.table_id_, "test1");
ASSERT_EQ(fileToMerge.dimension_, 123);
meta::TableFilesSchema resultTableFilesSchema;
std::vector<size_t> ids;
ids.push_back(tableFileSchema.id_);
status = impl.GetTableFiles(tableFileSchema.table_id_, ids, resultTableFilesSchema);
ASSERT_TRUE(status.ok());
ASSERT_EQ(resultTableFilesSchema.size(), 1);
meta::TableFileSchema resultTableFileSchema = resultTableFilesSchema[0];
// ASSERT_EQ(resultTableFileSchema.id_, tableFileSchema.id_);
ASSERT_EQ(resultTableFileSchema.table_id_, tableFileSchema.table_id_);
ASSERT_EQ(resultTableFileSchema.file_id_, tableFileSchema.file_id_);
ASSERT_EQ(resultTableFileSchema.file_type_, tableFileSchema.file_type_);
ASSERT_EQ(resultTableFileSchema.size_, tableFileSchema.size_);
ASSERT_EQ(resultTableFileSchema.date_, tableFileSchema.date_);
ASSERT_EQ(resultTableFileSchema.engine_type_, tableFileSchema.engine_type_);
ASSERT_EQ(resultTableFileSchema.dimension_, tableFileSchema.dimension_);
tableFileSchema.size_ = 234;
meta::TableSchema schema3;
schema3.table_id_ = "test3";
schema3.dimension_ = 321;
status = impl.CreateTable(schema3);
ASSERT_TRUE(status.ok());
meta::TableFileSchema tableFileSchema2;
tableFileSchema2.table_id_ = "test3";
tableFileSchema2.size_ = 345;
status = impl.CreateTableFile(tableFileSchema2);
ASSERT_TRUE(status.ok());
meta::TableFilesSchema filesToUpdate;
filesToUpdate.emplace_back(tableFileSchema);
filesToUpdate.emplace_back(tableFileSchema2);
status = impl.UpdateTableFile(tableFileSchema);
ASSERT_TRUE(status.ok());
uint64_t resultSize;
status = impl.Size(resultSize);
// std::cout << status.ToString() << std::endl;
ASSERT_TRUE(status.ok());
ASSERT_EQ(resultSize, tableFileSchema.size_ + tableFileSchema2.size_);
uint64_t countResult;
status = impl.Count(tableFileSchema.table_id_, countResult);
ASSERT_TRUE(status.ok());
status = impl.DropAll();
ASSERT_TRUE(status.ok());
}
TEST_F(MySQLTest, GROUP_TEST) {
TEST_F(MySQLTest, TABLE_TEST) {
DBMetaOptions options;
try {
options = getDBMetaOptions();
@ -210,38 +33,37 @@ TEST_F(MySQLTest, GROUP_TEST) {
int mode = Options::MODE::SINGLE;
meta::MySQLMetaImpl impl(options, mode);
auto table_id = "meta_test_group";
auto table_id = "meta_test_table";
meta::TableSchema group;
group.table_id_ = table_id;
auto status = impl.CreateTable(group);
meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl.CreateTable(table);
ASSERT_TRUE(status.ok());
auto gid = group.id_;
group.id_ = -1;
status = impl.DescribeTable(group);
auto gid = table.id_;
table.id_ = -1;
status = impl.DescribeTable(table);
ASSERT_TRUE(status.ok());
ASSERT_EQ(group.id_, gid);
ASSERT_EQ(group.table_id_, table_id);
ASSERT_EQ(table.id_, gid);
ASSERT_EQ(table.table_id_, table_id);
group.table_id_ = "not_found";
status = impl.DescribeTable(group);
table.table_id_ = "not_found";
status = impl.DescribeTable(table);
ASSERT_TRUE(!status.ok());
group.table_id_ = table_id;
status = impl.CreateTable(group);
table.table_id_ = table_id;
status = impl.CreateTable(table);
ASSERT_TRUE(status.ok());
group.table_id_ = "";
status = impl.CreateTable(group);
table.table_id_ = "";
status = impl.CreateTable(table);
ASSERT_TRUE(status.ok());
status = impl.DropAll();
ASSERT_TRUE(status.ok());
}
TEST_F(MySQLTest, table_file_TEST) {
TEST_F(MySQLTest, TABLE_FILE_TEST) {
DBMetaOptions options;
try {
options = getDBMetaOptions();
@ -253,17 +75,16 @@ TEST_F(MySQLTest, table_file_TEST) {
int mode = Options::MODE::SINGLE;
meta::MySQLMetaImpl impl(options, mode);
auto table_id = "meta_test_group";
auto table_id = "meta_test_table";
meta::TableSchema group;
group.table_id_ = table_id;
group.dimension_ = 256;
auto status = impl.CreateTable(group);
meta::TableSchema table;
table.table_id_ = table_id;
table.dimension_ = 256;
auto status = impl.CreateTable(table);
meta::TableFileSchema table_file;
table_file.table_id_ = group.table_id_;
table_file.table_id_ = table.table_id_;
status = impl.CreateTableFile(table_file);
// std::cout << status.ToString() << std::endl;
ASSERT_TRUE(status.ok());
ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW);
@ -332,15 +153,15 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) {
int mode = Options::MODE::SINGLE;
meta::MySQLMetaImpl impl(options, mode);
auto table_id = "meta_test_group";
auto table_id = "meta_test_table";
meta::TableSchema group;
group.table_id_ = table_id;
auto status = impl.CreateTable(group);
meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl.CreateTable(table);
meta::TableFilesSchema files;
meta::TableFileSchema table_file;
table_file.table_id_ = group.table_id_;
table_file.table_id_ = table.table_id_;
auto cnt = 100;
long ts = utils::GetMicroSecTimeStamp();
@ -391,13 +212,13 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DISK) {
auto impl = meta::MySQLMetaImpl(options, mode);
auto table_id = "meta_test_group";
meta::TableSchema group;
group.table_id_ = table_id;
auto status = impl.CreateTable(group);
meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl.CreateTable(table);
meta::TableFilesSchema files;
meta::TableFileSchema table_file;
table_file.table_id_ = group.table_id_;
table_file.table_id_ = table.table_id_;
auto cnt = 10;
auto each_size = 2UL;
@ -445,9 +266,9 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) {
auto table_id = "meta_test_group";
meta::TableSchema group;
group.table_id_ = table_id;
auto status = impl.CreateTable(group);
meta::TableSchema table;
table.table_id_ = table_id;
auto status = impl.CreateTable(table);
int new_files_cnt = 4;
int raw_files_cnt = 5;
@ -455,7 +276,7 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) {
int index_files_cnt = 7;
meta::TableFileSchema table_file;
table_file.table_id_ = group.table_id_;
table_file.table_id_ = table.table_id_;
for (auto i=0; i<new_files_cnt; ++i) {
status = impl.CreateTableFile(table_file);
@ -488,7 +309,7 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) {
ASSERT_EQ(files.size(), to_index_files_cnt);
meta::DatePartionedTableFilesSchema dated_files;
status = impl.FilesToMerge(group.table_id_, dated_files);
status = impl.FilesToMerge(table.table_id_, dated_files);
ASSERT_TRUE(status.ok());
ASSERT_EQ(dated_files[table_file.date_].size(), raw_files_cnt);