test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-02 20:09:57 +08:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master'

Browse Source
This commit is contained in:
quicksilver 2019-12-16 14:12:48 +08:00
commit 60d1da8abb
10 changed files with 337 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -9,6 +9,7 @@ Please mark all change in change log and use the issue from GitHub
## Feature
- \#343 - Add Opentracing
- \#665 - Support get/set config via CLI
## Improvement

21
core/src/sdk/grpc/ClientProxy.cpp
View File

@ -334,7 +334,7 @@ ClientProxy::ServerStatus() const {
try {
std::string dummy;
Status status = client_ptr_->Cmd(dummy, "");
Status status = client_ptr_->Cmd(dummy, "status");
return "server alive";
} catch (std::exception& ex) {
return "connection lost";
@ -456,4 +456,23 @@ ClientProxy::DropPartition(const PartitionParam& partition_param) {
}
}
Status
ClientProxy::GetConfig(const std::string& node_name, std::string& value) const {
try {
return client_ptr_->Cmd(value, "get " + node_name);
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Fail to get config: " + node_name);
}
}
Status
ClientProxy::SetConfig(const std::string& node_name, const std::string& value) const {
try {
std::string dummy;
return client_ptr_->Cmd(dummy, "set " + node_name + " " + value);
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Fail to set config: " + node_name);
}
}
} // namespace milvus

6
core/src/sdk/grpc/ClientProxy.h
View File

@ -104,6 +104,12 @@ class ClientProxy : public Connection {
Status
DropPartition(const PartitionParam& partition_param) override;
Status
GetConfig(const std::string& node_name, std::string& value) const override;
Status
SetConfig(const std::string& node_name, const std::string& value) const override;
private:
std::shared_ptr<::grpc::Channel> channel_;

26
core/src/sdk/include/MilvusApi.h
View File

@ -444,6 +444,32 @@ class Connection {
*/
virtual Status
DropPartition(const PartitionParam& param) = 0;
/**
* @brief Get config method
*
* This method is used to set config.
*
* @param node_name, config node name.
* @param value, config value.
*
* @return Indicate if this operation is successful.
*/
virtual Status
GetConfig(const std::string& node_name, std::string& value) const = 0;
/**
* @brief Set config method
*
* This method is used to set config.
*
* @param node_name, config node name.
* @param value, config value.
*
* @return Indicate if this operation is successful.
*/
virtual Status
SetConfig(const std::string& node_name, const std::string& value) const = 0;
};
} // namespace milvus

9
core/src/sdk/interface/ConnectionImpl.cpp
View File

@ -161,4 +161,13 @@ ConnectionImpl::DropPartition(const PartitionParam& param) {
return client_proxy_->DropPartition(param);
}
Status
ConnectionImpl::GetConfig(const std::string& node_name, std::string& value) const {
return client_proxy_->GetConfig(node_name, value);
}
Status
ConnectionImpl::SetConfig(const std::string& node_name, const std::string& value) const {
return client_proxy_->SetConfig(node_name, value);
}
} // namespace milvus

6
core/src/sdk/interface/ConnectionImpl.h
View File

@ -106,6 +106,12 @@ class ConnectionImpl : public Connection {
Status
DropPartition(const PartitionParam& param) override;
Status
GetConfig(const std::string& node_name, std::string& value) const override;
Status
SetConfig(const std::string& node_name, const std::string& value) const override;
private:
std::shared_ptr<ClientProxy> client_proxy_;
};

102
core/src/server/Config.cpp
View File

@ -33,7 +33,7 @@
namespace milvus {
namespace server {
constexpr uint64_t GB = 1UL << 30;
constexpr int64_t GB = 1UL << 30;
static const std::unordered_map<std::string, std::string> milvus_config_version_map({{"0.6.0", "0.1"}});
@ -400,6 +400,92 @@ Config::PrintAll() {
PrintConfigSection(CONFIG_GPU_RESOURCE);
}
Status
Config::GetConfigCli(const std::string& parent_key, const std::string& child_key, std::string& value) {
if (!ConfigNodeValid(parent_key, child_key)) {
std::string str = "Config node invalid: " + parent_key + CONFIG_NODE_DELIMITER + child_key;
return Status(SERVER_UNEXPECTED_ERROR, str);
}
return GetConfigValueInMem(parent_key, child_key, value);
}
Status
Config::SetConfigCli(const std::string& parent_key, const std::string& child_key, const std::string& value) {
if (!ConfigNodeValid(parent_key, child_key)) {
std::string str = "Config node invalid: " + parent_key + CONFIG_NODE_DELIMITER + child_key;
return Status(SERVER_UNEXPECTED_ERROR, str);
}
if (parent_key == CONFIG_SERVER) {
return Status(SERVER_UNSUPPORTED_ERROR, "Not support set server_config");
} else if (parent_key == CONFIG_DB) {
return Status(SERVER_UNSUPPORTED_ERROR, "Not support set db_config");
} else if (parent_key == CONFIG_METRIC) {
return Status(SERVER_UNSUPPORTED_ERROR, "Not support set metric_config");
} else if (parent_key == CONFIG_CACHE) {
if (child_key == CONFIG_CACHE_CPU_CACHE_CAPACITY) {
return SetCacheConfigCpuCacheCapacity(value);
} else if (child_key == CONFIG_CACHE_CPU_CACHE_THRESHOLD) {
return SetCacheConfigCpuCacheThreshold(value);
} else if (child_key == CONFIG_CACHE_CACHE_INSERT_DATA) {
return SetCacheConfigCacheInsertData(value);
}
} else if (parent_key == CONFIG_ENGINE) {
if (child_key == CONFIG_ENGINE_USE_BLAS_THRESHOLD) {
return SetEngineConfigUseBlasThreshold(value);
} else if (child_key == CONFIG_ENGINE_OMP_THREAD_NUM) {
return SetEngineConfigOmpThreadNum(value);
#ifdef MILVUS_GPU_VERSION
} else if (child_key == CONFIG_ENGINE_GPU_SEARCH_THRESHOLD) {
return SetEngineConfigGpuSearchThreshold(value);
#endif
}
#ifdef MILVUS_GPU_VERSION
} else if (parent_key == CONFIG_GPU_RESOURCE) {
if (child_key == CONFIG_GPU_RESOURCE_ENABLE) {
return SetGpuResourceConfigEnable(value);
} else if (child_key == CONFIG_GPU_RESOURCE_CACHE_CAPACITY) {
return SetGpuResourceConfigCacheCapacity(value);
} else if (child_key == CONFIG_GPU_RESOURCE_CACHE_THRESHOLD) {
return SetGpuResourceConfigCacheThreshold(value);
} else if (child_key == CONFIG_GPU_RESOURCE_SEARCH_RESOURCES) {
return SetGpuResourceConfigSearchResources(value);
} else if (child_key == CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES) {
return SetGpuResourceConfigBuildIndexResources(value);
}
#endif
} else if (parent_key == CONFIG_TRACING) {
return Status(SERVER_UNSUPPORTED_ERROR, "Not support set tracing_config");
}
}
Status
Config::HandleConfigCli(std::string& result, const std::string& cmd) {
std::vector<std::string> tokens;
std::vector<std::string> nodes;
server::StringHelpFunctions::SplitStringByDelimeter(cmd, " ", tokens);
if (tokens[0] == "get") {
if (tokens.size() != 2) {
return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd);
}
server::StringHelpFunctions::SplitStringByDelimeter(tokens[1], CONFIG_NODE_DELIMITER, nodes);
if (nodes.size() != 2) {
return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd);
}
return GetConfigCli(nodes[0], nodes[1], result);
} else if (tokens[0] == "set") {
if (tokens.size() != 3) {
return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd);
}
server::StringHelpFunctions::SplitStringByDelimeter(tokens[1], CONFIG_NODE_DELIMITER, nodes);
if (nodes.size() != 2) {
return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd);
}
return SetConfigCli(nodes[0], nodes[1], tokens[2]);
} else {
return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd);
}
}
////////////////////////////////////////////////////////////////////////////////
Status
Config::CheckConfigVersion(const std::string& value) {
@ -804,6 +890,17 @@ Config::GetConfigNode(const std::string& name) {
return GetConfigRoot().GetChild(name);
}
bool
Config::ConfigNodeValid(const std::string& parent_key, const std::string& child_key) {
if (config_map_.find(parent_key) == config_map_.end()) {
return false;
}
if (config_map_[parent_key].count(child_key) == 0) {
return false;
}
return true;
}
Status
Config::GetConfigValueInMem(const std::string& parent_key, const std::string& child_key, std::string& value) {
std::lock_guard<std::mutex> lock(mutex_);
@ -815,10 +912,11 @@ Config::GetConfigValueInMem(const std::string& parent_key, const std::string& ch
return Status(SERVER_UNEXPECTED_ERROR, "key not exist");
}
void
Status
Config::SetConfigValueInMem(const std::string& parent_key, const std::string& child_key, const std::string& value) {
std::lock_guard<std::mutex> lock(mutex_);
config_map_[parent_key][child_key] = value;
return Status::OK();
}
////////////////////////////////////////////////////////////////////////////////

12
core/src/server/Config.h
View File

@ -28,6 +28,7 @@
namespace milvus {
namespace server {
static const char* CONFIG_NODE_DELIMITER = ".";
static const char* CONFIG_VERSION = "version";
/* server config */
@ -56,6 +57,7 @@ static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0";
static const char* CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size";
static const char* CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT = "4";
static const char* CONFIG_DB_PRELOAD_TABLE = "preload_table";
static const char* CONFIG_DB_PRELOAD_TABLE_DEFAULT = "";
/* cache config */
static const char* CONFIG_CACHE = "cache_config";
@ -120,18 +122,26 @@ class Config {
ResetDefaultConfig();
void
PrintAll();
Status
HandleConfigCli(std::string& result, const std::string& cmd);
private:
ConfigNode&
GetConfigRoot();
ConfigNode&
GetConfigNode(const std::string& name);
bool
ConfigNodeValid(const std::string& parent_key, const std::string& child_key);
Status
GetConfigValueInMem(const std::string& parent_key, const std::string& child_key, std::string& value);
void
Status
SetConfigValueInMem(const std::string& parent_key, const std::string& child_key, const std::string& value);
void
PrintConfigSection(const std::string& config_node_name);
Status
GetConfigCli(const std::string& parent_key, const std::string& child_key, std::string& value);
Status
SetConfigCli(const std::string& parent_key, const std::string& child_key, const std::string& value);
///////////////////////////////////////////////////////////////////////////
Status

10
core/src/server/delivery/request/CmdRequest.cpp
View File

@ -38,9 +38,12 @@ Status
CmdRequest::OnExecute() {
std::string hdr = "CmdRequest(cmd=" + cmd_ + ")";
TimeRecorderAuto rc(hdr);
Status stat = Status::OK();
if (cmd_ == "version") {
result_ = MILVUS_VERSION;
} else if (cmd_ == "status") {
result_ = "OK";
} else if (cmd_ == "tasktable") {
result_ = scheduler::ResMgrInst::GetInstance()->DumpTaskTables();
} else if (cmd_ == "mode") {
@ -49,11 +52,14 @@ CmdRequest::OnExecute() {
#else
result_ = "CPU";
#endif
} else if (cmd_.substr(0, 3) == "set" || cmd_.substr(0, 3) == "get") {
server::Config& config = server::Config::GetInstance();
stat = config.HandleConfigCli(result_, cmd_);
} else {
result_ = "OK";
result_ = "Unknown command";
}
return Status::OK();
return stat;
}
} // namespace server

150
core/unittest/server/test_config.cpp
View File

@ -36,6 +36,8 @@ static constexpr uint64_t GB = MB * 1024;
} // namespace
namespace ms = milvus::server;
TEST_F(ConfigTest, CONFIG_TEST) {
milvus::server::ConfigMgr* config_mgr = milvus::server::YamlConfigMgr::GetInstance();
@ -308,6 +310,154 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) {
#endif
}
std::string gen_get_command(const std::string& parent_node, const std::string& child_node) {
std::string cmd = "get " + parent_node + ms::CONFIG_NODE_DELIMITER + child_node;
return cmd;
}
std::string gen_set_command(const std::string& parent_node, const std::string& child_node, const std::string& value) {
std::string cmd = "set " + parent_node + ms::CONFIG_NODE_DELIMITER + child_node + " " + value;
return cmd;
}
TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) {
std::string config_path(CONFIG_PATH);
milvus::server::Config& config = milvus::server::Config::GetInstance();
milvus::Status s;
std::string get_cmd, set_cmd;
std::string result, dummy;
/* server config */
std::string server_addr = "192.168.1.155";
get_cmd = gen_get_command(ms::CONFIG_SERVER, ms::CONFIG_SERVER_ADDRESS);
set_cmd = gen_set_command(ms::CONFIG_SERVER, ms::CONFIG_SERVER_ADDRESS, server_addr);
s = config.HandleConfigCli(dummy, set_cmd);
ASSERT_FALSE(s.ok());
s = config.HandleConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
/* db config */
std::string db_primary_path = "/home/zilliz";
get_cmd = gen_get_command(ms::CONFIG_DB, ms::CONFIG_DB_PRIMARY_PATH);
set_cmd = gen_set_command(ms::CONFIG_DB, ms::CONFIG_DB_PRIMARY_PATH, db_primary_path);
s = config.HandleConfigCli(dummy, set_cmd);
ASSERT_FALSE(s.ok());
s = config.HandleConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
/* metric config */
std::string metric_enable_monitor = "false";
get_cmd = gen_get_command(ms::CONFIG_METRIC, ms::CONFIG_METRIC_ENABLE_MONITOR);
set_cmd = gen_set_command(ms::CONFIG_METRIC, ms::CONFIG_METRIC_ENABLE_MONITOR, metric_enable_monitor);
s = config.HandleConfigCli(dummy, set_cmd);
ASSERT_FALSE(s.ok());
s = config.HandleConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
/* cache config */
std::string cache_cpu_cache_capacity = "5";
get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_CAPACITY);
set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_CAPACITY, cache_cpu_cache_capacity);
s = config.HandleConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.HandleConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(result == cache_cpu_cache_capacity);
std::string cache_cpu_cache_threshold = "0.1";
get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_THRESHOLD);
set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_THRESHOLD, cache_cpu_cache_threshold);
s = config.HandleConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.HandleConfigCli(result, get_cmd);
ASSERT_TRUE(result == cache_cpu_cache_threshold);
std::string cache_insert_data = "true";
get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CACHE_INSERT_DATA);
set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CACHE_INSERT_DATA, cache_insert_data);
s = config.HandleConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.HandleConfigCli(result, get_cmd);
ASSERT_TRUE(result == cache_insert_data);
/* engine config */
std::string engine_use_blas_threshold = "50";
get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_USE_BLAS_THRESHOLD);
set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_USE_BLAS_THRESHOLD, engine_use_blas_threshold);
s = config.HandleConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.HandleConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(result == engine_use_blas_threshold);
std::string engine_omp_thread_num = "8";
get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_OMP_THREAD_NUM);
set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_OMP_THREAD_NUM, engine_omp_thread_num);
s = config.HandleConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.HandleConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(result == engine_omp_thread_num);
#ifdef MILVUS_GPU_VERSION
std::string engine_gpu_search_threshold = "800";
get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD);
set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, engine_gpu_search_threshold);
s = config.HandleConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.HandleConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(result == engine_gpu_search_threshold);
/* gpu resource config */
std::string resource_enable_gpu = "true";
get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_ENABLE);
set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_ENABLE, resource_enable_gpu);
s = config.HandleConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.HandleConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(result == resource_enable_gpu);
std::string gpu_cache_capacity = "1";
get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_CAPACITY);
set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_CAPACITY, gpu_cache_capacity);
s = config.HandleConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.HandleConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(result == gpu_cache_capacity);
std::string gpu_cache_threshold = "0.2";
get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_THRESHOLD);
set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, gpu_cache_threshold);
s = config.HandleConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.HandleConfigCli(result, get_cmd);
ASSERT_TRUE(result == gpu_cache_threshold);
std::string search_resources = "gpu0";
get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES);
set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, search_resources);
s = config.HandleConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.HandleConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(result == search_resources);
std::string build_index_resources = "gpu0";
get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES);
set_cmd =
gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, build_index_resources);
s = config.HandleConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.HandleConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(result == build_index_resources);
#endif
}
TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) {
std::string config_path(CONFIG_PATH);
milvus::server::Config& config = milvus::server::Config::GetInstance();