format code by clang-tidy

Former-commit-id: f9e606566555ad381b8e95e1a4e2572fc61678f0
This commit is contained in:
starlord 2019-09-28 12:36:14 +08:00
parent 464a341523
commit aabd91a156
38 changed files with 532 additions and 229 deletions

View File

@ -23,3 +23,5 @@ AccessModifierOffset: -3
AlwaysBreakAfterReturnType: All AlwaysBreakAfterReturnType: All
AllowShortBlocksOnASingleLine: false AllowShortBlocksOnASingleLine: false
AllowShortFunctionsOnASingleLine: false AllowShortFunctionsOnASingleLine: false
AllowShortIfStatementsOnASingleLine: false
AlignTrailingComments: true

View File

@ -47,7 +47,7 @@ class YamlConfigMgr : public ConfigMgr {
SetConfigValue(const YAML::Node& node, const std::string& key, ConfigNode& config); SetConfigValue(const YAML::Node& node, const std::string& key, ConfigNode& config);
bool bool
SetChildConfig(const YAML::Node& node, const std::string& name, ConfigNode& config); SetChildConfig(const YAML::Node& node, const std::string& child_name, ConfigNode& config);
bool bool
SetSequence(const YAML::Node& node, const std::string& child_name, ConfigNode& config); SetSequence(const YAML::Node& node, const std::string& child_name, ConfigNode& config);

View File

@ -46,12 +46,14 @@ struct Cfg {
const int64_t &k, const int64_t &k,
const int64_t &gpu_id, const int64_t &gpu_id,
METRICTYPE type) METRICTYPE type)
: d(dim), k(k), gpu_id(gpu_id), metric_type(type) {} : metric_type(type), k(k), gpu_id(gpu_id), d(dim) {}
Cfg() = default; Cfg() = default;
virtual bool virtual bool
CheckValid(){}; CheckValid(){
return true;
};
}; };
using Config = std::shared_ptr<Cfg>; using Config = std::shared_ptr<Cfg>;

View File

@ -127,10 +127,10 @@ class Dataset {
//} //}
private: private:
SchemaPtr array_schema_;
SchemaPtr tensor_schema_;
std::vector<ArrayPtr> array_; std::vector<ArrayPtr> array_;
SchemaPtr array_schema_;
std::vector<TensorPtr> tensor_; std::vector<TensorPtr> tensor_;
SchemaPtr tensor_schema_;
//Config meta_; //Config meta_;
}; };

View File

@ -25,9 +25,14 @@ namespace zilliz {
namespace knowhere { namespace knowhere {
faiss::MetricType GetMetricType(METRICTYPE &type) { faiss::MetricType GetMetricType(METRICTYPE &type) {
if (type == METRICTYPE::L2) return faiss::METRIC_L2; if (type == METRICTYPE::L2) {
if (type == METRICTYPE::IP) return faiss::METRIC_INNER_PRODUCT; return faiss::METRIC_L2;
if (type == METRICTYPE::INVALID) KNOWHERE_THROW_MSG("Metric type is invalid"); }
if (type == METRICTYPE::IP) {
return faiss::METRIC_INNER_PRODUCT;
}
KNOWHERE_THROW_MSG("Metric type is invalid");
} }

View File

@ -52,12 +52,15 @@ struct IVFCfg : public Cfg {
const int64_t &nlist, const int64_t &nlist,
const int64_t &nprobe, const int64_t &nprobe,
METRICTYPE type) METRICTYPE type)
: nlist(nlist), nprobe(nprobe), Cfg(dim, k, gpu_id, type) {} : Cfg(dim, k, gpu_id, type), nlist(nlist), nprobe(nprobe) {
}
IVFCfg() = default; IVFCfg() = default;
bool bool
CheckValid() override {}; CheckValid() override {
return true;
};
}; };
using IVFConfig = std::shared_ptr<IVFCfg>; using IVFConfig = std::shared_ptr<IVFCfg>;
@ -72,12 +75,15 @@ struct IVFSQCfg : public IVFCfg {
const int64_t &nprobe, const int64_t &nprobe,
const int64_t &nbits, const int64_t &nbits,
METRICTYPE type) METRICTYPE type)
: nbits(nbits), IVFCfg(dim, k, gpu_id, nlist, nprobe, type) {} : IVFCfg(dim, k, gpu_id, nlist, nprobe, type), nbits(nbits) {
}
IVFSQCfg() = default; IVFSQCfg() = default;
bool bool
CheckValid() override {}; CheckValid() override {
return true;
};
}; };
using IVFSQConfig = std::shared_ptr<IVFSQCfg>; using IVFSQConfig = std::shared_ptr<IVFSQCfg>;
@ -98,12 +104,15 @@ struct IVFPQCfg : public IVFCfg {
const int64_t &nbits, const int64_t &nbits,
const int64_t &m, const int64_t &m,
METRICTYPE type) METRICTYPE type)
: nbits(nbits), m(m), IVFCfg(dim, k, gpu_id, nlist, nprobe, type) {} : IVFCfg(dim, k, gpu_id, nlist, nprobe, type), m(m), nbits(nbits) {
}
IVFPQCfg() = default; IVFPQCfg() = default;
bool bool
CheckValid() override {}; CheckValid() override {
return true;
};
}; };
using IVFPQConfig = std::shared_ptr<IVFPQCfg>; using IVFPQConfig = std::shared_ptr<IVFPQCfg>;
@ -123,13 +132,17 @@ struct NSGCfg : public IVFCfg {
const int64_t &out_degree, const int64_t &out_degree,
const int64_t &candidate_size, const int64_t &candidate_size,
METRICTYPE type) METRICTYPE type)
: knng(knng), search_length(search_length), out_degree(out_degree), candidate_pool_size(candidate_size), : IVFCfg(dim, k, gpu_id, nlist, nprobe, type),
IVFCfg(dim, k, gpu_id, nlist, nprobe, type) {} knng(knng), search_length(search_length),
out_degree(out_degree), candidate_pool_size(candidate_size) {
}
NSGCfg() = default; NSGCfg() = default;
bool bool
CheckValid() override {}; CheckValid() override {
return true;
};
}; };
using NSGConfig = std::shared_ptr<NSGCfg>; using NSGConfig = std::shared_ptr<NSGCfg>;

View File

@ -81,7 +81,7 @@ void NsgIndex::Build_with_ids(size_t nb, const float *data, const long *ids, con
//>> Debug code //>> Debug code
/// ///
int total_degree = 0; int total_degree = 0;
for (int i = 0; i < ntotal; ++i) { for (size_t i = 0; i < ntotal; ++i) {
total_degree += nsg[i].size(); total_degree += nsg[i].size();
} }
@ -172,7 +172,7 @@ void NsgIndex::GetNeighbors(const float *query,
for (size_t i = 0; i < init_ids.size(); ++i) { for (size_t i = 0; i < init_ids.size(); ++i) {
node_t id = init_ids[i]; node_t id = init_ids[i];
if (id >= ntotal) { if (id >= static_cast<node_t>(ntotal)) {
KNOWHERE_THROW_MSG("Build Index Error, id > ntotal"); KNOWHERE_THROW_MSG("Build Index Error, id > ntotal");
continue; continue;
} }
@ -262,7 +262,7 @@ void NsgIndex::GetNeighbors(const float *query, std::vector<Neighbor> &resset, s
for (size_t i = 0; i < init_ids.size(); ++i) { for (size_t i = 0; i < init_ids.size(); ++i) {
node_t id = init_ids[i]; node_t id = init_ids[i];
if (id >= ntotal) { if (id >= static_cast<node_t>(ntotal)) {
KNOWHERE_THROW_MSG("Build Index Error, id > ntotal"); KNOWHERE_THROW_MSG("Build Index Error, id > ntotal");
continue; continue;
} }
@ -350,7 +350,7 @@ void NsgIndex::GetNeighbors(const float *query,
node_t id = init_ids[i]; node_t id = init_ids[i];
//assert(id < ntotal); //assert(id < ntotal);
if (id >= ntotal) { if (id >= static_cast<node_t>(ntotal)) {
KNOWHERE_THROW_MSG("Build Index Error, id > ntotal"); KNOWHERE_THROW_MSG("Build Index Error, id > ntotal");
continue; continue;
} }
@ -461,7 +461,7 @@ void NsgIndex::Link() {
//} //}
///// /////
for (int i = 0; i < ntotal; ++i) { for (size_t i = 0; i < ntotal; ++i) {
nsg[i].shrink_to_fit(); nsg[i].shrink_to_fit();
} }
} }
@ -483,7 +483,9 @@ void NsgIndex::SyncPrune(size_t n,
unsigned cursor = 0; unsigned cursor = 0;
std::sort(pool.begin(), pool.end()); std::sort(pool.begin(), pool.end());
std::vector<Neighbor> result; std::vector<Neighbor> result;
if (pool[cursor].id == n) cursor++; if (pool[cursor].id == static_cast<node_t>(n)) {
cursor++;
}
result.push_back(pool[cursor]); // init result with nearest neighbor result.push_back(pool[cursor]); // init result with nearest neighbor
SelectEdge(cursor, pool, result, true); SelectEdge(cursor, pool, result, true);
@ -518,7 +520,7 @@ void NsgIndex::InterInsert(unsigned n, std::vector<std::mutex> &mutex_vec, float
int duplicate = false; int duplicate = false;
{ {
LockGuard lk(mutex_vec[current_neighbor]); LockGuard lk(mutex_vec[current_neighbor]);
for (int j = 0; j < out_degree; ++j) { for (size_t j = 0; j < out_degree; ++j) {
if (nsn_dist_pool[j] == -1) break; if (nsn_dist_pool[j] == -1) break;
// 保证至少有一条边能连回来 // 保证至少有一条边能连回来
@ -551,14 +553,14 @@ void NsgIndex::InterInsert(unsigned n, std::vector<std::mutex> &mutex_vec, float
{ {
LockGuard lk(mutex_vec[current_neighbor]); LockGuard lk(mutex_vec[current_neighbor]);
for (int j = 0; j < result.size(); ++j) { for (size_t j = 0; j < result.size(); ++j) {
nsn_id_pool[j] = result[j].id; nsn_id_pool[j] = result[j].id;
nsn_dist_pool[j] = result[j].distance; nsn_dist_pool[j] = result[j].distance;
} }
} }
} else { } else {
LockGuard lk(mutex_vec[current_neighbor]); LockGuard lk(mutex_vec[current_neighbor]);
for (int j = 0; j < out_degree; ++j) { for (size_t j = 0; j < out_degree; ++j) {
if (nsn_dist_pool[j] == -1) { if (nsn_dist_pool[j] == -1) {
nsn_id_pool.push_back(current_as_neighbor.id); nsn_id_pool.push_back(current_as_neighbor.id);
nsn_dist_pool[j] = current_as_neighbor.distance; nsn_dist_pool[j] = current_as_neighbor.distance;
@ -605,9 +607,11 @@ void NsgIndex::CheckConnectivity() {
boost::dynamic_bitset<> has_linked{ntotal, 0}; boost::dynamic_bitset<> has_linked{ntotal, 0};
int64_t linked_count = 0; int64_t linked_count = 0;
while (linked_count < ntotal) { while (linked_count < static_cast<int64_t>(ntotal)) {
DFS(root, has_linked, linked_count); DFS(root, has_linked, linked_count);
if (linked_count >= ntotal) break; if (linked_count >= static_cast<int64_t>(ntotal)) {
break;
}
FindUnconnectedNode(has_linked, root); FindUnconnectedNode(has_linked, root);
} }
} }
@ -697,7 +701,7 @@ void NsgIndex::Search(const float *query,
} else{ } else{
//#pragma omp parallel for schedule(dynamic, 50) //#pragma omp parallel for schedule(dynamic, 50)
#pragma omp parallel for #pragma omp parallel for
for (int i = 0; i < nq; ++i) { for (unsigned int i = 0; i < nq; ++i) {
// TODO(linxj): when to use openmp // TODO(linxj): when to use openmp
auto single_query = query + i * dim; auto single_query = query + i * dim;
GetNeighbors(single_query, resset[i], nsg, &params); GetNeighbors(single_query, resset[i], nsg, &params);
@ -705,8 +709,8 @@ void NsgIndex::Search(const float *query,
} }
rc.ElapseFromBegin("cost"); rc.ElapseFromBegin("cost");
for (int i = 0; i < nq; ++i) { for (unsigned int i = 0; i < nq; ++i) {
for (int j = 0; j < k; ++j) { for (unsigned int j = 0; j < k; ++j) {
//ids[i * k + j] = resset[i][j].id; //ids[i * k + j] = resset[i][j].id;
// Fix(linxj): bug, reset[i][j] out of range // Fix(linxj): bug, reset[i][j] out of range

View File

@ -29,7 +29,7 @@ namespace algo {
// TODO: impl search && insert && return insert pos. why not just find and swap? // TODO: impl search && insert && return insert pos. why not just find and swap?
int InsertIntoPool(Neighbor *addr, unsigned K, Neighbor nn) { int InsertIntoPool(Neighbor *addr, unsigned K, Neighbor nn) {
//>> Fix: Add assert //>> Fix: Add assert
for (int i = 0; i < K; ++i) { for (unsigned int i = 0; i < K; ++i) {
assert(addr[i].id != nn.id); assert(addr[i].id != nn.id);
} }

View File

@ -38,8 +38,8 @@ void DataGen::Generate(const int &dim, const int &nb, const int &nq) {
this->dim = dim; this->dim = dim;
GenAll(dim, nb, xb, ids, nq, xq); GenAll(dim, nb, xb, ids, nq, xq);
assert(xb.size() == dim * nb); assert(xb.size() == (size_t)dim * nb);
assert(xq.size() == dim * nq); assert(xq.size() == (size_t)dim * nq);
base_dataset = generate_dataset(nb, dim, xb.data(), ids.data()); base_dataset = generate_dataset(nb, dim, xb.data(), ids.data());
query_dataset = generate_query_dataset(nq, dim, xq.data()); query_dataset = generate_query_dataset(nq, dim, xq.data());
@ -47,7 +47,7 @@ void DataGen::Generate(const int &dim, const int &nb, const int &nq) {
} }
zilliz::knowhere::DatasetPtr DataGen::GenQuery(const int &nq) { zilliz::knowhere::DatasetPtr DataGen::GenQuery(const int &nq) {
xq.resize(nq * dim); xq.resize(nq * dim);
for (size_t i = 0; i < nq * dim; ++i) { for (int i = 0; i < nq * dim; ++i) {
xq[i] = xb[i]; xq[i] = xb[i];
} }
return generate_query_dataset(nq, dim, xq.data()); return generate_query_dataset(nq, dim, xq.data());
@ -72,7 +72,7 @@ void GenAll(const int64_t &dim,
const int64_t &nq, const int64_t &nq,
float *xq) { float *xq) {
GenBase(dim, nb, xb, ids); GenBase(dim, nb, xb, ids);
for (size_t i = 0; i < nq * dim; ++i) { for (int64_t i = 0; i < nq * dim; ++i) {
xq[i] = xb[i]; xq[i] = xb[i];
} }
} }

View File

@ -638,7 +638,8 @@ DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, const m
ENGINE_LOG_DEBUG << "Merging file " << file_schema.file_id_; ENGINE_LOG_DEBUG << "Merging file " << file_schema.file_id_;
index_size = index->Size(); index_size = index->Size();
if (index_size >= file_schema.index_file_size_) break; if (index_size >= file_schema.index_file_size_)
break;
} }
// step 3: serialize to disk // step 3: serialize to disk

View File

@ -53,9 +53,9 @@ struct ArchiveConf {
private: private:
void void
ParseCritirias(const std::string& type); ParseCritirias(const std::string& criterias);
void void
ParseType(const std::string& criterias); ParseType(const std::string& type);
std::string type_; std::string type_;
CriteriaT criterias_; CriteriaT criterias_;

View File

@ -143,14 +143,14 @@ GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file
if (boost::filesystem::exists(file_path)) { if (boost::filesystem::exists(file_path)) {
table_file.location_ = file_path; table_file.location_ = file_path;
return Status::OK(); return Status::OK();
} else { }
for (auto& path : options.slave_paths_) {
parent_path = ConstructParentFolder(path, table_file); for (auto& path : options.slave_paths_) {
file_path = parent_path + "/" + table_file.file_id_; parent_path = ConstructParentFolder(path, table_file);
if (boost::filesystem::exists(file_path)) { file_path = parent_path + "/" + table_file.file_id_;
table_file.location_ = file_path; if (boost::filesystem::exists(file_path)) {
return Status::OK(); table_file.location_ = file_path;
} return Status::OK();
} }
} }

View File

@ -342,7 +342,8 @@ Status
ExecutionEngineImpl::Init() { ExecutionEngineImpl::Init() {
server::Config& config = server::Config::GetInstance(); server::Config& config = server::Config::GetInstance();
Status s = config.GetDBConfigBuildIndexGPU(gpu_num_); Status s = config.GetDBConfigBuildIndexGPU(gpu_num_);
if (!s.ok()) return s; if (!s.ok())
return s;
return Status::OK(); return Status::OK();
} }

View File

@ -39,12 +39,12 @@ main(int argc, char* argv[]) {
std::cout << std::endl << "Welcome to use Milvus by Zilliz!" << std::endl; std::cout << std::endl << "Welcome to use Milvus by Zilliz!" << std::endl;
std::cout << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << " built at " << BUILD_TIME << std::endl; std::cout << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << " built at " << BUILD_TIME << std::endl;
static struct option long_options[] = {{"conf_file", required_argument, 0, 'c'}, static struct option long_options[] = {{"conf_file", required_argument, nullptr, 'c'},
{"log_conf_file", required_argument, 0, 'l'}, {"log_conf_file", required_argument, nullptr, 'l'},
{"help", no_argument, 0, 'h'}, {"help", no_argument, nullptr, 'h'},
{"daemon", no_argument, 0, 'd'}, {"daemon", no_argument, nullptr, 'd'},
{"pid_file", required_argument, 0, 'p'}, {"pid_file", required_argument, nullptr, 'p'},
{NULL, 0, 0, 0}}; {nullptr, 0, nullptr, 0}};
int option_index = 0; int option_index = 0;
int64_t start_daemonized = 0; int64_t start_daemonized = 0;

View File

@ -33,13 +33,20 @@ PrometheusMetrics::Init() {
try { try {
Config& config = Config::GetInstance(); Config& config = Config::GetInstance();
Status s = config.GetMetricConfigEnableMonitor(startup_); Status s = config.GetMetricConfigEnableMonitor(startup_);
if (!s.ok()) return s.code(); if (!s.ok()) {
if (!startup_) return SERVER_SUCCESS; return s.code();
}
if (!startup_) {
return SERVER_SUCCESS;
}
// Following should be read from config file. // Following should be read from config file.
std::string bind_address; std::string bind_address;
s = config.GetMetricConfigPrometheusPort(bind_address); s = config.GetMetricConfigPrometheusPort(bind_address);
if (!s.ok()) return s.code(); if (!s.ok()) {
return s.code();
}
const std::string uri = std::string("/tmp/metrics"); const std::string uri = std::string("/tmp/metrics");
const std::size_t num_threads = 2; const std::size_t num_threads = 2;
@ -58,21 +65,30 @@ PrometheusMetrics::Init() {
void void
PrometheusMetrics::CPUUsagePercentSet() { PrometheusMetrics::CPUUsagePercentSet() {
if (!startup_) return; if (!startup_) {
return;
}
double usage_percent = server::SystemInfo::GetInstance().CPUPercent(); double usage_percent = server::SystemInfo::GetInstance().CPUPercent();
CPU_usage_percent_.Set(usage_percent); CPU_usage_percent_.Set(usage_percent);
} }
void void
PrometheusMetrics::RAMUsagePercentSet() { PrometheusMetrics::RAMUsagePercentSet() {
if (!startup_) return; if (!startup_) {
return;
}
double usage_percent = server::SystemInfo::GetInstance().MemoryPercent(); double usage_percent = server::SystemInfo::GetInstance().MemoryPercent();
RAM_usage_percent_.Set(usage_percent); RAM_usage_percent_.Set(usage_percent);
} }
void void
PrometheusMetrics::GPUPercentGaugeSet() { PrometheusMetrics::GPUPercentGaugeSet() {
if (!startup_) return; if (!startup_) {
return;
}
int numDevice = server::SystemInfo::GetInstance().num_device(); int numDevice = server::SystemInfo::GetInstance().num_device();
std::vector<uint64_t> used_total = server::SystemInfo::GetInstance().GPUMemoryTotal(); std::vector<uint64_t> used_total = server::SystemInfo::GetInstance().GPUMemoryTotal();
std::vector<uint64_t> used_memory = server::SystemInfo::GetInstance().GPUMemoryUsed(); std::vector<uint64_t> used_memory = server::SystemInfo::GetInstance().GPUMemoryUsed();
@ -86,7 +102,10 @@ PrometheusMetrics::GPUPercentGaugeSet() {
void void
PrometheusMetrics::GPUMemoryUsageGaugeSet() { PrometheusMetrics::GPUMemoryUsageGaugeSet() {
if (!startup_) return; if (!startup_) {
return;
}
std::vector<uint64_t> values = server::SystemInfo::GetInstance().GPUMemoryUsed(); std::vector<uint64_t> values = server::SystemInfo::GetInstance().GPUMemoryUsed();
constexpr uint64_t MtoB = 1024 * 1024; constexpr uint64_t MtoB = 1024 * 1024;
int numDevice = server::SystemInfo::GetInstance().num_device(); int numDevice = server::SystemInfo::GetInstance().num_device();
@ -100,7 +119,9 @@ PrometheusMetrics::GPUMemoryUsageGaugeSet() {
void void
PrometheusMetrics::AddVectorsPerSecondGaugeSet(int num_vector, int dim, double time) { PrometheusMetrics::AddVectorsPerSecondGaugeSet(int num_vector, int dim, double time) {
// MB/s // MB/s
if (!startup_) return; if (!startup_) {
return;
}
int64_t MtoB = 1024 * 1024; int64_t MtoB = 1024 * 1024;
int64_t size = num_vector * dim * 4; int64_t size = num_vector * dim * 4;
@ -109,7 +130,10 @@ PrometheusMetrics::AddVectorsPerSecondGaugeSet(int num_vector, int dim, double t
void void
PrometheusMetrics::QueryIndexTypePerSecondSet(std::string type, double value) { PrometheusMetrics::QueryIndexTypePerSecondSet(std::string type, double value) {
if (!startup_) return; if (!startup_) {
return;
}
if (type == "IVF") { if (type == "IVF") {
query_index_IVF_type_per_second_gauge_.Set(value); query_index_IVF_type_per_second_gauge_.Set(value);
} else if (type == "IDMap") { } else if (type == "IDMap") {
@ -119,19 +143,27 @@ PrometheusMetrics::QueryIndexTypePerSecondSet(std::string type, double value) {
void void
PrometheusMetrics::ConnectionGaugeIncrement() { PrometheusMetrics::ConnectionGaugeIncrement() {
if (!startup_) return; if (!startup_) {
return;
}
connection_gauge_.Increment(); connection_gauge_.Increment();
} }
void void
PrometheusMetrics::ConnectionGaugeDecrement() { PrometheusMetrics::ConnectionGaugeDecrement() {
if (!startup_) return; if (!startup_) {
return;
}
connection_gauge_.Decrement(); connection_gauge_.Decrement();
} }
void void
PrometheusMetrics::OctetsSet() { PrometheusMetrics::OctetsSet() {
if (!startup_) return; if (!startup_) {
return;
}
// get old stats and reset them // get old stats and reset them
uint64_t old_inoctets = SystemInfo::GetInstance().get_inoctets(); uint64_t old_inoctets = SystemInfo::GetInstance().get_inoctets();
@ -147,14 +179,19 @@ PrometheusMetrics::OctetsSet() {
auto now_time = std::chrono::system_clock::now(); auto now_time = std::chrono::system_clock::now();
auto total_microsecond = METRICS_MICROSECONDS(old_time, now_time); auto total_microsecond = METRICS_MICROSECONDS(old_time, now_time);
auto total_second = total_microsecond * micro_to_second; auto total_second = total_microsecond * micro_to_second;
if (total_second == 0) return; if (total_second == 0) {
return;
}
inoctets_gauge_.Set((in_and_out_octets.first - old_inoctets) / total_second); inoctets_gauge_.Set((in_and_out_octets.first - old_inoctets) / total_second);
outoctets_gauge_.Set((in_and_out_octets.second - old_outoctets) / total_second); outoctets_gauge_.Set((in_and_out_octets.second - old_outoctets) / total_second);
} }
void void
PrometheusMetrics::CPUCoreUsagePercentSet() { PrometheusMetrics::CPUCoreUsagePercentSet() {
if (!startup_) return; if (!startup_) {
return;
}
std::vector<double> cpu_core_percent = server::SystemInfo::GetInstance().CPUCorePercent(); std::vector<double> cpu_core_percent = server::SystemInfo::GetInstance().CPUCorePercent();
@ -166,7 +203,9 @@ PrometheusMetrics::CPUCoreUsagePercentSet() {
void void
PrometheusMetrics::GPUTemperature() { PrometheusMetrics::GPUTemperature() {
if (!startup_) return; if (!startup_) {
return;
}
std::vector<uint64_t> GPU_temperatures = server::SystemInfo::GetInstance().GPUTemperature(); std::vector<uint64_t> GPU_temperatures = server::SystemInfo::GetInstance().GPUTemperature();
@ -178,7 +217,9 @@ PrometheusMetrics::GPUTemperature() {
void void
PrometheusMetrics::CPUTemperature() { PrometheusMetrics::CPUTemperature() {
if (!startup_) return; if (!startup_) {
return;
}
std::vector<float> CPU_temperatures = server::SystemInfo::GetInstance().CPUTemperature(); std::vector<float> CPU_temperatures = server::SystemInfo::GetInstance().CPUTemperature();

View File

@ -31,7 +31,8 @@ namespace server {
void void
SystemInfo::Init() { SystemInfo::Init() {
if (initialized_) return; if (initialized_)
return;
initialized_ = true; initialized_ = true;
@ -45,7 +46,8 @@ SystemInfo::Init() {
file = fopen("/proc/cpuinfo", "r"); file = fopen("/proc/cpuinfo", "r");
num_processors_ = 0; num_processors_ = 0;
while (fgets(line, 128, file) != NULL) { while (fgets(line, 128, file) != NULL) {
if (strncmp(line, "processor", 9) == 0) num_processors_++; if (strncmp(line, "processor", 9) == 0)
num_processors_++;
if (strncmp(line, "physical", 8) == 0) { if (strncmp(line, "physical", 8) == 0) {
num_physical_processors_ = ParseLine(line); num_physical_processors_ = ParseLine(line);
} }
@ -116,7 +118,8 @@ SystemInfo::GetProcessUsedMemory() {
double double
SystemInfo::MemoryPercent() { SystemInfo::MemoryPercent() {
if (!initialized_) Init(); if (!initialized_)
Init();
return (double)(GetProcessUsedMemory() * 100) / (double)total_ram_; return (double)(GetProcessUsedMemory() * 100) / (double)total_ram_;
} }
@ -171,7 +174,8 @@ SystemInfo::getTotalCpuTime(std::vector<uint64_t>& work_time_array) {
double double
SystemInfo::CPUPercent() { SystemInfo::CPUPercent() {
if (!initialized_) Init(); if (!initialized_)
Init();
struct tms time_sample; struct tms time_sample;
clock_t now; clock_t now;
double percent; double percent;
@ -195,7 +199,8 @@ SystemInfo::CPUPercent() {
std::vector<uint64_t> std::vector<uint64_t>
SystemInfo::GPUMemoryTotal() { SystemInfo::GPUMemoryTotal() {
// get GPU usage percent // get GPU usage percent
if (!initialized_) Init(); if (!initialized_)
Init();
std::vector<uint64_t> result; std::vector<uint64_t> result;
nvmlMemory_t nvmlMemory; nvmlMemory_t nvmlMemory;
for (int i = 0; i < num_device_; ++i) { for (int i = 0; i < num_device_; ++i) {
@ -209,7 +214,8 @@ SystemInfo::GPUMemoryTotal() {
std::vector<uint64_t> std::vector<uint64_t>
SystemInfo::GPUTemperature() { SystemInfo::GPUTemperature() {
if (!initialized_) Init(); if (!initialized_)
Init();
std::vector<uint64_t> result; std::vector<uint64_t> result;
for (int i = 0; i < num_device_; i++) { for (int i = 0; i < num_device_; i++) {
nvmlDevice_t device; nvmlDevice_t device;
@ -241,7 +247,8 @@ SystemInfo::CPUTemperature() {
std::vector<uint64_t> std::vector<uint64_t>
SystemInfo::GPUMemoryUsed() { SystemInfo::GPUMemoryUsed() {
// get GPU memory used // get GPU memory used
if (!initialized_) Init(); if (!initialized_)
Init();
std::vector<uint64_t> result; std::vector<uint64_t> result;
nvmlMemory_t nvmlMemory; nvmlMemory_t nvmlMemory;

View File

@ -48,7 +48,7 @@ class ResourceMgr {
Add(ResourcePtr&& resource); Add(ResourcePtr&& resource);
bool bool
Connect(const std::string& res1, const std::string& res2, Connection& connection); Connect(const std::string& name1, const std::string& name2, Connection& connection);
void void
Clear(); Clear();

View File

@ -30,7 +30,8 @@ get_neighbours(const ResourcePtr& self) {
std::vector<ResourcePtr> neighbours; std::vector<ResourcePtr> neighbours;
for (auto& neighbour_node : self->GetNeighbours()) { for (auto& neighbour_node : self->GetNeighbours()) {
auto node = neighbour_node.neighbour_node.lock(); auto node = neighbour_node.neighbour_node.lock();
if (not node) continue; if (not node)
continue;
auto resource = std::static_pointer_cast<Resource>(node); auto resource = std::static_pointer_cast<Resource>(node);
// if (not resource->HasExecutor()) continue; // if (not resource->HasExecutor()) continue;
@ -45,7 +46,8 @@ get_neighbours_with_connetion(const ResourcePtr& self) {
std::vector<std::pair<ResourcePtr, Connection>> neighbours; std::vector<std::pair<ResourcePtr, Connection>> neighbours;
for (auto& neighbour_node : self->GetNeighbours()) { for (auto& neighbour_node : self->GetNeighbours()) {
auto node = neighbour_node.neighbour_node.lock(); auto node = neighbour_node.neighbour_node.lock();
if (not node) continue; if (not node)
continue;
auto resource = std::static_pointer_cast<Resource>(node); auto resource = std::static_pointer_cast<Resource>(node);
// if (not resource->HasExecutor()) continue; // if (not resource->HasExecutor()) continue;

View File

@ -92,7 +92,8 @@ uint64_t
Resource::NumOfTaskToExec() { Resource::NumOfTaskToExec() {
uint64_t count = 0; uint64_t count = 0;
for (auto& task : task_table_) { for (auto& task : task_table_) {
if (task->state == TaskTableItemState::LOADED) ++count; if (task->state == TaskTableItemState::LOADED)
++count;
} }
return count; return count;
} }
@ -102,7 +103,8 @@ Resource::pick_task_load() {
auto indexes = task_table_.PickToLoad(10); auto indexes = task_table_.PickToLoad(10);
for (auto index : indexes) { for (auto index : indexes) {
// try to set one task loading, then return // try to set one task loading, then return
if (task_table_.Load(index)) return task_table_.Get(index); if (task_table_.Load(index))
return task_table_.Get(index);
// else try next // else try next
} }
return nullptr; return nullptr;
@ -113,7 +115,8 @@ Resource::pick_task_execute() {
auto indexes = task_table_.PickToExecute(3); auto indexes = task_table_.PickToExecute(3);
for (auto index : indexes) { for (auto index : indexes) {
// try to set one task executing, then return // try to set one task executing, then return
if (task_table_.Execute(index)) return task_table_.Get(index); if (task_table_.Execute(index))
return task_table_.Get(index);
// else try next // else try next
} }
return nullptr; return nullptr;

View File

@ -26,7 +26,7 @@ namespace scheduler {
class XDeleteTask : public Task { class XDeleteTask : public Task {
public: public:
explicit XDeleteTask(const scheduler::DeleteJobPtr& job); explicit XDeleteTask(const scheduler::DeleteJobPtr& delete_job);
void void
Load(LoadType type, uint8_t device_id) override; Load(LoadType type, uint8_t device_id) override;

View File

@ -31,9 +31,8 @@ UriCheck(const std::string& uri) {
size_t index = uri.find_first_of(':', 0); size_t index = uri.find_first_of(':', 0);
if (index == std::string::npos) { if (index == std::string::npos) {
return false; return false;
} else {
return true;
} }
return true;
} }
Status Status
@ -45,11 +44,11 @@ ClientProxy::Connect(const ConnectParam& param) {
connected_ = true; connected_ = true;
client_ptr_ = std::make_shared<GrpcClient>(channel_); client_ptr_ = std::make_shared<GrpcClient>(channel_);
return Status::OK(); return Status::OK();
} else {
std::string reason = "connect failed!";
connected_ = false;
return Status(StatusCode::NotConnected, reason);
} }
std::string reason = "connect failed!";
connected_ = false;
return Status(StatusCode::NotConnected, reason);
} }
Status Status
@ -134,7 +133,7 @@ ClientProxy::CreateIndex(const IndexParam& index_param) {
// TODO: add index params // TODO: add index params
::milvus::grpc::IndexParam grpc_index_param; ::milvus::grpc::IndexParam grpc_index_param;
grpc_index_param.set_table_name(index_param.table_name); grpc_index_param.set_table_name(index_param.table_name);
grpc_index_param.mutable_index()->set_index_type((int32_t)index_param.index_type); grpc_index_param.mutable_index()->set_index_type(static_cast<int32_t>(index_param.index_type));
grpc_index_param.mutable_index()->set_nlist(index_param.nlist); grpc_index_param.mutable_index()->set_nlist(index_param.nlist);
return client_ptr_->CreateIndex(grpc_index_param); return client_ptr_->CreateIndex(grpc_index_param);
} catch (std::exception& ex) { } catch (std::exception& ex) {

View File

@ -85,7 +85,7 @@ class Status {
MoveFrom(Status& s); MoveFrom(Status& s);
private: private:
const char* state_ = nullptr; char* state_ = nullptr;
}; // Status }; // Status
} // namespace milvus } // namespace milvus

View File

@ -28,7 +28,7 @@ Status::Status(StatusCode code, const std::string& msg) {
// 4 bytes store message length // 4 bytes store message length
// the left bytes store message string // the left bytes store message string
const uint32_t length = (uint32_t)msg.size(); const uint32_t length = (uint32_t)msg.size();
char* result = new char[length + sizeof(length) + CODE_WIDTH]; auto result = new char[length + sizeof(length) + CODE_WIDTH];
memcpy(result, &code, CODE_WIDTH); memcpy(result, &code, CODE_WIDTH);
memcpy(result + CODE_WIDTH, &length, sizeof(length)); memcpy(result + CODE_WIDTH, &length, sizeof(length));
memcpy(result + sizeof(length) + CODE_WIDTH, msg.data(), length); memcpy(result + sizeof(length) + CODE_WIDTH, msg.data(), length);
@ -75,7 +75,7 @@ Status::CopyFrom(const Status& s) {
memcpy(&length, s.state_ + CODE_WIDTH, sizeof(length)); memcpy(&length, s.state_ + CODE_WIDTH, sizeof(length));
int buff_len = length + sizeof(length) + CODE_WIDTH; int buff_len = length + sizeof(length) + CODE_WIDTH;
state_ = new char[buff_len]; state_ = new char[buff_len];
memcpy((void*)state_, (void*)s.state_, buff_len); memcpy(state_, s.state_, buff_len);
} }
void void

View File

@ -77,100 +77,146 @@ Config::ValidateConfig() {
/* server config */ /* server config */
std::string server_addr; std::string server_addr;
s = GetServerConfigAddress(server_addr); s = GetServerConfigAddress(server_addr);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
std::string server_port; std::string server_port;
s = GetServerConfigPort(server_port); s = GetServerConfigPort(server_port);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
std::string server_mode; std::string server_mode;
s = GetServerConfigDeployMode(server_mode); s = GetServerConfigDeployMode(server_mode);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
std::string server_time_zone; std::string server_time_zone;
s = GetServerConfigTimeZone(server_time_zone); s = GetServerConfigTimeZone(server_time_zone);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
/* db config */ /* db config */
std::string db_primary_path; std::string db_primary_path;
s = GetDBConfigPrimaryPath(db_primary_path); s = GetDBConfigPrimaryPath(db_primary_path);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
std::string db_secondary_path; std::string db_secondary_path;
s = GetDBConfigSecondaryPath(db_secondary_path); s = GetDBConfigSecondaryPath(db_secondary_path);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
std::string db_backend_url; std::string db_backend_url;
s = GetDBConfigBackendUrl(db_backend_url); s = GetDBConfigBackendUrl(db_backend_url);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
int32_t db_archive_disk_threshold; int32_t db_archive_disk_threshold;
s = GetDBConfigArchiveDiskThreshold(db_archive_disk_threshold); s = GetDBConfigArchiveDiskThreshold(db_archive_disk_threshold);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
int32_t db_archive_days_threshold; int32_t db_archive_days_threshold;
s = GetDBConfigArchiveDaysThreshold(db_archive_days_threshold); s = GetDBConfigArchiveDaysThreshold(db_archive_days_threshold);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
int32_t db_insert_buffer_size; int32_t db_insert_buffer_size;
s = GetDBConfigInsertBufferSize(db_insert_buffer_size); s = GetDBConfigInsertBufferSize(db_insert_buffer_size);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
int32_t db_build_index_gpu; int32_t db_build_index_gpu;
s = GetDBConfigBuildIndexGPU(db_build_index_gpu); s = GetDBConfigBuildIndexGPU(db_build_index_gpu);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
/* metric config */ /* metric config */
bool metric_enable_monitor; bool metric_enable_monitor;
s = GetMetricConfigEnableMonitor(metric_enable_monitor); s = GetMetricConfigEnableMonitor(metric_enable_monitor);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
std::string metric_collector; std::string metric_collector;
s = GetMetricConfigCollector(metric_collector); s = GetMetricConfigCollector(metric_collector);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
std::string metric_prometheus_port; std::string metric_prometheus_port;
s = GetMetricConfigPrometheusPort(metric_prometheus_port); s = GetMetricConfigPrometheusPort(metric_prometheus_port);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
/* cache config */ /* cache config */
int32_t cache_cpu_cache_capacity; int32_t cache_cpu_cache_capacity;
s = GetCacheConfigCpuCacheCapacity(cache_cpu_cache_capacity); s = GetCacheConfigCpuCacheCapacity(cache_cpu_cache_capacity);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
float cache_cpu_cache_threshold; float cache_cpu_cache_threshold;
s = GetCacheConfigCpuCacheThreshold(cache_cpu_cache_threshold); s = GetCacheConfigCpuCacheThreshold(cache_cpu_cache_threshold);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
int32_t cache_gpu_cache_capacity; int32_t cache_gpu_cache_capacity;
s = GetCacheConfigGpuCacheCapacity(cache_gpu_cache_capacity); s = GetCacheConfigGpuCacheCapacity(cache_gpu_cache_capacity);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
float cache_gpu_cache_threshold; float cache_gpu_cache_threshold;
s = GetCacheConfigGpuCacheThreshold(cache_gpu_cache_threshold); s = GetCacheConfigGpuCacheThreshold(cache_gpu_cache_threshold);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
bool cache_insert_data; bool cache_insert_data;
s = GetCacheConfigCacheInsertData(cache_insert_data); s = GetCacheConfigCacheInsertData(cache_insert_data);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
/* engine config */ /* engine config */
int32_t engine_use_blas_threshold; int32_t engine_use_blas_threshold;
s = GetEngineConfigUseBlasThreshold(engine_use_blas_threshold); s = GetEngineConfigUseBlasThreshold(engine_use_blas_threshold);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
int32_t engine_omp_thread_num; int32_t engine_omp_thread_num;
s = GetEngineConfigOmpThreadNum(engine_omp_thread_num); s = GetEngineConfigOmpThreadNum(engine_omp_thread_num);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
/* resource config */ /* resource config */
std::string resource_mode; std::string resource_mode;
s = GetResourceConfigMode(resource_mode); s = GetResourceConfigMode(resource_mode);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
std::vector<std::string> resource_pool; std::vector<std::string> resource_pool;
s = GetResourceConfigPool(resource_pool); s = GetResourceConfigPool(resource_pool);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
return Status::OK(); return Status::OK();
} }
@ -181,75 +227,119 @@ Config::ResetDefaultConfig() {
/* server config */ /* server config */
s = SetServerConfigAddress(CONFIG_SERVER_ADDRESS_DEFAULT); s = SetServerConfigAddress(CONFIG_SERVER_ADDRESS_DEFAULT);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
s = SetServerConfigPort(CONFIG_SERVER_PORT_DEFAULT); s = SetServerConfigPort(CONFIG_SERVER_PORT_DEFAULT);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
s = SetServerConfigDeployMode(CONFIG_SERVER_DEPLOY_MODE_DEFAULT); s = SetServerConfigDeployMode(CONFIG_SERVER_DEPLOY_MODE_DEFAULT);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
s = SetServerConfigTimeZone(CONFIG_SERVER_TIME_ZONE_DEFAULT); s = SetServerConfigTimeZone(CONFIG_SERVER_TIME_ZONE_DEFAULT);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
/* db config */ /* db config */
s = SetDBConfigPrimaryPath(CONFIG_DB_PRIMARY_PATH_DEFAULT); s = SetDBConfigPrimaryPath(CONFIG_DB_PRIMARY_PATH_DEFAULT);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
s = SetDBConfigSecondaryPath(CONFIG_DB_SECONDARY_PATH_DEFAULT); s = SetDBConfigSecondaryPath(CONFIG_DB_SECONDARY_PATH_DEFAULT);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
s = SetDBConfigBackendUrl(CONFIG_DB_BACKEND_URL_DEFAULT); s = SetDBConfigBackendUrl(CONFIG_DB_BACKEND_URL_DEFAULT);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
s = SetDBConfigArchiveDiskThreshold(CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); s = SetDBConfigArchiveDiskThreshold(CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
s = SetDBConfigArchiveDaysThreshold(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); s = SetDBConfigArchiveDaysThreshold(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
s = SetDBConfigInsertBufferSize(CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); s = SetDBConfigInsertBufferSize(CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
s = SetDBConfigBuildIndexGPU(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); s = SetDBConfigBuildIndexGPU(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
/* metric config */ /* metric config */
s = SetMetricConfigEnableMonitor(CONFIG_METRIC_ENABLE_MONITOR_DEFAULT); s = SetMetricConfigEnableMonitor(CONFIG_METRIC_ENABLE_MONITOR_DEFAULT);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
s = SetMetricConfigCollector(CONFIG_METRIC_COLLECTOR_DEFAULT); s = SetMetricConfigCollector(CONFIG_METRIC_COLLECTOR_DEFAULT);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
s = SetMetricConfigPrometheusPort(CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); s = SetMetricConfigPrometheusPort(CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
/* cache config */ /* cache config */
s = SetCacheConfigCpuCacheCapacity(CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT); s = SetCacheConfigCpuCacheCapacity(CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
s = SetCacheConfigCpuCacheThreshold(CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT); s = SetCacheConfigCpuCacheThreshold(CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
s = SetCacheConfigGpuCacheCapacity(CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT); s = SetCacheConfigGpuCacheCapacity(CONFIG_CACHE_GPU_CACHE_CAPACITY_DEFAULT);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
s = SetCacheConfigGpuCacheThreshold(CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT); s = SetCacheConfigGpuCacheThreshold(CONFIG_CACHE_GPU_CACHE_THRESHOLD_DEFAULT);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
s = SetCacheConfigCacheInsertData(CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); s = SetCacheConfigCacheInsertData(CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
/* engine config */ /* engine config */
s = SetEngineConfigUseBlasThreshold(CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT); s = SetEngineConfigUseBlasThreshold(CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
s = SetEngineConfigOmpThreadNum(CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); s = SetEngineConfigOmpThreadNum(CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
/* resource config */ /* resource config */
s = SetResourceConfigMode(CONFIG_RESOURCE_MODE_DEFAULT); s = SetResourceConfigMode(CONFIG_RESOURCE_MODE_DEFAULT);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
return Status::OK(); return Status::OK();
} }
@ -423,13 +513,16 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) {
CommonUtil::GetSystemMemInfo(total_mem, free_mem); CommonUtil::GetSystemMemInfo(total_mem, free_mem);
if (cpu_cache_capacity >= total_mem) { if (cpu_cache_capacity >= total_mem) {
return Status(SERVER_INVALID_ARGUMENT, "Cache config cpu_cache_capacity exceed system memory: " + value); return Status(SERVER_INVALID_ARGUMENT, "Cache config cpu_cache_capacity exceed system memory: " + value);
} else if (cpu_cache_capacity > (double)total_mem * 0.9) { } else if (cpu_cache_capacity > static_cast<double>(total_mem * 0.9)) {
std::cerr << "Warning: cpu_cache_capacity value is too big" << std::endl; std::cerr << "Warning: cpu_cache_capacity value is too big" << std::endl;
} }
int32_t buffer_value; int32_t buffer_value;
Status s = GetDBConfigInsertBufferSize(buffer_value); Status s = GetDBConfigInsertBufferSize(buffer_value);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
int64_t insert_buffer_size = buffer_value * GB; int64_t insert_buffer_size = buffer_value * GB;
if (insert_buffer_size + cpu_cache_capacity >= total_mem) { if (insert_buffer_size + cpu_cache_capacity >= total_mem) {
return Status(SERVER_INVALID_ARGUMENT, "Sum of cpu_cache_capacity and buffer_size exceed system memory"); return Status(SERVER_INVALID_ARGUMENT, "Sum of cpu_cache_capacity and buffer_size exceed system memory");
@ -459,7 +552,10 @@ Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) {
uint64_t gpu_cache_capacity = std::stoi(value) * GB; uint64_t gpu_cache_capacity = std::stoi(value) * GB;
int gpu_index; int gpu_index;
Status s = GetDBConfigBuildIndexGPU(gpu_index); Status s = GetDBConfigBuildIndexGPU(gpu_index);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
size_t gpu_memory; size_t gpu_memory;
if (!ValidationUtil::GetGpuMemory(gpu_index, gpu_memory).ok()) { if (!ValidationUtil::GetGpuMemory(gpu_index, gpu_memory).ok()) {
return Status(SERVER_UNEXPECTED_ERROR, return Status(SERVER_UNEXPECTED_ERROR,
@ -507,12 +603,13 @@ Status
Config::CheckEngineConfigOmpThreadNum(const std::string& value) { Config::CheckEngineConfigOmpThreadNum(const std::string& value) {
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid engine config omp_thread_num: " + value); return Status(SERVER_INVALID_ARGUMENT, "Invalid engine config omp_thread_num: " + value);
} else { }
int32_t omp_thread = std::stoi(value);
uint32_t sys_thread_cnt = 8; int32_t omp_thread = std::stoi(value);
if (omp_thread > CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) { uint32_t sys_thread_cnt = 8;
return Status(SERVER_INVALID_ARGUMENT, "Invalid engine config omp_thread_num: " + value); CommonUtil::GetSystemAvailableThreads(sys_thread_cnt);
} if (omp_thread > static_cast<int32_t>(sys_thread_cnt)) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid engine config omp_thread_num: " + value);
} }
return Status::OK(); return Status::OK();
} }
@ -548,9 +645,8 @@ Config::GetConfigValueInMem(const std::string& parent_key, const std::string& ch
config_map_[parent_key].find(child_key) != config_map_[parent_key].end()) { config_map_[parent_key].find(child_key) != config_map_[parent_key].end()) {
value = config_map_[parent_key][child_key]; value = config_map_[parent_key][child_key];
return Status::OK(); return Status::OK();
} else {
return Status(SERVER_UNEXPECTED_ERROR, "key not exist");
} }
return Status(SERVER_UNEXPECTED_ERROR, "key not exist");
} }
void void
@ -849,7 +945,10 @@ Status
Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { Config::GetDBConfigArchiveDiskThreshold(int32_t& value) {
std::string str = GetDBConfigStrArchiveDiskThreshold(); std::string str = GetDBConfigStrArchiveDiskThreshold();
Status s = CheckDBConfigArchiveDiskThreshold(str); Status s = CheckDBConfigArchiveDiskThreshold(str);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
value = std::stoi(str); value = std::stoi(str);
return Status::OK(); return Status::OK();
} }
@ -858,7 +957,10 @@ Status
Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { Config::GetDBConfigArchiveDaysThreshold(int32_t& value) {
std::string str = GetDBConfigStrArchiveDaysThreshold(); std::string str = GetDBConfigStrArchiveDaysThreshold();
Status s = CheckDBConfigArchiveDaysThreshold(str); Status s = CheckDBConfigArchiveDaysThreshold(str);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
value = std::stoi(str); value = std::stoi(str);
return Status::OK(); return Status::OK();
} }
@ -867,7 +969,10 @@ Status
Config::GetDBConfigInsertBufferSize(int32_t& value) { Config::GetDBConfigInsertBufferSize(int32_t& value) {
std::string str = GetDBConfigStrInsertBufferSize(); std::string str = GetDBConfigStrInsertBufferSize();
Status s = CheckDBConfigInsertBufferSize(str); Status s = CheckDBConfigInsertBufferSize(str);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
value = std::stoi(str); value = std::stoi(str);
return Status::OK(); return Status::OK();
} }
@ -876,7 +981,10 @@ Status
Config::GetDBConfigBuildIndexGPU(int32_t& value) { Config::GetDBConfigBuildIndexGPU(int32_t& value) {
std::string str = GetDBConfigStrBuildIndexGPU(); std::string str = GetDBConfigStrBuildIndexGPU();
Status s = CheckDBConfigBuildIndexGPU(str); Status s = CheckDBConfigBuildIndexGPU(str);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
value = std::stoi(str); value = std::stoi(str);
return Status::OK(); return Status::OK();
} }
@ -885,7 +993,10 @@ Status
Config::GetMetricConfigEnableMonitor(bool& value) { Config::GetMetricConfigEnableMonitor(bool& value) {
std::string str = GetMetricConfigStrEnableMonitor(); std::string str = GetMetricConfigStrEnableMonitor();
Status s = CheckMetricConfigEnableMonitor(str); Status s = CheckMetricConfigEnableMonitor(str);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
std::transform(str.begin(), str.end(), str.begin(), ::tolower); std::transform(str.begin(), str.end(), str.begin(), ::tolower);
value = (str == "true" || str == "on" || str == "yes" || str == "1"); value = (str == "true" || str == "on" || str == "yes" || str == "1");
return Status::OK(); return Status::OK();
@ -907,7 +1018,10 @@ Status
Config::GetCacheConfigCpuCacheCapacity(int32_t& value) { Config::GetCacheConfigCpuCacheCapacity(int32_t& value) {
std::string str = GetCacheConfigStrCpuCacheCapacity(); std::string str = GetCacheConfigStrCpuCacheCapacity();
Status s = CheckCacheConfigCpuCacheCapacity(str); Status s = CheckCacheConfigCpuCacheCapacity(str);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
value = std::stoi(str); value = std::stoi(str);
return Status::OK(); return Status::OK();
} }
@ -916,7 +1030,10 @@ Status
Config::GetCacheConfigCpuCacheThreshold(float& value) { Config::GetCacheConfigCpuCacheThreshold(float& value) {
std::string str = GetCacheConfigStrCpuCacheThreshold(); std::string str = GetCacheConfigStrCpuCacheThreshold();
Status s = CheckCacheConfigCpuCacheThreshold(str); Status s = CheckCacheConfigCpuCacheThreshold(str);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
value = std::stof(str); value = std::stof(str);
return Status::OK(); return Status::OK();
} }
@ -925,7 +1042,10 @@ Status
Config::GetCacheConfigGpuCacheCapacity(int32_t& value) { Config::GetCacheConfigGpuCacheCapacity(int32_t& value) {
std::string str = GetCacheConfigStrGpuCacheCapacity(); std::string str = GetCacheConfigStrGpuCacheCapacity();
Status s = CheckCacheConfigGpuCacheCapacity(str); Status s = CheckCacheConfigGpuCacheCapacity(str);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
value = std::stoi(str); value = std::stoi(str);
return Status::OK(); return Status::OK();
} }
@ -934,7 +1054,10 @@ Status
Config::GetCacheConfigGpuCacheThreshold(float& value) { Config::GetCacheConfigGpuCacheThreshold(float& value) {
std::string str = GetCacheConfigStrGpuCacheThreshold(); std::string str = GetCacheConfigStrGpuCacheThreshold();
Status s = CheckCacheConfigGpuCacheThreshold(str); Status s = CheckCacheConfigGpuCacheThreshold(str);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
value = std::stof(str); value = std::stof(str);
return Status::OK(); return Status::OK();
} }
@ -943,7 +1066,10 @@ Status
Config::GetCacheConfigCacheInsertData(bool& value) { Config::GetCacheConfigCacheInsertData(bool& value) {
std::string str = GetCacheConfigStrCacheInsertData(); std::string str = GetCacheConfigStrCacheInsertData();
Status s = CheckCacheConfigCacheInsertData(str); Status s = CheckCacheConfigCacheInsertData(str);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
std::transform(str.begin(), str.end(), str.begin(), ::tolower); std::transform(str.begin(), str.end(), str.begin(), ::tolower);
value = (str == "true" || str == "on" || str == "yes" || str == "1"); value = (str == "true" || str == "on" || str == "yes" || str == "1");
return Status::OK(); return Status::OK();
@ -953,7 +1079,10 @@ Status
Config::GetEngineConfigUseBlasThreshold(int32_t& value) { Config::GetEngineConfigUseBlasThreshold(int32_t& value) {
std::string str = GetEngineConfigStrUseBlasThreshold(); std::string str = GetEngineConfigStrUseBlasThreshold();
Status s = CheckEngineConfigUseBlasThreshold(str); Status s = CheckEngineConfigUseBlasThreshold(str);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
value = std::stoi(str); value = std::stoi(str);
return Status::OK(); return Status::OK();
} }
@ -962,7 +1091,10 @@ Status
Config::GetEngineConfigOmpThreadNum(int32_t& value) { Config::GetEngineConfigOmpThreadNum(int32_t& value) {
std::string str = GetEngineConfigStrOmpThreadNum(); std::string str = GetEngineConfigStrOmpThreadNum();
Status s = CheckEngineConfigOmpThreadNum(str); Status s = CheckEngineConfigOmpThreadNum(str);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
value = std::stoi(str); value = std::stoi(str);
return Status::OK(); return Status::OK();
} }
@ -985,7 +1117,10 @@ Config::GetResourceConfigPool(std::vector<std::string>& value) {
Status Status
Config::SetServerConfigAddress(const std::string& value) { Config::SetServerConfigAddress(const std::string& value) {
Status s = CheckServerConfigAddress(value); Status s = CheckServerConfigAddress(value);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value);
return Status::OK(); return Status::OK();
} }
@ -993,7 +1128,10 @@ Config::SetServerConfigAddress(const std::string& value) {
Status Status
Config::SetServerConfigPort(const std::string& value) { Config::SetServerConfigPort(const std::string& value) {
Status s = CheckServerConfigPort(value); Status s = CheckServerConfigPort(value);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value);
return Status::OK(); return Status::OK();
} }
@ -1001,7 +1139,10 @@ Config::SetServerConfigPort(const std::string& value) {
Status Status
Config::SetServerConfigDeployMode(const std::string& value) { Config::SetServerConfigDeployMode(const std::string& value) {
Status s = CheckServerConfigDeployMode(value); Status s = CheckServerConfigDeployMode(value);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value); SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value);
return Status::OK(); return Status::OK();
} }
@ -1009,7 +1150,10 @@ Config::SetServerConfigDeployMode(const std::string& value) {
Status Status
Config::SetServerConfigTimeZone(const std::string& value) { Config::SetServerConfigTimeZone(const std::string& value) {
Status s = CheckServerConfigTimeZone(value); Status s = CheckServerConfigTimeZone(value);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value);
return Status::OK(); return Status::OK();
} }
@ -1018,7 +1162,10 @@ Config::SetServerConfigTimeZone(const std::string& value) {
Status Status
Config::SetDBConfigPrimaryPath(const std::string& value) { Config::SetDBConfigPrimaryPath(const std::string& value) {
Status s = CheckDBConfigPrimaryPath(value); Status s = CheckDBConfigPrimaryPath(value);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value); SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value);
return Status::OK(); return Status::OK();
} }
@ -1026,7 +1173,10 @@ Config::SetDBConfigPrimaryPath(const std::string& value) {
Status Status
Config::SetDBConfigSecondaryPath(const std::string& value) { Config::SetDBConfigSecondaryPath(const std::string& value) {
Status s = CheckDBConfigSecondaryPath(value); Status s = CheckDBConfigSecondaryPath(value);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value); SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value);
return Status::OK(); return Status::OK();
} }
@ -1034,7 +1184,10 @@ Config::SetDBConfigSecondaryPath(const std::string& value) {
Status Status
Config::SetDBConfigBackendUrl(const std::string& value) { Config::SetDBConfigBackendUrl(const std::string& value) {
Status s = CheckDBConfigBackendUrl(value); Status s = CheckDBConfigBackendUrl(value);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value);
return Status::OK(); return Status::OK();
} }
@ -1042,7 +1195,10 @@ Config::SetDBConfigBackendUrl(const std::string& value) {
Status Status
Config::SetDBConfigArchiveDiskThreshold(const std::string& value) { Config::SetDBConfigArchiveDiskThreshold(const std::string& value) {
Status s = CheckDBConfigArchiveDiskThreshold(value); Status s = CheckDBConfigArchiveDiskThreshold(value);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value); SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value);
return Status::OK(); return Status::OK();
} }
@ -1050,7 +1206,10 @@ Config::SetDBConfigArchiveDiskThreshold(const std::string& value) {
Status Status
Config::SetDBConfigArchiveDaysThreshold(const std::string& value) { Config::SetDBConfigArchiveDaysThreshold(const std::string& value) {
Status s = CheckDBConfigArchiveDaysThreshold(value); Status s = CheckDBConfigArchiveDaysThreshold(value);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value);
return Status::OK(); return Status::OK();
} }
@ -1058,7 +1217,10 @@ Config::SetDBConfigArchiveDaysThreshold(const std::string& value) {
Status Status
Config::SetDBConfigInsertBufferSize(const std::string& value) { Config::SetDBConfigInsertBufferSize(const std::string& value) {
Status s = CheckDBConfigInsertBufferSize(value); Status s = CheckDBConfigInsertBufferSize(value);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value); SetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value);
return Status::OK(); return Status::OK();
} }
@ -1066,7 +1228,10 @@ Config::SetDBConfigInsertBufferSize(const std::string& value) {
Status Status
Config::SetDBConfigBuildIndexGPU(const std::string& value) { Config::SetDBConfigBuildIndexGPU(const std::string& value) {
Status s = CheckDBConfigBuildIndexGPU(value); Status s = CheckDBConfigBuildIndexGPU(value);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value); SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value);
return Status::OK(); return Status::OK();
} }
@ -1075,7 +1240,10 @@ Config::SetDBConfigBuildIndexGPU(const std::string& value) {
Status Status
Config::SetMetricConfigEnableMonitor(const std::string& value) { Config::SetMetricConfigEnableMonitor(const std::string& value) {
Status s = CheckMetricConfigEnableMonitor(value); Status s = CheckMetricConfigEnableMonitor(value);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_ENABLE_MONITOR, value); SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_ENABLE_MONITOR, value);
return Status::OK(); return Status::OK();
} }
@ -1083,7 +1251,10 @@ Config::SetMetricConfigEnableMonitor(const std::string& value) {
Status Status
Config::SetMetricConfigCollector(const std::string& value) { Config::SetMetricConfigCollector(const std::string& value) {
Status s = CheckMetricConfigCollector(value); Status s = CheckMetricConfigCollector(value);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_COLLECTOR, value); SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_COLLECTOR, value);
return Status::OK(); return Status::OK();
} }
@ -1091,7 +1262,10 @@ Config::SetMetricConfigCollector(const std::string& value) {
Status Status
Config::SetMetricConfigPrometheusPort(const std::string& value) { Config::SetMetricConfigPrometheusPort(const std::string& value) {
Status s = CheckMetricConfigPrometheusPort(value); Status s = CheckMetricConfigPrometheusPort(value);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_PROMETHEUS_PORT, value); SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_PROMETHEUS_PORT, value);
return Status::OK(); return Status::OK();
} }
@ -1100,7 +1274,10 @@ Config::SetMetricConfigPrometheusPort(const std::string& value) {
Status Status
Config::SetCacheConfigCpuCacheCapacity(const std::string& value) { Config::SetCacheConfigCpuCacheCapacity(const std::string& value) {
Status s = CheckCacheConfigCpuCacheCapacity(value); Status s = CheckCacheConfigCpuCacheCapacity(value);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_CACHE_CAPACITY, value); SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_CACHE_CAPACITY, value);
return Status::OK(); return Status::OK();
} }
@ -1108,7 +1285,10 @@ Config::SetCacheConfigCpuCacheCapacity(const std::string& value) {
Status Status
Config::SetCacheConfigCpuCacheThreshold(const std::string& value) { Config::SetCacheConfigCpuCacheThreshold(const std::string& value) {
Status s = CheckCacheConfigCpuCacheThreshold(value); Status s = CheckCacheConfigCpuCacheThreshold(value);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value); SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_CACHE_THRESHOLD, value);
return Status::OK(); return Status::OK();
} }
@ -1116,7 +1296,10 @@ Config::SetCacheConfigCpuCacheThreshold(const std::string& value) {
Status Status
Config::SetCacheConfigGpuCacheCapacity(const std::string& value) { Config::SetCacheConfigGpuCacheCapacity(const std::string& value) {
Status s = CheckCacheConfigGpuCacheCapacity(value); Status s = CheckCacheConfigGpuCacheCapacity(value);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_CACHE_CAPACITY, value); SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_CACHE_CAPACITY, value);
return Status::OK(); return Status::OK();
} }
@ -1124,7 +1307,10 @@ Config::SetCacheConfigGpuCacheCapacity(const std::string& value) {
Status Status
Config::SetCacheConfigGpuCacheThreshold(const std::string& value) { Config::SetCacheConfigGpuCacheThreshold(const std::string& value) {
Status s = CheckCacheConfigGpuCacheThreshold(value); Status s = CheckCacheConfigGpuCacheThreshold(value);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value); SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_CACHE_THRESHOLD, value);
return Status::OK(); return Status::OK();
} }
@ -1132,7 +1318,10 @@ Config::SetCacheConfigGpuCacheThreshold(const std::string& value) {
Status Status
Config::SetCacheConfigCacheInsertData(const std::string& value) { Config::SetCacheConfigCacheInsertData(const std::string& value) {
Status s = CheckCacheConfigCacheInsertData(value); Status s = CheckCacheConfigCacheInsertData(value);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CACHE_INSERT_DATA, value); SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CACHE_INSERT_DATA, value);
return Status::OK(); return Status::OK();
} }
@ -1141,7 +1330,10 @@ Config::SetCacheConfigCacheInsertData(const std::string& value) {
Status Status
Config::SetEngineConfigUseBlasThreshold(const std::string& value) { Config::SetEngineConfigUseBlasThreshold(const std::string& value) {
Status s = CheckEngineConfigUseBlasThreshold(value); Status s = CheckEngineConfigUseBlasThreshold(value);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value); SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_USE_BLAS_THRESHOLD, value);
return Status::OK(); return Status::OK();
} }
@ -1149,7 +1341,10 @@ Config::SetEngineConfigUseBlasThreshold(const std::string& value) {
Status Status
Config::SetEngineConfigOmpThreadNum(const std::string& value) { Config::SetEngineConfigOmpThreadNum(const std::string& value) {
Status s = CheckEngineConfigOmpThreadNum(value); Status s = CheckEngineConfigOmpThreadNum(value);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_OMP_THREAD_NUM, value); SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_OMP_THREAD_NUM, value);
return Status::OK(); return Status::OK();
} }
@ -1158,7 +1353,10 @@ Config::SetEngineConfigOmpThreadNum(const std::string& value) {
Status Status
Config::SetResourceConfigMode(const std::string& value) { Config::SetResourceConfigMode(const std::string& value) {
Status s = CheckResourceConfigMode(value); Status s = CheckResourceConfigMode(value);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_RESOURCE_MODE, value); SetConfigValueInMem(CONFIG_DB, CONFIG_RESOURCE_MODE, value);
return Status::OK(); return Status::OK();
} }

View File

@ -30,9 +30,6 @@ namespace zilliz {
namespace milvus { namespace milvus {
namespace server { namespace server {
DBWrapper::DBWrapper() {
}
Status Status
DBWrapper::StartService() { DBWrapper::StartService() {
Config& config = Config::GetInstance(); Config& config = Config::GetInstance();
@ -41,27 +38,37 @@ DBWrapper::StartService() {
engine::DBOptions opt; engine::DBOptions opt;
s = config.GetDBConfigBackendUrl(opt.meta_.backend_uri_); s = config.GetDBConfigBackendUrl(opt.meta_.backend_uri_);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
std::string path; std::string path;
s = config.GetDBConfigPrimaryPath(path); s = config.GetDBConfigPrimaryPath(path);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
opt.meta_.path_ = path + "/db"; opt.meta_.path_ = path + "/db";
std::string db_slave_path; std::string db_slave_path;
s = config.GetDBConfigSecondaryPath(db_slave_path); s = config.GetDBConfigSecondaryPath(db_slave_path);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta_.slave_paths_); StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta_.slave_paths_);
// cache config // cache config
s = config.GetCacheConfigCacheInsertData(opt.insert_cache_immediately_); s = config.GetCacheConfigCacheInsertData(opt.insert_cache_immediately_);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
std::string mode; std::string mode;
s = config.GetServerConfigDeployMode(mode); s = config.GetServerConfigDeployMode(mode);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
if (mode == "single") { if (mode == "single") {
opt.mode_ = engine::DBOptions::MODE::SINGLE; opt.mode_ = engine::DBOptions::MODE::SINGLE;
@ -78,14 +85,17 @@ DBWrapper::StartService() {
// engine config // engine config
int32_t omp_thread; int32_t omp_thread;
s = config.GetEngineConfigOmpThreadNum(omp_thread); s = config.GetEngineConfigOmpThreadNum(omp_thread);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
if (omp_thread > 0) { if (omp_thread > 0) {
omp_set_num_threads(omp_thread); omp_set_num_threads(omp_thread);
SERVER_LOG_DEBUG << "Specify openmp thread number: " << omp_thread; SERVER_LOG_DEBUG << "Specify openmp thread number: " << omp_thread;
} else { } else {
uint32_t sys_thread_cnt = 8; uint32_t sys_thread_cnt = 8;
if (CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) { if (CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) {
omp_thread = (int32_t)ceil(sys_thread_cnt * 0.5); omp_thread = static_cast<int32_t>(ceil(sys_thread_cnt * 0.5));
omp_set_num_threads(omp_thread); omp_set_num_threads(omp_thread);
} }
} }
@ -93,20 +103,29 @@ DBWrapper::StartService() {
// init faiss global variable // init faiss global variable
int32_t use_blas_threshold; int32_t use_blas_threshold;
s = config.GetEngineConfigUseBlasThreshold(use_blas_threshold); s = config.GetEngineConfigUseBlasThreshold(use_blas_threshold);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
faiss::distance_compute_blas_threshold = use_blas_threshold; faiss::distance_compute_blas_threshold = use_blas_threshold;
// set archive config // set archive config
engine::ArchiveConf::CriteriaT criterial; engine::ArchiveConf::CriteriaT criterial;
int32_t disk, days; int32_t disk, days;
s = config.GetDBConfigArchiveDiskThreshold(disk); s = config.GetDBConfigArchiveDiskThreshold(disk);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
if (disk > 0) { if (disk > 0) {
criterial[engine::ARCHIVE_CONF_DISK] = disk; criterial[engine::ARCHIVE_CONF_DISK] = disk;
} }
s = config.GetDBConfigArchiveDaysThreshold(days); s = config.GetDBConfigArchiveDaysThreshold(days);
if (!s.ok()) return s; if (!s.ok()) {
return s;
}
if (days > 0) { if (days > 0) {
criterial[engine::ARCHIVE_CONF_DAYS] = days; criterial[engine::ARCHIVE_CONF_DAYS] = days;
} }

View File

@ -28,7 +28,7 @@ namespace server {
class DBWrapper { class DBWrapper {
private: private:
DBWrapper(); DBWrapper() = default;
~DBWrapper() = default; ~DBWrapper() = default;
public: public:

View File

@ -146,7 +146,7 @@ Server::Daemonize() {
void void
Server::Start() { Server::Start() {
if (daemonized_) { if (daemonized_ != 0) {
Daemonize(); Daemonize();
} }

View File

@ -227,7 +227,7 @@ class GrpcRequestHandler final : public ::milvus::grpc::MilvusService::Service {
*/ */
::grpc::Status ::grpc::Status
ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request, ShowTables(::grpc::ServerContext* context, const ::milvus::grpc::Command* request,
::milvus::grpc::TableNameList* table_name_list) override; ::milvus::grpc::TableNameList* response) override;
/** /**
* @brief Give the server status * @brief Give the server status

View File

@ -163,7 +163,8 @@ GrpcRequestScheduler::Stop() {
} }
for (auto iter : execute_threads_) { for (auto iter : execute_threads_) {
if (iter == nullptr) continue; if (iter == nullptr)
continue;
iter->join(); iter->join();
} }

View File

@ -69,7 +69,7 @@ class GrpcBaseTask {
OnExecute() = 0; OnExecute() = 0;
Status Status
SetStatus(ErrorCode error_code, const std::string& msg); SetStatus(ErrorCode error_code, const std::string& error_msg);
protected: protected:
mutable std::mutex finish_mtx_; mutable std::mutex finish_mtx_;

View File

@ -159,7 +159,7 @@ CreateTableTask::OnExecute() {
// step 2: construct table schema // step 2: construct table schema
engine::meta::TableSchema table_info; engine::meta::TableSchema table_info;
table_info.table_id_ = schema_->table_name(); table_info.table_id_ = schema_->table_name();
table_info.dimension_ = (uint16_t)schema_->dimension(); table_info.dimension_ = static_cast<uint16_t>(schema_->dimension());
table_info.index_file_size_ = schema_->index_file_size(); table_info.index_file_size_ = schema_->index_file_size();
table_info.metric_type_ = schema_->metric_type(); table_info.metric_type_ = schema_->metric_type();
@ -446,13 +446,13 @@ InsertTask::OnExecute() {
// all user provide id, or all internal id // all user provide id, or all internal id
bool user_provide_ids = !insert_param_->row_id_array().empty(); bool user_provide_ids = !insert_param_->row_id_array().empty();
// user already provided id before, all insert action require user id // user already provided id before, all insert action require user id
if ((table_info.flag_ & engine::meta::FLAG_MASK_HAS_USERID) && !user_provide_ids) { if ((table_info.flag_ & engine::meta::FLAG_MASK_HAS_USERID) != 0 && !user_provide_ids) {
return Status(SERVER_ILLEGAL_VECTOR_ID, return Status(SERVER_ILLEGAL_VECTOR_ID,
"Table vector ids are user defined, please provide id for this batch"); "Table vector ids are user defined, please provide id for this batch");
} }
// user didn't provided id before, no need to provide user id // user didn't provided id before, no need to provide user id
if ((table_info.flag_ & engine::meta::FLAG_MASK_NO_USERID) && user_provide_ids) { if ((table_info.flag_ & engine::meta::FLAG_MASK_NO_USERID) != 0 && user_provide_ids) {
return Status(SERVER_ILLEGAL_VECTOR_ID, return Status(SERVER_ILLEGAL_VECTOR_ID,
"Table vector ids are auto generated, no need to provide id for this batch"); "Table vector ids are auto generated, no need to provide id for this batch");
} }
@ -487,12 +487,12 @@ InsertTask::OnExecute() {
rc.ElapseFromBegin("prepare vectors data"); rc.ElapseFromBegin("prepare vectors data");
// step 5: insert vectors // step 5: insert vectors
auto vec_count = (uint64_t)insert_param_->row_record_array_size(); auto vec_count = static_cast<uint64_t>(insert_param_->row_record_array_size());
std::vector<int64_t> vec_ids(insert_param_->row_id_array_size(), 0); std::vector<int64_t> vec_ids(insert_param_->row_id_array_size(), 0);
if (!insert_param_->row_id_array().empty()) { if (!insert_param_->row_id_array().empty()) {
const int64_t* src_data = insert_param_->row_id_array().data(); const int64_t* src_data = insert_param_->row_id_array().data();
int64_t* target_data = vec_ids.data(); int64_t* target_data = vec_ids.data();
memcpy(target_data, src_data, (size_t)(sizeof(int64_t) * insert_param_->row_id_array_size())); memcpy(target_data, src_data, static_cast<size_t>(sizeof(int64_t) * insert_param_->row_id_array_size()));
} }
status = DBWrapper::DB()->InsertVectors(insert_param_->table_name(), vec_count, vec_f.data(), vec_ids); status = DBWrapper::DB()->InsertVectors(insert_param_->table_name(), vec_count, vec_f.data(), vec_ids);
@ -710,7 +710,7 @@ CountTableTask::OnExecute() {
return status; return status;
} }
row_count_ = (int64_t)row_count; row_count_ = static_cast<int64_t>(row_count);
rc.ElapseFromBegin("total cost"); rc.ElapseFromBegin("total cost");
} catch (std::exception& ex) { } catch (std::exception& ex) {

View File

@ -41,7 +41,7 @@ class CreateTableTask : public GrpcBaseTask {
Create(const ::milvus::grpc::TableSchema* schema); Create(const ::milvus::grpc::TableSchema* schema);
protected: protected:
explicit CreateTableTask(const ::milvus::grpc::TableSchema* request); explicit CreateTableTask(const ::milvus::grpc::TableSchema* schema);
Status Status
OnExecute() override; OnExecute() override;
@ -57,7 +57,7 @@ class HasTableTask : public GrpcBaseTask {
Create(const std::string& table_name, bool& has_table); Create(const std::string& table_name, bool& has_table);
protected: protected:
HasTableTask(const std::string& request, bool& has_table); HasTableTask(const std::string& table_name, bool& has_table);
Status Status
OnExecute() override; OnExecute() override;
@ -104,10 +104,10 @@ class DropTableTask : public GrpcBaseTask {
class CreateIndexTask : public GrpcBaseTask { class CreateIndexTask : public GrpcBaseTask {
public: public:
static BaseTaskPtr static BaseTaskPtr
Create(const ::milvus::grpc::IndexParam* index_Param); Create(const ::milvus::grpc::IndexParam* index_param);
protected: protected:
explicit CreateIndexTask(const ::milvus::grpc::IndexParam* index_Param); explicit CreateIndexTask(const ::milvus::grpc::IndexParam* index_param);
Status Status
OnExecute() override; OnExecute() override;
@ -136,10 +136,10 @@ class ShowTablesTask : public GrpcBaseTask {
class InsertTask : public GrpcBaseTask { class InsertTask : public GrpcBaseTask {
public: public:
static BaseTaskPtr static BaseTaskPtr
Create(const ::milvus::grpc::InsertParam* insert_Param, ::milvus::grpc::VectorIds* record_ids_); Create(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids);
protected: protected:
InsertTask(const ::milvus::grpc::InsertParam* insert_Param, ::milvus::grpc::VectorIds* record_ids_); InsertTask(const ::milvus::grpc::InsertParam* insert_param, ::milvus::grpc::VectorIds* record_ids);
Status Status
OnExecute() override; OnExecute() override;

View File

@ -59,7 +59,9 @@ CommonUtil::GetSystemAvailableThreads(uint32_t& thread_count) {
// threadCnt = std::thread::hardware_concurrency(); // threadCnt = std::thread::hardware_concurrency();
thread_count = sysconf(_SC_NPROCESSORS_CONF); thread_count = sysconf(_SC_NPROCESSORS_CONF);
thread_count *= THREAD_MULTIPLY_CPU; thread_count *= THREAD_MULTIPLY_CPU;
if (thread_count == 0) thread_count = 8; if (thread_count == 0) {
thread_count = 8;
}
return true; return true;
} }
@ -162,9 +164,9 @@ CommonUtil::GetFileSize(const std::string& path) {
struct stat file_info; struct stat file_info;
if (stat(path.c_str(), &file_info) < 0) { if (stat(path.c_str(), &file_info) < 0) {
return 0; return 0;
} else {
return (uint64_t)file_info.st_size;
} }
return static_cast<uint64_t>(file_info.st_size);
} }
std::string std::string

View File

@ -29,7 +29,7 @@ Status::Status(StatusCode code, const std::string& msg) {
// 4 bytes store message length // 4 bytes store message length
// the left bytes store message string // the left bytes store message string
const uint32_t length = (uint32_t)msg.size(); const uint32_t length = (uint32_t)msg.size();
char* result = new char[length + sizeof(length) + CODE_WIDTH]; auto result = new char[length + sizeof(length) + CODE_WIDTH];
std::memcpy(result, &code, CODE_WIDTH); std::memcpy(result, &code, CODE_WIDTH);
std::memcpy(result + CODE_WIDTH, &length, sizeof(length)); std::memcpy(result + CODE_WIDTH, &length, sizeof(length));
memcpy(result + sizeof(length) + CODE_WIDTH, msg.data(), length); memcpy(result + sizeof(length) + CODE_WIDTH, msg.data(), length);
@ -76,7 +76,7 @@ Status::CopyFrom(const Status& s) {
memcpy(&length, s.state_ + CODE_WIDTH, sizeof(length)); memcpy(&length, s.state_ + CODE_WIDTH, sizeof(length));
int buff_len = length + sizeof(length) + CODE_WIDTH; int buff_len = length + sizeof(length) + CODE_WIDTH;
state_ = new char[buff_len]; state_ = new char[buff_len];
memcpy((void*)state_, (void*)s.state_, buff_len); memcpy(state_, s.state_, buff_len);
} }
void void

View File

@ -71,7 +71,7 @@ class Status {
MoveFrom(Status& s); MoveFrom(Status& s);
private: private:
const char* state_ = nullptr; char* state_ = nullptr;
}; // Status }; // Status
} // namespace milvus } // namespace milvus

View File

@ -70,7 +70,8 @@ inline ThreadPool::ThreadPool(size_t threads, size_t queue_size) : max_queue_siz
{ {
std::unique_lock<std::mutex> lock(this->queue_mutex_); std::unique_lock<std::mutex> lock(this->queue_mutex_);
this->condition_.wait(lock, [this] { return this->stop || !this->tasks_.empty(); }); this->condition_.wait(lock, [this] { return this->stop || !this->tasks_.empty(); });
if (this->stop && this->tasks_.empty()) return; if (this->stop && this->tasks_.empty())
return;
task = std::move(this->tasks_.front()); task = std::move(this->tasks_.front());
this->tasks_.pop(); this->tasks_.pop();
} }
@ -95,7 +96,8 @@ ThreadPool::enqueue(F&& f, Args&&... args) -> std::future<typename std::result_o
std::unique_lock<std::mutex> lock(queue_mutex_); std::unique_lock<std::mutex> lock(queue_mutex_);
this->condition_.wait(lock, [this] { return this->tasks_.size() < max_queue_size_; }); this->condition_.wait(lock, [this] { return this->tasks_.size() < max_queue_size_; });
// don't allow enqueueing after stopping the pool // don't allow enqueueing after stopping the pool
if (stop) throw std::runtime_error("enqueue on stopped ThreadPool"); if (stop)
throw std::runtime_error("enqueue on stopped ThreadPool");
tasks_.emplace([task]() { (*task)(); }); tasks_.emplace([task]() { (*task)(); });
} }

View File

@ -39,7 +39,8 @@ TimeRecorder::GetTimeSpanStr(double span) {
void void
TimeRecorder::PrintTimeRecord(const std::string& msg, double span) { TimeRecorder::PrintTimeRecord(const std::string& msg, double span) {
std::string str_log; std::string str_log;
if (!header_.empty()) str_log += header_ + ": "; if (!header_.empty())
str_log += header_ + ": ";
str_log += msg; str_log += msg;
str_log += " ("; str_log += " (";
str_log += TimeRecorder::GetTimeSpanStr(span); str_log += TimeRecorder::GetTimeSpanStr(span);

View File

@ -83,8 +83,8 @@ ValidationUtil::ValidateTableDimension(int64_t dimension) {
Status Status
ValidationUtil::ValidateTableIndexType(int32_t index_type) { ValidationUtil::ValidateTableIndexType(int32_t index_type) {
int engine_type = (int)engine::EngineType(index_type); int engine_type = static_cast<int>(engine::EngineType(index_type));
if (engine_type <= 0 || engine_type > (int)engine::EngineType::MAX_VALUE) { if (engine_type <= 0 || engine_type > static_cast<int>(engine::EngineType::MAX_VALUE)) {
std::string msg = "Invalid index type: " + std::to_string(index_type); std::string msg = "Invalid index type: " + std::to_string(index_type);
SERVER_LOG_ERROR << msg; SERVER_LOG_ERROR << msg;
return Status(SERVER_INVALID_INDEX_TYPE, msg); return Status(SERVER_INVALID_INDEX_TYPE, msg);
@ -117,7 +117,8 @@ ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) {
Status Status
ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) { ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) {
if (metric_type != (int32_t)engine::MetricType::L2 && metric_type != (int32_t)engine::MetricType::IP) { if (metric_type != static_cast<int32_t>(engine::MetricType::L2) &&
metric_type != static_cast<int32_t>(engine::MetricType::IP)) {
std::string msg = "Invalid metric type: " + std::to_string(metric_type); std::string msg = "Invalid metric type: " + std::to_string(metric_type);
SERVER_LOG_ERROR << msg; SERVER_LOG_ERROR << msg;
return Status(SERVER_INVALID_INDEX_METRIC_TYPE, msg); return Status(SERVER_INVALID_INDEX_METRIC_TYPE, msg);
@ -151,7 +152,7 @@ Status
ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) { ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) {
int num_devices = 0; int num_devices = 0;
auto cuda_err = cudaGetDeviceCount(&num_devices); auto cuda_err = cudaGetDeviceCount(&num_devices);
if (cuda_err) { if (cuda_err != cudaSuccess) {
std::string msg = "Failed to get gpu card number, cuda error:" + std::to_string(cuda_err); std::string msg = "Failed to get gpu card number, cuda error:" + std::to_string(cuda_err);
SERVER_LOG_ERROR << msg; SERVER_LOG_ERROR << msg;
return Status(SERVER_UNEXPECTED_ERROR, msg); return Status(SERVER_UNEXPECTED_ERROR, msg);
@ -222,9 +223,8 @@ ValidationUtil::ValidateStringIsBool(const std::string& str) {
if (s == "true" || s == "on" || s == "yes" || s == "1" || s == "false" || s == "off" || s == "no" || s == "0" || if (s == "true" || s == "on" || s == "yes" || s == "1" || s == "false" || s == "off" || s == "no" || s == "0" ||
s.empty()) { s.empty()) {
return Status::OK(); return Status::OK();
} else {
return Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str);
} }
return Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str);
} }
Status Status