test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-01 03:18:29 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

Merge branch 'branch-0.3.0' into 'branch-0.3.0'

Browse Source 
MS - 130 Add prometheus_test

See merge request megasearch/vecwise_engine!131

Former-commit-id: ebb3c84b3a76f185658a293b91521cd98c0dbc56
This commit is contained in:
jinhai 2019-06-28 16:42:47 +08:00
commit ec2a868ed5
12 changed files with 130 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cpp/CHANGELOG.md
View File

@ -46,7 +46,7 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-85 - add NetIO metric
- MS-96 - add new query interface for specified files
- MS-97 - Add S3 SDK for MinIO Storage
- MS-130 - Add prometheus_test
## Task
- MS-74 - Change README.md in cpp
- MS-88 - Add support for arm architecture

12
cpp/conf/log_config.conf
View File

@ -1,6 +1,6 @@
* GLOBAL:
FORMAT = "%datetime | %level | %logger | %msg"
FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-global.log"
FILENAME = "/opt/milvus/logs/milvus-%datetime{%H:%m}-global.log"
ENABLED = true
TO_FILE = true
TO_STANDARD_OUTPUT = false
@ -8,12 +8,12 @@
PERFORMANCE_TRACKING = false
MAX_LOG_FILE_SIZE = 2097152 ## Throw log files away after 2MB
* DEBUG:
FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-debug.log"
FILENAME = "/opt/milvus/logs/milvus-%datetime{%H:%m}-debug.log"
ENABLED = true
* WARNING:
FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-warning.log"
FILENAME = "/opt/milvus/logs/milvus-%datetime{%H:%m}-warning.log"
* TRACE:
FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-trace.log"
FILENAME = "/opt/milvus/logs/milvus-%datetime{%H:%m}-trace.log"
* VERBOSE:
FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg"
TO_FILE = false
@ -21,7 +21,7 @@
## Error logs
* ERROR:
ENABLED = true
FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-error.log"
FILENAME = "/opt/milvus/logs/milvus-%datetime{%H:%m}-error.log"
* FATAL:
ENABLED = true
FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-fatal.log"
FILENAME = "/opt/milvus/logs/milvus-%datetime{%H:%m}-fatal.log"

4
cpp/conf/server_config.yaml
View File

@ -5,7 +5,7 @@ server_config:
mode: single # milvus deployment type: single, cluster
db_config:
db_path: /tmp/milvus # milvus data storage path
db_path: /opt/milvus # milvus data storage path
db_backend_url: http://127.0.0.1 # meta database uri
index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB
archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB
@ -22,7 +22,7 @@ metric_config:
license_config: # license configure
license_path: "/tmp/milvus/system.license" # license file path
license_path: "/opt/milvus/system.license" # license file path
cache_config: # cache configure
cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory

27
cpp/src/metrics/MetricBase.h
View File

@ -22,40 +22,21 @@ class MetricsBase{
}
virtual ServerError Init() {};
virtual void AddGroupSuccessTotalIncrement(double value = 1) {};
virtual void AddGroupFailTotalIncrement(double value = 1) {};
virtual void HasGroupSuccessTotalIncrement(double value = 1) {};
virtual void HasGroupFailTotalIncrement(double value = 1) {};
virtual void GetGroupSuccessTotalIncrement(double value = 1) {};
virtual void GetGroupFailTotalIncrement(double value = 1) {};
virtual void GetGroupFilesSuccessTotalIncrement(double value = 1) {};
virtual void GetGroupFilesFailTotalIncrement(double value = 1) {};
virtual void AddVectorsSuccessTotalIncrement(double value = 1) {};
virtual void AddVectorsFailTotalIncrement(double value = 1) {};
virtual void AddVectorsDurationHistogramOberve(double value) {};
virtual void SearchSuccessTotalIncrement(double value = 1) {};
virtual void SearchFailTotalIncrement(double value = 1) {};
virtual void SearchDurationHistogramObserve(double value) {};
virtual void RawFileSizeHistogramObserve(double value) {};
virtual void IndexFileSizeHistogramObserve(double value) {};
virtual void BuildIndexDurationSecondsHistogramObserve(double value) {};
virtual void AllBuildIndexDurationSecondsHistogramObserve(double value) {};
virtual void CacheUsageGaugeIncrement(double value = 1) {};
virtual void CacheUsageGaugeDecrement(double value = 1) {};
virtual void CacheUsageGaugeSet(double value) {};
virtual void MetaVisitTotalIncrement(double value = 1) {};
virtual void MetaVisitDurationSecondsHistogramObserve(double value) {};
virtual void MemUsagePercentGaugeSet(double value) {};
virtual void MemUsagePercentGaugeIncrement(double value = 1) {};
virtual void MemUsagePercentGaugeDecrement(double value = 1) {};
virtual void MemUsageTotalGaugeSet(double value) {};
virtual void MemUsageTotalGaugeIncrement(double value = 1) {};
virtual void MemUsageTotalGaugeDecrement(double value = 1) {};
virtual void MetaAccessTotalIncrement(double value = 1) {};
virtual void MetaAccessDurationSecondsHistogramObserve(double value) {};
virtual void FaissDiskLoadDurationSecondsHistogramObserve(double value) {};
virtual void FaissDiskLoadSizeBytesHistogramObserve(double value) {};
virtual void FaissDiskLoadIOSpeedHistogramObserve(double value) {};
virtual void CacheAccessTotalIncrement(double value = 1) {};
virtual void MemTableMergeDurationSecondsHistogramObserve(double value) {};
virtual void SearchIndexDataDurationSecondsHistogramObserve(double value) {};

23
cpp/src/metrics/PrometheusMetrics.cpp
View File

@ -60,9 +60,6 @@ PrometheusMetrics::GPUPercentGaugeSet() {
if(!startup_) return;
int numDevide = server::SystemInfo::GetInstance().num_device();
std::vector<unsigned int> values = server::SystemInfo::GetInstance().GPUPercent();
// for (int i = 0; i < numDevide; ++i) {
// GPU_percent_gauges_[i].Set(static_cast<double>(values[i]));
// }
if(numDevide >= 1) GPU0_percent_gauge_.Set(static_cast<double>(values[0]));
if(numDevide >= 2) GPU1_percent_gauge_.Set(static_cast<double>(values[1]));
if(numDevide >= 3) GPU2_percent_gauge_.Set(static_cast<double>(values[2]));
@ -77,13 +74,10 @@ PrometheusMetrics::GPUPercentGaugeSet() {
void PrometheusMetrics::GPUMemoryUsageGaugeSet() {
if(!startup_) return;
int numDevide = server::SystemInfo::GetInstance().num_device();
std::vector<unsigned long long> values = server::SystemInfo::GetInstance().GPUMemoryUsed();
constexpr unsigned long long MtoB = 1024*1024;
int numDevice = values.size();
// for (int i = 0; i < numDevice; ++i) {
// GPU_memory_usage_gauges_[i].Set(values[i]/MtoB);
// }
if(numDevice >=1) GPU0_memory_usage_gauge_.Set(values[0]/MtoB);
if(numDevice >=2) GPU1_memory_usage_gauge_.Set(values[1]/MtoB);
if(numDevice >=3) GPU2_memory_usage_gauge_.Set(values[2]/MtoB);
@ -146,21 +140,6 @@ void PrometheusMetrics::OctetsSet() {
outoctets_gauge_.Set((in_and_out_octets.second-old_outoctets)/total_second);
}
//void PrometheusMetrics::GpuPercentInit() {
// int num_device = SystemInfo::GetInstance().num_device();
// constexpr char device_number[] = "DeviceNum";
// for(int i = 0; i < num_device; ++ i) {
// GPU_percent_gauges_.emplace_back(GPU_percent_.Add({{device_number,std::to_string(i)}}));
// }
//
//}
//void PrometheusMetrics::GpuMemoryInit() {
// int num_device = SystemInfo::GetInstance().num_device();
// constexpr char device_number[] = "DeviceNum";
// for(int i = 0; i < num_device; ++ i) {
// GPU_memory_usage_gauges_.emplace_back(GPU_memory_usage_.Add({{device_number,std::to_string(i)}}));
// }
//}
}

31
cpp/src/metrics/PrometheusMetrics.h
View File

@ -34,10 +34,6 @@ class PrometheusMetrics: public MetricsBase {
public:
static PrometheusMetrics &
GetInstance() {
// switch(MetricCollectorType) {
// case: prometheus::
// static
// }
static PrometheusMetrics instance;
return instance;
}
@ -49,46 +45,21 @@ class PrometheusMetrics: public MetricsBase {
std::shared_ptr<prometheus::Exposer> exposer_ptr_;
std::shared_ptr<prometheus::Registry> registry_ = std::make_shared<prometheus::Registry>();
bool startup_ = false;
// void GpuPercentInit();
// void GpuMemoryInit();
public:
void AddGroupSuccessTotalIncrement(double value = 1.0) override { if(startup_) add_group_success_total_.Increment(value);};
void AddGroupFailTotalIncrement(double value = 1.0) override { if(startup_) add_group_fail_total_.Increment(value);};
void HasGroupSuccessTotalIncrement(double value = 1.0) override { if(startup_) has_group_success_total_.Increment(value);};
void HasGroupFailTotalIncrement(double value = 1.0) override { if(startup_) has_group_fail_total_.Increment(value);};
void GetGroupSuccessTotalIncrement(double value = 1.0) override { if(startup_) get_group_success_total_.Increment(value);};
void GetGroupFailTotalIncrement(double value = 1.0) override { if(startup_) get_group_fail_total_.Increment(value);};
void GetGroupFilesSuccessTotalIncrement(double value = 1.0) override { if(startup_) get_group_files_success_total_.Increment(value);};
void GetGroupFilesFailTotalIncrement(double value = 1.0) override { if(startup_) get_group_files_fail_total_.Increment(value);};
void SetStartup(bool startup) {startup_ = startup;};
void AddVectorsSuccessTotalIncrement(double value = 1.0) override { if(startup_) add_vectors_success_total_.Increment(value);};
void AddVectorsFailTotalIncrement(double value = 1.0) override { if(startup_) add_vectors_fail_total_.Increment(value);};
void AddVectorsDurationHistogramOberve(double value) override { if(startup_) add_vectors_duration_histogram_.Observe(value);};
void SearchSuccessTotalIncrement(double value = 1.0) override { if(startup_) search_success_total_.Increment(value);};
void SearchFailTotalIncrement(double value = 1.0) override { if(startup_) search_fail_total_.Increment(value); };
void SearchDurationHistogramObserve(double value) override { if(startup_) search_duration_histogram_.Observe(value);};
void RawFileSizeHistogramObserve(double value) override { if(startup_) raw_files_size_histogram_.Observe(value);};
void IndexFileSizeHistogramObserve(double value) override { if(startup_) index_files_size_histogram_.Observe(value);};
void BuildIndexDurationSecondsHistogramObserve(double value) override { if(startup_) build_index_duration_seconds_histogram_.Observe(value);};
void AllBuildIndexDurationSecondsHistogramObserve(double value) override { if(startup_) all_build_index_duration_seconds_histogram_.Observe(value);};
void CacheUsageGaugeIncrement(double value = 1.0) override { if(startup_) cache_usage_gauge_.Increment(value);};
void CacheUsageGaugeDecrement(double value = 1.0) override { if(startup_) cache_usage_gauge_.Decrement(value);};
void CacheUsageGaugeSet(double value) override { if(startup_) cache_usage_gauge_.Set(value);};
// void MetaVisitTotalIncrement(double value = 1.0) override { meta_visit_total_.Increment(value);};
// void MetaVisitDurationSecondsHistogramObserve(double value) override { meta_visit_duration_seconds_histogram_.Observe(value);};
void MemUsagePercentGaugeSet(double value) override { if(startup_) mem_usage_percent_gauge_.Set(value);};
void MemUsagePercentGaugeIncrement(double value = 1.0) override { if(startup_) mem_usage_percent_gauge_.Increment(value);};
void MemUsagePercentGaugeDecrement(double value = 1.0) override { if(startup_) mem_usage_percent_gauge_.Decrement(value);};
void MemUsageTotalGaugeSet(double value) override { if(startup_) mem_usage_total_gauge_.Set(value);};
void MemUsageTotalGaugeIncrement(double value = 1.0) override { if(startup_) mem_usage_total_gauge_.Increment(value);};
void MemUsageTotalGaugeDecrement(double value = 1.0) override { if(startup_) mem_usage_total_gauge_.Decrement(value);};
void MetaAccessTotalIncrement(double value = 1) override { if(startup_) meta_access_total_.Increment(value);};
void MetaAccessDurationSecondsHistogramObserve(double value) override { if(startup_) meta_access_duration_seconds_histogram_.Observe(value);};
void FaissDiskLoadDurationSecondsHistogramObserve(double value) override { if(startup_) faiss_disk_load_duration_seconds_histogram_.Observe(value);};
void FaissDiskLoadSizeBytesHistogramObserve(double value) override { if(startup_) faiss_disk_load_size_bytes_histogram_.Observe(value);};
// void FaissDiskLoadIOSpeedHistogramObserve(double value) { if(startup_) faiss_disk_load_IO_speed_histogram_.Observe(value);};
void FaissDiskLoadIOSpeedGaugeSet(double value) override { if(startup_) faiss_disk_load_IO_speed_gauge_.Set(value);};
void CacheAccessTotalIncrement(double value = 1) override { if(startup_) cache_access_total_.Increment(value);};

40
cpp/src/metrics/SystemInfo.cpp
View File

@ -135,46 +135,6 @@ SystemInfo::CPUPercent() {
return percent;
}
//std::unordered_map<int,std::vector<double>>
//SystemInfo::GetGPUMemPercent(){
// // return GPUID: MEM%
//
// //write GPU info to a file
// system("nvidia-smi pmon -c 1 > GPUInfo.txt");
// int pid = (int)getpid();
//
// //parse line
// std::ifstream read_file;
// read_file.open("GPUInfo.txt");
// std::string line;
// while(getline(read_file, line)){
// std::vector<std::string> words = split(line);
// // 0 1 2 3 4 5 6 7
// //words stand for gpuindex, pid, type, sm, mem, enc, dec, command respectively
// if(std::stoi(words[1]) != pid) continue;
// int GPUindex = std::stoi(words[0]);
// double sm_percent = std::stod(words[3]);
// double mem_percent = std::stod(words[4]);
//
// }
//
//}
//std::vector<std::string>
//SystemInfo::split(std::string input) {
// std::vector<std::string> words;
// input += " ";
// int word_start = 0;
// for (int i = 0; i < input.size(); ++i) {
// if(input[i] != ' ') continue;
// if(input[i] == ' ') {
// word_start = i + 1;
// continue;
// }
// words.push_back(input.substr(word_start,i-word_start));
// }
// return words;
//}
std::vector<unsigned int>
SystemInfo::GPUPercent() {

2
cpp/src/metrics/SystemInfo.h
View File

@ -59,8 +59,6 @@ class SystemInfo {
double MemoryPercent();
double CPUPercent();
std::pair<unsigned long long , unsigned long long > Octets();
// std::unordered_map<int,std::vector<double>> GetGPUMemPercent() {};
// std::vector<std::string> split(std::string input) {};
std::vector<unsigned int> GPUPercent();
std::vector<unsigned long long> GPUMemoryUsed();

3
cpp/unittest/metrics/CMakeLists.txt
View File

@ -55,8 +55,9 @@ set(count_test_src
${wrapper_src}
${metrics_src}
metrics_test.cpp
prometheus_test.cpp
../db/utils.cpp
)
metricbase_test.cpp)
add_executable(metrics_test ${count_test_src} ${require_files} )

56
cpp/unittest/metrics/metricbase_test.cpp Normal file
View File

@ -0,0 +1,56 @@
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#include "metrics/Metrics.h"
#include <gtest/gtest.h>
#include <iostream>
using namespace zilliz::milvus;
TEST(MetricbaseTest, Metricbase_Test){
server::MetricsBase instance = server::MetricsBase::GetInstance();
instance.Init();
server::SystemInfo::GetInstance().Init();
instance.AddVectorsSuccessTotalIncrement();
instance.AddVectorsFailTotalIncrement();
instance.AddVectorsDurationHistogramOberve(1.0);
instance.RawFileSizeHistogramObserve(1.0);
instance.IndexFileSizeHistogramObserve(1.0);
instance.BuildIndexDurationSecondsHistogramObserve(1.0);
instance.CacheUsageGaugeSet(1.0);
instance.MetaAccessTotalIncrement();
instance.MetaAccessDurationSecondsHistogramObserve(1.0);
instance.FaissDiskLoadDurationSecondsHistogramObserve(1.0);
instance.FaissDiskLoadSizeBytesHistogramObserve(1.0);
instance.FaissDiskLoadIOSpeedGaugeSet(1.0);
instance.CacheAccessTotalIncrement();
instance.MemTableMergeDurationSecondsHistogramObserve(1.0);
instance.SearchIndexDataDurationSecondsHistogramObserve(1.0);
instance.SearchRawDataDurationSecondsHistogramObserve(1.0);
instance.IndexFileSizeTotalIncrement();
instance.RawFileSizeTotalIncrement();
instance.IndexFileSizeGaugeSet(1.0);
instance.RawFileSizeGaugeSet(1.0);
instance.QueryResponseSummaryObserve(1.0);
instance.DiskStoreIOSpeedGaugeSet(1.0);
instance.DataFileSizeGaugeSet(1.0);
instance.AddVectorsSuccessGaugeSet(1.0);
instance.AddVectorsFailGaugeSet(1.0);
instance.QueryVectorResponseSummaryObserve(1.0, 1);
instance.QueryVectorResponsePerSecondGaugeSet(1.0);
instance.CPUUsagePercentSet();
instance.RAMUsagePercentSet();
instance.QueryResponsePerSecondGaugeSet(1.0);
instance.GPUPercentGaugeSet();
instance.GPUMemoryUsageGaugeSet();
instance.AddVectorsPerSecondGaugeSet(1,1,1);
instance.QueryIndexTypePerSecondSet("IVF", 1.0);
instance.ConnectionGaugeIncrement();
instance.ConnectionGaugeDecrement();
instance.KeepingAliveCounterIncrement();
instance.OctetsSet();
}

1
cpp/unittest/metrics/metrics_test.cpp
View File

@ -24,7 +24,6 @@
using namespace zilliz::milvus;
TEST_F(DBTest, Metric_Tes) {
server::SystemInfo::GetInstance().Init();

57
cpp/unittest/metrics/prometheus_test.cpp Normal file
View File

@ -0,0 +1,57 @@
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#include "metrics/PrometheusMetrics.h"
#include <gtest/gtest.h>
#include <iostream>
using namespace zilliz::milvus;
TEST(PrometheusTest, Prometheus_Test){
server::PrometheusMetrics instance = server::PrometheusMetrics::GetInstance();
instance.Init();
instance.SetStartup(true);
server::SystemInfo::GetInstance().Init();
instance.AddVectorsSuccessTotalIncrement();
instance.AddVectorsFailTotalIncrement();
instance.AddVectorsDurationHistogramOberve(1.0);
instance.RawFileSizeHistogramObserve(1.0);
instance.IndexFileSizeHistogramObserve(1.0);
instance.BuildIndexDurationSecondsHistogramObserve(1.0);
instance.CacheUsageGaugeSet(1.0);
instance.MetaAccessTotalIncrement();
instance.MetaAccessDurationSecondsHistogramObserve(1.0);
instance.FaissDiskLoadDurationSecondsHistogramObserve(1.0);
instance.FaissDiskLoadSizeBytesHistogramObserve(1.0);
instance.FaissDiskLoadIOSpeedGaugeSet(1.0);
instance.CacheAccessTotalIncrement();
instance.MemTableMergeDurationSecondsHistogramObserve(1.0);
instance.SearchIndexDataDurationSecondsHistogramObserve(1.0);
instance.SearchRawDataDurationSecondsHistogramObserve(1.0);
instance.IndexFileSizeTotalIncrement();
instance.RawFileSizeTotalIncrement();
instance.IndexFileSizeGaugeSet(1.0);
instance.RawFileSizeGaugeSet(1.0);
instance.QueryResponseSummaryObserve(1.0);
instance.DiskStoreIOSpeedGaugeSet(1.0);
instance.DataFileSizeGaugeSet(1.0);
instance.AddVectorsSuccessGaugeSet(1.0);
instance.AddVectorsFailGaugeSet(1.0);
instance.QueryVectorResponseSummaryObserve(1.0, 1);
instance.QueryVectorResponsePerSecondGaugeSet(1.0);
instance.CPUUsagePercentSet();
instance.RAMUsagePercentSet();
instance.QueryResponsePerSecondGaugeSet(1.0);
instance.GPUPercentGaugeSet();
instance.GPUMemoryUsageGaugeSet();
instance.AddVectorsPerSecondGaugeSet(1,1,1);
instance.QueryIndexTypePerSecondSet("IVF", 1.0);
instance.ConnectionGaugeIncrement();
instance.ConnectionGaugeDecrement();
instance.KeepingAliveCounterIncrement();
instance.OctetsSet();
}