test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-05 05:18:52 +08:00
Code Issues Actions 14 Packages Projects Releases Wiki Activity

use rocksdb column family

Browse Source 
Former-commit-id: 8cbc5d91e02e6bbc0eff1e329b4c76687d2c1f74
This commit is contained in:
groot 2019-05-13 15:17:05 +08:00
parent 354f68a96f
commit 56291b218d
5 changed files with 101 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
cpp/conf/server_config.yaml
View File

@ -1,11 +1,11 @@
server_config:
address: 0.0.0.0
address: 127.0.0.1
port: 33001
transfer_protocol: json #optional: binary, compact, json, debug
server_mode: thread_pool #optional: simple, thread_pool
db_config:
db_path: /var/vecwise
db_path: /tmp/vecwise
db_backend_url: http://127.0.0.1
db_flush_interval: 5 #unit: second
idmapper_max_open_file: 128
@ -13,7 +13,7 @@ db_config:
log_config:
global:
format: "%datetime | %level | %logger | %msg"
filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-global.log"
filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-global.log"
enabled: true
to_file: true
to_standard_output: true
@ -21,22 +21,22 @@ log_config:
performance_tracking: false
max_log_file_size: 2097152 # throw log files away after 2mb
debug:
filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-debug.log"
filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-debug.log"
enabled: true
warning:
filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-warning.log"
filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-warning.log"
trace:
filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-trace.log"
filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-trace.log"
verbose:
format: "%datetime{%d/%m/%y} | %level-%vlevel | %msg"
to_file: false
to_standard_output: true
error:
enabled: false
filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-error.log"
filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-error.log"
fatal:
enabled: false
filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-fatal.log"
filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-fatal.log"
cache_config:
cpu_cache_capacity: 16 # unit: GB

12
cpp/conf/vecwise_engine_log.conf
View File

@ -1,6 +1,6 @@
* GLOBAL:
FORMAT = "%datetime | %level | %logger | %msg"
FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-global.log"
FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-global.log"
ENABLED = true
TO_FILE = true
TO_STANDARD_OUTPUT = true
@ -8,12 +8,12 @@
PERFORMANCE_TRACKING = false
MAX_LOG_FILE_SIZE = 2097152 ## Throw log files away after 2MB
* DEBUG:
FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-debug.log"
FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-debug.log"
ENABLED = true
* WARNING:
FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-warning.log"
FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-warning.log"
* TRACE:
FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-trace.log"
FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-trace.log"
* VERBOSE:
FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg"
TO_FILE = false
@ -21,7 +21,7 @@
## Error logs
* ERROR:
ENABLED = false
FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-error.log"
FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-error.log"
* FATAL:
ENABLED = false
FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-fatal.log"
FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-fatal.log"

81
cpp/src/server/VecIdMapper.cpp
View File

@ -13,6 +13,8 @@
#include "rocksdb/slice.h"
#include "rocksdb/options.h"
#include <exception>
namespace zilliz {
namespace vecwise {
namespace server {
@ -36,26 +38,33 @@ SimpleIdMapper::~SimpleIdMapper() {
}
ServerError SimpleIdMapper::Put(const std::string& nid, const std::string& sid) {
ids_[nid] = sid;
//not thread-safe
ServerError SimpleIdMapper::Put(const std::string& nid, const std::string& sid, const std::string& group) {
ID_MAPPING& mapping = id_groups_[group];
mapping[nid] = sid;
return SERVER_SUCCESS;
}
ServerError SimpleIdMapper::Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid) {
//not thread-safe
ServerError SimpleIdMapper::Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid, const std::string& group) {
if(nid.size() != sid.size()) {
return SERVER_INVALID_ARGUMENT;
}
ID_MAPPING& mapping = id_groups_[group];
for(size_t i = 0; i < nid.size(); i++) {
ids_[nid[i]] = sid[i];
mapping[nid[i]] = sid[i];
}
return SERVER_SUCCESS;
}
ServerError SimpleIdMapper::Get(const std::string& nid, std::string& sid) const {
auto iter = ids_.find(nid);
if(iter == ids_.end()) {
//not thread-safe
ServerError SimpleIdMapper::Get(const std::string& nid, std::string& sid, const std::string& group) const {
ID_MAPPING& mapping = id_groups_[group];
auto iter = mapping.find(nid);
if(iter == mapping.end()) {
return SERVER_INVALID_ARGUMENT;
}
@ -64,13 +73,16 @@ ServerError SimpleIdMapper::Get(const std::string& nid, std::string& sid) const
return SERVER_SUCCESS;
}
ServerError SimpleIdMapper::Get(const std::vector<std::string>& nid, std::vector<std::string>& sid) const {
//not thread-safe
ServerError SimpleIdMapper::Get(const std::vector<std::string>& nid, std::vector<std::string>& sid, const std::string& group) const {
sid.clear();
ID_MAPPING& mapping = id_groups_[group];
ServerError err = SERVER_SUCCESS;
for(size_t i = 0; i < nid.size(); i++) {
auto iter = ids_.find(nid[i]);
if(iter == ids_.end()) {
auto iter = mapping.find(nid[i]);
if(iter == mapping.end()) {
sid.push_back("");
SERVER_LOG_ERROR << "ID mapper failed to find id: " << nid[i];
err = SERVER_INVALID_ARGUMENT;
@ -83,8 +95,10 @@ ServerError SimpleIdMapper::Get(const std::vector<std::string>& nid, std::vector
return err;
}
ServerError SimpleIdMapper::Delete(const std::string& nid) {
ids_.erase(nid);
//not thread-safe
ServerError SimpleIdMapper::Delete(const std::string& nid, const std::string& group) {
ID_MAPPING& mapping = id_groups_[group];
mapping.erase(nid);
return SERVER_SUCCESS;
}
@ -104,12 +118,43 @@ RocksIdMapper::RocksIdMapper() {
options.create_if_missing = true;
options.max_open_files = config.GetInt32Value(CONFIG_DB_IDMAPPER_MAX_FILE, 128);
//load column families
std::vector<std::string> column_names;
rocksdb::Status s = rocksdb::DB::ListColumnFamilies(options, db_path, &column_names);
if (!s.ok()) {
SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString();
}
std::vector<rocksdb::ColumnFamilyDescriptor> column_families;
for(auto& column_name : column_names) {
rocksdb::ColumnFamilyDescriptor desc;
desc.name = column_name;
column_families.emplace_back(desc);
}
// open DB
rocksdb::Status s = rocksdb::DB::Open(options, db_path, &db_);
std::vector<rocksdb::ColumnFamilyHandle*> column_handles;
s = rocksdb::DB::Open(options, db_path, column_families, &column_handles, &db_);
if(!s.ok()) {
SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString();
db_ = nullptr;
}
try {
rocksdb::ColumnFamilyHandle *cf;
s = db_->CreateColumnFamily(rocksdb::ColumnFamilyOptions(), "222", &cf);
if (!s.ok()) {
SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString();
}
std::vector<std::string> column_families;
s = db_->ListColumnFamilies(options, db_path, &column_families);
if (!s.ok()) {
SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString();
}
} catch(std::exception& ex) {
std::cout << ex.what() << std::endl;
}
}
RocksIdMapper::~RocksIdMapper() {
if(db_) {
@ -118,7 +163,7 @@ RocksIdMapper::~RocksIdMapper() {
}
}
ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid) {
ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid, const std::string& group) {
if(db_ == nullptr) {
return SERVER_NULL_POINTER;
}
@ -134,7 +179,7 @@ ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid) {
return SERVER_SUCCESS;
}
ServerError RocksIdMapper::Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid) {
ServerError RocksIdMapper::Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid, const std::string& group) {
if(nid.size() != sid.size()) {
return SERVER_INVALID_ARGUMENT;
}
@ -150,7 +195,7 @@ ServerError RocksIdMapper::Put(const std::vector<std::string>& nid, const std::v
return err;
}
ServerError RocksIdMapper::Get(const std::string& nid, std::string& sid) const {
ServerError RocksIdMapper::Get(const std::string& nid, std::string& sid, const std::string& group) const {
if(db_ == nullptr) {
return SERVER_NULL_POINTER;
}
@ -165,7 +210,7 @@ ServerError RocksIdMapper::Get(const std::string& nid, std::string& sid) const {
return SERVER_SUCCESS;
}
ServerError RocksIdMapper::Get(const std::vector<std::string>& nid, std::vector<std::string>& sid) const {
ServerError RocksIdMapper::Get(const std::vector<std::string>& nid, std::vector<std::string>& sid, const std::string& group) const {
sid.clear();
ServerError err = SERVER_SUCCESS;
@ -185,7 +230,7 @@ ServerError RocksIdMapper::Get(const std::vector<std::string>& nid, std::vector<
return err;
}
ServerError RocksIdMapper::Delete(const std::string& nid) {
ServerError RocksIdMapper::Delete(const std::string& nid, const std::string& group) {
if(db_ == nullptr) {
return SERVER_NULL_POINTER;
}

33
cpp/src/server/VecIdMapper.h
View File

@ -25,14 +25,14 @@ public:
virtual ~IVecIdMapper(){}
virtual ServerError Put(const std::string& nid, const std::string& sid) = 0;
virtual ServerError Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid) = 0;
virtual ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") = 0;
virtual ServerError Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid, const std::string& group = "") = 0;
virtual ServerError Get(const std::string& nid, std::string& sid) const = 0;
virtual ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const = 0;
//NOTE: the 'sid' will be cleared at begin of the function
virtual ServerError Get(const std::vector<std::string>& nid, std::vector<std::string>& sid) const = 0;
virtual ServerError Get(const std::vector<std::string>& nid, std::vector<std::string>& sid, const std::string& group = "") const = 0;
virtual ServerError Delete(const std::string& nid) = 0;
virtual ServerError Delete(const std::string& nid, const std::string& group = "") = 0;
};
class SimpleIdMapper : public IVecIdMapper{
@ -40,16 +40,17 @@ public:
SimpleIdMapper();
~SimpleIdMapper();
ServerError Put(const std::string& nid, const std::string& sid) override;
ServerError Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid) override;
ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") override;
ServerError Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid, const std::string& group = "") override;
ServerError Get(const std::string& nid, std::string& sid) const override;
ServerError Get(const std::vector<std::string>& nid, std::vector<std::string>& sid) const override;
ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const override;
ServerError Get(const std::vector<std::string>& nid, std::vector<std::string>& sid, const std::string& group = "") const override;
ServerError Delete(const std::string& nid) override;
ServerError Delete(const std::string& nid, const std::string& group = "") override;
private:
std::unordered_map<std::string, std::string> ids_;
using ID_MAPPING = std::unordered_map<std::string, std::string>;
mutable std::unordered_map<std::string, ID_MAPPING> id_groups_;
};
class RocksIdMapper : public IVecIdMapper{
@ -57,13 +58,13 @@ public:
RocksIdMapper();
~RocksIdMapper();
ServerError Put(const std::string& nid, const std::string& sid) override;
ServerError Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid) override;
ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") override;
ServerError Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid, const std::string& group = "") override;
ServerError Get(const std::string& nid, std::string& sid) const override;
ServerError Get(const std::vector<std::string>& nid, std::vector<std::string>& sid) const override;
ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const override;
ServerError Get(const std::vector<std::string>& nid, std::vector<std::string>& sid, const std::string& group = "") const override;
ServerError Delete(const std::string& nid) override;
ServerError Delete(const std::string& nid, const std::string& group = "") override;
private:
rocksdb::DB* db_;

10
cpp/unittest/server/idmapper_test.cpp
View File

@ -44,9 +44,13 @@ TEST(IdMapperTest, IDMAPPER_TEST) {
nid.clear();
sid.clear();
const int64_t count = 1000000;
for(int64_t i = 0; i < count; i++) {
nid.push_back(std::to_string(i+100000));
sid.push_back("val_" + std::to_string(i));
{
server::TimeRecorder rc("prepare id data");
for (int64_t i = 0; i < count; i++) {
nid.push_back(std::to_string(i + 100000));
sid.push_back("val_" + std::to_string(i));
}
rc.Record("done!");
}
{