test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-05 05:18:52 +08:00
Code Issues Actions 14 Packages Projects Releases Wiki Activity

Merge branch 'release-v1' of 192.168.1.105:jinhai/vecwise_engine into release-v1

Browse Source 
Former-commit-id: 4170b07a53013331e4afcafe8f00680b302a6db8
This commit is contained in:
xj.lin 2019-05-14 15:37:49 +08:00
commit 8b11855e90
22 changed files with 69 additions and 3997 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
cpp/conf/server_config.yaml
View File

@ -5,7 +5,7 @@ server_config:
server_mode: thread_pool #optional: simple, thread_pool
db_config:
db_path: /var/vecwise
db_path: /tmp/vecwise
db_backend_url: http://127.0.0.1
db_flush_interval: 5 #unit: second
idmapper_max_open_file: 128
@ -13,7 +13,7 @@ db_config:
log_config:
global:
format: "%datetime | %level | %logger | %msg"
filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-global.log"
filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-global.log"
enabled: true
to_file: true
to_standard_output: true
@ -21,22 +21,22 @@ log_config:
performance_tracking: false
max_log_file_size: 2097152 # throw log files away after 2mb
debug:
filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-debug.log"
filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-debug.log"
enabled: true
warning:
filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-warning.log"
filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-warning.log"
trace:
filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-trace.log"
filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-trace.log"
verbose:
format: "%datetime{%d/%m/%y} | %level-%vlevel | %msg"
to_file: false
to_standard_output: true
error:
enabled: false
filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-error.log"
filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-error.log"
fatal:
enabled: false
filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-fatal.log"
filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-fatal.log"
cache_config:
cpu_cache_capacity: 16 # unit: GB

12
cpp/conf/vecwise_engine_log.conf
View File

@ -1,6 +1,6 @@
* GLOBAL:
FORMAT = "%datetime | %level | %logger | %msg"
FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-global.log"
FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-global.log"
ENABLED = true
TO_FILE = true
TO_STANDARD_OUTPUT = true
@ -8,12 +8,12 @@
PERFORMANCE_TRACKING = false
MAX_LOG_FILE_SIZE = 2097152 ## Throw log files away after 2MB
* DEBUG:
FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-debug.log"
FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-debug.log"
ENABLED = true
* WARNING:
FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-warning.log"
FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-warning.log"
* TRACE:
FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-trace.log"
FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-trace.log"
* VERBOSE:
FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg"
TO_FILE = false
@ -21,7 +21,7 @@
## Error logs
* ERROR:
ENABLED = false
FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-error.log"
FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-error.log"
* FATAL:
ENABLED = false
FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-fatal.log"
FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-fatal.log"

26
cpp/src/server/VecServiceHandler.cpp
View File

@ -17,6 +17,8 @@ namespace zilliz {
namespace vecwise {
namespace server {
using namespace megasearch;
namespace {
class TimeRecordWrapper {
public:
@ -38,17 +40,17 @@ namespace {
}
const std::map<ServerError, zilliz::VecErrCode::type>& ErrorMap() {
static const std::map<ServerError, zilliz::VecErrCode::type> code_map = {
{SERVER_UNEXPECTED_ERROR, zilliz::VecErrCode::ILLEGAL_ARGUMENT},
{SERVER_NULL_POINTER, zilliz::VecErrCode::ILLEGAL_ARGUMENT},
{SERVER_INVALID_ARGUMENT, zilliz::VecErrCode::ILLEGAL_ARGUMENT},
{SERVER_FILE_NOT_FOUND, zilliz::VecErrCode::ILLEGAL_ARGUMENT},
{SERVER_NOT_IMPLEMENT, zilliz::VecErrCode::ILLEGAL_ARGUMENT},
{SERVER_BLOCKING_QUEUE_EMPTY, zilliz::VecErrCode::ILLEGAL_ARGUMENT},
{SERVER_GROUP_NOT_EXIST, zilliz::VecErrCode::GROUP_NOT_EXISTS},
{SERVER_INVALID_TIME_RANGE, zilliz::VecErrCode::ILLEGAL_TIME_RANGE},
{SERVER_INVALID_VECTOR_DIMENSION, zilliz::VecErrCode::ILLEGAL_VECTOR_DIMENSION},
const std::map<ServerError, VecErrCode::type>& ErrorMap() {
static const std::map<ServerError, VecErrCode::type> code_map = {
{SERVER_UNEXPECTED_ERROR, VecErrCode::ILLEGAL_ARGUMENT},
{SERVER_NULL_POINTER, VecErrCode::ILLEGAL_ARGUMENT},
{SERVER_INVALID_ARGUMENT, VecErrCode::ILLEGAL_ARGUMENT},
{SERVER_FILE_NOT_FOUND, VecErrCode::ILLEGAL_ARGUMENT},
{SERVER_NOT_IMPLEMENT, VecErrCode::ILLEGAL_ARGUMENT},
{SERVER_BLOCKING_QUEUE_EMPTY, VecErrCode::ILLEGAL_ARGUMENT},
{SERVER_GROUP_NOT_EXIST, VecErrCode::GROUP_NOT_EXISTS},
{SERVER_INVALID_TIME_RANGE, VecErrCode::ILLEGAL_TIME_RANGE},
{SERVER_INVALID_VECTOR_DIMENSION, VecErrCode::ILLEGAL_VECTOR_DIMENSION},
};
return code_map;
@ -82,7 +84,7 @@ namespace {
task_ptr->WaitToFinish();
ServerError err = task_ptr->ErrorCode();
if (err != SERVER_SUCCESS) {
zilliz::VecException ex;
VecException ex;
ex.__set_code(ErrorMap().at(err));
std::string msg = task_ptr->ErrorMsg();
if(msg.empty()){

2
cpp/src/server/VecServiceHandler.h
View File

@ -22,6 +22,8 @@ namespace zilliz {
namespace vecwise {
namespace server {
using namespace megasearch;
class VecServiceHandler : virtual public VecServiceIf {
public:
VecServiceHandler() {

1
cpp/src/server/VecServiceTask.h
View File

@ -19,6 +19,7 @@ namespace zilliz {
namespace vecwise {
namespace server {
using namespace megasearch;
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class AddGroupTask : public BaseTask {
public:

2
cpp/src/thrift/gen-cpp/VecService.cpp
View File

@ -6,7 +6,7 @@
*/
#include "VecService.h"
namespace zilliz {
namespace megasearch {
VecService_add_group_args::~VecService_add_group_args() throw() {

2
cpp/src/thrift/gen-cpp/VecService.h
View File

@ -11,7 +11,7 @@
#include <thrift/async/TConcurrentClientSyncInfo.h>
#include "megasearch_types.h"
namespace zilliz {
namespace megasearch {
#ifdef _MSC_VER
#pragma warning( push )

2
cpp/src/thrift/gen-cpp/VecService_server.skeleton.cpp
View File

@ -12,7 +12,7 @@ using namespace ::apache::thrift::protocol;
using namespace ::apache::thrift::transport;
using namespace ::apache::thrift::server;
using namespace ::zilliz;
using namespace ::megasearch;
class VecServiceHandler : virtual public VecServiceIf {
public:

2
cpp/src/thrift/gen-cpp/megasearch_constants.cpp
View File

@ -6,7 +6,7 @@
*/
#include "megasearch_constants.h"
namespace zilliz {
namespace megasearch {
const megasearchConstants g_megasearch_constants;

2
cpp/src/thrift/gen-cpp/megasearch_constants.h
View File

@ -9,7 +9,7 @@
#include "megasearch_types.h"
namespace zilliz {
namespace megasearch {
class megasearchConstants {
public:

2
cpp/src/thrift/gen-cpp/megasearch_types.cpp
View File

@ -11,7 +11,7 @@
#include <thrift/TToString.h>
namespace zilliz {
namespace megasearch {
int _kVecErrCodeValues[] = {
VecErrCode::SUCCESS,

2
cpp/src/thrift/gen-cpp/megasearch_types.h
View File

@ -18,7 +18,7 @@
#include <thrift/stdcxx.h>
namespace zilliz {
namespace megasearch {
struct VecErrCode {
enum type {

47
cpp/src/thrift/gen-py/py_sample.py
View File

@ -1,22 +1,15 @@
import time
import struct
from zilliz import VecService
from megasearch import VecService
from thrift import Thrift
#Note: pip install thrift
from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol, TCompactProtocol, TJSONProtocol
def print_time_cost(desc, t):
time_now = time.time()
print(desc + ' cost ', time_now - t, ' sec')
return time_now
def test_vecwise():
def test_megasearch():
try:
time_start = time.time()
#connect
transport = TSocket.TSocket('localhost', 33001)
transport = TTransport.TBufferedTransport(transport)
@ -24,26 +17,13 @@ def test_vecwise():
client = VecService.Client(protocol)
transport.open()
time_start = print_time_cost('connection', time_start)
print("connected");
#add group
group = VecService.VecGroup("py_group_" + time.strftime('%H%M%S'), 256)
group = VecService.VecGroup("test_" + time.strftime('%H%M%S'), 256)
client.add_group(group)
time_start = print_time_cost('add group', time_start)
#build vectors
# vec_list = VecService.VecTensorList([])
# for i in range(10000):
# vec = VecService.VecTensor("normal_"+str(i), [])
# for k in range(group.dimension):
# vec.tensor.append(k)
# vec_list.tensor_list.append(vec)
#time_start = print_time_cost('build normal vectors', time_start)
#add vectors
#client.add_vector_batch(group.id, vec_list)
#time_start = print_time_cost('add normal vectors', time_start))
print("group added");
# build binary vectors
bin_vec_list = VecService.VecBinaryTensorList([])
@ -55,14 +35,12 @@ def test_vecwise():
bin_vec.tensor = struct.pack(str(group.dimension)+"d", *a)
bin_vec_list.tensor_list.append(bin_vec)
time_start = print_time_cost('build binary vectors', time_start)
# add vectors
client.add_binary_vector_batch(group.id, bin_vec_list)
time_start = print_time_cost('add binary vectors', time_start)
time.sleep(5)
time_start = print_time_cost('sleep 5 seconds', time_start)
wait_storage = 5
print("wait {} seconds for persisting data".format(wait_storage))
time.sleep(wait_storage)
# search vector
a = []
@ -72,18 +50,19 @@ def test_vecwise():
bin_vec.tensor = struct.pack(str(group.dimension) + "d", *a)
filter = VecService.VecSearchFilter()
res = VecService.VecSearchResult()
print("begin search ...");
res = client.search_binary_vector(group.id, 5, bin_vec, filter)
time_start = print_time_cost('search binary vectors', time_start)
print('result count: ' + str(len(res.result_list)))
for item in res.result_list:
print(item.uid)
transport.close()
time_start = print_time_cost('close connection', time_start)
print("disconnected");
except VecService.VecException as ex:
print(ex.reason)
test_vecwise()
test_megasearch()

187
cpp/src/thrift/gen-py/zilliz/VecService-remote
View File

@ -1,187 +0,0 @@
#!/usr/bin/env python
#
# Autogenerated by Thrift Compiler (0.11.0)
#
# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
#
# options string: py
#
import sys
import pprint
if sys.version_info[0] > 2:
from urllib.parse import urlparse
else:
from urlparse import urlparse
from thrift.transport import TTransport, TSocket, TSSLSocket, THttpClient
from thrift.protocol.TBinaryProtocol import TBinaryProtocol
from zilliz import VecService
from zilliz.ttypes import *
if len(sys.argv) <= 1 or sys.argv[1] == '--help':
print('')
print('Usage: ' + sys.argv[0] + ' [-h host[:port]] [-u url] [-f[ramed]] [-s[sl]] [-novalidate] [-ca_certs certs] [-keyfile keyfile] [-certfile certfile] function [arg1 [arg2...]]')
print('')
print('Functions:')
print(' void add_group(VecGroup group)')
print(' VecGroup get_group(string group_id)')
print(' void del_group(string group_id)')
print(' string add_vector(string group_id, VecTensor tensor)')
print(' add_vector_batch(string group_id, VecTensorList tensor_list)')
print(' string add_binary_vector(string group_id, VecBinaryTensor tensor)')
print(' add_binary_vector_batch(string group_id, VecBinaryTensorList tensor_list)')
print(' VecSearchResult search_vector(string group_id, i64 top_k, VecTensor tensor, VecSearchFilter filter)')
print(' VecSearchResultList search_vector_batch(string group_id, i64 top_k, VecTensorList tensor_list, VecSearchFilter filter)')
print(' VecSearchResult search_binary_vector(string group_id, i64 top_k, VecBinaryTensor tensor, VecSearchFilter filter)')
print(' VecSearchResultList search_binary_vector_batch(string group_id, i64 top_k, VecBinaryTensorList tensor_list, VecSearchFilter filter)')
print('')
sys.exit(0)
pp = pprint.PrettyPrinter(indent=2)
host = 'localhost'
port = 9090
uri = ''
framed = False
ssl = False
validate = True
ca_certs = None
keyfile = None
certfile = None
http = False
argi = 1
if sys.argv[argi] == '-h':
parts = sys.argv[argi + 1].split(':')
host = parts[0]
if len(parts) > 1:
port = int(parts[1])
argi += 2
if sys.argv[argi] == '-u':
url = urlparse(sys.argv[argi + 1])
parts = url[1].split(':')
host = parts[0]
if len(parts) > 1:
port = int(parts[1])
else:
port = 80
uri = url[2]
if url[4]:
uri += '?%s' % url[4]
http = True
argi += 2
if sys.argv[argi] == '-f' or sys.argv[argi] == '-framed':
framed = True
argi += 1
if sys.argv[argi] == '-s' or sys.argv[argi] == '-ssl':
ssl = True
argi += 1
if sys.argv[argi] == '-novalidate':
validate = False
argi += 1
if sys.argv[argi] == '-ca_certs':
ca_certs = sys.argv[argi+1]
argi += 2
if sys.argv[argi] == '-keyfile':
keyfile = sys.argv[argi+1]
argi += 2
if sys.argv[argi] == '-certfile':
certfile = sys.argv[argi+1]
argi += 2
cmd = sys.argv[argi]
args = sys.argv[argi + 1:]
if http:
transport = THttpClient.THttpClient(host, port, uri)
else:
if ssl:
socket = TSSLSocket.TSSLSocket(host, port, validate=validate, ca_certs=ca_certs, keyfile=keyfile, certfile=certfile)
else:
socket = TSocket.TSocket(host, port)
if framed:
transport = TTransport.TFramedTransport(socket)
else:
transport = TTransport.TBufferedTransport(socket)
protocol = TBinaryProtocol(transport)
client = VecService.Client(protocol)
transport.open()
if cmd == 'add_group':
if len(args) != 1:
print('add_group requires 1 args')
sys.exit(1)
pp.pprint(client.add_group(eval(args[0]),))
elif cmd == 'get_group':
if len(args) != 1:
print('get_group requires 1 args')
sys.exit(1)
pp.pprint(client.get_group(args[0],))
elif cmd == 'del_group':
if len(args) != 1:
print('del_group requires 1 args')
sys.exit(1)
pp.pprint(client.del_group(args[0],))
elif cmd == 'add_vector':
if len(args) != 2:
print('add_vector requires 2 args')
sys.exit(1)
pp.pprint(client.add_vector(args[0], eval(args[1]),))
elif cmd == 'add_vector_batch':
if len(args) != 2:
print('add_vector_batch requires 2 args')
sys.exit(1)
pp.pprint(client.add_vector_batch(args[0], eval(args[1]),))
elif cmd == 'add_binary_vector':
if len(args) != 2:
print('add_binary_vector requires 2 args')
sys.exit(1)
pp.pprint(client.add_binary_vector(args[0], eval(args[1]),))
elif cmd == 'add_binary_vector_batch':
if len(args) != 2:
print('add_binary_vector_batch requires 2 args')
sys.exit(1)
pp.pprint(client.add_binary_vector_batch(args[0], eval(args[1]),))
elif cmd == 'search_vector':
if len(args) != 4:
print('search_vector requires 4 args')
sys.exit(1)
pp.pprint(client.search_vector(args[0], eval(args[1]), eval(args[2]), eval(args[3]),))
elif cmd == 'search_vector_batch':
if len(args) != 4:
print('search_vector_batch requires 4 args')
sys.exit(1)
pp.pprint(client.search_vector_batch(args[0], eval(args[1]), eval(args[2]), eval(args[3]),))
elif cmd == 'search_binary_vector':
if len(args) != 4:
print('search_binary_vector requires 4 args')
sys.exit(1)
pp.pprint(client.search_binary_vector(args[0], eval(args[1]), eval(args[2]), eval(args[3]),))
elif cmd == 'search_binary_vector_batch':
if len(args) != 4:
print('search_binary_vector_batch requires 4 args')
sys.exit(1)
pp.pprint(client.search_binary_vector_batch(args[0], eval(args[1]), eval(args[2]), eval(args[3]),))
else:
print('Unrecognized method %s' % cmd)
sys.exit(1)
transport.close()

2548
cpp/src/thrift/gen-py/zilliz/VecService.py
View File

File diff suppressed because it is too large Load Diff

1
cpp/src/thrift/gen-py/zilliz/__init__.py
View File

@ -1 +0,0 @@
__all__ = ['ttypes', 'constants', 'VecService']

14
cpp/src/thrift/gen-py/zilliz/constants.py
View File

@ -1,14 +0,0 @@
#
# Autogenerated by Thrift Compiler (0.11.0)
#
# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
#
# options string: py
#
from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException
from thrift.protocol.TProtocol import TProtocolException
from thrift.TRecursive import fix_spec
import sys
from .ttypes import *

1165
cpp/src/thrift/gen-py/zilliz/ttypes.py
View File

File diff suppressed because it is too large Load Diff

20
cpp/src/thrift/megasearch.thrift
View File

@ -3,16 +3,16 @@
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
namespace cl zilliz
namespace cpp zilliz
namespace py zilliz
namespace d zilliz
namespace dart zilliz
namespace java zilliz
namespace perl zilliz
namespace php zilliz
namespace haxe zilliz
namespace netcore zilliz
namespace cl megasearch
namespace cpp megasearch
namespace py megasearch
namespace d megasearch
namespace dart megasearch
namespace java megasearch
namespace perl megasearch
namespace php megasearch
namespace haxe megasearch
namespace netcore megasearch
enum VecErrCode {
SUCCESS = 0,

2
cpp/test_client/src/ClientSession.cpp
View File

@ -26,6 +26,8 @@ namespace zilliz {
namespace vecwise {
namespace client {
using namespace megasearch;
using namespace ::apache::thrift;
using namespace ::apache::thrift::protocol;
using namespace ::apache::thrift::transport;

2
cpp/test_client/src/ClientSession.h
View File

@ -13,7 +13,7 @@ namespace zilliz {
namespace vecwise {
namespace client {
using VecServiceClientPtr = std::shared_ptr<VecServiceClient>;
using VecServiceClientPtr = std::shared_ptr<megasearch::VecServiceClient>;
class ClientSession {
public:

11
cpp/test_client/src/ClientTest.cpp
View File

@ -15,6 +15,7 @@
#include <time.h>
using namespace megasearch;
using namespace zilliz;
using namespace zilliz::vecwise;
using namespace zilliz::vecwise::client;
@ -129,23 +130,23 @@ TEST(AddVector, CLIENT_TEST) {
//verify get invalid group
try {
std::string id;
zilliz::VecTensor tensor;
VecTensor tensor;
for(int32_t i = 0; i < VEC_DIMENSION; i++) {
tensor.tensor.push_back(0.5);
}
session.interface()->add_vector(id, GetGroupID(), tensor);
} catch (zilliz::VecException& ex) {
} catch (VecException& ex) {
CLIENT_LOG_ERROR << "request encounter exception: " << ex.what();
ASSERT_EQ(ex.code, zilliz::VecErrCode::ILLEGAL_ARGUMENT);
ASSERT_EQ(ex.code, VecErrCode::ILLEGAL_ARGUMENT);
}
try {
VecGroup temp_group;
session.interface()->get_group(temp_group, GetGroupID());
//ASSERT_TRUE(temp_group.id.empty());
} catch (zilliz::VecException& ex) {
} catch (VecException& ex) {
CLIENT_LOG_ERROR << "request encounter exception: " << ex.what();
ASSERT_EQ(ex.code, zilliz::VecErrCode::GROUP_NOT_EXISTS);
ASSERT_EQ(ex.code, VecErrCode::GROUP_NOT_EXISTS);
}
//add group