From 84110d2684f794a9ec76c2567f696c382b1723a3 Mon Sep 17 00:00:00 2001 From: Cai Yudong Date: Thu, 25 Feb 2021 17:35:36 +0800 Subject: [PATCH] Add tests/benchmark and tests/python_test using new python SDK Signed-off-by: Cai Yudong --- internal/datanode/data_sync_service.go | 12 +- internal/datanode/flow_graph_dd_node.go | 6 +- internal/datanode/flow_graph_dd_node_test.go | 2 +- .../datanode/flow_graph_filter_dm_node.go | 9 +- internal/datanode/flow_graph_gc_node.go | 7 +- .../datanode/flow_graph_insert_buffer_node.go | 6 +- .../flow_graph_insert_buffer_node_test.go | 2 +- internal/dataservice/server.go | 6 +- internal/distributed/indexservice/service.go | 30 +- internal/distributed/masterservice/server.go | 2 +- internal/indexnode/indexnode.go | 26 +- internal/msgstream/msgstream.go | 2 +- internal/msgstream/pulsarms/msg_test.go | 2 +- .../msgstream/pulsarms/pulsar_msgstream.go | 67 +- .../pulsarms/pulsar_msgstream_test.go | 6 +- internal/msgstream/rmqms/rmq_msgstream.go | 8 +- .../msgstream/rmqms/rmq_msgstream_test.go | 2 +- internal/proxyservice/timetick.go | 9 +- internal/querynode/data_sync_service.go | 14 +- internal/querynode/flow_graph_dd_node.go | 9 +- .../querynode/flow_graph_filter_dm_node.go | 9 +- internal/querynode/flow_graph_gc_node.go | 7 +- internal/querynode/flow_graph_insert_node.go | 6 +- .../querynode/flow_graph_service_time_node.go | 6 +- internal/querynode/query_node.go | 35 +- internal/querynode/search_service.go | 2 +- internal/timesync/timesync.go | 4 +- internal/util/flowgraph/flow_graph.go | 14 +- internal/util/flowgraph/flow_graph_test.go | 46 +- internal/util/flowgraph/input_node.go | 16 +- internal/util/flowgraph/node.go | 84 +- internal/util/trace/util.go | 35 +- tests/benchmark/README.md | 39 + tests/benchmark/__init__.py | 0 tests/benchmark/assets/Parameters.png | Bin 0 -> 50835 bytes .../gpu_search_performance_random50m-yaml.png | Bin 0 -> 66119 bytes ...milvus-nightly-performance-new-jenkins.png | Bin 0 -> 45034 bytes .../ci/function/file_transfer.groovy | 10 + tests/benchmark/ci/jenkinsfile/cleanup.groovy | 13 + .../ci/jenkinsfile/cleanupShards.groovy | 13 + .../ci/jenkinsfile/deploy_shards_test.groovy | 21 + .../ci/jenkinsfile/deploy_test.groovy | 19 + tests/benchmark/ci/jenkinsfile/notify.groovy | 15 + .../ci/jenkinsfile/publishDailyImages.groovy | 46 + tests/benchmark/ci/main_jenkinsfile | 148 ++ .../pod_containers/milvus-testframework.yaml | 13 + tests/benchmark/ci/publish_jenkinsfile | 104 + tests/benchmark/ci/scripts/yaml_processor.py | 536 +++++ tests/benchmark/client.py | 460 +++++ tests/benchmark/docker_runner.py | 366 ++++ tests/benchmark/docker_utils.py | 126 ++ tests/benchmark/executors/__init__.py | 3 + tests/benchmark/executors/shell.py | 4 + tests/benchmark/handlers/__init__.py | 0 tests/benchmark/helm_utils.py | 370 ++++ tests/benchmark/k8s_runner.py | 927 +++++++++ tests/benchmark/local_runner.py | 732 +++++++ tests/benchmark/locust_file.py | 30 + tests/benchmark/locust_flush_task.py | 33 + tests/benchmark/locust_get_entity_task.py | 36 + tests/benchmark/locust_insert_task.py | 33 + tests/benchmark/locust_search_task.py | 46 + tests/benchmark/locust_task.py | 37 + tests/benchmark/locust_tasks.py | 45 + tests/benchmark/locust_test.py | 18 + tests/benchmark/locust_user.py | 70 + tests/benchmark/main.py | 199 ++ tests/benchmark/mix_task.py | 42 + tests/benchmark/operation.py | 10 + tests/benchmark/parser.py | 85 + tests/benchmark/requirements.txt | 12 + tests/benchmark/results/__init__.py | 11 + tests/benchmark/results/reporter.py | 0 tests/benchmark/runner.py | 369 ++++ tests/benchmark/runners/__init__.py | 11 + tests/benchmark/runners/locust_runner.py | 75 + tests/benchmark/scheduler/010_data.json | 65 + tests/benchmark/scheduler/011_data.json | 62 + .../scheduler/011_data_acc_debug.json | 11 + .../scheduler/011_data_gpu_build.json | 11 + .../benchmark/scheduler/011_data_insert.json | 11 + .../scheduler/011_data_search_debug.json | 11 + tests/benchmark/scheduler/011_delete.json | 15 + tests/benchmark/scheduler/080_data.json | 65 + tests/benchmark/scheduler/acc.json | 15 + tests/benchmark/scheduler/build.json | 11 + tests/benchmark/scheduler/clean.json | 11 + tests/benchmark/scheduler/debug.json | 11 + tests/benchmark/scheduler/default_config.json | 53 + tests/benchmark/scheduler/file_size.json | 11 + tests/benchmark/scheduler/filter.json | 11 + tests/benchmark/scheduler/idc.json | 11 + tests/benchmark/scheduler/insert.json | 11 + tests/benchmark/scheduler/jaccard.json | 11 + tests/benchmark/scheduler/locust.json | 11 + .../benchmark/scheduler/locust_mix_debug.json | 10 + tests/benchmark/scheduler/loop.json | 10 + tests/benchmark/scheduler/search.json | 11 + tests/benchmark/scheduler/shards.json | 18 + tests/benchmark/scheduler/shards_ann.json | 10 + tests/benchmark/scheduler/shards_debug.json | 15 + .../benchmark/scheduler/shards_stability.json | 10 + tests/benchmark/scheduler/stability.json | 11 + tests/benchmark/search_task.py | 50 + .../suites/011_add_flush_performance.yaml | 20 + .../suites/011_cluster_cpu_accuracy_ann.yaml | 336 ++++ tests/benchmark/suites/011_cpu_accuracy.yaml | 55 + .../suites/011_cpu_accuracy_ann.yaml | 260 +++ tests/benchmark/suites/011_cpu_build.yaml | 40 + .../suites/011_cpu_build_binary.yaml | 11 + tests/benchmark/suites/011_cpu_search.yaml | 255 +++ .../suites/011_cpu_search_binary.yaml | 49 + .../suites/011_cpu_search_debug.yaml | 26 + .../suites/011_cpu_search_sift10m_filter.yaml | 97 + .../suites/011_cpu_search_sift50m.yaml | 98 + tests/benchmark/suites/011_gpu_accuracy.yaml | 61 + .../suites/011_gpu_accuracy_ann.yaml | 165 ++ tests/benchmark/suites/011_gpu_build.yaml | 21 + tests/benchmark/suites/011_gpu_search.yaml | 251 +++ .../suites/011_gpu_search_sift10m_filter.yaml | 122 ++ .../suites/011_gpu_search_sift50m.yaml | 121 ++ tests/benchmark/suites/011_gpu_stability.yaml | 39 + tests/benchmark/suites/011_insert_data.yaml | 57 + .../suites/011_insert_performance.yaml | 113 ++ tests/benchmark/suites/011_locust_insert.yaml | 33 + tests/benchmark/suites/011_locust_search.yaml | 43 + .../suites/011_search_stability.yaml | 20 + .../benchmark/suites/cluster_locust_mix.yaml | 47 + tests/benchmark/suites/cpu_accuracy.yaml | 61 + tests/benchmark/suites/cpu_accuracy_ann.yaml | 212 ++ .../suites/cpu_build_performance.yaml | 19 + tests/benchmark/suites/cpu_search_binary.yaml | 67 + .../cpu_search_performance_jaccard.yaml | 20 + .../cpu_search_performance_sift50m.yaml | 20 + tests/benchmark/suites/gpu_accuracy.yaml | 41 + tests/benchmark/suites/gpu_accuracy_ann.yaml | 172 ++ .../suites/gpu_accuracy_ann_debug.yaml | 24 + .../benchmark/suites/gpu_accuracy_sift1b.yaml | 59 + .../gpu_build_performance_jaccard50m.yaml | 20 + .../suites/gpu_search_performance.yaml | 247 +++ .../gpu_search_performance_jaccard50m.yaml | 22 + .../gpu_search_performance_sift50m.yaml | 146 ++ .../suites/gpu_search_stability.yaml | 23 + .../suites/gpu_stability_sift50m.yaml | 27 + tests/benchmark/suites/insert_binary.yaml | 39 + .../suites/insert_performance_deep1b.yaml | 87 + .../suites/locust_cluster_search.yaml | 45 + tests/benchmark/suites/locust_insert.yaml | 23 + tests/benchmark/suites/locust_mix.yaml | 47 + tests/benchmark/suites/locust_search.yaml | 49 + tests/benchmark/suites/loop_stability.yaml | 17 + tests/benchmark/suites/shards_ann_debug.yaml | 25 + .../suites/shards_insert_performance.yaml | 17 + .../shards_insert_performance_sift1m.yaml | 19 + .../suites/shards_loop_stability.yaml | 16 + .../shards_search_performance_sift1m.yaml | 12 + tests/benchmark/task/task.py | 0 tests/benchmark/test_loop.py | 52 + tests/benchmark/utils.py | 259 +++ tests/python_test/.dockerignore | 14 + tests/python_test/.gitignore | 13 + tests/python_test/Dockerfile | 15 + tests/python_test/README.md | 62 + .../collection/test_collection_count.py | 575 ++++++ .../collection/test_collection_logic.py | 138 ++ .../collection/test_collection_stats.py | 356 ++++ .../collection/test_create_collection.py | 290 +++ .../collection/test_describe_collection.py | 187 ++ .../collection/test_drop_collection.py | 103 + .../collection/test_has_collection.py | 101 + .../collection/test_list_collections.py | 94 + .../collection/test_load_collection.py | 573 ++++++ tests/python_test/conftest.py | 194 ++ tests/python_test/constants.py | 22 + tests/python_test/docker-entrypoint.sh | 9 + tests/python_test/entity/test_delete.py | 473 +++++ .../entity/test_get_entity_by_id.py | 666 ++++++ tests/python_test/entity/test_insert.py | 1090 ++++++++++ .../entity/test_list_id_in_segment.py | 318 +++ tests/python_test/entity/test_search.py | 1782 +++++++++++++++++ tests/python_test/pytest.ini | 14 + tests/python_test/requirements.txt | 12 + tests/python_test/requirements_cluster.txt | 25 + tests/python_test/run.sh | 4 + tests/python_test/stability/test_mysql.py | 43 + tests/python_test/stability/test_restart.py | 315 +++ tests/python_test/test_compact.py | 722 +++++++ tests/python_test/test_config.py | 1402 +++++++++++++ tests/python_test/test_connect.py | 232 +++ tests/python_test/test_flush.py | 353 ++++ tests/python_test/test_index.py | 832 ++++++++ tests/python_test/test_mix.py | 161 ++ tests/python_test/test_partition.py | 425 ++++ tests/python_test/test_ping.py | 129 ++ tests/python_test/utils.py | 1001 +++++++++ 195 files changed, 22966 insertions(+), 187 deletions(-) create mode 100644 tests/benchmark/README.md create mode 100644 tests/benchmark/__init__.py create mode 100644 tests/benchmark/assets/Parameters.png create mode 100644 tests/benchmark/assets/gpu_search_performance_random50m-yaml.png create mode 100644 tests/benchmark/assets/milvus-nightly-performance-new-jenkins.png create mode 100644 tests/benchmark/ci/function/file_transfer.groovy create mode 100644 tests/benchmark/ci/jenkinsfile/cleanup.groovy create mode 100644 tests/benchmark/ci/jenkinsfile/cleanupShards.groovy create mode 100644 tests/benchmark/ci/jenkinsfile/deploy_shards_test.groovy create mode 100644 tests/benchmark/ci/jenkinsfile/deploy_test.groovy create mode 100644 tests/benchmark/ci/jenkinsfile/notify.groovy create mode 100644 tests/benchmark/ci/jenkinsfile/publishDailyImages.groovy create mode 100644 tests/benchmark/ci/main_jenkinsfile create mode 100644 tests/benchmark/ci/pod_containers/milvus-testframework.yaml create mode 100644 tests/benchmark/ci/publish_jenkinsfile create mode 100755 tests/benchmark/ci/scripts/yaml_processor.py create mode 100644 tests/benchmark/client.py create mode 100644 tests/benchmark/docker_runner.py create mode 100644 tests/benchmark/docker_utils.py create mode 100644 tests/benchmark/executors/__init__.py create mode 100644 tests/benchmark/executors/shell.py create mode 100644 tests/benchmark/handlers/__init__.py create mode 100644 tests/benchmark/helm_utils.py create mode 100644 tests/benchmark/k8s_runner.py create mode 100644 tests/benchmark/local_runner.py create mode 100644 tests/benchmark/locust_file.py create mode 100644 tests/benchmark/locust_flush_task.py create mode 100644 tests/benchmark/locust_get_entity_task.py create mode 100644 tests/benchmark/locust_insert_task.py create mode 100644 tests/benchmark/locust_search_task.py create mode 100644 tests/benchmark/locust_task.py create mode 100644 tests/benchmark/locust_tasks.py create mode 100644 tests/benchmark/locust_test.py create mode 100644 tests/benchmark/locust_user.py create mode 100644 tests/benchmark/main.py create mode 100644 tests/benchmark/mix_task.py create mode 100644 tests/benchmark/operation.py create mode 100644 tests/benchmark/parser.py create mode 100644 tests/benchmark/requirements.txt create mode 100644 tests/benchmark/results/__init__.py create mode 100644 tests/benchmark/results/reporter.py create mode 100644 tests/benchmark/runner.py create mode 100644 tests/benchmark/runners/__init__.py create mode 100644 tests/benchmark/runners/locust_runner.py create mode 100644 tests/benchmark/scheduler/010_data.json create mode 100644 tests/benchmark/scheduler/011_data.json create mode 100644 tests/benchmark/scheduler/011_data_acc_debug.json create mode 100644 tests/benchmark/scheduler/011_data_gpu_build.json create mode 100644 tests/benchmark/scheduler/011_data_insert.json create mode 100644 tests/benchmark/scheduler/011_data_search_debug.json create mode 100644 tests/benchmark/scheduler/011_delete.json create mode 100644 tests/benchmark/scheduler/080_data.json create mode 100644 tests/benchmark/scheduler/acc.json create mode 100644 tests/benchmark/scheduler/build.json create mode 100644 tests/benchmark/scheduler/clean.json create mode 100644 tests/benchmark/scheduler/debug.json create mode 100644 tests/benchmark/scheduler/default_config.json create mode 100644 tests/benchmark/scheduler/file_size.json create mode 100644 tests/benchmark/scheduler/filter.json create mode 100644 tests/benchmark/scheduler/idc.json create mode 100644 tests/benchmark/scheduler/insert.json create mode 100644 tests/benchmark/scheduler/jaccard.json create mode 100644 tests/benchmark/scheduler/locust.json create mode 100644 tests/benchmark/scheduler/locust_mix_debug.json create mode 100644 tests/benchmark/scheduler/loop.json create mode 100644 tests/benchmark/scheduler/search.json create mode 100644 tests/benchmark/scheduler/shards.json create mode 100644 tests/benchmark/scheduler/shards_ann.json create mode 100644 tests/benchmark/scheduler/shards_debug.json create mode 100644 tests/benchmark/scheduler/shards_stability.json create mode 100644 tests/benchmark/scheduler/stability.json create mode 100644 tests/benchmark/search_task.py create mode 100644 tests/benchmark/suites/011_add_flush_performance.yaml create mode 100644 tests/benchmark/suites/011_cluster_cpu_accuracy_ann.yaml create mode 100644 tests/benchmark/suites/011_cpu_accuracy.yaml create mode 100644 tests/benchmark/suites/011_cpu_accuracy_ann.yaml create mode 100644 tests/benchmark/suites/011_cpu_build.yaml create mode 100644 tests/benchmark/suites/011_cpu_build_binary.yaml create mode 100644 tests/benchmark/suites/011_cpu_search.yaml create mode 100644 tests/benchmark/suites/011_cpu_search_binary.yaml create mode 100644 tests/benchmark/suites/011_cpu_search_debug.yaml create mode 100644 tests/benchmark/suites/011_cpu_search_sift10m_filter.yaml create mode 100644 tests/benchmark/suites/011_cpu_search_sift50m.yaml create mode 100644 tests/benchmark/suites/011_gpu_accuracy.yaml create mode 100644 tests/benchmark/suites/011_gpu_accuracy_ann.yaml create mode 100644 tests/benchmark/suites/011_gpu_build.yaml create mode 100644 tests/benchmark/suites/011_gpu_search.yaml create mode 100644 tests/benchmark/suites/011_gpu_search_sift10m_filter.yaml create mode 100644 tests/benchmark/suites/011_gpu_search_sift50m.yaml create mode 100644 tests/benchmark/suites/011_gpu_stability.yaml create mode 100644 tests/benchmark/suites/011_insert_data.yaml create mode 100644 tests/benchmark/suites/011_insert_performance.yaml create mode 100644 tests/benchmark/suites/011_locust_insert.yaml create mode 100644 tests/benchmark/suites/011_locust_search.yaml create mode 100644 tests/benchmark/suites/011_search_stability.yaml create mode 100644 tests/benchmark/suites/cluster_locust_mix.yaml create mode 100644 tests/benchmark/suites/cpu_accuracy.yaml create mode 100644 tests/benchmark/suites/cpu_accuracy_ann.yaml create mode 100644 tests/benchmark/suites/cpu_build_performance.yaml create mode 100644 tests/benchmark/suites/cpu_search_binary.yaml create mode 100644 tests/benchmark/suites/cpu_search_performance_jaccard.yaml create mode 100644 tests/benchmark/suites/cpu_search_performance_sift50m.yaml create mode 100644 tests/benchmark/suites/gpu_accuracy.yaml create mode 100644 tests/benchmark/suites/gpu_accuracy_ann.yaml create mode 100644 tests/benchmark/suites/gpu_accuracy_ann_debug.yaml create mode 100644 tests/benchmark/suites/gpu_accuracy_sift1b.yaml create mode 100644 tests/benchmark/suites/gpu_build_performance_jaccard50m.yaml create mode 100644 tests/benchmark/suites/gpu_search_performance.yaml create mode 100644 tests/benchmark/suites/gpu_search_performance_jaccard50m.yaml create mode 100644 tests/benchmark/suites/gpu_search_performance_sift50m.yaml create mode 100644 tests/benchmark/suites/gpu_search_stability.yaml create mode 100644 tests/benchmark/suites/gpu_stability_sift50m.yaml create mode 100644 tests/benchmark/suites/insert_binary.yaml create mode 100644 tests/benchmark/suites/insert_performance_deep1b.yaml create mode 100644 tests/benchmark/suites/locust_cluster_search.yaml create mode 100644 tests/benchmark/suites/locust_insert.yaml create mode 100644 tests/benchmark/suites/locust_mix.yaml create mode 100644 tests/benchmark/suites/locust_search.yaml create mode 100644 tests/benchmark/suites/loop_stability.yaml create mode 100644 tests/benchmark/suites/shards_ann_debug.yaml create mode 100644 tests/benchmark/suites/shards_insert_performance.yaml create mode 100644 tests/benchmark/suites/shards_insert_performance_sift1m.yaml create mode 100644 tests/benchmark/suites/shards_loop_stability.yaml create mode 100644 tests/benchmark/suites/shards_search_performance_sift1m.yaml create mode 100644 tests/benchmark/task/task.py create mode 100644 tests/benchmark/test_loop.py create mode 100644 tests/benchmark/utils.py create mode 100644 tests/python_test/.dockerignore create mode 100644 tests/python_test/.gitignore create mode 100644 tests/python_test/Dockerfile create mode 100644 tests/python_test/README.md create mode 100644 tests/python_test/collection/test_collection_count.py create mode 100644 tests/python_test/collection/test_collection_logic.py create mode 100644 tests/python_test/collection/test_collection_stats.py create mode 100644 tests/python_test/collection/test_create_collection.py create mode 100644 tests/python_test/collection/test_describe_collection.py create mode 100644 tests/python_test/collection/test_drop_collection.py create mode 100644 tests/python_test/collection/test_has_collection.py create mode 100644 tests/python_test/collection/test_list_collections.py create mode 100644 tests/python_test/collection/test_load_collection.py create mode 100644 tests/python_test/conftest.py create mode 100644 tests/python_test/constants.py create mode 100755 tests/python_test/docker-entrypoint.sh create mode 100644 tests/python_test/entity/test_delete.py create mode 100644 tests/python_test/entity/test_get_entity_by_id.py create mode 100644 tests/python_test/entity/test_insert.py create mode 100644 tests/python_test/entity/test_list_id_in_segment.py create mode 100644 tests/python_test/entity/test_search.py create mode 100644 tests/python_test/pytest.ini create mode 100644 tests/python_test/requirements.txt create mode 100644 tests/python_test/requirements_cluster.txt create mode 100644 tests/python_test/run.sh create mode 100644 tests/python_test/stability/test_mysql.py create mode 100644 tests/python_test/stability/test_restart.py create mode 100644 tests/python_test/test_compact.py create mode 100644 tests/python_test/test_config.py create mode 100644 tests/python_test/test_connect.py create mode 100644 tests/python_test/test_flush.py create mode 100644 tests/python_test/test_index.py create mode 100644 tests/python_test/test_mix.py create mode 100644 tests/python_test/test_partition.py create mode 100644 tests/python_test/test_ping.py create mode 100644 tests/python_test/utils.py diff --git a/internal/datanode/data_sync_service.go b/internal/datanode/data_sync_service.go index d883b3003e..3ca4820644 100644 --- a/internal/datanode/data_sync_service.go +++ b/internal/datanode/data_sync_service.go @@ -85,14 +85,14 @@ func (dsService *dataSyncService) initNodes() { var insertBufferNode Node = newInsertBufferNode(dsService.ctx, mt, dsService.replica, dsService.idAllocator, dsService.msFactory) var gcNode Node = newGCNode(dsService.replica) - dsService.fg.AddNode(&dmStreamNode) - dsService.fg.AddNode(&ddStreamNode) + dsService.fg.AddNode(dmStreamNode) + dsService.fg.AddNode(ddStreamNode) - dsService.fg.AddNode(&filterDmNode) - dsService.fg.AddNode(&ddNode) + dsService.fg.AddNode(filterDmNode) + dsService.fg.AddNode(ddNode) - dsService.fg.AddNode(&insertBufferNode) - dsService.fg.AddNode(&gcNode) + dsService.fg.AddNode(insertBufferNode) + dsService.fg.AddNode(gcNode) // dmStreamNode err = dsService.fg.SetEdges(dmStreamNode.Name(), diff --git a/internal/datanode/flow_graph_dd_node.go b/internal/datanode/flow_graph_dd_node.go index 00afd3df98..0135ee4ba0 100644 --- a/internal/datanode/flow_graph_dd_node.go +++ b/internal/datanode/flow_graph_dd_node.go @@ -66,7 +66,7 @@ func (ddNode *ddNode) Name() string { return "ddNode" } -func (ddNode *ddNode) Operate(in []*Msg) []*Msg { +func (ddNode *ddNode) Operate(ctx context.Context, in []Msg) ([]Msg, context.Context) { //fmt.Println("Do filterDdNode operation") if len(in) != 1 { @@ -74,7 +74,7 @@ func (ddNode *ddNode) Operate(in []*Msg) []*Msg { // TODO: add error handling } - msMsg, ok := (*in[0]).(*MsgStreamMsg) + msMsg, ok := in[0].(*MsgStreamMsg) if !ok { log.Println("type assertion failed for MsgStreamMsg") // TODO: add error handling @@ -141,7 +141,7 @@ func (ddNode *ddNode) Operate(in []*Msg) []*Msg { } var res Msg = ddNode.ddMsg - return []*Msg{&res} + return []Msg{res}, ctx } func (ddNode *ddNode) flush() { diff --git a/internal/datanode/flow_graph_dd_node_test.go b/internal/datanode/flow_graph_dd_node_test.go index 46b2468b14..8fdac3ad62 100644 --- a/internal/datanode/flow_graph_dd_node_test.go +++ b/internal/datanode/flow_graph_dd_node_test.go @@ -154,5 +154,5 @@ func TestFlowGraphDDNode_Operate(t *testing.T) { tsMessages = append(tsMessages, msgstream.TsMsg(&dropPartitionMsg)) msgStream := flowgraph.GenerateMsgStreamMsg(tsMessages, Timestamp(0), Timestamp(3), make([]*internalpb2.MsgPosition, 0)) var inMsg Msg = msgStream - ddNode.Operate([]*Msg{&inMsg}) + ddNode.Operate(ctx, []Msg{inMsg}) } diff --git a/internal/datanode/flow_graph_filter_dm_node.go b/internal/datanode/flow_graph_filter_dm_node.go index c5171a6be3..752c916725 100644 --- a/internal/datanode/flow_graph_filter_dm_node.go +++ b/internal/datanode/flow_graph_filter_dm_node.go @@ -1,6 +1,7 @@ package datanode import ( + "context" "log" "math" @@ -18,7 +19,7 @@ func (fdmNode *filterDmNode) Name() string { return "fdmNode" } -func (fdmNode *filterDmNode) Operate(in []*Msg) []*Msg { +func (fdmNode *filterDmNode) Operate(ctx context.Context, in []Msg) ([]Msg, context.Context) { //fmt.Println("Do filterDmNode operation") if len(in) != 2 { @@ -26,13 +27,13 @@ func (fdmNode *filterDmNode) Operate(in []*Msg) []*Msg { // TODO: add error handling } - msgStreamMsg, ok := (*in[0]).(*MsgStreamMsg) + msgStreamMsg, ok := in[0].(*MsgStreamMsg) if !ok { log.Println("type assertion failed for MsgStreamMsg") // TODO: add error handling } - ddMsg, ok := (*in[1]).(*ddMsg) + ddMsg, ok := in[1].(*ddMsg) if !ok { log.Println("type assertion failed for ddMsg") // TODO: add error handling @@ -69,7 +70,7 @@ func (fdmNode *filterDmNode) Operate(in []*Msg) []*Msg { iMsg.startPositions = append(iMsg.startPositions, msgStreamMsg.StartPositions()...) iMsg.gcRecord = ddMsg.gcRecord var res Msg = &iMsg - return []*Msg{&res} + return []Msg{res}, ctx } func (fdmNode *filterDmNode) filterInvalidInsertMessage(msg *msgstream.InsertMsg) *msgstream.InsertMsg { diff --git a/internal/datanode/flow_graph_gc_node.go b/internal/datanode/flow_graph_gc_node.go index b5b3277876..1732dbb6eb 100644 --- a/internal/datanode/flow_graph_gc_node.go +++ b/internal/datanode/flow_graph_gc_node.go @@ -1,6 +1,7 @@ package datanode import ( + "context" "log" ) @@ -13,7 +14,7 @@ func (gcNode *gcNode) Name() string { return "gcNode" } -func (gcNode *gcNode) Operate(in []*Msg) []*Msg { +func (gcNode *gcNode) Operate(ctx context.Context, in []Msg) ([]Msg, context.Context) { //fmt.Println("Do gcNode operation") if len(in) != 1 { @@ -21,7 +22,7 @@ func (gcNode *gcNode) Operate(in []*Msg) []*Msg { // TODO: add error handling } - gcMsg, ok := (*in[0]).(*gcMsg) + gcMsg, ok := in[0].(*gcMsg) if !ok { log.Println("type assertion failed for gcMsg") // TODO: add error handling @@ -35,7 +36,7 @@ func (gcNode *gcNode) Operate(in []*Msg) []*Msg { } } - return nil + return nil, ctx } func newGCNode(replica Replica) *gcNode { diff --git a/internal/datanode/flow_graph_insert_buffer_node.go b/internal/datanode/flow_graph_insert_buffer_node.go index 354eb45803..c526946a61 100644 --- a/internal/datanode/flow_graph_insert_buffer_node.go +++ b/internal/datanode/flow_graph_insert_buffer_node.go @@ -85,7 +85,7 @@ func (ibNode *insertBufferNode) Name() string { return "ibNode" } -func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg { +func (ibNode *insertBufferNode) Operate(ctx context.Context, in []Msg) ([]Msg, context.Context) { // log.Println("=========== insert buffer Node Operating") if len(in) != 1 { @@ -93,7 +93,7 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg { // TODO: add error handling } - iMsg, ok := (*in[0]).(*insertMsg) + iMsg, ok := in[0].(*insertMsg) if !ok { log.Println("Error: type assertion failed for insertMsg") // TODO: add error handling @@ -472,7 +472,7 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg { timeRange: iMsg.timeRange, } - return []*Msg{&res} + return []Msg{res}, ctx } func (ibNode *insertBufferNode) flushSegment(segID UniqueID, partitionID UniqueID, collID UniqueID) error { diff --git a/internal/datanode/flow_graph_insert_buffer_node_test.go b/internal/datanode/flow_graph_insert_buffer_node_test.go index 527f7d577b..450e961c24 100644 --- a/internal/datanode/flow_graph_insert_buffer_node_test.go +++ b/internal/datanode/flow_graph_insert_buffer_node_test.go @@ -53,7 +53,7 @@ func TestFlowGraphInsertBufferNode_Operate(t *testing.T) { iBNode := newInsertBufferNode(ctx, newMetaTable(), replica, idFactory, msFactory) inMsg := genInsertMsg() var iMsg flowgraph.Msg = &inMsg - iBNode.Operate([]*flowgraph.Msg{&iMsg}) + iBNode.Operate(ctx, []flowgraph.Msg{iMsg}) } func genInsertMsg() insertMsg { diff --git a/internal/dataservice/server.go b/internal/dataservice/server.go index 6c700e0985..06375a2791 100644 --- a/internal/dataservice/server.go +++ b/internal/dataservice/server.go @@ -329,7 +329,7 @@ func (s *Server) startStatsChannel(ctx context.Context) { return default: } - msgPack := statsStream.Consume() + msgPack, _ := statsStream.Consume() for _, msg := range msgPack.Msgs { statistics, ok := msg.(*msgstream.SegmentStatisticsMsg) if !ok { @@ -358,7 +358,7 @@ func (s *Server) startSegmentFlushChannel(ctx context.Context) { return default: } - msgPack := flushStream.Consume() + msgPack, _ := flushStream.Consume() for _, msg := range msgPack.Msgs { if msg.Type() != commonpb.MsgType_kSegmentFlushDone { continue @@ -393,7 +393,7 @@ func (s *Server) startDDChannel(ctx context.Context) { return default: } - msgPack := ddStream.Consume() + msgPack, _ := ddStream.Consume() for _, msg := range msgPack.Msgs { if err := s.ddHandler.HandleDDMsg(msg); err != nil { log.Error("handle dd msg error", zap.Error(err)) diff --git a/internal/distributed/indexservice/service.go b/internal/distributed/indexservice/service.go index a95c0f2a8f..b5ea9b0998 100644 --- a/internal/distributed/indexservice/service.go +++ b/internal/distributed/indexservice/service.go @@ -2,18 +2,21 @@ package grpcindexservice import ( "context" + "fmt" + "io" "log" "net" "strconv" "sync" - "github.com/zilliztech/milvus-distributed/internal/proto/internalpb2" - "github.com/zilliztech/milvus-distributed/internal/proto/milvuspb" - "github.com/zilliztech/milvus-distributed/internal/util/funcutil" - + "github.com/opentracing/opentracing-go" + "github.com/uber/jaeger-client-go/config" "github.com/zilliztech/milvus-distributed/internal/indexservice" "github.com/zilliztech/milvus-distributed/internal/proto/commonpb" "github.com/zilliztech/milvus-distributed/internal/proto/indexpb" + "github.com/zilliztech/milvus-distributed/internal/proto/internalpb2" + "github.com/zilliztech/milvus-distributed/internal/proto/milvuspb" + "github.com/zilliztech/milvus-distributed/internal/util/funcutil" "github.com/zilliztech/milvus-distributed/internal/util/typeutil" "google.golang.org/grpc" ) @@ -30,6 +33,8 @@ type Server struct { loopCtx context.Context loopCancel func() loopWg sync.WaitGroup + + closer io.Closer } func (s *Server) Run() error { @@ -71,6 +76,9 @@ func (s *Server) start() error { } func (s *Server) Stop() error { + if err := s.closer.Close(); err != nil { + return err + } if s.impl != nil { s.impl.Stop() } @@ -191,5 +199,19 @@ func NewServer(ctx context.Context) (*Server, error) { grpcErrChan: make(chan error), } + cfg := &config.Configuration{ + ServiceName: "index_service", + Sampler: &config.SamplerConfig{ + Type: "const", + Param: 1, + }, + } + tracer, closer, err := cfg.NewTracer() + if err != nil { + panic(fmt.Sprintf("ERROR: cannot init Jaeger: %v\n", err)) + } + opentracing.SetGlobalTracer(tracer) + s.closer = closer + return s, nil } diff --git a/internal/distributed/masterservice/server.go b/internal/distributed/masterservice/server.go index da86d6506a..fc1e3f3874 100644 --- a/internal/distributed/masterservice/server.go +++ b/internal/distributed/masterservice/server.go @@ -68,7 +68,7 @@ func NewServer(ctx context.Context, factory msgstream.Factory) (*Server, error) //TODO cfg := &config.Configuration{ - ServiceName: "proxy_service", + ServiceName: "master_service", Sampler: &config.SamplerConfig{ Type: "const", Param: 1, diff --git a/internal/indexnode/indexnode.go b/internal/indexnode/indexnode.go index 994c449638..5aa01ff089 100644 --- a/internal/indexnode/indexnode.go +++ b/internal/indexnode/indexnode.go @@ -2,11 +2,15 @@ package indexnode import ( "context" + "fmt" + "io" "log" "time" "github.com/zilliztech/milvus-distributed/internal/proto/milvuspb" + "github.com/opentracing/opentracing-go" + "github.com/uber/jaeger-client-go/config" "github.com/zilliztech/milvus-distributed/internal/errors" "github.com/zilliztech/milvus-distributed/internal/kv" miniokv "github.com/zilliztech/milvus-distributed/internal/kv/minio" @@ -40,6 +44,8 @@ type NodeImpl struct { // Add callback functions at different stages startCallbacks []func() closeCallbacks []func() + + closer io.Closer } func NewNodeImpl(ctx context.Context) (*NodeImpl, error) { @@ -57,9 +63,7 @@ func NewNodeImpl(ctx context.Context) (*NodeImpl, error) { } func (i *NodeImpl) Init() error { - log.Println("AAAAAAAAAAAAAAAAA", i.serviceClient) err := funcutil.WaitForComponentHealthy(i.serviceClient, "IndexService", 10, time.Second) - log.Println("BBBBBBBBB", i.serviceClient) if err != nil { return err @@ -87,6 +91,21 @@ func (i *NodeImpl) Init() error { return err } + // TODO + cfg := &config.Configuration{ + ServiceName: fmt.Sprintf("index_node_%d", Params.NodeID), + Sampler: &config.SamplerConfig{ + Type: "const", + Param: 1, + }, + } + tracer, closer, err := cfg.NewTracer() + if err != nil { + panic(fmt.Sprintf("ERROR: cannot init Jaeger: %v\n", err)) + } + opentracing.SetGlobalTracer(tracer) + i.closer = closer + connectMinIOFn := func() error { option := &miniokv.Option{ Address: Params.MinIOAddress, @@ -126,6 +145,9 @@ func (i *NodeImpl) Start() error { // Close closes the server. func (i *NodeImpl) Stop() error { + if err := i.closer.Close(); err != nil { + return err + } i.loopCancel() if i.sched != nil { i.sched.Close() diff --git a/internal/msgstream/msgstream.go b/internal/msgstream/msgstream.go index 9826c58183..8a92d989d4 100644 --- a/internal/msgstream/msgstream.go +++ b/internal/msgstream/msgstream.go @@ -32,7 +32,7 @@ type MsgStream interface { Produce(context.Context, *MsgPack) error Broadcast(context.Context, *MsgPack) error - Consume() *MsgPack + Consume() (*MsgPack, context.Context) Seek(offset *MsgPosition) error } diff --git a/internal/msgstream/pulsarms/msg_test.go b/internal/msgstream/pulsarms/msg_test.go index 5d986fa138..48496c99c4 100644 --- a/internal/msgstream/pulsarms/msg_test.go +++ b/internal/msgstream/pulsarms/msg_test.go @@ -160,7 +160,7 @@ func TestStream_task_Insert(t *testing.T) { } receiveCount := 0 for { - result := outputStream.Consume() + result, _ := outputStream.Consume() if len(result.Msgs) > 0 { msgs := result.Msgs for _, v := range msgs { diff --git a/internal/msgstream/pulsarms/pulsar_msgstream.go b/internal/msgstream/pulsarms/pulsar_msgstream.go index c93cbf479a..157f0a53de 100644 --- a/internal/msgstream/pulsarms/pulsar_msgstream.go +++ b/internal/msgstream/pulsarms/pulsar_msgstream.go @@ -5,14 +5,12 @@ import ( "path/filepath" "reflect" "strconv" - "strings" "sync" "time" "github.com/apache/pulsar-client-go/pulsar" "github.com/golang/protobuf/proto" - "go.uber.org/zap" - + "github.com/opentracing/opentracing-go" "github.com/zilliztech/milvus-distributed/internal/errors" "github.com/zilliztech/milvus-distributed/internal/log" "github.com/zilliztech/milvus-distributed/internal/msgstream" @@ -21,6 +19,7 @@ import ( "github.com/zilliztech/milvus-distributed/internal/proto/internalpb2" "github.com/zilliztech/milvus-distributed/internal/util/trace" "github.com/zilliztech/milvus-distributed/internal/util/typeutil" + "go.uber.org/zap" ) type TsMsg = msgstream.TsMsg @@ -52,6 +51,8 @@ type PulsarMsgStream struct { pulsarBufSize int64 consumerLock *sync.Mutex consumerReflects []reflect.SelectCase + + scMap *sync.Map } func newPulsarMsgStream(ctx context.Context, @@ -92,6 +93,7 @@ func newPulsarMsgStream(ctx context.Context, consumerReflects: consumerReflects, consumerLock: &sync.Mutex{}, wait: &sync.WaitGroup{}, + scMap: &sync.Map{}, } return stream, nil @@ -182,29 +184,6 @@ func (ms *PulsarMsgStream) Close() { } } -type propertiesReaderWriter struct { - ppMap map[string]string -} - -func (ppRW *propertiesReaderWriter) Set(key, val string) { - // The GRPC HPACK implementation rejects any uppercase keys here. - // - // As such, since the HTTP_HEADERS format is case-insensitive anyway, we - // blindly lowercase the key (which is guaranteed to work in the - // Inject/Extract sense per the OpenTracing spec). - key = strings.ToLower(key) - ppRW.ppMap[key] = val -} - -func (ppRW *propertiesReaderWriter) ForeachKey(handler func(key, val string) error) error { - for k, val := range ppRW.ppMap { - if err := handler(k, val); err != nil { - return err - } - } - return nil -} - func (ms *PulsarMsgStream) Produce(ctx context.Context, msgPack *MsgPack) error { tsMsgs := msgPack.Msgs if len(tsMsgs) <= 0 { @@ -316,18 +295,31 @@ func (ms *PulsarMsgStream) Broadcast(ctx context.Context, msgPack *MsgPack) erro return nil } -func (ms *PulsarMsgStream) Consume() *MsgPack { +func (ms *PulsarMsgStream) Consume() (*MsgPack, context.Context) { for { select { case cm, ok := <-ms.receiveBuf: if !ok { log.Debug("buf chan closed") - return nil + return nil, nil } - return cm + var ctx context.Context + var opts []opentracing.StartSpanOption + for _, msg := range cm.Msgs { + sc, loaded := ms.scMap.LoadAndDelete(msg.ID()) + if loaded { + opts = append(opts, opentracing.ChildOf(sc.(opentracing.SpanContext))) + } + } + if len(opts) != 0 { + ctx = context.Background() + } + sp, ctx := trace.StartSpanFromContext(ctx, opts...) + sp.Finish() + return cm, ctx case <-ms.ctx.Done(): log.Debug("context closed") - return nil + return nil, nil } } } @@ -360,8 +352,15 @@ func (ms *PulsarMsgStream) receiveMsg(consumer Consumer) { MsgID: typeutil.PulsarMsgIDToString(pulsarMsg.ID()), }) + sp, ok := trace.ExtractFromPulsarMsgProperties(tsMsg, pulsarMsg.Properties()) + if ok { + ms.scMap.Store(tsMsg.ID(), sp.Context()) + } + msgPack := MsgPack{Msgs: []TsMsg{tsMsg}} ms.receiveBuf <- &msgPack + + sp.Finish() } } } @@ -687,12 +686,18 @@ func (ms *PulsarTtMsgStream) findTimeTick(consumer Consumer, log.Error("Failed to unmarshal tsMsg", zap.Error(err)) continue } + // set pulsar info to tsMsg tsMsg.SetPosition(&msgstream.MsgPosition{ ChannelName: filepath.Base(pulsarMsg.Topic()), MsgID: typeutil.PulsarMsgIDToString(pulsarMsg.ID()), }) + sp, ok := trace.ExtractFromPulsarMsgProperties(tsMsg, pulsarMsg.Properties()) + if ok { + ms.scMap.Store(tsMsg.ID(), sp.Context()) + } + ms.unsolvedMutex.Lock() ms.unsolvedBuf[consumer] = append(ms.unsolvedBuf[consumer], tsMsg) ms.unsolvedMutex.Unlock() @@ -701,8 +706,10 @@ func (ms *PulsarTtMsgStream) findTimeTick(consumer Consumer, findMapMutex.Lock() eofMsgMap[consumer] = tsMsg.(*TimeTickMsg).Base.Timestamp findMapMutex.Unlock() + sp.Finish() return } + sp.Finish() } } } diff --git a/internal/msgstream/pulsarms/pulsar_msgstream_test.go b/internal/msgstream/pulsarms/pulsar_msgstream_test.go index d9aa513be2..ca9480061d 100644 --- a/internal/msgstream/pulsarms/pulsar_msgstream_test.go +++ b/internal/msgstream/pulsarms/pulsar_msgstream_test.go @@ -223,7 +223,7 @@ func initPulsarTtStream(pulsarAddress string, func receiveMsg(outputStream msgstream.MsgStream, msgCount int) { receiveCount := 0 for { - result := outputStream.Consume() + result, _ := outputStream.Consume() if len(result.Msgs) > 0 { msgs := result.Msgs for _, v := range msgs { @@ -607,13 +607,13 @@ func TestStream_PulsarTtMsgStream_Seek(t *testing.T) { assert.Nil(t, err) outputStream.Consume() - receivedMsg := outputStream.Consume() + receivedMsg, _ := outputStream.Consume() for _, position := range receivedMsg.StartPositions { outputStream.Seek(position) } err = inputStream.Broadcast(ctx, &msgPack5) assert.Nil(t, err) - seekMsg := outputStream.Consume() + seekMsg, _ := outputStream.Consume() for _, msg := range seekMsg.Msgs { assert.Equal(t, msg.BeginTs(), uint64(14)) } diff --git a/internal/msgstream/rmqms/rmq_msgstream.go b/internal/msgstream/rmqms/rmq_msgstream.go index 8561035a64..3c463f588c 100644 --- a/internal/msgstream/rmqms/rmq_msgstream.go +++ b/internal/msgstream/rmqms/rmq_msgstream.go @@ -219,18 +219,18 @@ func (ms *RmqMsgStream) Broadcast(ctx context.Context, msgPack *MsgPack) error { return nil } -func (ms *RmqMsgStream) Consume() *msgstream.MsgPack { +func (ms *RmqMsgStream) Consume() (*msgstream.MsgPack, context.Context) { for { select { case cm, ok := <-ms.receiveBuf: if !ok { log.Println("buf chan closed") - return nil + return nil, nil } - return cm + return cm, nil case <-ms.ctx.Done(): log.Printf("context closed") - return nil + return nil, nil } } } diff --git a/internal/msgstream/rmqms/rmq_msgstream_test.go b/internal/msgstream/rmqms/rmq_msgstream_test.go index 3e19f3d7b3..a35b1e4788 100644 --- a/internal/msgstream/rmqms/rmq_msgstream_test.go +++ b/internal/msgstream/rmqms/rmq_msgstream_test.go @@ -239,7 +239,7 @@ func initRmqTtStream(producerChannels []string, func receiveMsg(outputStream msgstream.MsgStream, msgCount int) { receiveCount := 0 for { - result := outputStream.Consume() + result, _ := outputStream.Consume() if len(result.Msgs) > 0 { msgs := result.Msgs for _, v := range msgs { diff --git a/internal/proxyservice/timetick.go b/internal/proxyservice/timetick.go index da22a7181f..92cf0c46fc 100644 --- a/internal/proxyservice/timetick.go +++ b/internal/proxyservice/timetick.go @@ -5,11 +5,9 @@ import ( "log" "sync" - "github.com/zilliztech/milvus-distributed/internal/proto/commonpb" - - "github.com/zilliztech/milvus-distributed/internal/proto/internalpb2" - "github.com/zilliztech/milvus-distributed/internal/msgstream" + "github.com/zilliztech/milvus-distributed/internal/proto/commonpb" + "github.com/zilliztech/milvus-distributed/internal/proto/internalpb2" ) type ( @@ -58,6 +56,9 @@ func (tt *TimeTickImpl) Start() error { }, } msgPack.Msgs = append(msgPack.Msgs, timeTickMsg) + for _, msg := range msgPack.Msgs { + log.Println("msg type xxxxxxxxxxxxxxxxxxxxxxxx", msg.Type()) + } for _, channel := range tt.channels { err = channel.Broadcast(tt.ctx, &msgPack) if err != nil { diff --git a/internal/querynode/data_sync_service.go b/internal/querynode/data_sync_service.go index e0ebdc83f6..6ec7a7745e 100644 --- a/internal/querynode/data_sync_service.go +++ b/internal/querynode/data_sync_service.go @@ -56,15 +56,15 @@ func (dsService *dataSyncService) initNodes() { var serviceTimeNode node = newServiceTimeNode(dsService.ctx, dsService.replica, dsService.msFactory) var gcNode node = newGCNode(dsService.replica) - dsService.fg.AddNode(&dmStreamNode) - dsService.fg.AddNode(&ddStreamNode) + dsService.fg.AddNode(dmStreamNode) + dsService.fg.AddNode(ddStreamNode) - dsService.fg.AddNode(&filterDmNode) - dsService.fg.AddNode(&ddNode) + dsService.fg.AddNode(filterDmNode) + dsService.fg.AddNode(ddNode) - dsService.fg.AddNode(&insertNode) - dsService.fg.AddNode(&serviceTimeNode) - dsService.fg.AddNode(&gcNode) + dsService.fg.AddNode(insertNode) + dsService.fg.AddNode(serviceTimeNode) + dsService.fg.AddNode(gcNode) // dmStreamNode var err = dsService.fg.SetEdges(dmStreamNode.Name(), diff --git a/internal/querynode/flow_graph_dd_node.go b/internal/querynode/flow_graph_dd_node.go index 99cd70cf60..69a3eeefed 100644 --- a/internal/querynode/flow_graph_dd_node.go +++ b/internal/querynode/flow_graph_dd_node.go @@ -1,6 +1,7 @@ package querynode import ( + "context" "log" "github.com/golang/protobuf/proto" @@ -19,15 +20,15 @@ func (ddNode *ddNode) Name() string { return "ddNode" } -func (ddNode *ddNode) Operate(in []*Msg) []*Msg { - //fmt.Println("Do ddNode operation") +func (ddNode *ddNode) Operate(ctx context.Context, in []Msg) ([]Msg, context.Context) { + //fmt.Println("Do filterDmNode operation") if len(in) != 1 { log.Println("Invalid operate message input in ddNode, input length = ", len(in)) // TODO: add error handling } - msMsg, ok := (*in[0]).(*MsgStreamMsg) + msMsg, ok := in[0].(*MsgStreamMsg) if !ok { log.Println("type assertion failed for MsgStreamMsg") // TODO: add error handling @@ -72,7 +73,7 @@ func (ddNode *ddNode) Operate(in []*Msg) []*Msg { //} var res Msg = ddNode.ddMsg - return []*Msg{&res} + return []Msg{res}, ctx } func (ddNode *ddNode) createCollection(msg *msgstream.CreateCollectionMsg) { diff --git a/internal/querynode/flow_graph_filter_dm_node.go b/internal/querynode/flow_graph_filter_dm_node.go index f50a4dab31..7e75a561dc 100644 --- a/internal/querynode/flow_graph_filter_dm_node.go +++ b/internal/querynode/flow_graph_filter_dm_node.go @@ -1,6 +1,7 @@ package querynode import ( + "context" "log" "math" @@ -18,7 +19,7 @@ func (fdmNode *filterDmNode) Name() string { return "fdmNode" } -func (fdmNode *filterDmNode) Operate(in []*Msg) []*Msg { +func (fdmNode *filterDmNode) Operate(ctx context.Context, in []Msg) ([]Msg, context.Context) { //fmt.Println("Do filterDmNode operation") if len(in) != 2 { @@ -26,13 +27,13 @@ func (fdmNode *filterDmNode) Operate(in []*Msg) []*Msg { // TODO: add error handling } - msgStreamMsg, ok := (*in[0]).(*MsgStreamMsg) + msgStreamMsg, ok := in[0].(*MsgStreamMsg) if !ok { log.Println("type assertion failed for MsgStreamMsg") // TODO: add error handling } - ddMsg, ok := (*in[1]).(*ddMsg) + ddMsg, ok := in[1].(*ddMsg) if !ok { log.Println("type assertion failed for ddMsg") // TODO: add error handling @@ -63,7 +64,7 @@ func (fdmNode *filterDmNode) Operate(in []*Msg) []*Msg { iMsg.gcRecord = ddMsg.gcRecord var res Msg = &iMsg - return []*Msg{&res} + return []Msg{res}, ctx } func (fdmNode *filterDmNode) filterInvalidInsertMessage(msg *msgstream.InsertMsg) *msgstream.InsertMsg { diff --git a/internal/querynode/flow_graph_gc_node.go b/internal/querynode/flow_graph_gc_node.go index 2be079e447..63264b5ca4 100644 --- a/internal/querynode/flow_graph_gc_node.go +++ b/internal/querynode/flow_graph_gc_node.go @@ -1,6 +1,7 @@ package querynode import ( + "context" "log" ) @@ -13,7 +14,7 @@ func (gcNode *gcNode) Name() string { return "gcNode" } -func (gcNode *gcNode) Operate(in []*Msg) []*Msg { +func (gcNode *gcNode) Operate(ctx context.Context, in []Msg) ([]Msg, context.Context) { //fmt.Println("Do gcNode operation") if len(in) != 1 { @@ -21,7 +22,7 @@ func (gcNode *gcNode) Operate(in []*Msg) []*Msg { // TODO: add error handling } - _, ok := (*in[0]).(*gcMsg) + _, ok := in[0].(*gcMsg) if !ok { log.Println("type assertion failed for gcMsg") // TODO: add error handling @@ -47,7 +48,7 @@ func (gcNode *gcNode) Operate(in []*Msg) []*Msg { // } //} - return nil + return nil, ctx } func newGCNode(replica collectionReplica) *gcNode { diff --git a/internal/querynode/flow_graph_insert_node.go b/internal/querynode/flow_graph_insert_node.go index a47facef0d..23629399df 100644 --- a/internal/querynode/flow_graph_insert_node.go +++ b/internal/querynode/flow_graph_insert_node.go @@ -26,7 +26,7 @@ func (iNode *insertNode) Name() string { return "iNode" } -func (iNode *insertNode) Operate(in []*Msg) []*Msg { +func (iNode *insertNode) Operate(ctx context.Context, in []Msg) ([]Msg, context.Context) { // fmt.Println("Do insertNode operation") if len(in) != 1 { @@ -34,7 +34,7 @@ func (iNode *insertNode) Operate(in []*Msg) []*Msg { // TODO: add error handling } - iMsg, ok := (*in[0]).(*insertMsg) + iMsg, ok := in[0].(*insertMsg) if !ok { log.Println("type assertion failed for insertMsg") // TODO: add error handling @@ -90,7 +90,7 @@ func (iNode *insertNode) Operate(in []*Msg) []*Msg { gcRecord: iMsg.gcRecord, timeRange: iMsg.timeRange, } - return []*Msg{&res} + return []Msg{res}, ctx } func (iNode *insertNode) insert(insertData *InsertData, segmentID int64, wg *sync.WaitGroup) { diff --git a/internal/querynode/flow_graph_service_time_node.go b/internal/querynode/flow_graph_service_time_node.go index a164eb2bac..7a2005028e 100644 --- a/internal/querynode/flow_graph_service_time_node.go +++ b/internal/querynode/flow_graph_service_time_node.go @@ -19,7 +19,7 @@ func (stNode *serviceTimeNode) Name() string { return "stNode" } -func (stNode *serviceTimeNode) Operate(in []*Msg) []*Msg { +func (stNode *serviceTimeNode) Operate(ctx context.Context, in []Msg) ([]Msg, context.Context) { //fmt.Println("Do serviceTimeNode operation") if len(in) != 1 { @@ -27,7 +27,7 @@ func (stNode *serviceTimeNode) Operate(in []*Msg) []*Msg { // TODO: add error handling } - serviceTimeMsg, ok := (*in[0]).(*serviceTimeMsg) + serviceTimeMsg, ok := in[0].(*serviceTimeMsg) if !ok { log.Println("type assertion failed for serviceTimeMsg") // TODO: add error handling @@ -45,7 +45,7 @@ func (stNode *serviceTimeNode) Operate(in []*Msg) []*Msg { gcRecord: serviceTimeMsg.gcRecord, timeRange: serviceTimeMsg.timeRange, } - return []*Msg{&res} + return []Msg{res}, ctx } func (stNode *serviceTimeNode) sendTimeTick(ts Timestamp) error { diff --git a/internal/querynode/query_node.go b/internal/querynode/query_node.go index da546bf5ea..3cf30a5674 100644 --- a/internal/querynode/query_node.go +++ b/internal/querynode/query_node.go @@ -15,12 +15,12 @@ import "C" import ( "context" "fmt" + "github.com/opentracing/opentracing-go" + "github.com/uber/jaeger-client-go/config" "io" "log" "sync/atomic" - "github.com/opentracing/opentracing-go" - "github.com/uber/jaeger-client-go/config" "github.com/zilliztech/milvus-distributed/internal/errors" "github.com/zilliztech/milvus-distributed/internal/msgstream" "github.com/zilliztech/milvus-distributed/internal/msgstream/pulsarms" @@ -91,20 +91,6 @@ func NewQueryNode(ctx context.Context, queryNodeID UniqueID, factory msgstream.F msFactory: factory, } - cfg := &config.Configuration{ - ServiceName: fmt.Sprintf("query_node_%d", node.QueryNodeID), - Sampler: &config.SamplerConfig{ - Type: "const", - Param: 1, - }, - } - tracer, closer, err := cfg.NewTracer() - if err != nil { - panic(fmt.Sprintf("ERROR: cannot init Jaeger: %v\n", err)) - } - opentracing.SetGlobalTracer(tracer) - node.closer = closer - node.replica = newCollectionReplicaImpl() node.UpdateStateCode(internalpb2.StateCode_ABNORMAL) return node @@ -167,6 +153,20 @@ func (node *QueryNode) Init() error { fmt.Println("QueryNodeID is", Params.QueryNodeID) + cfg := &config.Configuration{ + ServiceName: fmt.Sprintf("query_node_%d", node.QueryNodeID), + Sampler: &config.SamplerConfig{ + Type: "const", + Param: 1, + }, + } + tracer, closer, err := cfg.NewTracer() + if err != nil { + panic(fmt.Sprintf("ERROR: cannot init Jaeger: %v\n", err)) + } + opentracing.SetGlobalTracer(tracer) + node.closer = closer + if node.masterClient == nil { log.Println("WARN: null master service detected") } @@ -212,9 +212,6 @@ func (node *QueryNode) Start() error { } func (node *QueryNode) Stop() error { - if err := node.closer.Close(); err != nil { - return err - } node.UpdateStateCode(internalpb2.StateCode_ABNORMAL) node.queryNodeLoopCancel() diff --git a/internal/querynode/search_service.go b/internal/querynode/search_service.go index 14160200e9..cc8e88992f 100644 --- a/internal/querynode/search_service.go +++ b/internal/querynode/search_service.go @@ -121,7 +121,7 @@ func (ss *searchService) receiveSearchMsg() { case <-ss.ctx.Done(): return default: - msgPack := ss.searchMsgStream.Consume() + msgPack, _ := ss.searchMsgStream.Consume() if msgPack == nil || len(msgPack.Msgs) <= 0 { continue } diff --git a/internal/timesync/timesync.go b/internal/timesync/timesync.go index b61b5ea3da..c1ceccfb88 100644 --- a/internal/timesync/timesync.go +++ b/internal/timesync/timesync.go @@ -93,7 +93,7 @@ func (ttBarrier *softTimeTickBarrier) Start() { return default: } - ttmsgs := ttBarrier.ttStream.Consume() + ttmsgs, _ := ttBarrier.ttStream.Consume() if len(ttmsgs.Msgs) > 0 { for _, timetickmsg := range ttmsgs.Msgs { ttmsg := timetickmsg.(*ms.TimeTickMsg) @@ -156,7 +156,7 @@ func (ttBarrier *hardTimeTickBarrier) Start() { return default: } - ttmsgs := ttBarrier.ttStream.Consume() + ttmsgs, _ := ttBarrier.ttStream.Consume() if len(ttmsgs.Msgs) > 0 { log.Printf("receive tt msg") for _, timetickmsg := range ttmsgs.Msgs { diff --git a/internal/util/flowgraph/flow_graph.go b/internal/util/flowgraph/flow_graph.go index 83448cd813..6d91f201b5 100644 --- a/internal/util/flowgraph/flow_graph.go +++ b/internal/util/flowgraph/flow_graph.go @@ -13,11 +13,11 @@ type TimeTickedFlowGraph struct { nodeCtx map[NodeName]*nodeCtx } -func (fg *TimeTickedFlowGraph) AddNode(node *Node) { - nodeName := (*node).Name() +func (fg *TimeTickedFlowGraph) AddNode(node Node) { + nodeName := node.Name() nodeCtx := nodeCtx{ node: node, - inputChannels: make([]chan *Msg, 0), + inputChannels: make([]chan *MsgWithCtx, 0), downstreamInputChanIdx: make(map[string]int), } fg.nodeCtx[nodeName] = &nodeCtx @@ -50,8 +50,8 @@ func (fg *TimeTickedFlowGraph) SetEdges(nodeName string, in []string, out []stri errMsg := "Cannot find out node:" + n return errors.New(errMsg) } - maxQueueLength := (*outNode.node).MaxQueueLength() - outNode.inputChannels = append(outNode.inputChannels, make(chan *Msg, maxQueueLength)) + maxQueueLength := outNode.node.MaxQueueLength() + outNode.inputChannels = append(outNode.inputChannels, make(chan *MsgWithCtx, maxQueueLength)) currentNode.downstream[i] = outNode } @@ -70,8 +70,8 @@ func (fg *TimeTickedFlowGraph) Start() { func (fg *TimeTickedFlowGraph) Close() { for _, v := range fg.nodeCtx { // close message stream - if (*v.node).IsInputNode() { - inStream, ok := (*v.node).(*InputNode) + if v.node.IsInputNode() { + inStream, ok := v.node.(*InputNode) if !ok { log.Fatal("Invalid inputNode") } diff --git a/internal/util/flowgraph/flow_graph_test.go b/internal/util/flowgraph/flow_graph_test.go index 88c84af8a3..8feed307f0 100644 --- a/internal/util/flowgraph/flow_graph_test.go +++ b/internal/util/flowgraph/flow_graph_test.go @@ -47,19 +47,19 @@ func (m *intMsg) DownStreamNodeIdx() int { return 1 } -func intMsg2Msg(in []*intMsg) []*Msg { - out := make([]*Msg, 0) +func intMsg2Msg(in []*intMsg) []Msg { + out := make([]Msg, 0) for _, msg := range in { var m Msg = msg - out = append(out, &m) + out = append(out, m) } return out } -func msg2IntMsg(in []*Msg) []*intMsg { +func msg2IntMsg(in []Msg) []*intMsg { out := make([]*intMsg, 0) for _, msg := range in { - out = append(out, (*msg).(*intMsg)) + out = append(out, msg.(*intMsg)) } return out } @@ -68,43 +68,43 @@ func (a *nodeA) Name() string { return "NodeA" } -func (a *nodeA) Operate(in []*Msg) []*Msg { - return append(in, in...) +func (a *nodeA) Operate(ctx context.Context, in []Msg) ([]Msg, context.Context) { + return append(in, in...), nil } func (b *nodeB) Name() string { return "NodeB" } -func (b *nodeB) Operate(in []*Msg) []*Msg { +func (b *nodeB) Operate(ctx context.Context, in []Msg) ([]Msg, context.Context) { messages := make([]*intMsg, 0) for _, msg := range msg2IntMsg(in) { messages = append(messages, &intMsg{ num: math.Pow(msg.num, 2), }) } - return intMsg2Msg(messages) + return intMsg2Msg(messages), nil } func (c *nodeC) Name() string { return "NodeC" } -func (c *nodeC) Operate(in []*Msg) []*Msg { +func (c *nodeC) Operate(ctx context.Context, in []Msg) ([]Msg, context.Context) { messages := make([]*intMsg, 0) for _, msg := range msg2IntMsg(in) { messages = append(messages, &intMsg{ num: math.Sqrt(msg.num), }) } - return intMsg2Msg(messages) + return intMsg2Msg(messages), nil } func (d *nodeD) Name() string { return "NodeD" } -func (d *nodeD) Operate(in []*Msg) []*Msg { +func (d *nodeD) Operate(ctx context.Context, in []Msg) ([]Msg, context.Context) { messages := make([]*intMsg, 0) outLength := len(in) / 2 inMessages := msg2IntMsg(in) @@ -117,7 +117,7 @@ func (d *nodeD) Operate(in []*Msg) []*Msg { d.d = messages[0].num d.resChan <- d.d fmt.Println("flow graph result:", d.d) - return intMsg2Msg(messages) + return intMsg2Msg(messages), nil } func sendMsgFromCmd(ctx context.Context, fg *TimeTickedFlowGraph) { @@ -129,8 +129,12 @@ func sendMsgFromCmd(ctx context.Context, fg *TimeTickedFlowGraph) { time.Sleep(time.Millisecond * time.Duration(500)) var num = float64(rand.Int() % 100) var msg Msg = &intMsg{num: num} + var msgWithContext = &MsgWithCtx{ + ctx: ctx, + msg: msg, + } a := nodeA{} - fg.nodeCtx[a.Name()].inputChannels[0] <- &msg + fg.nodeCtx[a.Name()].inputChannels[0] <- msgWithContext fmt.Println("send number", num, "to node", a.Name()) res, ok := receiveResult(ctx, fg) if !ok { @@ -156,7 +160,7 @@ func sendMsgFromCmd(ctx context.Context, fg *TimeTickedFlowGraph) { func receiveResultFromNodeD(res *float64, fg *TimeTickedFlowGraph, wg *sync.WaitGroup) { d := nodeD{} node := fg.nodeCtx[d.Name()] - nd, ok := (*node.node).(*nodeD) + nd, ok := node.node.(*nodeD) if !ok { log.Fatal("not nodeD type") } @@ -167,7 +171,7 @@ func receiveResultFromNodeD(res *float64, fg *TimeTickedFlowGraph, wg *sync.Wait func receiveResult(ctx context.Context, fg *TimeTickedFlowGraph) (float64, bool) { d := nodeD{} node := fg.nodeCtx[d.Name()] - nd, ok := (*node.node).(*nodeD) + nd, ok := node.node.(*nodeD) if !ok { log.Fatal("not nodeD type") } @@ -211,10 +215,10 @@ func TestTimeTickedFlowGraph_Start(t *testing.T) { resChan: make(chan float64), } - fg.AddNode(&a) - fg.AddNode(&b) - fg.AddNode(&c) - fg.AddNode(&d) + fg.AddNode(a) + fg.AddNode(b) + fg.AddNode(c) + fg.AddNode(d) var err = fg.SetEdges(a.Name(), []string{}, @@ -250,7 +254,7 @@ func TestTimeTickedFlowGraph_Start(t *testing.T) { // init node A nodeCtxA := fg.nodeCtx[a.Name()] - nodeCtxA.inputChannels = []chan *Msg{make(chan *Msg, 10)} + nodeCtxA.inputChannels = []chan *MsgWithCtx{make(chan *MsgWithCtx, 10)} go fg.Start() diff --git a/internal/util/flowgraph/input_node.go b/internal/util/flowgraph/input_node.go index 0c0730f7f2..8682291025 100644 --- a/internal/util/flowgraph/input_node.go +++ b/internal/util/flowgraph/input_node.go @@ -1,9 +1,13 @@ package flowgraph import ( + "context" "log" + "github.com/opentracing/opentracing-go" + "github.com/zilliztech/milvus-distributed/internal/errors" "github.com/zilliztech/milvus-distributed/internal/msgstream" + "github.com/zilliztech/milvus-distributed/internal/util/trace" ) type InputNode struct { @@ -25,15 +29,19 @@ func (inNode *InputNode) InStream() *msgstream.MsgStream { } // empty input and return one *Msg -func (inNode *InputNode) Operate([]*Msg) []*Msg { +func (inNode *InputNode) Operate(ctx context.Context, msgs []Msg) ([]Msg, context.Context) { //fmt.Println("Do InputNode operation") - msgPack := (*inNode.inStream).Consume() + msgPack, ctx := (*inNode.inStream).Consume() + + sp, ctx := trace.StartSpanFromContext(ctx, opentracing.Tag{Key: "NodeName", Value: inNode.Name()}) + defer sp.Finish() // TODO: add status if msgPack == nil { log.Println("null msg pack") - return nil + trace.LogError(sp, errors.New("null msg pack")) + return nil, ctx } var msgStreamMsg Msg = &MsgStreamMsg{ @@ -43,7 +51,7 @@ func (inNode *InputNode) Operate([]*Msg) []*Msg { startPositions: msgPack.StartPositions, } - return []*Msg{&msgStreamMsg} + return []Msg{msgStreamMsg}, ctx } func NewInputNode(inStream *msgstream.MsgStream, nodeName string, maxQueueLength int32, maxParallelism int32) *InputNode { diff --git a/internal/util/flowgraph/node.go b/internal/util/flowgraph/node.go index c12da812fd..65a997952e 100644 --- a/internal/util/flowgraph/node.go +++ b/internal/util/flowgraph/node.go @@ -6,13 +6,16 @@ import ( "log" "sync" "time" + + "github.com/opentracing/opentracing-go" + "github.com/zilliztech/milvus-distributed/internal/util/trace" ) type Node interface { Name() string MaxQueueLength() int32 MaxParallelism() int32 - Operate(in []*Msg) []*Msg + Operate(ctx context.Context, in []Msg) ([]Msg, context.Context) IsInputNode() bool } @@ -22,9 +25,9 @@ type BaseNode struct { } type nodeCtx struct { - node *Node - inputChannels []chan *Msg - inputMessages []*Msg + node Node + inputChannels []chan *MsgWithCtx + inputMessages []Msg downstream []*nodeCtx downstreamInputChanIdx map[string]int @@ -32,10 +35,15 @@ type nodeCtx struct { NumCompletedTasks int64 } +type MsgWithCtx struct { + ctx context.Context + msg Msg +} + func (nodeCtx *nodeCtx) Start(ctx context.Context, wg *sync.WaitGroup) { - if (*nodeCtx.node).IsInputNode() { + if nodeCtx.node.IsInputNode() { // fmt.Println("start InputNode.inStream") - inStream, ok := (*nodeCtx.node).(*InputNode) + inStream, ok := nodeCtx.node.(*InputNode) if !ok { log.Fatal("Invalid inputNode") } @@ -46,19 +54,23 @@ func (nodeCtx *nodeCtx) Start(ctx context.Context, wg *sync.WaitGroup) { select { case <-ctx.Done(): wg.Done() - fmt.Println((*nodeCtx.node).Name(), "closed") + fmt.Println(nodeCtx.node.Name(), "closed") return default: // inputs from inputsMessages for Operate - inputs := make([]*Msg, 0) + inputs := make([]Msg, 0) - if !(*nodeCtx.node).IsInputNode() { - nodeCtx.collectInputMessages() + var msgCtx context.Context + var res []Msg + var sp opentracing.Span + if !nodeCtx.node.IsInputNode() { + msgCtx = nodeCtx.collectInputMessages() inputs = nodeCtx.inputMessages } - - n := *nodeCtx.node - res := n.Operate(inputs) + n := nodeCtx.node + res, msgCtx = n.Operate(msgCtx, inputs) + sp, msgCtx = trace.StartSpanFromContext(msgCtx) + sp.SetTag("node name", n.Name()) downstreamLength := len(nodeCtx.downstreamInputChanIdx) if len(nodeCtx.downstream) < downstreamLength { @@ -72,9 +84,10 @@ func (nodeCtx *nodeCtx) Start(ctx context.Context, wg *sync.WaitGroup) { w := sync.WaitGroup{} for i := 0; i < downstreamLength; i++ { w.Add(1) - go nodeCtx.downstream[i].ReceiveMsg(&w, res[i], nodeCtx.downstreamInputChanIdx[(*nodeCtx.downstream[i].node).Name()]) + go nodeCtx.downstream[i].ReceiveMsg(msgCtx, &w, res[i], nodeCtx.downstreamInputChanIdx[nodeCtx.downstream[i].node.Name()]) } w.Wait() + sp.Finish() } } } @@ -86,38 +99,54 @@ func (nodeCtx *nodeCtx) Close() { } } -func (nodeCtx *nodeCtx) ReceiveMsg(wg *sync.WaitGroup, msg *Msg, inputChanIdx int) { - nodeCtx.inputChannels[inputChanIdx] <- msg +func (nodeCtx *nodeCtx) ReceiveMsg(ctx context.Context, wg *sync.WaitGroup, msg Msg, inputChanIdx int) { + sp, ctx := trace.StartSpanFromContext(ctx) + defer sp.Finish() + nodeCtx.inputChannels[inputChanIdx] <- &MsgWithCtx{ctx: ctx, msg: msg} //fmt.Println((*nodeCtx.node).Name(), "receive to input channel ", inputChanIdx) wg.Done() } -func (nodeCtx *nodeCtx) collectInputMessages() { +func (nodeCtx *nodeCtx) collectInputMessages() context.Context { + var opts []opentracing.StartSpanOption + inputsNum := len(nodeCtx.inputChannels) - nodeCtx.inputMessages = make([]*Msg, inputsNum) + nodeCtx.inputMessages = make([]Msg, inputsNum) // init inputMessages, // receive messages from inputChannels, // and move them to inputMessages. for i := 0; i < inputsNum; i++ { channel := nodeCtx.inputChannels[i] - msg, ok := <-channel + msgWithCtx, ok := <-channel if !ok { // TODO: add status log.Println("input channel closed") - return + return nil } - nodeCtx.inputMessages[i] = msg + nodeCtx.inputMessages[i] = msgWithCtx.msg + if msgWithCtx.ctx != nil { + sp, _ := trace.StartSpanFromContext(msgWithCtx.ctx) + opts = append(opts, opentracing.ChildOf(sp.Context())) + sp.Finish() + } + } + + var ctx context.Context + var sp opentracing.Span + if len(opts) != 0 { + sp, ctx = trace.StartSpanFromContext(context.Background(), opts...) + defer sp.Finish() } // timeTick alignment check if len(nodeCtx.inputMessages) > 1 { - t := (*nodeCtx.inputMessages[0]).TimeTick() + t := nodeCtx.inputMessages[0].TimeTick() latestTime := t for i := 1; i < len(nodeCtx.inputMessages); i++ { - if t < (*nodeCtx.inputMessages[i]).TimeTick() { - latestTime = (*nodeCtx.inputMessages[i]).TimeTick() + if t < nodeCtx.inputMessages[i].TimeTick() { + latestTime = nodeCtx.inputMessages[i].TimeTick() //err := errors.New("Fatal, misaligned time tick," + // "t1=" + strconv.FormatUint(time, 10) + // ", t2=" + strconv.FormatUint((*nodeCtx.inputMessages[i]).TimeTick(), 10) + @@ -127,7 +156,7 @@ func (nodeCtx *nodeCtx) collectInputMessages() { } // wait for time tick for i := 0; i < len(nodeCtx.inputMessages); i++ { - for (*nodeCtx.inputMessages[i]).TimeTick() != latestTime { + for nodeCtx.inputMessages[i].TimeTick() != latestTime { channel := nodeCtx.inputChannels[i] select { case <-time.After(10 * time.Second): @@ -135,13 +164,14 @@ func (nodeCtx *nodeCtx) collectInputMessages() { case msg, ok := <-channel: if !ok { log.Println("input channel closed") - return + return nil } - nodeCtx.inputMessages[i] = msg + nodeCtx.inputMessages[i] = msg.msg } } } } + return ctx } func (node *BaseNode) MaxQueueLength() int32 { diff --git a/internal/util/trace/util.go b/internal/util/trace/util.go index 2234f5c04a..6b7b8e5a4e 100644 --- a/internal/util/trace/util.go +++ b/internal/util/trace/util.go @@ -16,7 +16,7 @@ import ( func StartSpanFromContext(ctx context.Context, opts ...opentracing.StartSpanOption) (opentracing.Span, context.Context) { if ctx == nil { - panic("StartSpanFromContext called with nil context") + return noopSpan(), ctx } var pcs [1]uintptr @@ -45,7 +45,7 @@ func StartSpanFromContext(ctx context.Context, opts ...opentracing.StartSpanOpti func StartSpanFromContextWithOperationName(ctx context.Context, operationName string, opts ...opentracing.StartSpanOption) (opentracing.Span, context.Context) { if ctx == nil { - panic("StartSpanFromContextWithOperationName called with nil context") + return noopSpan(), ctx } var pcs [1]uintptr @@ -109,9 +109,9 @@ func InjectContextToPulsarMsgProperties(sc opentracing.SpanContext, properties m tracer.Inject(sc, opentracing.TextMap, propertiesReaderWriter{properties}) } -func ExtractFromPulsarMsgProperties(msg msgstream.TsMsg, properties map[string]string) opentracing.Span { +func ExtractFromPulsarMsgProperties(msg msgstream.TsMsg, properties map[string]string) (opentracing.Span, bool) { if !allowTrace(msg) { - return noopSpan() + return noopSpan(), false } tracer := opentracing.GlobalTracer() sc, _ := tracer.Extract(opentracing.TextMap, propertiesReaderWriter{properties}) @@ -124,21 +124,42 @@ func ExtractFromPulsarMsgProperties(msg msgstream.TsMsg, properties map[string]s "HashKeys": msg.HashKeys(), "Position": msg.Position(), }} - return opentracing.StartSpan(name, opts...) + return opentracing.StartSpan(name, opts...), true } func MsgSpanFromCtx(ctx context.Context, msg msgstream.TsMsg, opts ...opentracing.StartSpanOption) (opentracing.Span, context.Context) { + if ctx == nil { + return noopSpan(), ctx + } if !allowTrace(msg) { return noopSpan(), ctx } - name := "send pulsar msg" + operationName := "send pulsar msg" opts = append(opts, opentracing.Tags{ "ID": msg.ID(), "Type": msg.Type(), "HashKeys": msg.HashKeys(), "Position": msg.Position(), }) - return StartSpanFromContextWithOperationName(ctx, name, opts...) + + var pcs [1]uintptr + n := runtime.Callers(2, pcs[:]) + if n < 1 { + span, ctx := opentracing.StartSpanFromContext(ctx, operationName, opts...) + span.LogFields(log.Error(errors.New("runtime.Callers failed"))) + return span, ctx + } + file, line := runtime.FuncForPC(pcs[0]).FileLine(pcs[0]) + + if parentSpan := opentracing.SpanFromContext(ctx); parentSpan != nil { + opts = append(opts, opentracing.ChildOf(parentSpan.Context())) + } + span := opentracing.StartSpan(operationName, opts...) + ctx = opentracing.ContextWithSpan(ctx, span) + + span.LogFields(log.String("filename", file), log.Int("line", line)) + + return span, ctx } type propertiesReaderWriter struct { diff --git a/tests/benchmark/README.md b/tests/benchmark/README.md new file mode 100644 index 0000000000..ec6cad9062 --- /dev/null +++ b/tests/benchmark/README.md @@ -0,0 +1,39 @@ +# Quick start + +### Description： + +This project is used to test performance/reliability/stability for milvus server +- Test cases can be organized with `yaml` +- Test can run with local mode or helm mode + +### Usage: +`pip install requirements.txt` + +if using local mode, the following libs is optional + +`pymongo==3.10.0` + +`kubernetes==10.0.1` + +### Demos： + +1. Local test： + + `python3 main.py --local --host=*.* --port=19530 --suite=suites/gpu_search_performance_random50m.yaml` + +### Definitions of test suites： + +Testers need to write test suite config if adding a customizised test into the current test framework + +1. search_performance: the test type，also we have`build_performance`,`insert_performance`,`accuracy`,`stability`,`search_stability` +2. tables: list of test cases +3. The following fields are in the `table` field： + - server: run host + - milvus: config in milvus + - collection_name: currently support one collection + - run_count: search count + - search_params: params of query + +## Test result： + +Test result will be uploaded if tests run in helm mode, and will be used to judge if the test run pass or failed diff --git a/tests/benchmark/__init__.py b/tests/benchmark/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/benchmark/assets/Parameters.png b/tests/benchmark/assets/Parameters.png new file mode 100644 index 0000000000000000000000000000000000000000..2ea108dc97e8079310c4358ea82b92e6e3075729 GIT binary patch literal 50835 zcmd?R2UL?;)HVu(f}<#7p^1PaR;q{;fdpqnP*Jhaq$N@W={-OQppJz;fPi!b5$U0Y z8VFJ%H39;mMrtUbgpvRu$$x?~|M#uC*1hZAe|_Iw|6O;jS(6aRdC$Ag-p_uXz4vR# zy}P;x_aEQS#l>~-<_+x!TwHtMTwFZ&_wj;HoVf*;xwuYp-PHcWz&C|5x`=rD8bxN4 z*q49B@aLSrYeRpp_D9R9+rc`Ia-@H#h4lJb5pGGAnJIend z8Q?#c#8v>i;Nr4>!*6oL8=cR>6BYulB$3$Z}q< zz7T2{dSVqg!3BAtN8zT37hMD#OySG=_4UcF*TEs8WT%3%aQu1`G>_w z`qJhSqF$^BOEbNF=T5;C-YjN(d^|V(N^+;lR8{-*>U6td)MGgO{Wu<-#;j-4*%N$P zn|}nXC6mGlfwWS00)*Zqju<&C$spnwRGVA4`+=gxUBuspHc|d0CzHld3O>Loh)e#bN38A-5jS4cS zQ59-e{%o}t5LYjg#jj4q>i2KZeH3{poyn?`gLdf=HA~eC@$6L1^>5dx{bsIm?RI-O zYY$b6z>kJ#JCW)jV?k_Y+)ibfRj`F+{m!&Py9A!J(@A=(Nt@|RE_R=6iNP-ol{>cL zsWZv0ExORvXj^f3_NPx)%VXH~)#WK`NruVJ_CUSmD*2zKMLA1ztmDqy-iX`z>aB#6 zFCwQQ3n}g?ntmly#gmhhwvvq00QQf>F?I`_UlSf;e12LbEcTeK(Lz57G(* zrk9;7<+#yBXN!aA5baZG$fc@(7jzAn6tj(*MAS1ZTCHLJb6F)(5FZLLCUTruY#p{; zV@lv+EVEh5#g7|b*|HHmy}i9}>32EtV&ZJH!Byk8Hskx_8DB1KPCv;1J{{}i>>MEg z-Graf^0DZd^Xeloi`d&mg6WRIa(#n?k39IzDqQ3k#QM#!`t!&Ry(+EwEWLz`f&%A3 zKB7Xt^2Aa^MyE!Y{O0w}mJNPqEqm}N~ zBbDxx@loR%q^+)uqCX3iaWfuOmB7`dp=<&6ioj$lVk~EKCulo6C{)f2V-Ubnt)oVm zv1&P2zHN4t$Eld(JygD+6+S~4T?V#wiwD{{|2Q~x8OEB1wMjHKyx`N5&`?%3^m!Yw zIsV#pi>aRuweaC$b=W~mKXTKWLXHZiGl27H1~P_8{hF9`)_S%eZ1QOOW|Lgd6V7!$ z6;yGnWNd$ad*iefK2p4W=cbV`4<$f1^27xsze4?vN2@gnMIW_g)~JEgukl%w_QTR> zv`~6d0U{VR{!&;PN@tMRfunfh=%@wNdv$9^4(0N5l()Vsz3OAd@9h3yFn_>~p6fH^%_=;Ck%z9m8|v%Fjx<4Tf%u~ri% zUu*3=HplvVk4+l4b1@cOgK8*>Z)^U%oO(9_Md{Aeb)CZoZ8l2Aqp(NM-8Bd(^5-M6 zStK^HnJhoUD&XPa5rmOP5hQFD;UL%xFx|DT@I|EP}9`C9lDp1TdUJ0 z6^%E}#vZ=!M*(JGHV6dXqgH7#1_|@fE*`!^1_7yh3G#B=o;=|47wzsGLJKQc=5JTm zAeIZa19&K9rFMiCGL&AG)VT2Fkhr?nFdxy>dd33*pr>E+lUy2u8Oz2gNFT!>AXv!+CW9B{|IYIM2(9t6pMQs+&t6Tc^?Oq>Q3@$Z> zT*g)|*D+lD-sgJ<^_D|%bxhi@J6;@NAg+_3jtd0-Lk-an$e?%*4-ZdHG>0h@hU<6Q z>)%4Uv`-yoZ)vf~)m;$I!NviXp{>vLQjzs^q#$e!0CyUqsK=w9!NoB}C7Z=axO{KS zd%$ABCpa}JNg8Q5$E1@a2MR6n81Ebpq7NVD6OhQ?+))UmhDEne0}tv*RHC-(2bN5% z1+B)W%Pzg*yEE?2ZaFpj`n;V<;oCDvg^q--|wD8Sg!qy%hmn@Ns z<;41+pk^7^KM?4m#kne)pMy9f%qZZi^Xi2gnz$#WgVkNr80X=Mx$N{vyL6NaYgK`L z$!BTgX>i2cA6^Un1!tnz09NZ}tqbjfBi{J3ne@pIuO&<|1fOLry#<)Q;$u1=XZ7Cx zX093l)SIt^HeXjqIYI=`ocK5O^bG*9)MhE3csCokzV~%rBAXG!cCB6ca$wDWsfyfI zhp@A=1L)~H{&I6YTg!q*C1|k&TCD+Wvg;`BC)C{DPi=t%sTYoojkP5Vc<=)|4-~bLXibNvk+A*hW!Si~BcKw2emvUu=T{|*-t1b6dXZ?>?v`GR+F>brNJ{h(2jT9zA1 z?A3(&aE;~KHH-jydS|$vX*Nh_xw04Q<1~F79C9hbrlTMmdDoL%0r;1sGpopHGK|gX z6L6_hf#5>dkw}Id+sto!E?ambkaO$bF*AI`gJLR8KYqXgGG;DKnila;=yh~#E@5F| zA#cGmp=Q6=8D65~&JR9Y5`(>y&VIN`q>mwsDAtleJJf2@vL0;vLF<}iz)C}C5Ix>8 z7#qm+2X}rFy4@WmzeQ?C)UBm{C9iq^#I{_sHIS9l**oM(5d8%UmHi7`!vLi={Ei+y z>Wx|gDdZ#w_R0w!N~}Kg5e-X7gP5%nCE1&Jkn<>|g#Hant*uu5HYnS;e%r8_EQ{wt zf;_o7AzTkqpV3l1n^hgu&$&5+3Kw1?h!c=_B1kM+g)%tWLulafVIZsLO?X1tTZ92- zmDH&^T6s~yCKp3DXkGWrat#2-5_(aASD#Dynx<4bA%UsUOCrHG{PcDc_rx%X<U2t7p{IhYeUFV4;2$|`|mDd82g$z$! z?VVX&Ez@pp8i~y$wbv~x5i}ptsLsyLXQEIm4SbGuguzm~%&e@esj9}tM&YtU-rb> zz$6!=M^C{J)ukV}8e6S}&~D0km<=q#tOP0jz=qaLbsVgZ}r@ zImyR>_26S--9~|{VZf6A2{p);R+kXbK*hxdhSU z$8*8D-|K!Eo*r+=7`SWT-LKeQ|LyOg(7O z2{X@tf_QCT_2e*6lc;{mZk+CNtvVW@S$=xPI2TeKcFB5kj9xcznTNvRY#Ra0*7Tj5 z9GhE#(B~!c9zvIo(Lv(yPOs0@jWR+N2*U#02Lp{bcf8PrSRp5^T{QdvCj7zF5A;pUFRHJek>Z_;dC$y;#v z!og5MSW&-W#Bhac6yP$30W41Tu4F9LB)bGND}opmNq9i0*p(9u9qpZYgbhhPY5+iQ@X=heIhhzV?!=unYua#v3j zQIk6S$nDOZkyFjNk@8?>ex)cZM#Bc0Lm`R&{^;rh+% zeUsZ;grGpPNNeRoXnd4p(E?Et2o~R&M7s%)7sCJrY~s+n>nNJeL9{~Mrqv)_!(-}3 zt96li)dN~_6oh7SEzuu|NA)!i!?xOta|2t=-RUEq`sQ^3 zTbLA%`VKC47(IyA*J65gP9`NKh4M-G!dsU&fbV^r<0XQK^qq>t#WE255{z84AjxuY!Gm1d;}KeG8yjg<6;1H^nIgQ`|x2c zkX+-}EkP97bt+_$mn~aNL*Nm1-+V8W=F95jmpRRtw`1Zw;u|CJ+Z=>02Fk z@rxuq7!1ZC8V%_Z;8Jju3DD^Aa%NTluN4&)-SzN@wV6&ew;#9+(%N(aEb@*0JjWg! zqPf!`xzmtAU~t$;Wx&>|vm~4Z)PKn`fNWGAWwJdY9$-&x0*SID zTh`ABnjRGuxSn|?irHpdS33;!KOiD8Fn*iYw#6Jf>J+f=5iAx<9V7&*0BW4Z!Fj9RPj|Y<#v}j$Q$cA(M?EzXz*NEdI42|I zKjJm%57%N%YRQWlfY822yEXga!-tZe`ASgD1N;qFzx01YmFvHtQ1-vs5;E#I%Nid! zd05>Ej<*5%42wB^lk;D{fy^b`-8*-hL9t~52*^?%3eZ}UfX)2#2W3yH7kMb(zZ*<} zI*7;gE$||!#r@AOzkhc5M?d;+IMf0SDZ?5+v%Bu;oaoVhZrb+7g8N)gR;AZmPiT4d|x-4*MtYw+WFhBgWz?_el86OdBm4t=;^HD)ExLi$- zq{%;*cQNKQ`FG8G|DSlfU>6KQoQg?SZE1;69;#LLp^ON&LSDG% z87Wt`zt++R{WiT)G`uwrokg_t4-OCy*U&yr#3ho`1J68&S`HO=en~EO4oj_}HH~*9 zN|QIdYGa1PT6Hnk&Bn1RuA@R4h~<#)Aus7&5aDw^u7&=2ubG2cwXA~l$pdyJ?p|!` zVn;HjYOU1Uj1q8K6#1a2a{JNGyQxUw#(c4s9M%jvp{b^EC@sv%flfJTQk7avZ`z@j z*qj()+)5$Vm(JD;M!l$DK@ac|oujlQG-Mz_3)b%IO>j!CZ_o(lh6KdIKk}|*7#?Au zg~8ZRg`MO)f-Tk;NNrc@e#?%6NN6~uPGT(FSK=J5{p0QiP#&i$e7re#jnX0Ez&kAS zC$3I;S3JWU<(iDsOFG~4{IYb#RxMLVYF68 zFnd2i!`nBAGPq``aDx}PhKxd zhp4f4oKUiYiArRkjy&Dxn|?gE9zIbsWATR>LjtYm$J{1RUw&O88LdL`O4*YgRZ8ya zp0bCQlGsW_MPG+zcEhdRqspl=#q&6sQO2}Hatw06!B33Tt4bhk99!Kgya_VBbTLRm z2)20=HaxjjsEKtO|1;%yik8HZx+N-jl(yL&1IvWXGv4x}hiO8*Q-5YZIN)*`FcMxL>%)YhJHQq0tc3n-0gQ0viO78Pt&8U}|U29ofE?d`BD`0S{~ zLSLw=%a~BqX?n|#ZsQ4qm(k|`++Z9x=63S1Pty`4N6o!@#4$DGC?7&$(<-|SXugWY zP6JW6D--2n8*JelXP*VijB==kfqr6g;+4929?An)kUM$P>jbH$bakKE`gRRXLZLe1 z`A>f|fGzk27p$K#?^72}#17x$tuD|&;H{B|bmoaN(Ki?KgMvrX3oag#Uql_hf|Cze z?2OVBLeeNX-3@9g8YM=%``<{Ptm;Y)JQFDqpy{({pQqkrB;)Z&Vp$g(J6>#`&Oo{qtT?$9=!mjnJAnLVSAm8%xVh~9d7At zRa^4e^V1Hd+PuW;wJlX0V}2ETh&#lPy^~{!YVjkrQ`|l|%e`N@X-DCeP=QT)X1 zRy#G}!Aaf8mGQ=2amf93aF}&}IffL$*k75qPC!yZ4IHX7i{6ec&_{AIqn(n2@^eZm zvi|;Ql{ULzizdgEc1mK9e!nxHl-XtTG-zG+R9ZoW3XWq6kj~WvUucAqkYQoJ%wE`#5zE2+XU5C%0G7qxohxD~I4xj01^IB>_=YU3ekLFBfN)n+ z2qX)3#AvABcZWC^Lr55fEmaL$1^LVs)xSgH(vY^xJ9+TDazDfg%XO0Q&lL;k;M@aK zZ=VQs(oKwZ(bbG^YntmA+p3qScempg8ne3(`>p&h`cqI$-G#s+eyEW<&IxIw|FLbv zcmvQM_wf!Yv24TSn7e3XvE3Y?;)Ch|H;BFeZ#@YR4{Y9eZ^ zp~2t;#B#iJbag?Y&@;P~+C~NWWd(jh6D;)U5Waiz+$qnYm49O1jKC%H^O83Yw*CM4 z?L4S)12P+Pdeo}Q+@(3{kC-H7wyHsl3uO2I*kS0;_IAGT|9Zb0(D=()pjOrdN|;i~ zojPY@<0~>O|D6Sh-i=n%I8pesPoAKz@%2e5rw82p@__IJ(0!@9zwXKH0E@(dMYL6t zBS2wXLEf@f1oW%rEvkP;dzrKn?w32ZKqHO$FBEM4cp)$Wn&ljM3aa8ip(pPWkL)*t zl0hePpvVeAb&6A@UKl9C13@+c+Q|R>l|1LKE+yXP^cCF~hsrB~?&dV(*MH|!>z_Una^tjVrfM>5^L2OZ zjbkj$!y$#y@ABy{&%Msfo2`wyke|9l(!$@Jm`h|(QgcL# z=-AiQPhffFh|3-?*A)9nMhvXkaAs!SIe$JKE7^+et_4$IRbZ&35YRW}N}dgkRu?9= zju-k#nOg8rroPs8t3=y}Oj!vLM@;xPFaxUd*>Ty!E>Y)m_|}%J`)l5pp|&qVR;|e@#nF;&D1vJukXybxEgsMyn`|+h^Q9 z`NrvtGgmwXpB$`m;`b8Ls!&#sWh$y&J|Q*d70#b!1&6tMl?!Qof^6%wWBsZ3h@vm% zx<$kbvYl>^d=acUvkZ-vV{gwjR6VP9yx1cc`PgGd~_DAzkWgHfhCiD5eZg&#uv zf>cy-K228>*P zag>K*!|&a75bW)6lDTB8JX@Eun!RN4MP>E?l zV%v2qFUQOo`dcQy`=7Rw=5f-6xt0mKT)Ve$ir5AZ< zkMp&0puDv@$ok@&_C}k0PqsGI!KDmA3;nTb+$!%8+-WznbY@uj!Fs7#7P&k`?rCsm zdPMB&?F=LXTl}>56R~V=%ok10@ufzXYqL6ix9jHE{q&X&0clBBs5i`MsO-WLYChtt zF0`vI;QV&=a+qAQ${jX+akLvTJFR9mW0xG1skJm(c0t8?J~BaON29ZT%WI_K=|iWX zb+=mIK&s@g!23-``O$s6#1B*vG5aEv*z?xt*)(j7hvoy&g`X_F=A0pAD6-E}aj5r_ z#cpUWBkCB8AMSWlgpD(;}0)`kcFOchpG) zo_7d+G+I*f4efZv?;xnt0Ks<^JYrJcjds_Ka3;H78}*LP9HD5XZY9eM6xjN2$Q%{?h)#~U9n!L`B-VzaeSF=pn8P>?>9Jh;eFhoqZn%vUE z#}H@^J4bq>Y8iATQZqGtF&Gcl6@H!sdiD@vn600B)nq9q| zD)v}15IQFBZ@i1J@ZbXAP~n6z{jweBeIANE7MitvO1K&8fSIWZRXy| zVO|3ri+tOel*K!)H@ok>n+vV;IYC!I^rm@{M}lNYdjgvhKmDtdfGn zwLytd3ney^ypBT1)Vp_cnYk8pD`^)05HX(fbG8bTgcI;#O2+ljfYV*^sN3?EcU^L> z;CzftZ>+fZEZ1XR2 zg1p2r5qq>!IVoX&ORlK4Zswwuy~2bU7dn0vV^oi5=o&q_qrNwai>5Y zpn_2=TR@O^5UU}FLPh#*LX(+0#q6O|4;G9j=>TgaG9AR7^@?9QoX% zK$H%&SNAu4vZPS`qaAbhLARpPHy?$aBSpFCnkA)~8Qho_F)_GtPBZEGkF76q`!L$8 zHq&}7R{9>{>NJUhtG^k$)mFYe98a|eAUli_G zTg>efXS$Q%hCR%r-wpb80Hu$;PhI{tNS4f}0!>`_zO%W#RjxOxW_j+Np9z%4LDYqlZp$0FYTj_GwD8%vUc2~C8Ub7#$bR|JqjE0?mX7D3Mjfb=!(;_CfWQ<+ zMqzxRNJqcO<8!*1_1MjR-v4QA2u=rTxm<(rpBjk?pPbg|+7Y9EMQ?L6_%uGtqPIif8d5~12@;k27aDEG&h)rbPC7+s6FgCzQ}hAR$blGGDp{c+k}LokYA$Nx#! z+N-SYl64sKEc&{M9uLJqz37e;6+{JV@meX*+^0qT=Jw`mb|972*|k%?nmPxMmVson z9OO5Kg?qZ52wjr%l$u{_4)Udar7&FPt{hoYA?_qo16Ic}22?nT00$VG#dj_G_MCW}k;7oCC`L5l7b-yvaoFd^1=}Lzj5l9$WD7OzrZQiJ5 z&P4kYNaovqS zi`N?NcJb9KT`MoAo;Ua;j|Cp5KUhw+fjc)&sC%7H)AG<9Q-f^^uhO1i+lP}GVjSNecB2J#y*p++V2^_TrsyL)sg@D06l)hj@UbBFo z#vzEwW=THYQlZ*mWY=?LBe>(Iqm#p1ThF}p%-l2YDN}LsYyH+b1q-Y2(h^swb64@$ zdh<0kN);BQn6@-_38^q}&6QBETC$6%Ci{70+fPa+j>Vg8XXFf4>yGBdSC0%q;A_;G zHAr#RR2X_D=z>wnR9M+AT;O^{;E#l=sJFu-BgZ@v6-hU=QSsnrwo1=U-h=6DRx-y1 z6@A)K0=gyad$l`0gUF*lwbdCgrG3Zt!Y)C;Ek5~g6k7imA`Ym=5Fj_jc1ivxI7deU zF$)G|aJ#m=yq6<9&z?PN6%g>#x<5YwV_JtnH?k{TOY-5vhcYNNaE@Gt|4A+@LZK3% z6ZvSaaaU}9{a0=`HEG+F4t}+{^dIt<^VR>-S@!?UT6F(QMG6!0YV@LGgm#9qf9X~n zR0P_pqHjwbfi)xB8o?D?Q(U;h+PNXePZBhY*P5`Tm}E?p-koI zH((ice_Hcb6Kc`lQTVIKQ%9T`ROSHuKH9DEcO=!X-aReqwyWOh zBhKAgM%zGsM;G2WeWW`&=`%y5^cu6fw@(ojUa;6bI4Z_7M}5bh-x86ukJ8SNqeVf=>pm2Jy~BUq?$lv9Y~mMWK~vS&X8V#e zM?$o{4&8}ou6fbd-4Rj8*nZHnI((T$*-ArpQN`zI2kop`dvnVPW-d-@w9dM_r%Z>` zGiri-8{!JLzS4}V_x^#^YJ7)~t#Xp~uNlh7T0xXzYqVZjSb~`hv^;Rgnb`Cp{QsgHkBs?M zB_VQ2`_vIRNOT%RbnELOFEB76zUYZ3#-9kOLgWVmkf>t!_w*l5O;xf+ojM}Uvj%nC z0@+rpv5Fg`+Na8_b589Z9aYwdJZD;MT{&_Hj7j5?H~`V5oqO?{_d0%-@;)xCxMA-8 z#crs`TQqk`M zqXvszpWH|h6@3_;_%+V_@fBFNr#iivGIJtUvAW3G<+8aNZfwajS}ejw@6?fyLXk%o zp))U0;EX%E1=)siKn3?83f7Rjg2Kjru|0NZcotJdA5c;KW`ym(>O zc*j?wXm33gs;Z8m+1}#q+#%M@$q1S;to(J&3b}n;b{1qje_d$0H}HJC*#3r-q=}nM z@GbK1N3jJj38o%uN2BLIYvRVnJQV>$scP`tuFpn(BHUl5Txn~*L0?A?CR|{8GY8!< zQihfLF>QYBQ9AnL^F!TxUu#`a{|$vjDPB}por*t%4r7W69Yl*Ou?2!N^vy()%^0Q1 zue>>%KF-;63t#r@FN23&>PCE3E+snMo|2l;!LLeLHEHwv+$Le*a80X09enHfY>Mg= z`JJ`s*;}Vib@#(Il{c$?FUEQq%FAeBT>Zl-vTJ$F!-`*s!DX-u0OBN^99H@}}>@l`#U0meW>}1Ej5pKxrV_)K-WCXqP1f56E z7y?ZvkAnhm3-B1xoiphd!u?u%^3?P?!y_#eAJ7<4tI;E; zre8M|dZA$7-%84v-ybv%{MH^tGZ2BhE?2z^bOZWBu2n>?7Qx}4A=+gdgwg%K?}wd; z|NO{q+%mV1Fl#87IWv>7$sx7uJ8|@*R1GYNC`#o8IzfV@0wi| zl?)R2(G_&_V-gmX2OBynfl?-{J1)xvE`~JBA@eX^IweI4+tPhnCY5C8#kF`zecjx8 zyQMJn{&Js#+E08Q$*JPxX?FIUlHPkwPEvdFDqov!zxCKDEwQ^;s%`nNsvC%Np|2|D zGbg9+*`YG6BZg0RuyRpWGpvt5rLJW{42$-3Wz|TVb0dM)?#L5%x%O=+IV3<{93s~Y zi&t68cnM>;4h!2Z$K11WC(EfuM|kBbRPVQ@jL02s8+i`z*1EQv?58=f^k8EwrAlP+ zLS2u?QdoAiZrjc&zi#12ojbP}>$Ak;s51&sy6ppHU3|c}@zT-J`!r{}KrFT+;bxx( zu0`8n+o8L}dU~WUf4c6k1x#AlVD_=Dn~t!Lx}WfqG32hX$?_55usiiFbvF9z>)8h? z_5!y#t1G2^@wXRJlUXmw9i$KOV_%Q)uEcQr{O5h0RXThvwU@q5kw|){_2kYWU4-8F zz?XAUp}Kf&v!L;Te!Hnt8W)+@4MVy<4F2x*q*7Lt@7#k@`ndv#de_0Xi_F9=~2}{Wm14rb#`_e-F@COvJYLrOarFAz{eN`g7thKn2ZZj!P_TNAx&{fSu=1+^i5U(z;>zf%#eSNb2BA z$0vU+gnsx?_r>1K9JwATTN$cy*7{>b!IZaIHLp}`BCC9@dKsA{`VLE)tsO#I+hlB7 zxH^N`yLY$ggvh_vB9$Q!AIyfr3Api$`R(+-yXTPgM%4$ubPS3}CW-fw(<^NFBl?2I zRwu7TJ1MNGhs4jOU?UpV7v74)trh>X$Ht|H7vENY_25PH@w>WD0>)Q3VXPl?|AEV? zlFY{F{ZT62Zk9#&z8=K+&$Rx11_y|>A>sOX7Nd1pUbZvf^QxV=^!4l%!?WE zwduX4sP-;&yqo&FNmtO~5+8zF@Zf>#6?r zD)48)hM-fE|Jc#bD`o3aN&gM`O7E-k8W&M!&*N>vyDU|z;|ZKff4(KJ3H%`F0$2wI zxY1Ze1*wqk;bAd|#o|jKE+W$W=#MOh$(#u`BN-l9^s549x1v^BWP^=eSq0{H&>u!- zx?UUW_-hIIkBxj%uJS#NOlco0`M1EH& zOcYd%g>nY%IU{MtMV1i((4URepUto{$(=2nF*GpMz1znEPmF+}P$`$O+6%jvDx2TU zTd}Q8U?g0G(+mUS#x38!f4?Pp1eovY^?w^%GO&mu@Bz-jj-y#Py{|=%eSiMk*1?4Y z)1-0GW8kCUB+ll4J`&r>>HEbcB5}L%7W{-Gc>cTETmL7j>;Ijum)SPAP6l#Dxn@C& z94lq|I(j z&Q0ZPUhUn!`bOKS7F}z5&}>4KU3YBnAVu}tJ1&RXu;Jl0=knMVopz_Ltkr@VPygEQ zYkYW2<1f!Yn0T=``{i^Cu^(%@(%& z&RE;$?~q}2#SxD5x*Pfiz6E}un|35M=kt~|ctGJEbS#j1Rg<~?%$GIw zkC=MD9}L)Ws~omy+{k|xtJtH=_S@h`AJ82JPYubnc7N6+3AWl=zc1aaD(Ft6L^XUE z7rB{jZF63bhvA>;I7S?hKxIx_Vtvaf|5WH7sz^KggvE(1CSv_73Oy~o`Y1b|n1FO! zmtf~8R`e=CFZ;Fi`ErFAaXC+N)fv$_k8D3p!PM(o3$!#82T{)^M>w4om%8?%d%*Cr zM8$fOTVhrz;Ja3*{fB4Rlp25fb%&W{yqh%dwtq&ZuZi*;^glQ6q%iaG5%r?O8kUMy8?YhI09 z<%M`#@8G)yZ6DzpeuOs!rK7uD5!qQ%9#q3R=Wes@YwwU*Ua+#x9n^g1TmDsLP#IRC zl@lVe#8HikM$`#1@VHs&YVYeM>!tDFp=UWr9|5#kpY0Y@zR7|iNl+WF8~d`aU$9*M zYYJBQshR6~`M?$2UH(qXgM-kE60xPm!JtYu*nTc-OaX!@x97C?ke}{`cR;d#w)dWM ztE@)PrPg?qZ(ku?_n)EhoS}DG z+c1VN^-`|?I}0$=GVshv+}MC@ZRNYPtgA|ZUt z__H}!YJAv;9k*FEEKnx!(Hfmu$L(`U+0(TvQshL`n~{;L39~niB|1Z26=<8m45~(F zf0wDmRt8nD|K% zt(el84?#XzOmv)CJ5~sr@3B8}?ollLKwXOG%!2^|77x}QZ)bxU}1g|lK*5D=**ZjVlx34|8n(J7Rx>ui7q^4hMa_hm);Myee zCghoaJ1+)&i=FP|y8F4dyfj?|o*=GJ@~JRejJqH9>68&g;-ceVe7U(pRT-u*6E(0> z61%~stf<#-kKp*n%9*eO=&+45hkTg2!QeQjhnI8Vv@A76kV_dem1C(&LN?sefrVbf+*P;r7TDinvdt1Zw zMj>76JF6Ml_s!zMI@J;%mP2}a+r!fAP^0l(wOTS|FcmFx#1%TqPTHcpPqOneGU@56 zZ_#l3fn_XNwX}Q<9wD8Vm_@COd6~PkEp_LK(j6UvYGneJK7CwYY&Zb*>vOVCw?J95 z%qT?D;Zm~wx?6T$Mv!J}dtHCz*Mn&4bLnQo**lhR4RS2!XsnRw-C-CmwIkfLI4&{% z2jlI(nam5lSjd$)0$;pJ9Nt&cV7qViap`Hfjn$=&#$3#CQu5x6tY|~5>NmI>j>w!U zs#tDaA3A{iP%+OZ(I5BuufOUAG~-|VlJETVie#>*#rl)8c7=@tzUBCVvF)6t8{|6{ zsx`;&Ch*}4QU}nGK5sA*8o6A9d1Z?b^0~=F&%E|azRlBnJ_OGjZmbM$l|ZAxq!0&! zi`RJbz=6@*Dztw8@6(B5vbqcSBb$_w(I@BQJ_+H#S0|nN3)owZO2@9zWaEC!iRqR}~MQ$&=mH_HlKwLD=j3Mc7d&f-L4ncv*W;x|#nA*K>PMa^+z-@oH z{G@L6w@UHn-Geo(KY8*`=bh`dxOD2_K)_+ z*+axgXT{vg%H2+mwHDVuXqWhG#Ny<~y4epu#)c#@cjj!r8)bHX8|^nmGk7R^H9wkW za&>C%)tN7a10mMA?;wuNt^=#+^^df_s!RPOpx=WY772H zv&xnHfB59#NYR@F}pP?4A(>2@zu+3x~VOXEbm9h#E;@OhEn)w(=xn7Pm7n3 zoVGMS#Y+g`Bla%@>WBEzMJ3-6cGZP@i8y}Z=F41qsgxDDj%xuwQI0bIH+P9`?;eR(gdh!+Udm5V98E z7IJ%Q;}dC+_jbDBEh`4JVw_d1f6E>7z)f9Dt#|z~8 zE^L=#u#(9D&|EElMGrgH-oo!OS(v_AdLJgR_{v54%=B{svgrYIvYupKg--_-lPF@gLu6xC!XRxPhczlQb8STT5igW>Isx(S^ zw@|EnF{X6|j9>PDEVv~vx?Q(sbv*)4{@b(S9F8~sp_*@s#g^kYe{R~J6_53XIAV}1 zIP*8S06&^_X%DOVoe|+t;7M zY2HmV=GCP&AaP1gBtz87IqNG zJPRXT>Q*?VbFy>vL@{qZh>oLhh2yU0KR@GQ#8+@0f(B1_?jc^|ae|9J@2k#d-ER2q zy<)V9;_mOkC1aBg`r(XLvLLQ$tq&3%D8_ z2QlnSMI~SCGYhZ(Be-0-<_hA^xhnZ(yAKO&CopnZq=UH=p9$lp0OFwXafH9`kh_oNkhEGU`~bayIV22sj*xC$EVro8 z%tJl8CvW`CR^#uNzyAEko&wJQLOINBv{qfg{|8nSvlCpVfHQ@#pHjDY94uEh<~Dwj zL0Zx!zYDhKK?Bcxk>|_hl{g|8!6jxg({ST_dW4^%PaWL?_L$en${re7*+2Yydml9 z9jJUjO9wgAdBZ{g9d&84sCHmJZ>sjwrkj0mTs~gUPV~jcj@RhP==}SJ9I{1KGSqpd ze{#2F%eC@h9~wYHr=fn)>#RkyaRdOn#)$BVKNR)*aj;h>PpiwDURU^bT<6a%2ffBE zoj2ve$1%BW8tF&d6H=xR(Ds-}<@vk5Z}t=Oh!$% zM4@jj{{6*c?Z$ng3O3Gtr8)lIy^Ir5_xoo~lLtU{b(}9~txi%l&-GTebgAu6a{MMM zb5spKHxEaiB{zV`Qy%gEHUbXL)c<^yW4+NvrQd|5j>FrooelGWCm-uRTeu~NoS%KN z_SlRTDSPjW$zannA7 z19^4QYm56S82tBP@<|IKHCzoX?VkRSQ~u3o7+c_%SH`?oWb$W@m+72=(_(sdN8}8q zrf{~73pw&8aacf%q1NE-X-s}>eaY=jlt#SuZoob9kaa>?B6FWWG1x_uTXSBwLWna^ zTf+m1qdWe=7OAhF&Td3MP){|IF2+hGIw%A@4Y&RUWL>*LXZo@j{LDh}M91j}nKaqi z693HMWEsPb=Uu6fKEKQmdl1`ZG4>f^ipuW)JYU-qk^^|U-ltp5BSTtRMUfbZFMWtc z*B$fb_S{@m0mp^*q0(2O4X^!Dg$<Qt-$2~9K$%<6h^!7sE_u*nM++*&3zs1^Y(+fx6$zdT6wnCE{A$-1I1_YM9 zx%@xad(W_@zNTL|pn?U&21vI7N)ZJqQdJa{BE5GN5JHm{2wfBu1XKj1i%3mK5C}bh zB1NhKN$8+R4GASA0Yc#20shZ(-_LW-b-iESbFTAVCm(>Fy;*DRHM3^T%x`9vfSs^q znP2nzX6#e!OOMd_KD8=Q^VE zEYO7yARmAGBf1iqRUN&xbXcu@Ep#woSxlOwFI2rtxAaS2fMFP2+)S63yohnX)?Wq2 z+ppHifiWS1b6be)meB~o%5V9(`M+`miNh|}cx!4ye@;HhOE#_N#n0(eA?%}aM4$02 z!UuB?yaT0xM#4?H13`s12Zaz@GFxbK+nL=$wPH zAbut*RAg_2_W-VZ`M&vK_Il8Ry|;UNgE2j7A)7z_m9ZTiF6i$#R28FRt9T#HcU{-NhYov7wBut>M_l?CYQ~ce!ux@cjYu2OT_N4Nv)%M zQvm{{AKdi;18qJ4sI${J2qgN?41>Z>?#&_Ky?k|Sm*qg9iQ7Q;yC2VUdjtf56psJj zX8+qHocn>TdiTcuf|1~G7FuTD)gmKs8C0(XOx5iZ_ENlSnT+d++VhRfH*TBvg+tNd zd03~#LzLfv4E-%^G4|i6a_l~~JO^)JGWYy!)0I6sAGDC>yd<9+Q0 zlP1QwL5{v5C!HN)957ICw|eO+DQs1tmul_@`?f15G`V5(SI_`P49t-Pu!P&bL>lL^ zt3e&mH=eY$^aZV!_olk3~SW7Y^ebr5Kr#zZorp#~KdUMvT2)IqD z!ty`ax4lg4FH`la^*B;;h_l1InN7XWRlhk%v1GI;`jzpUF0PpNymU9702J5Pcpa#PRk_e!QyTK}ZL+dfW-vFieJNjz3l`j; zUxt}l-MQvHai@%9nD~Ik3_sg+52m;T@fnms`@1{_^9Cv@n#$mBH{@jrC1=!;hBk&? zKM4F~eErMD=FrGEw@2+Dm>rFilCA1qdwcbmUA&T-QqT~@t5f#$mP){kyuN|>l??yu zRVFQ09-qEstloh8utL6(j+${%{HYW{wsOQuGo zZ#lW{PdUrPnh2KV=rhUAiZVj`eo0ynS9OgwOsSmq7uG`einS!7=Zb`?M>9kpdW7k? zjuAaLjH0jq_6d$INy@;$fA*961LBJV&;{BeuzBp(FmKW$>DSN2h<=3ZlMZG0vM0j# zkI2fTF;NCcWgViyqIaz$?8g&?JjKr%rgIl>nkZ8!!hqdQl2G_^U00YRQ3p;pWrxjYNALcFT&Gyty^#{)(-D=ZMoprAC=4gHnjp(9z$o6A z$wKH+J>>p@uAXIP*Y%emF!(C5$hg5I*UCUO;qXG>nMTD$U88UNKB`%2;s~jmcD|;SoyY?{6Ehqc=0!GPDwNFyQo#6+n>f2CVZKR9ZLE$2L~w~xlN|E$ z?sjx89w6BYyx9!xcqc&gc|c`OZ6!1xA*)iLjTRgou^c-&mF)_vv@n%Lf4Mw!sDh7@ z*>_;zg@yU{Z^MW?OJ%OH;;&+PN?<(sSG`a3f{JZIf>{=R%{Vn6mogUWSeokd1FYwQ z!6P*t!l$=*%XaDT@-iTBo|qr)hhvNuyDJ|h@C}Z%%@*T%D4rMZV{hOb@8`PoRC8{I z%}$T4WEK?QtY5Si`(D@mGAb#`R+3c4;{1??dRLrXy?hLD$6L1c+BNr@uX0r^3twht zI77)h3yq>O*uBMj$5;L6Ps>kdjy>7c365CQoPJ<{P(p}XeknVMa(WHDs!A!BHQ z?(4Ht&_ntbzau@~&{nx3^K{VLk&jpNlsF8>QB#yG1M9i>Vg3LXuYBQoO=TTlmv3G%9W~#UH6*yynYmmS0{<+!$Vb8{c41pBYB=h7P12(bgx@3OFam7- zZYGKGJkkb-M1$hS*U)uDB>h2LHA_>2Xjj^p`qy4Zo#L?@=m&<03u5)Wh3!KdmH$Gb zwl6c1Db8y9H_qp|wz=Ucb;2^7D;wp7*dY}XYos1{qLW$Hh^;ni(`eXwdo@87@VLku zYr7HyZ#56yAY}?E(ano{OHIk2J}Et1uZir&8}fxOEs?y#8rF7%`f)p9^QP2F$Ayjh z_ZVwGQ2fmgSnlYj;kBQReq(8d#^7Ri`(c*E9a zP`_L}nnis;tE+XNZ>q1Xr8Rs0k#k~<-D9(hu}T*o^0(5<4U_R$6pdeGRoS?!Y?3&uf-j0b^oUi0}`A@3Z_H>eHUUH__SM=T4o0=wmB39ch_py1h0pbRG*D37ML0hAGT z7?#`9$9I3VB=ZA2{#IHrp4_LaHixqf#dtK(^TtEBtr<1|P}TC^1`)A5#lmQsElQmZ zbi4Dc+SZzSC{P{CX^|0*01)%RPK*3U(J9F9j3QHj7J>nm{q1&7h1H~8O9Heu&kz93 z>?J_i%vPtx9WDSh!Wdt^*i$_L8w{}9Mr7^&v0x7W_q%&saYplh<*j;17~qsCM;Kq8 zO4P|eB?D~aw+gjQ?vw%UeRwXK)WvIhLYi%|?X7 za&LDInn+|DDN2=Dm_q9MzeZ77@ZP^V>wX7%-!;ilmrB>Y6*k(FY2?clFD@uGyEQc+ z-{a#AfR2Ku*})5GJ1WVI4<3AW#r8VQ-UvzCln^5|@R;mrK&0x*CR_uSv=+~Ked{JwvpV_$37~-)j96|L-i$EO- z9jPLo$ZVfEi>?Q{P0MDJ_PV1c)`>x9R9iIXT~U>GA3KZ2_?}!*c>EeQIJS8s#RSE- zRd0jLF(Fwvif|1`8KPq~oY^(IwePF1t7}@=VO=}<+qSEa0YQu2pFp*R@vFJ5Hnl_>pK*mo1%HQ&-&MZ%@e(j zz;M7zb#`Vl2hmeP;4t<>#F_>8uNfS)Wn-c|-{aSyszq))Lb#^;b7|WclZr^C^M9QV zXx=BK_S*RKc|5;n*;K0L?Hk%q-tV0(bnpFnub{2X>X1Fb*o6dEqUGaVe{?lwbwcX6sdW?|eYzyH?HNvt5Fw?>)Da zhlfbvCOt91uZqu5ed8AistS#l_i?Cn$jPcbRD1xKRSYFW%I%V6Hk zlTS0Obsl$My&JM2L!lH9X!nc&&1I_}YhrSl)~Ae}SIKqM&t|DABSi*vo>()$o2G2~ zway^0T+PIyjCItCD^W>}GzQVX;_IsxAUkB45D0wd6Ef{eS^%mH8pm!K&IS?&W4o!v zOyDk`Q18NT0bb;QWf|*J;x4OM4h_6^ggGKB*m+PxP(~^+QB~j) zQNFhW^J)#esI)&%r`!eApB3c(>dGWEPeyaQ%O0XjO8`i;&4r_UPL7sku~Y!rp%_|D z4`x29sxkC64GYF!Ke&Hj%g`1RW4)->H}xw9yR(%pk%JpRuotQB&ja1My3wjqdY3KC zJt%+4*Ib(_^2UmnYaKN#c&os%B4WHArXx~8K_29{OjrZtU&K^Rbx_mU>7v{w$`rtN z$M%~w_oQ$0^b~pxn}lBLoeWktZ7#cU3{bO%Xp$->n`*jTNFVA_!H>tELue++*VWE( z4fo&|`HkyRKK;05457ajDN7N_VI57K38`|0d+5K*9BwAKb&K$-j1&_*R!Wg}6(c-$ z#c#PyCzq=(ck(g7=GC>2wS0X6@oLz83&TC(Ko+ZUdoyRe?d%Ey`J*oeK9eH2c-i;2 zI~87YUSr3oJ7zt8rBRczW|7ipIL+Rqr|2s< z&EZ}zRil}jSxKok)njOl0u?kx@aHM0c;JM+^()N*MVKvZZA|2togt_1(d(HHgg2Fk z46XA=qB4BPOR^Z~GS9PjEa6fUj{t(|oUj9IgXI2@A>|EjGK{+${7#y-B|p#FRn;wy z2Yiy@ya%XdQsh}Q20PoEauphK@&s3w?@VNel_hwvcfLbgak!56MK?TSq>2`pH{Z6s zWX(h={N3|iQm!IdwM3&*Y~f&uhZ^hQ{R759=oD9>F7Mh2dceJyopQftdg~nkv=BUG za6Qc|#%kwEHj?WHv-Y0;C5zWV1eP{0>01l+?@DCT?q3Cd`}r zo3Yi)9=oW;L5#6kozVF4P;C_&!h^y*x#m;(g=-;(??O)zFJa>oGcC}BCtw_i3TKBC zc%??>u5a#>twkR^RI+%T6~HZjbd)&oUQnq-o0rO`fp?Td(6OiEa*Z|cb-fIzv7;(M zB_$;Ivr(Pkb9nQ5O1QLS**f%NI&QS5tDt%r^d*~itiVY2@p||Z{_$m1xyB?)_nDB< zPbq`T^tG{&;K(zLVuDgGROy@wMEx7h5HGrr+GCCf*Q z7oCP>?y8&5Cs#DN3~W6ZabWTaF=A&gv6*-ctMbUeHBk4)lE3Y(t-wL+W1P|ipWusP{PCWD9V4u4!1(!05WS)kaB&c3~M&e zISxtR;TyB&BLsoUaV2EB)%E4xLy0D~cOcWVh{3dOxw_$1?^u+ZSwe+a_2VZBDpDr> z{y{d|mls|YIo)L$C=>UH9sCJWsSU^P)q0mS|6HTC=g50X)t^o~9hh9`RC7n%%7c$uxM^@@cRm6l zsdnbh^jh4?TPC@xtjJze@Qn1|M@n}>r;8v4sw*38H^YoK;eWbLv6EKxd_EPNl-hT5 z<>4ZlCI!8?!LbDOr9Y(INucp1j<7yzmh!>BXcd+9>b5At{P;C)ILlk2TiI=J(5Yxa zi79*#5fT>)&4p&s(bw+eor^UKp))B$N#$ADHqw zV9kPc1lAuVr!o$|j&YrVc&OMw%e?7Pp6E&bd%N0VV)C5e*%;BO4d>^bod(z1q5Ep)Z_cqY)-N{$5km zBLirOYESernVLjs&ey4f6c1s=R!B)aN6EWlXB9q$O6Nou>)2#I?)S2|kLD&KODdmX z5pOIpYV#Xx!^X^TAnuJG%jzI&q+JtgV`eLgff_XjUo4K9bO6ghWJ9LPD@I@{eS|wBv@+P%Mqu%e3{B#e&4`%7S2<;1&;3rH0W@MM z-v&yaC`zga+pT{EIB$dh<$t`beOTgST})>y_Ss*TJf|HzHP+1ev^l0F%~$;N#dyA} z-YOvid>2s*{`yx;f~G3G@4w3p=TWwyhFb;d8a$6%)XP6u^6oKsNGw1`ABW5@H+V$= zOBxagstAs1)h5+s5TEXJI!ez}=C0J?15@39!k0c6EfbybBWCf&YBnvCnwc|35}ZD7 z+3Mt--Mb`E8up7a#$waht@Qxka?VMDd6Q+lFQ?8Cu|!R=>1WyHA6qm^0N1#%Rzli{ z#bX`dzYnUOTIlRIzFI9{H~#eKuI?SSNDKt3r}hGvfcfC}`VXfK^Nb9oRx4I?+1{tZ zGAYzA{edd=gX)G?Awox^<9{jEH>Ha;A_hpAF1fEmGu1W3Z1uhU^CUY;Nj?HynD<1J^HN|xloFyqQ1V7E-MD~Xs-46UyEfh3>ZMnx~RL%F|@~boV zDShao7k|6mB8w$!l^QkQ=)f??-E zb7Y~3M^T@^6NOAL>9W_v8Hz#QP16h$L$ZYq*-KlstNpMq#5Lja~kFImuH- ze;^CJ8%=7*3efs?fAZlHz^actn{(|a_W<9{km}hOcTVkD>qv(G^f*9r#=Qyr4jKX; z`sI5G=JV~1{psjjX)zO}R114##eYLvVFfAwZ?RSt**=bvq@<>rPS;zJ{~(^vGhgSn z6c=f(zdivt)Zdy^n3)vIbFk+lvz)X>>wdf9gyt;PW_CPBOQVd16(K;g#g6b1}RUpE6$7t z`Y|!+*%%-I;l(mj?@mN!n~A|5UPWqm28Zfrpfaor3^n0|1U?hM8k~lzunGBPx^*eZ zS|^vmNC{$v(BR_fD-uRK#^Vy5F{Dl_@~TTx{-E_*nMCmy7ak|BBHNn|@i0xE$k)#c zP3!Ectghl#MB-?o;QNywv+HCy!8mmp?W(k3)baE{4 zp_p8Lh%y6yEHjYNQ6pVeTF2l|3ZtgC@}*dD?WckN%bg#f&$MPZC7szJ=ox-sf$w0G zgL!FMS(jO5joUdfEbas4{gQ}p4;`w4mFl_g%jh`Cxtw{P2Tnx>*8@A{@X`3ve(~a4 z4uffkjkm+2U4gn4JRfxM3tvB$M^|GCPu`|~v^bPvQu|~IcYZ~G#(dA+=EsQH;0i1P zJFz%d3}Jl6Gl&#VS+J}=Mge|!$k5m@k}YReLSQ25r%!M_*$Z4^5qZXBe!UF9HpGlX z#J1#A{cCHh7JpR8kzkKGOlk861|04VcG1}^{1WBacnarToWmb=lU$tkMDcS*f9Vd$ zy40KVBOzJsNk)aJYacv1<*dt1el26(`eHT7>QCyqS3u%`iTw{!D5wYkR05k(o^GN& zT_CV7(sV|GuQamw0t5g1dNJsGfDM6dZwUbI3eelfJ%KJJTJ0CGIgaOS6R4Z{cY;wB z&JyBf2Ko-T0<7%FhgZB@vUG=o52d3jOgcEe17NW=Yt|czrA3d8wUYl+g8T)h@8qfm2q!Y^7zDkwz%Z?V_~+Vle@&2dmmBdrdo5lG9zbN6ZCAbk9|9OmDZv z=akao)Q%{*<1rU@6L14>WVAh%l;u*(7k9Me z9o)XlvdK3%t9|}5R&Az$#uf_X(>)~EJ!|Nj=>(#^l&la(zcUu&>MNUmnex5&d^gDp?p}s~qH~F> zeepZcMVJ(T+3zgr|6c|p?dT0aaw8T9-tts<{4cRt5ntEdMF1r70(1@T*w#loun};S z#`di3_6GO=2rya!5G}@oq^#|J;Dnb89i^+l0T^lj1{Uo^e1+XJm*Cm}4#3II3f(dX z4jBtw#UmTqE8PBx3)^Y^wbXTnV#5ee13;}_WW!d`cnD1gIKHelWOFSWfSCSm5G<1& zJ31yK#0}^c#&{M0lA?Bl)r5CQ7`CG-IVkW$Wo%zD@?wGQ6~@?J-~aXmJT^OD>;Y5J ze0JMwlkq)coIBr!F=D9yixyc0Zrlrfym5-L^#51tnE#W$f)+twscTQHVOH(pX%8+r zo(6(e3{v3Hj(1SqW?F$bAjFas$=3AdEy6jZjgn;T5Q_=yB@-Y65xw)I*U#G@g*L@F zwye*1@R<81jd5K=To_y8;lXgXrkhRp94t|2CuMjsqL+`zW~v4{)t|7`&Z;RVwhA*=G?S0!e1G2i#An8oxURF#`y;pJf_rLl)A8HUY;fB6KLKa&>Kv6 z5uF$*n6>^}r8?iz`uk*btGA1CD5V>4*I(SIV8ZfZjkNiX2^r>D^p$M40?>C53$sMr ztuS_n*dj&CBWl7by`IN7kF%9f{<=&$5vxG7P0vD7#$w0Z{I3l>$Gj3FUN8)@0n~^c zTfJ3ae_Ascfmv9+#vd;$Na=UO82@0sa>xoH>g3<-TL3}c@f2nS_0Wo*? zY-41JY>hVJlg`|7SSSQ!tG$2ULLZ?S7zPgeluo?EAuL~-3LQ|eqRn+%4o$!n|IDl# zF=YNkM!jtqidC%oSMSK{B1}SPh3@bk-%)nqK<%!<)B4^P($3SwOYR7{Vatc7>}7m9OYjPoj?#{|;2dNntw` z_ylPqGC#s8R6$hzFfu+{IuXkHgY>Ku zR-s`(;~~>%aS{a+p2EiA8SoLDYq8cH8%G&fv|p^=3|-wGOJ6iomv&`0(<+OyG>+!p z8cO`(&DklAQrD$`havS^i%RBh-ekM%XH>iFzvefSE4Qk@WC>vo0uG7?%`TP+!M1>E z*F>hjlkKC&R#Ucc$i(x=o(ZM#4H+s|>t<&#wi-BarlH#CYq0L7Hg|J0FWqSn$udxB zRhwUT*|z^O+y?SAa3u6lsbKV&A`+tXElZ`^YfHr|Pjkr`8rs}rF`%kuqlnuk1}907 zb!KY2M%{W2Z-wEB>hZ~pjcja=frM4MyTAu9!~+MdD&SKF9l5CE3WrJ>o&3GSP=daR z!(>o`w=q%37bVqbB7|vZCxm#m-!Z0lS*oi^Hd^qGE)3%Qt(LzL;zZpnQ%AV!{7=bs zcUCyse_bz>B)Rs;Wsu5c)jMKt<_TueiqctagkImIXvaSE>Y3s8kr`TBx)FvZj1vYM zWKzOBCW+es#V8H)(x%UYTjAC zmu-jLiSrLI03y?cBOXF(1^(Hj2_rsHrS@53J4oVqDOzaq0#qUS!Ve&AL(o!5k5zZG ztRjJYQHG~ft(Fpy(;M>Lm-@R^i(!;4uF{#|TrQ4uR!;{p0iE${C?G5$D6;avfDKCl=-mU*A&!z|8OXHVgT&#A4WO{eB43SI=6XXpC-GGJ@Txf= z>x}EnZT}r*+hS|8YExP7A$+3F{P>ed6cm(A(YXe#vQ)zUbqJljd9|m8#7s@mG<752 zmFdcKOjRDkQ&zqOF0OaE(ig?$S(=X&DKLo#=o{ohbBAf7Gbkyjr;}E5f|$biZ&&GF z^_)N3pEByK)No@`e~DFkQ#**9O{UYlJhLv>IJZLFNYV?sNb5IFk&GB$!{{$RQ*qC#}P<$Ucnaj4SI)lOr{+~H3swgQ3k zdvVB*^@V5HhUoYz;seh*hO)l&U=L*EX^Jn2w7-}k?>oBSqF^mpC1?75AN-$8*3%8D zx~JD*b;6An!F}rv@`VO8XS`E0ul4#>JJYb(R!-d{04@xo#%vfL$^XgIuVOna{WIPS!6C!)KnlRY=clw)>F6O7D^d){d2s3-mj=7Et%IYZ~IHGQZLp`z|MgX?hKR#lP5U zR^+eh-xPp@&vY#ub$Vq{t|PR>Y&m2GMMB_-O%1H(r$zsX z1yEe?vwP}3Qg1JD7pLEbtWvAH>{y$nTc|T zN@V-=4Z(9ed{8lv+4ZUES@sbdUMW@ohGQWQ!h(NA(2fLN-Q=bu>s!WZs6+U`U+b%G z{)HE1WQcmgoe~?R3VPg(HH8y3YuyVuo+oyZx1L=s@DY->6kDQOjktGlk>iHxnGxCN zmX53sZ6Vyovx*_4v($iw#hQR|>e5iP6=9qRM6eq8;@HOMH?r(z#Z>m&__R^>jo&}J zKzYnaz`fa70v<3JkCjiOZ6okkt^^XL|8n`xkSdXhm9|-n8w`G4oIBa=U?<{KcdYdC z?EIb7(T%b*bP#O}O!dL#s?@M``b#R5U3MG9#$;-6fMuj6t0OnJZAhP6n&ZI1T`@#- z16Al2aHQB$77DjN>GVbS%QDL=6WfE+sU@dWm`A z3$9D4Aqv->OBx$x!g8q6oaz{`>}b#ZC!n!v`i-Uf!`9mV(+66+P-oF}d8%Y~Nl!ig zXS{r_wOWZ3$1tuv4cTM3-AQ=*(|0BHWV)7Ad*tofkBzSlwin4%o7()^%eEWIH419D z0HX9`xw;GHyw)$eUQ(&P+gA^4HCdiL)~`l(A~a`B)m#mvPJPV~upA;$Coe}?K|~d7 zcK9#fL&y#X*oCz&Hpz*)#dq7x3UixVz~=?*%=d8}_w@-Q-)c%JqH(RQhGdT)w}W)M zdfTkeUiQ{|GrUg~%nqawOH4+KdP~D9U8thap+SpH+f@$1!0sR0<6{D~dA*YJLfbic zC-$Fx@%hcEcP-cQQh{m_g>d*g5oZI7moI>V5jRSXt*GBS%k}WM0l%B}Q&d%_3KiXS zou<-*^yI&raaA#4uq9)Wl`B1AwhEgnON3;?%{x+cS5^J>^UX2pg&)NiBgMj*TpM*r zn}dg3{GJ!wq@;}UI)MV0s$TgEs^YcsL&BNhDnDastTEP?B64w`Y$wWEXF~lRD4%_` zEu=%?cyW|3D(kuYyU-`6p+O-8S;!l7%a5AzU&*k=PAx)PseH@a$c&SH3vrPzc?$#e z#b^0AIk8AcmN?W~6g92}GEXSDr9zVMeB3JAEG){!$IdKN(N-uY7;5&*gLaS)WUv_Y z*+38g_`U~mDMYI8GouApQ~rno$M3v6ajcQ$vh75jS2_K9>tR3`@X8w3tc)Mp{3cPj zvGL`2U9sab!Mb}TT|GFRX(4-3 zWIkXGE|sxyfTe`!cynxYh#vo2l^ASHP6@A_BuyTDaVx$l_!~3}B)71(84bP+N4LBR z3A9BBVr=PWJCeswzf$&#=Wm1xg+d}IVYvq?_DdLhF35FdWq5n#{qfRy8Q?@N`$#&4 zt2)ov^6#!rG7mT-b4_5mCLT58q4yr}NhizLliYf8*7lb)UgZ}AqNTcZzoFu%i6BbC z-B7elWfALv63CbRC0f|O#yz?YoJc7wWb}cEFL}azA69VAef_lWhl~QaX(GC8y&Gv% zRo!QZF6_qNJJ~VzhZar~O9&?3AjA5@xArj)kb1jQoM3k) zO>fTIFu|Q#O!=7>Hln&@p&@$q)jxDr40MS)^S6y>^l_w>*+xD$%SDb3SzRd~(k4;K z5T7F`9fndQc2+c!F9`prxR%u;Td9mv^V{KdFTvFVOba3S6oiUKUBz6_)qMj6lUE*t z_@F@=$~gTXXl7i0vU}SUU6ArSg-~ksi7%@E`EcuxhZ8LiVT&cPOY5_4 zEAH1!dmAm>qURc}TXs&yfdJ1G|1XXscDc)sS`J+&xe!og{bunz4Ll_kW9`l}X4hpY z_q>ZM=IT_lKk07V2edxol1`PfOa$?ndK^Z_{ildk?YHYamuI;-EC^j?hwUOn9_x&Z z)-_xVLy6A!!-`Ds^K0&uPpC99n5wQ)_RS9=vz|TOg+d?@zLSWuDvSIpo~_>Z6XqSN zK-e-F=#t|dK0Kl({;$yWFctri+t)dw1skDFh1MOSX0R>-_8TIHr9{8gzU}9g)wS3D zh@7=mWdWF!DgL9)2%0s-))FaCe7%J^{YugA5pOc(hH%4Rp`~)#OHR{ZB7M^CCmR^= zP@S~e()rutSR+Eky0;LyMQvPsay4Fw8BmlwZ>wDnLyZ#?{p7JO!HP$)^J8LDb*h$@ zp(quJ>ISV*Ka@-l3MwXvOVByhPa2+0197eOe`SKZvL`9M%;Ka&tSasjb<90MY5nHQ zcj`StL%kqGN@N|LuYGNl4XQ?kF%Ot>CS>hDjg6VnAH_r=ZpF%Q?yJbFVCj$x@c3w& zcIZGz9-Y`tQBnvaXGWDh3sno)@FKL(XPFj$Pl{{?{N&zW(%b6QVzF`y!W7=~xgOeO zktHAtaQ#vB$jIX@PM^I6n)7Go%!23EbH1D3?5V$ri<B=c?KjV>nm zTf^L8Lu94xTtje$uH;f6AVM@LpXXVue{?9J{TFK&AebX#T}99K4Lp`E>{#g-V$YH- zYPWJf_^xj3LvX*38Wl)WvS0x>z-*MM{4`k7nWcruj5=%+*3t5$uV#mwb#aswrPj%m zB0h?SH{f_N#%2{5r4FMcMfXO%u@G%PRXx;%Gk)5&0e!1BkJk2drSx-mRK6o6N^$B4 zbu*DQB)ff?c_5-yXO{F)>X4lSx<-cb&M7aG1v?Zd+V$z=DdEIxX{?cR(XzAAaz~pY zBM+;o=A`2(yLSui-FU0g3S!L z0dsezJ6P;&0QqxQ{I;k1q<12Ao2l=!x!H@gv^jS;Ack(v88|sPDJnILEeB5qe|xO| zEP3Q*z(0Zt*z_*YbWodO&waJd6>EzW8$Fzy(RdKw^zqUWl|-2|r2KnLr}q!swv`PJ zXI`QO?i*;*{8ZJm9KQDA5Rm+wP*m-=&;O}Ik!Nr<^fj5-J$i4ZTPjSY*2}$~3t3%P zKQ@rPuI(6HN>-qHelGVNsKx56-_+%Rh;bq_dX)Q6 z0lz62Ojkr|hv#GDPrVJ#2$mUx(JjR2q znCiO%uToK8ZcFpL!xwd&+3W>>R1LaRBOU-vHlLXHH zZgo$$&&~ZhnwziVR8v)xsd( z1ljD6p*sAxw0`6KY~0`vgZ~hgJ#2Qu_hs1mD%;nee}=u&t5_-^t|p`j2Er-m>QqU* zsdOH3crxRx2xFJ<1cp5b^+-~-{~!tq1( z$mHF0L{T);quG(VzK*+XJ@MC19acM4IRcqN4n_ybb8#=(-&6PlquCmJv-~mxyEti`2^Y^E%X~iptc+GwbZ6Z ziZ^({RthX_wrTmNo;rK}uXm=eE5zgW^wZ9#tRT>_eZa0GD%>ezl@NFGNga#s)eK0w z@Zs(b3l)Jm?$1nF_9LLicPbYHCWa@p=-1&95a^u9qCNJ1$gz-jf-we>bDMY#x$V8<4v0 zdn5xup40hvhCRXys0nzy(tDa)wDHK$u{j89Nk#1!?OB6^V@8K)Yti0I+t<#8-H}*} zYrM3yIXvf{Nm>0d&8R*YD!0=@yNDH~6;_n{3~{+<^a^F+k43YWu~dDU|K=7?3HI?v z91i!>VJXm$x{-57jn&(+@tj=H-HB5{*F1cDlZ>PTvhfDi=Ocl9nzi2t?U>TvUu4=X zLeRLV0Xj1A`Pg2Lzu~R_9apQgU^;DlTVyZRM>L#4y0d#%`Og_w4De<~ZM)F6Aw-~d zH|szZMWQz zM=a3dk97c90`8IHLpPhO<0gQ(oIJgYrw+fkmA(6G-R}jXW8j?bWT1=4rtA&6>VMp1 zJkwq&0rau^Id+nv6zzOo9@?8)5a>HF^u2E%0u#IU=hc7S3>5ZZZ{9(m8`=O_b{{w+ z@_(BJ{ntrQxKt6D`yPm<7zPrQ+WCQs%SLAtfRGo>Y1Olco;T}HxgkVfb{W519QXNmJgt$T_V*>0wm0z zxyxGej*K;|9PUe-TK_FyWV?{Z?!d0DfyhpoX|1QPJlZ8$BjgUSQdT#YKv*0EIc{pf zQnFqV@)$cGv)7t9W7&Q;P-^gB9kG;-f22G61Uv*QFlxJ$ApQ9Cq1>_qqm_~M_8DYs zFPdhd{>sP1v3e@yPIZqHOFyOWk<|QfgHGF(UaA3TYemgR}xVj(Wc& zWdrr@@cL_mNFc%fl4X_YaQ;}Yc9Vwriy*({AtR0JB$N12LI|Il`y#!FG)Q^PTT8fX z;Py1BrqIQ_skos7YDlZUC{wur1CYZycX8`j`J<&S4Y^NiCHl;VvG*Q79)rlDgu9?c zBh!gGqUj`4quj-eZLM9#ALnF}2s`-@JOb^fySX3#nR`TRd>l5-qF)giUu}FPxmzsF z{D9xwLrcS+U+|ECtKj|sw{G?hHlD$|k}Ksss(s@jtv38>79bib_{0d=Q1&p4npI9^00mfzOKwPHBsfoxHum59;Bvhyr9!+$YKq%U zCxy{{oX8;OL0)1hO`oWV-qVD~)_^iF>E~b%tv?9)H6*+=qDz&lW7}2G?eOSo(N(ljFKy zKZqhuyD(5b{k&?h6iV!nL>qIFWu!Uf)6-GuS-FQ@{PP04q12d?-g~fxLfW}7tIiW$ zrpmwJDkfcrbjvt(AJ5h954UX?fIXTaqS{|N&5t_CJN_iPqGdW{bK3nUVs5GI)N2GdPdsPvQxD@Q0#Q%>)SWe0x)oDL9g5W_{=K{A3>Clk6L*e1qVdK z2?2)J-P*2{S~7jUhJDoX#{O^r2g$8%>7dKxV8HF9 zf*8RLJ0;xUIBW-y;5Fb&3O@1J%A8;6OaZDzolH}AD(+440)oBIPQVly$ znukaF{=fo*_BU@Yy2x}?1OhrRV|vpMGDJ|Tl&g=N(a^|8=i0v`I;>PTh@-JvH?Teh?!}*jT&gY6qPN+OHxeNW zqf86%w?~8;LvCi>gB|O?61bIJgcI%Htm-&-8b*FPnZ{%}aNQd>Kh5!h34bWLtHQBk zinUJbgoIdrWZ6xV8Z$$WdAEdO`%8d0XOKXR4G>d(leJ@WUMhwS?8z6*RYtjwh`q0k zxjX5kf)Nxlc8;%Xs&w$2W3NAY$c{s?V!K8AK`usIrag*f3|Qo)_}`5Sbla~@!1BMP<~p=RC+a-g`BAz{E}4G;^v}s zEzy-6)48Migk?nn3dr0tySf{D21v}e_PSvoc1g8tOJTz+&#|an;C=PiQJ0*82*47L zLmy_Y#Gc5&j>uZ6A1H~U-2XTQ?|$Cd;zLK3C3QY8``oC#64{dC6-t*O6!PYsylC2z zS=4%}v8H4Hg~t_G8dgxA>*m~9Cisli0YpZvI6M<&e6NIMhb=d+0ZZsXkMu<6drUAN zSkap~-yD|`?0nH7$;jCkgGU)xTX&jVGq~eB+UQzc`0niaz*>K|hJ5!)n-}H3BcD^8 zvgT~RM#Q=Ka>uvxX_(|{=z2`blP%(<4I^LsKzjgTw{hWvN}VnL_`Qp^h{92P;V{(b zm$Poo-$RzIun7gjAfudN!%AbrOO*$0V||?6x(3&n&3?=$r+57{a7PcLN>>i4Ybc_q zLlwmQ)r_grZ0Z_g9ywpWh;?ru2fZwt=CsR}_0SsktG~Ng4_G-A3_5`)7~W|>Mhh{N z&ABhM7&{l96&1uc?Z6RF^{=RRmx{hD^Vg`j7kpR_D1NKwPWrMbkTqx7tR7dKGE_NN zj1yVmB-3K`;)|6n&n^U7Sw5cuXQjMbALtuc*Iw+j1o$5Ktd%iDmQGpr z5Gq=Tw?Q2HvoTV;7;|@@LWpvBGby`2Eg^sX@1#YME5;7ximRUkV9KS#6z`70? z!z%pUYSl*7PL04qqE>$#rDy}28ZRo^r5Ld>DRB3T6E8=g3r4I+c4)rg#gjhMFD<-& zd__hsE@UpZ%9_0GUXEISY&d(HZ3a#UvshsSGwOdrB~NEUv?kr%p2kIG0|7a}XW~0U zBihoYn8h0CHybZACRxU`Oa81-b5+j+2+q4;iQC#6N@8o45S>QRyL zF%E}L+#ypQ805SB7r_n|ZKom{HJH(3_Vti~~E?Hnip3Q+*&tWl^-5Mq33`u%-jt~A@+vUcuvmsRQ^TVZFQI`o&X z#)`Hkv@qdEJ9_q+J@exKL?v9|enSYox3x9ue`=LH=(Y`|=l=FA2;H0%+F0D)@}T!@ zZ*u$(?3aIRB(m)`*>+=mn{G=7eq^~0{}*d0X~!B$d63Tt&pSI@K?Z70Xl-v%8_0N^ z6dfoX_b;XsU>%Jk0IpED)bv{~$zcwN3rybHqIu8(3+}pK;tqvD%natfL=PFySl8UG z+rcO~1}M$Bp<%8fpk_)l5PoR3f2Ow1~8Z7?W@mC6d&Tn7I$94)b_%aDOmkIG}^5cEEFz0?_-f(AK!`g~* z>C`n6wZH2h%loj7vtc!cc6AlDtv7m89lOg+i{DM0;yDo8h(QOVCe=E(%+I69Uz9SJ z2BrLc1WC>8#i+xx~W?200LM@_n{@Rjhke+5n#YWB{-2fog{@ znjpB)efb@#$Bo9jxObs{_|Z#c2370MieDSa`-cTU>4~dQP`c?FKS>(h_z_O@d#A>2 zypHHl+Zi|6BhH1z>>vvmKobJh+c*9+nL+&@F_r}-Hk2O25KTAeor*<^k}=Z=Q0_v7V@reoY}?^OCnfHbtFY9l~kHf1^R}z$nRk#iO*KqMi5d6XZ(Tz=zIAzbS(!RweE-=|;&Dap#g9%}Z z9ja-$m{R6)4jnyID5%L}@(+fPH;F(&zGG|%JiK34L=$-7AP0vIl+>JZAexSk&#SO^qXiU5%JjvY_z}BQZbe(9e=m}u*Q+`)Tf{)#5JY_Icw!nArUEfjhW##}I8pfBQBHXmVwE z+OcLC&f0#sT@(HIWsKBQf#|3>yZM{pG`E4K-x7xrC$Ai5*ba+($0WuF*Vc%kiFIdH z$z!_sqH==wohWW?|3J-9xbiS3`fkg>L2=o9uI_QapKFzmJ~oWdJ0Bq^7K6cbcK~Mn zBkndx57Oe6cxUF4WlM8*aVA~rYLUM2&4Fp0F<-!G0%qjXpD!PeY)R@n5;~!R)tUH2 z@9m;+=D;tVEn((ndhOR~Jg^TsYR}*m+s*x`>I0sk{_(o-$YP+(oyc^PPtJw77kBvP z+)y9Ys+IiJo3QfXU<*A}Q7YOLZgbby3}%p0=j+k!;owf!&ZW45*IvlW=e_rzt-k#| zGCcLy03>T!4^hksr%l&5Ft36;O>se*e$TY1*oQYo9b8iDD`t7e-&I!JLPN?*7}+N5_Yy{<&8wFtt4cH*Z$ucQE^i-? ztKm@QeRLoFuq&oBcR{hr zy3Zxi!MF9pjfuJ2fJ|bLw2R;8d}2@8a^U5ir`tylyhS9_<$3Eb(4#|vOm8SioAWXV zbVNJ1N4oKtyFR1rNSpP!Iy!B!IuBK63UXaK=;?(<{s8e_4w!0J7+K}bob~+jY<2N& z9;`h2hlhNfQ3xmGuy>3p9GX5a&Le(nvud0ULHqhTKjaR*Z8H9fyPf6`v1fYGuDR#J z3wfAI%WBr;ql0JaBmrakwm`CZI9Q=13;rUrcKhfnFNj+A`n!y(!;7Z-sFZ>;ErR@kozip3Eg1k_4$}5H`lG*@o1&~tF@gax_0SzRLCTf|MBDP5 zkarP19~Jq2IV9czG~%9^<%3K+@u{wI$O4;<`_>JJ6q`9hTH2K6n-kK}Pv9o))stb? zx;r>^Ys1)6H$+FJXHei-9ziCLT-e=N*dMvjwV_5EGN`LuZKpoO zr-QV@;*&gy_%y3rvgptYf6t#{;F+n6TaxfUQ!%lxy4R$eR?FN<*-kK$dn8z$pSulXZbtdc=Vj&P`f$#q z6ZYsJYh6YtbCo_d&Z^yNd#3SzhRoM@8e~X$9%U`oewDT2?xQp}{J>Fy!+1&|9WIbZ~+}EscG*IUKuZ(7;($AzyjDPaUg5-Eyl) zaL*IQbswN4lV~RzuNSBoA1-23J#nc%K%D*02eT-su)AKl(Xo1;Uu7klO->!8hbvOT zVCdP^@VCp~E~5PXaDMcm6s>|%k$6~@j{|<%w+dasb)h^_JwmM%Ru|#v1bkgd{ffS} ztCZWw!|hRS%2z`jV}w?^*KBAo%CcJ`q1au36J5{u zf#g@~XO?9D;Q0a7@|u^ITJ|`Z)4UI7s{m)Z))$BaoWM_aXD{)OQc15ZNTus8uAB8k z`Kh4XhNN^tGH#I&!3@mBfCH*dR#X)8yVb3*1SYyrc9z$mSL9q5oA?tB<> z(|NJM@GM7)s1ZKRei>V)#~4Z^T$vMNNXA1zdIRQutJk*MyxKiR3+`$D&`w*71|yTD znoUaAZ|Q`pWT!jR=s!M7={HD!Jw@OC)^^|E93l6&Y%+`dXCudT#E;2*)7!x#=+d?@R|VHUu2trW*O0iIcz z;MR^SZr-=Qvi}TkORj3cBznDfkJI&cbSXT!Xk-1{Ge@*dk5@&>OltVJBi|gK+`zC4 z&r`ga?fMQMcu2^@`YRw+0A>oY*(Tr5Yy>~8a5k4EPkM2D6;}4Dyc3B2$(CSev%Exu z^wlfSkRTC_?2&g>4QC&u<{y~OUyeew8oHyVJD2mb9>_{_^*=IU2QS1RW8Y)@8t+}R z<^CaLpKdtMd7!%AvkIXZ?8q#0Pago6ul8&mT9~i=IP|M0A!@$%%uSFqo}KSZ`_kM# zY>;~;@yGU0uMa()YNZDbjZMue`T8TbB9BhuUFuc~?L|aHD8VA5FM}@s2&rz7|67>0 z^c`U;QVOe#hq7y8#Hzpazv)pBKcH}|BU(tiPP#UlDLvz|v=Q=i%?d2Z#|rQM(Cf}% zB6ob!SngT~Ux4?C<|(NU@T{ZFY+wt_RCc+)JRQ>Io4)^t>LO{nm(hO6Q(b|&-_zRm zC5ZP$NPR#io~8!%J>lbSgz@1UIzxk@gCv0Yoi#8rbeItiKt;`LIbmPg8I421%BQ3J8WwzkR%ZE6C!bX3F% z|7ygsA_8;M2U6k|*d^d5f41xB-jTGKb!XO7l*WMUdc8aaKU5?1rPUG!X*wMyn;*{i zf}Io5UcSh)TrBZ!FMJQ2F(5e76QgD~{RK-On?%V!idAz8fbjXwS1=~SCaretX zapp)Gk4>&OTAmRDSQ+(P&YQ+l{-&_|r*F}q(&Dj4-$iO&Suo_CWIDYx(&PJ@{RH!< zR{PIz@06wQbxga~*<_O!4N4@%(LckLCHpX};~ zr^CkAwE>SEZxKHtpSB5*Qu!SUInZw5iEI^ga&%-|7E9P7`H`ViL(ZxC1F%#FU&^5- zV&^JK1gSQJA+BdPRmz+qMf5)2yj*V+d-VGlh}u(1BKeWM=bO2r{HaWEoFVJQm%Fgf z<_0UMX2y}`wZ2$x|16B^;7Jpb+tPK2o5eJ zq<-}DEaEADy}Ia0VDW*&{A6erke0_d&s2jj+&`60)J%6Fm@tZHE4R8I;A3xKIY)JG zpj$;XX{SlmUe5a{vhopCgxK4=lbP3m`!d5i28pN3yj&S-d5(?hrHFWz5Hnq>GF^x> zaol{ZTPnUEr$+K8`BRSpXOj&_A(RJA#%?yRq>6}o8W?}`)?7I|D)u~pwRdyIbZ+_f zU7JmlZhXX-#dp&9lSSRUYlzMQ$V7BoFCZCGczB_#;nGO>27mX2G2mx(i~3z&e(99% zH}%kbr05op_NhN*ht|g~kbX^|p$P^GI$V4YTPUO=R6J0umXgLv-G@*wYypPjK!$Hu z0c)c$!FPhh|FP}Igy;N_U>fJ;$Yjj56;94IhJ75P%45VGVL6V?9n(zA+JR`*t115h z(ehyYTEn;>Le&U#32RlR;e9XiqN|Aijz+9Lh*Q& z0M>Jo?M#2ALeq&9+f#RCmPI@RDQw<#SfZ@o?iFvTM$ z%|T4CL}luFb|MV>Gr)Xt)h!h6yuBX~R~}0w*nru-HFessZzZ(>*14S2p!I9hK=53Q z9}@ZtYdY6#KF(MK?bv-aLit0Wqa3h4j){!(+ZSp zm#w)UfNMb1xTa8<6yf{^AL21oyYhhVdY2wTK_XcpbM_vg3lq_%@|o;Cch3w-wt^|2 z8AF{2PuK0FzN)qU@$(-X7He_yP3=G&S4TXz>2n63P7lVvUX5HBX!(FAUdSnqT1?N)G zeIJLfXZ!lX>CpH`C#oMm$a>Qp0+5Q{X^v^}YxIZR+yHb?G?DKy7k@~H0^uQf-=s?C zkO7pGrB%x@;Zn_W1o-|SR^8?ZbB>s8Xn(3G-Z^SbQ@ia&;o?%C0>bu+K|0#<6q zgrJlI3#ab$4Fs+*{n4rQst%Lv%Rh(vF}5uLwxt+YwMx*xIi^E4@2m6jJ z>De^!i?*`0ly?iAuQWXF$aESJL5?})&pZ-8m+gSJ0nFmns#sNZTr{j3!H>?yvweJ$ zj_xqGWtC(Tr@x5Yb)9g+k*VlCPG?HY>Py^%n|k71ji3ODH(^@8gA67FpK$;w0)sg( zBMjX$Ov6-N$5tOO)#7|+eemZ;ElL|8OAMUP^dIaN_Fr(}0>l;+oVAOu(vv??0e%&B z=`^q*#9f)ZFsrmq!+Nb}u-y%*B>75P+R&j^y%yDId!LpJfq*Q@W*jlVz^){Ir9DJ!mp#b)dp3<5I{Rf?K8 zJ-m|Iv$GuO2p7LG7ekI3MFjTuGoc@Xqt=n3I~Z_7zLw8zQ-TChIt^9fBwMw418!J~ zZrgCSIUe_yGr`D8H=EV9rQFw3?^S-B-jfZ~cKI|+>21wyLK zVQ#Qa4NO=_UAg%}<*Q=GVx?&r_Er-V`is39fFI)yB;M~*AbifL&Bw%O@7iQKDL+j!3 zK22Ail8jfUeB4+4L`%YO)F@7SU@W8T(zr`jegulq7y!$9hOS82(r$8dlZHyt;Zrf& z^8^fRVob3O@gw4@w1+La;@Y;%6@xT-(IK<-D)|6nrenC?zrnz?NjWt6U_BcO7WEf# zR8KksOXd;s(T%auuN4a@caO*PZyA zCBKG&6qS9>iK|kB&Sp)Pj}FyDJ*`U163m_+;h5*=Mn}@bLC#W|=J*5q6;uw4sJ~A2 z@cjqDZJ3@y>E65N@sKwxJ39))B)Y=ETTr=vQ(JWZ?bvs5r3QPPD8-iJdlb$utOzOI z1xe%BnF9TP`twlrZW;Zn)uZ8%mcWnV-b&iof5nc3zrM-sk5@Lo+JFuq8fEbts+WBS zz}`ns8L=!WK@YW^DfjTofo=M2FZrIRFmRjhtrHTi9P)F5BZe`(L;t#61fc23-cO;H zri~Ja4e89r#-UA=O!=;P?oAW*gi)O~gUnN0uBK%?k*cbT4$e*cv27e+K5+wDWn!7p zZffA@?_Z0bO{+Q_Sf`0(meKr69i2~B^s~#1PQrQZ5vNn->_E)wvpWUyAqC~-MoqVC zgA=?dT*Jt3p~P~3AeYDUZgXl^Ca}=U;g;u*QVhmufNE9&d~SYjIHUrisi!K{NLk)U z{r#&u^YVi%0JDWaDlEg@1)_~!VGPB~tW)KNS9H+7d%f;;Ag+DSt4PCaB6K>Os;8sl z!`u9JvxQy|=_gtEwMf|?xLyaG%RuNfr0@%gv)TP-cU3Y@JhE-D!kYe;qeOyMK$cw= zr~_NawF75<^m9b{j$p0=uy1!LtiMDa!lXo>*c6%oSVqtI-FO`I@7Not=^%hhy!eN0 zv=ZW*L->+4a<86~5RDwC{QLLyJ{9q~`*GJZT)Sl80pn@^ehuuKvOdd=er>+Za?tYQ z<6Zunl>~n8KY#uI*nd5oW$sQPlfNm?yuCZ&OQ49zY{B-1{9h~$NLy%EVqv=dk^27$ zxUK?ZDGg@zKb>cU8TP*F?oqtDCW2&4l>i|d?+%KZxZpv|{AHPSBl zH!!@B4Rr2?#op@iPt@SQMqq-G7;sZBP<#{MH**27H0kKfUkvBfbJ_1%dVH^|0Aw_> zu_z*LUMBd9e(Li6W4^NkW&lh96g>lQ+JJF)7MPldkY+BuMv$t5a&Z_;` zz+DNg#O^@5GV7M&*2iCV-%5|zLhyKw*(e~>Cxi*q=54t8^`%N_OgzSA`-RS1onk@LCd;w*)?p+d^jleDAUfl*(zoH;oSluA}AhmD3!O^3Gzp z+GmaQ&y$kdNaX~Bxs^^loobehFAh~BH%bRc3)xp?8_v`xfT%LF^eUuTc7RpGoZ6UX z`ng*D@Oe?ss;zQSv6;%d4UqMexkCC&rCn-lWooD8FTI=nK7Xclk_|NNIfjG-GW`5b z?f24ptl?Ir*~%5u#i&fW!fGo;T|5QM^gV7S37@rAURIyG5p_hO)tYhIojVTg#( zerf>3=OGj>1AKp2*B+^J1~_<9RG*~J7Z7pcG}3=WQxY57*#ePeooXJIf?9ZEbtrRV z{mk`xp%W%%aujC$9w$P{eSm~GR@{nEZYnn^uPiy*-bQOlhu`-8RT#GIVR_i$hm)}Y;*fR*G z>0u*6C+p~+U3T?R{`Z{(fEr|XWmnHvur>~vs4D82WqkY1Aj7NSJ_MA@h|!O0@S~Q< zl~65ma1xy|Z#j$&BgBbAs{Dhy&q=_~R_t+d?Y-fe(m^z<-#3Z%vom1U+h@tf?0wLJs+Iit*WX$<)P(-;&5bg+&;AlVsAW9qpM4-y|zQ*I8 zFIXMVJr}mD4b*!8J|_X5E1~yY9(P7sP2~Ym)tx`$L_o2bA-uw6R{SVXi|R?Ns~*1- zLfEm%68AsM<;8I4_JoV!4Rn^jFd$LPaek-Hp^Hl46(ma3tmN`bK0Rg`Z>7`>*%+Ie zIsLD3M9l8fNTNHA*r(hjqDH24$i?~uE5z%_T5LFo**#_PIlyAT!wCMx0w{siIGlV; zLpvd(>b00!aS+(IcUX$y}j2 zyL=GJBg&Pm2U!;_nShXE^P(c2vFqffG*$>%=m}9yZn37MCN$p>o3S~NfNK{PS2qm2 zaI_;T({5LSV}zZg9(-j@5-kaU8eNDrN>y6u4(f)4G3__$a#%&FonPr@-L#|Cf8dPO zt{0tiNZdXEPAS9nvCo=WqbD=m6&~t+vZVZd^Xu6U=kiWWS-OM0fLb3-c0L1Ck|24)W=%UJan<=vZQbB z+6T}{N#L>=cJ#1f(M@FRWhMB^d#B9Pwho-m;@y)m0YCvr?i>3)IPrwA1y_K6WwEM& zqz!EGZL>rz#Rt1mPvGnh`FHIw$P*oJyT`32Rx_0giSMLhKSFog@G}v4Dbrq7ZsO1cVTJKp=nx;iwdqCLKg-BoQI> zBnk(R5(Nk33 zghc3+9SWA5FU-zG_uC8s*bo+DWG-A)syGMr)<9>bP(!PBVV8+{@)4LvrGH zQR!c|5swy*|GK?=>3{98P@`i^)=HsVboBIhf3tOqlT$qxq^ehMV&lhZ)D_`*S7gm)FQgBhX>*NiHid&-r83Rr4WDO)Ewo_Td7*WHb_S8Ufp14`zHyO;^55 znlV4O-{z;tye}WYRqP%`x;DN?#fd#bgZqj!eh!;%8-bS9Tbs4=vD*?9?0ok6HnWTV zOL&r&`$nO+XPC}h9hDr2fzg<6>}>Mle!CRAo^f`-&_{(VJFrKXb#jvdK>-y0yb&Ji zW(+m4vQa=zs2K^RK1V9Kq(r#++h*J;6S6_#O^OO921;|Cze8IM4JDJ|ATjB1Qhx-h z9sQs;=g>M`C5B*H)NMBZa!jB`5&wSOpGij6lfG9>W*H;>KG@D1m9??8z?B_h<0|56 zv;L|@9s$PT-)^RIX1+hPB_;O><60SR;y0Eoh!ZvL8$PMt$inxX65$sxEu9K*69Uu& zBC@X4p&nxqEhcC5(|!}rKq#Lauv9)Zbk#C=D{%H7mRcuec)62RK+jx zt?m}Voxs#6TM7Oh6KnD9x-M?yl!FAd9hqxibs~*aXY8F2D&TdhovT@!@1NnDysJXZ z+CHUNrffZ><37wRzc;Vw#r|w}om?ez0X?ff8jGB{f`R1WYqC5-mWfZlyj-;6s=DKc zUoK93X5DtH9U}SNMiTS=-F&;&{%M_vHn!g6yq7bUB`0X1@h2rS+O!nkeS5|v@96PM zX@lEx1WAc4v!{b@l8{}V47$eO0TX#3@kNsAp^%p^^#6Qu6}Z@#s-Y3a&*hkWbZ8(A zzK~0K+?U$-L3DhH!VY1kpolU9dqdVs_9#`&q(eG=Q7Z=5@+vL-K`y)17muGqCJ}Hc zeR)ND8i)VFB$Xk86bdF1t;RwO;1e--mYnwYH;v7n66|bzZq?XVQJzG&kTKM^ly=T8 z&Xg2^p6PW{ism@arc8EM`=bBZs5p`A3cp|>i4X-;uJi)eYWA^J`wSzmYCQw5ku>nK z*umT1(5c0YNUXB}yR0A=y+mKmqz)&Bt=^tt)b;tTOAqpDy|PPFpnY-*Fh}Oav=*o+ zK@RvDFta|BBlBXNh_RQyABT(b-DB5qO=D_2FXQZD{- zE{~M{>R3?0>%eyFc3bf`4d+W-sxCg=E%YXa#!5Y`fNZ&uE7MKe8M`0K3u*Spy3`lH zvp5-Z$+tXMd~a!-cAsA(xN1M7?1i8+?qan^S@@^a^4)pRRcU{#Nng<|_r3ksWQoF5 zh1-1MWzakN0~WNE)`|;pP9duUDDBFr9dVFfe$P8{w;0Y|IhbW!J@c`e+di$Lmh*Ou zL!CHX;y18<&WCc26iupXrrR4OOYN$7yXm30(2`0K_7mNgb|&RYN?)enndu0-2h)k? z5TZ68!#wb)+Ngxw(_*f)qY_thx&x|Z{ikNt23C*VMBP-Lem(Q-93qB-25uA+cz@= zl12)YhsUQs(68zp-${Aj-LV_fwgFO_Ee$M&TLg0)_tf)2O3pYqcXqLz>e!f|Ely$i z&^)c0Ee$wfS`(DZ48Au-T1gww>O<)dCRN)u_c6+NA>9WJ9^dc%%YT?smK8981@UyN z{+!*n%+C$r8wLW)A9T7q+7skrz=U}*2ue_?`;b@h<-+`9@$8{f1xxkg?fT z(9N~kp0hnJDGv2u&f%4RBGLA%x7XW1J$_7Z`9%b*xLv0K451I0)*d*GVD|p|81H(L z)cRnOglijitk~5}9$zsR@J_C+D&=f^lgBc~@C&$O6neOUuUraIqBeKQYsisjka4yx zS=?()XyxOs=EFEPl(<(4Ofv&M#Kdi!6yYMlBN;#0Xmc^V|U!6)jg?Seps{_&BE6_<(l_z3S`k(Sc5>01o z7GJD279F1gV_x>g$7VcQfZxAKA|Xk(J+tpRuLSv9FCX8U3l+D*Z&@~gosUZ=26fGw zvp+}}F-I+l&4bIm;9o|0njGna`Y(Gce5sh?VZ|DZC$7~RyXpn=o{ zlK}Vm*Tr=hLEa=ahsv-~9kY};**?%AZzA`=UD}54;R}rt6nZu;Wxt)sb7+&0;l%K? zp<8v4vuxT{5@|N=nw=0PT20u0P3i(x6ZTDitF1?gP#{}+H_Rp+Ei+x&fAiy+6xY55 z_|=p81= z16eqpHsFiaX*PvFywgMW9Xu2rd-+1!mRMTnxZx=Q&%|*|ow9lL<}7i#iuWT}KQ>Rx_P;tJc9^8)=s|FI;@fXb9u{&ykX?x~3>Ww`(N~^y78@Gy_tS9+B z`weK-cKmOK!^9WX>0$uv zJYtuALyF)gW%y4+zn!C=k$T(!2|zu!=ZKxZELomF`2ugwbn zD=60j9l%U)_LT$}vkohJpOaFh=`>bj$>wzW06@~+gK*)Wl0%TP0eB3- zWR`S$3?E}!%o<32)7V5AQ9%$~NypQ&`tlZU`w9O>#c-0%-QXX3@^%7yQLz}(BNl0xQ zr9Er0vJ>>P3|mS`68B2=}lDYy^#;#0|2Xa zR&Cl6g=plG=L&XK3bWbmyfK8+Px5@7x_)frJAl z0W!`sXJ_hx6tloqcm#8A^HI?FU z@aK^G`T-G?wOsQ%)RX&4CI8!gqgPf!x^RDsaII(PZWr97jt+(bPOgcWr6$Dea~BtNBzAb1gGPdaUM--uegLvSd!Un9S9 zPRhDyq(S&6?Noq+N5yD_ z&6X8fX|&Jq0wS~z_r@3jWV~;EzU$(C-Ub>sVpAwuN+?E3b1mF(uG;#@ubrICC|xkOwo$at5YOlu z4|tlHQw@O92;|9vB#g)2_dz8%tDd~uZR&;a1`=RDF*G9c!x76#OpPXVZzWz%dHPza zO|MrIH9+Qr+M%0eEqiiVw7=zk2Jp?zdnQO4lI`Sj`J};jl^m~N6;y1C6}P$ug-LgQ zx;#}+H1J;HB;ZX3YuevxN)#|TWyfVbDKaUh&@kuQcvI72r}j^S(^YcDMCpo5 znCA!Z|I9;4@V9fmg#6b}|9_AD`G1tFnx)|E`Sg!H??=3QNru7~f2mnHKnZ#M%cZ&h z|I*&^e>dR>!n`m<%hOmz`?;E$?+%_lTl1g6pVCUQ()^c1i*YtsF_J;ijG)%ieGtad zTz{wHqKfmmy!f^nvACIci=dND68_F%*N~05_^ps8O9Olp4jK zN%oG-Be+b%YbA2VsXNEFq17o-S)S+H&jx6#zqVeQ?nPtty4L8F-5179R-R}-T;>;ct*yFC{CMK9btm)fk*iJMgmSy; zb6-W`kWTlno|JSsZbRv^RxR)kuxay3@(1bI@hQH%VJEJ|9;RB*1-jl_jOmbF-I7>+k7EZmXQ|m-(vA>lAnfW_ODI zh}BkNxKm1yQ_79n93CLQ906klxbvHqmh6mMSi;9#4r<5oDEhJPv~ zBGNMwzGZfg*bw+jN&vtI#k7C)y?XLqH`~eCxQ()2Bj9nvEEuD0;?X?*UTz>vVg>4^ zh+LOVW0d3S@%kk3i}jxDHnIGb%4Nv%`Dn^Q|!Oy_7T|DN%l zQCiBKc{?`k9z6R;^tXKSJg)}5IORv-bIN5OQmjnwaXbV%Y=oNb+E(Uh8h@>xyQyKy zIk*2#(d}`&e+pr0|0wN3f_)6q$=4zrg@Veu-Z;Cw9jdYUUb>O)s?`oFe~ETBw(K#6 zIk)YEDHQZF&AAFnIWN!p5XRk4ByXKA6UpCHbVQhXlzJAD*@{MFo4d+KZ~p~cUFb%b zJme=nd$b&}z&vJ%zFB3s~M zMR^r{g?H!flmTaWDlOXWL(A#1JZFjju%DmH|9^EQ<^M94Ub}RFj2}P;!oW-vt>}gM z)+&}!JdCmH)n7^Ev$;@q^kqAx^uA7s}Ry9{#;y6y`o$HcbQNXvMG zJ8%f{GA?Fy$j6i#n6cRkv&-21CAqq~W(hsx_ibL9O98zvLFE!%ct(+|P=6tYdF>G7j;&(O~-%U7FAXH7|$hhQKC=Q5@94Ax6+6NiP^gz~R2;q(0!xj}ol zH~(5;j_wMb7nXy>lTlI{Yo>RmOiU?hrZb$}Dn-y0L#w{(BZ6H8X$RS_eL}9?gjOOq zKxuJZa>-up5X!=B;$u>xTEXH#vAOf=a$Q{a(}m+ZHDdUc3-qi*nqt1_Q}1y7gO)%K z!@xz~!my0Y+m8hT?Rt8xFiv7uieBFpn95Q*g_|T{L=|RFClxP8dD3S8+E>+XOiB#5 zCu-=5^QAKudPzE(F)XR(BNiD=PFvCHO@7~6F0EP9N7h60M3U*?$(H>by}zuz6G-N?bk;&I^~Yo)(5?e+ z>QlREhoaEQuI2c&?#DyT7jCxHbrhK0I#KPlx}p7$b9u{NZjW;VJNS+7I^wc81wG51 zurZ`YLm7+d)x5k7U^K<%z@J)?2{@&K!LFJEDWb>k(Ab;j^JSGhL~gCZ1i(#Z8k+AW zrH69>o|YU5te^&USSOQ3F707_h20X3BOdAxg2i!ll{v>JOn$t5uNQv8H= zy*=GFmyfcZ0y4rdR;ODX81S^UCe0d>maZCzjBe9FBlj(nBG>U(0mxm=?RUzi#yDvN z?6+`Rsn$0T5Vgkjm*mD&8wM{zYU zVFT@6zijw3r!UK!)d1O&9I!Gr8XEeyt&&_OmOK}dJxH`$y}O&3ALt8Z3j#-J%i-hG2kzu&z8x~zCl26mUaE@wo;p$!eK1*j)= zRw~t2Cn|8TaxfVa5JIoWt-jSkTU9-q_dx++kjwRU5D63xl{as%Q|}ywKFhx{ub+yR zCTUZrb}wo_On2s+-z;8?+F$wYXgye2KC8<nWMTk^IVxn)w;0sD_q2X%ynT z0PnGK@mh0*zu|-eE{fpLr@^hdvo(5u*`a^-3_I?k@Q*#NC*j(14iRZUv26GvDT z#DY8+l&VGPK=h`uy7mccPaU(fdRXeD*H&-|^63I@v(c?94hE1CwKCQ4Q8X%cBOR$0Mlr{}93-hliKnePnNRhv zLKEo&lMfHQYoDjJZj~|^8@7}AQ?$`8kka7#?6HJ7oN}a=O?!mTHQBBN<-R5 zp=GcJ6Ngb~JbdCgiIaXhlP9jSJ&b|g05QhO|3qAiKR}jezJStt3!?H!f(NAJ2s81_ zGkL?iVto8I8aF013%Jd!_Owlw+LLf@&e~cANm_A3vG=~4C4GpsnV=IG(*_>>8^io6Z5E5`LQm2Jg%6hQ z^h?uevQNRwiT!mV0aY8$Ev$L$fzO+JlLynTfmX!yh!z}fT)Nez@!>AT9A=9v7?Rv7 zd_}&kkZ2a>sh=`Wfrlh1gTYBl`ri%sMQhPA__us+ESTN|I29!ol)JQd#$mpqBKRIT zBP8z$yrOd>b}4RN@JyddsK1A>eMO~qF0xx%KkB&|fBIuG_>1;(WjL3VIPTiNliKX{ zvVqh<;*VyF0^VK8s4ASfcJYYPQxZY=U~3wuDEr<$y%aOXi}`0{VMN>0{z*czl;y6H zp@cB)SZRR_H}Clalkk})_{3H}dJ?X02X5dShg@(<$LnebJnYX<5(}U(EC}WEO5O+V zZzZXaGEK*@FR~OXXSD@;f`^vF(BG$mk0I7(=Qrl)I?WuNB|p)XflSa)2v2xeK(2O? z89Fa&%Nz@hn{9BgrHVB5%1{HX>VXxm3ecp4=;S~Bg4agx8QQql{9-Dexc__5!rQhv+Ga7k_Ef+~-D$)%KN(5) zTAwJsim`XS@6ceq_mk)2T75B*{oA)!I_#{HI=n@Fvf25q`hH`PaN}$ zC7`w#(+Rh(;BGRSs3O_96dlQ6A7#p)Za$8)mOC8)y2vL`s>vbo(;6$lDXDr*vSU{; z)PO2)D>1|9SIup637!ok)?_0-dxO`Bo7}cDew~IzvOgU<(!s4?glo%FY!KSE1!8{2 zIGr(z!yM8LCHT4ZgOE@$yeu!okL?n6xu;N=$JS_srGWU``hq)G}>H%9k3d!d4qU{4vV95w`R~=eD2=fx1Mfsjhk}J zat9WAmsk4&*8=i4k{bWMgZ`SZxYrWh)#o5TzR(M$d^~<>jcjYTr}@sBQ2p`~Vb6I& z9Tj2@U^KOTkj{jqe-4>E)v;+dL@r+%>)G|^R-8^?g4f2=+Z~%3n{0kFzE;U@O||Mt zblEqx8AkA2h_Z5MeAvxV=whk!mgb5ad>9{QDOWMXht&lju3II)m!Y#XK6Z+Gh6r1$ zaSm#He=O3{^{dX!e`3k6>{1lBrfC_qZ0~B4<6uQFtdF*OVx>Vu3FRw(A}v(53A1Ar zjsCqMNi9~Z%qK6$dhP5IiNfH-VxhCXZ7wn`sU%9fa5pmeWOWJMKt3@%&2RnUh?=r{ z1?msYRoPf+1V-nhxo-OjBj@QVSEc!Csw2;ne*BxXRRjQ2HWCNC=cZb+`)INFkmK7p zr2#BIPgE_0Ao-HH!eB3Pa^Af4sCOV3|RAesdj>YP&l>|w&p%GRiRdw=YuT9F&t(d zPb6<5gGonvC!LKf&1a8v&HIZ{As z1ozN%0B6LE1rE@mkzx&CJU_hlLTI=M3Eyc$p8b%TR~>@-(jOUC4_1T2Qyajvah)p@ z`-2os35Tcs>cg%lUy9uW;gIY?bb#a%*E(3deBXfvpIV7R+Y^amC=L!#Ir=$7iMfJB zd0uk`efM2+A>%6epA{rxgiUXF_gsWc?bfuB+u-Q+o4O)Ya_da3&s@JrPfyoEZ#Arc zNeHKE2mYeP0WwbxQVOWq?;Vv+HCrquT?k5K{B=mn+q$0gg7lBc$j4nXE>WWDqodFu znev<$g3n9Z9nK{zn6s_Oq$cSHx<+bYsHZh952^SsJ&FE6Ewufh)5b8AJsy115Z%DH zTWkG`RrqKBeP;y1NI#ad_Cb}hax#4 zO7bkQ`98>Y>Z;rreC0kxOAe2s=S!4&KP;(NeBM})D#(YGHm#cY1kRe+`vs73!8rs- zNx(B>VWHPHYAkN&1nF-50Z7TN&*Ap$g2~7;mcuNsi*Wc#AS6bZF&TD2cW4wj@Up&p z&2xUXcmoAVLx=jn+7jP6!6^RBlA#Kw-!Iop%@IKpW}ebAZ%i-J5eHC+p?x*Po9R7Ggse23y%*n?4|FWYlvWS~m>euCu4 z*tbt|XpY+38pT$J!9h)2>s|Bw&#~11E~={7R_i7$4@ET-2uE#OV0L*4vKf-{#Gb9k zlJUOlxyBk-i^TU<%a(nC+hwE!c|lht!7Po~%*qAK z=q9u4M_`Wofm{cJ?Uvx^?~JepR_O?BV=iY3i4Gkf)10QAS_x(h)nEKHxt`PY#jM&` zf(J?K>YAJjKJS{D8h^r~Z{h+k1mxxA2D(6BO{o7T*azp7tiPOzpho`=ixD(Pi@QFo8^x8 ze@C?g&+8Y-p$6&{gAO!+t(rxP2k}u}S*c}CssMiVyg!WLRe5*`WAA~Uwd2S-CkM2` z#yahmo5s1K1re@=J)n#3>q4(A7lSU-+B^!e7M1SgEy|nL199Q_d zou~qY=8n(9R`MFe9GYJn`cDd)z8AULB4YM#3^(<$!OqTHN87`{aYf~|9LXxofJ4l3 z*V4?6sTfW5(_38pXlBP+nee&%EZW%)@T;{-_cP;!Vv;8++TSGCXliUbt5T%l(9NoS zm^R0trB6P5&y}YivsCSIIyR`MJZMOQB*R<%#Kt_ti@vTrj9feKhx``56Gf6`u%vVt zY6y{oZ>Y;Q0U=peH7n zx}lFrR8aedfgF+OzEc8dX*`AV^rv=5 z1w_@%qe!ZC5wGoOtqfi-y=PGkiLlNZf@mOyb-sOn<)I`n9^kC+XnxMoky;-8j_ zSuP)xE}|E!YUUbVduMoU?~3F58qyG;`O8~xL4T?`=`_9bHwVAD+d-`MJwLJC0YF$3 zmZc%x&!(wCR6NH#+vbW;8)6(CBJnuDs1((xVj$i692KlrM@Z8g7F-jlB2wvZKsB7& zoHd=YqsOX?AH=PqvwnmS#E%g2xsO(ZsspwdOI-+)S<}@Gsh;Ny80es%W>U(=>BF^{ zQRpWVtGJ3FOGr{WqwEEdL8^{im0o^d6K{ZQAlU|eeX%#Ib^mj$Ct3z{Fm0_$V=6)f z=ol3iqiKAoasNsTH91Ut3X~rW=NV|3pFyOmJXf5bF>~d;|D9eW0msvdB_z6fmn>f@ zI})1r^zP+>=~u(Tl6^7X&g2=`&$@mcqlwgo<;;!9!U8Nk_h|ugy6CyNWDU^%#Hp2P z*Go5+FLxmCvOu9h-<2<1L)L@aEM1KeIy#jc9S)Zxn|No_C0=b)tbsH>A}#vfe%8qU zYn|1=w1|a7(t{4wI_-pqCCJmB4-W#QrTz?RbLqeH0uMJ2SX+Ho!D*e!=yGkCRQcR}3U^{ETKw%F4@HsdHJ@bSgY&Ryl}MK|hIuiq zLMKjUnq!mZH#c8{0|jRjLjhi%*hc4)XKT?zQ#1J?YE>7?Dr=TP=6HOcT^rH7}w)n)8z*MANJ9daAL8We=SVBqhEaA{ET<#G==}Or{P)aCz-t zU-~kJG~MH{03{0Q08^&oyG3y|RQrt9h2C(AI&2h~^^+>9ZSRFl0NxfTf(-)4d^#f1 z*Kb*eFA1}#Hn{#sj?n%Ot?@(3pi^~{bJVfUxwXP(W_m`4I&0T<^c|s5wFvPA*Rh;zqF}kshocXF5Q{$M1#ADemXm1UKf_ep$of zO2Zq#fb9r|RttbvOS^BKSHEnRftWk1JhE<|cbs&Vth7V9Y59+DJV5`7LHnXh^h=2A z-69j(_|YruH~WI`Nyt?+I~i2t!p>39=5s`kc%_Dr=FL4jc7I*Q9w43Fp`N0yJGMO` zhyG4h6o0~uU>lALM)3RKF4NI@A(Tt&l z2(?A~#r^Ik-SNNfcvQOM9`1B)K2WK{bg~SiO60y(n{^N+ee!DAn9cxmMmrG}N+dy+ z96Z;iy|kRjUJI3vK(XBA$mQ)~@}u|{3&&kg?YxNu(j)V(qcxuXD^dfU$ou2Cu*0}N z{Y5xaF_p{uCY-qMWSjtLzRTooBee8Mn8Z;wd*QJTix*m!i|{nz+w2}$TPT}*4HPHJ zMMb+f0Zsz|+0O)ZkVw@Is{Fu{)JOM0d?Js0F4wmXvD2R>v+nBJ?JeXXGNgJ3(5xXP zsl&K>0bpKDlpv>OZE&8xnQLXnxB%O7aqp9tJ9>$q-tc8o!L5$#fiP~Xwnkfy8ZfAL z#Lw9qZ4-Md-t4vXOydW8Vndb3M4qAPVCLqMY@SyWhV{@Si)G2EZ z#e)?aHJfw~9ILp}`5+lDpz=rG8M+0}hm-=BI5JBuO&AA^vqQaPA7r#;l#<*!T$M-o z2A%K$Y4HF>lW5#P%vh4Cw60oicy0qZ+ET*Nok4OTf}FFIStB%0tohEHzlt?|n4iTO zMc%#?!w0M;7|g0KCHj!GK!_k4x`?7#ZHK5SKlY$f_Ir&cd$elEyfjCfev243TU_Tc z7DtG%Ns3y3VcPAaHb33mz+oT0keCBb3*}D;N$cJuZ02tDG4D3I;97SDa?Tfc4dsn| zPvkR`XErLH_^vxT8EP-$ zB2%@&cD+s6Om*q=5X~%qox?nEMZk8F3O8BjgxWZDj z)_hib@OEwg51?f``2aa?sQB&8qRH;CN*A+peu@d2S*)k>wB7gZ`=I;xgiNmFQ5>tk z?rP@2XZbOkkXpfsOEyn;Gz)t%WhPd}Ot{{T+tm)eem}e8<+{E`Hg;Eo4;V~jLvuNqI^`7k%1)4L z8NGxLqOoeaRLa0+a?4vR=#Um#&=^oX4ZTRbu`3v2ZP0vaqPiG*J?kDVv3Y<})hQ9o z^r|1-R~8Xp^jeeA&G@hK#d6l(d37KOeNqu!GZcJZ7X=XYHcdKWw~}&r02z2k9>LWo z0-)H1n1A2YmTJ#P5Xb#y^#i_qhQXNslxwjJpG8mPsNi!qazmPS(N#jz%fhcOR}2@6 zrHEFp5pu2qk8yU^cgAkWNe zJbhJsNTngWC!=_zpm9{Xd9#z1+5by9O2~N3V7glCAfMK&K2Pw4hvm9^`2|=pr-QOr z%|nn%Vz?w)4mhO?ej1<4{io(d)20SOREd1zN)I585s$NoYs&$EHSe|-2AO7Di5e0O z?cs553|JL(?OfhubS8*P4ig=D znma`|hE^&+zd=eT_`1;(FDIpXe>1f+*FY_(Ku`ta4s;#OoBy<8$xrwoAjVl}M=y}< zMchyFIZE-OaXcsR7#YMwKidC$ELVTFw=7F_#ua-twILN6_|)&4 zAU}jZGdUQe&zzul`a%pz1!i%}J?W|halNGVPRn+v1JVSN5shgt6_t_3-!0c_o;5`$XsbY${bw_R7v8T7^-?Bs zQN^MFWjt>+aqLkZuVCvm=WU@gbyFeNJ8k{CvQ zpT*o;iv?JJcUGQ`;1#8wmB}Gv%gqwKw^duZTqaeImygeV0Uv&$l;n}J%Ufg=sUh-A z;r%1$Q-3V?f3hu0kCXrwvEv(Xa9Fu0o2#YIFM*DCy1&^;(h6jb`g#SKCC!==0y$P_ z_@F`Ue%u(GAteX4&5&K{wRHE_Dn4RQG@rl~1DpZ20&DZq zPv48US}PZ_PrS?`0!{9$6y8OCP+Q!S-FYzeN4%ce4G8{zN=nsjqJc1v5x)%99yM!c za5pVk)=|GG>K+G6E)@o$NkSF_cqzuQ&u_8cQ#Y!@9+iMy;wT{xaFZJE(Ix^r>9`)A zvcL}**Cf`DStG0CM4qNSSyHQi>)mFWR({j-7>55=Nm&FQeafi}zHJUII;-b&nR5lk z74BPPkFC;Y=)O|J5^QeZ3Px-ClFnZxrvg6nHSldKd^%9$iAQLkpMuQzxh>`XAoOCM zwS(_mIp{F>%-m(i*sPw;K1tx@mf+kGT%XUkdT?y$No#YW4*7oDs#nR_AxK`rC;h_o z7lJcT_cQ4}j~Y6P_g5BgMeeTz#`iV>IK&%a_EC6O?Fe2sqT`{)EPfsl@s6htnf6pc{%%m2R|x%4sgTLrm^2qu5s(P z=GGozb8Tlv@sm4;IKkZA7o*VPr$D~xA@qpGEWy>VJ zSO?Qy$rLd1d`zySdB1ehy;)M^%7A@>b1IWa3ruLTo&2%{z^3MQn&`-Loid>nVtdqf z+B}Foi@>ZT4NsRHWFBNIJe2*oAPP)Ky#=Td)D3}KPodv>uE?n4B~ppz+OI_A62(47 z!5~q(H?=RioU9DL#Hz6me|*o-%1L!OFxhoB>QTzw>@S%tx-9YYCaP^-Ww?CRQ13cn zH50dBzV2Jt4$A%A3jjLj)swUi_+QCu?m(Xo#I4rVQoJULnIoAm9uuq`JZzAZjVuAD zO!bb5oG1@(p2m{HY^cSpx96`tP|7i#e5O;OMTcamufj4CoTv~`h*x|G1&^tFhSmtB zDfBUv$1i6%GvF!jR2|&d6&PO! zL;Ej`qpEG)7m8Mm^jhX?eAS|1u-g1M6_AsuE6u67V)hwAT^faiqH4+`uvQ zq@k0OmQ@ULWGXppc=ff)bWL#}y)wa$tT3xpz%>XYsIoR=jZZJCdPZ6?WI3!6CV<%n zmauvN>s*=aM8%AuUBzLaoB@TXTKJS^I}F@0R|l0J8bEa+fzaInxQ_37w`<) z2f@C0X=|{y;_*ZNN0mqj!@TH2kquTb6ao ztF}1iTcO!&;My5_%I)H@#}mEmKo}_jHKStNTpEe4URvPi)kzdkVB1+p>EIz#>a*a< z=iI~F?>G2t1wb-WK70JFfQ2MgEtxN1AS*xAL>QJ$+-@)WnW+c-o2hH#dq;K-wFN)j ze~kYQx;${XqC&Ie3zuSccUJRAvV2?ZG$z(EB>&@RIYn^sDwDI`AatE7;thTubN~`pMKPGjmeyD^{L!2w zsu)6q`SVE&7)`qljh^Ug9jY!Nbjd8!x$KMT1>z}}mpNZ~V!81^eXqn75BqeSqxSc* z{=g{ZBza!Ai&H!jWaEszs7#Zxmq@z}{;5$1cj|0PXOaGfbM;-6~kv_vm^y8T#;!ot#mCJ9KZs*WK?Cjbv zZ@adC^i2iGj|-ADa379{!Sxeo%T2DUyqA9--$gV3bGp9eN}9)TtZyjSumXZqZ@y2B zRH;`4e)Xsfc#p^D3l+Hxe)03&O4%%P0VbkOp@2~of8xbq|Izn3#;RThA$EBk%+%pP zDoHF}e|y0XfM4x1)X9l6{)V&T2wKQ>&+>jwxss+W*7Ku%`1Akb7lo*U`6eToD0=ZQ zys13P3-0EC{-4g&+0IS(T7b`~WMaSQIBjJr?+ImSk;8>M!J7VM4WAAzA8<&M?iYPf(qcod|=e)&_K6>=Xgm3=tNFM zOS}*cD5$*dAVu-7RC`s`5n$j%k&ObqA&%AR*KIY8<(mnuE%gY&SFpvVX*Aoyw(V?5 z?@vTyweP|P$}EE$Oa+Eq0c_~NFNi_YnsgYn;X)#whw~bK&X4i){{h+)vw3A zPc2ldCt2*PFwO*C*Z}l=)(+zcy9>jT68#r8aAtgMcqk1p{$+lFu%zt(@dP**Ix_wh zZ`L%Ox3Z{#jW^M)2Yc;<(6j_8A)fW5Rb+7ZdQz9#7&%0{z?3dm4@S4S*uM?u$oW#| zN`e)z->8WLn(0RdIT_Tcba2kk}7@8UI@D}q9c*0pY z+(Tk;sV>||yF!{M6r~Sz5B`x+btU`GtB{?62ytU zKibNd)a)=gEn*ft?tG`gsEV&<$!%iFWyQ|Y z&>Z@0|Jyd8K>cYQ4om(<3pIe1n(67j)91nm!8+m{KX&ChKu@$1tAoc1MX}#mC6kXy zM?W&#XN)SlKk29i8@j8J-xB5r7a+P?f)8H( z{v!paa2^fY{6Ty%mi;3Ej|Ixb%jV8&?91y0l~R?YjYAL6#cDaMfC)?{*Aq?hv+E70 z-PAjshZQRjPm4^Laa&3+t1Nu**SqV$^&aw)LinJ4y7>sswe0Aq0{bi2JT&+|QbC5|QnXHo>e0&g(^B@Bevt%DH=ij2)CNt72l zu$8}hyBhz{)aEsLyPw+5uexXC4n4X4;}Cqp^C}^9p~xkK+iSijtAS4$AN~{uCM*l$ zte-q1q{*9q+?!N;f%{MdETQQ{RVqCP>pF_75Pe2=q1*z9J4VWxF^(`3A|5D?8}73( zy;>1>t_Pl*)@LHUY{e{9RpZV>dB2wT`}z*NZ>Nq= z#}%8(;)G4ht{Y^0aAM@>t*G*{QG0q<~ABS&mDb)GpxP`iHi zekzj@uj?g;NfeA*c}&+{)Qe`ndj3GgRdyba@uOVtofd76KhZf7amd}Z{m|aw2YoXS zZ0~$521F9*0;ritdacI*5^@Uj&8dI}LRz7>R9#Vt*DB-8r(7E1qcb5YM{P63_ zo@sKkV$UhKoz?eguAxaG|AAPCD%@IK1HMVo;j#T~n?W2s(UkUGsMRsu-Ijf3?JkwI zoST6v2;uz}7j!O$o5((RC|%txCi7T?IqRiA>qd7smok%hHXKv4|Gjph*KW;_wp1=` zL9@;3z~c|kfgLf@p*I_U55CjPqetxNq@w(+Tce7{1Y&nK@}}%zYzP4*B%tc&k7c*S z*jxJqm6hJmStTjfq!v1|A3JS4=6Iuz(^(C5LUk~2&WBq!XugN0Fts1nI+~jlL}?CG zWB)XK#UEN>J@C`er*{VaNM+DtjiPzTzm1@=+%-4;s_Dlo=IrSVS#pBvZCPIIfMG~% z*Ke1!uMg5byNM@uaqjA8!g(_nF`~ikbqxC3mgPlWI`8Rd-i~JSy$Y+wItU zDKQw&w6`kI0i%bPVdadHV%>q2DfxP%KaEyX0aN<1BMd#SNY{cGcBuqh{3#ch(^NcO zoYZg#lz?IESIH7}<+&ie+Zlo-_cK@akjYYQ9lyt)J(oRM*yc3u8PW3%9;ioiwEN~&WrUg7rH-1Eb$_1A$v zXT*wX47qyA#0_<^zNG#wBgZXS3FpL&zFRJkw~0J_ z&WTNBDzTdue9qURp{Tg5P#N%~j29AU|aiac ?k5qV~^|m`=NXea9b{| zFK5IFI5A`I(S^X@@*HKePjSrbc%gUW3IoiZK-dZ*5 zE7IwjrM8g=vWFXzJZdHUV!GYcLQh{`C6z;!>+v8hBi-2dMGq34 zk!a07WbX>f_3iiwY%?8E>&bTbai&(qjA5&i?&Ws(4{SZg2PnxN9}VmA1%t*vM0Qr_ z95KE9MF=S!nl&Q;{cKYDYB^X}TK1Gyf? zHA!qQ(3CijQMNAb3sq>#BonmVPBBQ{0YaR`(HH(8o(|$a=BITnNgWq2p;e{x){FCp zR)s&Ovm$iz3WD#pU0wbuNSMeKK z(FfAub@}}mPG^58Qm(n6L+OOh<@t@|1H|V0(gPQHv2nYpdBphya`S!|Po{^3)9Szg zsO^=+X72J4i>iZvat*%}$vOwd9QELs12~+Vqi|K0Hl{lNV4KBxMg4uh&-X)P zrA_9r^m~{4nuYjeBX(LubKYT^f!bRYLna!oA{Q$|3YmB>!x_BK1L48olsedOwS?oWLRq^t> ziBlXV4rQzus%G9hn|W^|FdCZ&pFLmw`)Y3LY-ZjYoJ<OlxT-WoZIVB{1u zg&Q0##m9FWs~GyYNhXk;os6>z^(lyxQku~Z>aIv82zKIwWvREOyQsNEY9<9sarOA4 z2)?0)&>Oy1G)A2kG|*$0h4QuXTQ@{?vXH-JHO1KZYDi#nFH7J4N;#w9OnCnEuEtK( zqQMwa(-BgE@D2D%Kozdj;JzK)o7^HR8>`;XkrCxKvH^rg{Ghy#4T5?s|55KkS3(Lo zM~?1;&45l`a=I1o9)k8C2tVOoW}SEuUCC3Nfyowl@fbq!Q5w#272?W=HOzT zlKvFuR9wYI`n?Oew&2_S_vAYc0tyA17Lm&EmQ0GBCiRgWfp}8`?jOLIjfdhP^uXU+ zC0#Z}eTRnKm*4J#R)pW?F4|hvegmtTrampl;NEz3urM|^s$-%6&iigB#Z=bh%|8D~ z?{fx!b{MPbn-QwO0;HB1VB}Wf9PdmD|!6e?TD_7529VEz&n?ct%{jciB*E-?n+5 zwLx@J+}ef*(oW@D5?YN)a9@L{$Jta~Ldq&rUvd7iXj5ydN9e+sFIu=t-%|@cQE4W$ zo6wV0ofP}p_O`aWQIo>Mh+XGI8}wF%(xGc;;zvBW9F3IG`Qc<>P}e>&8bnu?ux#d5 z>LXU!uExpgcIew6EfUUt#dbCfn1_2Prp~n9)Le%ZD`J`34r~ z@()%7M@XQVi0Y9`cuIXe1)jXRWxaQbQNHOF`4(j>+4!DUm&U;%P1)qUK#^d7VB*M`VJM+|&OGU#~&ofIT0Ag(ZVtnw~c{IF`22 zoz9(qUS2-+hu6*W{znu{GiA!h&SE5_>G7pGhsVF`G{Bqltw;gd^dhIC!`RuzvSr(J z1g~({)z2Jk3d@YNNiV@2wJ@_P@BNvoI-b9gTT*D9tvCfa+Z3-;xt&#fPJ8tg9+$-4y|KUQ^r^I_ ziU%FBZJn?0{QlM!0yYt_jWmMAU}qbm_&j=MG@DzBKT<_dZ~9Ai`T0YmAs(#g3dC1o z^L$=h9CnEp3XYf1-`!OooaL0xcg-IVztkIMBD;LFbP5Xd@~ zcyIFPfO~a6LNsP2c#@xy{b=;L0D>0qbIq$7>9aTG6RWY!yx4KwV+pePAK4uc!Z&OXx(=y;*2om2nCOsV8RB9)vCZRIL%sxIgBpuyhg%-qc zqe<9rzMmOdL~xPG3dV+@f;O9#7wi1!A3k#e+JPJMrBS_q4BBqjg>+}tL_+I4yw(48 ziIe)x3`@w^Y0>Vmm%10`;46x!Ti4^SCvy+$o>>aee=lyZF0@s9?265CzuogbhXG*j z$s&NQOzBw@#}%R%SDo0HgA=&W$nUFCF?g4UIwQVY#dzC4fI(GqS|WdFH9f*-&&}kW z+J~A?T~t$FL3uqqcLbC4;cV*|zhH;HYQ~^resL1n=?Ftg?Lq36_u;#I=Jr;k1&48AD(R?=j|wQ?7n7#MU2AELW~YV9y`Rnjep8%)*Ap{&Wr-9A?%+omz3#xC9hu1xpd zxzf2R4wP(*Ae4)zV(!q5srVHIynxQNihHRgcmLq)R^-L5OBat|IIih?e*2?H3z+*3|PG(}HNY2*vSPh-(LWA~0qVNrRY{VWSboRptU3#|z@4FFNL(sdgy?<`OeCu$#zitm5)1voEYpX`0w$Ac&$(y&K!1V%Vm+0$xJ}%#z zrzrx2h6O9WqFx7>KF_q3RoY~WIkRZCg_?JS(5IH7;P66k%)I>D3r44JLaxQb%Ge9A z3zd7^{~RjPp{ixRk8#Tj8oGYnnq8iiyxcrfgL>w>dfx86Q=qFpJ{fmQHdp!DMb2WC zS0u%9R)(}*)d2nF&mDHkz3^dp zbPL(pZh*V(wd7pXOd}}QRunPwz|OES;fD=~ki_|8!u{RFdyb4M``p$g*`Ciax4+ex z_u~}@QR-}KT$5<~A2#vsFP^FG6NLH?_+CT~H4E^C!_|(2fMM56)iVixwwY9SAX#E$ zL+|ov&G$``@?CLqrS6_e3#Pwg1*o-^I%}P0$umu^7bu2Fk{(KApj(##AOXD>Bn!6R zsFW~IoUA(gkNdL{@Ym66nwL|%aBUDxA}9GJ);QQcgKyR|w2N=4%1Lt2soK>znI@Td zHoV*3lm(IjqPS9F;7{Q~`LI`3EWMJqiT1VWv23=gc`cJaik3>?G=T6~NcnM9Ko!rn zdnb;Oql%0<|KvklvWXhZ^HJ(~6MP>*vwD_L0)Ph01 z;TK>ubHgRgcbauVW1(+yg=ZA+LyRrX_;FQ26{Y&TMaLX!eASQz8II{x<__;QKmZ^f zh3AFcA-UTb22=b9O5)A%XhteB$4<0HcrjL|q_=o{Lu6R`WBzc{oWc9syZv;0jXs^mNSw^I6)!^A~#Y~4Pm-B?IPTOb`qstI2 zU_!4d{q4_%F9O<&hgi3cX5IQ=Ul96@o1ViVYj6A4yIPz`eV~Bd#!o^Z)Y$ucC(eI?ezk6)t5ZwLK0~qPIzsArZx+ZOOHdq; z-2(YW{2WvtUERCYks~(eyNO>fGRKfi`HL{z!{x$&fyYr~uHGh-gtR~AOQHTyKm-7Y z+g=Cuu>$#-&sM4X zt&Je0n`w^WXUgVTCEqTy8+e-8B9e`IX3H#LOFXzmO7YE9h3j69T8Q(g<=CyJJ;V35 zke?k0vqdm>LD}#JsyVUj3SZ$yal|LL-;2R*26mc!9dkI+#eD$V+ncLiW)Z*tFt*HL z;3?1T%U~+)C=ni$7Ft~P9!}73x!#|7t?<0Xy}|NPlP)W6BS>8|Ki%%h3T@`e8BS-! z&DH18EB1eR0oM5izye#og09?U?oGG{H^tsvwdcPlFu7vV)*|>*xoGgI%?=+_mGOH} ztzu_V9poU^Vg(bPK=Y^Nio`Uj1~vZx3#)UT_^#)>I#H=YBgr7(~6`wzk%O zdH07Xk!t#^CFOy%gHcuOZV*E_vKFenkU4tvoq@32s(15>_n^C@rF-zhqeV0uTyS%v z)lKQ=b5xAKOkzg{pNTQlg&2%ZDMQx3yTg7+Wma#>>wM0jnZ&|xiFjVD2bW}}$(rdC zeNV&5?CP)SDCuT>p=o8Pn%VhE)Z3SMhEs+?pSdV?EifWUDer*v*!mwdLm!I}W8AS& z^k#A${s49op=SGz;9wsnPuT~%KiJf3|CWW;#qZC$Idv{6qsNbM1p>HUvIW=ig#Q;@ z->R|p@i{Lx^w4ZWcuYMw`<%~G4&IVm)l4)*UZ3f_Eq^B-q6wm>UXR1dkcxC{3Or1 z=3=Zt-KMF9XO6e1R>I1WC#zC5xKW3_Fwri(-%q($Ks~iPl!DQ9>-U&W@}ncWNBkXw zXL1R|BXeDd&DnoJbW{(~=+l3T(Y+IB+ZbKqC0Iaf9dtmp&S6Ay+_~^0OGEWsFypUN zPx>m3GzU@h=1<6{%*mk}HRUm{PTxev(;}t|vfTGKEX?UtA5u97s0ksd)qLdnpT^hk z^vSNF?x(Bx#VleHh4M`;EQcssHBivebmR_7uI5{?d@QN(50O|Q(+~9(-|$_k_)^47 zbH0sP#fRQqz8T)m0GVs`vw-;v=S_OTxMu2xpledGui5)s`S)B~#)*P{qGQM+WQKk_ z*!anRe0%u?fKr6&p~>s*opKvJ#Z2|y!1(<>)dBQA(vzW9BazecAFICxagWPPNFBk5 zh`4IlJl+nT1VCy>M$T1|V=4b>R<$<^Hn2HTzV&rZ(7(OvNeO+te1|mny~w!?sS(P( z`78i67;Gx4!#=Yl%8errd_~R=lN$+gf!1t;-^^CSt}DVIW8NE8sIb)++?ArlS+O2B zKm9eF1@WJl`%?3u2C|qBz1wSm54)$LhF=48N-^M{m)_Om8LRKa!gUiHzkFX4p_ky_ z-c|b0>)$%fFW;=EOKKOQfZ*Sa75o<>>MWdf@VA!Ml~hkP^z%zds0bw)I-fK2{25n6 z%>Em1xBr8;dx-ZBV$~gH1^rHfr8kR$NaT*3{&+lU2+2Vm5X8n=t#m2*m49QQZuMSW zi{K1p9q~>`vAKo+oq)^eN}9x0Ty%vKOtppyN=zJ5KxgNMJ)|-hVPUg+^FGV1y5r*^ zU?(e9=0C4u`f!<3vlHcEyj1n8Afmg_e?spY_FK>!{%uw{QZIBT55*pW#ogL2D|{vE zm|r8=2FGTnEuL$do#x6T&0tSP^P`51&F9W>*s}r4}|S)ou`a)}R-1v*X8zz6AnP_iVasR;n_KT=8-O{X3d3=_o1e&Aoo7_sF?b6;ROr+SZdFMSw6Us5~>O7Y~rU z2iKY6g&$4u#HaH;;#;xn4m8+sZ;xJ>ANmDC@y`@pj$2X?Sd7)LN@tz$&(*%RY-yb#NMIEQ_;H4y=&%`*`cM;SgweH z6+nS}WF+Y)ekLL+th@Kxyuw_h5z49I`=?!9?+7>uMl$Mwa*8;40~7(n97%3ZNyAco zv=Q=~06JCc*si0AqZ={GFMN&8yQ?Xf#*HY*r>3zew*h+Zh&lg|)XbD`WXqhADiN6+ zAR*FP9)5J?Y)+K^u@(Ih%YCBSqG2DjwB0XVLWc37ZOVb%cA*pV@-qFKoRzhbaNO~F zm(_tbVz%Ii8b)eh{0BQ*dTTOG?YDk*#tJUbYqPpBa9Y#Anu%Tq<&6U@Da_F1mB0W- z{O0D}0U~pAnucp)yrsL*nAD{u?6;aJnAn7@`OKL$a`N(^#^$gLvF5bE3QU;US!1?v zfHf{`rNP>HEDbq!j8=Q<&-ROC8^wBj^(I&^hpr`8Z${sWVv*qpzIUgjOhU@b^r1q0 z6E|DGW%E7JBDzy}qX{RQq)470H>#3SAQt6@mczsuQr8&0L=4&{=*+6@paxs|z5lnn zJx-1Th+}7%D&MRuJcTQgN&vysPieUVX9%4B%c1C`{tUZ9!XFebGo zqrKFTZdzj=P2DGionp>)%+Gs(>>9UEY(k!?t^JO`WK)imdgyEi3(F zKZpGbH12833mK)n4u5aLIbCw?#s|4lL*9+*)cMlWpl|hj=d5GyucfBCq1YSlf`QP# z`K{>6f@EUJ^VeH2UyJUwh4QVT%d%wbVsyWiX#cUH%vWt6PufgT8&Zo3BT*@M5Z8!$ z2`v~COzD#z(c+Q)v^fV-6-nh97c4$o`i2h+;rsqV_*P#9F{ZIS`Xwq~_Z#7MuAPG) zq1l!r^(qO6REo8~@wXx3Z}&B+p$5t|0nDEXDg?7Nu!i?Fvy3+{N!;+QiqLJT_X7rU z32SY zW+o$(%ooQ#Og^Au_`;ntzgbU;p_!TCgBuY8mdGliYi%b;-C_OHFWM*pzm3;^3b^(;|fC zVmOrjoIasWOx$#GaB=OqJoGIQ5)H%w_VLQ^V?${K6t6#$LstPaqev6IgHKXe&Zx&< zVQ3$?I-vb>#XKYXnPRIZ)2DF-gpWY$%B6oZh)TSR{c@%o&=Uue)9!G=T5F3pPcyI9 z;|&>OL?WEggr;L$`ym2>JuWuDBBT)59cU99$eMtw`e6G#rZ=s;v}BKd?{dvXAAO(^ zgz2ZD8a*)!`k|8q>_2Ij!f=`?6UMs<>3) z2Sj~2vf|ac4cgt0p$hBG9B$(K{xc^JHsK84;aS!zwKLv4Pb>9`QV7scLOY6{2(=&^bBf zj9I%$j_8XLHNTa;kECKqxq8|zjec=dCHu(LL2;%*02GLe_tJL$t=oIPfJ#o~^7;wo z@P`bw;(L~|iEjp;z74HinTco0haw_9*a^vLA<4mDFg{-q-rVgVe5@Nz4MSySA~u}A zS`J&5m;J~N%hfN;tQYWC&^aiqkaWWiG>)E>R`5cOQhT~7)vT1ZAdI!%v@ko7SNX_d zU_zRX|^^TQU{ z+#YqHYQHwcWcFc?@ipl>e3XLw(nGLG9RgeD8J?@_hQ>=d*xdQjIt&MpiOq1jzFcDt zwKikxPxd(aLy;21=kC4hVB_N%<6!iPMk$pjN!NP6(0Qh`xu=sYuv6?)5(A*6 zEYZE?No0p@E?^J&;s$50(guDB7)+{aEHTTme6`%#4#8-r4<4f#$!P^u;DQUvevEPr z%BLhUO!%8h7x%`EWEJ_Jtjx$!khLS!3CNV0H*O5F%u@@Ci@h7K(I@=;*HvBGA}^)q zMLs|(T@$d77%6Ib$`9wTzyWUtaoiYTVt{TR@Ike@e0V;bc83!-SUPG_P&QVRMDbbJ zKY@!WU#~YCbD%jAc9&yOJ@;IZf1q~`tce6mpqYvdc;;%fo%iZG9+vOFIbEc8HTnCS z!k96KV@ux>a4S6PX#bF{YVR&U`VcbgGbkJ6VC1s>0oFW=s0{1X0mXDjW81r6(NoTq zm(L-;>VRatdDPV-QCvh;k<595(TiJdvhZZDzWWuftpRvZxDJ# zG6d6|b{$#6Jlsv(hqpVK)VU;|N6zO19KxAm1A-p^+ez=rQ_-}zGEE#4UW?So=O2f= zzwgysh<#KT`)Glwzg{$4(QvmN&7_N9Hz(8RMmuIVUZ6WSVXbz%1DJ(AyBs!?*EMRA z-9QNeuu|GA*=V?QC-{7f(r|L+He*dV@Gc@gF6rvkEy9<6a$}nWxR5P?rP^JTPYDZ$ zD|CO|BBUR1XPQsS4Ox`1g*{)*o&A1(Y+(t5t$~ZsdcY-rXiFLmkabB?iIPyrn7O9| zQYAdx9xbiUm#XS_{p(a>Pk(61E7K;viTzN$@P69p{Q)QSZ+5;2M1wx08EE|K_UxBq z4Ee8fq$K%=p#Kz8zdbN9EN9i6-Rti}A#;R@4p4Sl4lG{fgdx@i5b{`it!IAiAC1*DW;j12XCYX=(a$+9i$BWYm<_co&NBc7Wm=H_Z~fI@M0d`=2+Uyt`TCdiICXxq$X{=Wvf9N~e%(Z1; zPTiQ}R@FsN=_u)RaS6q^yMK&+F0$w+OVM9S`0@#IuT&oR>eg9%6j~3)c8Kpm5Bbq6wx^DudW+({ka!+q?uVka{*~v%tIM(dQzB_%!%f~ z7fbz=q}~YwHcsc2Hd6XvT2*zaPL^2{CO*g3^36_))*kl*o|TzVNojy;X=YYm792~ zW&{K~I_49eID8mL8#d}@nB5%Nqf;vG1`g4LWD<25TnBEH88=cQ3Iosn#&lHkWXLJ)S8< zR1qJOt{=O1kC<+s0*2-e8GGvXo73EZZ|DN5AT5j*h8Z~DtijVpnUYd^6ku_gsM2(D zOnP2>ZT2EK{f-?|ejsX3P}55nOs9T+ynIQ(R;bH#z-d_se8~DgK2%@f%s&|;D$;Yc zaa{BC4@tri3r-q%JEF_`k$|mt3Iw$QvQ1p?SpM5R5Nq`CK`wM^1o!WQTbJ1(`*Lev zW&P)WiWy-0#@-xe?$_g$j{5mKY^dn~%yW+9 zZm(YvzGT8b$JcxYf1gy${azltc?z6io4SC@V=i2KT|H(@xG4Bh9i`hcCD?J5sl&Gq zc1?42y?50&qK#sMkChuXx?I6LKXxQVk3}s6wXt4V<)X{d@ZmvjMekfZ?T&&R#BlGl zF$WJQOu~PF64Bq4zV!~vkA~o%+WUsr$qt~2m3yN5&y-bOi25BsiK<3GYLNS)JSsz$ z&z0s`ZqX83i`}$&IVW5tf;%>OlZ!RQ*CELk1Tb7JG1NRp0$cFV2KwCp=y z-b0BqzVe;T;d-QOo7zpICmL~gl`2e$3SFsUy$0(_m+bLf&iEjzuQuXfGJe7=|NU(0 zY^y&l-(Lba1QNO$&AwRHoF^&YgWl-uqGrl~cVAq0w7uf6{~_^5o&6>8U*Pvq z^c+4)!@ITJ@!|Kw!=q{>ueJ#<`!LjWcy3v8saN_Bgk7f3rE>Hx^(erWL>fQDLf_R( z$KU+vW_gDWI9=i4?rR>2Thl-^6eNm%owE1(%E*xY4^~0dLWpiCBO8jA^n*ek+ND0S z@^TRF$~0_HE%Uo?-C|4mysHL47of9$JX>e@}IT9eTe{=Rt^#Ox$%2BLoT2(-s@Xt+ka*0hHIOSS=CyFk; z<{u%3pLBQBNJ~>GXzNfj+nk|BOglWus$vdXTp0Wuh0Rji)Kt-YXqlM{m-$a?U3H-1 zg5Y5akz~ zYHkY|0?Q`oCd!SkQ>3opO4A7wY_@Fzx$eBtb$_F>z@oEv{eV;8z)x2{(_qIA_2Fwi zrv2Hb0*n?6TGh(KRL5g)dCsG@g<>F$@PsvgQOAUGNOyUR+I@x~Y$)+S8Bw(!sX^y7 z8y8*U{$DW&-}nB-AWSKHeQ5unc(cqEHTbw*eb%i9%lF%z%kAH#D4X=~QNNM2?(~Q~ zZ<{m9jJ~UovRqO1oq7F}fH}~|ZMF%|Yuq1q&sYV*cuergC_6!@B@=wQ?ni;7$O{ZgfUWbBI2+UloA zqLoFC4@1#hy)o%JbRWysV9lLm<-H?T+|y#-ts}O$!PT~NeF!hW@!kVu4+v7f7D5gt z(dwAqY(pb={P=~G%WQt#{g}nvEf7OFk&w!Y*LrM+%?B0~=DbT!-tc6$>(zuwi$zLr zRwhSd9ZS9k9GQGcIA)bFrm__}sT^SGtMhenX`K#-_4O|{0ETT+^R_X^{bucVkYr3!)X@2g@}!Gd3^)&J=31X`e=5N^I0^Ogp zdKb!OWaGVZo^Q$JsZ9~Zjk7GPf8NO6dZt>DRn3&Wy;;*-i0Y(_$7PSGrS+Pc#MQHX zem%cdwE7j5s(ZjM>`Ynv)RJ5r+WkUV$^-L_w>}?H6J4Mq@P38!pU3i2ZM=CYxH!As z3xu4FL(kRtJlySNR^9F@>5NEozOWs)=vk1%d-H}AUieU(0zWiV>dba=RF!Q#n-@&q6E`A- zm#BLuh>K0`t8KiG&C(DO#XU=9>*@eb4gP_9b~ydlR6*#@bnK@p_cp)?3bddoWIvdIEQ6yYrcno5OwSy3Sgy(EGLu~`??yi z44YZKo2ua9$y*4GnJ9b0M)d~RKvvSc;%W)oSTEfFm(l+Li-w!+$n%c;;yx`GNCITN z%_V^^C2Xz3BW-^C&$^45HIkIdnjQ-!znIy4Vn+i1q`I92u`=@9>fkhFdR=JPVk1z6 z6&tm(JpBU|qwmz@QGtYslb6xL4oRPZXs_;{bWvkY#GqHF!1%u1vcZaZ;T9bL;rM#* zFE7A0HSn8KYdF^4New?){{8BlepWCVZCRH;Is3-nDQRq=V$mefgs*M4t?;oF`NRZ}P9bP~@B;HvbE=w);T+Y>wI>;gdMw_l})1QW<0Bi(2B1Yg z)QCFf4_$p!Q>)>GmKywVB-mgJ6+hM#5GGFEvfL8qcfik26iqczQyKDp2Y|r8cAT+o ztdF4E*F&b=(hf^eR!$*e+Ad9ssQ%+6Fsfcv|L-t9s@~3b!uc1*cP7+4&6VUl5quX1 zUIKlCNBk7!$sCUiX}wtD_BTNc^d{+&3RI2b+!YXURDQ?K>9dERIXYd8TwlllgwI9L zgVDZYIk`aW7e=ZgV!i2{?@8J?(MYKRaACy1MR+;)i<(-eaWP9 z_(F-!SH6FNbK6hOyyfQ`K=ku(aQ;W<&G~9em6B^+I|YB4_$Qc$=Ls%G^T3T+siVaw zaBzZtDY<6W4l3h)?(5;1s=8U1EHzG$nDBKZ+Ji6>gnMJ?QM!Cm6Tl0fNAjt1zz!>l z>;CYnw(vkdh<@db(?T3T0XQ%o{Qi5`J+0;acO>q++`1(Xn$Z+BU~OkK6mzaQ#E`ze zi~zXJIxL+14tCeR!x^V7Z@r&8jp+WdKRM)YQvlS$4}KYA5r;1ylM^IFFpmEv!xIg= zQfKk3m^_#9@<#%rSml0B@s(R2eYi~^ol9p1S^j`TJV7n@$7(7FG_L&yTM;=JXhnrb z0(h)4Y2^fL{P4?3$pLB4(DN&%OSHcw{ol&|mh|zbNH5?Z&}?H+@@(o{)YuvEoe^~5 zFL0y9Ela(esL?K*?}CpZ?_mUOn3xI}cF-#bAZVi(QukMxVz;j(vo7e$ za;hYdtsh$S*XfHIojw0VKB+T&Y(>5)Onx3D3SQhH3wDvA3K`%0cElBS1{{-D=yeSI zVGtN)<4pk8rPuLgAj^L@L}sTov?~SKsIuxc?3h2kH9uH~AO8os0RDfa|0)kH0yhs! zoAYhB?_=-j#%jqZJ4;%3X>VvQp^B!EEof2H%Fq&!BnjT11yo~ZiN37M*q#LcdO2{|`BO+N z6e=&==<$A!lNJkmZ(ZQksl+Y6ddE&U>gzFJpOLNsGl2ev7)XZpTym$U`=0$v;a-&5>w{2wXyIStMZOkUAOLriP$aQWtB z(WnW_%Wob~KBf#~EDJx}yC|~e2>G`S{`+4Txm%ssqM>tj!1&0Nh9WVYC{^X%|6vVk>b*e1Cp>WSU(~@&5VFdqs{0R&L2R;!vjt&`<~$NfHuO;C|X+a zbehRmzQnFlox2|IMUdGePZyVbLM*%qcf!4si;;TUZg!hbbywhs6~ER)R-9Li%M0!B z!5=s9@VfymA{$~9ZkA^rQN1MfNma*F0Vp@33g!2vE5ag_H~nwxJYSM=YPl=(t{{-u z&hbc?1IC~VU)}8;u~9DVD_0~mc-61?ODT34emET({8YUcnx>mJUYEd3c$(fDf90qAP)mOw6CQ`?sL7)@8(P0dJg)idk{b)(IQzE( zE}O^ls)yyXtbL^}&N^!gvHnbKW*ZWv(EZ@!ADBPWLwVkMRM6N35RA1dqmYD@c;Nfe zLOy?bWkn=vL_X9ll9lc<$nwueL%00%{zE@<{iSy$fYMz@g!kUr043medqq4eK{ovx z$O5&P1=_wD$oKAjJ>c-eeZ=NnPoR+WYOa5ir}dECIrayZSA_4{2|NGgPr- z&n|7iAG1R8uS$}foBiy(DrxGKyF+Wl64HW9&NAoI{t`+cu}b}lmN`O^^cO#@VS8Fde&`da6Zw^qfo9{5oZ zyR`vfSE!>rxBWs|iv@gS%$=g=4Ff*)ZkJs@n9BB6l~TD6bc@?-X5Fe~86v<2>6a2& zx?$^|$Miv6VNLfSZm>RwsAdqqZEqX{Vd-l$V%s$DXO-OI4oq0 zB6aEAnm)Pddd!ge7NF4=oh&qJt`5+s{Gw0_^%ojtXZstC?l%1!jZ&zebh(c5*?edI zvCwwz$dkVzDKWc|LIPIvB{A$O{IGT9Vz7GeV%ka}mR!;KUj%425!MD^%QFA{9DAHG~x&n8F=j z{cQ^8waNI1{Bo<}js3%~1V5-+q>qx$Zyq>B#jMApKGar#&@jNZGKBh6F-Xd%jG-r`XY$h zzB4im|BGsj#=VfH#{N+edQNOGph~66``!ly*l6Nl5Vz0i75|5~c$}AvP+O{TwEmWA ze2%!xb8M@7jk=v2Yw_8b^Q3-prWN;`SViC_W1t=nz>;e~Qrbt*qn5*BqHBH>IZ-V9 zg>mook*|5=ac$O67}y>PFa7h9eOMxCzy&bR_l`Lv)3OQ8YM}PC?a5W4f2DS_O~|4d zIk^>^7RKk>A3%c-?%$P9%mjJhilfEz+Vzkdx*9r92aF%y8p+f8?DLm6d_`i6%&Drg zT)14AYT}vel|DCdaWv07`%>=~M!oH=di?oJ(J4^s zPQ9#{-siKjA2e*k7Qo-At=#O_AUE5)|2z3h9C|)x7n@!(u~m6KBzlF{u?OU7=MhP3 zEAQCO?KxDH#Ms5w*7zXqzB0CS#XKH3m#TFV`LSgniF=?bIqfK=86|*=BAa_7W%n!k zakZbieBLi3lEJ&)C$xRd2jH5q_3w%ZyFuY$prOtuOrtoNLaqiAq4;Vc|K8xWDmSUC>{6&eSpGNy@wb z6`m8eJzW*udL2DK$uQNOZAZTKJk>ud}AEnZ0EY9nBiG| z!?T+jH(^SMB4B zw%F&A?|IJ;hIu@2+rny{>wjalXFLCe)uK^IS5#%GapD*4Qhdx^?D2H()u^bB%gACL zJHFxw>1IJpl2GwnU3tl%-=AGn5-B7N6}{!_4rH56_TGcCNv;d}*V5n5+fLk1kU5DE zNknlVpLN6UVq0eDVu%N;xknk=!9y6e_mj@}djz`Rl=9*rN(5Os$4aRVe20hHTj=mM zt%QhTn_c2(KC?#8q4mC_E9R5`X(=18XH97cP^QDBAG<5hn4h)0;%gFn;6pp@9!r_T zk@^I$Tq+Mhzft@reR*m9+0LA7dbgupbUbl#u^u2Y(*dd!X=a}BPRDf1cI|z~QkBB{ zWOu#|vo0taa!mPQC>zU_ePpizu5RsI@d!)aQ%pm8Y3fv4Qax-#Guh-tm;v3FQ@Wau zHhR|RgM?|tcKFY7D4((U_2}|_MCHmXL`CM4cZ&TsEJH0~@#H^9*_Q$J28?5K}3In5O-=4cJ zIYkod`L(%&D7b4A4mdt0&V zAnq34Yx?GO^I{0z;Oq-@?chZD$kwMmF8vLGV)_9DVi3A)~#%8U2ex-P>OI=a?5aAVJNZBzc|1|;#V%J|7H)s3bVPt zweJA(rm4mjw*zIJ*&%#CXE_K!ZzosE-bhb$_z@=~g(;nPUAs z6{YQS*8|-72^jMRDX}Ua$D8z~(?vh_a>GKSp)HUZN^HEMd``Ui8V>0GtXe|7v2A51 z0V8gGu81!&Fv+_1`gHqQnpXMZF1}R1kT347Jxo63IS(M!ryg`EfGN>cp?tdV9&eKu;a;9C)q67mjf()Whsw zb8HlZn+44()7~JZKpTG{kNzD*{;k_Ja}yyT*_i zIlIfH3P$<2ZmPBgg!|^IXG;SHp1#JHuoV*L5uTSQp#0xkIsAN&auX6C{ad8;AwI)H z+l^qAuGuL-F?Ved(;uvvJvcOpu#9*O3L%T8+0jA>nj@^TEeP#pmO~D3%V63E#k1lC z+||iVuN5Lw&yKvrP#JT;M-@55z>)kQ(=@^Y013>a#}vjh<@_s;F+SZ+fa;%o*$>KZ zdJm!Gpe7NHttisxvrBb!+1V}@hr&8Fi0gtv|K7UUdb23TR=#I)aM_V$HE|JE~CxaU{e5O1tX`DP>r&RAxe<3a9ed6M0P^$^IY zQ_XAbA9Gq!0);f_eg)fiyBApX?som!nl$_V*_z_wOP>Vv7_4H%M>^9AbBg<7l|vl? ztH=`NF=$1i{dOgc%KQS)-Aavo`5Z?>{uR)Dx17}^4AF60h-wGx9Nm-Mbe!DcCM}E z{|RJkg9aDdW<|k4(7S&Q`TwsSZh$&zde?>NKjxUT=3?eO(08+)t&vHM8PVcrl3h~F z1&%`>yS0alzjC!5dt$b#Uypw#0)2cepEE@!eJ|uQC;!=!8jpupUve@^?n+*@EK!;+ zJm@qKA=gZnkyrR|;%#i8;UT);l~H))9qY-&3js@2bKZbWnBTGRSLqDZD}Qyz4xR(Q zdU!`I+vmTwvHzWqj%wZ@UlQrXH$^U2cwQgG$~cwSh<7}tsi@60Y*}}9tOg%|Jq`Q3 z*P&cZz&w#>e7mwpEl$gCx6DC_JVJzw6SNw+ns`H0J+Ui``>)@7AI1JAaNFX6;_>yY zx7)1P4p*_4=qsl5^#v~@amK~(Ei~T}0{wapkEAGJx@EK60T=Ua@KlBi!DQaN>&8qt zcGsR#tF|kI84-iyhkFah%4$jimaS=}{gsnKw<435ZD{0VqxHOo(_0=;P@7nIT>7qo zIn3AB9Ir>1I?`!Fwy_!~Thp%j&$z;#BZ__R=xFVH_t(wngLWP4Jf`rZY^2@Uwlyio z=b**iA>!T~t8oXV?qTC6lFlKx=!hiM_IGQ=eT?f@51jmUA@jqZA+_O!nJ;jK?pNv} z^%{;{{`f8ut#-5OHlJf+ZwDht(0Xo~esvS~l0`v-F9Eg%K36v(G2@czTf!8Oy-$93 z$#Q)heH6Q{DL>Tf5#DM$m|!dnd!nxK>+z1Jf)13C*JE$}fDBeJJZI-Wmwy!NQ6m|r zC@p!pxYwN1gYKp}=l#dOu5fPry33x8MLJaMsg|gcm&^3WinBp3-Tsrgo$}5Q*M|Hc z-zzb^9Z7;(<%_-k|87mm-v5WW_l|06d)r0bHryb%Q4nde6+r|6=}kaIKtV)MdWrNB zs&q(#Wm`asNC~}n0+CJv1eK=rn$RO41PCnzNC@pLiu-}~=Q~mG`fy;nINl+jzi&IY1m}JSH}?MXLnxB7)&gf3&vfF~>j*ywY@KW; zr^}#(V>Jl%*KiajLBi@+SK32?Z^f`t%W>T+-lVDBobeA8LqhM63x5v~)MMSTSrIA} zYZS0GV5*-wd7Fe7ZV+0Cm$3b3BT*li)`ovu>z$-{KULgbHVt|xRFupLsxIqxH~5|f zm4IY>uKs+%?NYf|eZ2cjWU!Gkk8csL#OLH1puS|0d~)Rzh?K6p+zp#MvsBa)xDsBb%9Xx*r|Qk#cvdX+ZI8wVGc+@zyi-^^{`@tMz??+c-i@k}$U|xt zZ`qt+x=iDkX*I_zarTF&PaXR;cR;II>=w{xhzO2_(*oGp8fI)N=C4t#)<4@RMlOZ( zTyrl4nP50kGjl}T;({DBoE5()`2J8zjvRIq*x~l_*GES#0*xPZ@j2bE=NP|DK;NYibawgP9XJTG`-O7UQP7Ha>L4B)N^zCa_W>GeZJKfn=`L$^=3x6LwXblImS#!!xWj0bV#5ig-F;X~X2)xy>aet(QEw z4cNawG*^0rDAhI@mC!f`YR&gTgz-=+Z6VIVl#}~v;1(Ge(9g&w!^i2ooL7B7*&Unq zH@j|<%JmC5Kne9o@BIQh@$7pIOQ{*A3;|06rb5c1^_r6}VWv0xEr0&wH+n<3@a7EK zOr1p$>tM#hn7Rh^Ui3yuLQmzQ)2SdP#O#%IHPF#lL*jO0&fZMdnCanFt8#@rwScD1 z80BptofFc1ApNH<%M$*=%y*?g3TM*2mh0`;y1BoNW`4sVM~4}Th@8nyU*)Ty+6qZn zU$1L(Ji3=xr$aQK^|sU`_7W1d*JYi{z)={sU%3GIM`fj;>SN)aRB?TNfh;d47tZpE zg_-X4b{#%Ylcp2yoqLfh;>zr5Jtv~KNgdmX0$1bLsnS1duLgu*An9flDpUt>PL&NXC5$rlrC%(vx zL8ki@^{15_gOSPsiF(Z&STf9|Z9{((>`@W4||K5|kp`fTC3DK7e z48P7sUeZyV1dy@kwrUVn)(P6s@Z?1&`8TVBG^02wdT{#@kAi81>r#&`C9xHrY8#A1 zTHi#OxY_vGO)^Ofpt!utmHFhe0-Mz@5u-SAMT;}fa5QNrDTH<9F*$oSO>9dd^ zw_wWfb7ypej(ZpSNI`}NRa0pxf9mv<%SoHYir|-)g(;kC-x_hD;4|e!Au)X%zYQ0Q zs|mjDwSQAfak%4T0dwzoyY36DSU?JIS-?v@)BSJdeb>!r z2QcEru^}jSKJ?NQ^o0WB+j-Ha!A6NV_xhqTu2e1o@H*#diW;3Qy}`DbwrkJ4(|)-USzg?zh~mwd3(w zozBG6j&^d_-+qASQSJ+?qqe%3O`7HyhohL0I?_8M{cPar*kUA4+NDiD<@3%asU0?S z`KSya!G?3qtpT?#;_P{Ps!8J-w<6R&aN2291X3rnE|C*o+%l}LN0KVcNL?G7?apal zYA=2uR0^C9oE+*=+|pT@O!0scOmKuZ%)Kw5rD%gN!k9sed`dy2UM)6yaw%f!>$YOP zRU&!c$%_Z9O|%$xo?Wr~#RR(|U|hK+TAB?iHN|@6(XKFNVpYx(-G(wap6Up5klz$v zkNs%iVbSch{@AHwD;?m}DmT=jw3TF3-PdASO;b99Lfq31KItYe`&uSdtNtIN)vV#s znXg6yz~pIiPsHlex*t_@oCI_qq%G7hv;?lmmaJ|1)|O>xh7n$ObIq<1hgOoA8xKLs zumtLF#YMjVs-nTS5{P|9TKf}G<)``un?zA9{%=0(QSDFbYl0%O-uB)Y%(RX7mh_F+=doTKig>V2$+757Z-lD z6R_qtE7Sr93*8V`k_HHzxgsC1avf{A8?l_L9L}C-Va7kR3J0GXj9R{}*A7OmL0$Ul znL*GDJ|&OyZJmiC3)78**!Res@wd~mbgPZRTkPh*G{#SWqn8Nij>_99oN;z%+N>`0 z!R;hy_B#PX(4nf%ILj@?64SCeb*XTVw2WO~oB0|0r%!Jk+ubo4ua{5Wb4eIgAgZe3 z+J)ucG}>F-ii!>SG4V1O0ZYWS?(y}k%%BNy(5*)TGWur0W0z7 z5nxd5e}v{(wc;}p87P$8CU*b;yaoOflYV*qbmm9Uy7=_Q=BEdJ?N_7JC%@^HOOmK^{Db3)Ps?qPXDKj#qr7cm;*Q%py zuBREL*128)Lu8i0jw8IzcE%PORoNVCLP?kNcubn}QfyAd3uCYpl+GD-h!EY+-+N%g}=>6`N}fxL;_gnTI#NFf=2 zl%&^S;M#W{RKQ7i(||h(aj{M|YUQ!s6=ucD=6CNs=Tl&KKG)yR`REguqu$EYped~I z6v{5htkaWr>v1ElRJ6RP-C?4ZiSAh`*qZ?}gM-G|jZT^l!$)bNWT$!Al!Yxgw|z9# z7*9}BoHO-KeUT!kY%};Gr?7VZR{0VEm?Dc5&aecRMs#84)Ftj1)1^{o;PJanqAQdc z#DC_9;8!mkaPl!l2rl~Q8nhD2==725Pg{>b6`6#emHMl2#Zm*^fsKqZ%q5<@x)$ys z4DwErk-fGy_KdI~3pXH&;WjzSjUmoq0I777{Kh23e7X}2{%QBHom3t?Sl-b!CEuZT zH$V$gU+9SMTuXFSQIZLXWhydq54_#1W(smOmf*jtm>ATF&p?0{mUQCwyQT?R4XVnig$a~qE z&bbly7z|PL^F#i$vt(|LK4#(*JlW>pTejj})CE1vg$-4A@mvq{q*0*E@gqx8p*(x2V}jvmxoI0Z=J#Y>oUOO#=EhITl9 z7)>TMQ%J?t&Wo|7-e?w&kZ7jM8lx5Fb)1WE5v*c|6l^d$%Fn-3X>q3gJ~V*% z&W28G+xKjb`tGr90qPx-PFY;Rv{liDl;am_zQ2+TJy%T$?(K*@&&|m-&!)ZJpPp@P z^<9)8%+`+tjPi|1tw`ixsrd~an-D9XuXYXbWj(hk?j3&-R=*oG%Hn2o8gTM}ee4X4 z%+d*XW3^?pQ`a2{$A*ktdgs^f4bQB#zTAD4R((UR*mcLBLWcjXG%bx;rT(F3=ze^lXS-O52qqd7o%bB z&Z(o?vu|7BHv*E{MyyJowpD0$AT(y1w|7c5Lq}~?GN~UcG)^(@v6^vM8}}=Xk=2PeCcyeh+P5xEos7-% z*qK`cYtMWq!%()l9{Nu8hr{5S$h{wFXa!M&1s90TlW%D=m6+9a`)W!ArcqaXF^ZNR z`Ojb!qq%BJ_;5DuQy35?av99+C_0-kXJs6QqD79*4RihE6A$G>3ul)*YF}2LrwDj2 zBrDOet!}?+ozH#s4x{Lo*;)H4c!O)$YtH zq#WYHk}HOBIf?Z=%OJ#lp-=zqTAW0#?{X+zid^Ugp`56Ys~KVLM8QoOoP*H!HYyAu zFwjLc?%a@`*fzy)=u=S4osDB1;;rTvn4!hCw$fXRt6Pd_Koiehv^%#KVWnPU26g{= zO~IQ5|HoH$5X$KI5lvL*0Sjf!Lt+gbuPW|v={EKOKsAvTmCe{XBW&*-#M+md_F`>YBs1s* zNWVShU`t+^(u7d{r@tfa6Edj-wTd}Q#kw-#moP``ij0+I_MdAHZ7?vssb*NsItHz# zig`Af^>z37x*jpwGNH;{*fT9Z;OhV1A;JA;{ci^v|LayoQhU!J%h-UaRR@ACAbRc2 zNg7k9(WJe`-X8FY%dJ2N%+~h3Ri|;Nea3io=JCYfjvvn+r4HaXYd59=!SGkVq4jT5LYa=g*wz!Qd z@Av8m2LQN5vEXoH7b)eB75)`Gj^IF}^U1wBI!6k5G5qJttduAn=O|ug+R39*6g={Z z=3ZJEaQOxUO?+UAizwZGW;}a$dSlfkzFrz>UlU!3~TYhqN>eUtAbWyBU#a4Jp>*5&FEc>qpo}v2uuo|f$!yUb`iTP3pO^|qr>2n9*AkBax86f#Lz7-M=F))u zQs>xnA<|Y`)?V>M;Oe>nAg-#@v#IRD40hSZJ+v{C+mHqCYEpEG1h~EYX(dlHIl0n; zq_;`)t_CG6m<2P|v|Qxf2T_1!kpmc1Rj%3g=O9>JEAOY=`jbn-+>-DU%c1B_Q$6FR z1$vh%R<|>`;=pvZ?R4mmp^=QH-k%v;_sj@zZAms5>_Mbi`J67S07Q83sqz_Mh$8$Z zwo8tYbX%E@myoS+ zjT0L8-qIGH(`HEl3ZIf;zD0L)SPeRZk!C|PRRuqLFzgDC<^5MaGgX>U%r4Koz2AsP zIdc;i!TwZx3`DM~BJB3#S@XVTA=bH>Gmj;j1@=;U{Zo@;$pk2zkx@=G`^R575@)@{@ z!#!&e-&@fcTnnYi-pJtzDho@A3P*nQq>V?f`_q;8N4zf@t_@#Za~?mt=X(LyGChek z_r3-`pis(!0nX&sWPsJ*1N{IrLl02g#qvt^sw8jv&!qcMv*0OF zA-6xeakoJ0&tUjm7>h8^-z*I<9POuwk(0K1-5t*@lR7vp80RzxoEi0g;iCXM!oEq{ zp+=mQ*({%`26~2pEsQ1iQVjEk&4Bf@--o}OJEHynmh{(_fi7h zmMsu_)B@I-xqXqnWTl(?{df*K1!(7cA)ma5J-+mB{6dhoOdAY-To(H}v@p|y?XM2T z$Muc(%0cITeV<4-A>{Vu=Lvn@^g#JvqyYgi2T|ono^bWG?!fvQNS!(`wNbPWJ>5x| z`M^CGx5inQhIKyCd5Fh9GMPTz*WPi7?~VmGr^}`$QQg%Wlu!#UDqf}D5A?3vxu4Z3 z4>_El&nUng^EIxOS2lZX_l1UCf>yxv`7tiu%6-jD6e&ZQYJ%qzmZ!fu!*&9g@Jrm{ zymp(63vaOVarsxrPc%ZqJ{a?3Y*iM__}~+^)CaRO-;X)RDZd{LXF0C(>p`l3DH^!| z)}V4}(y?q&CAmtDIc`vgG8$T+e8Dw|CCxm`nCVO@zqKsuF`afbRWAqbXNBoO4Rro)i(0w^<)%LS!9e#@0u+QQZC59=!^(a+s)UHTSc+&fX%m{rI-IKjoU&m9( zb=2S=Z=?GDPmJ*|MxYRwrK)c?NwC2ezIiruM+d3sZcgvetl3Au=z*@I|COfv#oTp` z0N-+2Y5ktUF6_z%+6m-faww>MNgW#ddw^u@klrJ`l`rGUo5R~}PRL(m=|pSQ^c2Zi z{j-r;w_MYjCo++D{Ga{91a&iN}uGkamRpFSA?#;#t<;&T|{}b(Jgc@gd(AxK2&0Xwr?0Cc_FZR7)(Vl;kQ z_aExXLGA8$g-9Bjr4^+AY zf1>6{3q>Wrq~zB^Kl1$QuaEx!q>>Ki^xs@oFuhHgEO%el(?`w{I_&*U@%}kNU*;?P z!n;0VYvoOUjAVLw=4f*QaDEk_ddNy#N0Q~@I9qm1mQK&+<|~KnwFzTwz{S|wdriIG zF@>Z(lAI!8=H4OIp1g`YhZc{3dWQW0a*r^pp#tn{3E?qtOmNyf3b58$wVzWfp;w~fg#zQG8c3?4T}lPoE{cC&6y0t^oPn+NsT-z0sQzAR>U7tVsuVS# z0#^X%g0*EUW4*02@(gmdQ%@+0aTYvJNa~COtc3!)%xkS^!iV3B>e-?6R$>xdNC3aS zhfkfL5TcBO1(F5~_|JD!u3lOgC@uzlKf~HK2UsGuu4w?ADX)=nhAj7@CYx7*w-C!6q&A(F)OuR$4B?K5iCQwZl*NdR!+i03o{B3qT_{Y6!y3UL-AMF&lA8bte+ zwWII$JkV`jijkC+$m{5e9YWxFEEM)^`GHG+S;=(6@)!?GC%$+ch52DQQ>?Tr`xrW88F0 zTYAE?t~)5Rttav9D9R{+bI%7c{{}MXe%BhC(5?L9lJn-~v%2B#oP~rl(U9{(%;r6` z?f~9uHS4ALhoP~U6H8Or#QB-tj?am(D=pk)4D9B`e)U~OiHfy7Btn-*7ZCZfFi^)V zxgDmuAay&S4C?XZJKP;^^$>`!)+yc>>vT$RTx|GMrOrk#|3A8Krz&sa<&V->{E?Bt z%{xv~bke-(iW1q*s~C5=Yyc2$BauT}K7oeGil9nz?sa;sKPLL!wb`Zo882BQj=iLd zgj(0A`!$Yp)cBlQ=6uI6tG0kd)HHZek^9hjl*Jw&J4iMW(*!6=AZ>$*RNi=n zmrR3xMf>$viJ0iPT907TqSvT6-5P;qKD<^kyR&F#Q2|xRUj$A zbP*hc?DiqAL@O+I)l75xb8M0JX|}CO=Fw+) zqm1kY^{OYfU>n=O4K*Gquo4iO2-&9(6BzRR!eQL*1(1d3C+$+6zaRmeKOl{RB|8R-6Ixgwq{tn0vn*nkKXBUOmgMpG}-Nnc_&_vsg@@{vrc=U(JEAGTgHi9^Lg#n!3=tJ+l(W*4-}D!(pD)b zDD^&txe0<9y##p4;?k$h4ILY-!o$r;IuJY6$~)6yT15iJU&wk*2mcq>lH|&th0}~& zcf#V|H|1m9y&G^{@16Rj(mA+Fo1k~nY9FjqY}HCijR-y~O=t@%uVhlW$Gu{Jwljz3 zzOq}24qFg9_x^iqf=>FT`E5vTPEx`p?zX~f3pLfKV$sR(D&(F?&Y>is(-vWXuaU)w zHv>SWM2<}wGe^nfk~-Wu{N-;7G^oFm*-oOqt;53vY#mEe|yX@r90Gf2BFgbD3`_}$cS+F3sAwc9&;VQ02Gxcb_==8SD z2(K)J(cc(z>(8YccNQ&J16z@X6Y!TvozzXm=FmfW*!LY!(62W4C@|)5<;9+ zWAlSPWlN7Oi)81M0Ib6AZ>(a&q%_#dW2i!>s~jIe3at?wSrz;4_*dLmvAG+N%B1WDJ-eqdg!haIM__&7 zPqZIuYam^ptnr2y6V_I3?uh&uEZ=Sf$Gx9$dZ0z3t%aACT_ko>T0j{~D z8_h*3x4&M31^v|~fHqwVF#)jfMqEeqh~L0wT=2l{{(-oD*Vk49--_jF;X6B&=O<5| zlw4k_V^u<|950v-LE2V1K9981zu@=e4>h2z8M|rMLft8;Qm~I<@H8UnW8*gIonzyQnGs*s}q_i8t;dN<;WO^?}rE^V@ ztCw`!Sl9S!F-7A9&wz~dko}Smj>7z5nx2FS7)EDeyy=TzfCL#c;aq<*$%ppGlHO=L$q)xC5>dq=<< z(zk+WE!aD{tJQ^v|8zO7M-`w)gi(zpg|M_}?rpJ;3dy?pK}b^n2xAlE1bE2Y-DtC7 zS$u0q4hJ#wNmAu7EAE(lz3KNvu>&Bd3a2N6q|BWqP*YqXNw1wCUJb)!&H^_q@D>2Z#mMnvQ(xe zm@&3A0&qLD+%e@%xZuvpkOoe^HmEDWL7`x?q{vHTANrSd&7|_gXG@F*YfU(SUW@>e zQbre37rLb|O>8C1UQS=akOUe1&>!!R zIC}Ugfd%W?2rF9)xHRN9XApOp>zh)m_FD_SxjcM88s&J&(QI8(e=^i)#bqt5b&_GS z0BS@_#qet+_I=@0h(VdXS*m$5Dy3!&4o+r&#le^5DpN=bjFsV6g^yRuSq;9(mc<*x zx!oSOmmH%5)0)k*ElV0UwR8c%N}*(_;MLddWGjd;Wz0=a8&c?r-~edbImqs`7y!y$ zH+9()k6Q)2fPxa71oJy~KB$!}&fTzfpC}Q1>smpbdl#boRsw^@-$GbJ3jGo}SwMj_ zL#da%n3#B0{cnc7cEo9Y+{IZF1XbKDVDLo=sbBH0KR`n8JxWh0Gs>z^1IPJ@RhIb1 z1em6^5x0V6AHmvj?3Vy&F_wG%?P|tPj?`;Vncan|vD{hHN$@P?O@Z+wxY4TL_aNie zZr{dD6x2Snce`_rFJ`l>-X2Bu7indB04C);>_sSIuu2_kjkrDe%T>ZmW6inyEZVkx z5QeyK)ObICOe>S5YRL5u)w4YxFmm;9tM*=9_`tACTZw2tJbn=(6#7ys#g(-4aXl2M z6;t~p7k*`d;KF1uo54QD>EalK)C<*{l=2^ZA$x#Z?s+d+S&}wQxYqY1z&~CYwcg;g zy_OaCoj!*9LYyMN+U^ENp~kUB;4JwIBvp^l9_G>DownJ;@@CvnGYSKc=T`lv`kM_D z{G$?Ippj9m3LFLSTen^S_lsS9>ZcJ=nLCUUAUNWl z+)w}Cck?D~l|w_29fJ-4hkLBs!MCplF~WxtEIJlo)*=VzJlg1G-#Ov2Na3=|j1c7f z@-XJk`i2@EeY1GsUZkK7XKx823_1I7ww@0~N~IvnQ(~E1jKxS6R+j1TR;IqSF}DDW z;6wqNmEJ2LO=KUw-IbLcQ{bj_B=hZUuuQc9-hG7j5!FFhj^bbMh+T;F&mr2bf)k9f zCEKH&U31l{X6)W0AM%_ajB`E9nLEK(+ArAKdwF4*g5hi(RS}OJ_39Gph$in6*&jse zNJyT1<(OdD)%Q@GvK1Gl&s#wAur+o5^1YEoQZy}_A;)v^BaxddY4t4H%_wFDd+UHQZ$;^qwmiX zsm_L=C4B4a>GeyLDW0>))$jhoTNxU)?H6~yzm-98UMW8_3m&V``U4ai^^mDe{jw8M zw1)i@k@_d4G@bHLiy-v<-E3EI5I4n>>aAS4o374UdRZ|9=~X9_In*>*P@I3;`9B~| zr%+bL)id*?Av9>Jy$KlXBa%`R-~!FUIW+HddFbn(48R>qq_Dl#WVSBD+;r$@DVs={ z3=D6xx6FFU>7vy(M|)?bK(I>R3ehB@^g$s6SDBlLig63GWG{v!c2)e;gv+dYWCJd~ z8MQ+l`c&-rnwnqM6aIGoLmVSV*zQ6=YW>wv0NKvmqCTe1TzE2(y~$WvqDynH!$VDk zTE>{zoP01ODSxcZJhuw1i^q?fTdN!>R%#f2y_({x|bwMGLwUfO}V((ach9Los(&8 zJ=aDZl0K&=&ffj)flG?B)wwyp;-VHfhg4aOI1;Z?ztL=Z*;C_VM2MrM1rVAh4u^b` z5i3+SN7%a~Bk+bqAklB3fQO*gU_d6Uh3RQF;O4!XMeSJXu<-Yv_Jhk05g#6!kFD`z zG(P?t=7om@@M)J`zI-A04`7LABL%z7%T}g zL!uLEO`2xk8VnJ+`(mIN=+GiG6X`no)fC)}o9i?XW(N@iUCWFip|kJnzHS3fCnYNY z0tE4g39#NZzoNkLJ(FAIwvRt0KWj3G**C#3zI}zSy^qL02eIza0G}W4fA+%w*cJgG z>n!`!z#d%yfc7`{KRW^gQiM!z#=R?|dxd_|Yo#l}=c2|`#szGl&{(1etzOU-bRWC= zC=0XzA(9G0`+p_w9%KUV$?gA+{T_Sm0N4MMtsw`${9pa4z404;1M`{AMmBoY#{}pN zyL-*&QElXnuW zRF~fWz$F3KTaRWro$o9^$3adPnjCai5iK+SR9E9H5fC3JR@lMp0jjq`i&rCX7e%!k z=L^BsFpVUuK`YeWKRKD=2yjUr(derZ!D=yU&3#oc&WbRfyGf7996?t!@3xcK3#cuB zLCx(Hhipf~3$FcQZr&jaTiHQ)2R|D@HZ$_Y$@qKI6%KbF)g0gicfWL{$4H21k31)v znZ@_GPQCoO9AeJ4Fx*9KT)AE9aVb5sb*IA$@iD@0TS+WU=o|nUiWRkneSkIYEDbi- z#Y{|1691Kyt9PR<^v+?5z@L~35p@W6LX_x*C&N_49ioDS#=1bjJPOsQLP~oh2j&MvcmyNsz}+b%wvlW&w1ADI`81_r*CklktBsuvT`LP znDytEM=;~sAm47LSZo)}#3ooEfK0Vt_^?|MFe9z|F?+meH!V)$Atx(K^QwB8o85f{ z7W4Qp$@OQxy!j^6e4EaX3xZ>|YM*?8Mbd{RG)orGmze3E!y@`hW0>3qlPFdP3kwJv zY=E#~Dg=3HTTjOj{OA#vi6??Lp>*eK;(htW^YUPLh+(O@3QfYiG5%zpo|&B08;573 z{)_IfSFjiD-xTLdU@mT}2kr*kI(FV5nzWcAyPV&w1 zeLXCBS)d}i>d9CEmF90-mCatRX5Es=elsmkSF?bFNE=P&Cf+Qty&!Z5nBNityx!~T zkR;C(vss`EFD=W5u^Ii7V8wqAfSwhk{Elr9dmK?SE&Dx^P%-}*6~3>XfTf%o%?vWp zcP4rHjSWpFwMxG^Us$e@z>kr4t&-B8Vyx7kLyu!O!u2#B+d^t1_Z_bL_wF z|DCG9Xq;9uI8l&E1#&E|jhq?V|lmw>e|{YdiRh)dEXd~*}|D=dUa8DxtOf@?&utMaze ztQ<}21}}Ns#e6r~E{8RRBI2JSLTY_z9{zXWZ^_5phUeKenSYIK$FKSSUo{`EeZ-l~ z&Ybkh%WlrJ)g?d=_x0R5+{^+<)W5lq|JHpy2d3%Y9SCN)p7FO57o4m)bGR9~j%7kf zP=1wbtL>vme)F z5%{JiFQ~W9xro}5;_iEDEb%GW=wb_x;Zzud)*QA6jg`pqj_X^g=40%5kS`&bE$vaQ zS#LF$7~lWRe*>xkz+hq4m2sv9y1F;f2$RUP5s%OiB5f;JUz6)qab;1&#x*bbu;O{Ac=G9#6D?hfDK?)O>j;G z54kRyBKOe#p^@fqf9?3fqbwR&& zDA^txbQ6g=0Vw^7N8(S_@a?UHtQ&J}rshEwQ_F3sf zg;6{%9txOLbvmeo^BX5ZB7Ra#xyzZTLJJJZ)hG7_oNGu8fpF*U-iB)d2zFLr^3*KAV-E^IG_5fpwsykLgi$aFsZOH1a8tE{q{454!la7`$<$t!j>R#kk zw+}A1IC#n_6an<_rlR%yb?Zs+xL7|Gc96Pg4-gRNSwSf>b`Rxw>d`mF3cpO{j_|)5 zDrFXeY)L2ZfFiq91DrdY{$j}U7-4oPT(f*@$AC32mreh1H&1i=eWjnGgz4mbpD6&~ z6(gSGNd4?Q>n8jDnn@i5EPh;Z zPGdhCM`B!0bn(mT7Htu>Lb5|MPGgY^Byg^uXBu4q=RYr%avgSbvt(5x{b?S1n#iyu zKL)MdwdX=ebx18$ijkx?8TxAvjaNK3Y3-<(u0nild%H_}olvdE;EOp|Cu<~*J+bR< zP`urVO0{O3+I0mtb5i+8$uUI$080ec00w`u3HQ}Xe)5;*nSY05>qu6+vk0!>0Kg|j z9P?=YI=}W3f9I*xYwd<#iqlB_0Kiln?^@;M0_+^(blKAy7|_xLSbvrxvHy;@N_jEn zvH>)qpO~23vYZHaiPTWhXvU{%j-sxGk9ZwVblt}(^l&P?UpY}5gxp&q=4bKJ<+}c} zFCL6*#mE1NIjQa`q+3M@(PwLCPpa>Y_t~P>G@;hF{dqu|btw#7|3;2_p!<&n3QAIk zUQQH5t^nu&vi}|Saagd;yaUYrYZxqWz6vqSE_KZCMs7vW39W@I+{DoAG7h8u6p2Yk zhKG(#?BiTMnXdcDj)Y3V(s?hNfnp`QjLAJuhn4OX&3W$i^kj!6DhDUTKvR@_{0VRS z4!K8+_1u%RS;X%tmu(JH^Xswge&&;=WEe3VxmJ_%f60L7+s}Xpj5PNIm?0SX;OCuK z*0wj~U3?Nl_t|dYjG5}ek?Tz#!(USKL$L6x3rFHZ-NJSi#WR9fMO~_F#L8+Uqw99R zHoK79PjGm~ZdEK%LIO2io6cCaQ$C zmSc+^bPBQDPew^+?;``~A4My?D_ReT;cP<2BOU>7LeeAn)*ggRT^{{bRn!USZazNV z#}}J$2V^0W1>r(!H)PQNn0RHXdU;mIVoN`Hk0NB;Al|7Z9^KLj%xXH!K^0PtT>i|kUp z!=<;ofwy3F%DW`cCFf+y3$qJ6QrpVgs#YVEI!edp=YwSgjKz;&s&RoSWIJ#b1clBK ziI0B-Zu3B|S4XE!^{cKu_-obVhQcEJ<-Q(Mj^Pcqh4p+fxRPP-smJ@il!p6RC3423 zne)l;05F@P4lrcfZ+7ctRV#NYv-y8=x;t^kch#NOUzPX5eqVGT1)k``;c_sT-JadU zq3kLHra@=|TsTjqDeow66LMcNio1tCU>aubS!0gJS+kdJiJNaD>Lr{{Ft?(P~7 z3c>K!>2vH#;NcHDhR9{>aO@goSM$JuOEm%{!Jq+Wtj7kDFg80q_+^ErD=E_|71^`0 z5+Y(Ep<;)PUZoBQMfUZzMfsmMx1TV(+e*qz&x@M)t|*KYC1QO=(s$TJV;kdxX8iYT z{<%3LQ|trDqO~U)Bx}?6LJVljw}bAXQ3;CR!?0AMt>AKT6W6{5Ze3&j_B_ZMDf zW!1vYPwG26Uc4>x(&>*=4X2JDy}k11H2*v1pK$u*zpR;m-fukh@%__t$1O{4xADL0 z)b_N!@bt~Y4TbAV2I3QAHa`;UPgg@^2>~ZLnv&!{d5*tsw<8nO_Rrk}XxIKzpop%! z+Wqi9=F5iQ3hdvV^u1!>^Gbfbxu}s zia0LKiFO`BVlR6oB?sH;;a<;{&`K?he1uKbg|@l~IzwEXx;UIOA>9gIMD%Bn=vQvbqhv_BEF%c?vLSc{8rYOq;9_TtBVV>%pL{?y79d_u z=y%8~N>}`_;JYTBVuBVjumjFPFFLV+&h_bEOBvzLx22Vx6fTMNM1K6 zLXY})O?!B_`&B8}AOAwk8q`r3_yS8GPt8zJcDwF#{#b>(I_~xjr&cCqltRzr%Dif0 zBJLE3Z*5p@S!txqe{OR-N!H)dz-zi+d%I7YQf7eTE*K`xT*}>u$Bmi{&#%9X`)oJ; zyyW1p7%thpmtz7k_!}C!EGDImRah3SHyTF<(|2~uUvqFfy4wob|#Pqx}0s;sd8&-*{0vNWH@3a%i#v69J$*e z;5V>^#bf?TgSBKCm+NQcKP>BUv^qs)FD;C?n@iiA?qRP%s|cMMw+nyiqTsH4r6J*D zH{Vob4d`$dGJz^qXX#%fPY6B3W{qBNx(;W0joeVXYjpJfLrplWi{0p+6c^_FTI>}& z4xd--neR*8YvJVuV{HQ;gC(vx2te8s#GDs>ojd0ExjMq)#}K#;LZY19sW+m-rDvW6 zz!FaDr6(k{b?(j*>y+juo5N{TB$Bpn6-w}bRrwbA@n`;MzS_w+yR7(?TCbCq0hjA> ziD}y2ro!WVol=fkb~k0to%|&+2YtmsLi5|Nttn*18fB&}mvcdj_@Sdy>Gk(^*K%y= z((Cbli=R)&Ae!HlKUCmfWqggCBFtz-@*h+5Emstc?`6B-owH^vdRhe+p?>Yu!uc3@ zc#7A#zSo6w9?kjzKeN4G75cd{fq+y7!P*)+Txiiyn2^CaxsF@qZ$KTFa}qj+^K%yC ziSlzpg{hehep27(ojM1wX5&*ey&89~1C4u`E7w;ThV0x+5(82lpTm>2` z3hBT&{aBBo#2l0rd;SDCU--`WvXuO-Q6nmcOn`UCSUv6zZ};g@DpQEHR|4%x!3_FC z0;6T?!p~AcpLyV*aL+;5?UaeLQ>h^;`b%tcCvqO&*p#byQ{2ow(~XQab()zi96dNA z{7{M7e#P0?(78m$aNh$8AlJ@+j_0&(r`X;6pp(*(UmKKXRqg8fJ1B|p?9-PhW=i%O zM~WJVb9Qol8(rH-4K3wYnR>;De_&Klll0GV9C}1W2QgNjeS_rikuDq8-|lCu{3;=y zQz}PBzn^cFS8GvktI&!Bc;x26+u#eLZa}uUomoX*kAtH$!Kl9V0DT z#a5Cvv+O1Qq*{dZ&2_5_%?alAHm-sSH|&pdhO6S#lX5xxy3BOrZ+(Of(OMZUBa3#rK3`ru7;L=uW{FO7oaU)tt9u z?~#t+gm#n$RZz~mJz+4uU$wTNX!qyhBdJoBzY5-KQtQXT`d3jU7Xv!V+DH9wa7qYD zdeV9yk^1a&#oe;Q<3jp@;_@i{9*2sXY$F==E=Fu>X5pjB^X>vD4|Q>2-c!XqCy{OW z+ZMAsEoXQoO1z`mYn|&0A3aKHpUrj<`i3%7*Dsih4)guJYr#glJY#t+E5_~i(wr8% zH)u2>W84=pZZA*(o`qWYkD5xWm*SeNuhgG0_t0anFXqC9%AIYC=l|DaNRJ5O)gF4UHw3|1~_dcRJ9+;?iVDjBFx0Klci++`qmS zsEb%)=kKH(p44^^DAs$WEN*!XF$SL;@JFzJ32ExRW8_nq90ON3lY&6yD)C@qnr@HL zAC}WzIz`0pi;W#+YxTHZV^CP{V~p+0O+w{hmvb*HtNkN!)kZQOE%Niu7h`iaHN7+K zVF;auf@+mGZp|}#Dr_5FeyEp41=+!hSB(mB$9@O#-H(iDi^)sSy%d`V`Pe=?k~^2S zJycGb$l2Bl^xfXxZVfCi7i2i6?p&RrUYhTdW$C7sXk9Ab4qq*@{eH^UyYtY&`H}kv z#**&qI9OgL=<76s1>^FWGM93@%XG8{f$RH+7gT)gFJ$wTQy?U)_w567Y7w+%Zy=>L{6Gw_5{pz!aK5JqmEV47WmC`VsF zEe;RVO2;kb@A#G9X!^CJ&x{Ls11I0^d)!BwfjVyrw+_RR=aYLy3l1)mUmqSh@_*o; z4erM1He0Wc!ZW>YB}>@(zVO{@Tz`N6JSiij@+;jQhsjZS+!Bj6W& zz6cD`i6V4Twvve!>*gd>#x~#-$jfUjLCIRY#abM_yHbg1P8QP%?0TH#Jey{WwNWaF zoNsm4=lM2jF&sI3H^HpDI>x0{HOY^V{F5Gi!;#ik(XTyaQ<^8OAjr`xRy{8}?6gXc z^$onrYm}6|DELO?4k770W38*_swB+$Lyeg99jn9xNgCC}dljn%F6fK-LgpY9Orl2ANWu;-OxG84-&7E)cnqir-%%KutA- z_4RrNGm-d=sFPsMTh#ml{BGgy_5-)CnK=oue_k#hC!(FbH~l4_Y2Jzb7*aMB!Rh2t zXA&nT<26pqUs}pU4x5fpsp>jsyxY!bN2aYz*;pm^-|vo!j=>jV=a+`Yi7!Ug!D~Js z+)oT*gPpkgz^q}tGPRm-dlZL~05m6!Zog>QDNylQpkVc=#yD7{fUK)p8JZp9VzT5x z_XnnE#8zExa92FJw|p>0?n0jO#R6rnZ~iVskI^T>dPd()4&Q{q*b5*!ISx!91HZuS z&jPM9SjPe^Ps_z{DbF`0|D(C{jB0Az_PAc_Q7I}a7>XX62NhJL#(;{VND-9|Az}cf zi;z$PM3G{lDAKFaA_Rz(fCNbdlok*WLZlNqp(7z7A>nPn^RDNP_ul6>Mn=ZYhn=zZ zT653&|INAgn(BPeUB}IIR@zp_tFJ)G0^^GW2L`5CSr(4(QA$A#BP@zzUhKETlHQ#u z>bOlsj9w8dlt>HplE|kFR9d}3m3+IOwoHKVntBD7oBMvELv6KT3kH?{od#%_;; z$Nj$U*$qP@0PT_0HFft~2j+5PjcIYI*_2GU2e{%marC2t|(lr+~LQZ#uBd%rT@rp|4#S1FY|% z-V*ukFb4tZNNWDF2#B}8#?YOoy{_%>^Mtj~Vvt9ZQvcer8ep7ulkI)`cjx=5f@Keptwp|{=DpTz-ZuBNpDvhN^ zp-qK_L*Gb_zU-RVB42W0wfCqtQJVx^)YvO`PGG3k>^m&~ptksse@OsK9e#vrn<&Mf z6w01nd$!-lMQ>?j?b&M;N^0g{?}-hcGcQQ|b3f)jJ;i3C`;4bcbYNDo>`D}KExV3m zVl**GZ?F&_v{v?gt@w3yo>5{&t%xUhVMa>{N&I)=8y8b_sz6V_jv&Rd$>vea*t9pbl!$2Nu#zd)6pyJ#Whk`P1^0gK-aYnQS~>a(e- z7~5xDd$0?6r%e0!M`FthyFUDPycWm4`vNoy)wMXwf$xL$S)Zw3G+Ftj2K4OKXmPcG zFu1$)L?L*rI(gk8((kMtn|)!*0rx$YlSrhcQFIF4doi+`;*Vwmwx@$-s(guaX-bmXf2`qxE2Y_440csskrYoH?9q| z$pIcG0s`7eZ#YYRc4zU$fv-(MDXyPxC-#i|DMTCtwJCK5zVfUgo7T{#?&cv4wcma` z?;T?u7~Fo>gnHD4wW8_IrC#%2W!;ikV;Bzv22Z~g3{)b=&) z-Q0>lfp?Z+|F||wmO8X?yjkY?fhL&*0YfdClbZdVSRI&Ge-xR#ypxn3dbw|k?kUsb zFsg>6PJU*o{W!FeX+{~3$X~A4XC4_AtfmIhochD!V^k{O%N8_FNM6ra0A?D>>FaV@ zi?5~@=2rx+iL^e)Vbx|Q?=*0FA9zxbR^d1H8jy$+@;PoCaCK+@UPA@&b@3a2@Jh<0l@@_@>JIk_v#?;T&6 zBX@$jXU0&ptcG(0RN8&v zQnPc|n^MI?yYd(b<7!_x6Nm+Fc*z^5b5~7Hg9TUK24!tSdb)1zF7FKlv_Rl~BR2|Y zpL6n2P|bNCCFS(!S`k6tKiAk3THn00aI|zfI>1K84_Yds*O$~#IYRb25e>BUMmP4; zU66JZSiwp?-%P}DKg2HIC9fz2d7LJl|Fm0&PG zIBiupv}GS+Kpx#Vg5sS8!S;~V2JIXy#zV$b2 z+IgU~?_vVB9T-q)uT=b{3V|S;Zg;Zn*q!dn88JNpP1gj;sc z{fHT~CjW;gG&?lRr>L(5d%%yZ$!|TSj2YOi99Zrw{o|o?)%BHh(s{L%su5l_HSH)D zPA+xnTGl+8zu1kv(LL61Z1*4)CSRGsE9f2555?`8=Y@#FPN9k^d$XIiAE%z#4cp5W z1^>h!YVw1MpP2Q|Gnts1vZ4gqvBsvynd^3E@qOR~_@m4bvCj};i{?ib;p{duLi zjL_X)e(Uc@`gKn>V&`yAV^HXlaVZdde4EEI{4*ZM$b1?tO-1W$>K{Q5-8=Jjg&15nOYpggT*Bhl`Qdl<6z~qxYDPT=a(Gv=& z(1wdc$~QZde^wot9|i4*iE|SwDdxF;Xtr@X4Spy%hs$iZ{`G~#Bg(;0tJ%BknkFAf zx{?fCnd@#1@MrgoS^f{0eyrP{!n}9%JdHK4uUq>8&80$LDMjnct3E?LL>A+N%}0h^ zKfD_q@-M}CNN#k@{tWb)jM=Vx(&Law2DRo|U4f6HW${KLj)ecHMbRT^Tg%@Q2>=?_ z`c(R%r~N10DKj0}HlcW$d1fIEo7rF(8SvK1EU#Q;Iu>Em!T2%LARhV+u>qTJB&_$k zqRj}TMbe4;wlfo)o*T5OFEPWub>0@Kw35y{11Y?&#ScDpRiATRklj_zv+YIF9wU%@ zLlZgBy*tj}roD+|o@;Kijf23tsFRfx38;_3HVQ{zS_&UF4Ui;9xh=uY1}gfN$a(Ye zdPn1@>KIvV)NE+B8hEde=^PIHe(J)e4g}RBYmY|M(FIVsW|YB1z-?xb&xro;FF~TS zAXaZYfZo~BnG41J?T}hU_2T0dkF*@{N0`M(XYMOKGs9VWqsz_q}q`5EVv- z1uLh1!*h%F<3a4?r+>hpl*WZQ7nT`8YYDKC)t(a4+6pm8+=bZ8b#*zKc~vZg!PZs< z&&-8%i!P7NGq&^(7@9vejkJQ^^pN1&q~h^IQ@NEE3XiVA@!IH`B>S&vcRJykyma+U z&I2xQc04iJeK+&%{U3^POBHXy-~jd<@mtDzQ}r!d)?L9^L@J})al%jD=t4NuRqYmu z+NqV5Sv;8hNxl&Q>2}B80rZG@r2-%l%@hL_K4XH5W-TBtMIBCqG4sYz)1?wcW1b-1 z|A9TdyokHynsHy%TltGa>;o4y$uY^Pw0N&R?ui#28?QFQp;FWC|cbFWXi1qi#of^%~+fdviM?a3YbIf6|r&4w{gS0hHN!Lkp&xX=Pa znl;47<}gR2zWt=t5v}fcHIGlqg-Y{}X;@T{I07(%A@O~hHPs}=J%l`q+E8 zCawa4B-2XOPk(&Oh?I^8YU2|qM$54W1+G4a9~^CkeX+@l0v0(AyTsq<|j2+qd%zzr$k8pZ28G2(+14tW! z0@&pJcH}%lBAezk!7}X7?D1=RdzeUtW_Ix!pQ9U+b%>C&|n|srQ*57)=GX;KTAQJD;m0(uHHEpm{t z3u4?IE1qiy9V=FEAd22+Jn0oHg_JXTrq|7RJ!_MabaH~=m~&1K9?9$(czP7uaNPux zQ}N>6%U0FDi8}QYTd3}w39xl)*v$=uUK9m>hWZH<1k7IVj6>O-6R|rdu>{^&hKGXd zgV|e6{T+`7r2#xnbV2%N%ir6E37nl?Fd)4CA0PqP9+-!DN}L?Q<-DC|k8c|I@jQZ}9GUjCE>)=c z?sA->Zzm>7;zqQwJM#M*+hxjt7U-nrF*1mN@6Kg}7S+M|Qr_hmXaN-v%i94$XSXfK z*WLh<>%UOe{a|pUa~}Vqc<+enW_@Yb)A>U`ic~^z-<=gLTZp`j-P~Fnff>eFf82L}flE>!gfqQQGNBh0#idw`$O6 zeUWXZHRFrqwUg&N!jrp{+CHqip!4}};{Lmc2Fj8X+HEQAm*Rd5Aj?db)P0ODRId)D zb0`S+Jk?Qw+NaZR#9=JHOnpwUm5%@lRodfwe>AsnrfJJWsISyc|^J^IUXV#*Y3a~!L@o`NyN@R6-x*5 zrAz)X^9Y@!Ix#3}j)ss`FZY^m^mOn|jD>{Gq?1UEX3fInGxw^glA_pPAV>W=Nox8+ zb)m(pWz@rVp!&7$b47ukHvcZ0O{*!u9?HTY_{K}Z>{*|MmsxOXpeD)rM^WWPo6=6G z-m|0Z@4j6h>_SP#75{|2CppE=CbC_hX4(B9RS_&$*l~i30Xt9hCQbVF+?*__5L;r# ziD1u^9dWKl*nf-YIU7OotjYI#5P}Ah$m#u*q;nbcA-Zavx;9D!5b`IXA!`E^h__UCRksE zL|A`AE#VHK598ZA?F6SWB~0+doPwgsac-$*MrGz|FV{N^uEYl$jtHDhJ%YI+J`AkO z3gqL$@9VNz%ZknE3(GN8o!zNu>NfpeA-bTcadDnu*O~XO@q1zs?tkU1-3v}!HrZLL zIQeE#R-C@WW$Mk|M}3t&m(b{h!o`mnou1k#kB?{U>r_pC1F9?Eulxck8Sn;BZCe)p zpQh#Gm{W)S@!O<3i}nlW?Lv8Lp5<98!?KqJd$nW5y!nw&H2yMSl*6%*M*@;TLbR!* zK*4akMClZJKvHQso#1v?wSd?ha#)O1xJ#c8kaT(;&!f!M3wsM{icJoN%o=?yCU{6P57ZgoHp^_SYG3QH~@2ZV0)o%ihL`RdZWHnXg)Yb^U zR)tR^YKxcth&xT9^%}Wz+SOaM6_KW1wT{*gf-e#^6kWq&ZleWU{gO8_fRCSEV!a3g zlFR6E+7$U<_}`Tz!S2?KP6J zXW=`H^pT%!k;dCAM*VB;$+^L`5rwzu|6xA{pTN2SEO-|LJ`UVE3c*8LE^6+U~F`+UT zr|H5S^;aVi*jnf=u-@eZWj^Tzqb*~KwoS4J0A1eI?K2!hTwCNqS_>1yjPh7jP7w2X z_M;*9aEk4UOrXI4EGm^^?$1+X)z`}10Opr3qIeVOLVsux@GUC5Oco`|*7A~SUv&Ix|I=6aQ&czagC}})*ESJzFq=rY%~E$C z{n&qYcJlaKb9^qN!wZ9s%{^wj9)=hiD$odP5b;8FLb-Bdu9R5_hsB*)j^$mVbZ3tYWXedC?QX$TB@HIAeKE0Jc`%+yC$xk8cy~&L&IV%)Etu! zM`nTVt*vIakn}m6w7DdD+gQ(q=>)dsEIF|TvvMtpMRtFju6j#ooeTy@czZS9M;tuAa-x==h2Mn| zC;)$p`u7YOJ&X;;?Q)$-#GPbYa}{?dVl7ji#PZj54j|5$G@M=kfPYy75MeT_Zw28@ zf0Ip46)`e&q)P3Gk^o_5Vw?@^1w+Cr!1Q+PYp6DP2jwH1ie3_1~x1 z8j()|l3%~1^5ABb#DB9TbtY*2BvbxhD)eD85KZ>{svB)#Q8t(p2!dNaAO8P)xT*=~ WSh`dh9#&c(sm@h{D+QPD2L2cS71Z(o literal 0 HcmV?d00001 diff --git a/tests/benchmark/assets/milvus-nightly-performance-new-jenkins.png b/tests/benchmark/assets/milvus-nightly-performance-new-jenkins.png new file mode 100644 index 0000000000000000000000000000000000000000..f03b24e647ee5c863aadcf4abf78e0ffd89712a9 GIT binary patch literal 45034 zcmeFZcUY5I_cn^%0TmfUlq#s8qM!nTv?wAdEdojefmo>m(tDyJ(xmqqq>7YC2Pqj) znh2pIHA3h}OMnp4&JH^7_|9);-uI91ob!F>I=Q?qjd_x1uf5k^`(F3D*W-UfTWv2Z z4=W1`%iinPRCHNbb_=txY;oGX6a0^iYriK8%Mq6ADwl7%CVm^vD=Zh*& z@SQrf&*#ePCnpZ<^EoVeEBCggmGc)F(;b$2CAaJhIDG;G0s;dn^@nzaFF8Mb;b>1H z>2VQQB$AM#;10jPp1!2Ycm4byEOdbbU?Bf`J#Dz*_pdkLA3Rrtw*K_q ze2>rWpWddNJ0N@l#^Wou&}-P%z_XX{qISXF$=3um4^@r5NwZi8?xmq>h*BPnXlu&I zLo_Zu8`@U!%r z1AAQ!7lH{PcUsZ<*AL~T!m-B{+wzq{{_Wlot3JEcx6PAkQpd5K46;R@&K1t(W*blG zFoY#S7qdV{E|VC>B1pqChhn2+6tCr*6fE5lQafmV&huUB)}JS|ot2d%UrL5U&`QrO z71twTvu3oI?Lh~YjlZ7so}J@}(_?ioxq3htR?Ne7RWL9q`Y0>MA2X3+eL&dA z!ctfXl6MM)f90Z%7(6))gRwl})AA+-+Z68~9VS9c37;T~#*85Qq~5eoQ$g%A!y%ub zycID~oRh4b5xxI5TXQE=$~!fMl=GjQ%hgPQ3TURlZtMGn`m*h9fk4=7f};fTO>7Hl zNbZ5RE0@V;JLCEFPG;@j^2dlYQ}&OlANAR7kka2Hwu}0WH2hdhrRllFgqd&4H3R$h zA*b;@3VF|0Ce3yR4ic~L%Rt3_W8}WQ^t0m~x-3)|CBn+liSOA?Nopoi zVMZxqq$254$VbREv;XD~Lu8F1c!$;nF`i$fu%_NdE1>BzslY6n@(UG;o!fat`mL2Q z(T6qBoX})aQIM!1=DqEeDYz9w2FoxJQsxntXe5;u6lVGCj(dJ}nRQPYb>L+(xA1tg z4qKUFx6TNi=FMnp_}f?SYwNSUa@Vw=kkVk~7h)xB9AkM^1fL0;1IQVp#dC&!OPQZB zUeZ^@PeP$`^~kzdh@12kfxG|*{xe+B&|<95ZnyNF2;uFGb}NF6f|0$8g7jy!e$;>U zCt-j=zR)XvLGhPHYt?7{Vf^cXH{5w$zq>RUC1 zkWVj;v!C^)ZkY^){KiMv6?=uw!bv#R?(UiFvNzXVYn#t*n5EXosj%pYW?Qq=D=ow( zuTv`J28wlqIfkWncfOpfp~*6r7Yi2z5G?aPt0^OKN<7`b~!>!Mpp&+ccY@ zken*^N!E?R=N{XDlk1uZE3r-Xh*Gl$t6D8DeuTAgkcoi%g84TpWJnoTEftkA{wjwCmsJ?O18Iw}cs?DzoGM0Vls z+KBp25Mf+-`R#sR<{wM)`G*fQbdn*g96WJ;Atl)g)r%Iq$R~382ZgVxGY^5rmEwK4 z#|KWTuq<8p+qr+0K|~%9mO9IUQWFdM1d$vOBab^iKBE$vt`LneHXqv;Nb^n3Y)N`P z?tMwukL9biX76_IL02DMCCjqci9H7`wEftbJlwZ`ROCKa2BUMiS<*gEPo!`?pO0s$r{g zN%W5(^CycMJE=N;EROyeGQugGe}8L>&0B9S_b>k=ZvFm~1yaBbS&Z+`1c6BLJ*55q z(NCki9S__+QY`2Szl+rwvj;&tgZZ+)>?Ga@5%*Ovw4A<)q?0dnpSRo?+n`bt`Z3w? zqha5j7|*V#6(jF6Q)w1DS8SJqq;#lPH#vpas4zjujIF|mC<%FXX5 zzB0wPOOo7=V#M={+K#VTChVAhn7Y?3ed66ju8Tnk!B@~ARqsi=O?CI2?YB~v{{k6DdzB1ejIO)3 zJ74Vj$ZvUrXU?usGH>m5I5%BpvH(4A|908@h9{Qql~nSX61HD!VoH3dCAlUD!V{B$ zspbn8XZwzIA*Dm8C2zy1jwv*iD@`~G&tb@Qu7&7$_l#0Asouw4?K2r6XR9GR_blkS z5-RI?u_<#tyPeuF{xsgO4a4_Xy3GUYI+=x+%i8!kY|8%Ec6~m(gDY=&F7DDgDEzR- z6jCppwmM^fM$d1_TlvLXfeH5TNJ|JJ@s`M4_m^Na{?u{x;6M#t#v68*yT{5XO{gte zQ=&chG@q^ zlt?zgb7rR+9Vz~*VHg?dvxsn1anc?EIO9XyK4{MJp2d*K&4o8SqjJciXQ{r2!2}a9 zxuXEKN_cTxbKJzOQVW^;YY$sEq1zko3Mp5uBHFBVX~J>!0jblbZ0tl00oA~?I(dkL zIz9EWF)PPg_ejXK9*M12Dk9Qx3}|;qaUpjBX+dtLhh!|*ynf)p4f9`2mSJ3 zkLHMZ-6b2rP)x6M5~`bV*hoCjMos<;C-{Q&0EU{rI;~K1w&Q-an7zl4#l@3(3$li@ zBSilWMVTKWd*}@>M`+fjmba-9lb-A49t$3~&8#0vtVEneK5|D&ePV9G z8_9lxjVR*aAf(7FH0r2=BT`1F!`s;^P5(RhxjK1%i`gA z;b#r6Vv;r76Nk}D>0s^4z}oL5el4}{y(y%8&~jO_S;?rznSUy)XTrr^X1NU8uJB{d zbn|7GxRN_tN)mQSirn1@0G?oqAS`tvWWBT3pvly6hADd}#u-f1>Gc_+cJVOOVi$$! zFWMAtt1{QM*%vqIsfWf_OP5;Lp)uTJ3nU zm)2(^gv@)JB$zfY%A-afx#>VXHexiXy0hy;>l|`yxO_d=BkD;eNv-H~E9zg#1u82M zEee~{Y&)K@pAmNrB8`j7^Pt12G-OOON?)`vYvK`mcn<6|RliAx-l;8A*B6E6vUL-G zA2*V3bnnf6p?(xmrgx$KgRbA*3gGr69l!Pm8B)x@!T>WzaZXS(U;iOfhms_wwKTyP;u zbR)rP8udNTF;aHPzm}Z6VSJ?8y|u0gc4s_5zdl?xtzNrD#{v#O0jx&9f%4V9Is|P- zgP%}o;)D;@t1OV+5512UC$VIiWIrj$k%4N51Z)K*A0Mjg@Ne$6cWtc0WcCrp@9=9lH0#Ck_a&u#+Y8 zJLnJGUg1UXXfELwM54>5>{v0{c|zYAC{fQvIErpH6^hY!s@XppLruFkjxKf4-`<}* zwOcKRdjr{r-gO=XI5Xx6Q{LIF;_0SLgi7PsdQ@g=-fw-1H|MExaZK~sy_k$FJoA=c z@YdrzwJ0nvq?~gy>%RYCGg(+@^a$jtP#x?h8%Oy49na_X;Uz%+Ri1CMYr2Sw8uzvY>JWJ*&fpwCF&lR8DtjEx`mK( z=G|N3%J07NXDEIWTu9shy0HmmCoz>JboOfa!Z!F*=9Jwm$Sk3MxqQ0H?EKNIbRlr5ygWp>$)h^sVtwo0J4 z*-7Je$Q_R>l+qcum8zr;R=YfnX60yeOUQM4^=pPf<}o=-$ukQ_BW$aB$L@yvN$-2~5~Mx*WvuhO?2 z!Po0wCVXWarS&Z^qfHb_4b^W-@i%4eNy$bb4%hH8o zF)R&` zrX~XAi|V(h`0oT$GXkb&l84Yq?IV~e357(*+~-5D3|Mth@NB1B1wjfL57?t}CuJ>lQKeA(`7j&o!_4Db-0+#)XA@$6I^l zYL?wEcVD8*jg?TxI}D4+h8_^zMAJ{lCk)=oDwr#ZRrflb?GdUNiTNJCdYZ8xuu|=a zR3WfG`x>DKVVu;8)T%O2P zr0iUkd+-S{xMs}PF|o?T?}ldupCv~X?iWPPexWmMjGl}j)2O;cd!gr9zV$CNj&J8h zUpvh${CVGiM1X{#zBAn;5;CMFq#P0k!7Gu4=VgN%2IbL<&3Fyu(=0>xr(nkXsp)hq zr}c-5{L0E7bLLNfaPR>B&%mpJcA&5?SYf=6BHmJ#e;}c9WYf`HjZ@HIW7nNE^pmxy=(0Y^!2S8 zL^VJX4EXG}TOQ$Rv`bbvN#keI0+Hl0v}x1viJ9wJ>r=l_M8TW1`<-77pW)K4c?71$ zngo7q zkd*VGS?b~A^k_wLS`;gXO=&YlOnt@Vr(;f3gH6u6zg1t`AFA9ylZ0P-18)l9ZDn!<^BHJuKPL<7Z&ukB8&b#kX!a8 zF4ON}-VSLKu9o7P;2xF}qJ1+%5^D?G!@A$&k{qp3q2J0K5tUNe*>!b~I+~yF^ESne zj+H|D@u&Xnwnvew)lsOJhlH%Df*}9k32Yi;Kye)XAbEW1S?>T;Dumjz*0$bEem#NH zzo~?c(E7ARsu~ji9VX~n3DZq=ZBxyN5xDj&>?*)R7KtGx0no?TrfEM^$|)mx}?Vt_JtdW z!XcE98J$4lsaI+BX#5_u_LD*>o4_X2Mcc_K>54kC+vx|npO)A*&nAB2F-XXoc{c4M zcj)HXv@C(&Xj`$PL@Z$0ckU}W0eu=fBG_8Q%%zR8B{@(#h_jq@XdR77(w$J8p_vJ< zxVsg1>QdTJPx8e_)A!dFH*$WPKqizS!_ik@+;*_q5G92sCJf?kOh zq-)zA5RPnPlm*lgS7mZuZH-~$P*CIxRCkWSwGPwmwHHru3EwPC7C`Ektm}Qbr}fDG zr|r=~K8c~vX8ILVABZKNm}4XKB^#1o?;l00&|gvBFk*KYCc$5yjhzXl(i!6Hwzv)t z(v&)`D3D8NnQI*Tik(0l+r^DnYg37m?Ly8A5cjIzH9$+Gwm4lpXO$c+#Klaa@Irk+N~-RtCWjHDZ{FC!No-T2X-PceC4*`EY1DZ%gwXL z#w{L28L)&b?_h%G2BG_WEBv%(cE#%C4uOg!X!GK~J?p(h#cG;(S*0LL3b$h_KG1GnREMrQ(lXEUcW@3 zw8|ORc7^^FTR?WE)-sme)|!+h-BRxk-RaT6F^;~4`wS&J85esiUIyDShY^b^u2wiF z*E^YX8KuyD|Hd~Dm#rcY3kN!Vs{H<=b%~?Mc>n-!kIpp5sw7XJysm@`9MJ8q0z zctgGO`I2r~7-p^K6ZGwel`2QvC4A52dx@u7bn8bi=QMvM7%WsjGjU`iTlJF&SA>+= z!L?pP^y33Dh4R%K{R#7tMtSa`Vk6NFK|49~_PriEb}L#$ZimO2#;f#Q4vgh1N6dEu zx~G*&ttzzts$Ft>(Q#>;&%N}sOHN<%rOt#fD&}^vzwAKH9{{02m2lMid$JF+qNY>d zq0%}$v%W^2@IOjU=%o7o8}ZTJq~e*YH8JA_6;zD1ekAlfy}Pf|%UKgT=K|M-g1nw) z=y${IOf9H@UNv4g)+Byr+key%1Y%+XbUU#1H} z`rFkyul2)e^)WkNBS0{SRGZW2NtYa%L7QC61M?@?cl9Z@)Pp4GxSMQcJWqZ2On-z( zQw==Yvaz~_Y=w*YF-yH*mL{HMmNaVXl3Q>gki>>8K{f%zFV)91fePdavfkDV=j}GqSbd_K)2T?x+iykF-M;MC`1C#ZcXN97I%mcneL^hH z^-0$L*SB@^_q>1aMtjf0n%>bB^Zn%@vxUVz3Rn|str~OMowjT= zhvybs23$g@ebR-iW^)U_zC;Fs9f$vT2sdvk|yLq|L13Y^sv|u=o-jARUXDTYNZsAURgE>!p+fpX%n8pwo|jaY7WJ<^!-9?Qj9niLGd&JghACQm1&&B66q&y`QK^opBV5EMRApD zYk1Ubb%ArWXM3FTi?;vnYe?xU7?5iH&tl#0%=RCo_w+edj&|XH=DnYa7p$nR9P-(1 z&OzSx7aTvR0)F?AOEI_IkGj87-d^nYZ#BZ-Z_6Bg^du@Dp}7+yA>&w7@X#rV=cR9M=NN00bD zwMjX;k69c2+3xQJh#!TR|2HmsZl7WI25*x$@i@2RD^_d03dleNjQ6<}ceiKLsYL5H zkGfatSIK0VBBUNhRTx>>{*nJ+qo8}dxC?8S3-(B#1zw-AmKNn6X2q<|$be$gU;Dy? z(@BgXz?+rbf1b}R+4uZ^@)xZU2CM7 zP}BxZ^^?X0#SdKYwC<5(>|8^Ou}$*a4A}X5Z55Vt?3I28dMaGN42A*~v!}3s93n9D zvPBAQ1QtoHn3>EfQQfQcQM7WoM`a_Me4X}wicybaxWrK{O77k@G+8@R;a*>7>%(V| zQomFizzWJv>sR znv%sQh2xynNj+RFd}>`^tJ8STsJN1-5kR?s_2@&U0m$@*;D*2APD&$msV+R4cv10u zC0!cmVMdhysoER@6cDWkgvS9|{j_~!ymq_0&m*P;q5U0|SoC%B_RQ$)>cKe=beZ4J zTpvqS;C~u8&qUFxXD+8(_2&F)+W-mn|*J)z-P+HhE#4 zs;wW+U`k(;e0F2Ykc<{gQNiE7oGjAz*INEiukZb57WRMqsqYx6zk<%Kw0F9Sr zP5`EjF1_Z&=SwYsQ2P2Y4K=~|P{^7YLypq-rO%F!IaTln3>=^JgrK|vJoyTtF+Kq%N}3qc06Y!y8ES6UYbfyq;Q1DMK(Z?~+3 zl|qV7LlN?v{D#S^3wlCzt+N~>COw5ggLT2n6YTL~u?SrYsosK?sNECF+XHPt$V`s` z#i8K1nTRc?|KW6~i;@GHx3g%aQOIh~T3!`E*nMcBL!S@aq~Fcn7cpypJV!$oh%4Hq z{)4!}i0mn>6akMrBxo*B<#rVMM;dx{(fHNhC|a07FY`YIJ)ywhfkje`by>1LOnOOw zAd2pV-0kQhAg^ey8z%*yZRObtByFN2{2;23eY^iIT?-g-)oQptr(?ljR>e!0<$T_M z4Cw|YexQslP)$~$X_6*;Ba&cAQgGA=GNX2olk9}WRNxpz#F%syN;6v{|L2iA%})Uy zuuc2t(^&cFW{oS0q-c$2*KYg-FncySgYqplxpy=nNcg`AAUj3Ih~^nOWU~g3)3+n58P{pF$jJ0zi*OfT6a&fA&w;Z6cKa4_r3>`rs9{V7vuqg3{sNE&$(t z#%Gq24OQba=l+M|>nHffRJZ>0_P>y={B-r^my^Q9G&ql6fBShQo2B>f(RI?MEG1l_ z+w*Ia9Z-AOcbRSb;`5(CF2Mg=lKv7dZlV?@p}h4w6x2TQm+|KR4n@q3e5rj-LwO~M zLDD=xl5?N;^A*MSd<66C*LpPa)ZmSlH#32A#Q!JGQ1@=Z>5h}K=4B?97wp7H z-TtT&5AkuVQjifcyKY!l4{eUwnO0yr=dGP*RJT_U(rH&#E^UbuiK6-1f7hj4cyNi{E~8k396Ewrn^|7HgQ0mmrHi^B z3)*Lh6}9)RmzH-FjkgWXD;9JLK{eKtwq!S?T|Q?jcgRBFjwSc8`Bz&s#<8l3HVhsz zys|I0JieztRsKC=34eWwKCI>dc z!lm@$RS0E8_n+g8bC&C$@X8_(hAq_(9aC3@Zh{h4;Y~Z&c1t78cGrY;3VTKvq-CX? zW;IUX#iM?ka`$AEZ>qk2mYn!u|5kDm3%^Bl|C25}qw^e4Jx`Y#*?kpL$cTA7Rr@Mf z)=-zY5hxS$EU|VaWv-ytRYqKCZfWe=H~Dxa1dx9_j>~ddlk|2zUw%YugC{FuR3mg> z4rzIMIS-skKI4_}34Q<`UOm4`hXXGaqiuR=^bBeB4w_c!4ZQSd@1%k%Ii+1h@gP2- zbqc+(+#(kLs`x!D`$==!QL}QY`njvCkxeM$jUl@9GA(Es`qjY;{K|Lf)D0eU42kPF zQv3_GIRCWXEg*kAY5Wl>(g|&a&H_$Pf=>Mv;*T`g%9 zfP^5TXZX%=P3pE?I=g2YN;e5rtLN^Avi|bP>!C~9I$M@2!*!gU);&krMJ_Tyqj3oz zFeCI1h6-x@o<+$sU&$2+TEY&{^f$Fk8dgIXhM@Y*h&2hqBDRo=MsPj!YR2~^sjfk8 z3c>QFq&ZM1{V1$mgT&3QbPAvYg3~OPMmmBH3Rij1n@isuth!?t2_VWXC0puO?NC&} zas9+mqsxZwNKRmjEz=eWI5G}BQ{ff2;l2_vL$5mDc_UDyukSi2U;XBvWetcGMve%`Q{sd@uw=%um0SqS;GPk<1QLR)D8-#42rsD{!7d>R# z398pO~Ui^~im%#npoa&UhkrkCc3-t~NNc)EqzF>ul+R zXNNBIAF-Ms#v2V5*5ctKJ{L|#FFrhSNMuC>$v*KZJc2r$dedCKW5L^r2G)eMAg{`+ z`m+HvhmU2O1|+~>$NUb~-(JE$-*Tl9x)yKLaLsIBLo|boqIGuZB+OvDQh?0MI@bzH z&+TKZp&h@m|Je{#AZ6(ESmsk}ySp|JZWPTW{4hfzek$Q|G@q0U|HW<4oFmobxZ1Hb z&+scVW^M3?07Sd{2g_=@uWpE;-N)%wE+-{Q;2I3*@;w1AsE-@!vnqnJ9#eY)I#^yl z$y^ezX80|OILusYk`=dCIO_JG#~oGMzheTCeLnp>rP_+E_n zSAMbLDqV{(BT(tVz!pAHkm}|MCta{v#x?A!ul=UGIw#vU|Mr(`_^FK(USSy^wjGxn zJ3BKG{J6M$Hboz$WT$GCM6gV7Us$?epB%&o=^35Mj+v`Hz4Q4!WjF!0dRMr?*j&)- zd_cln@frPX3B?52_$EPjG>Q~LydDM_TqSPMifL&K|Kw94=k`spE*Pa6+%R5ssfgK~3B)Sr+ zP}T|^8?7$khXCh5q%Lv^r;ifsJ#Bf#Jb>iqK*;)$oTcR)PG0fbs^4kpdZ2$10W_-~w`eOk8c9vdxJI+c`qOJZ8*|r)k!xn&8 zdNNX1>4pV{<)cXLyYZ8 zeRCAw?=qum|8*t-)XDLf5(r1JauA*@Q%l&wJ3o{YNPNcnicMUf0~ib9B!U;_OR#+F z%M7f*kD7jUR*TX-AgGoFnOB5K4PE`J@09R+HI$3Z6c4}OTPLbfRu2?jKR9wG`bV&k zxbBpgyvv>&vu}K|*`$k@(=Ze@;JQ1{eJhCHz5j~eydeW1+N}f0AuVVUuVbO$aKz<( z=V)YuqL$7C}m=J5Ob`87Ug~ODI~}nbcbqpY4J&V+8=AO{i6OFWQJ5 zhPlHqH%+Q-0{lIpc?vOWW6LcMs<8sG4=gN;v&-h;SAHzbAjS8gHB)le>PIB~p!`)ej`cVnX7@64_Uh>ORUf znjJYb+>N4EXWpil4k4?*FwvAqQPbBwOWBbdSLLen@D+43e1KQ%fpU$}TO zV>pVtus!sser6 zH6!ME=|wL?7FkrNAL-{>G_+4l$e;oEN>^eYEpvTv3%7o?J`Dkr5$`!yd1=Ze&1(D< zAn)C(lJ-Tk;O+qhBb5~S?1bknB!LuQe{SAebgb~2)uMu*bOj@Sq%`#3po@YHV;VVz zh8eCp6}wkai~R91jqa%Gcp-Uhjf+&m*OS1C#R1IAA6B@KLUQo^v42;ZzT{GQf97(; zr*txbHrw}mS^6X+--8|m`e>3FtEYYq&YD;_!6@$;%0u+t;U@#7_M2jcd4|dO0O&q> zvbicww0ho(soKPu(M;=CF8Hw`#F?yCt~_)q}e;-)70c+=L2>+BV=*xt5INq`e@!xI~{me_aE5 zT^`SE(y~-b!hr36<;>0uBu3_0JgWF2w2;1LSs=o6)J$4L4dHtr^JL5kwy<%4zQc@{ zQyKI0&L{OAO9+uxh(X=#yg&8*6mDpd(NEUUO{I<~;;`fbHxSxhDd|@05!r zzgRB4Jb@dtCABo8yey58plbfkoHQlBBTMBvUo-Bry7#*|?g5lh$J_PXzEe*U6zDzu zXZOFlup)(w5&iV?K`V@Rp1tPz*_#J;64>wLWlq$r-Uv+C16q@?E%d!o)f>jPrFsM! ztQw$GEwcr|z0ADu^Xi~8h1AjyQjWffF#Y_T=KE>G9k4?$S-sQN<|?W>t;`xL&34Gg zuzN5l@I3rfMqj3^{hx})-YppRNjl!QJszZKU0r~i{1Gia)6%vu>sBvRMaCXjpMoxW z3^WKP21kJss4iTC)PGJf?}N;pyUeqyTzt&E)Qut`5v)Sb)uU{*KnJvUhLCvMijfu- zHBeTLcwlei`TegBu1EtJ@i~vGASoQ~@b0#4S_!irGCtJTX73#s<}BnQZI^sp^dJV; z_L468UOc3jPpNWkSN!8TZnkR>F_kwi)SGkyiGqFd{(C9%ebF195|1~U-+nf6KFud!)M`lOYo`1gc zpGmAg-v9p$*niDRuR6GGH;*hFW;Tg-S^}>*(Dldhp?ec=>>^V*rj-|#X}nbgRi!bKkVw9 zPFeL7cDen~Wm;Pb$9Km9qNIorXw!)EDAWqJ%s6}uxmHj?JBnOqTLGZ7rRxTz#|Ynf zsQr&lFcNJ2;x7a<{=h*@OFwT7(kK+-`P<4WrFogyWq1@%RO11eq4LIouJsM)$5gnix$s(X*4W7ey70h}JAHQl#+^P8B4|d+2-ZY&I-hq< zWQ=P1!fLc+o>2`lv3<5o8VuoE***hdZ&HF~=X&4xj5isIw8N0~m!y}}!{8*oa_Wyr}zL0FUE$o-8C8g+e< zTliH8v}VQ1NIQVqpMjl+4=-ypKq{XrztCsnP&JhHv|9x|6CL!Zww1e?cn!D;C;x(% z0CUWd^5=gvU-9zCZL(v0Ke1yZ5tm_YL^JA zT851k+1@Acb>islqI$zYnhs+BsaMK7=q3q1Fwa^<1uc&v9|oxsS8NNga+K=p4T(&T z>pT{YN#fe3xL}BLjZI%z4)kw-eX#L{&Vgav{hHl)-p4i!z!T;z&ZCY!sx1V8vD2~Q zjQuykV&m5FAdv&YU)jFdFf?%x8>A?~w=WKGZ2S(byaSGGyoa6P&9jt@#OWpQLCI@vdlZAvB49X zO27}qLbZ%~(Q#^H)#wyIkcd5%XyMvbQ>W+NY%1{CWU=XELQBM1?-^?~`H`SNtmT7a z0pC|Tclds`aj1ehw;nlgOyhZ3kS#(On+(S{Bi#FO!$S)=S%Pm&{XinckGt@2VZfZ>SzZc_e zS|h&qVTJ85+>^ts^tb2t$)De3%Ob+)D@#vFRbmea zpe~06csi=h3;2gJ3YhKzYU6S^}YaCWq{K> zS$ZBdLuphTXGfp+q9JJOanH%v&SmckLkods)GGb$kA0WNd=vv{y2{x3rv6zlKwCf% ze%$fut$QT}T4>wwZ0Ne=VRLwO{-KM3rp45j8iLDUH~W`TQI#Y86`pf;E;*Np#>pCX+>_i(>1h!Z%l3w38TOq znC_;31N0DN`>#`r-+Bg1B3|>H_@9rpxLUF>9FYA(Sg^~bkb3hp2Y$~KYiB2=3&4Lv zqP<#XcZYLkjR5W8dkFfpWcS#h#HnM*-nrFPk27pil`EG=7O$DzE0Fq%f6Lej_Sgyb zfJa1Ac1qlY(!-*@o1Bek?H7}uI&WunUAGG_m0WUkct4JHzEFY$1a%K(4J~!uy~yxQ zwg!r7fQ??muF&xGQ7|VDKvY><4(xfZYFOfgJMx_?d(U*NQqY9Tlru!;d_`=fshFTns^BxH$33A+_+8EG#E~QHE8{2`F_q3i*vicJpW-raq;R$Y-HAJ=fLZa zs20;&B?D?Wq{Jr)A&GRRI{@|0`|_nyo@YWCHFD_sU6Sze@~V}FA7d3;Jy6Qq=? zRRdus(L8v1ZeT(T0kgu`1DVj#iJJDRlwmSJb8}_Fy=%HDGCj*P<~9~_C7WDXiPTe7 z`qEoh4}%>*rArNNBRDC#2|Vu&VR#C!R0WmnYZe^1r^gKbcNA?I0hGo|#1Hv$rp_gM zGb^1NY?*1zrWb-fDc7i(IZ{GnNFY}-9QibT3z!+&W5#SS{5;*_*YDjw*Fs>N-_SL< z#!}@M_;f;1+O71ihBNfR^%U9ITe|{&^#0ubuH^tG?W`IK^FIjVkF;D}s`$0s6Kap4 z3B$jySAG8s;Kn%Pzac4Gmr{-{h!=+#dKU1KKjeJ!sv0$_O?qQ`*{LUQxu({t;u(JG zFMD#LAD``c7T{U@Tjpkrb048c?Rv9|sW0nx#Lq=ehTCU|!V@g)AZ`l%T}KFJidF-V zE>cKkmKJ?UxT|lmFBm0em_$i7k%!)g@6%#3j6cGK$RBMxp64*j2-N33`>|#sqI6kh zP6n5KA85`0N)0l!K18tM1^eyQdezaR^>@%s-&U~ldrzudN&u7uz3F)>KYv?@#2M>^ z;{{`~y)y?+u8<6|fjB^_&7Tg@pfA0A2%o*;uEC$GVjwh4-;FW5JV^d?jnOf+mK8@HKY0^5Q1NLP``RbZB#LO+fC}Ta- zrZ4RTrow_zzJ_~{N)GnsizOS>Rhp@%Q@Vh3c`9gp{w+E zoOxAV&%$xQsSPz+pXRs*fXPjIXWj}&PC`!4J-w>H_scE^5ae|OiyM~Zoe56Oi${^Y zX*4k*>GRS;rjyvaah4!3IhLEMaE8w4+KGYNY#ILEd<3HgX91T{Fz|M-% zD=UXuNVNO%3sSt%N*-r>)}~uXY0hGMaE|~Zg$#kopPxw0GlM2fWZcqO7k*_sM_f%1 z_0YniPuauXGA!qAHqN*So9didqwFO3mQ8%8&$1yGL*fS!dQ!rqzmDl=V+9JNOW5GZr`IqwJam%^9Bn^_wX%4|nb!_hiH>)F30- z4!TMU*0gS{VaI9S)Ythn#YoaIN{gk4x7&KT7WE+WR-k~{&5wCVC4#2np9r@X#5?Y{ z{GR+`th42|v71ECJ>3l#v-FOdpmc#8NKAWV+p?|p%mL>xa1awVTDRs6 zrSdBtYc;N{_rQIcUyKQbNWEQAA%1w}418<(#VbiNo%1}1WyQ!Sb`j)s!M-x&M4#D+ z!V6t`0hgF?iZt^B8xs98z5NaN%;n=C(MsT@Ax~^Epc5ZaNGh8Sg<(qXP|%MtF;770 za$76T3uQcObJtRSk*$Yq^KXjOF8)y&vq#TE&40RW>d4PZc=4lTYTomo`*1QF>R@2o1@| za0c7)NdV}Z0TvtwM{V!S8nd2kA_5g0whZWV*eAt)Xm^7Pz~jYUj3Y>M-K;^3*hF1b zhq|gbk~!ZcNN5jpzKiyPCyPDo(4o-ih4Z~#vu}5fs8@Dk7TIfQF8vw4_v0>t^UA0O z?iI6;!?_roHX{MA0+qxE7)I6r6mS7G}V zl|Z6aF;a|lYLqz4__pT<9mmGVu+H>wn5*|t0eoDaOU+H%VEC(6Ss6?dP4q+ zmaDxHhz{8rNF7ylOOKkfEc{~bvcmQ9b=}uDGlPLLw|72u&7|n<_{X^M^gJ6JDF!A3Dm)_hwJjI9~MCGW5wSzh(5KC-B4?1 z4ED()*D0wWa9!^j)N-BP!2&|-N)1iE<$zK2J5sSVqhlE`>R#>x^zU9I4}=mAg_U|U z2`e>OAp*(o3in^uWvYTT==TjXLvY(#0smaGKOdSBBqF^K|k zY^H6MGRZxv&|a)M=tbknY#h8sRNSUBoubfbzy582-@Nb_+g*1Tkzdrd=?DrkK!aN! z)lH0SU9AmBEHKC)#w0yrwrK1LKG^6Sy?PwkTTn`i;Hg1MJ*AHFBd<_IX>L2dxS_aV z_t#koW=S=rKh}I7c#cA>A2#jQ>T`OrP(cZ3lpx$@KAP{&dyc)#C(7(--t{bf{}%JB z^TDr%7nTwoYJaJ^v}|nU|YR(#Wt1f zj$VpbBk7H^+cwi79xg2v8@E&TN#9w*cRN5B`I(e{g{;dH>!VQT(Uo z2%euGO2~4_3e40`FXormPx=r}--O;$0|Lh4YH?c5lMXMw))Y zgi*xu(ro;NU_v}R{N%`IuJuO<5AQIMmb~DSn$(%KGeXD(vOf`%JxP%$yghpVtxm?? zh1FdI?>POz+XiKjua5hH?;o5(%aW3^o~eGQagTYhXmK{)ye2rZN;W9a)v$lE-mIz% zCVq4rt9WntAKNi+Yyp(ns{n^MirQ>0Ks}ikE@tq3Y=%F1w9spNTDCE$6JNhwc zIm8C)Y}*=DG5jSDV_7}`ue%M84Y6Slpk8d8rE;m`^LmN?4KNzE=4G1y!F##IlNO)X zvo5xQ=YQVPTmp}F(h9}|&EE;T8qW@=RR$FY;L78V^9I>qK=<3%XPTqaZfllKDgkM| z_z@y?O?04U7WBK-XpLjfL7vcMdR!Skox_?RV!UF)N^#Tv5s=0&i5TVOZ}kWX3VX4V z0-YZe_)jMg`x=(H)l0h|yI$<-vYy|Pr`m*i@VE_51d+Cu`7w2U23dD{kVv|gY28jC zx>Uja&g#ZGdrpHJU6_*SF*Q0W^C7NOBIn*=hE$O93-m$ZJ;mna@j}27-nuN@TnRL!^I>VX+|bE^#sKz$ zHJvv-pCxv=p;q86&%V}}R^ImgFuE*C?zB?e95R}`%-#6tB1;diRYg$L|Ha;WM>Uzf zd*9Aja4aArpi&hJO+W=iX_0OLQBjZ@73n3Q^mfmP*bopAP!NKmfV4?cH%{swh_HcTlN_nBkL^xATV-!6BTr*Ky$WmT<*sPC& z@t8$!w$payNT!`-i5#8fe$T<6T2cnuNYYNBp;WF#b!qFG)>Y77yRH@}zIzt^D|)K}g8ut;t7iH5J(oTol!<01N9Z5e*#!*t5_CJJP2$wIzM zXW^+(_IJ;5|GAfY(_7IFGu_A}oPjyvK+OzPb8@yCw`q;Y!dD2AV_&~N$o3BA_G;G( z+3TejA0T@N`^j;x2kC35v9HI2*ZRN%dF&R#PnZ4kf%rKN6qfd-RAKmHEMfuP{e}o` zv_a+}q8jvyNk)I+%eq6RD~+*um&d9YdnZ>d|If0eu^=5bo)qF z^bO(&KR{5TXk1j}i+C2=wDnoA{Jd!tR zRXHaJZ?*G497XJx@g@dsJ;8tK9>FhR2?}7wuU|z0yT|9wA@2A+N{n^S;MsmGaCwyQ z=07nPa+lh_Z(?81YG6m~9Bf4(R!a|VJ9&W52B-a+o}rjUlfpvnB^;s0JDiJDHL1Bd z>zJhZ#?z-f!&Smr)%WbT*NS=FUC=HI$}x{v`!R5dgotEjzr4jLh=bi|j84r(xA{sJ0N%#O%k zxEagLj_Sbb_lThWmgTzB%L(x-zcxo>DaG_-$Wo54LEq#Mr~_FGqAy6 z76%GaPA?7mSmvINqEqy|>9)qQY1ZCM3hPvylgOHF?d2vdqi0W~Ke@XTd^;*Tf)bzA z#MKL;oGDa5%{%7KZkl!1^pshCtxo4%8dZ+nL@U*Y(A(rc2yTYiOkj?9jYrVyvvTuX z{D4ImTg!LBT7;L^;+ZQMOEvi=FzC4tncn z&|5^%Tapya(<&e7apkg$8U2d|JR0YBiQ#9QpT2f9K12}hl5u$(#wXAAxY;OG$apid ze|;mpK6-xAThqZC9SsvmxciE=P7}_IcIc0oOn=KSdho)iKOpadf0m^7nZ=Z_$~N@0 z51yJqlD4JxAuD72CZ&Y*}TBMlA7mWc{{5<3)5^+$H^FqQM&ihcO2C% z^AkMnK2h(ScaPeUHL;+g%+53yXp@_tMXJ2V+&9x2#BK%OoU9{|l5_WFPkYu5Iq#mGMUL>2BvbI3N5TdIXg~Q9tC?1M(73}os>G4=J&M?&6w#Vb6TKOxxVyGS+2DC z0}~Fr&w}3;JV7gGTFFC;s^BM(jR4vb*2Og3se_rApmSN}@f$=RUb{u96Uujzhq}wv zAgjIhNar)c6?K!whxlx8GO6h?+htT$7xss9NR%9LgJ|$*Ax>t$uujRXaY)KHWk!K0 zGfuMGT9$w9r&ob%z;!<~MF z+i%iZiB8#e?{ZK-tO`tw91=}>Sc!B~TZAlgm%#k@CI3RF)T1&5@rmWl+_0>?9-TL+ z={S>VhS={S1@|1x!6JH!@3(9L4%CwsGk=y!pkL&g&Nt3SzZo&Ld(81#YU%@Xu|FLb zk|=tK&62;k1pf}^2$OrE0ZvXvCsNto zm@S6$h`FG?F?BMsOw@;lJ`)WDo95#kX@(Rl0~EZLft94CQqA2>t{82=a^W&UKw>DT zKW5RW#nn|uWwmznVR3n}xtWEsjStJH-=@gd{huske~3QEXASF}7c(7qukLw)WY0Qh zFQ}aplfgP%UQ`2TvqhOo)KwKzTTH=tTzk>JQKz9HrqfQSn=`;BpWn#V#Tb8!f15|W zQfYN;*Opp0OTJfI#m4T{Z^}hvUyR-BCPlaeMK3O|VnDJ^;Sk=;zS16??nk<%J>i6J zGI@3#EN=sGAed(S~Br=%R9W$-SXQN^4H4yK+D*;rWJ4Agy-qiDcE@~TC}waH6J>76anq8x&S9Cw*R@D&F{n>@oq{c5~pDD2hE&tLbJAAt~fgN?Z zegaB!A&^Jk_2>G4XT}9wNXOw9vcGoSbst9c--v0RfSMhw)}Raixq!LDGy!3L{7JN zXkV=lV4+&!NtlHLrMBjHl6fM6l519u-K|7X@V!WkkZC&@ti^#yiKGGVFWUV{I8cJ` z;lg+NhhV>Kj|bew1nyuf%l>KNOYNS+#R0Qm(FJCnnmKmN;*^+8mL4W=vX0Q|n&g)A z>{b(@Mb2~1&1JbP&nQlyU1z^lY4DMEL+DV~r)U`JhPuZOcFbSJcN3qNz0bzILC?MD zVamV*Z-`A^JT661&stE=WVs_M9c5qTO?F)4OTgbh8K+awT!9_JCUym|_;c{!e@gxuzA+JX8LAlv%!Cm$8y^v|DvLzZyNU0n* zAD){hms9Z81tMlbae3aP5O6$$H?J`vRo->cw-+;B%!YL=QZOKT-G!JDyEQysZnutD zF1fhTS6P7gGAoDK`_Q+;(?HM48REY%`WCytKKlD?*arTQxeQsz8z&%);xbL&GPKAu zV${BLVGPdubtqQ2E7L2-uk13E1`rB2q1x|iaT#7M1@)yDu^*}P4>5JW ze~~{M85v+@y(wF^1Il(?t=#7EB{Wv%`hJo=O z0FQx)qBbO9G*56>4m`DXvgjv)%)@&%(s7fK8x|=-hJ0!F3w-$8Stjx(oYn069s%7L zja?s`(9wPuIn4<5%Dg61PcUX!eC;RYFr z@sQraoqn|f3({A;AbYEvW}ASAN_dUEb2f1SH2PqZxy(Wpc=yG0T!&O9FDW-J<*o0F zDCy}fG;v>|{hD5E5VD+9_c?Gknej<#UAn6FfLTQIoZ+0o$KZxDXQ(*gG}^Yq&<*Yz zV2sj2@Sh*@tyjQk_g#@vIiUf{0gf7ciacKhHF(a~ug8JaUq5wX<*9)xiO5m-sUa^# z@Ux;w4zcUsFw>M~qKa{dpdLXhI-;lxS@8N$@QyFi(}R@929JZ{K=^%ZmywFd$kyVL zp&{17*}42=pnfbn&KNF?TC#tSN|gr)?b9^}lZC0#+ak5=n$J$3jEQ1RT8UxP)m}|9 zeL*C2Fq)r+obQkKC-5i{s$3#2;s7bi-5yD&NINh>>hAY5@W%+>8#8k9e*cQ7W^KEN zt$ezjeLbO=?M+$KFlFGXX+lyPF8BQ6z#!J}l1=!R(!ooST(BLj24N=w6_K}P7fI;| zt$NsEQ#=?%7c+2IkZ!qsl(upp4T=W^C1QoDGZL^(b=!(=dt`c2Lc8Ve^JAtTWGJ(!7DTtGiSi+DyVt z^&|sigW^lVCtL0+fI8+$L!iL!TXl6q4mk5ARI5BvHwrE=xqM+(w9}) zT3{jgk3Y-*nfu5n3@pzYZl*8q{`sEp z+_~4tSsYE}F-pvxCTsX?)GL&SLR@UU=cpUL>xNS7$I!xs$mesfG0x-mTGEyk>s?+} zo^G<3b1V5nok5biuW9EO)QKyNIt3P`A4qC}rJQ+?6;|#jg#UPER1%kSq?$)KmQ%=O zX>PaFM4F$(?pO`dLw1?ussVD9@a<)gsdu)Vm;C7?4F$jg9+8A|0QqnS;QD@AnCI3%orL#~`*Z{sN#H0Hy`^_S@7v_BtPS>+`h8n2<&>xGIK(L#SyT z;B%SG>@S^YM(lb>d@C2&iIYcBE*zVyNfS-B0>9nGBibcyZDu(nTK1NoYLdJ(N&*HY zyDOr4Gt?676C?1|G^>iJAZ=10=Wr6DhlgBU~6T{!{lT)p!6A zI3m|d&a9=%frhwU=MQX0JQMtq?9)viU-f9#O%mHVe4LbV@+PQJYPNwO4BZ;@0bFJg zsLqbXrCOrKpK_KaOR4h(98x4%1KY)BVVY=#@<1M zrRqa~&H^I#I9`r=cx z!c;YMjzNZh>vjN|d<6Y|=a>hDtc!DEKJK?ol@%*i7(X`W|EdIO1~!YQfH=+eE_dyb z3_z1ty7oH%wE1ha?S>8OHcU*H$;4IJtM8)=jn)~h5r2|!J4P5|5Rl*YMn(ojbGzV? z38#RbE$zf=e+rG%e<|n?nqwQ*9XP)@Q=aaK7^C6Esw4fhbodfgC}px0s$lp(^U?9C z_o)Di#{%r=m=U$K$M(r{yZ*NC#I_jBMJHv^h{h=6Odm7XyonNp4{w>ifGsOMn%Ujl zGV}2FD~MNQQ9&g`o9x}k8VhqNZu*KRgHpBlxeB^W(}j9~KR#Nx`0c3M0^8$z{C=Hu1ZAV>BdYkVB;iScLO(T7#!~iOUdk4}U?=Z0 z-5=nj2)cQKs4yDBhxJ zPv*5xIt*f0gP3kXcLW#Ap8!CUU`Yc&vpeQ7LX1-;A6k(kza4=e;u?G}<52;41Z2}Q zkLLnRrXa+=#lckg^4R5R7Rt>VavpFHf!$f$!KtsHCXWIZ08)LOcdQmGFq`*e(9-~W z<*~E|t20v^Ig_4kcZbdwL%iK^D(<~3 zqB3#Km)TGd{ZdKIgL?oG3y;UIS$+2PZ^i5=pxPjf^IOq{K3!H0RJqlS2imf@S|mu+ z)=l@bKDu;IRwlV!_*0lP45He|U0FJB1i&dqE7O8Rq7FnG3AVydz@jeTwy!n7rrX-P zYn`9--@(~6|D0kF=mZ2T=nv(aX`PX9#;^Jswt-2b4g$7N*MctT@FLw7w+4}LV=pT{ z%dzdA%Td$&M6?(BLLUOoa#~=CV|J#IFP|}Q9JjI^-i+61n%631+JHH{7C@GP^jFba57KpwMF*q)`!-33B=&|MSP>48u4 zb=J+tTC9Zm1qrjTIg3=(NiH|$UgF;5D_E56gHq(*)b&p|e6~DF0!tCfNqadNsy(1& zl5Z1${@E!{thLnWrhJx@q$i#+X2}fUv{z|&IP8~GMJ0J?=m;#WlAZR6gKo0CprKRm zwEOkBAUXoeULbHNRR?I z#ix_pB`Xz=2F&*aIIEQTd^pm6;cDsO^0{6{2UgBQRMtQ-`&8u|9xt3mjtKq+XQ^v? zPVO>tw%gXq7+cu4ko)L5sBVP`e1zXtt~cnVGe5QvsmkmhB+tuQBt7&~fv(Tq0hc~$ zKJtiC`uw9mttjkz;WctHUV8<92I-Ce3|h@JdtVr4x&#c$`aGza}k&~Jt!!y6|WDA0+c zO-r^LzIXrGfgoTF))x6ZU7|i>zw51EiSgy|8!z|7V--HT8YbFBcQ zbP=~@xPhu`$bZTTGv{(BD2dG6>Zk(`IJ3vjXYD)r5Tqvz1H`VR$J1z`&y zo47ktqvn2iM=^S0qUV#vx!k7U;@*89U9UaVPIU`)=bQ$UR_jUnOd$n@#RWc&p+<$g ziPthB49;Jh+14dXDNqoSfY%fHeL0D^II&Xxrc6gOK zrFj1o((qmzJP=j(bP@31{t^RF!e;dSq7b$L7RoT3Po@YrLtlL!R@(5H|zR$bI%ad z?sDEH^o+|@d@%T4!bdjX^M~g5=fW^kq%}A_(_M5&C%^jy*rWi|oGD?5J@PMC#s z6j%n^y=82iUVzsl@K4L;@Si}7upuj(K4=k5(|`tXBL%%TVi9S#jyHAq9Bc68a&_xQ zR#x3H!$>UJ884;H`eWN{`^b)-Y*R@M(hvyd#R9l@8dE6|MMAGPT-lZtI2 zK=`5k`->!2)6FEK(W>l*#hR4{sj?>vRyyM$4l{#0*yoIrVq>s#)NU0^C@ z{W-cK=(dUglr~?-3s*W0HRMeeD$jdeZo0Hef{5JOH&wI8ZBeX=!IYjXGADV zZ!J`h?U`S6;PBA00_y%>voLJ8Jdf(Wz)+qOYVeMYy7E5t{j_U9fMX2!Us?kFcwvaT zecRL_>RI*ALrzi;yS_baoXfe^noFvV2UNXSs?i2av)iVwO3!;3%H(j|Xrh);`Ng^@ z`g2RwM}B!z=tTFL@EQrv!d5xSGG&}4i452s#SYvCfE(p$l?liRzh8Rna)EFE8Je5X z#4DpbBXVZrF72fY)zMk>kWU_PjMH*48JF;WIUMz-E4(_=sm`X`ws7`Kn5(L2@amV2 znmR@k(Wj0CHurFkf@?Fr)f7EltTaf>t5PQZq@%BiG+5t^9EyHGo?j$^rpw^=yxh>D z#Uy3aYbXB%D1+R&*v7Dd7g;wJoPm^t4ba_1uYJVmK6OmQ52#XSxI=;MT^j{RRYcj? zlwP9tc*Y*S<-Uh?t180y5+KT@dD$CsiN`*DTtV*CsdQ>Kje+fE>*DUtD~n`HNMxvFqcbA^7(ebt9X2+k{ka8OOV;_y#A99I>e< zD>WsAjWa_ST zN&~ylFWn3GMCH5C_&HhwmV2orXr6WIyxrpX-fT3WYyVe{`Ij8>PvPT=_VhoX;s2Fm z{#}&#e?k5FzgLc_aWAyMIn$dMn#Ud2cKvHsU3$@Wt#EXDv%>()_w~aHMxzV6u9dWf zhW6$r=F}>npD?PAR1X5ecp3L_JZoJJsC#g{t%Fc=MiNz-Wu=K7z+#3yo||r0;kj)B z24z>z=r~wJ$r7m+G6mepGy25@6$ll}U6;j~n1a?Tv2m}=QVu?KhL#fm@hSSG*C}N3 zuXIWlvu-NKxHZGwu_~09iEBQz!mR?URx5Tg?Lt@PeF3L z@bOpTM$vR>NQzV~53hhrAyS^=xB$cBhb0H2!DSbJWZL+dfK?EvFnu{&eUbWIKDAL9 zfD9dq&7n}DiWBln8P#X>B1*{}e1IAgqs}%%xHvHBZN)-U z%~3@l!m z;ajO$sh^M#nb`xbLtOPm31z}br(Tf}zTJWzvx|WH!cZWZF4`f-5 z+QGhVQMr47=&#q~b0{=tw-=qODu!R8FYUH;J5}72`8=vedlo)#Dr>?TsUO7?gCL)_gpjY5xZv`W4W%{oSaf$;F}ueQ4xNGGo8%D3wut z(m{70cp7mGLIuye9?o;S_?)}C@9JM#fNt{SdM4wf;oz~xNGIm*oZ`N?yWGYavt4f$ za5yj|4l*e;8;Q*Kfy(71zSEuWlZ#eh3v~e!$f}G-QSTbhPk+nS>_~YjWIBDKUrjiz zfFnGm%l4IF$6QEvMtp<#KZ6T0MMffw<6DTVk_E{!Kb`#H-)Bg4Dg3SChMh-O$a4TFM3 zf=xijNkbO!7+HB7 zfh88v+Q4-=wgV-&jiLt90#B)B*nO%(2aywcEALB={vn*U2kcXwLJc=!$Uj_L5~Hu8AzDc0B9vbYRPQ2gbbH-B{d)qiXR- zKyWP69Lr0rp`zATZn^ zHHk+z?6Fg_*&Ykc2zcD81a9LsOHSfE0yc9kgGtR}$eC|i`g|Z+2BqI^#GT0+J`gf4F@1^s7UUeKqCISAhy}HG~;6&7Sa!6Vr2_Tw+HUO=7g*@NS?T`Hg!N6Ky^( zgeTVx%C$?)fg*_ozVkqE6k{A&4|E53t_Hhn_gc#DC3eJ9FEYVFo2S7}viA^3`U#i- zQclmbA#(?7c-UPloN{nx@-g|s;{t%A9-@8RoK;jLdoC>mTLO`jj=g!N^=zEb)j;AqVfwkJ8u zBw&A@2Ytz!THd}y1V}{0iF`-mF9jwqlhYpat2#JMwspT{bRLN6DhS#5_i6H*dKRww z_VzUEIs|3_T38!lsTQ?e_T5(&Plc^YJ}l!gSIh%vM)0?!^lB)cgb#s(`DEP5q^Ut4 z^yc6u7#i$@E;OH2AtzH7H1%Z|oY?iC0%jmAhf3vMU}%LcJ;YhYs4R%9U*dAeVCK)|!RCxN#p;sU& zw0NqrfW;>eJ*xT)qOQr>Y`E*Z`J=PeOmF+}is%OuY(BpYPZqH9|KyX=F1`kj(_Rml zi4adrn=f6uJth+W8I3$~5+KDW;y7!m1ORm=CC`&Xbmp*u&>B{&8G?3 z-@IMD;UXw%2qQ}Qqw1^m)bb#3!{2bs92%yh0Pxjn$VW@rccx2ey&}*WKlwj=WmDx ztf)R@X8)eG?e$cVdJ~~O+H^0D@l7zCH*yAhVZzqzI*^j|Al9?5I}Rp%1L!=(a{~&| z^y=_!E1gew`_g{K9jilG6k75>Jji4>Bt6oIjC#51+kMa@-CRvyL9GV@aco)R_a}mE zsfTr*))CxTa5lSoim&$Zj0--Q!nRi!KT4|KGQ6;8r2s@pfc%tUzvOI|z;D=9!bur5 zk5SxAhj*opvv)4Z!8+WtcOFpvWM31V$i0~xNuyVZgHZ%_u4r-iI>|vGKTzg>n6V<0V5)h6**hq z0yU&Sqr2cnJ5i;E(K--X@A!^xFmrr$|VDnxR-vJ$1MPHQg$=-+b4NC>TxG1B=8Fi7p-t30WPj)T#2f;+mPgpG0=~{!e%Y2?#Ausfy<5GmKytoHvRz!g{cSdB5um%n1bjM;A$7T&g8B!$Ol(G)pez$JXdS^dh{1+R-`y|>N+;9* z07j@hm+dHAS_1*wh9-f;321&7li{+`{8|Xl(6m=zxtN+hUkJPVEWO<8;Fe9vK5d|) zN{^>mjF5;?2BI}mpHBcvQ9Y0mx|QO-o$p98E~tNSPw#?sx78{0M|efQjiR+(G@&2b zU$bSGoo!#r{McsOlR7$m=CC?dDRDNvKS|Jjx?&Ek)7FL&HCKB|?h{a6JXe)1$J zD00y`Oz|T!-`;lK!%I_%aJjC}YwP5pqis^BH0sNv;E|VZvDt&mZAhBUO^ZtOIz8ts zO&squ%UK8^S<-Lc^1-8IQ<|$AZTpO0!AxEsaRFTc`T7aq3GG0)9mEkc^dfhke^>_A zO*gI=D9 zs4+EjG7`l*tJm#RA|;gY^@Y;xL^e7(Prw!J5)!Qei!3uyWT9ll=S;K%trDW51R%zO zmodBDB&k@dmJ0oU#m7L+d_DX( z&+Ypw^e0ko+|9~O6CSIkb7rk}L9YQ%Wr7<4&2m7x%#bnFghByu>z7&fYRj-6b3NkI zg74ISl4+jz4nO7s+`#ioqlsfVC%SN(RuB%ELaD%58+L*~JDHwJ9&ZnKKH*oc`i zk>J0UYWhwCvK5B>9qKs}+*|;ciad?3o8%qxZRs@&&2BC&QUrwao4qsk*~2QYnP1SPei{@Hv2LB(e1s0^)ih% zjsEZvv2EA=$`hNiSa9+ygFF?J*=S^T8iL0egK;LFi6KzQfD*Y|@i#P=W4oIRVRGqQ zp63~#rwFG3!5qB$>f=0{QX_+mZc)_B;zuxt;#}2RAf1(c$m><-lHNr}b-B3|E;bH? zbs!LGVH@4#zP8?eD-G|q!x8WGRlVXBSKwa<)k`>6sph=`?hYP1c|*gtvhPccbrhw= z(f7rZbJa`^LZ45v+oOkMbe-<;E1vwmROcr}Pb-M2-6s@G9tTpdbz*l%M&k;e%J&9; z#9eKL?M%r0b#k z0MUHpn~KvJa({Ki%kGJ1VRwxh94+Q@8~8F%(LwV2MJsTUVEA+(0VC~FF1HBJsgAoS z=tOP%bFGzDHvyZTjbxTRy2X0*(e&{w*>rWeFOPWRR3SD9;2X$-{<%}<3j7^ia1Ct{ z8asf9t3K6lEFvqs=~(%=8OZ~4#6WS#e4u38h5|OQ=!w~k58&G~dK&|Ccn2?}ON$ln zOELU1p3DW%9zKoRJ#QNJXTQrR`*jdW!|;*Ui0Zx!u4>@LN8|^M3(ANkhb$ z66;LmP2B_urq~y5UmgA+HTsnjEICRi*NxU#?3(iC@oZZJ=75ZPMgzBH7YPrBJFN92 zGEFE+qMC29*#XZ||@vB+QXBCe^DrzIUDZ{6W zmf~aURyv&F56dh+r=xSJ$WdnbMgT~wCTQ|x?_r$5Y@-irzD`oW2f8N8MKsQxY*W5v z7*FOA7p*dYND?68fDiS8smBqLn2|*1uW%&nu*@^SH3lBThLHgm9Ys7Mbsl{CL8$q1 z_lF;ZnqMpBt?QjU4P-jx<}qoa~jA46OlrfhWBFVEeV#hg}6Ow8P};v48N zdo8W0fX4E(>l0@d#)YmI=MYWxz-sUUkkdpg75{R_064A!jakAVyCcO!N;TJIvykgS z6AXa5#TDKMLQeoB4Q?@-p@f1T;GN`un@>CdL1^`nTmQCR<}w!i0hoUHpB?Hyy!&6++kZo!Q^cQp>6!aw{@JSk=I%r|TMwN5 zrbV9Mc(A!bPa-#Qo&UyvTZ40%-v6Qgf9B)PHnRKGyV}mnV29@ipi~&o(~e)52~-rm zzR)m@gLsqP(H@+!CGH}rA6tx0Qntry$}&56tf;k&_>q~x-C+6Mj1f?UHrge51J8}e zPOG3Vs7kwTtsNqmk`tqf81~*LBcNgfass2h-Lve=)WvBP3)C{XM7QEOz&swaj+Y3| zt&AKnclLLYPkH7C#FNiH;95%3SPiY_F%7|HGewU7l4z!R!Xa3nW0ASM978X9$b2Hz!wF}6k0xK z;3$WDW1Ku^+5|W%B6pdi-k;=Zp1G9PwN5g}lb|{dB4&`jlwKo@fX;-peU(w#GJvgA|Q%=Yk zaQxN*0@6D8v7tc0%>VcD&ZNlyWPcx@dV7v{<_>tY&#uZT8MvkSh;dHv5{{MCZd$W& zgCi8*=5{`sn3J@Ju$*>&?;ovW<4cYa@!OCNDxM0rp&j_8-0=vw+6SsAXgim!dyl(t zG|L9-Qgc%0$XC*8dC0HyDAc>dbe+APq*CozPcjjW*lyvq$N@o}w99q=9+;^{F5K9L zN)Q)$?>Eqjf>Cn1vtV>bU?Msq&4 z2ISC3IJf11`TYd&!tqohaJjT4~W zJpWpIX=gqFR~dUH_{wAXLlwY1Zr)2{RZEMqqmgOMi4yzXW z@y0hf`kZVTR4ora@B6@JcS{ZZWwU#V(M{u{58fX;fc*nh0f=~DH;c%>U;&E9Sd=M1 zPl73PPlivnkw_uQWKijXQ53d?5zn=4@j$dM=D~2e>?KcSq@DE9N@eVH2Q6~|Gld66 z_T^9U6tt3;ou&U1aJ3;B6kXgO=qOx*Ma{oZI4j+ulwQw2&nYV7?*NTEPE9yAZkV&p zB%GOM7n;@o5@T0cjS64ZPDZIBWK_sD5r?S0EhGqpR`_P; z%ou{~eQifm?iPu`@|V2AA+lwb0VUs{sm34=Ojj7*IDjGoEFh^LHq(t=->aN%y>t4} z$Wa?$xh?P5SqR+Y`#5z`uB41{)zrk)>f;qVG6lJLZEDFIHi&-B9gz9`GtYFymrDC!U}_PU{9Vu*S%%FXB7|@AVA8}-HERzVjWqW+#=3`$Pa9D zcQo)esn3>$>CpOcHH zNNisw2n43V@Fj%tw-&3bhprun+8qP)x(euIU)sTkjU#MHMlYu}K+9x>-)E_UU754& z83tuqd_ow%HOwxxmx#;HytjpPFG_<5r9*IdK)K&a?ZT-n${;@dfoY1=*Y;xvz?eNx zP2gS*CK%#M_`0}XoEr8$)#c(|FV72X3R)QYT{y1srDddGsBV(E8SaA=07MB68P(e5 zf2u~w;Tv`X_6tZUce@w!^@F@+-}@^_4d`tFN<=RFWQActJ;%WHu$>!th#bQg7tYAy z8I4214AP8w5`qZ2HJ8NQ28&ibT98i?OvXg|Dj-|@c96=DllH>mvKGR`+^D{=lKO{n zydoC;!Y^J(VH{_as*@2>W-h8^M*>{3FQY|}zhze(C5^|KHeq)d-E`#G_0Mg$NdrX| ze@a-Y->tsa8`o*^U6B6U!p2{)b$>N@Hu_jmf0)zYoX~Xm;X&-5*wZv`k|+D66E{6s^EH0A}@;h-u`BU48g)K_1hL-O}s!lY0X2=6?X! zy)lVH1%3!z*GioMkt?;G5LUhAGElLyW`6=Ac2G!okJ4rp`~mKps@vzoc8P}-9C?aP zO;ALi{bJAxveDD<8VTM+XJO!(J!*9Gs*@cNEE$eUlix1N9R%bjSFaYC{_Mf|tL8Gv zgo=~uscwqGsE|n}!1!=;Qd!Kd2i6_FON}G)1X>F!M7wv~L{vPEg0Z@%-6tcbx zj{mTm7}f`>uhng28Ys{tKZIB=TI%UEiNdG|D{W|}DNmHt=w+w-( zuLec)#Vj-9e#?qN(#{(D{JmUfwpK``#Wc~#LZJE*yt?VqZNG&&W8WbW`w#o&Y%y#H zZh4U_ZhChNDKo9;7m<*Tt^Pu8gg$3t<`Bb~HevXUF$z+Qxj@9$E?GCE7^zcoPR6=c zQe;|jc{#IV?97efZu89lTK|da{|seebS-KXkYsF%?$;Dae z@i)(<@9Ro{Gn6j|Yjp>3*AmC@fRKIh_`x#&)%4Zr{_}VIwIgoIP1X90Vqd!ivV~MD zMCuC#wlt0K1NvV@z&Y$D^V975v7{T<)oy%aA3GDOwhmBOC$HxDl>!0w(X|h5rB&Ol zfYz!gYtU1{V>AMvV#3iD*){l-EGbjKNO1zgZ>E(TC-hXEU()@l9;7t^Ae zk)RVMhUYn~`BklGUpS}oxA63Nz|6f1@I82$1|bBF>p+5vfGp(^@C#T||Jc2t`}1AE zR@H?z>d)qEyO*c0uU2%ZVZ}_JhGy=aW@x%Zbs=gd{c5A&EC^^;MC|(U`QBbW%6z3g z0*$E!a^wTM?tIorgp$c`hE-!UufLFN=tchz7gkzCF|>$d%@Cba-UlzF_T)MfiEJF>+OX~Phg{nb@%Z#=_0#tD2zMxtq zqB5gCYZ5BhGl*zj`~Z zyt3Vb@%fTV`O7c=?rWG8=pHMMt+8t#1?)m}(M(@~8kiUn131Z6v^WTSTSatZFNd;y zQX3bF*71!p+65xjoEGn4SJJg~9XtrWaY+5-Sfa>~zEz_f#tNk9`=SMTEs$>9O^P?R zUumFNOU4-@I&8Z!Wh|a7VlxDDT54F~?gs$Ox&bijRF&M7&n+<@e+Ld)K;tlE%eda) zE+dih@($2_ABw06(!wBY05y4A)PT`saCqAbut0kH z>s5mN3!l^UT=$JxoeNOp+~-d<`m6XouUI3PD6w4b%B zdY4R;@-B@}dGUn4zc)b6%V5fh51}OW!|rCc(O|()VkJBI1u*PCEzL5ED<0Wno#lRV zq*!9~5Y$v>6q`yXfh<&3E>X}okO=JldjzlEo(zu3aB1L36K2yS_W^nf3SA4^Z@Mjs zw;7m`Zw@F5#aY)pZO81_)gRDPIs)=SfPhE9S3QsszyCmMy)z^IkDQi}Ki)3ibsNK+ z^xlE%Uz-c|_8Qe2;R|z3LY5H$pK1?X3S6<&m*D4DEcIDvdK?ghr&GnAR&&NIXTfuc z0&6c9LSQw@P68NbkF*0cT|dfRufj{w0A{Lf=F zG$;?SQDh{pX(asE9O3s!TR@&)9TzKhmlRRi_8h27u{{R7oe#j0DoPDpbErAv&Cj?_ zB12?NkIoMthLS$T^%*{4E+!jzF}fja8qg+v?gjh%&~uvTaSWe=gy$$ZCff zC+^QK^?1sCX~^r1&uEP59O8|nb*EYIy(qPippA?+?221|=Af3osNvpLxv?Zf>u%_Q zIe%v&*HD%nqemAC^GZjUuE~Fxo^>bt*D(?Lzj%{Y7h3-3^Cv8=OwRlQ{=6Q0`~KxwliyDEw**(->-CapHsD;$j=r~`Y*To_C%E7I=Y@ZK$nvu&`*d9Z zH~uQ_ex2KH$@t*=L@k}IhI_>hQo*tA4+i-9F_&VrDOG}nOOf>tXSGfEQFaKS_|M_*$&FSg00{FRonm1X~ z`_)MLU-Yo1)}X-J6~@^r{u)-SMvxCC`}P$m~TkuAg}Mq-HL{fmPj- z!7Vo2e~RB<`)%>mW85ddfNfnlACGC@O1A5C7YPWtmUffru`pXIR!&N0S%bo9?f>>8 zY&r5qVsVd8Q%VP-bth+uqaJA$qPWcY(}zD2$5B(ra4zmU2O_IYXt%`vp3y{-^|6vI z{qCIPQhmp8S|Q~8|J3>;XNSzV`(_jQ&v9E>8P^gQw9;e+X%eDYMl9}PY;z9{J2G}~ zBT9eZid^AY&YSszkKes6zO(9S)4%=3;GsM(ieKb~2hJPB5ACUaNn13@ATB zE6kG+?bhd0jRQrHWPRR!W~pp-;t+;)v3|*hvk(=SHT%?mTig|yQ59ABOg)kDT3!SUH*OJ`ga32{_-@I>M zgB-^5J+Htux#(P$Nf7q`~2+b_Me}9eg+r*H~rlI?}~N5-#-2G<=?xPA3xn~_x)z}WO|T` zMPa32;_oXOzqj2ojH>-0TlQJv+wO~R>*Dv;_1IfQU-}qtQvdAZ{xBDfX@{4uzh8Uf z$h)&jhxEjnpSvfN@X6mUIO^Kcq4ccz+r{+aUH`rv`yV+i6$6DXyefNq=Iwpq{(=vW z05zNirl0y1R^K=NbUprS))l2gGs`|t?@znoeq>+7@v6^n88v^K%bnd2-#ZU@i$~AX zhmCQif{IFuI-aJ0%9^W_uipP(b@Svq{d&XvdwVKBKRXKy6fBII$j|H7rmdDXU!1un z`-h7NFkYtT9(kW^YxmFl+q;zC-2LCCD(_gNbg0g<;`c^w!9bgDeRk)+U5@4dwsA-8 zzwhS?q(1-rQ+Zac@U~UO`)9s3XPz0(U%%*w>wSJ`t@oLXAkcdyL0T(3uf&{8|x#KGLFA^`I8G?V}&jOYWytyarX@Q+#d7C zd4*}b3uhc(aJ|#-`R`ZXlD1f_XPXe^V)6S&d|jG(yKduKm8y2trfUk*{PTfjAp``t zZq-+M6SjGV-*W$KqkWe9Ry&r9Om6-^-%jt*)!H|Y41c-LesE0cY zd;Y3Tm#fdb|3AI|{}(->-aH=l%`?u+Jz7`%C+_XVS3Sp5)+FDtn~~q2^!Ie$Tc|w; z_)UTKB-`fA|0(`;`^Q`NewAIay=Nl7{(SwCgEysr0!^&E|NnWPy;2gW+}@Y>`P&|A zNq9B9B)~-@Rk=?1yLEnf#z-SJoL8GrMO@~HEl*XG^fQQvr@Zjs{xSV&A*^bBSI!vY_8 nAmeTpKz+vGwHRgu!<1+L@4m8Ebm%P30m*v0`njxgN@xNA$36ZX literal 0 HcmV?d00001 diff --git a/tests/benchmark/ci/function/file_transfer.groovy b/tests/benchmark/ci/function/file_transfer.groovy new file mode 100644 index 0000000000..bebae14832 --- /dev/null +++ b/tests/benchmark/ci/function/file_transfer.groovy @@ -0,0 +1,10 @@ +def FileTransfer (sourceFiles, remoteDirectory, remoteIP, protocol = "ftp", makeEmptyDirs = true) { + if (protocol == "ftp") { + ftpPublisher masterNodeName: '', paramPublish: [parameterName: ''], alwaysPublishFromMaster: false, continueOnError: false, failOnError: true, publishers: [ + [configName: "${remoteIP}", transfers: [ + [asciiMode: false, cleanRemote: false, excludes: '', flatten: false, makeEmptyDirs: "${makeEmptyDirs}", noDefaultExcludes: false, patternSeparator: '[, ]+', remoteDirectory: "${remoteDirectory}", remoteDirectorySDF: false, removePrefix: '', sourceFiles: "${sourceFiles}"]], usePromotionTimestamp: true, useWorkspaceInPromotion: false, verbose: true + ] + ] + } +} +return this diff --git a/tests/benchmark/ci/jenkinsfile/cleanup.groovy b/tests/benchmark/ci/jenkinsfile/cleanup.groovy new file mode 100644 index 0000000000..ec38b34618 --- /dev/null +++ b/tests/benchmark/ci/jenkinsfile/cleanup.groovy @@ -0,0 +1,13 @@ +try { + def result = sh script: "helm status -n milvus ${env.HELM_RELEASE_NAME}", returnStatus: true + if (!result) { + sh "helm uninstall -n milvus ${env.HELM_RELEASE_NAME}" + } +} catch (exc) { + def result = sh script: "helm status -n milvus ${env.HELM_RELEASE_NAME}", returnStatus: true + if (!result) { + sh "helm uninstall -n milvus ${env.HELM_RELEASE_NAME}" + } + throw exc +} + diff --git a/tests/benchmark/ci/jenkinsfile/cleanupShards.groovy b/tests/benchmark/ci/jenkinsfile/cleanupShards.groovy new file mode 100644 index 0000000000..ac0f4a4336 --- /dev/null +++ b/tests/benchmark/ci/jenkinsfile/cleanupShards.groovy @@ -0,0 +1,13 @@ +try { + def result = sh script: "helm status -n milvus ${env.HELM_SHARDS_RELEASE_NAME}", returnStatus: true + if (!result) { + sh "helm uninstall -n milvus ${env.HELM_SHARDS_RELEASE_NAME}" + } +} catch (exc) { + def result = sh script: "helm status -n milvus ${env.HELM_SHARDS_RELEASE_NAME}", returnStatus: true + if (!result) { + sh "helm uninstall -n milvus ${env.HELM_SHARDS_RELEASE_NAME}" + } + throw exc +} + diff --git a/tests/benchmark/ci/jenkinsfile/deploy_shards_test.groovy b/tests/benchmark/ci/jenkinsfile/deploy_shards_test.groovy new file mode 100644 index 0000000000..2a4c88e91f --- /dev/null +++ b/tests/benchmark/ci/jenkinsfile/deploy_shards_test.groovy @@ -0,0 +1,21 @@ +timeout(time: 12, unit: 'HOURS') { + try { + dir ("milvus-helm") { + // sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' + // sh 'helm repo update' + checkout([$class: 'GitSCM', branches: [[name: "${HELM_BRANCH}"]], userRemoteConfigs: [[url: "${HELM_URL}", name: 'origin', refspec: "+refs/heads/${HELM_BRANCH}:refs/remotes/origin/${HELM_BRANCH}"]]]) + } + dir ("milvus_benchmark") { + print "Git clone url: ${TEST_URL}:${TEST_BRANCH}" + checkout([$class: 'GitSCM', branches: [[name: "${TEST_BRANCH}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "${TEST_URL}", name: 'origin', refspec: "+refs/heads/${TEST_BRANCH}:refs/remotes/origin/${TEST_BRANCH}"]]]) + print "Install requirements" + // sh "python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com" + sh "python3 -m pip install -r requirements.txt" + sh "python3 -m pip install git+${TEST_LIB_URL}" + sh "python3 main.py --image-version=${params.IMAGE_VERSION} --schedule-conf=scheduler/${params.SHARDS_CONFIG_FILE} --deploy-mode=${params.DEPLOY_MODE}" + } + } catch (exc) { + echo 'Deploy SHARDS Test Failed !' + throw exc + } +} diff --git a/tests/benchmark/ci/jenkinsfile/deploy_test.groovy b/tests/benchmark/ci/jenkinsfile/deploy_test.groovy new file mode 100644 index 0000000000..6f503ea489 --- /dev/null +++ b/tests/benchmark/ci/jenkinsfile/deploy_test.groovy @@ -0,0 +1,19 @@ +try { + dir ("milvus-helm") { + // sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' + // sh 'helm repo update' + checkout([$class: 'GitSCM', branches: [[name: "${HELM_BRANCH}"]], userRemoteConfigs: [[url: "${HELM_URL}", name: 'origin', refspec: "+refs/heads/${HELM_BRANCH}:refs/remotes/origin/${HELM_BRANCH}"]]]) + } + dir ("milvus_benchmark") { + print "Git clone url: ${TEST_URL}:${TEST_BRANCH}" + checkout([$class: 'GitSCM', branches: [[name: "${TEST_BRANCH}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "${TEST_URL}", name: 'origin', refspec: "+refs/heads/${TEST_BRANCH}:refs/remotes/origin/${TEST_BRANCH}"]]]) + print "Install requirements" + sh "python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com" + // sh "python3 -m pip install -r requirements.txt" + sh "python3 -m pip install git+${TEST_LIB_URL}" + sh "python3 main.py --image-version=${params.IMAGE_VERSION} --schedule-conf=scheduler/${params.CONFIG_FILE} --deploy-mode=${params.DEPLOY_MODE}" + } +} catch (exc) { + echo 'Deploy Test Failed !' + throw exc +} diff --git a/tests/benchmark/ci/jenkinsfile/notify.groovy b/tests/benchmark/ci/jenkinsfile/notify.groovy new file mode 100644 index 0000000000..0a257b8cd8 --- /dev/null +++ b/tests/benchmark/ci/jenkinsfile/notify.groovy @@ -0,0 +1,15 @@ +def notify() { + if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + // Send an email only if the build status has changed from green/unstable to red + emailext subject: '$DEFAULT_SUBJECT', + body: '$DEFAULT_CONTENT', + recipientProviders: [ + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'] + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS' + } +} +return this + diff --git a/tests/benchmark/ci/jenkinsfile/publishDailyImages.groovy b/tests/benchmark/ci/jenkinsfile/publishDailyImages.groovy new file mode 100644 index 0000000000..54ac5da9c4 --- /dev/null +++ b/tests/benchmark/ci/jenkinsfile/publishDailyImages.groovy @@ -0,0 +1,46 @@ +timeout(time: 30, unit: 'MINUTES') { + def imageName = "milvus/engine:${DOCKER_VERSION}" + def remoteImageName = "milvusdb/daily-build:${REMOTE_DOCKER_VERSION}" + def localDockerRegistryImage = "${params.LOCAL_DOKCER_REGISTRY_URL}/${imageName}" + def remoteDockerRegistryImage = "${params.REMOTE_DOKCER_REGISTRY_URL}/${remoteImageName}" + try { + deleteImages("${localDockerRegistryImage}", true) + + def pullSourceImageStatus = sh(returnStatus: true, script: "docker pull ${localDockerRegistryImage}") + + if (pullSourceImageStatus == 0) { + def renameImageStatus = sh(returnStatus: true, script: "docker tag ${localDockerRegistryImage} ${remoteImageName} && docker rmi ${localDockerRegistryImage}") + def sourceImage = docker.image("${remoteImageName}") + docker.withRegistry("https://${params.REMOTE_DOKCER_REGISTRY_URL}", "${params.REMOTE_DOCKER_CREDENTIALS_ID}") { + sourceImage.push() + sourceImage.push("${REMOTE_DOCKER_LATEST_VERSION}") + } + } else { + echo "\"${localDockerRegistryImage}\" image does not exist !" + } + } catch (exc) { + throw exc + } finally { + deleteImages("${localDockerRegistryImage}", true) + deleteImages("${remoteDockerRegistryImage}", true) + } +} + +boolean deleteImages(String imageName, boolean force) { + def imageNameStr = imageName.trim() + def isExistImage = sh(returnStatus: true, script: "docker inspect --type=image ${imageNameStr} 2>&1 > /dev/null") + if (isExistImage == 0) { + def deleteImageStatus = 0 + if (force) { + def imageID = sh(returnStdout: true, script: "docker inspect --type=image --format \"{{.ID}}\" ${imageNameStr}") + deleteImageStatus = sh(returnStatus: true, script: "docker rmi -f ${imageID}") + } else { + deleteImageStatus = sh(returnStatus: true, script: "docker rmi ${imageNameStr}") + } + + if (deleteImageStatus != 0) { + return false + } + } + return true +} diff --git a/tests/benchmark/ci/main_jenkinsfile b/tests/benchmark/ci/main_jenkinsfile new file mode 100644 index 0000000000..6084c562a9 --- /dev/null +++ b/tests/benchmark/ci/main_jenkinsfile @@ -0,0 +1,148 @@ +pipeline { + agent none + + options { + timestamps() + } + + parameters{ + string defaultValue: '0.11.1', description: 'server image version', name: 'IMAGE_VERSION', trim: true + choice choices: ['single', 'shards'], description: 'server deploy mode', name: 'DEPLOY_MODE' + string defaultValue: '011_data.json', description: 'test suite config yaml', name: 'CONFIG_FILE', trim: true + string defaultValue: 'shards.json', description: 'shards test suite config yaml', name: 'SHARDS_CONFIG_FILE', trim: true + string defaultValue: '09509e53-9125-4f5d-9ce8-42855987ad67', description: 'git credentials', name: 'GIT_USER', trim: true + } + + environment { + HELM_URL = "https://github.com/milvus-io/milvus-helm.git" + HELM_BRANCH = "0.11.1" + TEST_URL = "git@192.168.1.105:Test/milvus_benchmark.git" + TEST_BRANCH = "0.11.1" + TEST_LIB_URL = "http://192.168.1.105:6060/Test/milvus_metrics.git" + HELM_RELEASE_NAME = "milvus-benchmark-test-${env.BUILD_NUMBER}" + HELM_SHARDS_RELEASE_NAME = "milvus-shards-benchmark-test-${env.BUILD_NUMBER}" + } + + stages { + stage("Setup env") { + agent { + kubernetes { + label "test-benchmark-${env.JOB_NAME}-${env.BUILD_NUMBER}" + defaultContainer 'jnlp' + yaml """ + apiVersion: v1 + kind: Pod + metadata: + labels: + app: milvus + componet: test + spec: + containers: + - name: milvus-test-env + image: registry.zilliz.com/milvus/milvus-test-env:v0.3 + command: + - cat + tty: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/ + readOnly: true + - name: db-data-path + mountPath: /test + readOnly: false + nodeSelector: + kubernetes.io/hostname: idc-sh002 + tolerations: + - key: worker + operator: Equal + value: performance + effect: NoSchedule + volumes: + - name: kubeconf + secret: + secretName: test-cluster-config + - name: db-data-path + flexVolume: + driver: "fstab/cifs" + fsType: "cifs" + secretRef: + name: "cifs-test-secret" + options: + networkPath: "//172.16.70.249/test" + mountOptions: "vers=1.0" + """ + } + } + + stages { + stage("Publish Daily Docker images") { + steps { + container('milvus-test-env') { + script { + boolean isNightlyTest = isTimeTriggeredBuild() + if (isNightlyTest) { + build job: 'milvus-publish-daily-docker', parameters: [string(name: 'LOCAL_DOKCER_REGISTRY_URL', value: 'registry.zilliz.com'), string(name: 'REMOTE_DOKCER_REGISTRY_URL', value: 'registry-1.docker.io'), string(name: 'REMOTE_DOCKER_CREDENTIALS_ID', value: 'milvus-docker-access-token'), string(name: 'BRANCH', value: String.valueOf(IMAGE_VERSION))], wait: false + } else { + echo "Skip publish daily docker images ..." + } + } + } + } + } + + stage("Deploy Test") { + steps { + container('milvus-test-env') { + script { + print "In Deploy Test Stage" + if ("${params.DEPLOY_MODE}" == "single") { + load "${env.WORKSPACE}/ci/jenkinsfile/deploy_test.groovy" + } else { + load "${env.WORKSPACE}/ci/jenkinsfile/deploy_shards_test.groovy" + } + } + } + } + } + + stage ("Cleanup Env") { + steps { + container('milvus-test-env') { + script { + if ("${params.DEPLOY_MODE}" == "single") { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup.groovy" + } else { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanupShards.groovy" + } + } + } + } + } + } + post { + success { + script { + echo "Milvus benchmark test success !" + } + } + aborted { + script { + echo "Milvus benchmark test aborted !" + } + } + failure { + script { + echo "Milvus benchmark test failed !" + } + } + } + } + } +} + +boolean isTimeTriggeredBuild() { + if (currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause').size() != 0) { + return true + } + return false +} diff --git a/tests/benchmark/ci/pod_containers/milvus-testframework.yaml b/tests/benchmark/ci/pod_containers/milvus-testframework.yaml new file mode 100644 index 0000000000..6b1d6c7dfd --- /dev/null +++ b/tests/benchmark/ci/pod_containers/milvus-testframework.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: testframework +spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.2 + command: + - cat + tty: true diff --git a/tests/benchmark/ci/publish_jenkinsfile b/tests/benchmark/ci/publish_jenkinsfile new file mode 100644 index 0000000000..38f8996b00 --- /dev/null +++ b/tests/benchmark/ci/publish_jenkinsfile @@ -0,0 +1,104 @@ +pipeline { + agent none + + options { + timestamps() + } + + parameters{ + string defaultValue: 'registry.zilliz.com', description: 'Local Docker registry URL', name: 'LOCAL_DOKCER_REGISTRY_URL', trim: true + string defaultValue: 'registry-1.docker.io', description: 'Remote Docker registry URL', name: 'REMOTE_DOKCER_REGISTRY_URL', trim: true + string defaultValue: 'milvus-docker-access-token', description: 'Remote Docker credentials id', name: 'REMOTE_DOCKER_CREDENTIALS_ID', trim: true + string(defaultValue: "master", description: 'Milvus server version', name: 'BRANCH') + } + + environment { + DAILY_BUILD_VERSION = VersionNumber([ + versionNumberString : '${BUILD_DATE_FORMATTED, "yyyyMMdd"}' + ]); + } + + stages { + stage('Push Daily Docker Images') { + matrix { + agent none + axes { + axis { + name 'OS_NAME' + values 'centos7' + } + + axis { + name 'CPU_ARCH' + values 'amd64' + } + + axis { + name 'BINARY_VERSION' + values 'gpu', 'cpu' + } + } + + stages { + stage("Publish Docker Images") { + environment { + DOCKER_VERSION = "${params.BRANCH}-${BINARY_VERSION}-${OS_NAME}-release" + REMOTE_DOCKER_VERSION = "${params.BRANCH}-${OS_NAME}-${BINARY_VERSION}-${DAILY_BUILD_VERSION}" + REMOTE_DOCKER_LATEST_VERSION = "${params.BRANCH}-${OS_NAME}-${BINARY_VERSION}-latest" + } + + agent { + kubernetes { + label "${OS_NAME}-${BINARY_VERSION}-publish-${env.BUILD_NUMBER}" + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: publish + componet: docker +spec: + containers: + - name: publish-images + image: registry.zilliz.com/library/docker:v1.0.0 + securityContext: + privileged: true + command: + - cat + tty: true + resources: + limits: + memory: "4Gi" + cpu: "1.0" + requests: + memory: "2Gi" + cpu: "0.5" + volumeMounts: + - name: docker-sock + mountPath: /var/run/docker.sock + volumes: + - name: docker-sock + hostPath: + path: /var/run/docker.sock + """ + } + } + + stages { + stage('Publish') { + steps { + container('publish-images') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/publishDailyImages.groovy" + } + } + } + } + } + } + } + } + } + } +} diff --git a/tests/benchmark/ci/scripts/yaml_processor.py b/tests/benchmark/ci/scripts/yaml_processor.py new file mode 100755 index 0000000000..0e6d7dbbf4 --- /dev/null +++ b/tests/benchmark/ci/scripts/yaml_processor.py @@ -0,0 +1,536 @@ +#!/usr/bin/env python3 + +import sys +import argparse +from argparse import Namespace +import os, shutil +import getopt +from ruamel.yaml import YAML, yaml_object +from ruamel.yaml.comments import CommentedSeq, CommentedMap +from ruamel.yaml.tokens import CommentToken + +## +yaml = YAML(typ="rt") +## format yaml file +yaml.indent(mapping=2, sequence=4, offset=2) + + +############################################ +# Comment operation +# +############################################ +def _extract_comment(_comment): + """ + remove '#' at start of comment + """ + # if _comment is empty, do nothing + if not _comment: + return _comment + + # str_ = _comment.lstrip(" ") + str_ = _comment.strip() + str_ = str_.lstrip("#") + + return str_ + + +def _add_eol_comment(element, *args, **kwargs): + """ + add_eol_comment + args --> (comment, key) + """ + if element is None or \ + (not isinstance(element, CommentedMap) and + not isinstance(element, CommentedSeq)) or \ + args[0] is None or \ + len(args[0]) == 0: + return + + comment = args[0] + # comment is empty, do nothing + if not comment: + return + + key = args[1] + try: + element.yaml_add_eol_comment(*args, **kwargs) + except Exception: + element.ca.items.pop(key, None) + element.yaml_add_eol_comment(*args, **kwargs) + + +def _map_comment(_element, _key): + origin_comment = "" + token = _element.ca.items.get(_key, None) + if token is not None: + try: + origin_comment = token[2].value + except Exception: + try: + # comment is below element, add profix "#\n" + col = _element.lc.col + 2 + space_list = [" " for i in range(col)] + space_str = "".join(space_list) + + origin_comment = "\n" + "".join([space_str + t.value for t in token[3]]) + except Exception: + pass + + return origin_comment + + +def _seq_comment(_element, _index): + # get target comment + _comment = "" + token = _element.ca.items.get(_index, None) + if token is not None: + _comment = token[0].value + + return _comment + + +def _start_comment(_element): + _comment = "" + cmt = _element.ca.comment + try: + _comment = cmt[1][0].value + except Exception: + pass + + return _comment + + +def _comment_counter(_comment): + """ + + counter comment tips and split into list + """ + + x = lambda l: l.strip().strip("#").strip() + + _counter = [] + if _comment.startswith("\n"): + _counter.append("") + _counter.append(x(_comment[1:])) + + return _counter + elif _comment.startswith("#\n"): + _counter.append("") + _counter.append(x(_comment[2:])) + else: + index = _comment.find("\n") + _counter.append(x(_comment[:index])) + _counter.append(x(_comment[index + 1:])) + + return _counter + + +def _obtain_comment(_m_comment, _t_comment): + if not _m_comment or not _t_comment: + return _m_comment or _t_comment + + _m_counter = _comment_counter(_m_comment) + _t_counter = _comment_counter(_t_comment) + + if not _m_counter[0] and not _t_counter[1]: + comment = _t_comment + _m_comment + elif not _m_counter[1] and not _t_counter[0]: + comment = _m_comment + _t_comment + elif _t_counter[0] and _t_counter[1]: + comment = _t_comment + elif not _t_counter[0] and not _t_counter[1]: + comment = _m_comment + elif not _m_counter[0] and not _m_counter[1]: + comment = _t_comment + else: + if _t_counter[0]: + comment = _m_comment.replace(_m_counter[0], _t_counter[0], 1) + else: + comment = _m_comment.replace(_m_counter[1], _t_counter[1], 1) + + i = comment.find("\n\n") + while i >= 0: + comment = comment.replace("\n\n\n", "\n\n", 1) + i = comment.find("\n\n\n") + + return comment + + +############################################ +# Utils +# +############################################ +def _get_update_par(_args): + _dict = _args.__dict__ + + # file path + _in_file = _dict.get("f", None) or _dict.get("file", None) + # tips + _tips = _dict.get('tips', None) or "Input \"-h\" for more information" + # update + _u = _dict.get("u", None) or _dict.get("update", None) + # apppend + _a = _dict.get('a', None) or _dict.get('append', None) + # out stream group + _i = _dict.get("i", None) or _dict.get("inplace", None) + _o = _dict.get("o", None) or _dict.get("out_file", None) + + return _in_file, _u, _a, _i, _o, _tips + + +############################################ +# Element operation +# +############################################ +def update_map_element(element, key, value, comment, _type): + """ + element: + key: + value: + comment: + _type: value type. + """ + if element is None or not isinstance(element, CommentedMap): + print("Only key-value update support") + sys.exit(1) + + origin_comment = _map_comment(element, key) + + sub_element = element.get(key, None) + if isinstance(sub_element, CommentedMap) or isinstance(sub_element, CommentedSeq): + print("Only support update a single value") + + element.update({key: value}) + + comment = _obtain_comment(origin_comment, comment) + _add_eol_comment(element, _extract_comment(comment), key) + + +def update_seq_element(element, value, comment, _type): + if element is None or not isinstance(element, CommentedSeq): + print("Param `-a` only use to append yaml list") + sys.exit(1) + element.append(str(value)) + + comment = _obtain_comment("", comment) + _add_eol_comment(element, _extract_comment(comment), len(element) - 1) + + +def run_update(code, keys, value, comment, _app): + key_list = keys.split(".") + + space_str = ":\n " + key_str = "{}".format(key_list[0]) + for key in key_list[1:]: + key_str = key_str + space_str + key + space_str = space_str + " " + if not _app: + yaml_str = """{}: {}""".format(key_str, value) + else: + yaml_str = "{}{}- {}".format(key_str, space_str, value) + + if comment: + yaml_str = "{} # {}".format(yaml_str, comment) + + mcode = yaml.load(yaml_str) + + _merge(code, mcode) + + +def _update(code, _update, _app, _tips): + if not _update: + return code + + _update_list = [l.strip() for l in _update.split(",")] + for l in _update_list: + try: + variant, comment = l.split("#") + except ValueError: + variant = l + comment = None + + try: + keys, value = variant.split("=") + run_update(code, keys, value, comment, _app) + except ValueError: + print("Invalid format. print command \"--help\" get more info.") + sys.exit(1) + + return code + + +def _backup(in_file_p): + backup_p = in_file_p + ".bak" + + if os.path.exists(backup_p): + os.remove(backup_p) + + if not os.path.exists(in_file_p): + print("File {} not exists.".format(in_file_p)) + sys.exit(1) + + shutil.copyfile(in_file_p, backup_p) # 复制文件 + + +def _recovery(in_file_p): + backup_p = in_file_p + ".bak" + + if not os.path.exists(in_file_p): + print("File {} not exists.".format(in_file_p)) + sys.exit(1) + elif not os.path.exists(backup_p): + print("Backup file not exists") + sys.exit(0) + + os.remove(in_file_p) + + os.rename(backup_p, in_file_p) + + +# master merge target +def _merge(master, target): + if type(master) != type(target): + print("yaml format not match:\n") + yaml.dump(master, sys.stdout) + print("\n&&\n") + yaml.dump(target, sys.stdout) + + sys.exit(1) + + ## item is a sequence + if isinstance(target, CommentedSeq): + for index in range(len(target)): + # get target comment + target_comment = _seq_comment(target, index) + + master_index = len(master) + + target_item = target[index] + + if isinstance(target_item, CommentedMap): + merge_flag = False + for idx in range(len(master)): + if isinstance(master[idx], CommentedMap): + if master[idx].keys() == target_item.keys(): + _merge(master[idx], target_item) + # nonlocal merge_flag + master_index = idx + merge_flag = True + break + + if merge_flag is False: + master.append(target_item) + elif target_item not in master: + master.append(target[index]) + else: + # merge(master[index], target[index]) + pass + + # # remove enter signal in previous item + previous_comment = _seq_comment(master, master_index - 1) + _add_eol_comment(master, _extract_comment(previous_comment), master_index - 1) + + origin_comment = _seq_comment(master, master_index) + comment = _obtain_comment(origin_comment, target_comment) + if len(comment) > 0: + _add_eol_comment(master, _extract_comment(comment) + "\n\n", len(master) - 1) + + ## item is a map + elif isinstance(target, CommentedMap): + for item in target: + if item == "flag": + print("") + origin_comment = _map_comment(master, item) + target_comment = _map_comment(target, item) + + # get origin start comment + origin_start_comment = _start_comment(master) + + # get target start comment + target_start_comment = _start_comment(target) + + m = master.get(item, default=None) + if m is None or \ + (not (isinstance(m, CommentedMap) or + isinstance(m, CommentedSeq))): + master.update({item: target[item]}) + + else: + _merge(master[item], target[item]) + + comment = _obtain_comment(origin_comment, target_comment) + if len(comment) > 0: + _add_eol_comment(master, _extract_comment(comment), item) + + start_comment = _obtain_comment(origin_start_comment, target_start_comment) + if len(start_comment) > 0: + master.yaml_set_start_comment(_extract_comment(start_comment)) + + +def _save(_code, _file): + with open(_file, 'w') as wf: + yaml.dump(_code, wf) + + +def _load(_file): + with open(_file, 'r') as rf: + code = yaml.load(rf) + return code + + +############################################ +# sub parser process operation +# +############################################ +def merge_yaml(_args): + _dict = _args.__dict__ + + _m_file = _dict.get("merge_file", None) + _in_file, _u, _a, _i, _o, _tips = _get_update_par(_args) + + if not (_in_file and _m_file): + print(_tips) + sys.exit(1) + + code = _load(_in_file) + mcode = _load(_m_file) + + _merge(code, mcode) + + _update(code, _u, _a, _tips) + + if _i: + _backup(_in_file) + _save(code, _in_file) + elif _o: + _save(code, _o) + else: + print(_tips) + sys.exit(1) + + +def update_yaml(_args): + _in_file, _u, _a, _i, _o, _tips = _get_update_par(_args) + + if not _in_file or not _u: + print(_tips) + sys.exit(1) + + code = _load(_in_file) + + if _i and _o: + print(_tips) + sys.exit(1) + + _update(code, _u, _a, _tips) + + if _i: + _backup(_in_file) + _save(code, _in_file) + elif _o: + _save(code, _o) + + +def reset(_args): + _dict = _args.__dict__ + _f = _dict.get('f', None) or _dict.get('file', None) + + if _f: + _recovery(_f) + else: + _t = _dict.get('tips', None) or "Input \"-h\" for more information" + print(_t) + + +############################################ +# Cli operation +# +############################################ +def _set_merge_parser(_parsers): + """ + config merge parser + """ + + merge_parser = _parsers.add_parser("merge", help="merge with another yaml file") + + _set_merge_parser_arg(merge_parser) + _set_update_parser_arg(merge_parser) + + merge_parser.set_defaults( + function=merge_yaml, + tips=merge_parser.format_help() + ) + + +def _set_merge_parser_arg(_parser): + """ + config parser argument for merging + """ + + _parser.add_argument("-m", "--merge-file", help="indicate merge yaml file") + + +def _set_update_parser(_parsers): + """ + config merge parser + """ + + update_parser = _parsers.add_parser("update", help="update with another yaml file") + _set_update_parser_arg(update_parser) + + update_parser.set_defaults( + function=update_yaml, + tips=update_parser.format_help() + ) + + +def _set_update_parser_arg(_parser): + """ + config parser argument for updating + """ + + _parser.add_argument("-f", "--file", help="source yaml file") + _parser.add_argument('-u', '--update', help="update with args, instance as \"a.b.c=d# d comment\"") + _parser.add_argument('-a', '--append', action="store_true", help="append to a seq") + + group = _parser.add_mutually_exclusive_group() + group.add_argument("-o", "--out-file", help="indicate output yaml file") + group.add_argument("-i", "--inplace", action="store_true", help="indicate whether result store in origin file") + + +def _set_reset_parser(_parsers): + """ + config merge parser + """ + + reset_parser = _parsers.add_parser("reset", help="reset yaml file") + + # indicate yaml file + reset_parser.add_argument('-f', '--file', help="indicate input yaml file") + + reset_parser.set_defaults( + function=reset, + tips=reset_parser.format_help() + ) + + +def main(): + parser = argparse.ArgumentParser() + sub_parsers = parser.add_subparsers() + + # set merge command + _set_merge_parser(sub_parsers) + + # set update command + _set_update_parser(sub_parsers) + + # set reset command + _set_reset_parser(sub_parsers) + + # parse argument and run func + args = parser.parse_args() + args.function(args) + + +if __name__ == '__main__': + main() diff --git a/tests/benchmark/client.py b/tests/benchmark/client.py new file mode 100644 index 0000000000..e962de8048 --- /dev/null +++ b/tests/benchmark/client.py @@ -0,0 +1,460 @@ +import sys +import pdb +import random +import logging +import json +import time, datetime +import traceback +from multiprocessing import Process +from milvus import Milvus, DataType +import numpy as np +import utils + +logger = logging.getLogger("milvus_benchmark.client") + +SERVER_HOST_DEFAULT = "127.0.0.1" +SERVER_PORT_DEFAULT = 19530 +INDEX_MAP = { + "flat": "FLAT", + "ivf_flat": "IVF_FLAT", + "ivf_sq8": "IVF_SQ8", + "nsg": "NSG", + "ivf_sq8h": "IVF_SQ8_HYBRID", + "ivf_pq": "IVF_PQ", + "hnsw": "HNSW", + "annoy": "ANNOY", + "bin_flat": "BIN_FLAT", + "bin_ivf_flat": "BIN_IVF_FLAT", + "rhnsw_pq": "RHNSW_PQ", + "rhnsw_sq": "RHNSW_SQ" +} +epsilon = 0.1 + + +def time_wrapper(func): + """ + This decorator prints the execution time for the decorated function. + """ + + def wrapper(*args, **kwargs): + start = time.time() + # logger.debug("Milvus {} start".format(func.__name__)) + log = kwargs.get("log", True) + kwargs.pop("log", None) + result = func(*args, **kwargs) + end = time.time() + if log: + logger.debug("Milvus {} run in {}s".format(func.__name__, round(end - start, 2))) + return result + + return wrapper + + +class MilvusClient(object): + def __init__(self, collection_name=None, host=None, port=None, timeout=180): + self._collection_name = collection_name + start_time = time.time() + if not host: + host = SERVER_HOST_DEFAULT + if not port: + port = SERVER_PORT_DEFAULT + logger.debug(host) + logger.debug(port) + # retry connect remote server + i = 0 + while time.time() < start_time + timeout: + try: + self._milvus = Milvus( + host=host, + port=port, + try_connect=False, + pre_ping=False) + break + except Exception as e: + logger.error(str(e)) + logger.error("Milvus connect failed: %d times" % i) + i = i + 1 + time.sleep(i) + + if time.time() > start_time + timeout: + raise Exception("Server connect timeout") + # self._metric_type = None + + def __str__(self): + return 'Milvus collection %s' % self._collection_name + + def check_status(self, status): + if not status.OK(): + logger.error(status.message) + logger.error(self._milvus.server_status()) + logger.error(self.count()) + raise Exception("Status not ok") + + def check_result_ids(self, result): + for index, item in enumerate(result): + if item[0].distance >= epsilon: + logger.error(index) + logger.error(item[0].distance) + raise Exception("Distance wrong") + + # only support the given field name + def create_collection(self, dimension, data_type=DataType.FLOAT_VECTOR, auto_id=False, + collection_name=None, other_fields=None): + self._dimension = dimension + if not collection_name: + collection_name = self._collection_name + vec_field_name = utils.get_default_field_name(data_type) + fields = [{"name": vec_field_name, "type": data_type, "params": {"dim": dimension}}] + if other_fields: + other_fields = other_fields.split(",") + if "int" in other_fields: + fields.append({"name": utils.DEFAULT_INT_FIELD_NAME, "type": DataType.INT64}) + if "float" in other_fields: + fields.append({"name": utils.DEFAULT_FLOAT_FIELD_NAME, "type": DataType.FLOAT}) + create_param = { + "fields": fields, + "auto_id": auto_id} + try: + self._milvus.create_collection(collection_name, create_param) + logger.info("Create collection: <%s> successfully" % collection_name) + except Exception as e: + logger.error(str(e)) + raise + + def create_partition(self, tag, collection_name=None): + if not collection_name: + collection_name = self._collection_name + self._milvus.create_partition(collection_name, tag) + + def generate_values(self, data_type, vectors, ids): + values = None + if data_type in [DataType.INT32, DataType.INT64]: + values = ids + elif data_type in [DataType.FLOAT, DataType.DOUBLE]: + values = [(i + 0.0) for i in ids] + elif data_type in [DataType.FLOAT_VECTOR, DataType.BINARY_VECTOR]: + values = vectors + return values + + def generate_entities(self, vectors, ids=None, collection_name=None): + entities = [] + if collection_name is None: + collection_name = self._collection_name + info = self.get_info(collection_name) + for field in info["fields"]: + field_type = field["type"] + entities.append( + {"name": field["name"], "type": field_type, "values": self.generate_values(field_type, vectors, ids)}) + return entities + + @time_wrapper + def insert(self, entities, ids=None, collection_name=None): + tmp_collection_name = self._collection_name if collection_name is None else collection_name + try: + insert_ids = self._milvus.insert(tmp_collection_name, entities, ids=ids) + return insert_ids + except Exception as e: + logger.error(str(e)) + + def get_dimension(self): + info = self.get_info() + for field in info["fields"]: + if field["type"] in [DataType.FLOAT_VECTOR, DataType.BINARY_VECTOR]: + return field["params"]["dim"] + + def get_rand_ids(self, length): + segment_ids = [] + while True: + stats = self.get_stats() + segments = stats["partitions"][0]["segments"] + # random choice one segment + segment = random.choice(segments) + try: + segment_ids = self._milvus.list_id_in_segment(self._collection_name, segment["id"]) + except Exception as e: + logger.error(str(e)) + if not len(segment_ids): + continue + elif len(segment_ids) > length: + return random.sample(segment_ids, length) + else: + logger.debug("Reset length: %d" % len(segment_ids)) + return segment_ids + + # def get_rand_ids_each_segment(self, length): + # res = [] + # status, stats = self._milvus.get_collection_stats(self._collection_name) + # self.check_status(status) + # segments = stats["partitions"][0]["segments"] + # segments_num = len(segments) + # # random choice from each segment + # for segment in segments: + # status, segment_ids = self._milvus.list_id_in_segment(self._collection_name, segment["name"]) + # self.check_status(status) + # res.extend(segment_ids[:length]) + # return segments_num, res + + # def get_rand_entities(self, length): + # ids = self.get_rand_ids(length) + # status, get_res = self._milvus.get_entity_by_id(self._collection_name, ids) + # self.check_status(status) + # return ids, get_res + + def get(self): + get_ids = random.randint(1, 1000000) + self._milvus.get_entity_by_id(self._collection_name, [get_ids]) + + @time_wrapper + def get_entities(self, get_ids): + get_res = self._milvus.get_entity_by_id(self._collection_name, get_ids) + return get_res + + @time_wrapper + def delete(self, ids, collection_name=None): + tmp_collection_name = self._collection_name if collection_name is None else collection_name + self._milvus.delete_entity_by_id(tmp_collection_name, ids) + + def delete_rand(self): + delete_id_length = random.randint(1, 100) + count_before = self.count() + logger.debug("%s: length to delete: %d" % (self._collection_name, delete_id_length)) + delete_ids = self.get_rand_ids(delete_id_length) + self.delete(delete_ids) + self.flush() + logger.info("%s: count after delete: %d" % (self._collection_name, self.count())) + get_res = self._milvus.get_entity_by_id(self._collection_name, delete_ids) + for item in get_res: + assert not item + # if count_before - len(delete_ids) < self.count(): + # logger.error(delete_ids) + # raise Exception("Error occured") + + @time_wrapper + def flush(self,_async=False, collection_name=None): + tmp_collection_name = self._collection_name if collection_name is None else collection_name + self._milvus.flush([tmp_collection_name], _async=_async) + + @time_wrapper + def compact(self, collection_name=None): + tmp_collection_name = self._collection_name if collection_name is None else collection_name + status = self._milvus.compact(tmp_collection_name) + self.check_status(status) + + @time_wrapper + def create_index(self, field_name, index_type, metric_type, _async=False, index_param=None): + index_type = INDEX_MAP[index_type] + metric_type = utils.metric_type_trans(metric_type) + logger.info("Building index start, collection_name: %s, index_type: %s, metric_type: %s" % ( + self._collection_name, index_type, metric_type)) + if index_param: + logger.info(index_param) + index_params = { + "index_type": index_type, + "metric_type": metric_type, + "params": index_param + } + self._milvus.create_index(self._collection_name, field_name, index_params, _async=_async) + + # TODO: need to check + def describe_index(self, field_name): + # stats = self.get_stats() + info = self._milvus.describe_index(self._collection_name, field_name) + index_info = {"index_type": "flat", "index_param": None} + for field in info["fields"]: + for index in field['indexes']: + if not index or "index_type" not in index: + continue + else: + for k, v in INDEX_MAP.items(): + if index['index_type'] == v: + index_info['index_type'] = k + index_info['index_param'] = index['params'] + return index_info + return index_info + + def drop_index(self, field_name): + logger.info("Drop index: %s" % self._collection_name) + return self._milvus.drop_index(self._collection_name, field_name) + + @time_wrapper + def query(self, vector_query, filter_query=None, collection_name=None): + tmp_collection_name = self._collection_name if collection_name is None else collection_name + must_params = [vector_query] + if filter_query: + must_params.extend(filter_query) + query = { + "bool": {"must": must_params} + } + result = self._milvus.search(tmp_collection_name, query) + return result + + @time_wrapper + def load_and_query(self, vector_query, filter_query=None, collection_name=None): + tmp_collection_name = self._collection_name if collection_name is None else collection_name + must_params = [vector_query] + if filter_query: + must_params.extend(filter_query) + query = { + "bool": {"must": must_params} + } + self.load_collection(tmp_collection_name) + result = self._milvus.search(tmp_collection_name, query) + return result + + def get_ids(self, result): + idss = result._entities.ids + ids = [] + len_idss = len(idss) + len_r = len(result) + top_k = len_idss // len_r + for offset in range(0, len_idss, top_k): + ids.append(idss[offset: min(offset + top_k, len_idss)]) + return ids + + def query_rand(self, nq_max=100): + # for ivf search + dimension = 128 + top_k = random.randint(1, 100) + nq = random.randint(1, nq_max) + nprobe = random.randint(1, 100) + search_param = {"nprobe": nprobe} + query_vectors = [[random.random() for _ in range(dimension)] for _ in range(nq)] + metric_type = random.choice(["l2", "ip"]) + logger.info("%s, Search nq: %d, top_k: %d, nprobe: %d" % (self._collection_name, nq, top_k, nprobe)) + vec_field_name = utils.get_default_field_name() + vector_query = {"vector": {vec_field_name: { + "topk": top_k, + "query": query_vectors, + "metric_type": utils.metric_type_trans(metric_type), + "params": search_param} + }} + self.query(vector_query) + + def load_query_rand(self, nq_max=100): + # for ivf search + dimension = 128 + top_k = random.randint(1, 100) + nq = random.randint(1, nq_max) + nprobe = random.randint(1, 100) + search_param = {"nprobe": nprobe} + query_vectors = [[random.random() for _ in range(dimension)] for _ in range(nq)] + metric_type = random.choice(["l2", "ip"]) + logger.info("%s, Search nq: %d, top_k: %d, nprobe: %d" % (self._collection_name, nq, top_k, nprobe)) + vec_field_name = utils.get_default_field_name() + vector_query = {"vector": {vec_field_name: { + "topk": top_k, + "query": query_vectors, + "metric_type": utils.metric_type_trans(metric_type), + "params": search_param} + }} + self.load_and_query(vector_query) + + # TODO: need to check + def count(self, collection_name=None): + if collection_name is None: + collection_name = self._collection_name + row_count = self._milvus.get_collection_stats(collection_name)["row_count"] + logger.debug("Row count: %d in collection: <%s>" % (row_count, collection_name)) + return row_count + + def drop(self, timeout=120, collection_name=None): + timeout = int(timeout) + if collection_name is None: + collection_name = self._collection_name + logger.info("Start delete collection: %s" % collection_name) + self._milvus.drop_collection(collection_name) + i = 0 + while i < timeout: + try: + row_count = self.count(collection_name=collection_name) + if row_count: + time.sleep(1) + i = i + 1 + continue + else: + break + except Exception as e: + logger.debug(str(e)) + break + if i >= timeout: + logger.error("Delete collection timeout") + + def get_stats(self): + return self._milvus.get_collection_stats(self._collection_name) + + def get_info(self, collection_name=None): + # pdb.set_trace() + if collection_name is None: + collection_name = self._collection_name + return self._milvus.get_collection_info(collection_name) + + def show_collections(self): + return self._milvus.list_collections() + + def exists_collection(self, collection_name=None): + if collection_name is None: + collection_name = self._collection_name + res = self._milvus.has_collection(collection_name) + return res + + def clean_db(self): + collection_names = self.show_collections() + for name in collection_names: + self.drop(collection_name=name) + + @time_wrapper + def load_collection(self, collection_name=None): + if collection_name is None: + collection_name = self._collection_name + return self._milvus.load_collection(collection_name, timeout=3000) + + @time_wrapper + def release_collection(self, collection_name=None): + if collection_name is None: + collection_name = self._collection_name + return self._milvus.release_collection(collection_name, timeout=3000) + + @time_wrapper + def load_partitions(self, tag_names, collection_name=None): + if collection_name is None: + collection_name = self._collection_name + return self._milvus.load_partitions(collection_name, tag_names, timeout=3000) + + @time_wrapper + def release_partitions(self, tag_names, collection_name=None): + if collection_name is None: + collection_name = self._collection_name + return self._milvus.release_partitions(collection_name, tag_names, timeout=3000) + + # TODO: remove + # def get_server_version(self): + # return self._milvus.server_version() + + # def get_server_mode(self): + # return self.cmd("mode") + + # def get_server_commit(self): + # return self.cmd("build_commit_id") + + # def get_server_config(self): + # return json.loads(self.cmd("get_milvus_config")) + + # def get_mem_info(self): + # result = json.loads(self.cmd("get_system_info")) + # result_human = { + # # unit: Gb + # "memory_used": round(int(result["memory_used"]) / (1024 * 1024 * 1024), 2) + # } + # return result_human + + # def cmd(self, command): + # res = self._milvus._cmd(command) + # logger.info("Server command: %s, result: %s" % (command, res)) + # return res + + # @time_wrapper + # def set_config(self, parent_key, child_key, value): + # self._milvus.set_config(parent_key, child_key, value) + + # def get_config(self, key): + # return self._milvus.get_config(key) diff --git a/tests/benchmark/docker_runner.py b/tests/benchmark/docker_runner.py new file mode 100644 index 0000000000..406f3524d2 --- /dev/null +++ b/tests/benchmark/docker_runner.py @@ -0,0 +1,366 @@ +import os +import logging +import pdb +import time +import random +from multiprocessing import Process +import numpy as np +from client import MilvusClient +import utils +import parser +from runner import Runner + +logger = logging.getLogger("milvus_benchmark.docker") + + +class DockerRunner(Runner): + """run docker mode""" + def __init__(self, image): + super(DockerRunner, self).__init__() + self.image = image + + def run(self, definition, run_type=None): + if run_type == "performance": + for op_type, op_value in definition.items(): + # run docker mode + run_count = op_value["run_count"] + run_params = op_value["params"] + container = None + + if op_type == "insert": + if not run_params: + logger.debug("No run params") + continue + for index, param in enumerate(run_params): + logger.info("Definition param: %s" % str(param)) + collection_name = param["collection_name"] + volume_name = param["db_path_prefix"] + print(collection_name) + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + for k, v in param.items(): + if k.startswith("server."): + # Update server config + utils.modify_config(k, v, type="server", db_slave=None) + container = utils.run_server(self.image, test_type="remote", volume_name=volume_name, db_slave=None) + time.sleep(2) + milvus = MilvusClient(collection_name) + # Check has collection or not + if milvus.exists_collection(): + milvus.delete() + time.sleep(10) + milvus.create_collection(collection_name, dimension, index_file_size, metric_type) + # debug + # milvus.create_index("ivf_sq8", 16384) + res = self.do_insert(milvus, collection_name, data_type, dimension, collection_size, param["ni_per"]) + logger.info(res) + # wait for file merge + time.sleep(collection_size * dimension / 5000000) + # Clear up + utils.remove_container(container) + + elif op_type == "query": + for index, param in enumerate(run_params): + logger.info("Definition param: %s" % str(param)) + collection_name = param["dataset"] + volume_name = param["db_path_prefix"] + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + for k, v in param.items(): + if k.startswith("server."): + utils.modify_config(k, v, type="server") + container = utils.run_server(self.image, test_type="remote", volume_name=volume_name, db_slave=None) + time.sleep(2) + milvus = MilvusClient(collection_name) + logger.debug(milvus.show_collections()) + # Check has collection or not + if not milvus.exists_collection(): + logger.warning("Table %s not existed, continue exec next params ..." % collection_name) + continue + # parse index info + index_types = param["index.index_types"] + nlists = param["index.nlists"] + # parse top-k, nq, nprobe + top_ks, nqs, nprobes = parser.search_params_parser(param) + for index_type in index_types: + for nlist in nlists: + result = milvus.describe_index() + logger.info(result) + # milvus.drop_index() + # milvus.create_index(index_type, nlist) + result = milvus.describe_index() + logger.info(result) + logger.info(milvus.count()) + # preload index + milvus.preload_collection() + logger.info("Start warm up query") + res = self.do_query(milvus, collection_name, [1], [1], 1, 1) + logger.info("End warm up query") + # Run query test + for nprobe in nprobes: + logger.info("index_type: %s, nlist: %s, metric_type: %s, nprobe: %s" % (index_type, nlist, metric_type, nprobe)) + res = self.do_query(milvus, collection_name, top_ks, nqs, nprobe, run_count) + headers = ["Nq/Top-k"] + headers.extend([str(top_k) for top_k in top_ks]) + utils.print_collection(headers, nqs, res) + utils.remove_container(container) + + elif run_type == "insert_performance": + for op_type, op_value in definition.items(): + # run docker mode + run_count = op_value["run_count"] + run_params = op_value["params"] + container = None + if not run_params: + logger.debug("No run params") + continue + for index, param in enumerate(run_params): + logger.info("Definition param: %s" % str(param)) + collection_name = param["collection_name"] + volume_name = param["db_path_prefix"] + print(collection_name) + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + for k, v in param.items(): + if k.startswith("server."): + # Update server config + utils.modify_config(k, v, type="server", db_slave=None) + container = utils.run_server(self.image, test_type="remote", volume_name=volume_name, db_slave=None) + time.sleep(2) + milvus = MilvusClient(collection_name) + # Check has collection or not + if milvus.exists_collection(): + milvus.delete() + time.sleep(10) + milvus.create_collection(collection_name, dimension, index_file_size, metric_type) + # debug + # milvus.create_index("ivf_sq8", 16384) + res = self.do_insert(milvus, collection_name, data_type, dimension, collection_size, param["ni_per"]) + logger.info(res) + # wait for file merge + time.sleep(collection_size * dimension / 5000000) + # Clear up + utils.remove_container(container) + + elif run_type == "search_performance": + for op_type, op_value in definition.items(): + # run docker mode + run_count = op_value["run_count"] + run_params = op_value["params"] + container = None + for index, param in enumerate(run_params): + logger.info("Definition param: %s" % str(param)) + collection_name = param["dataset"] + volume_name = param["db_path_prefix"] + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + for k, v in param.items(): + if k.startswith("server."): + utils.modify_config(k, v, type="server") + container = utils.run_server(self.image, test_type="remote", volume_name=volume_name, db_slave=None) + time.sleep(2) + milvus = MilvusClient(collection_name) + logger.debug(milvus.show_collections()) + # Check has collection or not + if not milvus.exists_collection(): + logger.warning("Table %s not existed, continue exec next params ..." % collection_name) + continue + # parse index info + index_types = param["index.index_types"] + nlists = param["index.nlists"] + # parse top-k, nq, nprobe + top_ks, nqs, nprobes = parser.search_params_parser(param) + for index_type in index_types: + for nlist in nlists: + result = milvus.describe_index() + logger.info(result) + # milvus.drop_index() + # milvus.create_index(index_type, nlist) + result = milvus.describe_index() + logger.info(result) + logger.info(milvus.count()) + # preload index + milvus.preload_collection() + logger.info("Start warm up query") + res = self.do_query(milvus, collection_name, [1], [1], 1, 1) + logger.info("End warm up query") + # Run query test + for nprobe in nprobes: + logger.info("index_type: %s, nlist: %s, metric_type: %s, nprobe: %s" % (index_type, nlist, metric_type, nprobe)) + res = self.do_query(milvus, collection_name, top_ks, nqs, nprobe, run_count) + headers = ["Nq/Top-k"] + headers.extend([str(top_k) for top_k in top_ks]) + utils.print_collection(headers, nqs, res) + utils.remove_container(container) + + elif run_type == "accuracy": + """ + { + "dataset": "random_50m_1024_512", + "index.index_types": ["flat", ivf_flat", "ivf_sq8"], + "index.nlists": [16384], + "nprobes": [1, 32, 128], + "nqs": [100], + "top_ks": [1, 64], + "server.use_blas_threshold": 1100, + "server.cpu_cache_capacity": 256 + } + """ + for op_type, op_value in definition.items(): + if op_type != "query": + logger.warning("invalid operation: %s in accuracy test, only support query operation" % op_type) + break + run_count = op_value["run_count"] + run_params = op_value["params"] + container = None + + for index, param in enumerate(run_params): + logger.info("Definition param: %s" % str(param)) + collection_name = param["dataset"] + sift_acc = False + if "sift_acc" in param: + sift_acc = param["sift_acc"] + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + for k, v in param.items(): + if k.startswith("server."): + utils.modify_config(k, v, type="server") + volume_name = param["db_path_prefix"] + container = utils.run_server(self.image, test_type="remote", volume_name=volume_name, db_slave=None) + time.sleep(2) + milvus = MilvusClient(collection_name) + # Check has collection or not + if not milvus.exists_collection(): + logger.warning("Table %s not existed, continue exec next params ..." % collection_name) + continue + + # parse index info + index_types = param["index.index_types"] + nlists = param["index.nlists"] + # parse top-k, nq, nprobe + top_ks, nqs, nprobes = parser.search_params_parser(param) + if sift_acc is True: + # preload groundtruth data + true_ids_all = self.get_groundtruth_ids(collection_size) + acc_dict = {} + for index_type in index_types: + for nlist in nlists: + result = milvus.describe_index() + logger.info(result) + milvus.create_index(index_type, nlist) + # preload index + milvus.preload_collection() + # Run query test + for nprobe in nprobes: + logger.info("index_type: %s, nlist: %s, metric_type: %s, nprobe: %s" % (index_type, nlist, metric_type, nprobe)) + for top_k in top_ks: + for nq in nqs: + result_ids = [] + id_prefix = "%s_index_%s_nlist_%s_metric_type_%s_nprobe_%s_top_k_%s_nq_%s" % \ + (collection_name, index_type, nlist, metric_type, nprobe, top_k, nq) + if sift_acc is False: + self.do_query_acc(milvus, collection_name, top_k, nq, nprobe, id_prefix) + if index_type != "flat": + # Compute accuracy + base_name = "%s_index_flat_nlist_%s_metric_type_%s_nprobe_%s_top_k_%s_nq_%s" % \ + (collection_name, nlist, metric_type, nprobe, top_k, nq) + avg_acc = self.compute_accuracy(base_name, id_prefix) + logger.info("Query: <%s> accuracy: %s" % (id_prefix, avg_acc)) + else: + result_ids, result_distances = self.do_query_ids(milvus, collection_name, top_k, nq, nprobe) + debug_file_ids = "0.5.3_result_ids" + debug_file_distances = "0.5.3_result_distances" + with open(debug_file_ids, "w+") as fd: + total = 0 + for index, item in enumerate(result_ids): + true_item = true_ids_all[:nq, :top_k].tolist()[index] + tmp = set(item).intersection(set(true_item)) + total = total + len(tmp) + fd.write("query: N-%d, intersection: %d, total: %d\n" % (index, len(tmp), total)) + fd.write("%s\n" % str(item)) + fd.write("%s\n" % str(true_item)) + acc_value = self.get_recall_value(true_ids_all[:nq, :top_k].tolist(), result_ids) + logger.info("Query: <%s> accuracy: %s" % (id_prefix, acc_value)) + # # print accuracy collection + # headers = [collection_name] + # headers.extend([str(top_k) for top_k in top_ks]) + # utils.print_collection(headers, nqs, res) + + # remove container, and run next definition + logger.info("remove container, and run next definition") + utils.remove_container(container) + + elif run_type == "stability": + for op_type, op_value in definition.items(): + if op_type != "query": + logger.warning("invalid operation: %s in accuracy test, only support query operation" % op_type) + break + run_count = op_value["run_count"] + run_params = op_value["params"] + container = None + for index, param in enumerate(run_params): + logger.info("Definition param: %s" % str(param)) + collection_name = param["dataset"] + index_type = param["index_type"] + volume_name = param["db_path_prefix"] + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + + # set default test time + if "during_time" not in param: + during_time = 100 # seconds + else: + during_time = int(param["during_time"]) * 60 + # set default query process num + if "query_process_num" not in param: + query_process_num = 10 + else: + query_process_num = int(param["query_process_num"]) + + for k, v in param.items(): + if k.startswith("server."): + utils.modify_config(k, v, type="server") + + container = utils.run_server(self.image, test_type="remote", volume_name=volume_name, db_slave=None) + time.sleep(2) + milvus = MilvusClient(collection_name) + # Check has collection or not + if not milvus.exists_collection(): + logger.warning("Table %s not existed, continue exec next params ..." % collection_name) + continue + + start_time = time.time() + insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(10000)] + i = 0 + while time.time() < start_time + during_time: + i = i + 1 + processes = [] + # do query + # for i in range(query_process_num): + # milvus_instance = MilvusClient(collection_name) + # top_k = random.choice([x for x in range(1, 100)]) + # nq = random.choice([x for x in range(1, 100)]) + # nprobe = random.choice([x for x in range(1, 1000)]) + # # logger.info("index_type: %s, nlist: %s, metric_type: %s, nprobe: %s" % (index_type, nlist, metric_type, nprobe)) + # p = Process(target=self.do_query, args=(milvus_instance, collection_name, [top_k], [nq], [nprobe], run_count, )) + # processes.append(p) + # p.start() + # time.sleep(0.1) + # for p in processes: + # p.join() + milvus_instance = MilvusClient(collection_name) + top_ks = random.sample([x for x in range(1, 100)], 3) + nqs = random.sample([x for x in range(1, 1000)], 3) + nprobe = random.choice([x for x in range(1, 500)]) + res = self.do_query(milvus, collection_name, top_ks, nqs, nprobe, run_count) + if i % 10 == 0: + status, res = milvus_instance.insert(insert_vectors, ids=[x for x in range(len(insert_vectors))]) + if not status.OK(): + logger.error(status) + # status = milvus_instance.drop_index() + # if not status.OK(): + # logger.error(status) + # index_type = random.choice(["flat", "ivf_flat", "ivf_sq8"]) + milvus_instance.create_index(index_type, 16384) + result = milvus.describe_index() + logger.info(result) + # milvus_instance.create_index("ivf_sq8", 16384) + utils.remove_container(container) + + else: + logger.warning("Run type: %s not supported" % run_type) + diff --git a/tests/benchmark/docker_utils.py b/tests/benchmark/docker_utils.py new file mode 100644 index 0000000000..504a7f51c9 --- /dev/null +++ b/tests/benchmark/docker_utils.py @@ -0,0 +1,126 @@ +# def pull_image(image): +# registry = image.split(":")[0] +# image_tag = image.split(":")[1] +# client = docker.APIClient(base_url='unix://var/run/docker.sock') +# logger.info("Start pulling image: %s" % image) +# return client.pull(registry, image_tag) + + +# def run_server(image, mem_limit=None, timeout=30, test_type="local", volume_name=None, db_slave=None): +# import colors + +# client = docker.from_env() +# # if mem_limit is None: +# # mem_limit = psutil.virtual_memory().available +# # logger.info('Memory limit:', mem_limit) +# # cpu_limit = "0-%d" % (multiprocessing.cpu_count() - 1) +# # logger.info('Running on CPUs:', cpu_limit) +# for dir_item in ['logs', 'db']: +# try: +# os.mkdir(os.path.abspath(dir_item)) +# except Exception as e: +# pass + +# if test_type == "local": +# volumes = { +# os.path.abspath('conf'): +# {'bind': '/opt/milvus/conf', 'mode': 'ro'}, +# os.path.abspath('logs'): +# {'bind': '/opt/milvus/logs', 'mode': 'rw'}, +# os.path.abspath('db'): +# {'bind': '/opt/milvus/db', 'mode': 'rw'}, +# } +# elif test_type == "remote": +# if volume_name is None: +# raise Exception("No volume name") +# remote_log_dir = volume_name+'/logs' +# remote_db_dir = volume_name+'/db' + +# for dir_item in [remote_log_dir, remote_db_dir]: +# if not os.path.isdir(dir_item): +# os.makedirs(dir_item, exist_ok=True) +# volumes = { +# os.path.abspath('conf'): +# {'bind': '/opt/milvus/conf', 'mode': 'ro'}, +# remote_log_dir: +# {'bind': '/opt/milvus/logs', 'mode': 'rw'}, +# remote_db_dir: +# {'bind': '/opt/milvus/db', 'mode': 'rw'} +# } +# # add volumes +# if db_slave and isinstance(db_slave, int): +# for i in range(2, db_slave+1): +# remote_db_dir = volume_name+'/data'+str(i) +# if not os.path.isdir(remote_db_dir): +# os.makedirs(remote_db_dir, exist_ok=True) +# volumes[remote_db_dir] = {'bind': '/opt/milvus/data'+str(i), 'mode': 'rw'} + +# container = client.containers.run( +# image, +# volumes=volumes, +# runtime="nvidia", +# ports={'19530/tcp': 19530, '8080/tcp': 8080}, +# # environment=["OMP_NUM_THREADS=48"], +# # cpuset_cpus=cpu_limit, +# # mem_limit=mem_limit, +# # environment=[""], +# detach=True) + +# def stream_logs(): +# for line in container.logs(stream=True): +# logger.info(colors.color(line.decode().rstrip(), fg='blue')) + +# if sys.version_info >= (3, 0): +# t = threading.Thread(target=stream_logs, daemon=True) +# else: +# t = threading.Thread(target=stream_logs) +# t.daemon = True +# t.start() + +# logger.info('Container: %s started' % container) +# return container +# # exit_code = container.wait(timeout=timeout) +# # # Exit if exit code +# # if exit_code == 0: +# # return container +# # elif exit_code is not None: +# # print(colors.color(container.logs().decode(), fg='red')) + +# def restart_server(container): +# client = docker.APIClient(base_url='unix://var/run/docker.sock') + +# client.restart(container.name) +# logger.info('Container: %s restarted' % container.name) +# return container + + +# def remove_container(container): +# container.remove(force=True) +# logger.info('Container: %s removed' % container) + + +# def remove_all_containers(image): +# client = docker.from_env() +# try: +# for container in client.containers.list(): +# if image in container.image.tags: +# container.stop(timeout=30) +# container.remove(force=True) +# except Exception as e: +# logger.error("Containers removed failed") + + +# def container_exists(image): +# ''' +# Check if container existed with the given image name +# @params: image name +# @return: container if exists +# ''' +# res = False +# client = docker.from_env() +# for container in client.containers.list(): +# if image in container.image.tags: +# # True +# res = container +# return res + diff --git a/tests/benchmark/executors/__init__.py b/tests/benchmark/executors/__init__.py new file mode 100644 index 0000000000..dc96d174c5 --- /dev/null +++ b/tests/benchmark/executors/__init__.py @@ -0,0 +1,3 @@ + +class BaseExecutor(object): + pass \ No newline at end of file diff --git a/tests/benchmark/executors/shell.py b/tests/benchmark/executors/shell.py new file mode 100644 index 0000000000..cb425b9b54 --- /dev/null +++ b/tests/benchmark/executors/shell.py @@ -0,0 +1,4 @@ +from . import BaseExecutor + +class ShellExecutor(BaseExecutor): + pass \ No newline at end of file diff --git a/tests/benchmark/handlers/__init__.py b/tests/benchmark/handlers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/benchmark/helm_utils.py b/tests/benchmark/helm_utils.py new file mode 100644 index 0000000000..c69801a745 --- /dev/null +++ b/tests/benchmark/helm_utils.py @@ -0,0 +1,370 @@ +import os +import pdb +import time +import logging +import hashlib +from yaml import full_load, dump +import utils + +logger = logging.getLogger("milvus_benchmark.utils") +REGISTRY_URL = "registry.zilliz.com/milvus/engine" +IDC_NAS_URL = "//172.16.70.249/test" +NAS_URL = "//192.168.1.126/test" + + +def get_host_cpus(hostname): + from kubernetes import client, config + config.load_kube_config() + client.rest.logger.setLevel(logging.WARNING) + v1 = client.CoreV1Api() + cpus = v1.read_node(hostname).status.allocatable.get("cpu") + return cpus + + +# update values.yaml +def update_values(file_path, deploy_mode, hostname, milvus_config, server_config=None): + if not os.path.isfile(file_path): + raise Exception('File: %s not found' % file_path) + # 　bak values.yaml + file_name = os.path.basename(file_path) + bak_file_name = file_name + ".bak" + file_parent_path = os.path.dirname(file_path) + bak_file_path = file_parent_path + '/' + bak_file_name + if os.path.exists(bak_file_path): + os.system("cp %s %s" % (bak_file_path, file_path)) + else: + os.system("cp %s %s" % (file_path, bak_file_path)) + with open(file_path) as f: + values_dict = full_load(f) + f.close() + cluster = False + if "cluster" in milvus_config and milvus_config["cluster"]: + cluster = True + for k, v in milvus_config.items(): + if k.find("primary_path") != -1: + suffix_path = milvus_config["suffix_path"] if "suffix_path" in milvus_config else None + path_value = v + if suffix_path: + path_value = v + "_" + str(int(time.time())) + values_dict["primaryPath"] = path_value + values_dict['wal']['path'] = path_value + "/wal" + values_dict['logs']['path'] = path_value + "/logs" + # elif k.find("use_blas_threshold") != -1: + # values_dict['useBLASThreshold'] = int(v) + elif k.find("gpu_search_threshold") != -1: + values_dict['gpu']['gpuSearchThreshold'] = int(v) + if cluster: + values_dict['readonly']['gpu']['gpuSearchThreshold'] = int(v) + elif k.find("cpu_cache_capacity") != -1: + values_dict['cache']['cacheSize'] = v + if cluster: + values_dict['readonly']['cache']['cacheSize'] = v + # elif k.find("cache_insert_data") != -1: + # values_dict['cache']['cacheInsertData'] = v + elif k.find("insert_buffer_size") != -1: + values_dict['cache']['insertBufferSize'] = v + if cluster: + values_dict['readonly']['cache']['insertBufferSize'] = v + elif k.find("gpu_resource_config.enable") != -1: + values_dict['gpu']['enabled'] = v + if cluster: + values_dict['readonly']['gpu']['enabled'] = v + elif k.find("gpu_resource_config.cache_capacity") != -1: + values_dict['gpu']['cacheSize'] = v + if cluster: + values_dict['readonly']['gpu']['cacheSize'] = v + elif k.find("build_index_resources") != -1: + values_dict['gpu']['buildIndexDevices'] = v + if cluster: + values_dict['readonly']['gpu']['buildIndexDevices'] = v + elif k.find("search_resources") != -1: + values_dict['gpu']['searchDevices'] = v + if cluster: + values_dict['readonly']['gpu']['searchDevices'] = v + # wal + elif k.find("auto_flush_interval") != -1: + values_dict['storage']['autoFlushInterval'] = v + if cluster: + values_dict['readonly']['storage']['autoFlushInterval'] = v + elif k.find("wal_enable") != -1: + values_dict['wal']['enabled'] = v + + # if values_dict['nodeSelector']: + # logger.warning("nodeSelector has been set: %s" % str(values_dict['engine']['nodeSelector'])) + # return + values_dict["wal"]["recoveryErrorIgnore"] = True + # enable monitor + values_dict["metrics"]["enabled"] = True + values_dict["metrics"]["address"] = "192.168.1.237" + values_dict["metrics"]["port"] = 9091 + # only test avx2 + values_dict["extraConfiguration"].update({"engine": {"simd_type": "avx2"}}) + # stat_optimizer_enable + values_dict["extraConfiguration"]["engine"].update({"stat_optimizer_enable": False}) + + # enable read-write mode + if cluster: + values_dict["cluster"]["enabled"] = True + # update readonly log path + values_dict["readonly"]['logs']['path'] = values_dict['logs']['path'] + "/readonly" + if "readonly" in milvus_config: + if "replicas" in milvus_config["readonly"]: + values_dict["readonly"]["replicas"] = milvus_config["readonly"]["replicas"] + + use_external_mysql = False + if "external_mysql" in milvus_config and milvus_config["external_mysql"]: + use_external_mysql = True + # meta mysql + if use_external_mysql: + values_dict["mysql"]["enabled"] = False + # values_dict["mysql"]["persistence"]["enabled"] = True + # values_dict["mysql"]["persistence"]["existingClaim"] = hashlib.md5(path_value.encode(encoding='UTF-8')).hexdigest() + values_dict['externalMysql']['enabled'] = True + if deploy_mode == "local": + values_dict['externalMysql']["ip"] = "192.168.1.238" + else: + values_dict['externalMysql']["ip"] = "milvus-mysql.test" + values_dict['externalMysql']["port"] = 3306 + values_dict['externalMysql']["user"] = "root" + values_dict['externalMysql']["password"] = "milvus" + values_dict['externalMysql']["database"] = "db" + else: + values_dict["mysql"]["enabled"] = False + # update values.yaml with the given host + nas_url = NAS_URL + if hostname: + nas_url = IDC_NAS_URL + values_dict['nodeSelector'] = {'kubernetes.io/hostname': hostname} + cpus = server_config["cpus"] + + # set limit/request cpus in resources + values_dict["image"]['resources'] = { + "limits": { + # "cpu": str(int(cpus)) + ".0" + "cpu": str(int(cpus)) + ".0" + }, + "requests": { + # "cpu": str(int(cpus) // 2) + ".0" + "cpu": "4.0" + } + } + # update readonly resouces limits/requests + values_dict["readonly"]['resources'] = { + "limits": { + # "cpu": str(int(cpus)) + ".0" + "cpu": str(int(cpus)) + ".0" + }, + "requests": { + # "cpu": str(int(cpus) // 2) + ".0" + "cpu": "4.0" + } + } + values_dict['tolerations'] = [{ + "key": "worker", + "operator": "Equal", + "value": "performance", + "effect": "NoSchedule" + }] + # add extra volumes + values_dict['extraVolumes'] = [{ + 'name': 'test', + 'flexVolume': { + 'driver': "fstab/cifs", + 'fsType': "cifs", + 'secretRef': { + 'name': "cifs-test-secret" + }, + 'options': { + 'networkPath': nas_url, + 'mountOptions': "vers=1.0" + } + } + }] + values_dict['extraVolumeMounts'] = [{ + 'name': 'test', + 'mountPath': '/test' + }] + + # add extra volumes for mysql + # values_dict['mysql']['persistence']['enabled'] = True + # values_dict['mysql']['configurationFilesPath'] = "/etc/mysql/mysql.conf.d/" + # values_dict['mysql']['imageTag'] = '5.6' + # values_dict['mysql']['securityContext'] = { + # 'enabled': True} + # mysql_db_path = "/test" + if deploy_mode == "cluster" and use_external_mysql: + # mount_path = values_dict["primaryPath"]+'/data' + # long_str = '- name: test-mysql\n flexVolume:\n driver: fstab/cifs\n fsType: cifs\n secretRef:\n name: cifs-test-secret\n options:\n networkPath: //192.168.1.126/test\n mountOptions: vers=1.0' + # values_dict['mysql']['extraVolumes'] = literal_str(long_str) + # long_str_2 = "- name: test-mysql\n mountPath: %s" % mysql_db_path + # values_dict['mysql']['extraVolumeMounts'] = literal_str(long_str_2) + # mysql_cnf_str = '[mysqld]\npid-file=%s/mysql.pid\ndatadir=%s' % (mount_path, mount_path) + # values_dict['mysql']['configurationFiles'] = {} + # values_dict['mysql']['configurationFiles']['mysqld.cnf'] = literal_str(mysql_cnf_str) + + values_dict['mysql']['enabled'] = False + values_dict['externalMysql']['enabled'] = True + values_dict['externalMysql']["ip"] = "192.168.1.197" + values_dict['externalMysql']["port"] = 3306 + values_dict['externalMysql']["user"] = "root" + values_dict['externalMysql']["password"] = "Fantast1c" + values_dict['externalMysql']["database"] = "db" + + # logger.debug(values_dict) + # print(dump(values_dict)) + with open(file_path, 'w') as f: + dump(values_dict, f, default_flow_style=False) + f.close() + # DEBUG + with open(file_path) as f: + for line in f.readlines(): + line = line.strip("\n") + + +# deploy server +def helm_install_server(helm_path, deploy_mode, image_tag, image_type, name, namespace): + timeout = 300 + logger.debug("Server deploy mode: %s" % deploy_mode) + host = "%s.%s.svc.cluster.local" % (name, namespace) + if deploy_mode == "single": + install_cmd = "helm install \ + --set image.repository=%s \ + --set image.tag=%s \ + --set image.pullPolicy=Always \ + --set service.type=ClusterIP \ + -f ci/filebeat/values.yaml \ + --namespace %s \ + %s ." % (REGISTRY_URL, image_tag, namespace, name) + elif deploy_mode == "cluster": + install_cmd = "helm install \ + --set cluster.enabled=true \ + --set persistence.enabled=true \ + --set mishards.image.tag=test \ + --set mishards.image.pullPolicy=Always \ + --set image.repository=%s \ + --set image.tag=%s \ + --set image.pullPolicy=Always \ + --set service.type=ClusterIP \ + -f ci/filebeat/values.yaml \ + --namespace %s \ + %s ." % (REGISTRY_URL, image_tag, namespace, name) + logger.debug(install_cmd) + logger.debug(host) + if os.system("cd %s && %s" % (helm_path, install_cmd)): + logger.error("Helm install failed: %s" % name) + return None + time.sleep(30) + # config.load_kube_config() + # v1 = client.CoreV1Api() + # pod_name = None + # pod_id = None + # pods = v1.list_namespaced_pod(namespace) + # for i in pods.items: + # if i.metadata.name.find(name) != -1: + # pod_name = i.metadata.name + # pod_ip = i.status.pod_ip + # logger.debug(pod_name) + # logger.debug(pod_ip) + # return pod_name, pod_ip + return host + + +# delete server +@utils.retry(3) +def helm_del_server(name, namespace): + # logger.debug("Sleep 600s before uninstall server") + # time.sleep(600) + del_cmd = "helm uninstall -n milvus %s" % name + logger.info(del_cmd) + if os.system(del_cmd): + logger.error("Helm delete name:%s failed" % name) + return False + return True + + +def restart_server(helm_release_name, namespace): + res = True + timeout = 120000 + # service_name = "%s.%s.svc.cluster.local" % (helm_release_name, namespace) + config.load_kube_config() + v1 = client.CoreV1Api() + pod_name = None + # config_map_names = v1.list_namespaced_config_map(namespace, pretty='true') + # body = {"replicas": 0} + pods = v1.list_namespaced_pod(namespace) + for i in pods.items: + if i.metadata.name.find(helm_release_name) != -1 and i.metadata.name.find("mysql") == -1: + pod_name = i.metadata.name + break + # v1.patch_namespaced_config_map(config_map_name, namespace, body, pretty='true') + # status_res = v1.read_namespaced_service_status(helm_release_name, namespace, pretty='true') + logger.debug("Pod name: %s" % pod_name) + if pod_name is not None: + try: + v1.delete_namespaced_pod(pod_name, namespace) + except Exception as e: + logger.error(str(e)) + logger.error("Exception when calling CoreV1Api->delete_namespaced_pod") + res = False + return res + logger.error("Sleep 10s after pod deleted") + time.sleep(10) + # check if restart successfully + pods = v1.list_namespaced_pod(namespace) + for i in pods.items: + pod_name_tmp = i.metadata.name + logger.error(pod_name_tmp) + if pod_name_tmp == pod_name: + continue + elif pod_name_tmp.find(helm_release_name) == -1 or pod_name_tmp.find("mysql") != -1: + continue + else: + status_res = v1.read_namespaced_pod_status(pod_name_tmp, namespace, pretty='true') + logger.error(status_res.status.phase) + start_time = time.time() + ready_break = False + while time.time() - start_time <= timeout: + logger.error(time.time()) + status_res = v1.read_namespaced_pod_status(pod_name_tmp, namespace, pretty='true') + if status_res.status.phase == "Running": + logger.error("Already running") + ready_break = True + break + else: + time.sleep(5) + if time.time() - start_time > timeout: + logger.error("Restart pod: %s timeout" % pod_name_tmp) + res = False + return res + if ready_break: + break + else: + raise Exception("Pod: %s not found" % pod_name) + follow = True + pretty = True + previous = True # bool | Return previous terminated container logs. Defaults to false. (optional) + since_seconds = 56 # int | A relative time in seconds before the current time from which to show logs. If this value precedes the time a pod was started, only logs since the pod start will be returned. If this value is in the future, no logs will be returned. Only one of sinceSeconds or sinceTime may be specified. (optional) + timestamps = True # bool | If true, add an RFC3339 or RFC3339Nano timestamp at the beginning of every line of log output. Defaults to false. (optional) + container = "milvus" + # start_time = time.time() + # while time.time() - start_time <= timeout: + # try: + # api_response = v1.read_namespaced_pod_log(pod_name_tmp, namespace, container=container, follow=follow, + # pretty=pretty, previous=previous, since_seconds=since_seconds, + # timestamps=timestamps) + # logging.error(api_response) + # return res + # except Exception as e: + # logging.error("Exception when calling CoreV1Api->read_namespaced_pod_log: %s\n" % e) + # # waiting for server start + # time.sleep(2) + # # res = False + # # return res + # if time.time() - start_time > timeout: + # logging.error("Restart pod: %s timeout" % pod_name_tmp) + # res = False + return res + + +if __name__ == '__main__': + print(type(get_host_cpus("idc-sh002"))) diff --git a/tests/benchmark/k8s_runner.py b/tests/benchmark/k8s_runner.py new file mode 100644 index 0000000000..5e2828a7be --- /dev/null +++ b/tests/benchmark/k8s_runner.py @@ -0,0 +1,927 @@ +import os +import logging +import pdb +import time +import re +import random +import traceback +import json +import csv +import threading +from multiprocessing import Process +import numpy as np +from milvus import DataType +from yaml import full_load, dump +import concurrent.futures + +import locust_user +from client import MilvusClient +import parser +from runner import Runner +from milvus_metrics.api import report +from milvus_metrics.models import Env, Hardware, Server, Metric +import helm_utils +import utils + +logger = logging.getLogger("milvus_benchmark.k8s_runner") +namespace = "milvus" +default_port = 19530 +DELETE_INTERVAL_TIME = 5 +# INSERT_INTERVAL = 100000 +INSERT_INTERVAL = 50000 +BIG_FLUSH_INTERVAL = 3600 +DEFAULT_FLUSH_INTERVAL = 1 +timestamp = int(time.time()) +default_path = "/var/lib/milvus" + + +class K8sRunner(Runner): + """run docker mode""" + + def __init__(self): + super(K8sRunner, self).__init__() + self.service_name = utils.get_unique_name() + self.host = None + self.port = default_port + self.hostname = None + self.env_value = None + self.hardware = None + self.deploy_mode = None + + def init_env(self, milvus_config, server_config, server_host, deploy_mode, image_type, image_tag): + logger.debug("Tests run on server host:") + logger.debug(server_host) + self.hostname = server_host + self.deploy_mode = deploy_mode + if self.hostname: + try: + cpus = helm_utils.get_host_cpus(self.hostname) + except Exception as e: + logger.error(str(e)) + cpus = 64 + logger.debug(type(cpus)) + if server_config: + if "cpus" in server_config.keys(): + cpus = min(server_config["cpus"], int(cpus)) + else: + server_config.update({"cpus": cpus}) + else: + server_config = {"cpus": cpus} + self.hardware = Hardware(name=self.hostname, cpus=cpus) + # update values + helm_path = os.path.join(os.getcwd(), "../milvus-helm/charts/milvus") + values_file_path = helm_path + "/values.yaml" + if not os.path.exists(values_file_path): + raise Exception("File %s not existed" % values_file_path) + if milvus_config: + helm_utils.update_values(values_file_path, deploy_mode, server_host, milvus_config, server_config) + try: + logger.debug("Start install server") + self.host = helm_utils.helm_install_server(helm_path, deploy_mode, image_tag, image_type, self.service_name, + namespace) + except Exception as e: + logger.error("Helm install server failed: %s" % (str(e))) + logger.error(traceback.format_exc()) + logger.error(self.hostname) + self.clean_up() + return False + logger.debug(server_config) + # for debugging + if not self.host: + logger.error("Helm install server failed") + self.clean_up() + return False + return True + + def clean_up(self): + logger.debug("Start clean up: %s" % self.service_name) + helm_utils.helm_del_server(self.service_name, namespace) + + def report_wrapper(self, milvus_instance, env_value, hostname, collection_info, index_info, search_params, + run_params=None, server_config=None): + metric = Metric() + metric.set_run_id(timestamp) + metric.env = Env(env_value) + metric.env.OMP_NUM_THREADS = 0 + metric.hardware = self.hardware + # TODO: removed + # server_version = milvus_instance.get_server_version() + # server_mode = milvus_instance.get_server_mode() + # commit = milvus_instance.get_server_commit() + server_version = "0.12.0" + server_mode = self.deploy_mode + metric.server = Server(version=server_version, mode=server_mode, build_commit=None) + metric.collection = collection_info + metric.index = index_info + metric.search = search_params + metric.run_params = run_params + return metric + + def run(self, run_type, collection): + logger.debug(run_type) + logger.debug(collection) + collection_name = collection["collection_name"] if "collection_name" in collection else None + milvus_instance = MilvusClient(collection_name=collection_name, host=self.host) + + # TODO: removed + # self.env_value = milvus_instance.get_server_config() + # ugly implemention + # self.env_value = utils.convert_nested(self.env_value) + # self.env_value.pop("logs") + # self.env_value.pop("network") + self.env_value = collection + + if run_type == "insert_performance": + (data_type, collection_size, dimension, metric_type) = parser.collection_parser( + collection_name) + ni_per = collection["ni_per"] + build_index = collection["build_index"] + if milvus_instance.exists_collection(): + milvus_instance.drop() + time.sleep(10) + index_info = {} + search_params = {} + vector_type = self.get_vector_type(data_type) + other_fields = collection["other_fields"] if "other_fields" in collection else None + milvus_instance.create_collection(dimension, data_type=vector_type, + other_fields=other_fields) + if build_index is True: + index_type = collection["index_type"] + index_param = collection["index_param"] + index_info = { + "index_type": index_type, + "index_param": index_param + } + index_field_name = utils.get_default_field_name(vector_type) + milvus_instance.create_index(index_field_name, index_type, metric_type, index_param=index_param) + logger.debug(milvus_instance.describe_index()) + res = self.do_insert(milvus_instance, collection_name, data_type, dimension, collection_size, ni_per) + flush_time = 0.0 + if "flush" in collection and collection["flush"] == "no": + logger.debug("No manual flush") + else: + start_time = time.time() + milvus_instance.flush() + flush_time = time.time() - start_time + logger.debug(milvus_instance.count()) + collection_info = { + "dimension": dimension, + "metric_type": metric_type, + "dataset_name": collection_name, + "other_fields": other_fields, + "ni_per": ni_per + } + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, + search_params) + total_time = res["total_time"] + build_time = 0 + if build_index is True: + logger.debug("Start build index for last file") + start_time = time.time() + milvus_instance.create_index(index_field_name, index_type, metric_type, index_param=index_param) + build_time = time.time() - start_time + total_time = total_time + build_time + metric.metrics = { + "type": run_type, + "value": { + "total_time": total_time, + "qps": res["qps"], + "ni_time": res["ni_time"], + "flush_time": flush_time, + "build_time": build_time + } + } + report(metric) + + elif run_type == "build_performance": + (data_type, collection_size, dimension, metric_type) = parser.collection_parser( + collection_name) + index_type = collection["index_type"] + index_param = collection["index_param"] + collection_info = { + "dimension": dimension, + "metric_type": metric_type, + "dataset_name": collection_name + } + index_info = { + "index_type": index_type, + "index_param": index_param + } + if not milvus_instance.exists_collection(): + logger.error("Table name: %s not existed" % collection_name) + return + search_params = {} + vector_type = self.get_vector_type(data_type) + index_field_name = utils.get_default_field_name(vector_type) + start_time = time.time() + # drop index + logger.debug("Drop index") + milvus_instance.drop_index(index_field_name) + # start_mem_usage = milvus_instance.get_mem_info()["memory_used"] + # TODO: need to check + milvus_instance.create_index(index_field_name, index_type, metric_type, index_param=index_param) + logger.debug(milvus_instance.describe_index()) + logger.debug(milvus_instance.count()) + end_time = time.time() + # end_mem_usage = milvus_instance.get_mem_info()["memory_used"] + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, + search_params) + metric.metrics = { + "type": "build_performance", + "value": { + "build_time": round(end_time - start_time, 1), + } + } + report(metric) + + elif run_type == "delete_performance": + (data_type, collection_size, dimension, metric_type) = parser.collection_parser( + collection_name) + ni_per = collection["ni_per"] + auto_flush = collection["auto_flush"] if "auto_flush" in collection else True + search_params = {} + collection_info = { + "dimension": dimension, + "metric_type": metric_type, + "dataset_name": collection_name + } + if not milvus_instance.exists_collection(): + logger.error(milvus_instance.show_collections()) + logger.error("Table name: %s not existed" % collection_name) + return + length = milvus_instance.count() + logger.info(length) + index_info = milvus_instance.describe_index() + logger.info(index_info) + ids = [i for i in range(length)] + loops = int(length / ni_per) + milvus_instance.load_collection() + # TODO: remove + # start_mem_usage = milvus_instance.get_mem_info()["memory_used"] + start_time = time.time() + # if auto_flush is False: + # milvus_instance.set_config("storage", "auto_flush_interval", BIG_FLUSH_INTERVAL) + for i in range(loops): + delete_ids = ids[i * ni_per: i * ni_per + ni_per] + logger.debug("Delete %d - %d" % (delete_ids[0], delete_ids[-1])) + milvus_instance.delete(delete_ids) + logger.debug("Table row counts: %d" % milvus_instance.count()) + logger.debug("Table row counts: %d" % milvus_instance.count()) + start_flush_time = time.time() + milvus_instance.flush() + end_flush_time = time.time() + end_time = time.time() + # end_mem_usage = milvus_instance.get_mem_info()["memory_used"] + logger.debug("Table row counts: %d" % milvus_instance.count()) + # milvus_instance.set_config("storage", "auto_flush_interval", DEFAULT_FLUSH_INTERVAL) + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, + search_params) + delete_time = round(end_time - start_time, 1) + metric.metrics = { + "type": "delete_performance", + "value": { + "delete_time": delete_time, + "qps": round(collection_size / delete_time, 1) + } + } + if auto_flush is False: + flush_time = round(end_flush_time - start_flush_time, 1) + metric.metrics["value"].update({"flush_time": flush_time}) + report(metric) + + elif run_type == "get_ids_performance": + (data_type, collection_size, dimension, metric_type) = parser.collection_parser( + collection_name) + ids_length_per_segment = collection["ids_length_per_segment"] + if not milvus_instance.exists_collection(): + logger.error("Table name: %s not existed" % collection_name) + return + collection_info = { + "dimension": dimension, + "metric_type": metric_type, + "dataset_name": collection_name + } + search_params = {} + logger.info(milvus_instance.count()) + index_info = milvus_instance.describe_index() + logger.info(index_info) + for ids_num in ids_length_per_segment: + segment_num, get_ids = milvus_instance.get_rand_ids_each_segment(ids_num) + start_time = time.time() + get_res = milvus_instance.get_entities(get_ids) + total_time = time.time() - start_time + avg_time = total_time / segment_num + run_params = {"ids_num": ids_num} + logger.info( + "Segment num: %d, ids num per segment: %d, run_time: %f" % (segment_num, ids_num, total_time)) + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, + index_info, search_params, run_params=run_params) + metric.metrics = { + "type": run_type, + "value": { + "total_time": round(total_time, 1), + "avg_time": round(avg_time, 1) + } + } + report(metric) + + elif run_type == "search_performance": + (data_type, collection_size, dimension, metric_type) = parser.collection_parser( + collection_name) + run_count = collection["run_count"] + top_ks = collection["top_ks"] + nqs = collection["nqs"] + # filter_query = collection["filter"] if "filter" in collection else None + filters = collection["filters"] if "filters" in collection else [] + filter_query = [] + search_params = collection["search_params"] + fields = self.get_fields(milvus_instance, collection_name) + collection_info = { + "dimension": dimension, + "metric_type": metric_type, + "dataset_name": collection_name + "fields": fields + } + if not milvus_instance.exists_collection(): + logger.error("Table name: %s not existed" % collection_name) + return + + vector_type = self.get_vector_type(data_type) + vec_field_name = utils.get_default_field_name(vector_type) + logger.info(milvus_instance.count()) + index_info = milvus_instance.describe_index() + logger.info(index_info) + milvus_instance.load_collection() + logger.info("Start warm up query") + res = self.do_query(milvus_instance, collection_name, vec_field_name, [1], [1], 2, + search_param=search_params[0], filter_query=filter_query) + logger.info("End warm up query") + for search_param in search_params: + logger.info("Search param: %s" % json.dumps(search_param)) + if not filters: + filters.append(None) + for filter in filters: + filter_param = [] + if isinstance(filter, dict) and "range" in filter: + filter_query.append(eval(filter["range"])) + filter_param.append(filter["range"]) + if isinstance(filter, dict) and "term" in filter: + filter_query.append(eval(filter["term"])) + filter_param.append(filter["term"]) + logger.info("filter param: %s" % json.dumps(filter_param)) + res = self.do_query(milvus_instance, collection_name, vec_field_name, top_ks, nqs, run_count, + search_param, filter_query=filter_query) + headers = ["Nq/Top-k"] + headers.extend([str(top_k) for top_k in top_ks]) + logger.info("Search param: %s" % json.dumps(search_param)) + utils.print_table(headers, nqs, res) + for index_nq, nq in enumerate(nqs): + for index_top_k, top_k in enumerate(top_ks): + search_param_group = { + "nq": nq, + "topk": top_k, + "search_param": search_param, + "filter": filter_param + } + search_time = res[index_nq][index_top_k] + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, + collection_info, index_info, search_param_group) + metric.metrics = { + "type": "search_performance", + "value": { + "search_time": search_time + } + } + report(metric) + + elif run_type == "locust_insert_stress": + pass + + elif run_type in ["locust_search_performance", "locust_insert_performance", "locust_mix_performance"]: + (data_type, collection_size, dimension, metric_type) = parser.collection_parser( + collection_name) + ni_per = collection["ni_per"] + build_index = collection["build_index"] + if milvus_instance.exists_collection(): + milvus_instance.drop() + time.sleep(10) + index_info = {} + search_params = {} + vector_type = self.get_vector_type(data_type) + index_field_name = utils.get_default_field_name(vector_type) + milvus_instance.create_collection(dimension, data_type=vector_type, other_fields=None) + vector_type = self.get_vector_type(data_type) + vec_field_name = utils.get_default_field_name(vector_type) + if build_index is True: + index_type = collection["index_type"] + index_param = collection["index_param"] + index_info = { + "index_type": index_type, + "index_param": index_param + } + milvus_instance.create_index(index_field_name, index_type, metric_type, index_param=index_param) + logger.debug(milvus_instance.describe_index()) + if run_type in ["locust_search_performance", "locust_mix_performance"]: + res = self.do_insert(milvus_instance, collection_name, data_type, dimension, collection_size, ni_per) + if "flush" in collection and collection["flush"] == "no": + logger.debug("No manual flush") + else: + milvus_instance.flush() + if build_index is True: + logger.debug("Start build index for last file") + milvus_instance.create_index(index_field_name, index_type, metric_type, _async=True, + index_param=index_param) + logger.debug(milvus_instance.describe_index()) + logger.debug("Table row counts: %d" % milvus_instance.count()) + milvus_instance.load_collection() + logger.info("Start warm up query") + for i in range(2): + res = self.do_query(milvus_instance, collection_name, vec_field_name, [1], [1], 2, + search_param={"nprobe": 16}) + logger.info("End warm up query") + real_metric_type = utils.metric_type_trans(metric_type) + ### spawn locust requests + task = collection["task"] + connection_type = "single" + connection_num = task["connection_num"] + if connection_num > 1: + connection_type = "multi" + clients_num = task["clients_num"] + hatch_rate = task["hatch_rate"] + during_time = utils.timestr_to_int(task["during_time"]) + task_types = task["types"] + run_params = {"tasks": {}, "clients_num": clients_num, "spawn_rate": hatch_rate, "during_time": during_time} + for task_type in task_types: + run_params["tasks"].update({task_type["type"]: task_type["weight"] if "weight" in task_type else 1}) + + # . collect stats + locust_stats = locust_user.locust_executor(self.host, self.port, collection_name, + connection_type=connection_type, run_params=run_params) + logger.info(locust_stats) + collection_info = { + "dimension": dimension, + "metric_type": metric_type, + "dataset_name": collection_name + } + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, + search_params) + metric.metrics = { + "type": run_type, + "value": locust_stats} + report(metric) + + elif run_type == "search_ids_stability": + (data_type, collection_size, dimension, metric_type) = parser.collection_parser( + collection_name) + search_params = collection["search_params"] + during_time = collection["during_time"] + ids_length = collection["ids_length"] + ids = collection["ids"] + collection_info = { + "dimension": dimension, + "metric_type": metric_type, + "dataset_name": collection_name + } + if not milvus_instance.exists_collection(): + logger.error("Table name: %s not existed" % collection_name) + return + logger.info(milvus_instance.count()) + index_info = milvus_instance.describe_index() + logger.info(index_info) + g_top_k = int(collection["top_ks"].split("-")[1]) + l_top_k = int(collection["top_ks"].split("-")[0]) + g_id = int(ids.split("-")[1]) + l_id = int(ids.split("-")[0]) + g_id_length = int(ids_length.split("-")[1]) + l_id_length = int(ids_length.split("-")[0]) + + milvus_instance.load_collection() + # start_mem_usage = milvus_instance.get_mem_info()["memory_used"] + # logger.debug(start_mem_usage) + start_time = time.time() + while time.time() < start_time + during_time * 60: + search_param = {} + top_k = random.randint(l_top_k, g_top_k) + ids_num = random.randint(l_id_length, g_id_length) + ids_param = [random.randint(l_id_length, g_id_length) for _ in range(ids_num)] + for k, v in search_params.items(): + search_param[k] = random.randint(int(v.split("-")[0]), int(v.split("-")[1])) + logger.debug("Query top-k: %d, ids_num: %d, param: %s" % (top_k, ids_num, json.dumps(search_param))) + result = milvus_instance.query_ids(top_k, ids_param, search_param=search_param) + # end_mem_usage = milvus_instance.get_mem_info()["memory_used"] + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, + {}) + metric.metrics = { + "type": "search_ids_stability", + "value": { + "during_time": during_time, + } + } + report(metric) + + # for sift/deep datasets + # TODO: enable + elif run_type == "accuracy": + (data_type, collection_size, dimension, metric_type) = parser.collection_parser( + collection_name) + search_params = collection["search_params"] + # mapping to search param list + search_params = self.generate_combinations(search_params) + + top_ks = collection["top_ks"] + nqs = collection["nqs"] + collection_info = { + "dimension": dimension, + "metric_type": metric_type, + "dataset_name": collection_name + } + if not milvus_instance.exists_collection(): + logger.error("Table name: %s not existed" % collection_name) + return + logger.info(milvus_instance.count()) + index_info = milvus_instance.describe_index() + logger.info(index_info) + milvus_instance.load_collection() + true_ids_all = self.get_groundtruth_ids(collection_size) + vector_type = self.get_vector_type(data_type) + vec_field_name = utils.get_default_field_name(vector_type) + for search_param in search_params: + headers = ["Nq/Top-k"] + res = [] + for nq in nqs: + for top_k in top_ks: + tmp_res = [] + search_param_group = { + "nq": nq, + "topk": top_k, + "search_param": search_param, + "metric_type": metric_type + } + logger.info("Query params: %s" % json.dumps(search_param_group)) + result_ids = self.do_query_ids(milvus_instance, collection_name, vec_field_name, top_k, nq, + search_param=search_param) + # mem_used = milvus_instance.get_mem_info()["memory_used"] + acc_value = self.get_recall_value(true_ids_all[:nq, :top_k].tolist(), result_ids) + logger.info("Query accuracy: %s" % acc_value) + tmp_res.append(acc_value) + # logger.info("Memory usage: %s" % mem_used) + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, + index_info, search_param_group) + metric.metrics = { + "type": "accuracy", + "value": { + "acc": acc_value + } + } + report(metric) + # logger.info("Memory usage: %s" % mem_used) + res.append(tmp_res) + headers.extend([str(top_k) for top_k in top_ks]) + logger.info("Search param: %s" % json.dumps(search_param)) + utils.print_table(headers, nqs, res) + + elif run_type == "ann_accuracy": + hdf5_source_file = collection["source_file"] + collection_name = collection["collection_name"] + index_types = collection["index_types"] + index_params = collection["index_params"] + top_ks = collection["top_ks"] + nqs = collection["nqs"] + search_params = collection["search_params"] + # mapping to search param list + search_params = self.generate_combinations(search_params) + # mapping to index param list + index_params = self.generate_combinations(index_params) + + data_type, dimension, metric_type = parser.parse_ann_collection_name(collection_name) + collection_info = { + "dimension": dimension, + "metric_type": metric_type, + "dataset_name": collection_name + } + dataset = utils.get_dataset(hdf5_source_file) + if milvus_instance.exists_collection(collection_name): + logger.info("Re-create collection: %s" % collection_name) + milvus_instance.drop() + time.sleep(DELETE_INTERVAL_TIME) + true_ids = np.array(dataset["neighbors"]) + vector_type = self.get_vector_type_from_metric(metric_type) + vec_field_name = utils.get_default_field_name(vector_type) + real_metric_type = utils.metric_type_trans(metric_type) + + # re-create collection + if milvus_instance.exists_collection(collection_name): + milvus_instance.drop() + time.sleep(DELETE_INTERVAL_TIME) + milvus_instance.create_collection(dimension, data_type=vector_type) + insert_vectors = self.normalize(metric_type, np.array(dataset["train"])) + if len(insert_vectors) != dataset["train"].shape[0]: + raise Exception("Row count of insert vectors: %d is not equal to dataset size: %d" % ( + len(insert_vectors), dataset["train"].shape[0])) + logger.debug("The row count of entities to be inserted: %d" % len(insert_vectors)) + # Insert batch once + # milvus_instance.insert(insert_vectors) + loops = len(insert_vectors) // INSERT_INTERVAL + 1 + for i in range(loops): + start = i * INSERT_INTERVAL + end = min((i + 1) * INSERT_INTERVAL, len(insert_vectors)) + if start < end: + tmp_vectors = insert_vectors[start:end] + ids = [i for i in range(start, end)] + if not isinstance(tmp_vectors, list): + entities = milvus_instance.generate_entities(tmp_vectors.tolist(), ids) + res_ids = milvus_instance.insert(entities, ids=ids) + else: + entities = milvus_instance.generate_entities(tmp_vectors, ids) + res_ids = milvus_instance.insert(entities, ids=ids) + assert res_ids == ids + milvus_instance.flush() + res_count = milvus_instance.count() + logger.info("Table: %s, row count: %d" % (collection_name, res_count)) + if res_count != len(insert_vectors): + raise Exception("Table row count is not equal to insert vectors") + for index_type in index_types: + for index_param in index_params: + logger.debug("Building index with param: %s" % json.dumps(index_param)) + if milvus_instance.get_config("cluster.enable") == "true": + milvus_instance.create_index(vec_field_name, index_type, metric_type, _async=True, + index_param=index_param) + else: + milvus_instance.create_index(vec_field_name, index_type, metric_type, + index_param=index_param) + logger.info(milvus_instance.describe_index()) + logger.info("Start load collection: %s" % collection_name) + milvus_instance.load_collection() + logger.info("End load collection: %s" % collection_name) + index_info = { + "index_type": index_type, + "index_param": index_param + } + logger.debug(index_info) + warm_up = True + for search_param in search_params: + for nq in nqs: + query_vectors = self.normalize(metric_type, np.array(dataset["test"][:nq])) + if not isinstance(query_vectors, list): + query_vectors = query_vectors.tolist() + for top_k in top_ks: + search_param_group = { + "nq": len(query_vectors), + "topk": top_k, + "search_param": search_param, + "metric_type": metric_type + } + logger.debug(search_param_group) + vector_query = {"vector": {vec_field_name: { + "topk": top_k, + "query": query_vectors, + "metric_type": real_metric_type, + "params": search_param} + }} + for i in range(2): + result = milvus_instance.query(vector_query) + warm_up = False + logger.info("End warm up") + result = milvus_instance.query(vector_query) + result_ids = milvus_instance.get_ids(result) + acc_value = self.get_recall_value(true_ids[:nq, :top_k].tolist(), result_ids) + logger.info("Query ann_accuracy: %s" % acc_value) + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, + collection_info, index_info, search_param_group) + metric.metrics = { + "type": "ann_accuracy", + "value": { + "acc": acc_value + } + } + report(metric) + + elif run_type == "search_stability": + (data_type, collection_size, dimension, metric_type) = parser.collection_parser( + collection_name) + search_params = collection["search_params"] + during_time = collection["during_time"] + collection_info = { + "dimension": dimension, + "metric_type": metric_type, + "dataset_name": collection_name + } + if not milvus_instance.exists_collection(): + logger.error("Table name: %s not existed" % collection_name) + return + logger.info(milvus_instance.count()) + index_info = milvus_instance.describe_index() + logger.info(index_info) + g_top_k = int(collection["top_ks"].split("-")[1]) + g_nq = int(collection["nqs"].split("-")[1]) + l_top_k = int(collection["top_ks"].split("-")[0]) + l_nq = int(collection["nqs"].split("-")[0]) + milvus_instance.load_collection() + # start_mem_usage = milvus_instance.get_mem_info()["memory_used"] + # logger.debug(start_mem_usage) + start_row_count = milvus_instance.count() + logger.debug(milvus_instance.describe_index()) + logger.info(start_row_count) + vector_type = self.get_vector_type(data_type) + vec_field_name = utils.get_default_field_name(vector_type) + real_metric_type = utils.metric_type_trans(metric_type) + start_time = time.time() + while time.time() < start_time + during_time * 60: + search_param = {} + top_k = random.randint(l_top_k, g_top_k) + nq = random.randint(l_nq, g_nq) + for k, v in search_params.items(): + search_param[k] = random.randint(int(v.split("-")[0]), int(v.split("-")[1])) + query_vectors = [[random.random() for _ in range(dimension)] for _ in range(nq)] + logger.debug("Query nq: %d, top-k: %d, param: %s" % (nq, top_k, json.dumps(search_param))) + vector_query = {"vector": {vec_field_name: { + "topk": top_k, + "query": query_vectors[:nq], + "metric_type": real_metric_type, + "params": search_param} + }} + milvus_instance.query(vector_query) + # end_mem_usage = milvus_instance.get_mem_info()["memory_used"] + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, + {}) + metric.metrics = { + "type": "search_stability", + "value": { + "during_time": during_time, + } + } + report(metric) + + elif run_type == "loop_stability": + # init data + milvus_instance.clean_db() + pull_interval = collection["pull_interval"] + collection_num = collection["collection_num"] + concurrent = collection["concurrent"] if "concurrent" in collection else False + concurrent_num = collection_num + dimension = collection["dimension"] if "dimension" in collection else 128 + insert_xb = collection["insert_xb"] if "insert_xb" in collection else 100000 + index_types = collection["index_types"] if "index_types" in collection else ['ivf_sq8'] + index_param = {"nlist": 256} + collection_names = [] + milvus_instances_map = {} + insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(insert_xb)] + ids = [i for i in range(insert_xb)] + # initialize and prepare + for i in range(collection_num): + name = utils.get_unique_name(prefix="collection_%d_" % i) + collection_names.append(name) + metric_type = random.choice(["l2", "ip"]) + # default float_vector + milvus_instance = MilvusClient(collection_name=name, host=self.host) + milvus_instance.create_collection(dimension, other_fields=None) + index_type = random.choice(index_types) + field_name = utils.get_default_field_name() + milvus_instance.create_index(field_name, index_type, metric_type, index_param=index_param) + logger.info(milvus_instance.describe_index()) + insert_vectors = utils.normalize(metric_type, insert_vectors) + entities = milvus_instance.generate_entities(insert_vectors, ids) + res_ids = milvus_instance.insert(entities, ids=ids) + milvus_instance.flush() + milvus_instances_map.update({name: milvus_instance}) + logger.info(milvus_instance.describe_index()) + + # loop time unit: min -> s + pull_interval_seconds = pull_interval * 60 + tasks = ["insert_rand", "query_rand", "flush"] + i = 1 + while True: + logger.info("Loop time: %d" % i) + start_time = time.time() + while time.time() - start_time < pull_interval_seconds: + if concurrent: + threads = [] + for name in collection_names: + task_name = random.choice(tasks) + task_run = getattr(milvus_instances_map[name], task_name) + t = threading.Thread(target=task_run, args=()) + threads.append(t) + t.start() + for t in threads: + t.join() + # with concurrent.futures.ThreadPoolExecutor(max_workers=concurrent_num) as executor: + # future_results = {executor.submit(getattr(milvus_instances_map[mp[j][0]], mp[j][1])): j for j in range(concurrent_num)} + # for future in concurrent.futures.as_completed(future_results): + # future.result() + else: + tmp_collection_name = random.choice(collection_names) + task_name = random.choice(tasks) + logger.info(tmp_collection_name) + logger.info(task_name) + task_run = getattr(milvus_instances_map[tmp_collection_name], task_name) + task_run() + + logger.debug("Restart server") + helm_utils.restart_server(self.service_name, namespace) + # new connection + # for name in collection_names: + # milvus_instance = MilvusClient(collection_name=name, host=self.host) + # milvus_instances_map.update({name: milvus_instance}) + time.sleep(30) + i = i + 1 + + elif run_type == "stability": + (data_type, collection_size, dimension, metric_type) = parser.collection_parser( + collection_name) + during_time = collection["during_time"] + operations = collection["operations"] + collection_info = { + "dimension": dimension, + "metric_type": metric_type, + "dataset_name": collection_name + } + if not milvus_instance.exists_collection(): + logger.error(milvus_instance.show_collections()) + raise Exception("Table name: %s not existed" % collection_name) + logger.info(milvus_instance.count()) + index_info = milvus_instance.describe_index() + logger.info(index_info) + # start_mem_usage = milvus_instance.get_mem_info()["memory_used"] + start_row_count = milvus_instance.count() + logger.info(start_row_count) + vector_type = self.get_vector_type(data_type) + vec_field_name = utils.get_default_field_name(vector_type) + real_metric_type = utils.metric_type_trans(metric_type) + query_vectors = [[random.random() for _ in range(dimension)] for _ in range(10000)] + if "insert" in operations: + insert_xb = operations["insert"]["xb"] + if "delete" in operations: + delete_xb = operations["delete"]["xb"] + if "query" in operations: + g_top_k = int(operations["query"]["top_ks"].split("-")[1]) + l_top_k = int(operations["query"]["top_ks"].split("-")[0]) + g_nq = int(operations["query"]["nqs"].split("-")[1]) + l_nq = int(operations["query"]["nqs"].split("-")[0]) + search_params = operations["query"]["search_params"] + i = 0 + start_time = time.time() + while time.time() < start_time + during_time * 60: + i = i + 1 + q = self.gen_executors(operations) + for name in q: + try: + if name == "insert": + insert_ids = random.sample(list(range(collection_size)), insert_xb) + insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(insert_xb)] + entities = milvus_instance.generate_entities(insert_vectors, insert_ids) + milvus_instance.insert(entities, ids=insert_ids) + elif name == "delete": + delete_ids = random.sample(list(range(collection_size)), delete_xb) + milvus_instance.delete(delete_ids) + elif name == "query": + top_k = random.randint(l_top_k, g_top_k) + nq = random.randint(l_nq, g_nq) + search_param = {} + for k, v in search_params.items(): + search_param[k] = random.randint(int(v.split("-")[0]), int(v.split("-")[1])) + logger.debug("Query nq: %d, top-k: %d, param: %s" % (nq, top_k, json.dumps(search_param))) + vector_query = {"vector": {vec_field_name: { + "topk": top_k, + "query": query_vectors[:nq], + "metric_type": real_metric_type, + "params": search_param} + }} + result = milvus_instance.query(vector_query) + elif name in ["flush", "compact"]: + func = getattr(milvus_instance, name) + func() + logger.debug(milvus_instance.count()) + except Exception as e: + logger.error(name) + logger.error(str(e)) + raise + logger.debug("Loop time: %d" % i) + # end_mem_usage = milvus_instance.get_mem_info()["memory_used"] + end_row_count = milvus_instance.count() + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, + {}) + metric.metrics = { + "type": "stability", + "value": { + "during_time": during_time, + "row_count_increments": end_row_count - start_row_count + } + } + report(metric) + + elif run_type == "debug": + time.sleep(7200) + default_insert_vectors = [[random.random() for _ in range(128)] for _ in range(500000)] + interval = 50000 + for loop in range(1, 7): + insert_xb = loop * interval + insert_vectors = default_insert_vectors[:insert_xb] + insert_ids = [i for i in range(insert_xb)] + entities = milvus_instance.generate_entities(insert_vectors, insert_ids) + for j in range(5): + milvus_instance.insert(entities, ids=insert_ids) + time.sleep(10) + + else: + raise Exception("Run type not defined") + logger.debug("All test finished") diff --git a/tests/benchmark/local_runner.py b/tests/benchmark/local_runner.py new file mode 100644 index 0000000000..408fc95f52 --- /dev/null +++ b/tests/benchmark/local_runner.py @@ -0,0 +1,732 @@ +import os +import logging +import pdb +import string +import time +import random +import json +import csv +from multiprocessing import Process +import numpy as np +import concurrent.futures +from queue import Queue + +import locust_user +from milvus import DataType +from client import MilvusClient +from runner import Runner +import utils +import parser + + +DELETE_INTERVAL_TIME = 5 +INSERT_INTERVAL = 50000 +logger = logging.getLogger("milvus_benchmark.local_runner") + + +class LocalRunner(Runner): + """run local mode""" + def __init__(self, host, port): + super(LocalRunner, self).__init__() + self.host = host + self.port = port + + def run(self, run_type, collection): + logger.debug(run_type) + logger.debug(collection) + collection_name = collection["collection_name"] if "collection_name" in collection else None + milvus_instance = MilvusClient(collection_name=collection_name, host=self.host, port=self.port) + logger.info(milvus_instance.show_collections()) + # TODO: + # self.env_value = milvus_instance.get_server_config() + # ugly implemention + # self.env_value = utils.convert_nested(self.env_value) + # self.env_value.pop("logs") + # self.env_value.pop("network") + # logger.info(self.env_value) + + if run_type in ["insert_performance", "insert_flush_performance"]: + (data_type, collection_size, dimension, metric_type) = parser.collection_parser(collection_name) + ni_per = collection["ni_per"] + build_index = collection["build_index"] + if milvus_instance.exists_collection(): + milvus_instance.drop() + time.sleep(10) + vector_type = self.get_vector_type(data_type) + other_fields = collection["other_fields"] if "other_fields" in collection else None + milvus_instance.create_collection(dimension, data_type=vector_type, other_fields=other_fields) + if build_index is True: + index_type = collection["index_type"] + index_param = collection["index_param"] + index_field_name = utils.get_default_field_name(vector_type) + milvus_instance.create_index(index_field_name, index_type, metric_type, index_param=index_param) + res = self.do_insert(milvus_instance, collection_name, data_type, dimension, collection_size, ni_per) + milvus_instance.flush() + logger.debug("Table row counts: %d" % milvus_instance.count()) + if build_index is True: + logger.debug("Start build index for last file") + milvus_instance.create_index(index_field_name, index_type, metric_type, index_param=index_param) + + elif run_type == "delete_performance": + (data_type, collection_size, dimension, metric_type) = parser.collection_parser(collection_name) + ni_per = collection["ni_per"] + auto_flush = collection["auto_flush"] if "auto_flush" in collection else True + if not milvus_instance.exists_collection(): + logger.error(milvus_instance.show_collections()) + logger.error("Table: %s not found" % collection_name) + return + length = milvus_instance.count() + ids = [i for i in range(length)] + loops = int(length / ni_per) + if auto_flush is False: + milvus_instance.set_config("storage", "auto_flush_interval", BIG_FLUSH_INTERVAL) + for i in range(loops): + delete_ids = ids[i*ni_per: i*ni_per+ni_per] + logger.debug("Delete %d - %d" % (delete_ids[0], delete_ids[-1])) + milvus_instance.delete(delete_ids) + logger.debug("Table row counts: %d" % milvus_instance.count()) + logger.debug("Table row counts: %d" % milvus_instance.count()) + milvus_instance.flush() + logger.debug("Table row counts: %d" % milvus_instance.count()) + + elif run_type == "build_performance": + (data_type, collection_size, dimension, metric_type) = parser.collection_parser(collection_name) + index_type = collection["index_type"] + index_param = collection["index_param"] + if not milvus_instance.exists_collection(): + logger.error("Table name: %s not existed" % collection_name) + return + vector_type = self.get_vector_type(data_type) + index_field_name = utils.get_default_field_name(vector_type) + # drop index + logger.debug("Drop index") + milvus_instance.drop_index(index_field_name) + start_mem_usage = milvus_instance.get_mem_info()["memory_used"] + start_time = time.time() + milvus_instance.create_index(index_field_name, index_type, metric_type, index_param=index_param) + end_time = time.time() + logger.debug("Table row counts: %d" % milvus_instance.count()) + end_mem_usage = milvus_instance.get_mem_info()["memory_used"] + logger.debug("Diff memory: %s, current memory usage: %s, build time: %s" % ((end_mem_usage - start_mem_usage), end_mem_usage, round(end_time - start_time, 1))) + + elif run_type == "search_performance": + (data_type, collection_size, dimension, metric_type) = parser.collection_parser(collection_name) + run_count = collection["run_count"] + top_ks = collection["top_ks"] + nqs = collection["nqs"] + search_params = collection["search_params"] + filter_query = [] + filters = collection["filters"] if "filters" in collection else [] + # pdb.set_trace() + # ranges = collection["range"] if "range" in collection else None + # terms = collection["term"] if "term" in collection else None + # if ranges: + # filter_query.append(eval(ranges)) + # if terms: + # filter_query.append(eval(terms)) + vector_type = self.get_vector_type(data_type) + vec_field_name = utils.get_default_field_name(vector_type) + # for debugging + # time.sleep(3600) + if not milvus_instance.exists_collection(): + logger.error("Table name: %s not existed" % collection_name) + return + vector_type = self.get_vector_type(data_type) + vec_field_name = utils.get_default_field_name(vector_type) + logger.info(milvus_instance.count()) + result = milvus_instance.describe_index() + logger.info(result) + milvus_instance.preload_collection() + mem_usage = milvus_instance.get_mem_info()["memory_used"] + logger.info(mem_usage) + for search_param in search_params: + logger.info("Search param: %s" % json.dumps(search_param)) + filter_param = [] + if not filters: + filters.append(None) + for filter in filters: + if isinstance(filter, dict) and "range" in filter: + filter_query.append(eval(filter["range"])) + filter_param.append(filter["range"]) + if isinstance(filter, dict) and "term" in filter: + filter_query.append(eval(filter["term"])) + filter_param.append(filter["term"]) + logger.info("filter param: %s" % json.dumps(filter_param)) + res = self.do_query(milvus_instance, collection_name, vec_field_name, top_ks, nqs, run_count, search_param, filter_query) + headers = ["Nq/Top-k"] + headers.extend([str(top_k) for top_k in top_ks]) + logger.info("Search param: %s" % json.dumps(search_param)) + utils.print_table(headers, nqs, res) + mem_usage = milvus_instance.get_mem_info()["memory_used"] + logger.info(mem_usage) + + elif run_type == "locust_search_performance": + (data_type, collection_size, dimension, metric_type) = parser.collection_parser(collection_name) + ni_per = collection["ni_per"] + build_index = collection["build_index"] + vector_type = self.get_vector_type(data_type) + index_field_name = utils.get_default_field_name(vector_type) + # if build_index is True: + # index_type = collection["index_type"] + # index_param = collection["index_param"] + # # TODO: debug + # if milvus_instance.exists_collection(): + # milvus_instance.drop() + # time.sleep(10) + # other_fields = collection["other_fields"] if "other_fields" in collection else None + # milvus_instance.create_collection(dimension, data_type=vector_type, other_fields=other_fields) + # milvus_instance.create_index(index_field_name, index_type, metric_type, index_param=index_param) + # res = self.do_insert(milvus_instance, collection_name, data_type, dimension, collection_size, ni_per) + # milvus_instance.flush() + # logger.debug("Table row counts: %d" % milvus_instance.count()) + # if build_index is True: + # logger.debug("Start build index for last file") + # milvus_instance.create_index(index_field_name, index_type, metric_type, index_param=index_param) + real_metric_type = utils.metric_type_trans(metric_type) + ### spawn locust requests + task = collection["task"] + connection_type = "single" + connection_num = task["connection_num"] + if connection_num > 1: + connection_type = "multi" + clients_num = task["clients_num"] + hatch_rate = task["hatch_rate"] + during_time = utils.timestr_to_int(task["during_time"]) + task_types = task["types"] + # """ + # task: + # connection_num: 1 + # clients_num: 100 + # hatch_rate: 2 + # during_time: 5m + # types: + # - + # type: query + # weight: 1 + # params: + # top_k: 10 + # nq: 1 + # # filters: + # # - + # # range: + # # int64: + # # LT: 0 + # # GT: 1000000 + # search_param: + # nprobe: 16 + # """ + run_params = {"tasks": {}, "clients_num": clients_num, "spawn_rate": hatch_rate, "during_time": during_time} + for task_type in task_types: + run_params["tasks"].update({task_type["type"]: task_type["weight"] if "weight" in task_type else 1}) + + #. collect stats + locust_stats = locust_user.locust_executor(self.host, self.port, collection_name, connection_type=connection_type, run_params=run_params) + logger.info(locust_stats) + + elif run_type == "search_ids_stability": + (data_type, collection_size, dimension, metric_type) = parser.collection_parser(collection_name) + search_params = collection["search_params"] + during_time = collection["during_time"] + ids_length = collection["ids_length"] + ids = collection["ids"] + logger.info(milvus_instance.count()) + index_info = milvus_instance.describe_index() + logger.info(index_info) + g_top_k = int(collection["top_ks"].split("-")[1]) + l_top_k = int(collection["top_ks"].split("-")[0]) + g_id = int(ids.split("-")[1]) + l_id = int(ids.split("-")[0]) + g_id_length = int(ids_length.split("-")[1]) + l_id_length = int(ids_length.split("-")[0]) + + milvus_instance.preload_collection() + start_mem_usage = milvus_instance.get_mem_info()["memory_used"] + logger.debug(start_mem_usage) + start_time = time.time() + while time.time() < start_time + during_time * 60: + search_param = {} + top_k = random.randint(l_top_k, g_top_k) + ids_num = random.randint(l_id_length, g_id_length) + l_ids = random.randint(l_id, g_id-ids_num) + # ids_param = [random.randint(l_id_length, g_id_length) for _ in range(ids_num)] + ids_param = [id for id in range(l_ids, l_ids+ids_num)] + for k, v in search_params.items(): + search_param[k] = random.randint(int(v.split("-")[0]), int(v.split("-")[1])) + logger.debug("Query top-k: %d, ids_num: %d, param: %s" % (top_k, ids_num, json.dumps(search_param))) + result = milvus_instance.query_ids(top_k, ids_param, search_param=search_param) + end_mem_usage = milvus_instance.get_mem_info()["memory_used"] + metrics = { + "during_time": during_time, + "start_mem_usage": start_mem_usage, + "end_mem_usage": end_mem_usage, + "diff_mem": end_mem_usage - start_mem_usage, + } + logger.info(metrics) + + elif run_type == "search_performance_concurrents": + data_type, dimension, metric_type = parser.parse_ann_collection_name(collection_name) + hdf5_source_file = collection["source_file"] + use_single_connection = collection["use_single_connection"] + concurrents = collection["concurrents"] + top_ks = collection["top_ks"] + nqs = collection["nqs"] + search_params = self.generate_combinations(collection["search_params"]) + if not milvus_instance.exists_collection(): + logger.error("Table name: %s not existed" % collection_name) + return + logger.info(milvus_instance.count()) + result = milvus_instance.describe_index() + logger.info(result) + milvus_instance.preload_collection() + dataset = utils.get_dataset(hdf5_source_file) + for concurrent_num in concurrents: + top_k = top_ks[0] + for nq in nqs: + mem_usage = milvus_instance.get_mem_info()["memory_used"] + logger.info(mem_usage) + query_vectors = self.normalize(metric_type, np.array(dataset["test"][:nq])) + logger.debug(search_params) + for search_param in search_params: + logger.info("Search param: %s" % json.dumps(search_param)) + total_time = 0.0 + if use_single_connection is True: + connections = [MilvusClient(collection_name=collection_name, host=self.host, port=self.port)] + with concurrent.futures.ThreadPoolExecutor(max_workers=concurrent_num) as executor: + future_results = {executor.submit( + self.do_query_qps, connections[0], query_vectors, top_k, search_param=search_param) : index for index in range(concurrent_num)} + else: + connections = [MilvusClient(collection_name=collection_name, host=self.hos, port=self.port) for i in range(concurrent_num)] + with concurrent.futures.ThreadPoolExecutor(max_workers=concurrent_num) as executor: + future_results = {executor.submit( + self.do_query_qps, connections[index], query_vectors, top_k, search_param=search_param) : index for index in range(concurrent_num)} + for future in concurrent.futures.as_completed(future_results): + total_time = total_time + future.result() + qps_value = total_time / concurrent_num + logger.debug("QPS value: %f, total_time: %f, request_nums: %f" % (qps_value, total_time, concurrent_num)) + mem_usage = milvus_instance.get_mem_info()["memory_used"] + logger.info(mem_usage) + + elif run_type == "ann_accuracy": + hdf5_source_file = collection["source_file"] + collection_name = collection["collection_name"] + index_types = collection["index_types"] + index_params = collection["index_params"] + top_ks = collection["top_ks"] + nqs = collection["nqs"] + search_params = collection["search_params"] + # mapping to search param list + search_params = self.generate_combinations(search_params) + # mapping to index param list + index_params = self.generate_combinations(index_params) + data_type, dimension, metric_type = parser.parse_ann_collection_name(collection_name) + dataset = utils.get_dataset(hdf5_source_file) + true_ids = np.array(dataset["neighbors"]) + vector_type = self.get_vector_type_from_metric(metric_type) + vec_field_name = utils.get_default_field_name(vector_type) + real_metric_type = utils.metric_type_trans(metric_type) + + # re-create collection + if milvus_instance.exists_collection(collection_name): + milvus_instance.drop() + time.sleep(DELETE_INTERVAL_TIME) + milvus_instance.create_collection(dimension, data_type=vector_type) + insert_vectors = self.normalize(metric_type, np.array(dataset["train"])) + if len(insert_vectors) != dataset["train"].shape[0]: + raise Exception("Row count of insert vectors: %d is not equal to dataset size: %d" % (len(insert_vectors), dataset["train"].shape[0])) + logger.debug("The row count of entities to be inserted: %d" % len(insert_vectors)) + # insert batch once + # milvus_instance.insert(insert_vectors) + loops = len(insert_vectors) // INSERT_INTERVAL + 1 + for i in range(loops): + start = i*INSERT_INTERVAL + end = min((i+1)*INSERT_INTERVAL, len(insert_vectors)) + if start < end: + tmp_vectors = insert_vectors[start:end] + ids = [i for i in range(start, end)] + if not isinstance(tmp_vectors, list): + entities = milvus_instance.generate_entities(tmp_vectors.tolist(), ids) + res_ids = milvus_instance.insert(entities, ids=ids) + else: + entities = milvus_instance.generate_entities(tmp_vectors, ids) + res_ids = milvus_instance.insert(entities, ids=ids) + assert res_ids == ids + milvus_instance.flush() + res_count = milvus_instance.count() + logger.info("Table: %s, row count: %d" % (collection_name, res_count)) + if res_count != len(insert_vectors): + raise Exception("Table row count is not equal to insert vectors") + for index_type in index_types: + for index_param in index_params: + logger.debug("Building index with param: %s, metric_type: %s" % (json.dumps(index_param), metric_type)) + milvus_instance.create_index(vec_field_name, index_type, metric_type, index_param=index_param) + logger.info("Start preload collection: %s" % collection_name) + milvus_instance.preload_collection() + for search_param in search_params: + for nq in nqs: + query_vectors = self.normalize(metric_type, np.array(dataset["test"][:nq])) + if not isinstance(query_vectors, list): + query_vectors = query_vectors.tolist() + for top_k in top_ks: + logger.debug("Search nq: %d, top-k: %d, search_param: %s, metric_type: %s" % (nq, top_k, json.dumps(search_param), metric_type)) + vector_query = {"vector": {vec_field_name: { + "topk": top_k, + "query": query_vectors, + "metric_type": real_metric_type, + "params": search_param} + }} + result = milvus_instance.query(vector_query) + result_ids = milvus_instance.get_ids(result) + # pdb.set_trace() + acc_value = self.get_recall_value(true_ids[:nq, :top_k].tolist(), result_ids) + logger.info("Query ann_accuracy: %s" % acc_value) + + elif run_type == "accuracy": + (data_type, collection_size, dimension, metric_type) = parser.collection_parser(collection_name) + search_params = collection["search_params"] + # mapping to search param list + search_params = self.generate_combinations(search_params) + + top_ks = collection["top_ks"] + nqs = collection["nqs"] + collection_info = { + "dimension": dimension, + "metric_type": metric_type, + "dataset_name": collection_name + } + if not milvus_instance.exists_collection(): + logger.error("Table name: %s not existed" % collection_name) + return + logger.info(milvus_instance.count()) + index_info = milvus_instance.describe_index() + logger.info(index_info) + milvus_instance.preload_collection() + true_ids_all = self.get_groundtruth_ids(collection_size) + vector_type = self.get_vector_type(data_type) + vec_field_name = utils.get_default_field_name(vector_type) + for search_param in search_params: + headers = ["Nq/Top-k"] + res = [] + for nq in nqs: + tmp_res = [] + for top_k in top_ks: + search_param_group = { + "nq": nq, + "topk": top_k, + "search_param": search_param, + "metric_type": metric_type + } + logger.info("Query params: %s" % json.dumps(search_param_group)) + result_ids = self.do_query_ids(milvus_instance, collection_name, vec_field_name, top_k, nq, search_param=search_param) + mem_used = milvus_instance.get_mem_info()["memory_used"] + acc_value = self.get_recall_value(true_ids_all[:nq, :top_k].tolist(), result_ids) + logger.info("Query accuracy: %s" % acc_value) + tmp_res.append(acc_value) + logger.info("Memory usage: %s" % mem_used) + res.append(tmp_res) + headers.extend([str(top_k) for top_k in top_ks]) + logger.info("Search param: %s" % json.dumps(search_param)) + utils.print_table(headers, nqs, res) + + elif run_type == "stability": + (data_type, collection_size, dimension, metric_type) = parser.collection_parser(collection_name) + during_time = collection["during_time"] + operations = collection["operations"] + if not milvus_instance.exists_collection(): + logger.error(milvus_instance.show_collections()) + raise Exception("Table name: %s not existed" % collection_name) + milvus_instance.preload_collection() + start_mem_usage = milvus_instance.get_mem_info()["memory_used"] + start_row_count = milvus_instance.count() + logger.info(start_row_count) + vector_type = self.get_vector_type(data_type) + vec_field_name = utils.get_default_field_name(vector_type) + real_metric_type = utils.metric_type_trans(metric_type) + query_vectors = [[random.random() for _ in range(dimension)] for _ in range(10000)] + if "insert" in operations: + insert_xb = operations["insert"]["xb"] + if "delete" in operations: + delete_xb = operations["delete"]["xb"] + if "query" in operations: + g_top_k = int(operations["query"]["top_ks"].split("-")[1]) + l_top_k = int(operations["query"]["top_ks"].split("-")[0]) + g_nq = int(operations["query"]["nqs"].split("-")[1]) + l_nq = int(operations["query"]["nqs"].split("-")[0]) + search_params = operations["query"]["search_params"] + i = 0 + start_time = time.time() + while time.time() < start_time + during_time * 60: + i = i + 1 + q = self.gen_executors(operations) + for name in q: + try: + if name == "insert": + insert_ids = random.sample(list(range(collection_size)), insert_xb) + insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(insert_xb)] + entities = milvus_instance.generate_entities(insert_vectors, insert_ids) + milvus_instance.insert(entities, ids=insert_ids) + elif name == "delete": + delete_ids = random.sample(list(range(collection_size)), delete_xb) + milvus_instance.delete(delete_ids) + elif name == "query": + top_k = random.randint(l_top_k, g_top_k) + nq = random.randint(l_nq, g_nq) + search_param = {} + for k, v in search_params.items(): + search_param[k] = random.randint(int(v.split("-")[0]), int(v.split("-")[1])) + logger.debug("Query nq: %d, top-k: %d, param: %s" % (nq, top_k, json.dumps(search_param))) + vector_query = {"vector": {vec_field_name: { + "topk": top_k, + "query": query_vectors[:nq], + "metric_type": real_metric_type, + "params": search_param} + }} + result = milvus_instance.query(vector_query) + elif name in ["flush", "compact"]: + func = getattr(milvus_instance, name) + func() + logger.debug(milvus_instance.count()) + except Exception as e: + logger.error(name) + logger.error(str(e)) + raise + logger.debug("Loop time: %d" % i) + end_mem_usage = milvus_instance.get_mem_info()["memory_used"] + end_row_count = milvus_instance.count() + metrics = { + "during_time": during_time, + "start_mem_usage": start_mem_usage, + "end_mem_usage": end_mem_usage, + "diff_mem": end_mem_usage - start_mem_usage, + "row_count_increments": end_row_count - start_row_count + } + logger.info(metrics) + + elif run_type == "loop_stability": + # init data + milvus_instance.clean_db() + pull_interval = collection["pull_interval"] + collection_num = collection["collection_num"] + concurrent = collection["concurrent"] if "concurrent" in collection else False + concurrent_num = collection_num + dimension = collection["dimension"] if "dimension" in collection else 128 + insert_xb = collection["insert_xb"] if "insert_xb" in collection else 100000 + index_types = collection["index_types"] if "index_types" in collection else ['ivf_sq8'] + index_param = {"nlist": 256} + collection_names = [] + milvus_instances_map = {} + insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(insert_xb)] + ids = [i for i in range(insert_xb)] + # initialize and prepare + for i in range(collection_num): + name = utils.get_unique_name(prefix="collection_%d_" % i) + collection_names.append(name) + metric_type = random.choice(["l2", "ip"]) + # default float_vector + milvus_instance = MilvusClient(collection_name=name, host=self.host) + milvus_instance.create_collection(dimension, other_fields=None) + index_type = random.choice(index_types) + field_name = utils.get_default_field_name() + milvus_instance.create_index(field_name, index_type, metric_type, index_param=index_param) + logger.info(milvus_instance.describe_index()) + insert_vectors = utils.normalize(metric_type, insert_vectors) + entities = milvus_instance.generate_entities(insert_vectors, ids) + res_ids = milvus_instance.insert(entities, ids=ids) + milvus_instance.flush() + milvus_instances_map.update({name: milvus_instance}) + logger.info(milvus_instance.describe_index()) + + # loop time unit: min -> s + pull_interval_seconds = pull_interval * 60 + tasks = ["insert_rand", "delete_rand", "query_rand", "flush", "compact"] + i = 1 + while True: + logger.info("Loop time: %d" % i) + start_time = time.time() + while time.time() - start_time < pull_interval_seconds: + if concurrent: + threads = [] + for name in collection_names: + task_name = random.choice(tasks) + task_run = getattr(milvus_instances_map[name], task_name) + t = threading.Thread(target=task_run, args=()) + threads.append(t) + t.start() + for t in threads: + t.join() + # with concurrent.futures.ThreadPoolExecutor(max_workers=concurrent_num) as executor: + # future_results = {executor.submit(getattr(milvus_instances_map[mp[j][0]], mp[j][1])): j for j in range(concurrent_num)} + # for future in concurrent.futures.as_completed(future_results): + # future.result() + else: + tmp_collection_name = random.choice(collection_names) + task_name = random.choice(tasks) + logger.info(tmp_collection_name) + logger.info(task_name) + task_run = getattr(milvus_instances_map[tmp_collection_name], task_name) + task_run() + # new connection + # for name in collection_names: + # milvus_instance = MilvusClient(collection_name=name, host=self.host) + # milvus_instances_map.update({name: milvus_instance}) + i = i + 1 + + elif run_type == "locust_mix_performance": + (data_type, collection_size, dimension, metric_type) = parser.collection_parser( + collection_name) + ni_per = collection["ni_per"] + build_index = collection["build_index"] + vector_type = self.get_vector_type(data_type) + index_field_name = utils.get_default_field_name(vector_type) + # drop exists collection + if milvus_instance.exists_collection(): + milvus_instance.drop() + time.sleep(10) + # create collection + other_fields = collection["other_fields"] if "other_fields" in collection else None + milvus_instance.create_collection(dimension, data_type=DataType.FLOAT_VECTOR, collection_name=collection_name, other_fields=other_fields) + logger.info(milvus_instance.get_info()) + # insert entities + insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(ni_per)] + insert_ids = random.sample(list(range(collection_size)), ni_per) + insert_vectors = utils.normalize(metric_type, insert_vectors) + entities = milvus_instance.generate_entities(insert_vectors, insert_ids, collection_name) + milvus_instance.insert(entities, ids=insert_ids) + # flush + milvus_instance.flush() + logger.info(milvus_instance.get_stats()) + logger.debug("Table row counts: %d" % milvus_instance.count()) + # create index + if build_index is True: + index_type = collection["index_type"] + index_param = collection["index_param"] + logger.debug("Start build index for last file") + milvus_instance.create_index(index_field_name, index_type, metric_type, index_param) + logger.debug(milvus_instance.describe_index()) + # locust + task = collection["tasks"] + task_file = utils.get_unique_name() + task_file_script = task_file + '.py' + task_file_csv = task_file + '_stats.csv' + task_types = task["types"] + connection_type = "single" + connection_num = task["connection_num"] + if connection_num > 1: + connection_type = "multi" + clients_num = task["clients_num"] + hatch_rate = task["hatch_rate"] + during_time = task["during_time"] + def_strs = "" + # define def str + for task_type in task_types: + type = task_type["type"] + weight = task_type["weight"] + if type == "flush": + def_str = """ + @task(%d) + def flush(self): + client = get_client(collection_name) + client.flush(collection_name=collection_name) + """ % weight + if type == "compact": + def_str = """ + @task(%d) + def compact(self): + client = get_client(collection_name) + client.compact(collection_name) + """ % weight + if type == "query": + def_str = """ + @task(%d) + def query(self): + client = get_client(collection_name) + params = %s + X = [[random.random() for i in range(dim)] for i in range(params["nq"])] + vector_query = {"vector": {"%s": { + "topk": params["top_k"], + "query": X, + "metric_type": "%s", + "params": params["search_param"]}}} + client.query(vector_query, filter_query=params["filters"], collection_name=collection_name) + """ % (weight, task_type["params"], index_field_name, utils.metric_type_trans(metric_type)) + if type == "insert": + def_str = """ + @task(%d) + def insert(self): + client = get_client(collection_name) + params = %s + insert_ids = random.sample(list(range(100000)), params["nb"]) + insert_vectors = [[random.random() for _ in range(dim)] for _ in range(params["nb"])] + insert_vectors = utils.normalize("l2", insert_vectors) + entities = generate_entities(insert_vectors, insert_ids) + client.insert(entities,ids=insert_ids, collection_name=collection_name) + """ % (weight, task_type["params"]) + if type == "delete": + def_str = """ + @task(%d) + def delete(self): + client = get_client(collection_name) + ids = [random.randint(1, 1000000) for i in range(1)] + client.delete(ids, collection_name) + """ % weight + def_strs += def_str + print(def_strs) + # define locust code str + code_str = """ +import random +import json +from locust import User, task, between +from locust_task import MilvusTask +from client import MilvusClient +import utils + +host = '%s' +port = %s +collection_name = '%s' +dim = %s +connection_type = '%s' +m = MilvusClient(host=host, port=port) + + +def get_client(collection_name): + if connection_type == 'single': + return MilvusTask(m=m) + elif connection_type == 'multi': + return MilvusTask(connection_type='multi', host=host, port=port, collection_name=collection_name) + + +def generate_entities(vectors, ids): + return m.generate_entities(vectors, ids, collection_name) + + +class MixTask(User): + wait_time = between(0.001, 0.002) + %s + """ % (self.host, self.port, collection_name, dimension, connection_type, def_strs) + with open(task_file_script, "w+") as fd: + fd.write(code_str) + locust_cmd = "locust -f %s --headless --csv=%s -u %d -r %d -t %s" % ( + task_file_script, + task_file, + clients_num, + hatch_rate, + during_time) + logger.info(locust_cmd) + try: + res = os.system(locust_cmd) + except Exception as e: + logger.error(str(e)) + return + + # . retrieve and collect test statistics + metric = None + with open(task_file_csv, newline='') as fd: + dr = csv.DictReader(fd) + for row in dr: + if row["Name"] != "Aggregated": + continue + metric = row + logger.info(metric) + + else: + raise Exception("Run type not defined") + logger.debug("All test finished") diff --git a/tests/benchmark/locust_file.py b/tests/benchmark/locust_file.py new file mode 100644 index 0000000000..97f1f65d42 --- /dev/null +++ b/tests/benchmark/locust_file.py @@ -0,0 +1,30 @@ + +import random +from locust import HttpUser, task, between + + +collection_name = "random_1m_2048_512_ip_sq8" +headers = {'Content-Type': "application/json"} +url = '/collections/%s/vectors' % collection_name +top_k = 2 +nq = 1 +dim = 512 +vectors = [[random.random() for _ in range(dim)] for _ in range(nq)] +data = { + "search":{ + "topk": top_k, + "vectors": vectors, + "params": { + "nprobe": 1 + } + } +} + +class MyUser(HttpUser): + wait_time = between(0, 0.1) + host = "http://192.168.1.112:19122" + + @task + def search(self): + response = self.client.put(url=url, json=data, headers=headers, timeout=2) + print(response) diff --git a/tests/benchmark/locust_flush_task.py b/tests/benchmark/locust_flush_task.py new file mode 100644 index 0000000000..32de83ae78 --- /dev/null +++ b/tests/benchmark/locust_flush_task.py @@ -0,0 +1,33 @@ +import random +from locust import User, task, between +from locust_task import MilvusTask +from client import MilvusClient +from milvus import DataType + +connection_type = "single" +host = "192.168.1.6" +port = 19530 +collection_name = "create_collection_CZkkwJgo" +dim = 128 +nb = 50000 +m = MilvusClient(host=host, port=port, collection_name=collection_name) +m.clean_db() +m.create_collection(dim, data_type=DataType.FLOAT_VECTOR, auto_id=True, other_fields=None) +vectors = [[random.random() for _ in range(dim)] for _ in range(nb)] +entities = m.generate_entities(vectors) + + +class FlushTask(User): + wait_time = between(0.001, 0.002) + if connection_type == "single": + client = MilvusTask(m=m) + else: + client = MilvusTask(host=host, port=port, collection_name=collection_name) + + # def insert(self): + # self.client.insert(entities) + + @task(1) + def flush(self): + self.client.insert(entities) + self.client.flush(collection_name) diff --git a/tests/benchmark/locust_get_entity_task.py b/tests/benchmark/locust_get_entity_task.py new file mode 100644 index 0000000000..28df3daa76 --- /dev/null +++ b/tests/benchmark/locust_get_entity_task.py @@ -0,0 +1,36 @@ +import logging +import random +from locust import User, task, between +from locust_task import MilvusTask +from client import MilvusClient +from milvus import DataType + +connection_type = "single" +host = "192.168.1.6" +port = 19530 +collection_name = "sift_10m_100000_128_l2" +dim = 128 +m = MilvusClient(host=host, port=port, collection_name=collection_name) +# m.clean_db() +# m.create_collection(dim, data_type=DataType.FLOAT_VECTOR, auto_id=True, other_fields=None) +nb = 6000 +# vectors = [[random.random() for _ in range(dim)] for _ in range(nb)] +# entities = m.generate_entities(vectors) +ids = [i for i in range(nb)] + +class GetEntityTask(User): + wait_time = between(0.001, 0.002) + if connection_type == "single": + client = MilvusTask(m=m) + else: + client = MilvusTask(host=host, port=port, collection_name=collection_name) + + # def insert(self): + # self.client.insert(entities) + + @task(1) + def get_entity_by_id(self): + # num = random.randint(100, 200) + # get_ids = random.sample(ids, num) + self.client.get_entities([0]) + # logging.getLogger().info(len(get_res)) diff --git a/tests/benchmark/locust_insert_task.py b/tests/benchmark/locust_insert_task.py new file mode 100644 index 0000000000..fc2951a982 --- /dev/null +++ b/tests/benchmark/locust_insert_task.py @@ -0,0 +1,33 @@ +import random +from locust import User, task, between +from locust_task import MilvusTask +from client import MilvusClient +from milvus import DataType + +connection_type = "single" +host = "192.168.1.6" +port = 19530 +collection_name = "create_collection_hello" +dim = 128 +nb = 50000 +m = MilvusClient(host=host, port=port, collection_name=collection_name) +# m.clean_db() +m.create_collection(dim, data_type=DataType.FLOAT_VECTOR, auto_id=True, other_fields=None) +vectors = [[random.random() for _ in range(dim)] for _ in range(nb)] +entities = m.generate_entities(vectors) + + +class FlushTask(User): + wait_time = between(0.001, 0.002) + if connection_type == "single": + client = MilvusTask(m=m) + else: + client = MilvusTask(host=host, port=port, collection_name=collection_name) + + @task(1) + def insert(self): + self.client.insert(entities) + # @task(1) + # def create_partition(self): + # tag = 'tag_'.join(random.choice(string.ascii_letters) for _ in range(8)) + # self.client.create_partition(tag, collection_name) diff --git a/tests/benchmark/locust_search_task.py b/tests/benchmark/locust_search_task.py new file mode 100644 index 0000000000..f58cb0d6e7 --- /dev/null +++ b/tests/benchmark/locust_search_task.py @@ -0,0 +1,46 @@ +import random +from client import MilvusClient +from locust_task import MilvusTask +from locust import User, task, between + +connection_type = "single" +host = "172.16.50.9" +port = 19530 +collection_name = "sift_1m_2000000_128_l2_2" +m = MilvusClient(host=host, port=port, collection_name=collection_name) +dim = 128 +top_k = 5 +nq = 1 +X = [[random.random() for i in range(dim)] for i in range(nq)] +search_params = {"nprobe": 16} +vector_query = {"vector": {'float_vector': { + "topk": top_k, + "query": X, + "params": search_params, + 'metric_type': 'L2'}}} +# m.clean_db() + + +class QueryTask(User): + wait_time = between(0.001, 0.002) + + def preload(self): + self.client.preload_collection() + + @task(10) + def query(self): + if connection_type == "single": + client = MilvusTask(m=m, connection_type=connection_type) + elif connection_type == "multi": + client = MilvusTask(host, port, collection_name, connection_type=connection_type) + top_k = 10 + search_param = {"nprobe": 16} + X = [[random.random() for i in range(dim)]] + vector_query = {"vector": {"float_vector": { + "topk": top_k, + "query": X, + "metric_type": "L2", + "params": search_param} + }} + filter_query = None + client.query(vector_query, filter_query=filter_query, collection_name=collection_name) \ No newline at end of file diff --git a/tests/benchmark/locust_task.py b/tests/benchmark/locust_task.py new file mode 100644 index 0000000000..0330b3664b --- /dev/null +++ b/tests/benchmark/locust_task.py @@ -0,0 +1,37 @@ +import time +import pdb +import random +import logging +from locust import User, events +from client import MilvusClient + + +class MilvusTask(object): + def __init__(self, *args, **kwargs): + self.request_type = "grpc" + connection_type = kwargs.get("connection_type") + if connection_type == "single": + self.m = kwargs.get("m") + elif connection_type == "multi": + host = kwargs.get("host") + port = kwargs.get("port") + collection_name = kwargs.get("collection_name") + self.m = MilvusClient(host=host, port=port, collection_name=collection_name) + # logging.getLogger().error(id(self.m)) + + def __getattr__(self, name): + func = getattr(self.m, name) + + def wrapper(*args, **kwargs): + start_time = time.time() + try: + result = func(*args, **kwargs) + total_time = int((time.time() - start_time) * 1000) + events.request_success.fire(request_type=self.request_type, name=name, response_time=total_time, + response_length=0) + except Exception as e: + total_time = int((time.time() - start_time) * 1000) + events.request_failure.fire(request_type=self.request_type, name=name, response_time=total_time, + exception=e, response_length=0) + + return wrapper diff --git a/tests/benchmark/locust_tasks.py b/tests/benchmark/locust_tasks.py new file mode 100644 index 0000000000..277b9ebeea --- /dev/null +++ b/tests/benchmark/locust_tasks.py @@ -0,0 +1,45 @@ +import random +import time +import logging +from locust import TaskSet, task + +dim = 128 +X = [[random.random() for _ in range(dim)] for _ in range(1)] + + +class Tasks(TaskSet): + + @task + def query(self): + top_k = 10 + search_param = {"nprobe": 16} + X = [[random.random() for i in range(dim)]] + vector_query = {"vector": {"float_vector": { + "topk": top_k, + "query": X, + "metric_type": "L2", + "params": search_param} + }} + filter_query = None + self.client.query(vector_query, filter_query=filter_query, log=False) + + @task + def flush(self): + self.client.flush(log=False) + + @task + def get(self): + self.client.get() + + @task + def delete(self): + self.client.delete([random.randint(1, 1000000)], log=False) + + def insert(self): + ids = [random.randint(1, 10000000)] + entities = self.client.generate_entities(X, ids) + self.client.insert(entities, ids, log=False) + + @task + def insert_rand(self): + self.client.insert_rand(log=False) diff --git a/tests/benchmark/locust_test.py b/tests/benchmark/locust_test.py new file mode 100644 index 0000000000..95192a70ea --- /dev/null +++ b/tests/benchmark/locust_test.py @@ -0,0 +1,18 @@ +from locust_user import locust_executor +from client import MilvusClient +from milvus import DataType + + +if __name__ == "__main__": + connection_type = "single" + host = "192.168.1.239" + # host = "172.16.50.15" + port = 19530 + collection_name = "sift_1m_2000000_128_l2_2" + run_params = {"tasks": {"insert_rand": 5, "query": 10, "flush": 2}, "clients_num": 10, "spawn_rate": 2, "during_time": 3600} + dim = 128 + m = MilvusClient(host=host, port=port, collection_name=collection_name) + m.clean_db() + m.create_collection(dim, data_type=DataType.FLOAT_VECTOR, auto_id=False, other_fields=None) + + locust_executor(host, port, collection_name, run_params=run_params) diff --git a/tests/benchmark/locust_user.py b/tests/benchmark/locust_user.py new file mode 100644 index 0000000000..5053c4a59b --- /dev/null +++ b/tests/benchmark/locust_user.py @@ -0,0 +1,70 @@ +import logging +import random +import pdb +import gevent +import gevent.monkey +gevent.monkey.patch_all() + +from locust import User, between, events, stats +from locust.env import Environment +import locust.stats +from locust.stats import stats_printer, print_stats + +locust.stats.CONSOLE_STATS_INTERVAL_SEC = 30 +from locust.log import setup_logging, greenlet_exception_logger + +from locust_tasks import Tasks +from client import MilvusClient +from locust_task import MilvusTask + +logger = logging.getLogger("__locust__") + +class MyUser(User): + # task_set = None + wait_time = between(0.001, 0.002) + + +def locust_executor(host, port, collection_name, connection_type="single", run_params=None): + m = MilvusClient(host=host, port=port, collection_name=collection_name) + MyUser.tasks = {} + tasks = run_params["tasks"] + for op, weight in tasks.items(): + task = {eval("Tasks."+op): weight} + MyUser.tasks.update(task) + logger.error(MyUser.tasks) + # MyUser.tasks = {Tasks.query: 1, Tasks.flush: 1} + MyUser.client = MilvusTask(host=host, port=port, collection_name=collection_name, connection_type=connection_type, m=m) + env = Environment(events=events, user_classes=[MyUser]) + runner = env.create_local_runner() + # setup logging + # setup_logging("WARNING", "/dev/null") + setup_logging("WARNING", "/dev/null") + greenlet_exception_logger(logger=logger) + gevent.spawn(stats_printer(env.stats)) + # env.create_web_ui("127.0.0.1", 8089) + # gevent.spawn(stats_printer(env.stats), env, "test", full_history=True) + # events.init.fire(environment=env, runner=runner) + clients_num = run_params["clients_num"] + spawn_rate = run_params["spawn_rate"] + during_time = run_params["during_time"] + runner.start(clients_num, spawn_rate=spawn_rate) + gevent.spawn_later(during_time, lambda: runner.quit()) + runner.greenlet.join() + print_stats(env.stats) + result = { + "rps": round(env.stats.total.current_rps, 1), + "fail_ratio": env.stats.total.fail_ratio, + "max_response_time": round(env.stats.total.max_response_time, 1), + "min_response_time": round(env.stats.total.avg_response_time, 1) + } + runner.stop() + return result + + +if __name__ == '__main__': + connection_type = "single" + host = "192.168.1.112" + port = 19530 + collection_name = "sift_1m_2000000_128_l2_2" + run_params = {"tasks": {"query": 1, "flush": 1}, "clients_num": 1, "spawn_rate": 1, "during_time": 3} + locust_executor(host, port, collection_name, run_params=run_params) diff --git a/tests/benchmark/main.py b/tests/benchmark/main.py new file mode 100644 index 0000000000..1b2d8c9d18 --- /dev/null +++ b/tests/benchmark/main.py @@ -0,0 +1,199 @@ +import os +import sys +import time +from datetime import datetime +import pdb +import argparse +import logging +import traceback +from multiprocessing import Process +from queue import Queue +from logging import handlers +from yaml import full_load, dump +from local_runner import LocalRunner +from docker_runner import DockerRunner +import parser + +DEFAULT_IMAGE = "milvusdb/milvus:latest" +LOG_FOLDER = "logs" +NAMESPACE = "milvus" +LOG_PATH = "/test/milvus/benchmark/logs/" +BRANCH = "0.11.1" + +logger = logging.getLogger('milvus_benchmark') +logger.setLevel(logging.INFO) +# create file handler which logs even debug messages +fh = logging.FileHandler(LOG_PATH+'benchmark-{}-{:%Y-%m-%d}.log'.format(BRANCH, datetime.now())) +fh.setLevel(logging.DEBUG) +# create console handler with a higher log level +ch = logging.StreamHandler() +ch.setLevel(logging.INFO) +# create formatter and add it to the handlers +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +fh.setFormatter(formatter) +ch.setFormatter(formatter) +# add the handlers to the logger +logger.addHandler(fh) +logger.addHandler(ch) + +def positive_int(s): + i = None + try: + i = int(s) + except ValueError: + pass + if not i or i < 1: + raise argparse.ArgumentTypeError("%r is not a positive integer" % s) + return i + + +def get_image_tag(image_version, image_type): + return "%s-%s-centos7-release" % (image_version, image_type) + # return "%s-%s-centos7-release" % ("0.7.1", image_type) + # return "%s-%s-centos7-release" % ("PR-2780", image_type) + + +def queue_worker(queue): + from k8s_runner import K8sRunner + while not queue.empty(): + q = queue.get() + suite = q["suite"] + server_host = q["server_host"] + deploy_mode = q["deploy_mode"] + image_type = q["image_type"] + image_tag = q["image_tag"] + + with open(suite) as f: + suite_dict = full_load(f) + f.close() + logger.debug(suite_dict) + + run_type, run_params = parser.operations_parser(suite_dict) + collections = run_params["collections"] + for collection in collections: + # run tests + milvus_config = collection["milvus"] if "milvus" in collection else None + server_config = collection["server"] if "server" in collection else None + logger.debug(milvus_config) + logger.debug(server_config) + runner = K8sRunner() + if runner.init_env(milvus_config, server_config, server_host, deploy_mode, image_type, image_tag): + logger.debug("Start run tests") + try: + runner.run(run_type, collection) + except Exception as e: + logger.error(str(e)) + logger.error(traceback.format_exc()) + finally: + time.sleep(60) + runner.clean_up() + else: + logger.error("Runner init failed") + if server_host: + logger.debug("All task finished in queue: %s" % server_host) + + +def main(): + arg_parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + # helm mode with scheduler + arg_parser.add_argument( + "--image-version", + default="", + help="image version") + arg_parser.add_argument( + "--schedule-conf", + metavar='FILE', + default='', + help="load test schedule from FILE") + arg_parser.add_argument( + "--deploy-mode", + default='', + help="single node or multi nodes") + + # local mode + arg_parser.add_argument( + '--local', + action='store_true', + help='use local milvus server') + arg_parser.add_argument( + '--host', + help='server host ip param for local mode', + default='127.0.0.1') + arg_parser.add_argument( + '--port', + help='server port param for local mode', + default='19530') + arg_parser.add_argument( + '--suite', + metavar='FILE', + help='load test suite from FILE', + default='') + + args = arg_parser.parse_args() + + if args.schedule_conf: + if args.local: + raise Exception("Helm mode with scheduler and other mode are incompatible") + if not args.image_version: + raise Exception("Image version not given") + image_version = args.image_version + deploy_mode = args.deploy_mode + with open(args.schedule_conf) as f: + schedule_config = full_load(f) + f.close() + queues = [] + # server_names = set() + server_names = [] + for item in schedule_config: + server_host = item["server"] if "server" in item else "" + suite_params = item["suite_params"] + server_names.append(server_host) + q = Queue() + for suite_param in suite_params: + suite = "suites/"+suite_param["suite"] + image_type = suite_param["image_type"] + image_tag = get_image_tag(image_version, image_type) + q.put({ + "suite": suite, + "server_host": server_host, + "deploy_mode": deploy_mode, + "image_tag": image_tag, + "image_type": image_type + }) + queues.append(q) + logger.error(queues) + thread_num = len(server_names) + processes = [] + + for i in range(thread_num): + x = Process(target=queue_worker, args=(queues[i], )) + processes.append(x) + x.start() + time.sleep(10) + for x in processes: + x.join() + + # queue_worker(queues[0]) + + elif args.local: + # for local mode + host = args.host + port = args.port + suite = args.suite + with open(suite) as f: + suite_dict = full_load(f) + f.close() + logger.debug(suite_dict) + run_type, run_params = parser.operations_parser(suite_dict) + collections = run_params["collections"] + if len(collections) > 1: + raise Exception("Multi collections not supported in Local Mode") + collection = collections[0] + runner = LocalRunner(host, port) + logger.info("Start run local mode test, test type: %s" % run_type) + runner.run(run_type, collection) + + +if __name__ == "__main__": + main() diff --git a/tests/benchmark/mix_task.py b/tests/benchmark/mix_task.py new file mode 100644 index 0000000000..912989f85e --- /dev/null +++ b/tests/benchmark/mix_task.py @@ -0,0 +1,42 @@ +import random +from locust import User, task, between +from locust_task import MilvusTask +from client import MilvusClient + +connection_type = "single" +host = "192.168.1.29" +port = 19530 +collection_name = "sift_128_euclidean" +dim = 128 +m = MilvusClient(host=host, port=port, collection_name=collection_name) + + +class MixTask(User): + wait_time = between(0.001, 0.002) + print("in query task") + if connection_type == "single": + client = MilvusTask(m=m) + else: + client = MilvusTask(host=host, port=port, collection_name=collection_name) + + @task(30) + def query(self): + top_k = 10 + X = [[random.random() for i in range(dim)] for i in range(1)] + search_param = {"nprobe": 16} + self.client.query(X, top_k, search_param) + + @task(10) + def insert(self): + id = random.randint(10000000, 10000000000) + X = [[random.random() for i in range(dim)] for i in range(1)] + self.client.insert(X, ids=[id]) + + @task(1) + def flush(self): + self.client.flush() + + # @task(5) + # def delete(self): + # self.client.delete([random.randint(1, 1000000)]) + diff --git a/tests/benchmark/operation.py b/tests/benchmark/operation.py new file mode 100644 index 0000000000..348fa47f4a --- /dev/null +++ b/tests/benchmark/operation.py @@ -0,0 +1,10 @@ +from __future__ import absolute_import +import pdb +import time + +class Base(object): + pass + + +class Insert(Base): + pass \ No newline at end of file diff --git a/tests/benchmark/parser.py b/tests/benchmark/parser.py new file mode 100644 index 0000000000..428f1db0d4 --- /dev/null +++ b/tests/benchmark/parser.py @@ -0,0 +1,85 @@ +import pdb +import logging + +logger = logging.getLogger("milvus_benchmark.parser") + + +def operations_parser(operations): + if not operations: + raise Exception("No operations in suite defined") + for run_type, run_params in operations.items(): + logger.debug(run_type) + return (run_type, run_params) + + +def collection_parser(collection_name): + tmp = collection_name.split("_") + # if len(tmp) != 5: + # return None + data_type = tmp[0] + collection_size_unit = tmp[1][-1] + collection_size = tmp[1][0:-1] + if collection_size_unit == "m": + collection_size = int(collection_size) * 1000000 + elif collection_size_unit == "b": + collection_size = int(collection_size) * 1000000000 + dimension = int(tmp[2]) + metric_type = str(tmp[3]) + return (data_type, collection_size, dimension, metric_type) + + +def parse_ann_collection_name(collection_name): + data_type = collection_name.split("_")[0] + dimension = int(collection_name.split("_")[1]) + metric = collection_name.split("_")[2] + # metric = collection_name.attrs['distance'] + # dimension = len(collection_name["train"][0]) + if metric == "euclidean": + metric_type = "l2" + elif metric == "angular": + metric_type = "ip" + elif metric == "jaccard": + metric_type = "jaccard" + elif metric == "hamming": + metric_type = "hamming" + return (data_type, dimension, metric_type) + + +def search_params_parser(param): + # parse top-k, set default value if top-k not in param + if "top_ks" not in param: + top_ks = [10] + else: + top_ks = param["top_ks"] + if isinstance(top_ks, int): + top_ks = [top_ks] + elif isinstance(top_ks, list): + top_ks = list(top_ks) + else: + logger.warning("Invalid format top-ks: %s" % str(top_ks)) + + # parse nqs, set default value if nq not in param + if "nqs" not in param: + nqs = [10] + else: + nqs = param["nqs"] + if isinstance(nqs, int): + nqs = [nqs] + elif isinstance(nqs, list): + nqs = list(nqs) + else: + logger.warning("Invalid format nqs: %s" % str(nqs)) + + # parse nprobes + if "nprobes" not in param: + nprobes = [1] + else: + nprobes = param["nprobes"] + if isinstance(nprobes, int): + nprobes = [nprobes] + elif isinstance(nprobes, list): + nprobes = list(nprobes) + else: + logger.warning("Invalid format nprobes: %s" % str(nprobes)) + + return top_ks, nqs, nprobes diff --git a/tests/benchmark/requirements.txt b/tests/benchmark/requirements.txt new file mode 100644 index 0000000000..96d101ed5a --- /dev/null +++ b/tests/benchmark/requirements.txt @@ -0,0 +1,12 @@ +pymilvus-test>=0.5.0,<0.6.0 +scipy>=1.3.1 +scikit-learn>=0.19.1 +h5py>=2.7.1 +# influxdb==5.2.2 +pyyaml>=5.1 +tableprint==0.8.0 +ansicolors==1.1.8 +kubernetes==10.0.1 +# rq==1.2.0 +locust>=1.3.2 +pymongo==3.10.0 diff --git a/tests/benchmark/results/__init__.py b/tests/benchmark/results/__init__.py new file mode 100644 index 0000000000..81b55da1c5 --- /dev/null +++ b/tests/benchmark/results/__init__.py @@ -0,0 +1,11 @@ + +class Reporter(object): + def __init__(self): + pass + + def report(self, result): + pass + + +class BaseResult(object): + pass \ No newline at end of file diff --git a/tests/benchmark/results/reporter.py b/tests/benchmark/results/reporter.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/benchmark/runner.py b/tests/benchmark/runner.py new file mode 100644 index 0000000000..0a93cf9fa9 --- /dev/null +++ b/tests/benchmark/runner.py @@ -0,0 +1,369 @@ +import os +import threading +import logging +import pdb +import time +import random +import grpc +from multiprocessing import Process +from itertools import product +import numpy as np +import sklearn.preprocessing +from milvus import DataType +from client import MilvusClient +import utils +import parser + +logger = logging.getLogger("milvus_benchmark.runner") + +VECTORS_PER_FILE = 1000000 +SIFT_VECTORS_PER_FILE = 100000 +BINARY_VECTORS_PER_FILE = 2000000 + +MAX_NQ = 10001 +FILE_PREFIX = "binary_" + +# FOLDER_NAME = 'ann_1000m/source_data' +SRC_BINARY_DATA_DIR = '/test/milvus/raw_data/random/' +SIFT_SRC_DATA_DIR = '/test/milvus/raw_data/sift1b/' +DEEP_SRC_DATA_DIR = '/test/milvus/raw_data/deep1b/' +BINARY_SRC_DATA_DIR = '/test/milvus/raw_data/binary/' +SIFT_SRC_GROUNDTRUTH_DATA_DIR = SIFT_SRC_DATA_DIR + 'gnd' + +WARM_TOP_K = 1 +WARM_NQ = 1 +DEFAULT_DIM = 512 + + +GROUNDTRUTH_MAP = { + "1000000": "idx_1M.ivecs", + "2000000": "idx_2M.ivecs", + "5000000": "idx_5M.ivecs", + "10000000": "idx_10M.ivecs", + "20000000": "idx_20M.ivecs", + "50000000": "idx_50M.ivecs", + "100000000": "idx_100M.ivecs", + "200000000": "idx_200M.ivecs", + "500000000": "idx_500M.ivecs", + "1000000000": "idx_1000M.ivecs", +} + + +def gen_file_name(idx, dimension, data_type): + s = "%05d" % idx + fname = FILE_PREFIX + str(dimension) + "d_" + s + ".npy" + if data_type == "random": + fname = SRC_BINARY_DATA_DIR+fname + elif data_type == "sift": + fname = SIFT_SRC_DATA_DIR+fname + elif data_type == "deep": + fname = DEEP_SRC_DATA_DIR+fname + elif data_type == "binary": + fname = BINARY_SRC_DATA_DIR+fname + return fname + + +def get_vectors_from_binary(nq, dimension, data_type): + # use the first file, nq should be less than VECTORS_PER_FILE + if nq > MAX_NQ: + raise Exception("Over size nq") + if data_type == "random": + file_name = SRC_BINARY_DATA_DIR+'query_%d.npy' % dimension + elif data_type == "sift": + file_name = SIFT_SRC_DATA_DIR+'query.npy' + elif data_type == "deep": + file_name = DEEP_SRC_DATA_DIR+'query.npy' + elif data_type == "binary": + file_name = BINARY_SRC_DATA_DIR+'query.npy' + data = np.load(file_name) + vectors = data[0:nq].tolist() + return vectors + + +class Runner(object): + def __init__(self): + pass + + def gen_executors(self, operations): + l = [] + for name, operation in operations.items(): + weight = operation["weight"] if "weight" in operation else 1 + l.extend([name] * weight) + random.shuffle(l) + return l + + def get_vector_type(self, data_type): + vector_type = '' + if data_type in ["random", "sift", "deep", "glove"]: + vector_type = DataType.FLOAT_VECTOR + elif data_type in ["binary"]: + vector_type = DataType.BINARY_VECTOR + else: + raise Exception("Data type: %s not defined" % data_type) + return vector_type + + def get_vector_type_from_metric(self, metric_type): + vector_type = '' + if metric_type in ["hamming", "jaccard"]: + vector_type = DataType.BINARY_VECTOR + else: + vector_type = DataType.FLOAT_VECTOR + return vector_type + + def normalize(self, metric_type, X): + if metric_type == "ip": + logger.info("Set normalize for metric_type: %s" % metric_type) + X = sklearn.preprocessing.normalize(X, axis=1, norm='l2') + X = X.astype(np.float32) + elif metric_type == "l2": + X = X.astype(np.float32) + elif metric_type in ["jaccard", "hamming", "sub", "super"]: + tmp = [] + for item in X: + new_vector = bytes(np.packbits(item, axis=-1).tolist()) + tmp.append(new_vector) + X = tmp + return X + + def generate_combinations(self, args): + if isinstance(args, list): + args = [el if isinstance(el, list) else [el] for el in args] + return [list(x) for x in product(*args)] + elif isinstance(args, dict): + flat = [] + for k, v in args.items(): + if isinstance(v, list): + flat.append([(k, el) for el in v]) + else: + flat.append([(k, v)]) + return [dict(x) for x in product(*flat)] + else: + raise TypeError("No args handling exists for %s" % type(args).__name__) + + def do_insert(self, milvus, collection_name, data_type, dimension, size, ni): + ''' + @params: + mivlus: server connect instance + dimension: collection dimensionn + # index_file_size: size trigger file merge + size: row count of vectors to be insert + ni: row count of vectors to be insert each time + # store_id: if store the ids returned by call add_vectors or not + @return: + total_time: total time for all insert operation + qps: vectors added per second + ni_time: avarage insert operation time + ''' + bi_res = {} + total_time = 0.0 + qps = 0.0 + ni_time = 0.0 + if data_type == "random": + if dimension == 512: + vectors_per_file = VECTORS_PER_FILE + elif dimension == 4096: + vectors_per_file = 100000 + elif dimension == 16384: + vectors_per_file = 10000 + elif data_type == "sift": + vectors_per_file = SIFT_VECTORS_PER_FILE + elif data_type in ["binary"]: + vectors_per_file = BINARY_VECTORS_PER_FILE + else: + raise Exception("data_type: %s not supported" % data_type) + if size % vectors_per_file or size % ni: + raise Exception("Not invalid collection size or ni") + i = 0 + while i < (size // vectors_per_file): + vectors = [] + if vectors_per_file >= ni: + file_name = gen_file_name(i, dimension, data_type) + # logger.info("Load npy file: %s start" % file_name) + data = np.load(file_name) + # logger.info("Load npy file: %s end" % file_name) + for j in range(vectors_per_file // ni): + vectors = data[j*ni:(j+1)*ni].tolist() + if vectors: + # start insert vectors + start_id = i * vectors_per_file + j * ni + end_id = start_id + len(vectors) + logger.debug("Start id: %s, end id: %s" % (start_id, end_id)) + ids = [k for k in range(start_id, end_id)] + entities = milvus.generate_entities(vectors, ids) + ni_start_time = time.time() + try: + res_ids = milvus.insert(entities, ids=ids) + except grpc.RpcError as e: + if e.code() == grpc.StatusCode.UNAVAILABLE: + logger.debug("Retry insert") + def retry(): + res_ids = milvus.insert(entities, ids=ids) + + t0 = threading.Thread(target=retry) + t0.start() + t0.join() + logger.debug("Retry successfully") + raise e + assert ids == res_ids + # milvus.flush() + logger.debug(milvus.count()) + ni_end_time = time.time() + total_time = total_time + ni_end_time - ni_start_time + i += 1 + else: + vectors.clear() + loops = ni // vectors_per_file + for j in range(loops): + file_name = gen_file_name(loops*i+j, dimension, data_type) + data = np.load(file_name) + vectors.extend(data.tolist()) + if vectors: + start_id = i * vectors_per_file + end_id = start_id + len(vectors) + logger.info("Start id: %s, end id: %s" % (start_id, end_id)) + ids = [k for k in range(start_id, end_id)] + entities = milvus.generate_entities(vectors, ids) + ni_start_time = time.time() + try: + res_ids = milvus.insert(entities, ids=ids) + except grpc.RpcError as e: + if e.code() == grpc.StatusCode.UNAVAILABLE: + logger.debug("Retry insert") + def retry(): + res_ids = milvus.insert(entities, ids=ids) + + t0 = threading.Thread(target=retry) + t0.start() + t0.join() + logger.debug("Retry successfully") + raise e + + assert ids == res_ids + # milvus.flush() + logger.debug(milvus.count()) + ni_end_time = time.time() + total_time = total_time + ni_end_time - ni_start_time + i += loops + qps = round(size / total_time, 2) + ni_time = round(total_time / (size / ni), 2) + bi_res["total_time"] = round(total_time, 2) + bi_res["qps"] = qps + bi_res["ni_time"] = ni_time + return bi_res + + def do_query(self, milvus, collection_name, vec_field_name, top_ks, nqs, run_count=1, search_param=None, filter_query=None): + bi_res = [] + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + base_query_vectors = get_vectors_from_binary(MAX_NQ, dimension, data_type) + for nq in nqs: + tmp_res = [] + query_vectors = base_query_vectors[0:nq] + for top_k in top_ks: + avg_query_time = 0.0 + min_query_time = 0.0 + logger.info("Start query, query params: top-k: {}, nq: {}, actually length of vectors: {}".format(top_k, nq, len(query_vectors))) + for i in range(run_count): + logger.debug("Start run query, run %d of %s" % (i+1, run_count)) + start_time = time.time() + vector_query = {"vector": {vec_field_name: { + "topk": top_k, + "query": query_vectors, + "metric_type": utils.metric_type_trans(metric_type), + "params": search_param} + }} + query_res = milvus.query(vector_query, filter_query=filter_query) + interval_time = time.time() - start_time + if (i == 0) or (min_query_time > interval_time): + min_query_time = interval_time + logger.info("Min query time: %.2f" % min_query_time) + tmp_res.append(round(min_query_time, 2)) + bi_res.append(tmp_res) + return bi_res + + def do_query_qps(self, milvus, query_vectors, top_k, search_param): + start_time = time.time() + result = milvus.query(query_vectors, top_k, search_param) + end_time = time.time() + return end_time - start_time + + def do_query_ids(self, milvus, collection_name, vec_field_name, top_k, nq, search_param=None, filter_query=None): + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + base_query_vectors = get_vectors_from_binary(MAX_NQ, dimension, data_type) + query_vectors = base_query_vectors[0:nq] + logger.info("Start query, query params: top-k: {}, nq: {}, actually length of vectors: {}".format(top_k, nq, len(query_vectors))) + vector_query = {"vector": {vec_field_name: { + "topk": top_k, + "query": query_vectors, + "metric_type": utils.metric_type_trans(metric_type), + "params": search_param} + }} + query_res = milvus.query(vector_query, filter_query=filter_query) + result_ids = milvus.get_ids(query_res) + return result_ids + + def do_query_acc(self, milvus, collection_name, top_k, nq, id_store_name, search_param=None): + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + base_query_vectors = get_vectors_from_binary(MAX_NQ, dimension, data_type) + vectors = base_query_vectors[0:nq] + logger.info("Start query, query params: top-k: {}, nq: {}, actually length of vectors: {}".format(top_k, nq, len(vectors))) + query_res = milvus.query(vectors, top_k, search_param=None) + # if file existed, cover it + if os.path.isfile(id_store_name): + os.remove(id_store_name) + with open(id_store_name, 'a+') as fd: + for nq_item in query_res: + for item in nq_item: + fd.write(str(item.id)+'\t') + fd.write('\n') + + # compute and print accuracy + def compute_accuracy(self, flat_file_name, index_file_name): + flat_id_list = []; index_id_list = [] + logger.info("Loading flat id file: %s" % flat_file_name) + with open(flat_file_name, 'r') as flat_id_fd: + for line in flat_id_fd: + tmp_list = line.strip("\n").strip().split("\t") + flat_id_list.append(tmp_list) + logger.info("Loading index id file: %s" % index_file_name) + with open(index_file_name) as index_id_fd: + for line in index_id_fd: + tmp_list = line.strip("\n").strip().split("\t") + index_id_list.append(tmp_list) + if len(flat_id_list) != len(index_id_list): + raise Exception("Flat index result length: not match, Acc compute exiting ..." % (len(flat_id_list), len(index_id_list))) + # get the accuracy + return self.get_recall_value(flat_id_list, index_id_list) + + def get_recall_value(self, true_ids, result_ids): + """ + Use the intersection length + """ + sum_radio = 0.0 + for index, item in enumerate(result_ids): + # tmp = set(item).intersection(set(flat_id_list[index])) + tmp = set(true_ids[index]).intersection(set(item)) + sum_radio = sum_radio + len(tmp) / len(item) + # logger.debug(sum_radio) + return round(sum_radio / len(result_ids), 3) + + """ + Implementation based on: + https://github.com/facebookresearch/faiss/blob/master/benchs/datasets.py + """ + def get_groundtruth_ids(self, collection_size): + fname = GROUNDTRUTH_MAP[str(collection_size)] + fname = SIFT_SRC_GROUNDTRUTH_DATA_DIR + "/" + fname + a = np.fromfile(fname, dtype='int32') + d = a[0] + true_ids = a.reshape(-1, d + 1)[:, 1:].copy() + return true_ids + + def get_fields(self, milvus, collection_name): + fields = [] + info = milvus.get_info(collection_name) + for item in info["fields"]: + fields.append(item["name"]) + return fields + + # def get_filter_query(self, filter_query): + # for filter in filter_query: diff --git a/tests/benchmark/runners/__init__.py b/tests/benchmark/runners/__init__.py new file mode 100644 index 0000000000..4b082462d5 --- /dev/null +++ b/tests/benchmark/runners/__init__.py @@ -0,0 +1,11 @@ + + +class BaseRunner(object): + def __init__(self): + pass + + def set_up(self): + pass + + def tear_down(self): + pass diff --git a/tests/benchmark/runners/locust_runner.py b/tests/benchmark/runners/locust_runner.py new file mode 100644 index 0000000000..afe2b1e043 --- /dev/null +++ b/tests/benchmark/runners/locust_runner.py @@ -0,0 +1,75 @@ +import time +import random +from locust import Locust, TaskSet, events, task, between +from client import MilvusClient +from . import BasicRunner + + +dim = 128 +top_k = 10 +X = [[random.random() for i in range(dim)] for i in range(1)] +search_param = {"nprobe": 16} + + +class MilvusTask(object): + def __init__(self, type="single", args): + self.type = type + self.m = None + if type == "single": + self.m = MilvusClient(host=args["host"], port=args["port"], collection_name=args["collection_name"]) + elif type == "multi": + self.m = MilvusClient(host=args["m"]) + + def query(self, *args, **kwargs): + name = "milvus_search" + request_type = "grpc" + start_time = time.time() + try: + # result = self.m.getattr(*args, **kwargs) + status, result = self.m.query(*args, **kwargs) + except Exception as e: + total_time = int((time.time() - start_time) * 1000) + events.request_failure.fire(request_type=request_type, name=name, response_time=total_time, exception=e, response_length=0) + else: + if not status.OK: + total_time = int((time.time() - start_time) * 1000) + events.request_failure.fire(request_type=request_type, name=name, response_time=total_time, exception=e, response_length=0) + else: + total_time = int((time.time() - start_time) * 1000) + events.request_success.fire(request_type=request_type, name=name, response_time=total_time, response_length=0) + # In this example, I've hardcoded response_length=0. If we would want the response length to be + # reported correctly in the statistics, we would probably need to hook in at a lower level + + +class MilvusLocust(Locust): + def __init__(self, *args, **kwargs): + super(MilvusLocust, self).__init__(*args, **kwargs) + self.client = MilvusTask(self.host, self.port, self.collection_name) + + +class Query(MilvusLocust): + host = "192.168.1.183" + port = 19530 + collection_name = "sift_128_euclidean" + # m = MilvusClient(host=host, port=port, collection_name=collection_name) + wait_time = between(0.001, 0.002) + + class task_set(TaskSet): + @task + def query(self): + self.client.query(X, top_k, search_param) + + +class LocustRunner(BasicRunner): + """Only one client, not support M/S mode""" + def __init__(self, args): + # Start client with params including client number && last time && hatch rate ... + pass + + def set_up(self): + # helm install locust client + pass + + def tear_down(self): + # helm uninstall + pass diff --git a/tests/benchmark/scheduler/010_data.json b/tests/benchmark/scheduler/010_data.json new file mode 100644 index 0000000000..d7074b63f5 --- /dev/null +++ b/tests/benchmark/scheduler/010_data.json @@ -0,0 +1,65 @@ +[ + { + "server": "athena", + "suite_params": [ + { + "suite": "080_gpu_accuracy.yaml", + "image_type": "gpu" + }, + { + "suite": "080_search_stability.yaml", + "image_type": "gpu" + }, + { + "suite": "gpu_accuracy_ann.yaml", + "image_type": "gpu" + } + ] + }, + { + "server": "poseidon", + "suite_params": [ + { + "suite": "080_gpu_search.yaml", + "image_type": "gpu" + }, + { + "suite": "080_cpu_search.yaml", + "image_type": "cpu" + }, + { + "suite": "080_gpu_build.yaml", + "image_type": "gpu" + }, + { + "suite": "080_cpu_accuracy.yaml", + "image_type": "cpu" + }, + { + "suite": "locust_search.yaml", + "image_type": "cpu" + } + ] + }, + { + "server": "apollo", + "suite_params": [ + { + "suite": "cpu_accuracy_ann.yaml", + "image_type": "cpu" + }, + { + "suite": "080_cpu_build.yaml", + "image_type": "cpu" + }, + { + "suite": "080_insert_performance.yaml", + "image_type": "cpu" + }, + { + "suite": "add_flush_performance.yaml", + "image_type": "cpu" + } + ] + } +] diff --git a/tests/benchmark/scheduler/011_data.json b/tests/benchmark/scheduler/011_data.json new file mode 100644 index 0000000000..7b40300c58 --- /dev/null +++ b/tests/benchmark/scheduler/011_data.json @@ -0,0 +1,62 @@ +[ + { + "server": "idc-sh002", + "suite_params": [ + { + "suite": "011_cpu_accuracy_ann.yaml", + "image_type": "cpu" + }, + { + "suite": "011_gpu_accuracy_ann.yaml", + "image_type": "gpu" + } + ] + }, + { + "server": "idc-sh003", + "suite_params": [ + { + "suite": "locust_mix.yaml", + "image_type": "gpu" + } + ] + }, + { + "server": "idc-sh004", + "suite_params": [ + { + "suite": "011_insert_performance.yaml", + "image_type": "cpu" + }, + { + "suite": "011_gpu_accuracy.yaml", + "image_type": "gpu" + }, + { + "suite": "011_gpu_build.yaml", + "image_type": "gpu" + } + ] + }, + { + "server": "idc-sh005", + "suite_params": [ + { + "suite": "011_gpu_search.yaml", + "image_type": "gpu" + }, + { + "suite": "011_cpu_search.yaml", + "image_type": "cpu" + }, + { + "suite": "011_cpu_accuracy.yaml", + "image_type": "cpu" + }, + { + "suite": "011_locust_search.yaml", + "image_type": "cpu" + } + ] + } +] diff --git a/tests/benchmark/scheduler/011_data_acc_debug.json b/tests/benchmark/scheduler/011_data_acc_debug.json new file mode 100644 index 0000000000..3bb0df2259 --- /dev/null +++ b/tests/benchmark/scheduler/011_data_acc_debug.json @@ -0,0 +1,11 @@ +[ + { + "server": "apollo", + "suite_params": [ + { + "suite": "011_cpu_accuracy_ann.yaml", + "image_type": "cpu" + } + ] + } +] \ No newline at end of file diff --git a/tests/benchmark/scheduler/011_data_gpu_build.json b/tests/benchmark/scheduler/011_data_gpu_build.json new file mode 100644 index 0000000000..ed9642fa91 --- /dev/null +++ b/tests/benchmark/scheduler/011_data_gpu_build.json @@ -0,0 +1,11 @@ +[ + { + "server": "eros", + "suite_params": [ + { + "suite": "011_gpu_build_sift10m.yaml", + "image_type": "gpu" + } + ] + } +] \ No newline at end of file diff --git a/tests/benchmark/scheduler/011_data_insert.json b/tests/benchmark/scheduler/011_data_insert.json new file mode 100644 index 0000000000..c8bb875ac9 --- /dev/null +++ b/tests/benchmark/scheduler/011_data_insert.json @@ -0,0 +1,11 @@ +[ + { + "server": "eros", + "suite_params": [ + { + "suite": "011_insert_data.yaml", + "image_type": "cpu" + } + ] + } + ] \ No newline at end of file diff --git a/tests/benchmark/scheduler/011_data_search_debug.json b/tests/benchmark/scheduler/011_data_search_debug.json new file mode 100644 index 0000000000..dec44ac097 --- /dev/null +++ b/tests/benchmark/scheduler/011_data_search_debug.json @@ -0,0 +1,11 @@ +[ + { + "server": "athena", + "suite_params": [ + { + "suite": "011_gpu_search_debug.yaml", + "image_type": "gpu" + } + ] + } +] \ No newline at end of file diff --git a/tests/benchmark/scheduler/011_delete.json b/tests/benchmark/scheduler/011_delete.json new file mode 100644 index 0000000000..cc80004991 --- /dev/null +++ b/tests/benchmark/scheduler/011_delete.json @@ -0,0 +1,15 @@ +[ + { + "server": "apollo", + "suite_params": [ + { + "suite": "011_insert_performance.yaml", + "image_type": "cpu" + }, + { + "suite": "011_delete_performance.yaml", + "image_type": "cpu" + } + ] + } +] \ No newline at end of file diff --git a/tests/benchmark/scheduler/080_data.json b/tests/benchmark/scheduler/080_data.json new file mode 100644 index 0000000000..d7074b63f5 --- /dev/null +++ b/tests/benchmark/scheduler/080_data.json @@ -0,0 +1,65 @@ +[ + { + "server": "athena", + "suite_params": [ + { + "suite": "080_gpu_accuracy.yaml", + "image_type": "gpu" + }, + { + "suite": "080_search_stability.yaml", + "image_type": "gpu" + }, + { + "suite": "gpu_accuracy_ann.yaml", + "image_type": "gpu" + } + ] + }, + { + "server": "poseidon", + "suite_params": [ + { + "suite": "080_gpu_search.yaml", + "image_type": "gpu" + }, + { + "suite": "080_cpu_search.yaml", + "image_type": "cpu" + }, + { + "suite": "080_gpu_build.yaml", + "image_type": "gpu" + }, + { + "suite": "080_cpu_accuracy.yaml", + "image_type": "cpu" + }, + { + "suite": "locust_search.yaml", + "image_type": "cpu" + } + ] + }, + { + "server": "apollo", + "suite_params": [ + { + "suite": "cpu_accuracy_ann.yaml", + "image_type": "cpu" + }, + { + "suite": "080_cpu_build.yaml", + "image_type": "cpu" + }, + { + "suite": "080_insert_performance.yaml", + "image_type": "cpu" + }, + { + "suite": "add_flush_performance.yaml", + "image_type": "cpu" + } + ] + } +] diff --git a/tests/benchmark/scheduler/acc.json b/tests/benchmark/scheduler/acc.json new file mode 100644 index 0000000000..e4d47bd947 --- /dev/null +++ b/tests/benchmark/scheduler/acc.json @@ -0,0 +1,15 @@ +[ + { + "server": "poseidon", + "suite_params": [ + { + "suite": "crud_add.yaml", + "image_type": "gpu" + }, + { + "suite": "gpu_accuracy_sift1m.yaml", + "image_type": "gpu" + } + ] + } +] diff --git a/tests/benchmark/scheduler/build.json b/tests/benchmark/scheduler/build.json new file mode 100644 index 0000000000..f269669e1f --- /dev/null +++ b/tests/benchmark/scheduler/build.json @@ -0,0 +1,11 @@ +[ + { + "server": "eros", + "suite_params": [ + { + "suite": "011_gpu_build_sift1b.yaml", + "image_type": "gpu" + } + ] + } +] diff --git a/tests/benchmark/scheduler/clean.json b/tests/benchmark/scheduler/clean.json new file mode 100644 index 0000000000..be9cbbe231 --- /dev/null +++ b/tests/benchmark/scheduler/clean.json @@ -0,0 +1,11 @@ +[ + { + "server": "poseidon", + "suite_params": [ + { + "suite": "clean.yaml", + "image_type": "cpu" + } + ] + } +] diff --git a/tests/benchmark/scheduler/debug.json b/tests/benchmark/scheduler/debug.json new file mode 100644 index 0000000000..5028b53043 --- /dev/null +++ b/tests/benchmark/scheduler/debug.json @@ -0,0 +1,11 @@ +[ + { + "server": "idc-sh002", + "suite_params": [ + { + "suite": "011_cpu_search_sift10m.yaml", + "image_type": "cpu" + } + ] + } +] diff --git a/tests/benchmark/scheduler/default_config.json b/tests/benchmark/scheduler/default_config.json new file mode 100644 index 0000000000..6b76ddc4cd --- /dev/null +++ b/tests/benchmark/scheduler/default_config.json @@ -0,0 +1,53 @@ +[ + { + "server": "apollo", + "suite_params": [ + { + "suite": "cpu_accuracy_ann.yaml", + "image_type": "cpu" + } + ] + }, + { + "server": "poseidon", + "suite_params": [ + { + "suite": "gpu_search_performance.yaml", + "image_type": "gpu" + }, + { + "suite": "cpu_search_performance.yaml", + "image_type": "cpu" + }, + { + "suite": "insert_performance.yaml", + "image_type": "gpu" + }, + { + "suite": "gpu_accuracy.yaml", + "image_type": "gpu" + } + ] + }, + { + "server": "eros", + "suite_params": [ + { + "suite": "gpu_accuracy_ann.yaml", + "image_type": "gpu" + }, + { + "suite": "gpu_search_stability.yaml", + "image_type": "gpu" + }, + { + "suite": "gpu_build_performance.yaml", + "image_type": "gpu" + }, + { + "suite": "cpu_build_performance.yaml", + "image_type": "cpu" + } + ] + } +] \ No newline at end of file diff --git a/tests/benchmark/scheduler/file_size.json b/tests/benchmark/scheduler/file_size.json new file mode 100644 index 0000000000..f8da372e46 --- /dev/null +++ b/tests/benchmark/scheduler/file_size.json @@ -0,0 +1,11 @@ +[ + { + "server": "athena", + "suite_params": [ + { + "suite": "file_size.yaml", + "image_type": "cpu" + } + ] + } + ] diff --git a/tests/benchmark/scheduler/filter.json b/tests/benchmark/scheduler/filter.json new file mode 100644 index 0000000000..2a0baed660 --- /dev/null +++ b/tests/benchmark/scheduler/filter.json @@ -0,0 +1,11 @@ +[ + { + "server": "poseidon", + "suite_params": [ + { + "suite": "011_search_dsl.yaml", + "image_type": "gpu" + } + ] + } +] diff --git a/tests/benchmark/scheduler/idc.json b/tests/benchmark/scheduler/idc.json new file mode 100644 index 0000000000..598be6da15 --- /dev/null +++ b/tests/benchmark/scheduler/idc.json @@ -0,0 +1,11 @@ +[ + { + "server": "idc-sh004", + "suite_params": [ + { + "suite": "011_cpu_search_debug.yaml", + "image_type": "gpu" + } + ] + } +] diff --git a/tests/benchmark/scheduler/insert.json b/tests/benchmark/scheduler/insert.json new file mode 100644 index 0000000000..4df97657ab --- /dev/null +++ b/tests/benchmark/scheduler/insert.json @@ -0,0 +1,11 @@ +[ + { + "server": "idc-sh002", + "suite_params": [ + { + "suite": "011_insert_data.yaml", + "image_type": "cpu" + } + ] + } +] diff --git a/tests/benchmark/scheduler/jaccard.json b/tests/benchmark/scheduler/jaccard.json new file mode 100644 index 0000000000..5d5ebd6d13 --- /dev/null +++ b/tests/benchmark/scheduler/jaccard.json @@ -0,0 +1,11 @@ +[ + { + "server": "athena", + "suite_params": [ + { + "suite": "011_cpu_search_binary.yaml", + "image_type": "cpu" + } + ] + } +] diff --git a/tests/benchmark/scheduler/locust.json b/tests/benchmark/scheduler/locust.json new file mode 100644 index 0000000000..b82ccf65b0 --- /dev/null +++ b/tests/benchmark/scheduler/locust.json @@ -0,0 +1,11 @@ +[ + { + "server": "idc-sh002", + "suite_params": [ + { + "suite": "locust_cluster_search.yaml", + "image_type": "cpu" + } + ] + } +] diff --git a/tests/benchmark/scheduler/locust_mix_debug.json b/tests/benchmark/scheduler/locust_mix_debug.json new file mode 100644 index 0000000000..343d429338 --- /dev/null +++ b/tests/benchmark/scheduler/locust_mix_debug.json @@ -0,0 +1,10 @@ +[ + { + "suite_params": [ + { + "suite": "locust_mix.yaml", + "image_type": "cpu" + } + ] + } +] diff --git a/tests/benchmark/scheduler/loop.json b/tests/benchmark/scheduler/loop.json new file mode 100644 index 0000000000..02fc1e261d --- /dev/null +++ b/tests/benchmark/scheduler/loop.json @@ -0,0 +1,10 @@ +[ + { + "suite_params": [ + { + "suite": "loop_stability.yaml", + "image_type": "gpu" + } + ] + } +] diff --git a/tests/benchmark/scheduler/search.json b/tests/benchmark/scheduler/search.json new file mode 100644 index 0000000000..68dc7776ab --- /dev/null +++ b/tests/benchmark/scheduler/search.json @@ -0,0 +1,11 @@ +[ + { + "server": "athena", + "suite_params": [ + { + "suite": "011_cpu_search_sift1b.yaml", + "image_type": "cpu" + } + ] + } +] diff --git a/tests/benchmark/scheduler/shards.json b/tests/benchmark/scheduler/shards.json new file mode 100644 index 0000000000..2a9bb4352a --- /dev/null +++ b/tests/benchmark/scheduler/shards.json @@ -0,0 +1,18 @@ +[ + { + "suite_params": [ + { + "suite": "shards_insert_performance.yaml", + "image_type": "cpu" + }, + { + "suite": "shards_ann_debug.yaml", + "image_type": "cpu" + }, + { + "suite": "shards_loop_stability.yaml", + "image_type": "cpu" + } + ] + } +] diff --git a/tests/benchmark/scheduler/shards_ann.json b/tests/benchmark/scheduler/shards_ann.json new file mode 100644 index 0000000000..6282c234cc --- /dev/null +++ b/tests/benchmark/scheduler/shards_ann.json @@ -0,0 +1,10 @@ +[ + { + "suite_params": [ + { + "suite": "shards_ann_debug.yaml", + "image_type": "cpu" + } + ] + } +] diff --git a/tests/benchmark/scheduler/shards_debug.json b/tests/benchmark/scheduler/shards_debug.json new file mode 100644 index 0000000000..9e79ee7758 --- /dev/null +++ b/tests/benchmark/scheduler/shards_debug.json @@ -0,0 +1,15 @@ +[ + { + "server": "apollo", + "suite_params": [ + { + "suite": "shards_insert_performance_sift1m.yaml", + "image_type": "cpu" + }, + { + "suite": "shards_search_performance_sift1m.yaml", + "image_type": "cpu" + } + ] + } +] diff --git a/tests/benchmark/scheduler/shards_stability.json b/tests/benchmark/scheduler/shards_stability.json new file mode 100644 index 0000000000..c2b71d9472 --- /dev/null +++ b/tests/benchmark/scheduler/shards_stability.json @@ -0,0 +1,10 @@ +[ + { + "suite_params": [ + { + "suite": "shards_loop_stability.yaml", + "image_type": "cpu" + } + ] + } +] diff --git a/tests/benchmark/scheduler/stability.json b/tests/benchmark/scheduler/stability.json new file mode 100644 index 0000000000..497a161200 --- /dev/null +++ b/tests/benchmark/scheduler/stability.json @@ -0,0 +1,11 @@ +[ + { + "server": "eros", + "suite_params": [ + { + "suite": "gpu_search_stability.yaml", + "image_type": "gpu" + } + ] + } +] diff --git a/tests/benchmark/search_task.py b/tests/benchmark/search_task.py new file mode 100644 index 0000000000..fc40907c78 --- /dev/null +++ b/tests/benchmark/search_task.py @@ -0,0 +1,50 @@ +import random +import logging +from locust import User, task, between +from locust_task import MilvusTask +from client import MilvusClient +from milvus import DataType +import utils + +connection_type = "single" +host = "172.16.50.9" +port = 19530 +collection_name = "sift_5m_2000000_128_l2_2" +dim = 128 +m = MilvusClient(host=host, port=port, collection_name=collection_name) +# m.clean_db() +# m.create_collection(dim, data_type=DataType.FLOAT_VECTOR, auto_id=True, other_fields=None) +vectors = [[random.random() for _ in range(dim)] for _ in range(1000)] +entities = m.generate_entities(vectors) +ids = [i for i in range(10000000)] + + +class QueryTask(User): + wait_time = between(0.001, 0.002) + # if connection_type == "single": + # client = MilvusTask(m=m) + # else: + # client = MilvusTask(host=host, port=port, collection_name=collection_name) + client = MilvusTask(host, port, collection_name, connection_type=connection_type) + + # @task + # def query(self): + # top_k = 5 + # X = [[random.random() for i in range(dim)] for i in range(1)] + # search_param = {"nprobe": 16} + # self.client.query(X, top_k, search_param) + + @task(1) + def insert(self): + self.client.insert(entities) + + # @task(1) + # def create(self): + # collection_name = utils.get_unique_name(prefix="locust") + # self.client.create_collection(dim, data_type=DataType.FLOAT_VECTOR, auto_id=True, collection_name=collection_name, other_fields=None) + + # @task(1) + # def delete(self): + # delete_ids = random.sample(ids, 100) + # logging.error(delete_ids) + # self.client.delete(delete_ids) diff --git a/tests/benchmark/suites/011_add_flush_performance.yaml b/tests/benchmark/suites/011_add_flush_performance.yaml new file mode 100644 index 0000000000..b8a7090d9b --- /dev/null +++ b/tests/benchmark/suites/011_add_flush_performance.yaml @@ -0,0 +1,20 @@ +insert_flush_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_2m_128_128_l2_flush + cache_config.cpu_cache_capacity: 8 + cache_config.insert_buffer_size: 2 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + db_config.auto_flush_interval: 300 + collection_name: sift_2m_128_l2 + ni_per: 100000 diff --git a/tests/benchmark/suites/011_cluster_cpu_accuracy_ann.yaml b/tests/benchmark/suites/011_cluster_cpu_accuracy_ann.yaml new file mode 100644 index 0000000000..63f26c1307 --- /dev/null +++ b/tests/benchmark/suites/011_cluster_cpu_accuracy_ann.yaml @@ -0,0 +1,336 @@ +ann_accuracy: + collections: + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/cluster/sift_128_euclidean + suffix_path: true + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + cluster: true + readonly: + replicas: 2 + external_mysql: true + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_types: ['flat'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 512, 16384] + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/cluster/sift_128_euclidean + suffix_path: true + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + cluster: true + readonly: + replicas: 2 + external_mysql: true + + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_types: ['ivf_flat', 'ivf_sq8'] + index_params: + nlist: [1024, 16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/cluster/sift_128_euclidean + suffix_path: true + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + cluster: true + readonly: + replicas: 2 + external_mysql: true + + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_types: ['ivf_pq'] + index_params: + nlist: [16384] + m: [32] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/cluster/sift_128_euclidean + suffix_path: true + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + cluster: true + readonly: + replicas: 2 + external_mysql: true + + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_types: ['annoy'] + index_params: + n_trees: [8, 32] + top_ks: [10] + nqs: [10000] + search_params: + search_k: [50, 100, 500, 1000] + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/cluster/sift_128_euclidean + suffix_path: true + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + cluster: true + readonly: + replicas: 2 + external_mysql: true + + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_types: ['hnsw'] + index_params: + M: [16] + efConstruction: [500] + top_ks: [10] + nqs: [10000] + search_params: + ef: [16, 32, 64, 128, 256, 512] + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/cluster/glove_200_angular + suffix_path: true + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + cluster: true + readonly: + replicas: 2 + external_mysql: true + + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + index_types: ['flat'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 512, 16384] + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/cluster/glove_200_angular + suffix_path: true + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + cluster: true + readonly: + replicas: 2 + external_mysql: true + + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + index_types: ['ivf_flat', 'ivf_sq8'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/cluster/glove_200_angular + suffix_path: true + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + cluster: true + readonly: + replicas: 2 + external_mysql: true + + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + index_types: ['ivf_pq'] + index_params: + nlist: [16384] + m: [20] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - milvus: + db_config.primary_path: /test/milvus/db_data_011/cluster/glove_200_angular + suffix_path: true + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + cluster: true + readonly: + replicas: 2 + external_mysql: true + + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + index_types: ['annoy'] + index_params: + n_trees: [8, 32] + top_ks: [10] + nqs: [10000] + search_params: + search_k: [50, 100, 500, 1000] + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/cluster/glove_200_angular + suffix_path: true + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + cluster: true + readonly: + replicas: 2 + external_mysql: true + + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + index_types: ['hnsw'] + index_params: + M: [36] + efConstruction: [500] + top_ks: [10] + nqs: [10000] + search_params: + ef: [10, 16, 32, 64, 128, 256, 512] + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/cluster/glove_200_angular + suffix_path: true + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + cluster: true + readonly: + replicas: 2 + external_mysql: true + + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + index_types: ['nsg'] + index_params: + search_length: 45 + out_degree: 50 + candidate_pool_size: 300 + knng: 100 + top_ks: [10] + nqs: [10000] + search_params: + search_length: [50] diff --git a/tests/benchmark/suites/011_cpu_accuracy.yaml b/tests/benchmark/suites/011_cpu_accuracy.yaml new file mode 100644 index 0000000000..b254849708 --- /dev/null +++ b/tests/benchmark/suites/011_cpu_accuracy.yaml @@ -0,0 +1,55 @@ +accuracy: + collections: + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_128_l2_sq8 + cache_config.cpu_cache_capacity: 8GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: sift_10m_128_l2 + top_ks: [64] + nqs: [1000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_128_l2_sq8 + cache_config.cpu_cache_capacity: 8GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: sift_10m_128_l2 + top_ks: [64] + nqs: [1000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_128_l2_hnsw + cache_config.cpu_cache_capacity: 8GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: sift_10m_128_l2 + top_ks: [64] + nqs: [1000] + search_params: + ef: [64, 100, 200, 500, 700] diff --git a/tests/benchmark/suites/011_cpu_accuracy_ann.yaml b/tests/benchmark/suites/011_cpu_accuracy_ann.yaml new file mode 100644 index 0000000000..d7b736dd1f --- /dev/null +++ b/tests/benchmark/suites/011_cpu_accuracy_ann.yaml @@ -0,0 +1,260 @@ +ann_accuracy: + collections: + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_types: ['flat'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 512, 16384] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_types: ['ivf_flat', 'ivf_sq8'] + index_params: + nlist: [1024, 16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_types: ['ivf_pq'] + index_params: + nlist: [16384] + m: [32] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_types: ['annoy'] + index_params: + n_trees: [8, 32] + top_ks: [10] + nqs: [10000] + search_params: + search_k: [50, 100, 500, 1000] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_types: ['hnsw'] + index_params: + M: [16] + efConstruction: [500] + top_ks: [10] + nqs: [10000] + search_params: + ef: [16, 32, 64, 128, 256, 512] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + index_types: ['flat'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 512, 16384] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + index_types: ['ivf_flat', 'ivf_sq8'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + index_types: ['ivf_pq'] + index_params: + nlist: [16384] + m: [20] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + index_types: ['annoy'] + index_params: + n_trees: [8, 32] + top_ks: [10] + nqs: [10000] + search_params: + search_k: [50, 100, 500, 1000] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + index_types: ['hnsw'] + index_params: + M: [36] + efConstruction: [500] + top_ks: [10] + nqs: [10000] + search_params: + ef: [10, 16, 32, 64, 128, 256, 512] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + index_types: ['nsg'] + index_params: + search_length: 45 + out_degree: 50 + candidate_pool_size: 300 + knng: 100 + top_ks: [10] + nqs: [10000] + search_params: + search_length: [50] diff --git a/tests/benchmark/suites/011_cpu_build.yaml b/tests/benchmark/suites/011_cpu_build.yaml new file mode 100644 index 0000000000..17376451d7 --- /dev/null +++ b/tests/benchmark/suites/011_cpu_build.yaml @@ -0,0 +1,40 @@ +build_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/random_1m_1024_512_l2_ivf + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: random_1m_1024_512_l2 + index_type: ivf_flat + index_param: + nlist: 16384 + + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_1m_128_128_l2_pq + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_1m_128_128_l2 + index_type: ivf_pq + index_param: + nlist: 8092 + m: 32 diff --git a/tests/benchmark/suites/011_cpu_build_binary.yaml b/tests/benchmark/suites/011_cpu_build_binary.yaml new file mode 100644 index 0000000000..d3beb3bd5e --- /dev/null +++ b/tests/benchmark/suites/011_cpu_build_binary.yaml @@ -0,0 +1,11 @@ +build_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/binary_50m_512_jaccard_ivf + cache_config.cpu_cache_capacity: 8GB + gpu_resource_config.enable: false + collection_name: binary_50m_512_jaccard + index_type: bin_ivf_flat + index_param: + nlist: 2048 diff --git a/tests/benchmark/suites/011_cpu_search.yaml b/tests/benchmark/suites/011_cpu_search.yaml new file mode 100644 index 0000000000..e576b913e3 --- /dev/null +++ b/tests/benchmark/suites/011_cpu_search.yaml @@ -0,0 +1,255 @@ +search_performance: + collections: + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_1b_128_l2_sq8 + cache_config.cpu_cache_capacity: 150GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_1b_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + nprobe: 8 + - + nprobe: 32 + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_128_l2 + cache_config.cpu_cache_capacity: 64GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + nprobe: 8 + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_128_l2 + cache_config.cpu_cache_capacity: 64GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + nprobe: 8 + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_128_l2_ivf_flat + cache_config.cpu_cache_capacity: 64GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + nprobe: 8 + - + nprobe: 32 + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_128_l2_sq8 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + nprobe: 8 + - + nprobe: 32 + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_128_l2_pq + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + wal_enable: true + collection_name: sift_10m_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + nprobe: 8 + - + nprobe: 32 + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_128_l2_ivf_flat + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + wal_enable: true + collection_name: sift_10m_128_l2 + run_count: 2 + top_ks: [1, 1000] + nqs: [1, 100, 1000] + search_params: + - + nprobe: 8 + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_nsg + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + wal_enable: true + collection_name: sift_10m_100000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + search_length: 50 + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_annoy + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + wal_enable: true + collection_name: sift_10m_100000_128_l2 + run_count: 2 + top_ks: [1, 10, 100] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + search_k: 100 + - + search_k: 500 + - + search_k: 1000 + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/binary_50m_512_jaccard + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: binary_50m_512_jaccard + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/binary_50m_512_jaccard_ivf + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: binary_50m_512_jaccard + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 diff --git a/tests/benchmark/suites/011_cpu_search_binary.yaml b/tests/benchmark/suites/011_cpu_search_binary.yaml new file mode 100644 index 0000000000..6539999b4a --- /dev/null +++ b/tests/benchmark/suites/011_cpu_search_binary.yaml @@ -0,0 +1,49 @@ +search_performance: + collections: + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/binary_50m_512_jaccard + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: binary_50m_512_jaccard + run_count: 2 + top_ks: [10, 1, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/binary_50m_512_jaccard_ivf + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: binary_50m_512_jaccard + run_count: 2 + top_ks: [10, 1, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 diff --git a/tests/benchmark/suites/011_cpu_search_debug.yaml b/tests/benchmark/suites/011_cpu_search_debug.yaml new file mode 100644 index 0000000000..eca76ee3e4 --- /dev/null +++ b/tests/benchmark/suites/011_cpu_search_debug.yaml @@ -0,0 +1,26 @@ +search_performance: + collections: + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_1b_128_l2_sq8 + cache_config.cpu_cache_capacity: 150GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_1b_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + nprobe: 8 + - + nprobe: 32 diff --git a/tests/benchmark/suites/011_cpu_search_sift10m_filter.yaml b/tests/benchmark/suites/011_cpu_search_sift10m_filter.yaml new file mode 100644 index 0000000000..284aff9fdf --- /dev/null +++ b/tests/benchmark/suites/011_cpu_search_sift10m_filter.yaml @@ -0,0 +1,97 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_128_l2/ + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_128_l2 + run_count: 2 + top_ks: [1, 1000] + nqs: [1, 100, 1200] + search_params: + - + nprobe: 8 + - + server: + db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_128_l2_ivf_flat + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + + - + server: + db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_128_l2_sq8 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + - + server: + db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_128_l2_pq + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 \ No newline at end of file diff --git a/tests/benchmark/suites/011_cpu_search_sift50m.yaml b/tests/benchmark/suites/011_cpu_search_sift50m.yaml new file mode 100644 index 0000000000..6a96e0e721 --- /dev/null +++ b/tests/benchmark/suites/011_cpu_search_sift50m.yaml @@ -0,0 +1,98 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_128_l2 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_128_l2 + run_count: 2 + top_ks: [1, 1000] + nqs: [1, 100, 1200] + search_params: + - + nprobe: 8 + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_128_l2_ivf_flat_16384 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_128_l2_sq8_16384 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_128_l2_pq_16384 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 \ No newline at end of file diff --git a/tests/benchmark/suites/011_gpu_accuracy.yaml b/tests/benchmark/suites/011_gpu_accuracy.yaml new file mode 100644 index 0000000000..abd4d30ca1 --- /dev/null +++ b/tests/benchmark/suites/011_gpu_accuracy.yaml @@ -0,0 +1,61 @@ +accuracy: + collections: + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_128_l2_sq8 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_10m_128_l2 + top_ks: [64] + nqs: [1000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_128_l2 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_10m_128_l2 + top_ks: [64] + nqs: [1000] + search_params: + nprobe: [1] + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_128_ip + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_10m_128_ip + top_ks: [64] + nqs: [1000] + search_params: + nprobe: [1] diff --git a/tests/benchmark/suites/011_gpu_accuracy_ann.yaml b/tests/benchmark/suites/011_gpu_accuracy_ann.yaml new file mode 100644 index 0000000000..fb2fad25a4 --- /dev/null +++ b/tests/benchmark/suites/011_gpu_accuracy_ann.yaml @@ -0,0 +1,165 @@ +ann_accuracy: + collections: + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_types: ['flat', 'ivf_flat', 'ivf_sq8', 'ivf_sq8h'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_types: ['ivf_pq'] + index_params: + nlist: [16384] + m: [32] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_types: ['hnsw'] + index_params: + M: [16] + efConstruction: [500] + top_ks: [10] + nqs: [10000] + search_params: + ef: [16, 32, 64, 128, 256, 512] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + index_types: ['flat', 'ivf_flat', 'ivf_sq8', 'ivf_sq8h'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + index_types: ['hnsw'] + index_params: + M: [36] + efConstruction: [500] + top_ks: [10] + nqs: [10000] + search_params: + ef: [10, 16, 32, 64, 128, 256, 512] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/kosarak-27983-jaccard.hdf5 + collection_name: kosarak_27984_jaccard + index_types: ['bin_flat', 'bin_ivf_flat'] + index_params: + nlist: [2048] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + milvus: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-256-hamming.hdf5 + collection_name: sift_256_hamming + index_types: ['bin_flat', 'bin_ivf_flat'] + index_params: + nlist: [2048] + top_ks: [100] + nqs: [1000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] diff --git a/tests/benchmark/suites/011_gpu_build.yaml b/tests/benchmark/suites/011_gpu_build.yaml new file mode 100644 index 0000000000..488dc16b67 --- /dev/null +++ b/tests/benchmark/suites/011_gpu_build.yaml @@ -0,0 +1,21 @@ +build_performance: + collections: + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_sq8_4096 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_10m_100000_128_l2 + index_type: ivf_sq8 + index_param: + nlist: 4096 + diff --git a/tests/benchmark/suites/011_gpu_search.yaml b/tests/benchmark/suites/011_gpu_search.yaml new file mode 100644 index 0000000000..e717c6e325 --- /dev/null +++ b/tests/benchmark/suites/011_gpu_search.yaml @@ -0,0 +1,251 @@ +search_performance: + collections: + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_128_l2 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_128_l2_ivf_flat + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_128_l2_sq8 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_128_l2_sq8h + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_128_l2_pq + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_128_l2_hnsw + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_128_l2 + run_count: 2 + top_ks: [100] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + ef: 100 + - + ef: 200 + - + ef: 500 + - + ef: 1000 + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_annoy + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_100000_128_l2 + run_count: 2 + top_ks: [1, 10, 100] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + search_k: 100 + - + search_k: 500 + - + search_k: 1000 + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_100000_128_l2_nsg + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_100000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + search_length: 50 + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_1b_128_l2_sq8h + cache_config.cpu_cache_capacity: 150GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_1b_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_1b_128_l2_sq8 + cache_config.cpu_cache_capacity: 150GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_1b_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 diff --git a/tests/benchmark/suites/011_gpu_search_sift10m_filter.yaml b/tests/benchmark/suites/011_gpu_search_sift10m_filter.yaml new file mode 100644 index 0000000000..012153d983 --- /dev/null +++ b/tests/benchmark/suites/011_gpu_search_sift10m_filter.yaml @@ -0,0 +1,122 @@ +search_performance: + collections: +# - +# server: +# db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_128_l2/ +# cache_config.cpu_cache_capacity: 32GB +# engine_config.use_blas_threshold: 0 +# engine_config.gpu_search_threshold: 100 +# gpu_resource_config.enable: true +# gpu_resource_config.cache_capacity: 6GB +# gpu_resource_config.search_resources: +# - gpu0 +# - gpu1 +# gpu_resource_config.build_index_resources: +# - gpu0 +# - gpu1 +# wal_enable: true +# collection_name: sift_10m_128_l2 +# run_count: 2 +# top_ks: [1, 1000] +# nqs: [1, 100, 1200] +# search_params: +# - +# nprobe: 8 +# - +# server: +# db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_128_l2_ivf_flat +# cache_config.cpu_cache_capacity: 32GB +# engine_config.use_blas_threshold: 0 +# engine_config.gpu_search_threshold: 100 +# gpu_resource_config.enable: true +# gpu_resource_config.cache_capacity: 6GB +# gpu_resource_config.search_resources: +# - gpu0 +# - gpu1 +# gpu_resource_config.build_index_resources: +# - gpu0 +# - gpu1 +# wal_enable: true +# collection_name: sift_10m_128_l2 +# run_count: 2 +# top_ks: [1, 10, 100, 1000] +# nqs: [1, 10, 100, 1000, 1200] +# search_params: +# - +# nprobe: 8 +# - +# nprobe: 32 + +# - +# server: +# db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_128_l2_sq8 +# cache_config.cpu_cache_capacity: 32GB +# engine_config.use_blas_threshold: 0 +# engine_config.gpu_search_threshold: 100 +# gpu_resource_config.enable: true +# gpu_resource_config.cache_capacity: 6GB +# gpu_resource_config.search_resources: +# - gpu0 +# - gpu1 +# gpu_resource_config.build_index_resources: +# - gpu0 +# - gpu1 +# wal_enable: true +# collection_name: sift_10m_128_l2 +# run_count: 2 +# top_ks: [1, 10, 100, 1000] +# nqs: [1, 10, 100, 1000, 1200] +# search_params: +# - +# nprobe: 8 +# - +# nprobe: 32 + - + server: + db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_128_l2_sq8h + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_10m_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + filters: + - + term: "{'term': {'float': {'values': [float(i) for i in range(collection_size // 1000)]}}}" + search_params: + - + nprobe: 32 +# - +# server: +# db_config.primary_path: /test/milvus/db_data_011/filter/sift_10m_128_l2_pq +# cache_config.cpu_cache_capacity: 32GB +# engine_config.use_blas_threshold: 0 +# engine_config.gpu_search_threshold: 100 +# gpu_resource_config.enable: true +# gpu_resource_config.cache_capacity: 6GB +# gpu_resource_config.search_resources: +# - gpu0 +# - gpu1 +# gpu_resource_config.build_index_resources: +# - gpu0 +# - gpu1 +# wal_enable: true +# collection_name: sift_10m_128_l2 +# run_count: 2 +# top_ks: [1, 10, 100, 1000] +# nqs: [1, 10, 100, 1000, 1200] +# search_params: +# - +# nprobe: 8 +# - +# nprobe: 32 \ No newline at end of file diff --git a/tests/benchmark/suites/011_gpu_search_sift50m.yaml b/tests/benchmark/suites/011_gpu_search_sift50m.yaml new file mode 100644 index 0000000000..7aae432ca2 --- /dev/null +++ b/tests/benchmark/suites/011_gpu_search_sift50m.yaml @@ -0,0 +1,121 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_100000_128_l2 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_100000_128_l2 + run_count: 2 + top_ks: [1, 1000] + nqs: [1, 100, 1200] + search_params: + - + nprobe: 8 + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_100000_128_l2_ivf_flat + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_100000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_100000_128_l2_sq8 + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_100000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_100000_128_l2_sq8h + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_100000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_100000_128_l2_pq + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_50m_100000_128_l2 + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 1000, 1200] + search_params: + - + nprobe: 8 + - + nprobe: 32 \ No newline at end of file diff --git a/tests/benchmark/suites/011_gpu_stability.yaml b/tests/benchmark/suites/011_gpu_stability.yaml new file mode 100644 index 0000000000..9e7ec4f006 --- /dev/null +++ b/tests/benchmark/suites/011_gpu_stability.yaml @@ -0,0 +1,39 @@ +stability: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_011/sift_10m_1024_128_l2_sq8_stability + cache_config.cpu_cache_capacity: 64GB + cache_config.cache_insert_data: true + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 50 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_1m_50000_128_l2_2 + during_time: 5 + operations: + insert: + weight: 4 + xb: 100 + delete: + weight: 4 + xb: 100 + flush: + weight: 1 + # async: true + compact: + weight: 1 + # # async: true + query: + weight: 2 + # async: true + top_ks: 1-100 + nqs: 1-100 + search_params: + nprobe: 1-100 diff --git a/tests/benchmark/suites/011_insert_data.yaml b/tests/benchmark/suites/011_insert_data.yaml new file mode 100644 index 0000000000..f36fe0a875 --- /dev/null +++ b/tests/benchmark/suites/011_insert_data.yaml @@ -0,0 +1,57 @@ +insert_performance: + collections: + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/cluster/sift_10m_128_l2 + cache_config.cpu_cache_capacity: 4GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true +# cluster: true +# external_mysql: true + collection_name: sift_10m_128_l2_011 +# other_fields: int,float + ni_per: 50000 + build_index: false + index_type: ivf_sq8 + index_param: + nlist: 1024 + # - + # server: + # db_config.primary_path: /test/milvus/db_data_011/sift_50m_100000_128_l2 + # cache_config.cpu_cache_capacity: 4GB + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 1 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4GB + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # wal_enable: true + # collection_name: sift_50m_100000_128_l2 + # ni_per: 50000 + # build_index: false + # index_type: ivf_sq8 + # index_param: + # nlist: 1024 +# - +# server: +# db_config.primary_path: /test/milvus/db_data_011/sift_1b_524288_128_l2_debug +# collection_name: sift_1b_524288_128_l2 +# ni_per: 100000 +# # flush: no +# build_index: false +# index_type: ivf_sq8 +# index_param: +# nlist: 4096 diff --git a/tests/benchmark/suites/011_insert_performance.yaml b/tests/benchmark/suites/011_insert_performance.yaml new file mode 100644 index 0000000000..0bd3643408 --- /dev/null +++ b/tests/benchmark/suites/011_insert_performance.yaml @@ -0,0 +1,113 @@ +insert_performance: + collections: + - + milvus: + cache_config.cpu_cache_capacity: 8GB + engine_config.use_blas_threshold: 1100 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + wal_enable: true + collection_name: sift_5m_128_l2 + ni_per: 50000 + build_index: false + index_type: ivf_sq8 + index_param: + nlist: 1024 + + - + milvus: + cache_config.cpu_cache_capacity: 8GB + engine_config.use_blas_threshold: 1100 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + wal_enable: true + collection_name: sift_5m_128_l2 + ni_per: 50000 + build_index: true + index_type: ivf_sq8 + index_param: + nlist: 1024 + + # - + # server: + # cache_config.cpu_cache_capacity: 8GB + # engine_config.use_blas_threshold: 1100 + # gpu_resource_config.enable: false + # gpu_resource_config.cache_capacity: 4GB + # gpu_resource_config.search_resources: + # - gpu0 + # gpu_resource_config.build_index_resources: + # - gpu0 + # wal_enable: true + # collection_name: sift_5m_100000_128_l2 + # ni_per: 50000 + # flush: no + # build_index: false + # index_type: ivf_sq8 + # index_param: + # nlist: 1024 + + # - + # server: + # cache_config.cpu_cache_capacity: 8GB + # engine_config.use_blas_threshold: 1100 + # gpu_resource_config.enable: false + # gpu_resource_config.cache_capacity: 4GB + # gpu_resource_config.search_resources: + # - gpu0 + # gpu_resource_config.build_index_resources: + # - gpu0 + # wal_enable: true + # collection_name: sift_5m_100000_128_l2 + # ni_per: 100000 + # flush: no + # build_index: false + # index_type: ivf_sq8 + # index_param: + # nlist: 1024 + + # - + # server: + # cache_config.cpu_cache_capacity: 8GB + # engine_config.use_blas_threshold: 1100 + # gpu_resource_config.enable: false + # gpu_resource_config.cache_capacity: 4GB + # gpu_resource_config.search_resources: + # - gpu0 + # gpu_resource_config.build_index_resources: + # - gpu0 + # wal_enable: true + # collection_name: sift_5m_100000_128_l2 + # ni_per: 200000 + # flush: no + # build_index: false + # index_type: ivf_sq8 + # index_param: + # nlist: 1024 + + # - + # server: + # cache_config.cpu_cache_capacity: 8GB + # engine_config.use_blas_threshold: 1100 + # gpu_resource_config.enable: false + # gpu_resource_config.cache_capacity: 4GB + # gpu_resource_config.search_resources: + # - gpu0 + # gpu_resource_config.build_index_resources: + # - gpu0 + # wal_enable: true + # collection_name: sift_5m_100000_128_l2 + # ni_per: 500000 + # flush: no + # build_index: false + # index_type: ivf_sq8 + # index_param: + # nlist: 1024 diff --git a/tests/benchmark/suites/011_locust_insert.yaml b/tests/benchmark/suites/011_locust_insert.yaml new file mode 100644 index 0000000000..1d4b799978 --- /dev/null +++ b/tests/benchmark/suites/011_locust_insert.yaml @@ -0,0 +1,33 @@ +locust_insert_performance: + collections: + - + milvus: + mysql: true + db_config.primary_path: /test/milvus/db_data_011/insert_sift_1m_128_l2_2 + cache_config.cpu_cache_capacity: 8GB + cache_config.insert_buffer_size: 2GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + wal_enable: true + collection_name: sift_1m_128_l2_011 + ni_per: 50000 + build_index: false + index_type: ivf_sq8 + index_param: + nlist: 1024 + task: + connection_num: 1 + clients_num: 10 + hatch_rate: 2 + during_time: 30 + types: + - + type: insert + weight: 1 + params: None diff --git a/tests/benchmark/suites/011_locust_search.yaml b/tests/benchmark/suites/011_locust_search.yaml new file mode 100644 index 0000000000..fa84559975 --- /dev/null +++ b/tests/benchmark/suites/011_locust_search.yaml @@ -0,0 +1,43 @@ +locust_search_performance: + collections: + - + milvus: + cache_config.cpu_cache_capacity: 8GB + cache_config.insert_buffer_size: 2GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_1m_128_l2_2 + ni_per: 50000 + build_index: true + index_type: ivf_sq8 + index_param: + nlist: 16384 + task: + connection_num: 1 + clients_num: 100 + hatch_rate: 2 + during_time: 300 + types: + - + type: query + weight: 1 + params: + top_k: 10 + nq: 1 + # filters: + # - + # range: + # int64: + # LT: 0 + # GT: 1000000 + search_param: + nprobe: 16 diff --git a/tests/benchmark/suites/011_search_stability.yaml b/tests/benchmark/suites/011_search_stability.yaml new file mode 100644 index 0000000000..840b9378eb --- /dev/null +++ b/tests/benchmark/suites/011_search_stability.yaml @@ -0,0 +1,20 @@ +search_stability: + collections: + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/sift_50m_128_l2_sq8_16384_stability + cache_config.cpu_cache_capacity: 32GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: sift_50m_128_l2 + during_time: 180 + top_ks: 1-200 + nqs: 1-200 + search_params: + nprobe: 1-100 diff --git a/tests/benchmark/suites/cluster_locust_mix.yaml b/tests/benchmark/suites/cluster_locust_mix.yaml new file mode 100644 index 0000000000..cb1b457644 --- /dev/null +++ b/tests/benchmark/suites/cluster_locust_mix.yaml @@ -0,0 +1,47 @@ +locust_mix_performance: + collections: + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/cluster/locust_mix + suffix_path: true + cache_config.insert_buffer_size: 2GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + gpu0 + gpu_resource_config.build_index_resources: + gpu0 + wal_enable: true + external_mysql: true + cluster: true + readonly: + replicas: 2 + collection_name: sift_1m_500000_128_l2 + ni_per: 50000 + build_index: true + index_type: ivf_sq8 + index_param: + nlist: 1024 + task: + types: + - type: flush + weight: 1 + - + type: query + weight: 30 + params: + top_k: 10 + nq: 100 + search_param: + nprobe: 16 + - + type: insert + weight: 10 + params: + nb: 1 + connection_num: 1 + clients_num: 10 + hatch_rate: 2 + during_time: 600 diff --git a/tests/benchmark/suites/cpu_accuracy.yaml b/tests/benchmark/suites/cpu_accuracy.yaml new file mode 100644 index 0000000000..d780178089 --- /dev/null +++ b/tests/benchmark/suites/cpu_accuracy.yaml @@ -0,0 +1,61 @@ +accuracy: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_7/sift_50m_2048_128_ip_sq8_wal + cache_config.cpu_cache_capacity: 30 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_50m_2048_128_ip + top_ks: [64] + nqs: [1000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + server: + db_config.primary_path: /test/milvus/db_data_7/sift_50m_2048_128_l2_hnsw_wal + cache_config.cpu_cache_capacity: 64 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_50m_2048_128_l2 + top_ks: [64] + nqs: [1000] + search_params: + ef: [64, 100, 200, 500, 700] + + - + server: + db_config.primary_path: /test/milvus/db_data_7/sift_1b_2048_128_l2_sq8_wal + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 6 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_1b_2048_128_l2 + top_ks: [64] + nqs: [1000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] diff --git a/tests/benchmark/suites/cpu_accuracy_ann.yaml b/tests/benchmark/suites/cpu_accuracy_ann.yaml new file mode 100644 index 0000000000..b980186ae0 --- /dev/null +++ b/tests/benchmark/suites/cpu_accuracy_ann.yaml @@ -0,0 +1,212 @@ +ann_accuracy: + collections: + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_types: ['flat'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 512, 16384] + + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_types: ['ivf_flat', 'ivf_sq8'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_types: ['ivf_pq'] + index_params: + nlist: [16384] + m: [32] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_types: ['annoy'] + index_params: + n_trees: [8, 32] + top_ks: [10] + nqs: [10000] + search_params: + search_k: [50, 100, 500, 1000] + + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_types: ['hnsw'] + index_params: + M: [16] + efConstruction: [500] + top_ks: [10] + nqs: [10000] + search_params: + ef: [16, 32, 64, 128, 256, 512] + + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + index_types: ['flat'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 512, 16384] + + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + index_types: ['ivf_flat', 'ivf_sq8'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + index_types: ['ivf_pq'] + index_params: + nlist: [16384] + m: [20] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + index_types: ['hnsw'] + index_params: + M: [36] + efConstruction: [500] + top_ks: [10] + nqs: [10000] + search_params: + ef: [10, 16, 32, 64, 128, 256, 512] diff --git a/tests/benchmark/suites/cpu_build_performance.yaml b/tests/benchmark/suites/cpu_build_performance.yaml new file mode 100644 index 0000000000..660aff14f5 --- /dev/null +++ b/tests/benchmark/suites/cpu_build_performance.yaml @@ -0,0 +1,19 @@ +build_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_070/sift_50m_1024_128_l2_sq8h_wal + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_50m_1024_128_l2 + index_type: ivf_sq8h + nlist: 16384 diff --git a/tests/benchmark/suites/cpu_search_binary.yaml b/tests/benchmark/suites/cpu_search_binary.yaml new file mode 100644 index 0000000000..37aa2d0a35 --- /dev/null +++ b/tests/benchmark/suites/cpu_search_binary.yaml @@ -0,0 +1,67 @@ +search_performance: + collections: + # - + # server: + # db_config.primary_path: /test/milvus/db_data_7/sub_50m_512_512_sub_wal + # cache_config.cpu_cache_capacity: 32 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 1 + # gpu_resource_config.enable: false + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # gpu_resource_config.build_index_resources: + # - gpu0 + # collection_name: sub_50m_512_512_sub + # run_count: 2 + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + # search_params: + # - + # nprobe: 8 + # - + # nprobe: 32 + + # - + # server: + # db_config.primary_path: /test/milvus/db_data_7/super_50m_512_512_super_wal + # cache_config.cpu_cache_capacity: 32 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 1 + # gpu_resource_config.enable: false + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # gpu_resource_config.build_index_resources: + # - gpu0 + # collection_name: super_50m_512_512_super + # run_count: 2 + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + # search_params: + # - + # nprobe: 8 + # - + # nprobe: 32 + + - + server: + db_config.primary_path: /test/milvus/db_data_7/jaccard_50m_512_512_jaccard_wal + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 0 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: jaccard_50m_512_512_jaccard + run_count: 2 + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + search_params: + - + nprobe: 8 + - + nprobe: 32 diff --git a/tests/benchmark/suites/cpu_search_performance_jaccard.yaml b/tests/benchmark/suites/cpu_search_performance_jaccard.yaml new file mode 100644 index 0000000000..0266cababe --- /dev/null +++ b/tests/benchmark/suites/cpu_search_performance_jaccard.yaml @@ -0,0 +1,20 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_070/jaccard_50m_512_512_jaccard_wal + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: jaccard_50m_512_512_jaccard + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] diff --git a/tests/benchmark/suites/cpu_search_performance_sift50m.yaml b/tests/benchmark/suites/cpu_search_performance_sift50m.yaml new file mode 100644 index 0000000000..b74bb9e56e --- /dev/null +++ b/tests/benchmark/suites/cpu_search_performance_sift50m.yaml @@ -0,0 +1,20 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8 + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: sift_50m_1024_128_l2 + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] diff --git a/tests/benchmark/suites/gpu_accuracy.yaml b/tests/benchmark/suites/gpu_accuracy.yaml new file mode 100644 index 0000000000..2f5a086210 --- /dev/null +++ b/tests/benchmark/suites/gpu_accuracy.yaml @@ -0,0 +1,41 @@ +accuracy: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_7/sift_50m_2048_128_l2_sq8_wal + cache_config.cpu_cache_capacity: 30 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_50m_2048_128_l2 + top_ks: [64] + nqs: [1000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + server: + db_config.primary_path: /test/milvus/db_data_7/sift_1b_2048_128_l2_sq8h_wal + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_1b_2048_128_l2 + top_ks: [64] + nqs: [1000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] diff --git a/tests/benchmark/suites/gpu_accuracy_ann.yaml b/tests/benchmark/suites/gpu_accuracy_ann.yaml new file mode 100644 index 0000000000..900abf73ed --- /dev/null +++ b/tests/benchmark/suites/gpu_accuracy_ann.yaml @@ -0,0 +1,172 @@ +ann_accuracy: + collections: + - + server: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_file_sizes: [1024] + index_types: ['flat', 'ivf_flat', 'ivf_sq8', 'ivf_sq8h'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 16384] + + - + server: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_file_sizes: [1024] + index_types: ['ivf_pq'] + index_params: + nlist: [16384] + m: [32] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + server: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_file_sizes: [256] + index_types: ['hnsw'] + index_params: + M: [16] + efConstruction: [500] + top_ks: [10] + nqs: [10000] + search_params: + ef: [16, 32, 64, 128, 256, 512] + + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + index_file_sizes: [1024] + index_types: ['flat', 'ivf_flat', 'ivf_sq8', 'ivf_sq8h'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 16384] + + - + server: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + index_file_sizes: [256] + index_types: ['hnsw'] + index_params: + M: [36] + efConstruction: [500] + top_ks: [10] + nqs: [10000] + search_params: + ef: [10, 16, 32, 64, 128, 256, 512] + + - + server: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/kosarak-27983-jaccard.hdf5 + collection_name: kosarak_27984_jaccard + index_file_sizes: [1024] + index_types: ['flat', 'ivf_flat'] + index_params: + nlist: [2048] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + server: + cache_config.cpu_cache_capacity: 16GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-256-hamming.hdf5 + collection_name: sift_256_hamming + index_file_sizes: [1024] + index_types: ['flat', 'ivf_flat'] + index_params: + nlist: [2048] + top_ks: [100] + nqs: [1000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] diff --git a/tests/benchmark/suites/gpu_accuracy_ann_debug.yaml b/tests/benchmark/suites/gpu_accuracy_ann_debug.yaml new file mode 100644 index 0000000000..4748845edb --- /dev/null +++ b/tests/benchmark/suites/gpu_accuracy_ann_debug.yaml @@ -0,0 +1,24 @@ +ann_accuracy: + collections: + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_types: ['flat'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 512, 16384] diff --git a/tests/benchmark/suites/gpu_accuracy_sift1b.yaml b/tests/benchmark/suites/gpu_accuracy_sift1b.yaml new file mode 100644 index 0000000000..dbfe2abe8c --- /dev/null +++ b/tests/benchmark/suites/gpu_accuracy_sift1b.yaml @@ -0,0 +1,59 @@ +accuracy: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8 + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_1b_2048_128_l2 + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [64] + nqs: [1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8h + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_1b_2048_128_l2 + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [64] + nqs: [1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_pq + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_1b_2048_128_l2 + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [64] + nqs: [1000] \ No newline at end of file diff --git a/tests/benchmark/suites/gpu_build_performance_jaccard50m.yaml b/tests/benchmark/suites/gpu_build_performance_jaccard50m.yaml new file mode 100644 index 0000000000..fdf9cccd26 --- /dev/null +++ b/tests/benchmark/suites/gpu_build_performance_jaccard50m.yaml @@ -0,0 +1,20 @@ +build_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_7/jaccard_50m_512_512_jaccard_ivf_wal_debug + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: jaccard_50m_512_512_jaccard + index_type: ivf_flat + index_param: + nlist: 2048 diff --git a/tests/benchmark/suites/gpu_search_performance.yaml b/tests/benchmark/suites/gpu_search_performance.yaml new file mode 100644 index 0000000000..78f6334a7f --- /dev/null +++ b/tests/benchmark/suites/gpu_search_performance.yaml @@ -0,0 +1,247 @@ +search_performance: + collections: + # sift_50m + - + server: + db_config.primary_path: /test/milvus/db_data_gpu_crud/sift_50m_1024_128_l2_ivf + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_50m_1024_128_l2 + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu_crud/sift_50m_1024_128_l2_sq8 + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_50m_1024_128_l2 + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu_crud/sift_50m_1024_128_l2_sq8h + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_50m_1024_128_l2 + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_pq + # cache_config.cpu_cache_capacity: 32 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_50m_1024_128_l2 + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu_crud/sift_50m_1024_128_l2_nsg + # cache_config.cpu_cache_capacity: 50 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_50m_1024_128_l2 + # run_count: 2 + # search_params: + # nprobes: [8] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + + # random_50m + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu_crud/random_50m_1024_512_ip_ivf + # cache_config.cpu_cache_capacity: 110 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: random_50m_1024_512_ip + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8 + # cache_config.cpu_cache_capacity: 30 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: random_50m_1024_512_ip + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8h + # cache_config.cpu_cache_capacity: 30 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: random_50m_1024_512_ip + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_nsg + # cache_config.cpu_cache_capacity: 200 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 6 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: random_50m_1024_512_ip + # run_count: 2 + # search_params: + # nprobes: [8] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + + # sift_1b + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu_crud/sift_1b_1024_128_l2_sq8 + # cache_config.cpu_cache_capacity: 150 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_1b_1024_128_l2 + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu_crud/sift_1b_2048_128_l2_sq8h + # cache_config.cpu_cache_capacity: 150 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_1b_2048_128_l2 + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_pq + # cache_config.cpu_cache_capacity: 150 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_1b_2048_128_l2 + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] diff --git a/tests/benchmark/suites/gpu_search_performance_jaccard50m.yaml b/tests/benchmark/suites/gpu_search_performance_jaccard50m.yaml new file mode 100644 index 0000000000..c9a1ed99be --- /dev/null +++ b/tests/benchmark/suites/gpu_search_performance_jaccard50m.yaml @@ -0,0 +1,22 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/jaccard_50m_128_512_jaccard_ivf + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: jaccard_50m_128_512_jaccard + run_count: 1 + search_params: + nprobes: [8, 32] + top_ks: [1, 16, 64, 128, 256, 512, 1000] + nqs: [1, 10, 100, 200, 500, 1000] \ No newline at end of file diff --git a/tests/benchmark/suites/gpu_search_performance_sift50m.yaml b/tests/benchmark/suites/gpu_search_performance_sift50m.yaml new file mode 100644 index 0000000000..4f49bf4937 --- /dev/null +++ b/tests/benchmark/suites/gpu_search_performance_sift50m.yaml @@ -0,0 +1,146 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_ivf + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_50m_1024_128_l2 + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8 + # cache_config.cpu_cache_capacity: 16 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_50m_1024_128_l2 + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8h + # cache_config.cpu_cache_capacity: 16 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_50m_1024_128_l2 + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + + # git issue num: #626 + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_pq + # cache_config.cpu_cache_capacity: 32 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_50m_1024_128_l2 + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_nsg + # cache_config.cpu_cache_capacity: 50 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_50m_1024_128_l2 + # run_count: 2 + # search_params: + # nprobes: [8] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_8192 + # cache_config.cpu_cache_capacity: 16 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_50m_1024_128_l2 + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_4096 + # cache_config.cpu_cache_capacity: 16 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # collection_name: sift_50m_1024_128_l2 + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] \ No newline at end of file diff --git a/tests/benchmark/suites/gpu_search_stability.yaml b/tests/benchmark/suites/gpu_search_stability.yaml new file mode 100644 index 0000000000..a41ea817cd --- /dev/null +++ b/tests/benchmark/suites/gpu_search_stability.yaml @@ -0,0 +1,23 @@ +search_stability: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu_crud/sift_50m_1024_128_l2_sq8 + cache_config.cpu_cache_capacity: 50 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + - gpu2 + - gpu3 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: sift_50m_1024_128_l2 + during_time: 240 + search_params: + nprobes: 1-200 + top_ks: 1-200 + nqs: 1-200 diff --git a/tests/benchmark/suites/gpu_stability_sift50m.yaml b/tests/benchmark/suites/gpu_stability_sift50m.yaml new file mode 100644 index 0000000000..56faf7b205 --- /dev/null +++ b/tests/benchmark/suites/gpu_stability_sift50m.yaml @@ -0,0 +1,27 @@ +stability: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8h_stability + cache_config.cpu_cache_capacity: 64 + cache_config.cache_insert_data: true + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + collection_name: sift_50m_1024_128_l2 + during_time: 480 + search_params: + nprobes: 1-200 + top_ks: 1-200 + nqs: 1-200 + # length of insert vectors + insert_xb: 100000 + # insert after search 4 times + insert_interval: 4 \ No newline at end of file diff --git a/tests/benchmark/suites/insert_binary.yaml b/tests/benchmark/suites/insert_binary.yaml new file mode 100644 index 0000000000..79fa2b3567 --- /dev/null +++ b/tests/benchmark/suites/insert_binary.yaml @@ -0,0 +1,39 @@ +insert_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_7/sub_50m_512_512_sub_wal + cache_config.cpu_cache_capacity: 8 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: sub_50m_512_512_sub + ni_per: 100000 + build_index: false + index_type: flat + index_param: + nlist: 2048 + + - + server: + db_config.primary_path: /test/milvus/db_data_7/super_50m_512_512_super_wal + cache_config.cpu_cache_capacity: 8 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: super_50m_512_512_super + ni_per: 100000 + build_index: false + index_type: flat + index_param: + nlist: 2048 diff --git a/tests/benchmark/suites/insert_performance_deep1b.yaml b/tests/benchmark/suites/insert_performance_deep1b.yaml new file mode 100644 index 0000000000..27dc83c63e --- /dev/null +++ b/tests/benchmark/suites/insert_performance_deep1b.yaml @@ -0,0 +1,87 @@ +insert_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/deep_1b_1024_96_ip_ivf + cache_config.cpu_cache_capacity: 8 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: deep_1b_1024_96_ip + ni_per: 100000 + build_index: false + # index_type: ivf_flat + # nlist: 16384 + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/deep_1b_1024_96_ip_sq8 + cache_config.cpu_cache_capacity: 8 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: deep_1b_1024_96_ip + ni_per: 100000 + build_index: false + # index_type: ivf_sq8 + # nlist: 16384 + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/deep_1b_1024_96_ip_sq8h + cache_config.cpu_cache_capacity: 8 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: deep_1b_1024_96_ip + ni_per: 100000 + build_index: false + # index_type: ivf_sq8h + # nlist: 16384 + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/deep_1b_1024_96_ip_pq + cache_config.cpu_cache_capacity: 8 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: deep_1b_1024_96_ip + ni_per: 100000 + build_index: false + # index_type: ivf_pq + # nlist: 16384 + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/deep_1b_1024_96_ip_nsg + cache_config.cpu_cache_capacity: 8 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + collection_name: deep_1b_1024_96_ip + ni_per: 100000 + build_index: false + # index_type: nsg + # nlist: 16384 \ No newline at end of file diff --git a/tests/benchmark/suites/locust_cluster_search.yaml b/tests/benchmark/suites/locust_cluster_search.yaml new file mode 100644 index 0000000000..d7641819a2 --- /dev/null +++ b/tests/benchmark/suites/locust_cluster_search.yaml @@ -0,0 +1,45 @@ +locust_search_performance: + collections: + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/cluster/sift_1m_128_l2_2 + suffix_path: true + cache_config.cpu_cache_capacity: 8GB + cache_config.insert_buffer_size: 2GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + collection_name: sift_1m_128_l2_2 + ni_per: 50000 + build_index: true + index_type: ivf_sq8 + index_param: + nlist: 16384 + task: + connection_num: 1 + clients_num: 100 + hatch_rate: 10 + during_time: 10 + types: + - + type: query + weight: 1 + params: + top_k: 10 + nq: 1 + # filters: + # - + # range: + # int64: + # LT: 0 + # GT: 1000000 + search_param: + nprobe: 16 diff --git a/tests/benchmark/suites/locust_insert.yaml b/tests/benchmark/suites/locust_insert.yaml new file mode 100644 index 0000000000..cdbdeaca20 --- /dev/null +++ b/tests/benchmark/suites/locust_insert.yaml @@ -0,0 +1,23 @@ +locust_insert_performance: + collections: + - + server: + cache_config.cpu_cache_capacity: 8GB + cache_config.insert_buffer_size: 2GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + task: + type: insert + connection_num: 1 + clients_num: 10 + hatch_rate: 5 + during_time: 2m diff --git a/tests/benchmark/suites/locust_mix.yaml b/tests/benchmark/suites/locust_mix.yaml new file mode 100644 index 0000000000..71be3afd6a --- /dev/null +++ b/tests/benchmark/suites/locust_mix.yaml @@ -0,0 +1,47 @@ +locust_mix_performance: + collections: + - + milvus: + cache_config.insert_buffer_size: 2GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + gpu0 + gpu_resource_config.build_index_resources: + gpu0 + wal_enable: true + collection_name: sift_1m_500000_128_l2 + other_fields: int + ni_per: 50000 + build_index: true + index_type: ivf_sq8 + index_param: + nlist: 1024 + task: + types: + - type: flush + weight: 1 + - + type: query + weight: 20 + params: + top_k: 10 + nq: 100 + filters: + - range: + int64: + LT: 0 + GT: 1000000 + search_param: + nprobe: 16 + - + type: insert + weight: 10 + params: + nb: 1 + connection_num: 1 + clients_num: 10 + hatch_rate: 2 + during_time: 600 diff --git a/tests/benchmark/suites/locust_search.yaml b/tests/benchmark/suites/locust_search.yaml new file mode 100644 index 0000000000..eac80389c3 --- /dev/null +++ b/tests/benchmark/suites/locust_search.yaml @@ -0,0 +1,49 @@ +locust_search_performance: + collections: + - + milvus: + db_config.primary_path: /test/milvus/db_data_011/cluster/sift_1m_128_l2_2 + suffix_path: true + cache_config.cpu_cache_capacity: 8GB + cache_config.insert_buffer_size: 2GB + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4GB + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + wal_enable: true + cluster: true + external_mysql: true + readonly: + replicas: 1 + collection_name: sift_1m_128_l2_2 + ni_per: 50000 + build_index: true + index_type: ivf_sq8 + index_param: + nlist: 16384 + task: + connection_num: 2 + clients_num: 100 + hatch_rate: 10 + during_time: 3600 + types: + - + type: query + weight: 1 + params: + top_k: 10 + nq: 1 + # filters: + # - + # range: + # int64: + # LT: 0 + # GT: 1000000 + search_param: + nprobe: 16 diff --git a/tests/benchmark/suites/loop_stability.yaml b/tests/benchmark/suites/loop_stability.yaml new file mode 100644 index 0000000000..a304695feb --- /dev/null +++ b/tests/benchmark/suites/loop_stability.yaml @@ -0,0 +1,17 @@ +loop_stability: + collections: + - + server: + suffix_path: true + db_config.primary_path: /test/milvus/db_data_11/loop_stability + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 10 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 2GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + wal_enable: true + pull_interval: 20 + collection_num: 4 diff --git a/tests/benchmark/suites/shards_ann_debug.yaml b/tests/benchmark/suites/shards_ann_debug.yaml new file mode 100644 index 0000000000..ba5db54eb4 --- /dev/null +++ b/tests/benchmark/suites/shards_ann_debug.yaml @@ -0,0 +1,25 @@ +ann_accuracy: + collections: + - + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_file_sizes: [1024] + index_types: ['flat', 'ivf_sq8'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 32, 512] + + - + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + index_file_sizes: [1024] + index_types: ['flat', 'ivf_sq8'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 32, 512] diff --git a/tests/benchmark/suites/shards_insert_performance.yaml b/tests/benchmark/suites/shards_insert_performance.yaml new file mode 100644 index 0000000000..eada67a3b4 --- /dev/null +++ b/tests/benchmark/suites/shards_insert_performance.yaml @@ -0,0 +1,17 @@ +insert_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_8/shards_sift_1m_128_128_l2_insert + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + wal_enable: true + collection_name: sift_1m_128_128_l2 + ni_per: 10000 + build_index: false + index_type: flat diff --git a/tests/benchmark/suites/shards_insert_performance_sift1m.yaml b/tests/benchmark/suites/shards_insert_performance_sift1m.yaml new file mode 100644 index 0000000000..dbc2929fda --- /dev/null +++ b/tests/benchmark/suites/shards_insert_performance_sift1m.yaml @@ -0,0 +1,19 @@ +insert_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_010/shards_sift_1m_128_128_l2_insert + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + wal_enable: true + collection_name: sift_1m_1024_128_l2 + ni_per: 10000 + build_index: true + index_type: ivf_sq8 + index_param: + nlist: 16384 diff --git a/tests/benchmark/suites/shards_loop_stability.yaml b/tests/benchmark/suites/shards_loop_stability.yaml new file mode 100644 index 0000000000..4494b8f4b3 --- /dev/null +++ b/tests/benchmark/suites/shards_loop_stability.yaml @@ -0,0 +1,16 @@ +loop_stability: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_8/shards_loop_stability + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 10 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 2GB + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + wal_enable: true + pull_interval: 2 + collection_num: 2 diff --git a/tests/benchmark/suites/shards_search_performance_sift1m.yaml b/tests/benchmark/suites/shards_search_performance_sift1m.yaml new file mode 100644 index 0000000000..71ed51ce72 --- /dev/null +++ b/tests/benchmark/suites/shards_search_performance_sift1m.yaml @@ -0,0 +1,12 @@ +search_performance: + collections: + - + server: + db_config.primary_path: /test/milvus/db_data_010/shards_sift_1m_128_128_l2_insert + wal_enable: true + collection_name: sift_1m_1024_128_l2 + run_count: 2 + top_ks: [1, 10, 100] + nqs: [1, 10, 100] + search_params: + - nprobe: 8 diff --git a/tests/benchmark/task/task.py b/tests/benchmark/task/task.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/benchmark/test_loop.py b/tests/benchmark/test_loop.py new file mode 100644 index 0000000000..224c796a08 --- /dev/null +++ b/tests/benchmark/test_loop.py @@ -0,0 +1,52 @@ +import time +import random +import logging +from client import MilvusClient +import utils + + +if __name__ == "__main__": + milvus_instance = MilvusClient() + milvus_instance.clean_db() + p_num = 1 + dimension = 128 + insert_xb = 100000 + index_types = ['flat'] + index_param = {"nlist": 2048} + collection_names = [] + milvus_instances_map = {} + insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(insert_xb)] + + for i in range(collection_num): + name = utils.get_unique_name(prefix="collection_") + print(name) + collection_names.append(name) + metric_type = "ip" + # metric_type = random.choice(["l2", "ip"]) + index_file_size = random.randint(10, 100) + milvus_instance.create_collection(name, dimension, index_file_size, metric_type) + milvus_instance = MilvusClient(collection_name=name) + index_type = random.choice(index_types) + milvus_instance.create_index(index_type, index_param=index_param) + insert_vectors = utils.normalize(metric_type, insert_vectors) + milvus_instance.insert(insert_vectors) + milvus_instance.flush() + milvus_instances_map.update({name: milvus_instance}) + print(milvus_instance.describe_index(), milvus_instance.describe(), milvus_instance.count()) + + # tasks = ["insert_rand", "delete_rand", "query_rand", "flush", "compact"] + tasks = ["insert_rand", "query_rand", "flush"] + i = 1 + while True: + print("Loop time: %d" % i) + start_time = time.time() + while time.time() - start_time < pull_interval_seconds: + # choose collection + tmp_collection_name = random.choice(collection_names) + # choose task from task + task_name = random.choice(tasks) + # print(tmp_collection_name, task_name) + func = getattr(milvus_instances_map[tmp_collection_name], task_name) + func() + print("Restart") + i = i + 1 diff --git a/tests/benchmark/utils.py b/tests/benchmark/utils.py new file mode 100644 index 0000000000..3ee7f5e743 --- /dev/null +++ b/tests/benchmark/utils.py @@ -0,0 +1,259 @@ +# -*- coding: utf-8 -*- +import os +import sys +import pdb +import time +import json +import datetime +import argparse +import threading +import logging +import string +import random +# import multiprocessing +import numpy as np +# import psutil +import sklearn.preprocessing +import h5py +# import docker +from yaml import full_load, dump +import yaml +import tableprint as tp +from pprint import pprint +from milvus import DataType + +logger = logging.getLogger("milvus_benchmark.utils") + +DEFAULT_F_FIELD_NAME = 'float_vector' +DEFAULT_B_FIELD_NAME = 'binary_vector' +DEFAULT_INT_FIELD_NAME = 'int64' +DEFAULT_FLOAT_FIELD_NAME = 'float' + +METRIC_MAP = { + "l2": "L2", + "ip": "IP", + "jaccard": "JACCARD", + "hamming": "HAMMING", + "sub": "SUBSTRUCTURE", + "super": "SUPERSTRUCTURE" +} + + +def metric_type_trans(metric_type): + if metric_type in METRIC_MAP.keys(): + return METRIC_MAP[metric_type] + else: + raise Exception("metric_type: %s not in METRIC_MAP" % metric_type) + + +def timestr_to_int(time_str): + time_int = 0 + if isinstance(time_str, int) or time_str.isdigit(): + time_int = int(time_str) + elif time_str.endswith("s"): + time_int = int(time_str.split("s")[0]) + elif time_str.endswith("m"): + time_int = int(time_str.split("m")[0]) * 60 + elif time_str.endswith("h"): + time_int = int(time_str.split("h")[0]) * 60 * 60 + else: + raise Exception("%s not support" % time_str) + return time_int + + +class literal_str(str): pass + + +def change_style(style, representer): + def new_representer(dumper, data): + scalar = representer(dumper, data) + scalar.style = style + return scalar + + return new_representer + + +from yaml.representer import SafeRepresenter + +# represent_str does handle some corner cases, so use that +# instead of calling represent_scalar directly +represent_literal_str = change_style('|', SafeRepresenter.represent_str) + +yaml.add_representer(literal_str, represent_literal_str) + + +def retry(times): + """ + This decorator prints the execution time for the decorated function. + """ + def wrapper(func): + def newfn(*args, **kwargs): + attempt = 0 + while attempt < times: + try: + print("retry {} times".format(attempt+1)) + result = func(*args, **kwargs) + if result: + break + else: + logger.error("Retry failed") + raise Exception("Result false") + except Exception as e: + logger.info(str(e)) + time.sleep(3) + attempt += 1 + return func(*args, **kwargs) + return newfn + return wrapper + + +def timestr_to_int(time_str): + time_int = 0 + if isinstance(time_str, int) or time_str.isdigit(): + time_int = int(time_str) + elif time_str.endswith("s"): + time_int = int(time_str.split("s")[0]) + elif time_str.endswith("m"): + time_int = int(time_str.split("m")[0]) * 60 + elif time_str.endswith("h"): + time_int = int(time_str.split("h")[0]) * 60 * 60 + else: + raise Exception("%s not support" % time_str) + return time_int + + +def get_default_field_name(data_type=DataType.FLOAT_VECTOR): + if data_type == DataType.FLOAT_VECTOR: + field_name = DEFAULT_F_FIELD_NAME + elif data_type == DataType.BINARY_VECTOR: + field_name = DEFAULT_B_FIELD_NAME + elif data_type == DataType.INT64: + field_name = DEFAULT_INT_FIELD_NAME + elif data_type == DataType.FLOAT: + field_name = DEFAULT_FLOAT_FIELD_NAME + else: + logger.error(data_type) + raise Exception("Not supported data type") + return field_name + + +def normalize(metric_type, X): + if metric_type == "ip": + logger.info("Set normalize for metric_type: %s" % metric_type) + X = sklearn.preprocessing.normalize(X, axis=1, norm='l2') + X = X.tolist() + elif metric_type in ["jaccard", "hamming", "sub", "super"]: + tmp = [] + for index, item in enumerate(X): + new_vector = bytes(np.packbits(item, axis=-1).tolist()) + tmp.append(new_vector) + X = tmp + return X + + +def convert_nested(dct): + def insert(dct, lst): + for x in lst[:-2]: + dct[x] = dct = dct.get(x, dict()) + dct.update({lst[-2]: lst[-1]}) + + # empty dict to store the result + + result = dict() + + # create an iterator of lists + # representing nested or hierarchial flow + lsts = ([*k.split("."), v] for k, v in dct.items()) + + # insert each list into the result + for lst in lsts: + insert(result, lst) + return result + + +def get_unique_name(prefix=None): + if prefix is None: + prefix = "milvus-benchmark-test-" + return prefix + "".join(random.choice(string.ascii_letters + string.digits) for _ in range(8)).lower() + + +def get_current_time(): + return time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()) + + +def print_table(headers, columns, data): + bodys = [] + for index, value in enumerate(columns): + tmp = [value] + tmp.extend(data[index]) + bodys.append(tmp) + tp.table(bodys, headers) + + +def get_dataset(hdf5_file_path): + if not os.path.exists(hdf5_file_path): + raise Exception("%s not existed" % hdf5_file_path) + dataset = h5py.File(hdf5_file_path) + return dataset + + +def modify_config(k, v, type=None, file_path="conf/server_config.yaml", db_slave=None): + if not os.path.isfile(file_path): + raise Exception('File: %s not found' % file_path) + with open(file_path) as f: + config_dict = full_load(f) + f.close() + if config_dict: + if k.find("use_blas_threshold") != -1: + config_dict['engine_config']['use_blas_threshold'] = int(v) + elif k.find("use_gpu_threshold") != -1: + config_dict['engine_config']['gpu_search_threshold'] = int(v) + elif k.find("cpu_cache_capacity") != -1: + config_dict['cache_config']['cpu_cache_capacity'] = int(v) + elif k.find("enable_gpu") != -1: + config_dict['gpu_resource_config']['enable'] = v + elif k.find("gpu_cache_capacity") != -1: + config_dict['gpu_resource_config']['cache_capacity'] = int(v) + elif k.find("index_build_device") != -1: + config_dict['gpu_resource_config']['build_index_resources'] = v + elif k.find("search_resources") != -1: + config_dict['resource_config']['resources'] = v + + # if db_slave: + # config_dict['db_config']['db_slave_path'] = MULTI_DB_SLAVE_PATH + with open(file_path, 'w') as f: + dump(config_dict, f, default_flow_style=False) + f.close() + else: + raise Exception('Load file:%s error' % file_path) + + +# update server_config.yaml +def update_server_config(file_path, server_config): + if not os.path.isfile(file_path): + raise Exception('File: %s not found' % file_path) + with open(file_path) as f: + values_dict = full_load(f) + f.close() + for k, v in server_config.items(): + if k.find("primary_path") != -1: + values_dict["db_config"]["primary_path"] = v + elif k.find("use_blas_threshold") != -1: + values_dict['engine_config']['use_blas_threshold'] = int(v) + elif k.find("gpu_search_threshold") != -1: + values_dict['engine_config']['gpu_search_threshold'] = int(v) + elif k.find("cpu_cache_capacity") != -1: + values_dict['cache_config']['cpu_cache_capacity'] = int(v) + elif k.find("cache_insert_data") != -1: + values_dict['cache_config']['cache_insert_data'] = v + elif k.find("enable") != -1: + values_dict['gpu_resource_config']['enable'] = v + elif k.find("gpu_cache_capacity") != -1: + values_dict['gpu_resource_config']['cache_capacity'] = int(v) + elif k.find("build_index_resources") != -1: + values_dict['gpu_resource_config']['build_index_resources'] = v + elif k.find("search_resources") != -1: + values_dict['gpu_resource_config']['search_resources'] = v + with open(file_path, 'w') as f: + dump(values_dict, f, default_flow_style=False) + f.close() diff --git a/tests/python_test/.dockerignore b/tests/python_test/.dockerignore new file mode 100644 index 0000000000..c97d9d043c --- /dev/null +++ b/tests/python_test/.dockerignore @@ -0,0 +1,14 @@ +node_modules +npm-debug.log +Dockerfile* +docker-compose* +.dockerignore +.git +.gitignore +.env +*/bin +*/obj +README.md +LICENSE +.vscode +__pycache__ \ No newline at end of file diff --git a/tests/python_test/.gitignore b/tests/python_test/.gitignore new file mode 100644 index 0000000000..9bd7345e51 --- /dev/null +++ b/tests/python_test/.gitignore @@ -0,0 +1,13 @@ +.python-version +.pytest_cache +__pycache__ +.vscode +.idea + +test_out/ +*.pyc + +db/ +logs/ + +.coverage diff --git a/tests/python_test/Dockerfile b/tests/python_test/Dockerfile new file mode 100644 index 0000000000..dbc3f14ced --- /dev/null +++ b/tests/python_test/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.6.8-jessie + +LABEL Name=megasearch_engine_test Version=0.0.1 + +WORKDIR /app +COPY . /app + +RUN apt-get update && apt-get install -y --no-install-recommends \ + libc-dev build-essential && \ + python3 -m pip install -r requirements.txt && \ + apt-get remove --purge -y && \ + rm -rf /var/lib/apt/lists/* + +ENTRYPOINT [ "/app/docker-entrypoint.sh" ] +CMD [ "start" ] \ No newline at end of file diff --git a/tests/python_test/README.md b/tests/python_test/README.md new file mode 100644 index 0000000000..3e8b45a18c --- /dev/null +++ b/tests/python_test/README.md @@ -0,0 +1,62 @@ +## Requirements +* python 3.6.8+ +* pip install -r requirements.txt + +## How to Build Test Env +```shell +sudo docker pull registry.zilliz.com/milvus/milvus-test:v0.2 +sudo docker run -it -v /home/zilliz:/home/zilliz -d registry.zilliz.com/milvus/milvus-test:v0.2 +``` + +## How to Create Test Env docker in k8s +```shell +# 1. start milvus k8s pod +cd milvus-helm/charts/milvus +helm install --wait --timeout 300s \ + --set image.repository=registry.zilliz.com/milvus/engine \ + --set persistence.enabled=true \ + --set image.tag=PR-3818-gpu-centos7-release \ + --set image.pullPolicy=Always \ + --set service.type=LoadBalancer \ + -f ci/db_backend/mysql_gpu_values.yaml \ + -f ci/filebeat/values.yaml \ + -f test.yaml \ + --namespace milvus \ + milvus-ci-pr-3818-1-single-centos7-gpu . + +# 2. remove milvus k8s pod +helm uninstall -n milvus milvus-test + +# 3. check k8s pod status +kubectl get svc -n milvus -w milvus-test + +# 4. login to pod +kubectl get pods --namespace milvus +kubectl exec -it milvus-test-writable-6cc49cfcd4-rbrns -n milvus bash +``` + +## How to Run Test cases +```shell +# Test level-1 cases +pytest . --level=1 --ip=127.0.0.1 --port=19530 + +# Test level-1 cases in 'test_connect.py' only +pytest test_connect.py --level=1 +``` + +## How to list test cases +```shell +# List all cases +pytest --dry-run -qq + +# Collect all cases with docstring +pytest --collect-only -qq + +# Create test report with allure +pytest --alluredir=test_out . -q -v +allure serve test_out + ``` + +## Contribution getting started +* Follow PEP-8 for naming and black for formatting. + diff --git a/tests/python_test/collection/test_collection_count.py b/tests/python_test/collection/test_collection_count.py new file mode 100644 index 0000000000..7e2bece23f --- /dev/null +++ b/tests/python_test/collection/test_collection_count.py @@ -0,0 +1,575 @@ +import pdb +import copy +import logging +import itertools +from time import sleep +import threading +from multiprocessing import Process +import sklearn.preprocessing + +import pytest +from utils import * +from constants import * + +uid = "collection_count" +tag = "collection_count_tag" + +class TestCollectionCount: + """ + params means different nb, the nb value may trigger merge, or not + """ + @pytest.fixture( + scope="function", + params=[ + 1, + 1000, + 2001 + ], + ) + def insert_count(self, request): + yield request.param + + """ + generate valid create_index params + """ + @pytest.fixture( + scope="function", + params=gen_simple_index() + ) + def get_simple_index(self, request, connect): + if str(connect._cmd("mode")[1]) == "CPU": + if request.param["index_type"] in index_cpu_not_support(): + pytest.skip("sq8h not support in cpu mode") + request.param.update({"metric_type": "L2"}) + return request.param + + def test_collection_count(self, connect, collection, insert_count): + ''' + target: test collection rows_count is correct or not + method: create collection and add vectors in it, + assert the value returned by count_entities method is equal to length of vectors + expected: the count is equal to the length of vectors + ''' + entities = gen_entities(insert_count) + res = connect.insert(collection, entities) + connect.flush([collection]) + # res = connect.count_entities(collection) + # assert res == insert_count + stats = connect.get_collection_stats(collection) + assert stats["row_count"] == insert_count + + def test_collection_count_partition(self, connect, collection, insert_count): + ''' + target: test collection rows_count is correct or not + method: create collection, create partition and add vectors in it, + assert the value returned by count_entities method is equal to length of vectors + expected: the count is equal to the length of vectors + ''' + entities = gen_entities(insert_count) + connect.create_partition(collection, tag) + res_ids = connect.insert(collection, entities, partition_tag=tag) + connect.flush([collection]) + # res = connect.count_entities(collection) + # assert res == insert_count + stats = connect.get_collection_stats(collection) + assert stats["row_count"] == insert_count + + # def test_collection_count_multi_partitions_A(self, connect, collection, insert_count): + # ''' + # target: test collection rows_count is correct or not + # method: create collection, create partitions and add entities in it, + # assert the value returned by count_entities method is equal to length of entities + # expected: the count is equal to the length of entities + # ''' + # new_tag = "new_tag" + # entities = gen_entities(insert_count) + # connect.create_partition(collection, tag) + # connect.create_partition(collection, new_tag) + # res_ids = connect.insert(collection, entities) + # connect.flush([collection]) + # # res = connect.count_entities(collection) + # # assert res == insert_count + # stats = connect.get_collection_stats(collection) + # assert stats["row_count"] == insert_count + + # def test_collection_count_multi_partitions_B(self, connect, collection, insert_count): + # ''' + # target: test collection rows_count is correct or not + # method: create collection, create partitions and add entities in one of the partitions, + # assert the value returned by count_entities method is equal to length of entities + # expected: the count is equal to the length of entities + # ''' + # new_tag = "new_tag" + # entities = gen_entities(insert_count) + # connect.create_partition(collection, tag) + # connect.create_partition(collection, new_tag) + # res_ids = connect.insert(collection, entities, partition_tag=tag) + # connect.flush([collection]) + # # res = connect.count_entities(collection) + # # assert res == insert_count + # stats = connect.get_collection_stats(collection) + # assert stats["row_count"] == insert_count + + # def test_collection_count_multi_partitions_C(self, connect, collection, insert_count): + # ''' + # target: test collection rows_count is correct or not + # method: create collection, create partitions and add entities in one of the partitions, + # assert the value returned by count_entities method is equal to length of entities + # expected: the count is equal to the length of vectors + # ''' + # new_tag = "new_tag" + # entities = gen_entities(insert_count) + # connect.create_partition(collection, tag) + # connect.create_partition(collection, new_tag) + # res_ids = connect.insert(collection, entities) + # res_ids_2 = connect.insert(collection, entities, partition_tag=tag) + # connect.flush([collection]) + # # res = connect.count_entities(collection) + # # assert res == insert_count * 2 + # stats = connect.get_collection_stats(collection) + # assert stats["row_count"] == insert_count * 2 + + # def test_collection_count_multi_partitions_D(self, connect, collection, insert_count): + # ''' + # target: test collection rows_count is correct or not + # method: create collection, create partitions and add entities in one of the partitions, + # assert the value returned by count_entities method is equal to length of entities + # expected: the collection count is equal to the length of entities + # ''' + # new_tag = "new_tag" + # entities = gen_entities(insert_count) + # connect.create_partition(collection, tag) + # connect.create_partition(collection, new_tag) + # res_ids = connect.insert(collection, entities, partition_tag=tag) + # res_ids2 = connect.insert(collection, entities, partition_tag=new_tag) + # connect.flush([collection]) + # # res = connect.count_entities(collection) + # # assert res == insert_count * 2 + # stats = connect.get_collection_stats(collection) + # assert stats["row_count"] == insert_count * 2 + + def test_collection_count_after_index_created(self, connect, collection, get_simple_index, insert_count): + ''' + target: test count_entities, after index have been created + method: add vectors in db, and create index, then calling count_entities with correct params + expected: count_entities raise exception + ''' + entities = gen_entities(insert_count) + res = connect.insert(collection, entities) + connect.flush([collection]) + connect.create_index(collection, default_float_vec_field_name, get_simple_index) + # res = connect.count_entities(collection) + # assert res == insert_count + stats = connect.get_collection_stats(collection) + assert stats["row_count"] == insert_count + + def test_count_without_connection(self, collection, dis_connect): + ''' + target: test count_entities, without connection + method: calling count_entities with correct params, with a disconnected instance + expected: count_entities raise exception + ''' + with pytest.raises(Exception) as e: + dis_connect.count_entities(collection) + + def test_collection_count_no_vectors(self, connect, collection): + ''' + target: test collection rows_count is correct or not, if collection is empty + method: create collection and no vectors in it, + assert the value returned by count_entities method is equal to 0 + expected: the count is equal to 0 + ''' + # res = connect.count_entities(collection) + # assert res == 0 + stats = connect.get_collection_stats(collection) + assert stats["row_count"] == 0 + + +class TestCollectionCountIP: + """ + params means different nb, the nb value may trigger merge, or not + """ + @pytest.fixture( + scope="function", + params=[ + 1, + 1000, + 2001 + ], + ) + def insert_count(self, request): + yield request.param + + """ + generate valid create_index params + """ + @pytest.fixture( + scope="function", + params=gen_simple_index() + ) + def get_simple_index(self, request, connect): + if str(connect._cmd("mode")[1]) == "CPU": + if request.param["index_type"] in index_cpu_not_support(): + pytest.skip("sq8h not support in cpu mode") + request.param.update({"metric_type": "IP"}) + return request.param + + def test_collection_count_after_index_created(self, connect, collection, get_simple_index, insert_count): + ''' + target: test count_entities, after index have been created + method: add vectors in db, and create index, then calling count_entities with correct params + expected: count_entities raise exception + ''' + entities = gen_entities(insert_count) + res = connect.insert(collection, entities) + connect.flush([collection]) + connect.create_index(collection, field_name, get_simple_index) + # res = connect.count_entities(collection) + # assert res == insert_count + stats = connect.get_collection_stats(collection) + assert stats["row_count"] == insert_count + +class TestCollectionCountBinary: + """ + params means different nb, the nb value may trigger merge, or not + """ + @pytest.fixture( + scope="function", + params=[ + 1, + 1000, + 2001 + ], + ) + def insert_count(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_simple_index() + ) + def get_jaccard_index(self, request, connect): + if request.param["index_type"] in binary_support(): + request.param["metric_type"] = "JACCARD" + return request.param + else: + pytest.skip("Skip index") + + @pytest.fixture( + scope="function", + params=gen_simple_index() + ) + def get_hamming_index(self, request, connect): + if request.param["index_type"] in binary_support(): + request.param["metric_type"] = "HAMMING" + return request.param + else: + pytest.skip("Skip index") + + @pytest.fixture( + scope="function", + params=gen_simple_index() + ) + def get_substructure_index(self, request, connect): + if request.param["index_type"] == "FLAT": + request.param["metric_type"] = "SUBSTRUCTURE" + return request.param + else: + pytest.skip("Skip index") + + @pytest.fixture( + scope="function", + params=gen_simple_index() + ) + def get_superstructure_index(self, request, connect): + if request.param["index_type"] == "FLAT": + request.param["metric_type"] = "SUPERSTRUCTURE" + return request.param + else: + pytest.skip("Skip index") + + def test_collection_count(self, connect, binary_collection, insert_count): + ''' + target: test collection rows_count is correct or not + method: create collection and add entities in it, + assert the value returned by count_entities method is equal to length of entities + expected: the count is equal to the length of entities + ''' + raw_vectors, entities = gen_binary_entities(insert_count) + res = connect.insert(binary_collection, entities) + logging.getLogger().info(len(res)) + connect.flush([binary_collection]) + # res = connect.count_entities(binary_collection) + # assert res == insert_count + stats = connect.get_collection_stats(binary_collection) + assert stats["row_count"] == insert_count + + def test_collection_count_partition(self, connect, binary_collection, insert_count): + ''' + target: test collection rows_count is correct or not + method: create collection, create partition and add entities in it, + assert the value returned by count_entities method is equal to length of entities + expected: the count is equal to the length of entities + ''' + raw_vectors, entities = gen_binary_entities(insert_count) + connect.create_partition(binary_collection, tag) + res_ids = connect.insert(binary_collection, entities, partition_tag=tag) + connect.flush([binary_collection]) + # res = connect.count_entities(binary_collection) + # assert res == insert_count + stats = connect.get_collection_stats(binary_collection) + assert stats["row_count"] == insert_count + + # @pytest.mark.level(2) + # def test_collection_count_multi_partitions_A(self, connect, binary_collection, insert_count): + # ''' + # target: test collection rows_count is correct or not + # method: create collection, create partitions and add entities in it, + # assert the value returned by count_entities method is equal to length of entities + # expected: the count is equal to the length of entities + # ''' + # new_tag = "new_tag" + # raw_vectors, entities = gen_binary_entities(insert_count) + # connect.create_partition(binary_collection, tag) + # connect.create_partition(binary_collection, new_tag) + # res_ids = connect.insert(binary_collection, entities) + # connect.flush([binary_collection]) + # # res = connect.count_entities(binary_collection) + # # assert res == insert_count + # stats = connect.get_collection_stats(binary_collection) + # assert stats["row_count"] == insert_count + + # @pytest.mark.level(2) + # def test_collection_count_multi_partitions_B(self, connect, binary_collection, insert_count): + # ''' + # target: test collection rows_count is correct or not + # method: create collection, create partitions and add entities in one of the partitions, + # assert the value returned by count_entities method is equal to length of entities + # expected: the count is equal to the length of entities + # ''' + # new_tag = "new_tag" + # raw_vectors, entities = gen_binary_entities(insert_count) + # connect.create_partition(binary_collection, tag) + # connect.create_partition(binary_collection, new_tag) + # res_ids = connect.insert(binary_collection, entities, partition_tag=tag) + # connect.flush([binary_collection]) + # # res = connect.count_entities(binary_collection) + # # assert res == insert_count + # stats = connect.get_collection_stats(binary_collection) + # assert stats["row_count"] == insert_count + + # def test_collection_count_multi_partitions_C(self, connect, binary_collection, insert_count): + # ''' + # target: test collection rows_count is correct or not + # method: create collection, create partitions and add entities in one of the partitions, + # assert the value returned by count_entities method is equal to length of entities + # expected: the count is equal to the length of entities + # ''' + # new_tag = "new_tag" + # raw_vectors, entities = gen_binary_entities(insert_count) + # connect.create_partition(binary_collection, tag) + # connect.create_partition(binary_collection, new_tag) + # res_ids = connect.insert(binary_collection, entities) + # res_ids_2 = connect.insert(binary_collection, entities, partition_tag=tag) + # connect.flush([binary_collection]) + # # res = connect.count_entities(binary_collection) + # # assert res == insert_count * 2 + # stats = connect.get_collection_stats(binary_collection) + # assert stats["row_count"] == insert_count * 2 + + # @pytest.mark.level(2) + # def test_collection_count_multi_partitions_D(self, connect, binary_collection, insert_count): + # ''' + # target: test collection rows_count is correct or not + # method: create collection, create partitions and add entities in one of the partitions, + # assert the value returned by count_entities method is equal to length of entities + # expected: the collection count is equal to the length of entities + # ''' + # new_tag = "new_tag" + # raw_vectors, entities = gen_binary_entities(insert_count) + # connect.create_partition(binary_collection, tag) + # connect.create_partition(binary_collection, new_tag) + # res_ids = connect.insert(binary_collection, entities, partition_tag=tag) + # res_ids2 = connect.insert(binary_collection, entities, partition_tag=new_tag) + # connect.flush([binary_collection]) + # # res = connect.count_entities(binary_collection) + # # assert res == insert_count * 2 + # stats = connect.get_collection_stats(binary_collection) + # assert stats["row_count"] == insert_count * 2 + + # TODO: need to update and enable + def _test_collection_count_after_index_created(self, connect, binary_collection, get_jaccard_index, insert_count): + ''' + target: test count_entities, after index have been created + method: add vectors in db, and create index, then calling count_entities with correct params + expected: count_entities raise exception + ''' + raw_vectors, entities = gen_binary_entities(insert_count) + res = connect.insert(binary_collection, entities) + connect.flush([binary_collection]) + connect.create_index(binary_collection, field_name, get_jaccard_index) + # res = connect.count_entities(binary_collection) + # assert res == insert_count + stats = connect.get_collection_stats(binary_collection) + assert stats["row_count"] == insert_count + + # TODO: need to update and enable + def _test_collection_count_after_index_created(self, connect, binary_collection, get_hamming_index, insert_count): + ''' + target: test count_entities, after index have been created + method: add vectors in db, and create index, then calling count_entities with correct params + expected: count_entities raise exception + ''' + raw_vectors, entities = gen_binary_entities(insert_count) + res = connect.insert(binary_collection, entities) + connect.flush([binary_collection]) + connect.create_index(binary_collection, field_name, get_hamming_index) + # res = connect.count_entities(binary_collection) + # assert res == insert_count + stats = connect.get_collection_stats(binary_collection) + assert stats["row_count"] == insert_count + + def test_collection_count_no_entities(self, connect, binary_collection): + ''' + target: test collection rows_count is correct or not, if collection is empty + method: create collection and no vectors in it, + assert the value returned by count_entities method is equal to 0 + expected: the count is equal to 0 + ''' + # res = connect.count_entities(binary_collection) + # assert res == 0 + stats = connect.get_collection_stats(binary_collection) + assert stats["row_count"] == 0 + + +class TestCollectionMultiCollections: + """ + params means different nb, the nb value may trigger merge, or not + """ + @pytest.fixture( + scope="function", + params=[ + 1, + 1000, + 2001 + ], + ) + def insert_count(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_simple_index() + ) + def get_jaccard_index(self, request, connect): + if request.param["index_type"] in binary_support(): + request.param["metric_type"] = "JACCARD" + return request.param + else: + pytest.skip("Skip index") + + @pytest.fixture( + scope="function", + params=gen_simple_index() + ) + def get_hamming_index(self, request, connect): + if request.param["index_type"] in binary_support(): + request.param["metric_type"] = "HAMMING" + return request.param + else: + pytest.skip("Skip index") + + @pytest.fixture( + scope="function", + params=gen_simple_index() + ) + def get_substructure_index(self, request, connect): + if request.param["index_type"] == "FLAT": + request.param["metric_type"] = "SUBSTRUCTURE" + return request.param + else: + pytest.skip("Skip index") + + @pytest.fixture( + scope="function", + params=gen_simple_index() + ) + def get_superstructure_index(self, request, connect): + if request.param["index_type"] == "FLAT": + request.param["metric_type"] = "SUPERSTRUCTURE" + return request.param + else: + pytest.skip("Skip index") + + def test_collection_count_multi_collections_l2(self, connect, insert_count): + ''' + target: test collection rows_count is correct or not with multiple collections of L2 + method: create collection and add entities in it, + assert the value returned by count_entities method is equal to length of entities + expected: the count is equal to the length of entities + ''' + entities = gen_entities(insert_count) + collection_list = [] + collection_num = 20 + for i in range(collection_num): + collection_name = gen_unique_str(uid) + collection_list.append(collection_name) + connect.create_collection(collection_name, default_fields) + res = connect.insert(collection_name, entities) + connect.flush(collection_list) + for i in range(collection_num): + # res = connect.count_entities(collection_list[i]) + # assert res == insert_count + stats = connect.get_collection_stats(collection_list[i]) + assert stats["row_count"] == insert_count + + @pytest.mark.level(2) + def test_collection_count_multi_collections_binary(self, connect, binary_collection, insert_count): + ''' + target: test collection rows_count is correct or not with multiple collections of JACCARD + method: create collection and add entities in it, + assert the value returned by count_entities method is equal to length of entities + expected: the count is equal to the length of entities + ''' + raw_vectors, entities = gen_binary_entities(insert_count) + res = connect.insert(binary_collection, entities) + collection_list = [] + collection_num = 20 + for i in range(collection_num): + collection_name = gen_unique_str(uid) + collection_list.append(collection_name) + connect.create_collection(collection_name, default_binary_fields) + res = connect.insert(collection_name, entities) + connect.flush(collection_list) + for i in range(collection_num): + # res = connect.count_entities(collection_list[i]) + # assert res == insert_count + stats = connect.get_collection_stats(collection_list[i]) + assert stats["row_count"] == insert_count + + @pytest.mark.level(2) + def test_collection_count_multi_collections_mix(self, connect): + ''' + target: test collection rows_count is correct or not with multiple collections of JACCARD + method: create collection and add entities in it, + assert the value returned by count_entities method is equal to length of entities + expected: the count is equal to the length of entities + ''' + collection_list = [] + collection_num = 20 + for i in range(0, int(collection_num / 2)): + collection_name = gen_unique_str(uid) + collection_list.append(collection_name) + connect.create_collection(collection_name, default_fields) + res = connect.insert(collection_name, default_entities) + for i in range(int(collection_num / 2), collection_num): + collection_name = gen_unique_str(uid) + collection_list.append(collection_name) + connect.create_collection(collection_name, default_binary_fields) + res = connect.insert(collection_name, default_binary_entities) + connect.flush(collection_list) + for i in range(collection_num): + # res = connect.count_entities(collection_list[i]) + # assert res == default_nb + stats = connect.get_collection_stats(collection_list[i]) + assert stats["row_count"] == default_nb diff --git a/tests/python_test/collection/test_collection_logic.py b/tests/python_test/collection/test_collection_logic.py new file mode 100644 index 0000000000..e0503e4305 --- /dev/null +++ b/tests/python_test/collection/test_collection_logic.py @@ -0,0 +1,138 @@ +import pdb +import pytest +import logging +import itertools +from time import sleep +from multiprocessing import Process +from utils import * + +uid = "collection_logic" + +def create_collection(connect, **params): + connect.create_collection(params["collection_name"], const.default_fields) + +def search_collection(connect, **params): + status, result = connect.search( + params["collection_name"], + params["top_k"], + params["query_vectors"], + params={"nprobe": params["nprobe"]}) + return status + +def load_collection(connect, **params): + connect.load_collection(params["collection_name"]) + +def has(connect, **params): + status, result = connect.has_collection(params["collection_name"]) + return status + +def show(connect, **params): + status, result = connect.list_collections() + return status + +def delete(connect, **params): + status = connect.drop_collection(params["collection_name"]) + return status + +def describe(connect, **params): + status, result = connect.get_collection_info(params["collection_name"]) + return status + +def rowcount(connect, **params): + status, result = connect.count_entities(params["collection_name"]) + return status + +def create_index(connect, **params): + status = connect.create_index(params["collection_name"], params["index_type"], params["index_param"]) + return status + +func_map = { + # 0:has, + 1:show, + 10:create_collection, + 11:describe, + 12:rowcount, + 13:search_collection, + 14:load_collection, + 15:create_index, + 30:delete +} + +def gen_sequence(): + raw_seq = func_map.keys() + result = itertools.permutations(raw_seq) + for x in result: + yield x + + +class TestCollectionLogic(object): + @pytest.mark.parametrize("logic_seq", gen_sequence()) + @pytest.mark.level(2) + def _test_logic(self, connect, logic_seq, args): + if args["handler"] == "HTTP": + pytest.skip("Skip in http mode") + if self.is_right(logic_seq): + self.execute(logic_seq, connect) + else: + self.execute_with_error(logic_seq, connect) + self.tear_down(connect) + + def is_right(self, seq): + if sorted(seq) == seq: + return True + + not_created = True + has_deleted = False + for i in range(len(seq)): + if seq[i] > 10 and not_created: + return False + elif seq [i] > 10 and has_deleted: + return False + elif seq[i] == 10: + not_created = False + elif seq[i] == 30: + has_deleted = True + + return True + + def execute(self, logic_seq, connect): + basic_params = self.gen_params() + for i in range(len(logic_seq)): + # logging.getLogger().info(logic_seq[i]) + f = func_map[logic_seq[i]] + status = f(connect, **basic_params) + assert status.OK() + + def execute_with_error(self, logic_seq, connect): + basic_params = self.gen_params() + + error_flag = False + for i in range(len(logic_seq)): + f = func_map[logic_seq[i]] + status = f(connect, **basic_params) + if not status.OK(): + # logging.getLogger().info(logic_seq[i]) + error_flag = True + break + assert error_flag == True + + def tear_down(self, connect): + names = connect.list_collections()[1] + for name in names: + connect.drop_collection(name) + + def gen_params(self): + collection_name = gen_unique_str(uid) + top_k = 1 + vectors = gen_vectors(2, dim) + param = {'collection_name': collection_name, + 'dimension': dim, + 'metric_type': "L2", + 'nprobe': 1, + 'top_k': top_k, + 'index_type': "IVF_SQ8", + 'index_param': { + 'nlist': 16384 + }, + 'query_vectors': vectors} + return param diff --git a/tests/python_test/collection/test_collection_stats.py b/tests/python_test/collection/test_collection_stats.py new file mode 100644 index 0000000000..569833d463 --- /dev/null +++ b/tests/python_test/collection/test_collection_stats.py @@ -0,0 +1,356 @@ +import time +import pdb +import threading +import logging +from multiprocessing import Pool, Process + +import pytest +from utils import * +from constants import * + +uid = "get_collection_stats" + +class TestGetCollectionStats: + """ + ****************************************************************** + The following cases are used to test `collection_stats` function + ****************************************************************** + """ + + @pytest.fixture( + scope="function", + params=gen_invalid_strs() + ) + def get_collection_name(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_simple_index() + ) + def get_simple_index(self, request, connect): + if str(connect._cmd("mode")) == "CPU": + if request.param["index_type"] in index_cpu_not_support(): + pytest.skip("CPU not support index_type: ivf_sq8h") + return request.param + + @pytest.fixture( + scope="function", + params=gen_binary_index() + ) + def get_jaccard_index(self, request, connect): + logging.getLogger().info(request.param) + if request.param["index_type"] in binary_support(): + request.param["metric_type"] = "JACCARD" + return request.param + else: + pytest.skip("Skip index Temporary") + + def test_get_collection_stats_name_not_existed(self, connect, collection): + ''' + target: get collection stats where collection name does not exist + method: call collection_stats with a random collection_name, which is not in db + expected: status not ok + ''' + collection_name = gen_unique_str(uid) + connect.create_collection(collection_name, default_fields) + connect.get_collection_stats(collection_name) + connect.drop_collection(collection_name) + with pytest.raises(Exception) as e: + connect.get_collection_stats(collection_name) + + @pytest.mark.level(2) + def test_get_collection_stats_name_invalid(self, connect, get_collection_name): + ''' + target: get collection stats where collection name is invalid + method: call collection_stats with invalid collection_name + expected: status not ok + ''' + collection_name = get_collection_name + with pytest.raises(Exception) as e: + stats = connect.get_collection_stats(collection_name) + + def test_get_collection_stats_empty(self, connect, collection): + ''' + target: get collection stats where no entity in collection + method: call collection_stats in empty collection + expected: segment = [] + ''' + stats = connect.get_collection_stats(collection) + assert stats["row_count"] == 0 + # assert len(stats["partitions"]) == 1 + # assert stats["partitions"][0]["tag"] == default_partition_name + # assert stats["partitions"][0]["row_count"] == 0 + + def test_get_collection_stats_batch(self, connect, collection): + ''' + target: get row count with collection_stats + method: add entities, check count in collection info + expected: count as expected + ''' + ids = connect.insert(collection, default_entities) + connect.flush([collection]) + stats = connect.get_collection_stats(collection) + assert stats["row_count"] == default_nb + # assert len(stats["partitions"]) == 1 + # assert stats["partitions"][0]["tag"] == default_partition_name + # assert stats["partitions"][0]["row_count"] == default_nb + + def test_get_collection_stats_single(self, connect, collection): + ''' + target: get row count with collection_stats + method: add entity one by one, check count in collection info + expected: count as expected + ''' + nb = 10 + for i in range(nb): + ids = connect.insert(collection, default_entity) + connect.flush([collection]) + stats = connect.get_collection_stats(collection) + assert stats["row_count"] == nb + # assert len(stats["partitions"]) == 1 + # assert stats["partitions"][0]["tag"] == default_partition_name + # assert stats["partitions"][0]["row_count"] == nb + + @pytest.mark.skip("delete_by_id not support yet") + def test_get_collection_stats_after_delete(self, connect, collection): + ''' + target: get row count with collection_stats + method: add and delete entities, check count in collection info + expected: status ok, count as expected + ''' + ids = connect.insert(collection, default_entities) + status = connect.flush([collection]) + delete_ids = [ids[0], ids[-1]] + connect.delete_entity_by_id(collection, delete_ids) + connect.flush([collection]) + stats = connect.get_collection_stats(collection) + assert stats["row_count"] == default_nb - 2 + assert stats["partitions"][0]["row_count"] == default_nb - 2 + assert stats["partitions"][0]["segments"][0]["data_size"] > 0 + + # TODO: enable + @pytest.mark.level(2) + @pytest.mark.skip("no compact") + def test_get_collection_stats_after_compact_parts(self, connect, collection): + ''' + target: get row count with collection_stats + method: add and delete entities, and compact collection, check count in collection info + expected: status ok, count as expected + ''' + delete_length = 1000 + ids = connect.insert(collection, default_entities) + status = connect.flush([collection]) + delete_ids = ids[:delete_length] + connect.delete_entity_by_id(collection, delete_ids) + connect.flush([collection]) + stats = connect.get_collection_stats(collection) + logging.getLogger().info(stats) + assert stats["row_count"] == default_nb - delete_length + compact_before = stats["partitions"][0]["segments"][0]["data_size"] + connect.compact(collection) + stats = connect.get_collection_stats(collection) + logging.getLogger().info(stats) + compact_after = stats["partitions"][0]["segments"][0]["data_size"] + assert compact_before == compact_after + + @pytest.mark.skip("no compact") + def test_get_collection_stats_after_compact_delete_one(self, connect, collection): + ''' + target: get row count with collection_stats + method: add and delete one entity, and compact collection, check count in collection info + expected: status ok, count as expected + ''' + ids = connect.insert(collection, default_entities) + status = connect.flush([collection]) + delete_ids = ids[:1] + connect.delete_entity_by_id(collection, delete_ids) + connect.flush([collection]) + stats = connect.get_collection_stats(collection) + logging.getLogger().info(stats) + compact_before = stats["partitions"][0]["row_count"] + connect.compact(collection) + stats = connect.get_collection_stats(collection) + logging.getLogger().info(stats) + compact_after = stats["partitions"][0]["row_count"] + # pdb.set_trace() + assert compact_before == compact_after + + def test_get_collection_stats_partition(self, connect, collection): + ''' + target: get partition info in a collection + method: call collection_stats after partition created and check partition_stats + expected: status ok, vectors added to partition + ''' + connect.create_partition(collection, default_tag) + ids = connect.insert(collection, default_entities, partition_tag=default_tag) + connect.flush([collection]) + stats = connect.get_collection_stats(collection) + assert stats["row_count"] == default_nb + # assert len(stats["partitions"]) == 2 + # assert stats["partitions"][1]["tag"] == default_tag + # assert stats["partitions"][1]["row_count"] == default_nb + + def test_get_collection_stats_partitions(self, connect, collection): + ''' + target: get partition info in a collection + method: create two partitions, add vectors in one of the partitions, call collection_stats and check + expected: status ok, vectors added to one partition but not the other + ''' + new_tag = "new_tag" + connect.create_partition(collection, default_tag) + connect.create_partition(collection, new_tag) + ids = connect.insert(collection, default_entities, partition_tag=default_tag) + connect.flush([collection]) + stats = connect.get_collection_stats(collection) + assert stats["row_count"] == default_nb + # for partition in stats["partitions"]: + # if partition["tag"] == default_tag: + # assert partition["row_count"] == default_nb + # else: + # assert partition["row_count"] == 0 + ids = connect.insert(collection, default_entities, partition_tag=new_tag) + connect.flush([collection]) + stats = connect.get_collection_stats(collection) + assert stats["row_count"] == default_nb * 2 + # for partition in stats["partitions"]: + # if partition["tag"] in [default_tag, new_tag]: + # assert partition["row_count"] == default_nb + ids = connect.insert(collection, default_entities) + connect.flush([collection]) + stats = connect.get_collection_stats(collection) + assert stats["row_count"] == default_nb * 3 + + # TODO: assert metric type in stats response + def test_get_collection_stats_after_index_created(self, connect, collection, get_simple_index): + ''' + target: test collection info after index created + method: create collection, add vectors, create index and call collection_stats + expected: status ok, index created and shown in segments + ''' + ids = connect.insert(collection, default_entities) + connect.flush([collection]) + connect.create_index(collection, default_float_vec_field_name, get_simple_index) + stats = connect.get_collection_stats(collection) + logging.getLogger().info(stats) + assert stats["row_count"] == default_nb + # for file in stats["partitions"][0]["segments"][0]["files"]: + # if file["name"] == default_float_vec_field_name and "index_type" in file: + # assert file["data_size"] > 0 + # assert file["index_type"] == get_simple_index["index_type"] + # break + + # TODO: assert metric type in stats response + def test_get_collection_stats_after_index_created_ip(self, connect, collection, get_simple_index): + ''' + target: test collection info after index created + method: create collection, add vectors, create index and call collection_stats + expected: status ok, index created and shown in segments + ''' + get_simple_index["metric_type"] = "IP" + ids = connect.insert(collection, default_entities) + connect.flush([collection]) + get_simple_index.update({"metric_type": "IP"}) + connect.create_index(collection, default_float_vec_field_name, get_simple_index) + stats = connect.get_collection_stats(collection) + assert stats["row_count"] == default_nb + # for file in stats["partitions"][0]["segments"][0]["files"]: + # if file["name"] == default_float_vec_field_name and "index_type" in file: + # assert file["data_size"] > 0 + # assert file["index_type"] == get_simple_index["index_type"] + # break + + # TODO: assert metric type in stats response + def test_get_collection_stats_after_index_created_jac(self, connect, binary_collection, get_jaccard_index): + ''' + target: test collection info after index created + method: create collection, add binary entities, create index and call collection_stats + expected: status ok, index created and shown in segments + ''' + ids = connect.insert(binary_collection, default_binary_entities) + connect.flush([binary_collection]) + connect.create_index(binary_collection, "binary_vector", get_jaccard_index) + stats = connect.get_collection_stats(binary_collection) + assert stats["row_count"] == default_nb + # for file in stats["partitions"][0]["segments"][0]["files"]: + # if file["name"] == default_float_vec_field_name and "index_type" in file: + # assert file["data_size"] > 0 + # assert file["index_type"] == get_simple_index["index_type"] + # break + + def test_get_collection_stats_after_create_different_index(self, connect, collection): + ''' + target: test collection info after index created repeatedly + method: create collection, add vectors, create index and call collection_stats multiple times + expected: status ok, index info shown in segments + ''' + ids = connect.insert(collection, default_entities) + connect.flush([collection]) + for index_type in ["IVF_FLAT", "IVF_SQ8"]: + connect.create_index(collection, default_float_vec_field_name, + {"index_type": index_type, "params":{"nlist": 1024}, "metric_type": "L2"}) + stats = connect.get_collection_stats(collection) + assert stats["row_count"] == default_nb + # for file in stats["partitions"][0]["segments"][0]["files"]: + # if file["name"] == default_float_vec_field_name and "index_type" in file: + # assert file["data_size"] > 0 + # assert file["index_type"] == index_type + # break + + def test_collection_count_multi_collections(self, connect): + ''' + target: test collection rows_count is correct or not with multiple collections of L2 + method: create collection and add entities in it, + assert the value returned by count_entities method is equal to length of entities + expected: row count in segments + ''' + collection_list = [] + collection_num = 10 + for i in range(collection_num): + collection_name = gen_unique_str(uid) + collection_list.append(collection_name) + connect.create_collection(collection_name, default_fields) + res = connect.insert(collection_name, default_entities) + connect.flush(collection_list) + for i in range(collection_num): + stats = connect.get_collection_stats(collection_list[i]) + # assert stats["partitions"][0]["row_count"] == default_nb + assert stats["row_count"] == default_nb + connect.drop_collection(collection_list[i]) + + @pytest.mark.level(2) + def test_collection_count_multi_collections_indexed(self, connect): + ''' + target: test collection rows_count is correct or not with multiple collections of L2 + method: create collection and add entities in it, + assert the value returned by count_entities method is equal to length of entities + expected: row count in segments + ''' + collection_list = [] + collection_num = 10 + for i in range(collection_num): + collection_name = gen_unique_str(uid) + collection_list.append(collection_name) + connect.create_collection(collection_name, default_fields) + res = connect.insert(collection_name, default_entities) + connect.flush(collection_list) + if i % 2: + connect.create_index(collection_name, default_float_vec_field_name, + {"index_type": "IVF_SQ8", "params":{"nlist": 1024}, "metric_type": "L2"}) + else: + connect.create_index(collection_name, default_float_vec_field_name, + {"index_type": "IVF_FLAT","params":{"nlist": 1024}, "metric_type": "L2"}) + for i in range(collection_num): + stats = connect.get_collection_stats(collection_list[i]) + assert stats["row_count"] == default_nb + # if i % 2: + # for file in stats["partitions"][0]["segments"][0]["files"]: + # if file["name"] == default_float_vec_field_name and "index_type" in file: + # assert file["index_type"] == "IVF_SQ8" + # break + # else: + # for file in stats["partitions"][0]["segments"][0]["files"]: + # if file["name"] == default_float_vec_field_name and "index_type" in file: + # assert file["index_type"] == "IVF_FLAT" + # break + connect.drop_collection(collection_list[i]) diff --git a/tests/python_test/collection/test_create_collection.py b/tests/python_test/collection/test_create_collection.py new file mode 100644 index 0000000000..22c8ae80e8 --- /dev/null +++ b/tests/python_test/collection/test_create_collection.py @@ -0,0 +1,290 @@ +import pdb +import copy +import logging +import itertools +import time +import threading +from multiprocessing import Process +import sklearn.preprocessing + +import pytest +from utils import * +from constants import * + +uid = "create_collection" + +class TestCreateCollection: + """ + ****************************************************************** + The following cases are used to test `create_collection` function + ****************************************************************** + """ + @pytest.fixture( + scope="function", + params=gen_single_filter_fields() + ) + def get_filter_field(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_single_vector_fields() + ) + def get_vector_field(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_segment_row_limits() + ) + def get_segment_row_limit(self, request): + yield request.param + + def test_create_collection_fields(self, connect, get_filter_field, get_vector_field): + ''' + target: test create normal collection with different fields + method: create collection with diff fields: metric/field_type/... + expected: no exception raised + ''' + filter_field = get_filter_field + logging.getLogger().info(filter_field) + vector_field = get_vector_field + collection_name = gen_unique_str(uid) + fields = { + "fields": [filter_field, vector_field], + # "segment_row_limit": default_segment_row_limit + } + logging.getLogger().info(fields) + connect.create_collection(collection_name, fields) + assert connect.has_collection(collection_name) + + @pytest.mark.skip("no segment_row_limit") + def test_create_collection_segment_row_limit(self, connect, get_segment_row_limit): + ''' + target: test create normal collection with different fields + method: create collection with diff segment_row_limit + expected: no exception raised + ''' + collection_name = gen_unique_str(uid) + fields = copy.deepcopy(default_fields) + # fields["segment_row_limit"] = get_segment_row_limit + connect.create_collection(collection_name, fields) + assert connect.has_collection(collection_name) + + def test_create_collection_after_insert(self, connect, collection): + ''' + target: test insert vector, then create collection again + method: insert vector and create collection + expected: error raised + ''' + # pdb.set_trace() + connect.insert(collection, default_entity) + + with pytest.raises(Exception) as e: + connect.create_collection(collection, default_fields) + + def test_create_collection_after_insert_flush(self, connect, collection): + ''' + target: test insert vector, then create collection again + method: insert vector and create collection + expected: error raised + ''' + connect.insert(collection, default_entity) + connect.flush([collection]) + with pytest.raises(Exception) as e: + connect.create_collection(collection, default_fields) + + # TODO: assert exception + def test_create_collection_without_connection(self, dis_connect): + ''' + target: test create collection, without connection + method: create collection with correct params, with a disconnected instance + expected: error raised + ''' + collection_name = gen_unique_str(uid) + with pytest.raises(Exception) as e: + dis_connect.create_collection(collection_name, default_fields) + + def test_create_collection_existed(self, connect): + ''' + target: test create collection but the collection name have already existed + method: create collection with the same collection_name + expected: error raised + ''' + collection_name = gen_unique_str(uid) + connect.create_collection(collection_name, default_fields) + with pytest.raises(Exception) as e: + connect.create_collection(collection_name, default_fields) + + def test_create_after_drop_collection(self, connect, collection): + ''' + target: create with the same collection name after collection dropped + method: delete, then create + expected: create success + ''' + connect.drop_collection(collection) + time.sleep(2) + connect.create_collection(collection, default_fields) + + @pytest.mark.level(2) + def test_create_collection_multithread(self, connect): + ''' + target: test create collection with multithread + method: create collection using multithread, + expected: collections are created + ''' + threads_num = 8 + threads = [] + collection_names = [] + + def create(): + collection_name = gen_unique_str(uid) + collection_names.append(collection_name) + connect.create_collection(collection_name, default_fields) + for i in range(threads_num): + t = TestThread(target=create, args=()) + threads.append(t) + t.start() + time.sleep(0.2) + for t in threads: + t.join() + + for item in collection_names: + assert item in connect.list_collections() + connect.drop_collection(item) + + +class TestCreateCollectionInvalid(object): + """ + Test creating collections with invalid params + """ + @pytest.fixture( + scope="function", + params=gen_invalid_metric_types() + ) + def get_metric_type(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_invalid_ints() + ) + def get_segment_row_limit(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_invalid_ints() + ) + def get_dim(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_invalid_strs() + ) + def get_invalid_string(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_invalid_field_types() + ) + def get_field_type(self, request): + yield request.param + + @pytest.mark.level(2) + @pytest.mark.skip("no segment_row_limit") + def test_create_collection_with_invalid_segment_row_limit(self, connect, get_segment_row_limit): + collection_name = gen_unique_str() + fields = copy.deepcopy(default_fields) + fields["segment_row_limit"] = get_segment_row_limit + with pytest.raises(Exception) as e: + connect.create_collection(collection_name, fields) + + @pytest.mark.level(2) + def test_create_collection_with_invalid_dimension(self, connect, get_dim): + dimension = get_dim + collection_name = gen_unique_str() + fields = copy.deepcopy(default_fields) + fields["fields"][-1]["params"]["dim"] = dimension + with pytest.raises(Exception) as e: + connect.create_collection(collection_name, fields) + + @pytest.mark.level(2) + def test_create_collection_with_invalid_collection_name(self, connect, get_invalid_string): + collection_name = get_invalid_string + with pytest.raises(Exception) as e: + connect.create_collection(collection_name, default_fields) + + @pytest.mark.level(2) + def test_create_collection_with_empty_collection_name(self, connect): + collection_name = '' + with pytest.raises(Exception) as e: + connect.create_collection(collection_name, default_fields) + + @pytest.mark.level(2) + def test_create_collection_with_none_collection_name(self, connect): + collection_name = None + with pytest.raises(Exception) as e: + connect.create_collection(collection_name, default_fields) + + def test_create_collection_no_dimension(self, connect): + ''' + target: test create collection with no dimension params + method: create collection with correct params + expected: create status return ok + ''' + collection_name = gen_unique_str(uid) + fields = copy.deepcopy(default_fields) + fields["fields"][-1]["params"].pop("dim") + with pytest.raises(Exception) as e: + connect.create_collection(collection_name, fields) + + @pytest.mark.skip("no segment_row_limit") + def test_create_collection_no_segment_row_limit(self, connect): + ''' + target: test create collection with no segment_row_limit params + method: create collection with correct params + expected: use default default_segment_row_limit + ''' + collection_name = gen_unique_str(uid) + fields = copy.deepcopy(default_fields) + fields.pop("segment_row_limit") + connect.create_collection(collection_name, fields) + res = connect.get_collection_info(collection_name) + logging.getLogger().info(res) + assert res["segment_row_limit"] == default_server_segment_row_limit + + # TODO: assert exception + def test_create_collection_limit_fields(self, connect): + collection_name = gen_unique_str(uid) + limit_num = 64 + fields = copy.deepcopy(default_fields) + for i in range(limit_num): + field_name = gen_unique_str("field_name") + field = {"name": field_name, "type": DataType.INT64} + fields["fields"].append(field) + with pytest.raises(Exception) as e: + connect.create_collection(collection_name, fields) + + # TODO: assert exception + @pytest.mark.level(2) + def test_create_collection_invalid_field_name(self, connect, get_invalid_string): + collection_name = gen_unique_str(uid) + fields = copy.deepcopy(default_fields) + field_name = get_invalid_string + field = {"name": field_name, "type": DataType.INT64} + fields["fields"].append(field) + with pytest.raises(Exception) as e: + connect.create_collection(collection_name, fields) + + # TODO: assert exception + def test_create_collection_invalid_field_type(self, connect, get_field_type): + collection_name = gen_unique_str(uid) + fields = copy.deepcopy(default_fields) + field_type = get_field_type + field = {"name": "test_field", "type": field_type} + fields["fields"].append(field) + with pytest.raises(Exception) as e: + connect.create_collection(collection_name, fields) diff --git a/tests/python_test/collection/test_describe_collection.py b/tests/python_test/collection/test_describe_collection.py new file mode 100644 index 0000000000..a62a652214 --- /dev/null +++ b/tests/python_test/collection/test_describe_collection.py @@ -0,0 +1,187 @@ +import pytest +import logging +import time +from utils import * +from constants import * + +uid = "describe_collection" + + +class TestDescribeCollection: + + @pytest.fixture( + scope="function", + params=gen_single_filter_fields() + ) + def get_filter_field(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_single_vector_fields() + ) + def get_vector_field(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_simple_index() + ) + def get_simple_index(self, request, connect): + logging.getLogger().info(request.param) + if str(connect._cmd("mode")) == "CPU": + if request.param["index_type"] in index_cpu_not_support(): + pytest.skip("sq8h not support in CPU mode") + return request.param + + """ + ****************************************************************** + The following cases are used to test `describe_collection` function, no data in collection + ****************************************************************** + """ + + def test_collection_fields(self, connect, get_filter_field, get_vector_field): + ''' + target: test create normal collection with different fields, check info returned + method: create collection with diff fields: metric/field_type/..., calling `describe_collection` + expected: no exception raised, and value returned correct + ''' + filter_field = get_filter_field + vector_field = get_vector_field + collection_name = gen_unique_str(uid) + fields = { + "fields": [filter_field, vector_field], + # "segment_row_limit": default_segment_row_limit + } + connect.create_collection(collection_name, fields) + res = connect.describe_collection(collection_name) + assert res['auto_id'] == True + # assert res['segment_row_limit'] == default_segment_row_limit + assert len(res["fields"]) == 2 + for field in res["fields"]: + if field["type"] == filter_field: + assert field["name"] == filter_field["name"] + elif field["type"] == vector_field: + assert field["name"] == vector_field["name"] + assert field["params"] == vector_field["params"] + + def test_describe_collection_after_index_created(self, connect, collection, get_simple_index): + connect.create_index(collection, default_float_vec_field_name, get_simple_index) + res = connect.describe_collection(collection) + for field in res["fields"]: + if field["name"] == default_float_vec_field_name: + index = field["indexes"][0] + assert index["index_type"] == get_simple_index["index_type"] + assert index["metric_type"] == get_simple_index["metric_type"] + + @pytest.mark.level(2) + def test_describe_collection_without_connection(self, collection, dis_connect): + ''' + target: test get collection info, without connection + method: calling get collection info with correct params, with a disconnected instance + expected: get collection info raise exception + ''' + with pytest.raises(Exception) as e: + dis_connect.describe_collection(collection) + + def test_describe_collection_not_existed(self, connect): + ''' + target: test if collection not created + method: random a collection name, create this collection then drop it, + assert the value returned by describe_collection method + expected: False + ''' + collection_name = gen_unique_str(uid) + connect.create_collection(collection_name, default_fields) + connect.describe_collection(collection_name) + connect.drop_collection(collection_name) + with pytest.raises(Exception) as e: + connect.describe_collection(collection_name) + + @pytest.mark.level(2) + def test_describe_collection_multithread(self, connect): + ''' + target: test create collection with multithread + method: create collection using multithread, + expected: collections are created + ''' + threads_num = 4 + threads = [] + collection_name = gen_unique_str(uid) + connect.create_collection(collection_name, default_fields) + + def get_info(): + connect.describe_collection(collection_name) + + for i in range(threads_num): + t = TestThread(target=get_info) + threads.append(t) + t.start() + time.sleep(0.2) + for t in threads: + t.join() + + """ + ****************************************************************** + The following cases are used to test `describe_collection` function, and insert data in collection + ****************************************************************** + """ + + def test_describe_collection_fields_after_insert(self, connect, get_filter_field, get_vector_field): + ''' + target: test create normal collection with different fields, check info returned + method: create collection with diff fields: metric/field_type/..., calling `describe_collection` + expected: no exception raised, and value returned correct + ''' + filter_field = get_filter_field + vector_field = get_vector_field + collection_name = gen_unique_str(uid) + fields = { + "fields": [filter_field, vector_field], + # "segment_row_limit": default_segment_row_limit + } + connect.create_collection(collection_name, fields) + entities = gen_entities_by_fields(fields["fields"], default_nb, vector_field["params"]["dim"]) + res_ids = connect.insert(collection_name, entities) + connect.flush([collection_name]) + res = connect.describe_collection(collection_name) + assert res['auto_id'] == True + # assert res['segment_row_limit'] == default_segment_row_limit + assert len(res["fields"]) == 2 + for field in res["fields"]: + if field["type"] == filter_field: + assert field["name"] == filter_field["name"] + elif field["type"] == vector_field: + assert field["name"] == vector_field["name"] + assert field["params"] == vector_field["params"] + + + +class TestDescribeCollectionInvalid(object): + """ + Test describe collection with invalid params + """ + @pytest.fixture( + scope="function", + params=gen_invalid_strs() + ) + def get_collection_name(self, request): + yield request.param + + @pytest.mark.level(2) + def test_describe_collection_with_invalid_collection_name(self, connect, get_collection_name): + collection_name = get_collection_name + with pytest.raises(Exception) as e: + connect.describe_collection(collection_name) + + @pytest.mark.level(2) + def test_describe_collection_with_empty_collection_name(self, connect): + collection_name = '' + with pytest.raises(Exception) as e: + connect.describe_collection(collection_name) + + @pytest.mark.level(2) + def test_describe_collection_with_none_collection_name(self, connect): + collection_name = None + with pytest.raises(Exception) as e: + connect.describe_collection(collection_name) diff --git a/tests/python_test/collection/test_drop_collection.py b/tests/python_test/collection/test_drop_collection.py new file mode 100644 index 0000000000..a83be4ef2f --- /dev/null +++ b/tests/python_test/collection/test_drop_collection.py @@ -0,0 +1,103 @@ +import pdb +import pytest +import logging +import itertools +from time import sleep +import threading +from multiprocessing import Process +from utils import * +from constants import * + +uid = "drop_collection" + +class TestDropCollection: + """ + ****************************************************************** + The following cases are used to test `drop_collection` function + ****************************************************************** + """ + def test_drop_collection(self, connect, collection): + ''' + target: test delete collection created with correct params + method: create collection and then delete, + assert the value returned by delete method + expected: status ok, and no collection in collections + ''' + connect.drop_collection(collection) + time.sleep(2) + assert not connect.has_collection(collection) + + def test_drop_collection_without_connection(self, collection, dis_connect): + ''' + target: test describe collection, without connection + method: drop collection with correct params, with a disconnected instance + expected: drop raise exception + ''' + with pytest.raises(Exception) as e: + dis_connect.drop_collection(collection) + + def test_drop_collection_not_existed(self, connect): + ''' + target: test if collection not created + method: random a collection name, which not existed in db, + assert the exception raised returned by drp_collection method + expected: False + ''' + collection_name = gen_unique_str(uid) + with pytest.raises(Exception) as e: + connect.drop_collection(collection_name) + + @pytest.mark.level(2) + def test_create_drop_collection_multithread(self, connect): + ''' + target: test create and drop collection with multithread + method: create and drop collection using multithread, + expected: collections are created, and dropped + ''' + threads_num = 8 + threads = [] + collection_names = [] + + def create(): + collection_name = gen_unique_str(uid) + collection_names.append(collection_name) + connect.create_collection(collection_name, default_fields) + connect.drop_collection(collection_name) + for i in range(threads_num): + t = TestThread(target=create, args=()) + threads.append(t) + t.start() + time.sleep(0.2) + for t in threads: + t.join() + + for item in collection_names: + assert not connect.has_collection(item) + + +class TestDropCollectionInvalid(object): + """ + Test has collection with invalid params + """ + @pytest.fixture( + scope="function", + params=gen_invalid_strs() + ) + def get_collection_name(self, request): + yield request.param + + @pytest.mark.level(2) + def test_drop_collection_with_invalid_collection_name(self, connect, get_collection_name): + collection_name = get_collection_name + with pytest.raises(Exception) as e: + connect.has_collection(collection_name) + + def test_drop_collection_with_empty_collection_name(self, connect): + collection_name = '' + with pytest.raises(Exception) as e: + connect.has_collection(collection_name) + + def test_drop_collection_with_none_collection_name(self, connect): + collection_name = None + with pytest.raises(Exception) as e: + connect.has_collection(collection_name) diff --git a/tests/python_test/collection/test_has_collection.py b/tests/python_test/collection/test_has_collection.py new file mode 100644 index 0000000000..e85dc5b1a2 --- /dev/null +++ b/tests/python_test/collection/test_has_collection.py @@ -0,0 +1,101 @@ +import pdb +import pytest +import logging +import itertools +import threading +import time +from multiprocessing import Process +from utils import * +from constants import * + +uid = "has_collection" + +class TestHasCollection: + """ + ****************************************************************** + The following cases are used to test `has_collection` function + ****************************************************************** + """ + def test_has_collection(self, connect, collection): + ''' + target: test if the created collection existed + method: create collection, assert the value returned by has_collection method + expected: True + ''' + assert connect.has_collection(collection) + + @pytest.mark.level(2) + def test_has_collection_without_connection(self, collection, dis_connect): + ''' + target: test has collection, without connection + method: calling has collection with correct params, with a disconnected instance + expected: has collection raise exception + ''' + with pytest.raises(Exception) as e: + assert dis_connect.has_collection(collection) + + def test_has_collection_not_existed(self, connect): + ''' + target: test if collection not created + method: random a collection name, create this collection then drop it, + assert the value returned by has_collection method + expected: False + ''' + collection_name = gen_unique_str(uid) + connect.create_collection(collection_name, default_fields) + assert connect.has_collection(collection_name) + connect.drop_collection(collection_name) + assert not connect.has_collection(collection_name) + + @pytest.mark.level(2) + def test_has_collection_multithread(self, connect): + ''' + target: test create collection with multithread + method: create collection using multithread, + expected: collections are created + ''' + threads_num = 4 + threads = [] + collection_name = gen_unique_str(uid) + connect.create_collection(collection_name, default_fields) + + def has(): + assert connect.has_collection(collection_name) + # assert not assert_collection(connect, collection_name) + for i in range(threads_num): + t = TestThread(target=has, args=()) + threads.append(t) + t.start() + time.sleep(0.2) + for t in threads: + t.join() + + +class TestHasCollectionInvalid(object): + """ + Test has collection with invalid params + """ + @pytest.fixture( + scope="function", + params=gen_invalid_strs() + ) + def get_collection_name(self, request): + yield request.param + + @pytest.mark.level(2) + def test_has_collection_with_invalid_collection_name(self, connect, get_collection_name): + collection_name = get_collection_name + with pytest.raises(Exception) as e: + connect.has_collection(collection_name) + + @pytest.mark.level(2) + def test_has_collection_with_empty_collection_name(self, connect): + collection_name = '' + with pytest.raises(Exception) as e: + connect.has_collection(collection_name) + + @pytest.mark.level(2) + def test_has_collection_with_none_collection_name(self, connect): + collection_name = None + with pytest.raises(Exception) as e: + connect.has_collection(collection_name) diff --git a/tests/python_test/collection/test_list_collections.py b/tests/python_test/collection/test_list_collections.py new file mode 100644 index 0000000000..084605f624 --- /dev/null +++ b/tests/python_test/collection/test_list_collections.py @@ -0,0 +1,94 @@ +import pytest +import time +from utils import * +from constants import * + +uid = "list_collections" + + +class TestListCollections: + """ + ****************************************************************** + The following cases are used to test `list_collections` function + ****************************************************************** + """ + + def test_list_collections(self, connect, collection): + ''' + target: test list collections + method: create collection, assert the value returned by list_collections method + expected: True + ''' + assert collection in connect.list_collections() + + def test_list_collections_multi_collections(self, connect): + ''' + target: test list collections + method: create collection, assert the value returned by list_collections method + expected: True + ''' + collection_num = 50 + for i in range(collection_num): + collection_name = gen_unique_str(uid) + connect.create_collection(collection_name, default_fields) + assert collection_name in connect.list_collections() + + @pytest.mark.level(2) + def test_list_collections_without_connection(self, dis_connect): + ''' + target: test list collections, without connection + method: calling list collections with correct params, with a disconnected instance + expected: list collections raise exception + ''' + with pytest.raises(Exception) as e: + dis_connect.list_collections() + + def test_list_collections_not_existed(self, connect): + ''' + target: test if collection not created + method: random a collection name, create this collection then drop it, + assert the value returned by list_collections method + expected: False + ''' + collection_name = gen_unique_str(uid) + connect.create_collection(collection_name, default_fields) + assert collection_name in connect.list_collections() + connect.drop_collection(collection_name) + assert collection_name not in connect.list_collections() + + # TODO: make sure to run this case in the end + @pytest.mark.level(2) + def test_list_collections_no_collection(self, connect): + ''' + target: test show collections is correct or not, if no collection in db + method: delete all collections, + assert the value returned by list_collections method is equal to [] + expected: the status is ok, and the result is equal to [] + ''' + result = connect.list_collections() + if result: + for collection_name in result: + assert connect.has_collection(collection_name) + + @pytest.mark.level(2) + def test_list_collections_multithread(self, connect): + ''' + target: test list collection with multithread + method: list collection using multithread, + expected: list collections correctly + ''' + threads_num = 10 + threads = [] + collection_name = gen_unique_str(uid) + connect.create_collection(collection_name, default_fields) + + def _list(): + assert collection_name in connect.list_collections() + + for i in range(threads_num): + t = TestThread(target=_list) + threads.append(t) + t.start() + time.sleep(0.2) + for t in threads: + t.join() diff --git a/tests/python_test/collection/test_load_collection.py b/tests/python_test/collection/test_load_collection.py new file mode 100644 index 0000000000..7bb7f17575 --- /dev/null +++ b/tests/python_test/collection/test_load_collection.py @@ -0,0 +1,573 @@ +import pdb +import pytest +import logging +import itertools +from time import sleep +import threading +from multiprocessing import Process +from utils import * +from constants import * + +uid = "load_collection" +field_name = default_float_vec_field_name +default_single_query = { + "bool": { + "must": [ + {"vector": {field_name: {"topk": default_top_k, "query": gen_vectors(1, default_dim), "metric_type": "L2", + "params": {"nprobe": 10}}}} + ] + } +} + + +class TestLoadCollection: + """ + ****************************************************************** + The following cases are used to test `load_collection` function + ****************************************************************** + """ + @pytest.fixture( + scope="function", + params=gen_simple_index() + ) + def get_simple_index(self, request, connect): + if str(connect._cmd("mode")) == "CPU": + if request.param["index_type"] in index_cpu_not_support(): + pytest.skip("sq8h not support in cpu mode") + return request.param + + @pytest.fixture( + scope="function", + params=gen_binary_index() + ) + def get_binary_index(self, request, connect): + logging.getLogger().info(request.param) + if request.param["index_type"] in binary_support(): + return request.param + else: + pytest.skip("Skip index Temporary") + + def test_load_collection_after_index(self, connect, collection, get_simple_index): + ''' + target: test load collection, after index created + method: insert and create index, load collection with correct params + expected: no error raised + ''' + connect.insert(collection, default_entities) + connect.flush([collection]) + logging.getLogger().info(get_simple_index) + connect.create_index(collection, default_float_vec_field_name, get_simple_index) + connect.load_collection(collection) + connect.release_collection(collection) + + @pytest.mark.level(2) + def test_load_collection_after_index_binary(self, connect, binary_collection, get_binary_index): + ''' + target: test load binary_collection, after index created + method: insert and create index, load binary_collection with correct params + expected: no error raised + ''' + connect.insert(binary_collection, default_binary_entities) + connect.flush([binary_collection]) + for metric_type in binary_metrics(): + logging.getLogger().info(metric_type) + get_binary_index["metric_type"] = metric_type + if get_binary_index["index_type"] == "BIN_IVF_FLAT" and metric_type in structure_metrics(): + with pytest.raises(Exception) as e: + connect.create_index(binary_collection, default_binary_vec_field_name, get_binary_index) + else: + connect.create_index(binary_collection, default_binary_vec_field_name, get_binary_index) + connect.load_collection(binary_collection) + connect.release_collection(binary_collection) + + def test_load_empty_collection(self, connect, collection): + ''' + target: test load collection + method: no entities in collection, load collection with correct params + expected: load success + ''' + connect.load_collection(collection) + connect.release_collection(collection) + + @pytest.mark.level(2) + def test_load_collection_dis_connect(self, dis_connect, collection): + ''' + target: test load collection, without connection + method: load collection with correct params, with a disconnected instance + expected: load raise exception + ''' + with pytest.raises(Exception) as e: + dis_connect.load_collection(collection) + + @pytest.mark.level(2) + def test_release_collection_dis_connect(self, dis_connect, collection): + ''' + target: test release collection, without connection + method: release collection with correct params, with a disconnected instance + expected: release raise exception + ''' + with pytest.raises(Exception) as e: + dis_connect.release_collection(collection) + + @pytest.mark.level(2) + def test_load_collection_not_existed(self, connect, collection): + collection_name = gen_unique_str(uid) + with pytest.raises(Exception) as e: + connect.load_collection(collection_name) + + @pytest.mark.level(2) + def test_release_collection_not_existed(self, connect, collection): + collection_name = gen_unique_str(uid) + with pytest.raises(Exception) as e: + connect.release_collection(collection_name) + + def test_release_collection_not_load(self, connect, collection): + """ + target: test release collection without load + method: + expected: raise exception + """ + ids = connect.insert(collection, default_entities) + assert len(ids) == default_nb + connect.flush([collection]) + with pytest.raises(Exception) as e: + connect.release_collection(collection) + + @pytest.mark.level(2) + def test_load_release_collection(self, connect, collection): + collection_name = gen_unique_str(uid) + connect.create_collection(collection_name, default_fields) + connect.insert(collection_name, default_entities) + connect.flush([collection_name]) + connect.load_collection(collection_name) + connect.release_collection(collection_name) + connect.drop_collection(collection_name) + with pytest.raises(Exception) as e: + connect.load_collection(collection_name) + with pytest.raises(Exception) as e: + connect.release_collection(collection_name) + + def test_release_collection_after_drop(self, connect, collection): + """ + target: test release collection after drop + method: insert and flush, then release collection after load and drop + expected: raise exception + """ + ids = connect.insert(collection, default_entities) + assert len(ids) == default_nb + connect.flush([collection]) + connect.load_collection(collection) + connect.drop_collection(collection) + with pytest.raises(Exception) as e: + connect.release_collection(collection) + + # TODO + def test_load_collection_without_flush(self, connect, collection): + """ + target: test load collection without flush + method: insert entities without flush, then load collection + expected: load collection failed + """ + ids = connect.insert(collection, default_entities) + assert len(ids) == default_nb + with pytest.raises(Exception) as e: + connect.load_collection(collection) + + # TODO + def _test_load_collection_larger_than_memory(self): + """ + target: test load collection when memory less than collection size + method: i don't know + expected: raise exception + """ + + def test_load_collection_release_part_partitions(self, connect, collection): + """ + target: test release part partitions after load collection + method: load collection and release part partitions + expected: released partitions search empty + """ + ids = connect.insert(collection, default_entities) + assert len(ids) == default_nb + connect.create_partition(collection, default_tag) + ids = connect.insert(collection, default_entities, partition_tag=default_tag) + assert len(ids) == default_nb + connect.flush([collection]) + connect.load_collection(collection) + connect.release_partitions(collection, [default_tag]) + res = connect.search(collection, default_single_query, partition_tags=[default_tag]) + assert len(res[0]) == 0 + res = connect.search(collection, default_single_query, partition_tags=[default_partition_name]) + assert len(res[0]) == default_top_k + + def test_load_collection_release_part_partitions(self, connect, collection): + """ + target: test release all partitions after load collection + method: load collection and release all partitions + expected: search empty + """ + ids = connect.insert(collection, default_entities) + assert len(ids) == default_nb + connect.create_partition(collection, default_tag) + ids = connect.insert(collection, default_entities, partition_tag=default_tag) + assert len(ids) == default_nb + connect.flush([collection]) + connect.load_collection(collection) + connect.release_partitions(collection, [default_partition_name, default_tag]) + res = connect.search(collection, default_single_query) + assert len(res[0]) == 0 + + def test_load_partitions_release_collection(self, connect, collection): + """ + target: test release collection after load partitions + method: insert entities into partitions, search empty after load partitions and release collection + expected: search result empty + """ + connect.create_partition(collection, default_tag) + ids = connect.insert(collection, default_entities, partition_tag=default_tag) + assert len(ids) == default_nb + connect.flush([collection]) + connect.load_collection(collection) + connect.release_partitions(collection, [default_tag]) + res = connect.search(collection, default_single_query) + assert len(res[0]) == 0 + + +class TestReleaseAdvanced: + + def _test_release_collection_during_searching(self, connect, collection): + """ + target: test release collection during searching + method: insert entities into collection, flush and load collection, release collection during searching + expected: + """ + nq = 1000 + top_k = 1 + connect.insert(collection, default_entities) + connect.flush([collection]) + connect.load_collection(collection) + query, _ = gen_query_vectors(field_name, default_entities, top_k, nq) + res = connect.search(collection, query, _async=True) + connect.release_collection(collection) + res = connect.search(collection, default_single_query) + assert len(res[0]) == 0 + + def _test_release_partition_during_searching(self, connect, collection): + """ + target: test release partition during searching + method: insert entities into partition, flush and load partition, release partition during searching + expected: + """ + nq = 1000 + top_k = 1 + connect.create_partition(collection, default_tag) + query, _ = gen_query_vectors(field_name, default_entities, top_k, nq) + connect.insert(collection, default_entities, partition_tag=default_tag) + connect.flush([collection]) + connect.load_partitions(collection, [default_tag]) + res = connect.search(collection, query, _async=True) + connect.release_partitions(collection, [default_tag]) + res = connect.search(collection, default_single_query) + assert len(res[0]) == 0 + + def _test_release_collection_during_searching_A(self, connect, collection): + """ + target: test release collection during searching + method: insert entities into partition, flush and load partition, release collection during searching + expected: + """ + nq = 1000 + top_k = 1 + connect.create_partition(collection, default_tag) + query, _ = gen_query_vectors(field_name, default_entities, top_k, nq) + connect.insert(collection, default_entities, partition_tag=default_tag) + connect.flush([collection]) + connect.load_partitions(collection, [default_tag]) + res = connect.search(collection, query, _async=True) + connect.release_collection(collection) + res = connect.search(collection, default_single_query) + assert len(res[0]) == 0 + + def _test_release_collection_during_loading(self, connect, collection): + """ + target: test release collection during loading + method: insert entities into collection, flush, release collection during loading + expected: + """ + connect.insert(collection, default_entities) + connect.flush([collection]) + + def load(collection): + connect.load_collection(collection) + t = threading.Thread(target=load, (collection, )) + t.start() + connect.release_collection(collection) + res = connect.search(collection, default_single_query) + assert len(res[0]) == 0 + + def _test_release_partition_during_loading(self, connect, collection): + """ + target: test release partition during loading + method: insert entities into partition, flush, release partition during loading + expected: + """ + connect.create_partition(collection, default_tag) + connect.insert(collection, default_entities, partition_tag=default_tag) + connect.flush([collection]) + + def load(collection): + connect.load_collection(collection) + t = threading.Thread(target=load, (collection, )) + t.start() + connect.release_partitions(collection, [default_tag]) + res = connect.search(collection, default_single_query) + assert len(res[0]) == 0 + + def _test_release_collection_during_inserting(self, connect, collection): + """ + target: test release collection during inserting + method: load collection, do release collection during inserting + expected: + """ + connect.insert(collection, default_entities) + connect.flush([collection]) + connect.load_collection(collection) + + def insert(collection): + connect.insert(collection, default_entities) + t = threading.Thread(target=insert, (collection, )) + t.start() + connect.release_collection(collection) + res = connect.search(collection, default_single_query) + assert len(res[0]) == 0 + + def _test_release_collection_during_indexing(self, connect, collection): + """ + target: test release collection during building index + method: insert and flush, load collection, do release collection during creating index + expected: + """ + pass + + def _test_release_collection_during_droping_index(self, connect, collection): + """ + target: test release collection during droping index + method: insert, create index and flush, load collection, do release collection during droping index + expected: + """ + pass + + +class TestLoadCollectionInvalid(object): + """ + Test load collection with invalid params + """ + @pytest.fixture( + scope="function", + params=gen_invalid_strs() + ) + def get_collection_name(self, request): + yield request.param + + @pytest.mark.level(2) + def test_load_collection_with_invalid_collection_name(self, connect, get_collection_name): + collection_name = get_collection_name + with pytest.raises(Exception) as e: + connect.load_collection(collection_name) + + @pytest.mark.level(2) + def test_release_collection_with_invalid_collection_name(self, connect, get_collection_name): + collection_name = get_collection_name + with pytest.raises(Exception) as e: + connect.release_collection(collection_name) + + +class TestLoadPartition: + """ + ****************************************************************** + The following cases are used to test `load_collection` function + ****************************************************************** + """ + @pytest.fixture( + scope="function", + params=gen_simple_index() + ) + def get_simple_index(self, request, connect): + if str(connect._cmd("mode")) == "CPU": + if request.param["index_type"] in index_cpu_not_support(): + pytest.skip("sq8h not support in cpu mode") + return request.param + + @pytest.fixture( + scope="function", + params=gen_binary_index() + ) + def get_binary_index(self, request, connect): + logging.getLogger().info(request.param) + if request.param["index_type"] in binary_support(): + return request.param + else: + pytest.skip("Skip index Temporary") + + def test_load_partition_after_index(self, connect, collection, get_simple_index): + ''' + target: test load collection, after index created + method: insert and create index, load collection with correct params + expected: no error raised + ''' + connect.create_partition(collection, default_tag) + ids = connect.insert(collection, default_entities, partition_tag=default_tag) + assert len(ids) == default_nb + connect.flush([collection]) + connect.create_index(collection, default_float_vec_field_name, get_simple_index) + connect.load_partitions(collection, [default_tag]) + res = connect.search(collection, default_single_query) + assert len(res[0]) == default_top_k + + @pytest.mark.level(2) + def test_load_partition_after_index_binary(self, connect, binary_collection, get_binary_index): + ''' + target: test load binary_collection, after index created + method: insert and create index, load binary_collection with correct params + expected: no error raised + ''' + connect.create_partition(binary_collection, default_tag) + ids = connect.insert(binary_collection, default_binary_entities, partition_tag=[default_tag]) + assert len(ids) == default_nb + connect.flush([binary_collection]) + for metric_type in binary_metrics(): + logging.getLogger().info(metric_type) + get_binary_index["metric_type"] = metric_type + if get_binary_index["index_type"] == "BIN_IVF_FLAT" and metric_type in structure_metrics(): + with pytest.raises(Exception) as e: + connect.create_index(binary_collection, default_binary_vec_field_name, get_binary_index) + else: + connect.create_index(binary_collection, default_binary_vec_field_name, get_binary_index) + connect.load_partitions(binary_collection, [default_tag]) + + def test_load_empty_partition(self, connect, collection): + ''' + target: test load collection + method: no entities in collection, load collection with correct params + expected: load success + ''' + connect.create_partition(collection, default_tag) + connect.load_partitions(collection, [default_tag]) + res = connect.search(collection, default_single_query) + assert len(res[0]) == 0 + + @pytest.mark.level(2) + def test_load_collection_dis_connect(self, connect, dis_connect, collection): + ''' + target: test load collection, without connection + method: load collection with correct params, with a disconnected instance + expected: load raise exception + ''' + connect.create_partition(collection, default_tag) + with pytest.raises(Exception) as e: + dis_connect.load_partitions(collection, [default_tag]) + + @pytest.mark.level(2) + def test_release_partition_dis_connect(self, connect, dis_connect, collection): + ''' + target: test release collection, without connection + method: release collection with correct params, with a disconnected instance + expected: release raise exception + ''' + connect.create_partition(collection, default_tag) + connect.load_partitions(collection, [default_tag]) + with pytest.raises(Exception) as e: + dis_connect.release_partitions(collection, [default_tag]) + + @pytest.mark.level(2) + def test_load_partition_not_existed(self, connect, collection): + partition_name = gen_unique_str(uid) + with pytest.raises(Exception) as e: + connect.load_partitions(collection, [partition_name]) + + @pytest.mark.level(2) + def test_release_partition_not_existed(self, connect, collection): + partition_name = gen_unique_str(uid) + with pytest.raises(Exception) as e: + connect.release_partitions(collection, [partition_name]) + + def test_release_partition_not_load(self, connect, collection): + """ + target: test release collection without load + method: + expected: raise exception + """ + connect.create_partition(collection, default_tag) + ids = connect.insert(collection, default_entities, partition_tag=[default_tag]) + assert len(ids) == default_nb + connect.flush([collection]) + with pytest.raises(Exception) as e: + connect.release_partitions(collection, [default_tag]) + + @pytest.mark.level(2) + def test_load_release_after_drop(self, connect, collection): + connect.create_partition(collection, default_tag) + ids = connect.insert(collection, default_entities, partition_tag=[default_tag]) + connect.flush([collection_name]) + connect.load_partitions(collection, [default_tag]) + connect.release_partitions(collection, [default_tag]) + connect.drop_partition(collection, default_tag) + with pytest.raises(Exception) as e: + connect.load_partitions(collection, [default_tag]) + with pytest.raises(Exception) as e: + connect.release_partitions(collection, [default_tag]) + + def test_release_partition_after_drop(self, connect, collection): + """ + target: test release collection after drop + method: insert and flush, then release collection after load and drop + expected: raise exception + """ + connect.create_partition(collection, default_tag) + ids = connect.insert(collection, default_entities, partition_tag=[default_tag]) + connect.flush([collection_name]) + connect.load_partitions(collection, [default_tag]) + connect.drop_partition(collection, default_tag) + with pytest.raises(Exception) as e: + connect.release_partitions(collection, [default_tag]) + + def test_load_release_after_collection_drop(self, connect, collection): + """ + target: test release collection after drop + method: insert and flush, then release collection after load and drop + expected: raise exception + """ + connect.create_partition(collection, default_tag) + ids = connect.insert(collection, default_entities, partition_tag=[default_tag]) + connect.flush([collection_name]) + connect.load_partitions(collection, [default_tag]) + connect.release_partitions(collection, [default_tag]) + connect.drop_collection(collection) + with pytest.raises(Exception) as e: + connect.load_partitions(collection, [default_tag]) + with pytest.raises(Exception) as e: + connect.release_partitions(collection, [default_tag]) + + +class TestLoadPartitionInvalid(object): + """ + Test load collection with invalid params + """ + @pytest.fixture( + scope="function", + params=gen_invalid_strs() + ) + def get_partition_name(self, request): + yield request.param + + @pytest.mark.level(2) + def test_load_partition_with_invalid_partition_name(self, connect, collection, get_partition_name): + partition_name = get_partition_name + with pytest.raises(Exception) as e: + connect.load_partitions(collection, [partition_name]) + + @pytest.mark.level(2) + def test_release_partition_with_invalid_partition_name(self, connect, collection, get_partition_name): + partition_name = get_partition_name + with pytest.raises(Exception) as e: + connect.load_partitions(collection, [partition_name]) diff --git a/tests/python_test/conftest.py b/tests/python_test/conftest.py new file mode 100644 index 0000000000..456c3dd7a0 --- /dev/null +++ b/tests/python_test/conftest.py @@ -0,0 +1,194 @@ +import pdb +import logging +import socket +import pytest +from utils import gen_unique_str +from milvus import Milvus, DataType +from utils import * + +timeout = 60 +dimension = 128 +delete_timeout = 60 + + +def pytest_addoption(parser): + parser.addoption("--ip", action="store", default="localhost") + parser.addoption("--service", action="store", default="") + parser.addoption("--port", action="store", default=19530) + parser.addoption("--http-port", action="store", default=19121) + parser.addoption("--handler", action="store", default="GRPC") + parser.addoption("--tag", action="store", default="all", help="only run tests matching the tag.") + parser.addoption('--dry-run', action='store_true', default=False) + + +def pytest_configure(config): + # register an additional marker + config.addinivalue_line( + "markers", "tag(name): mark test to run only matching the tag" + ) + + +def pytest_runtest_setup(item): + tags = list() + for marker in item.iter_markers(name="tag"): + for tag in marker.args: + tags.append(tag) + if tags: + cmd_tag = item.config.getoption("--tag") + if cmd_tag != "all" and cmd_tag not in tags: + pytest.skip("test requires tag in {!r}".format(tags)) + + +def pytest_runtestloop(session): + if session.config.getoption('--dry-run'): + for item in session.items: + print(item.nodeid) + return True + + +def check_server_connection(request): + ip = request.config.getoption("--ip") + port = request.config.getoption("--port") + + connected = True + if ip and (ip not in ['localhost', '127.0.0.1']): + try: + socket.getaddrinfo(ip, port, 0, 0, socket.IPPROTO_TCP) + except Exception as e: + print("Socket connnet failed: %s" % str(e)) + connected = False + return connected + + +@pytest.fixture(scope="module") +def connect(request): + ip = request.config.getoption("--ip") + service_name = request.config.getoption("--service") + port = request.config.getoption("--port") + http_port = request.config.getoption("--http-port") + handler = request.config.getoption("--handler") + if handler == "HTTP": + port = http_port + try: + milvus = get_milvus(host=ip, port=port, handler=handler) + # reset_build_index_threshold(milvus) + except Exception as e: + logging.getLogger().error(str(e)) + pytest.exit("Milvus server can not connected, exit pytest ...") + def fin(): + try: + milvus.close() + pass + except Exception as e: + logging.getLogger().info(str(e)) + request.addfinalizer(fin) + return milvus + + +@pytest.fixture(scope="module") +def dis_connect(request): + ip = request.config.getoption("--ip") + service_name = request.config.getoption("--service") + port = request.config.getoption("--port") + http_port = request.config.getoption("--http-port") + handler = request.config.getoption("--handler") + if handler == "HTTP": + port = http_port + milvus = get_milvus(host=ip, port=port, handler=handler) + milvus.close() + return milvus + + +@pytest.fixture(scope="module") +def args(request): + ip = request.config.getoption("--ip") + service_name = request.config.getoption("--service") + port = request.config.getoption("--port") + http_port = request.config.getoption("--http-port") + handler = request.config.getoption("--handler") + if handler == "HTTP": + port = http_port + args = {"ip": ip, "port": port, "handler": handler, "service_name": service_name} + return args + + +@pytest.fixture(scope="module") +def milvus(request): + ip = request.config.getoption("--ip") + port = request.config.getoption("--port") + http_port = request.config.getoption("--http-port") + handler = request.config.getoption("--handler") + if handler == "HTTP": + port = http_port + return get_milvus(host=ip, port=port, handler=handler) + + +@pytest.fixture(scope="function") +def collection(request, connect): + ori_collection_name = getattr(request.module, "collection_id", "test") + collection_name = gen_unique_str(ori_collection_name) + try: + default_fields = gen_default_fields() + connect.create_collection(collection_name, default_fields) + except Exception as e: + pytest.exit(str(e)) + def teardown(): + if connect.has_collection(collection_name): + connect.drop_collection(collection_name, timeout=delete_timeout) + request.addfinalizer(teardown) + assert connect.has_collection(collection_name) + return collection_name + + +# customised id +@pytest.fixture(scope="function") +def id_collection(request, connect): + ori_collection_name = getattr(request.module, "collection_id", "test") + collection_name = gen_unique_str(ori_collection_name) + try: + fields = gen_default_fields(auto_id=False) + connect.create_collection(collection_name, fields) + except Exception as e: + pytest.exit(str(e)) + def teardown(): + if connect.has_collection(collection_name): + connect.drop_collection(collection_name, timeout=delete_timeout) + request.addfinalizer(teardown) + assert connect.has_collection(collection_name) + return collection_name + + +@pytest.fixture(scope="function") +def binary_collection(request, connect): + ori_collection_name = getattr(request.module, "collection_id", "test") + collection_name = gen_unique_str(ori_collection_name) + try: + fields = gen_binary_default_fields() + connect.create_collection(collection_name, fields) + except Exception as e: + pytest.exit(str(e)) + def teardown(): + collection_names = connect.list_collections() + if connect.has_collection(collection_name): + connect.drop_collection(collection_name, timeout=delete_timeout) + request.addfinalizer(teardown) + assert connect.has_collection(collection_name) + return collection_name + + +# customised id +@pytest.fixture(scope="function") +def binary_id_collection(request, connect): + ori_collection_name = getattr(request.module, "collection_id", "test") + collection_name = gen_unique_str(ori_collection_name) + try: + fields = gen_binary_default_fields(auto_id=False) + connect.create_collection(collection_name, fields) + except Exception as e: + pytest.exit(str(e)) + def teardown(): + if connect.has_collection(collection_name): + connect.drop_collection(collection_name, timeout=delete_timeout) + request.addfinalizer(teardown) + assert connect.has_collection(collection_name) + return collection_name diff --git a/tests/python_test/constants.py b/tests/python_test/constants.py new file mode 100644 index 0000000000..d3eae745f7 --- /dev/null +++ b/tests/python_test/constants.py @@ -0,0 +1,22 @@ +import utils + +default_fields = utils.gen_default_fields() +default_binary_fields = utils.gen_binary_default_fields() + +default_entity = utils.gen_entities(1) +default_raw_binary_vector, default_binary_entity = utils.gen_binary_entities(1) + +default_entity_row = utils.gen_entities_rows(1) +default_raw_binary_vector_row, default_binary_entity_row = utils.gen_binary_entities_rows(1) + + +default_entities = utils.gen_entities(utils.default_nb) +default_raw_binary_vectors, default_binary_entities = utils.gen_binary_entities(utils.default_nb) + + +default_entities_new = utils.gen_entities_new(utils.default_nb) +default_raw_binary_vectors_new, default_binary_entities_new = utils.gen_binary_entities_new(utils.default_nb) + + +default_entities_rows = utils.gen_entities_rows(utils.default_nb) +default_raw_binary_vectors_rows, default_binary_entities_rows = utils.gen_binary_entities_rows(utils.default_nb) \ No newline at end of file diff --git a/tests/python_test/docker-entrypoint.sh b/tests/python_test/docker-entrypoint.sh new file mode 100755 index 0000000000..af9ba0ba66 --- /dev/null +++ b/tests/python_test/docker-entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -e + +if [ "$1" = 'start' ]; then + tail -f /dev/null +fi + +exec "$@" \ No newline at end of file diff --git a/tests/python_test/entity/test_delete.py b/tests/python_test/entity/test_delete.py new file mode 100644 index 0000000000..1cc78e4028 --- /dev/null +++ b/tests/python_test/entity/test_delete.py @@ -0,0 +1,473 @@ +import time +import random +import pdb +import copy +import threading +import logging +from multiprocessing import Pool, Process +import pytest +from utils import * +from constants import * + +field_name = default_float_vec_field_name +default_single_query = { + "bool": { + "must": [ + {"vector": {field_name: {"topk": 10, "metric_type":"L2", "query": gen_vectors(1, default_dim), "params": {"nprobe": 10}}}} + ] + } +} + + +class TestDeleteBase: + """ + ****************************************************************** + The following cases are used to test `delete_entity_by_id` function + ****************************************************************** + """ + + @pytest.fixture( + scope="function", + params=gen_simple_index() + ) + def get_simple_index(self, request, connect): + if str(connect._cmd("mode")) == "GPU": + if not request.param["index_type"] not in ivf(): + pytest.skip("Only support index_type: idmap/ivf") + if str(connect._cmd("mode")) == "CPU": + if request.param["index_type"] in index_cpu_not_support(): + pytest.skip("CPU not support index_type: ivf_sq8h") + return request.param + + @pytest.fixture( + scope="function", + params=[ + 1, + 2000 + ], + ) + def insert_count(self, request): + yield request.param + + def test_delete_entity_id_not_exised(self, connect, collection): + ''' + target: test delete entity, params entity_id not existed + method: add entity and delete + expected: status DELETED + ''' + ids = connect.bulk_insert(collection, default_entity) + connect.flush([collection]) + status = connect.delete_entity_by_id(collection, [0]) + assert status + + def test_delete_empty_collection(self, connect, collection): + ''' + target: test delete entity, params collection_name not existed + method: add entity and delete + expected: status DELETED + ''' + status = connect.delete_entity_by_id(collection, [0]) + assert status + + def test_delete_entity_collection_not_existed(self, connect, collection): + ''' + target: test delete entity, params collection_name not existed + method: add entity and delete + expected: error raised + ''' + collection_new = gen_unique_str() + with pytest.raises(Exception) as e: + status = connect.delete_entity_by_id(collection_new, [0]) + + def test_delete_entity_collection_not_existed(self, connect, collection): + ''' + target: test delete entity, params collection_name not existed + method: add entity and delete + expected: error raised + ''' + ids = connect.bulk_insert(collection, default_entity) + connect.flush([collection]) + collection_new = gen_unique_str() + with pytest.raises(Exception) as e: + status = connect.delete_entity_by_id(collection_new, [0]) + + def test_insert_delete(self, connect, collection, insert_count): + ''' + target: test delete entity + method: add entities and delete + expected: no error raised + ''' + entities = gen_entities(insert_count) + ids = connect.bulk_insert(collection, entities) + connect.flush([collection]) + delete_ids = [ids[0]] + status = connect.delete_entity_by_id(collection, delete_ids) + assert status + connect.flush([collection]) + res_count = connect.count_entities(collection) + assert res_count == insert_count - 1 + + def test_insert_delete_A(self, connect, collection): + ''' + target: test delete entity + method: add entities and delete one in collection, and one not in collection + expected: no error raised + ''' + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + delete_ids = [ids[0], 1] + status = connect.delete_entity_by_id(collection, delete_ids) + assert status + connect.flush([collection]) + res_count = connect.count_entities(collection) + assert res_count == default_nb - 1 + + def test_insert_delete_B(self, connect, id_collection): + ''' + target: test delete entity + method: add entities with the same ids, and delete the id in collection + expected: no error raised, all entities deleted + ''' + ids = [1 for i in range(default_nb)] + res_ids = connect.bulk_insert(id_collection, default_entities, ids) + connect.flush([id_collection]) + delete_ids = [1] + status = connect.delete_entity_by_id(id_collection, delete_ids) + assert status + connect.flush([id_collection]) + res_count = connect.count_entities(id_collection) + assert res_count == 0 + + def test_delete_exceed_limit(self, connect, collection): + ''' + target: test delete entity + method: add one entity and delete two ids + expected: error raised + ''' + ids = connect.bulk_insert(collection, default_entity) + connect.flush([collection]) + delete_ids = [ids[0], 1] + status = connect.delete_entity_by_id(collection, delete_ids) + connect.flush([collection]) + res_count = connect.count_entities(collection) + assert res_count == 0 + + def test_flush_after_delete(self, connect, collection): + ''' + target: test delete entity + method: add entities and delete, then flush + expected: entity deleted and no error raised + ''' + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + delete_ids = [ids[0], ids[-1]] + status = connect.delete_entity_by_id(collection, delete_ids) + assert status + connect.flush([collection]) + res_count = connect.count_entities(collection) + assert res_count == default_nb - len(delete_ids) + + def test_flush_after_delete_binary(self, connect, binary_collection): + ''' + target: test delete entity + method: add entities and delete, then flush + expected: entity deleted and no error raised + ''' + ids = connect.bulk_insert(binary_collection, default_binary_entities) + connect.flush([binary_collection]) + delete_ids = [ids[0], ids[-1]] + status = connect.delete_entity_by_id(binary_collection, delete_ids) + assert status + connect.flush([binary_collection]) + res_count = connect.count_entities(binary_collection) + assert res_count == default_nb - len(delete_ids) + + def test_insert_delete_binary(self, connect, binary_collection): + ''' + method: add entities and delete + expected: status DELETED + ''' + ids = connect.bulk_insert(binary_collection, default_binary_entities) + connect.flush([binary_collection]) + delete_ids = [ids[0], ids[-1]] + status = connect.delete_entity_by_id(binary_collection, delete_ids) + + def test_insert_same_ids_after_delete(self, connect, id_collection): + ''' + method: add entities and delete + expected: status DELETED + note: Not flush after delete + ''' + insert_ids = [i for i in range(default_nb)] + ids = connect.bulk_insert(id_collection, default_entities, insert_ids) + connect.flush([id_collection]) + delete_ids = [ids[0], ids[-1]] + status = connect.delete_entity_by_id(id_collection, delete_ids) + assert status + new_ids = connect.bulk_insert(id_collection, default_entity, [ids[0]]) + assert new_ids == [ids[0]] + connect.flush([id_collection]) + res_count = connect.count_entities(id_collection) + assert res_count == default_nb - 1 + + def test_insert_same_ids_after_delete_binary(self, connect, binary_id_collection): + ''' + method: add entities, with the same id and delete the ids + expected: status DELETED, all id deleted + ''' + insert_ids = [i for i in range(default_nb)] + ids = connect.bulk_insert(binary_id_collection, default_binary_entities, insert_ids) + connect.flush([binary_id_collection]) + delete_ids = [ids[0], ids[-1]] + status = connect.delete_entity_by_id(binary_id_collection, delete_ids) + assert status + new_ids = connect.bulk_insert(binary_id_collection, default_binary_entity, [ids[0]]) + assert new_ids == [ids[0]] + connect.flush([binary_id_collection]) + res_count = connect.count_entities(binary_id_collection) + assert res_count == default_nb - 1 + + def test_search_after_delete(self, connect, collection): + ''' + target: test delete entity + method: add entities and delete, then search + expected: entity deleted and no error raised + ''' + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + delete_ids = [ids[0], ids[-1]] + status = connect.delete_entity_by_id(collection, delete_ids) + assert status + connect.flush([collection]) + query = copy.deepcopy(default_single_query) + query["bool"]["must"][0]["vector"][field_name]["query"] =\ + [default_entity[-1]["values"][0], default_entities[-1]["values"][1], default_entities[-1]["values"][-1]] + res = connect.search(collection, query) + logging.getLogger().debug(res) + assert len(res) == len(query["bool"]["must"][0]["vector"][field_name]["query"]) + assert res[0]._distances[0] > epsilon + assert res[1]._distances[0] < epsilon + assert res[2]._distances[0] > epsilon + + def test_create_index_after_delete(self, connect, collection, get_simple_index): + ''' + method: add entitys and delete, then create index + expected: vectors deleted, index created + ''' + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + delete_ids = [ids[0], ids[-1]] + status = connect.delete_entity_by_id(collection, delete_ids) + connect.create_index(collection, field_name, get_simple_index) + # assert index info + + def test_delete_multiable_times(self, connect, collection): + ''' + method: add entities and delete id serveral times + expected: entities deleted + ''' + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + delete_ids = [ids[0], ids[-1]] + status = connect.delete_entity_by_id(collection, delete_ids) + assert status + connect.flush([collection]) + for i in range(10): + status = connect.delete_entity_by_id(collection, delete_ids) + assert status + + def test_index_insert_batch_delete_get(self, connect, collection, get_simple_index): + ''' + method: create index, insert entities, and delete + expected: entities deleted + ''' + connect.create_index(collection, field_name, get_simple_index) + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + delete_ids = [ids[0], ids[-1]] + status = connect.delete_entity_by_id(collection, delete_ids) + assert status + connect.flush([collection]) + res_count = connect.count_entities(collection) + assert res_count == default_nb - len(delete_ids) + res_get = connect.get_entity_by_id(collection, delete_ids) + assert res_get[0] is None + + # TODO: disable + @pytest.mark.level(2) + def _test_index_insert_single_delete_get(self, connect, id_collection): + ''' + method: insert entities, and delete + expected: entities deleted + ''' + ids = [i for i in range(default_nb)] + for i in range(default_nb): + connect.bulk_insert(id_collection, default_entity, [ids[i]]) + connect.flush([id_collection]) + delete_ids = [ids[0], ids[-1]] + status = connect.delete_entity_by_id(id_collection, delete_ids) + assert status + connect.flush([id_collection]) + res_count = connect.count_entities(id_collection) + assert res_count == default_nb - len(delete_ids) + + """ + ****************************************************************** + The following cases are used to test `delete_entity_by_id` function, with tags + ****************************************************************** + """ + + def test_insert_tag_delete(self, connect, collection): + ''' + method: add entitys with given tag, delete entities with the return ids + expected: entities deleted + ''' + connect.create_partition(collection, default_tag) + ids = connect.bulk_insert(collection, default_entities, partition_tag=default_tag) + connect.flush([collection]) + delete_ids = [ids[0], ids[-1]] + status = connect.delete_entity_by_id(collection, delete_ids) + assert status + connect.flush([collection]) + res_count = connect.count_entities(collection) + assert res_count == default_nb - 2 + + def test_insert_default_tag_delete(self, connect, collection): + ''' + method: add entitys, delete entities with the return ids + expected: entities deleted + ''' + connect.create_partition(collection, default_tag) + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + delete_ids = [ids[0], ids[-1]] + status = connect.delete_entity_by_id(collection, delete_ids) + assert status + connect.flush([collection]) + res_count = connect.count_entities(collection) + assert res_count == default_nb - 2 + + def test_insert_tags_delete(self, connect, collection): + ''' + method: add entitys with given two tags, delete entities with the return ids + expected: entities deleted + ''' + tag_new = "tag_new" + connect.create_partition(collection, default_tag) + connect.create_partition(collection, tag_new) + ids = connect.bulk_insert(collection, default_entities, partition_tag=default_tag) + ids_new = connect.bulk_insert(collection, default_entities, partition_tag=tag_new) + connect.flush([collection]) + delete_ids = [ids[0], ids_new[0]] + status = connect.delete_entity_by_id(collection, delete_ids) + assert status + connect.flush([collection]) + res_count = connect.count_entities(collection) + assert res_count == 2 * (default_nb - 1) + + def test_insert_tags_index_delete(self, connect, collection, get_simple_index): + """ + method: add entitys with given tag, create index, delete entities with the return ids + expected: entities deleted + """ + tag_new = "tag_new" + connect.create_partition(collection, default_tag) + connect.create_partition(collection, tag_new) + ids = connect.bulk_insert(collection, default_entities, partition_tag=default_tag) + ids_new = connect.bulk_insert(collection, default_entities, partition_tag=tag_new) + connect.flush([collection]) + connect.create_index(collection, field_name, get_simple_index) + delete_ids = [ids[0], ids_new[0]] + status = connect.delete_entity_by_id(collection, delete_ids) + assert status + connect.flush([collection]) + res_count = connect.count_entities(collection) + assert res_count == 2 * (default_nb - 1) + + def test_insert_delete_loop(self, connect, collection): + """ + target: test loop insert and delete entities + method: loop insert entities into two segments, and delete entities cross segments. + expected: count is correct + """ + loop = 2 + for i in range(loop): + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + status = connect.delete_entity_by_id(collection, ids[100:default_nb - 100]) + connect.flush([collection]) + res_count = connect.count_entities(collection) + assert res_count == loop * 200 + + def test_search_delete_loop(self, connect, collection): + """ + target: test loop search and delete entities + method: loop search and delete cross segments + expected: ok + """ + loop = 2 + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + ni = default_nb // loop + for i in range(loop): + res = connect.search(collection, default_single_query) + status = connect.delete_entity_by_id(collection, ids[i * ni:(i + 1) * ni]) + assert status + connect.flush([collection]) + res_count = connect.count_entities(collection) + assert res_count == 0 + + def test_count_delete_loop(self, connect, collection): + """ + target: test loop search and delete entities + method: loop search and delete cross segments + expected: ok + """ + loop = 2 + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + ni = default_nb // loop + for i in range(loop): + connect.count_entities(collection) + status = connect.delete_entity_by_id(collection, ids[i * ni:(i + 1) * ni]) + assert status + connect.flush([collection]) + res_count = connect.count_entities(collection) + assert res_count == 0 + + +class TestDeleteInvalid(object): + """ + Test adding vectors with invalid vectors + """ + + @pytest.fixture( + scope="function", + params=gen_invalid_ints() + ) + def gen_entity_id(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_invalid_strs() + ) + def get_collection_name(self, request): + yield request.param + + @pytest.mark.level(1) + def test_delete_entity_id_invalid(self, connect, collection, gen_entity_id): + invalid_id = gen_entity_id + with pytest.raises(Exception) as e: + status = connect.delete_entity_by_id(collection, [invalid_id]) + + def test_delete_entity_ids_invalid(self, connect, collection, gen_entity_id): + invalid_id = gen_entity_id + with pytest.raises(Exception) as e: + status = connect.delete_entity_by_id(collection, [1, invalid_id]) + + @pytest.mark.level(2) + def test_delete_entity_with_invalid_collection_name(self, connect, get_collection_name): + collection_name = get_collection_name + with pytest.raises(Exception) as e: + status = connect.delete_entity_by_id(collection_name, [1]) diff --git a/tests/python_test/entity/test_get_entity_by_id.py b/tests/python_test/entity/test_get_entity_by_id.py new file mode 100644 index 0000000000..afe6d479a6 --- /dev/null +++ b/tests/python_test/entity/test_get_entity_by_id.py @@ -0,0 +1,666 @@ +import time +import random +import pdb +import copy +import logging +from multiprocessing import Pool, Process +import concurrent.futures +from threading import current_thread +import pytest +from utils import * +from constants import * + +default_single_query = { + "bool": { + "must": [ + {"vector": { + default_float_vec_field_name: {"topk": 10, "query": gen_vectors(1, default_dim), "params": {"nprobe": 10}}}} + ] + } +} + +class TestGetBase: + """ + ****************************************************************** + The following cases are used to test `get_entity_by_id` function + ****************************************************************** + """ + + @pytest.fixture( + scope="function", + params=gen_simple_index() + ) + def get_simple_index(self, request, connect): + if str(connect._cmd("mode")) == "CPU": + if request.param["index_type"] in index_cpu_not_support(): + pytest.skip("sq8h not support in CPU mode") + return request.param + + @pytest.fixture( + scope="function", + params=[ + 1, + 500 + ], + ) + def get_pos(self, request): + yield request.param + + def test_get_entity(self, connect, collection, get_pos): + ''' + target: test.get_entity_by_id, get one + method: add entity, and get + expected: entity returned equals insert + ''' + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + res_count = connect.count_entities(collection) + assert res_count == default_nb + get_ids = [ids[get_pos]] + res = connect.get_entity_by_id(collection, get_ids) + assert_equal_vector(res[0].get(default_float_vec_field_name), default_entities[-1]["values"][get_pos]) + + def test_get_entity_multi_ids(self, connect, collection, get_pos): + ''' + target: test.get_entity_by_id, get one + method: add entity, and get + expected: entity returned equals insert + ''' + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + get_ids = ids[:get_pos] + res = connect.get_entity_by_id(collection, get_ids) + for i in range(get_pos): + assert_equal_vector(res[i].get(default_float_vec_field_name), default_entities[-1]["values"][i]) + + def test_get_entity_parts_ids(self, connect, collection): + ''' + target: test.get_entity_by_id, some ids in collection, some ids not + method: add entity, and get + expected: entity returned equals insert + ''' + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + get_ids = [ids[0], 1, ids[-1]] + res = connect.get_entity_by_id(collection, get_ids) + assert_equal_vector(res[0].get(default_float_vec_field_name), default_entities[-1]["values"][0]) + assert_equal_vector(res[-1].get(default_float_vec_field_name), default_entities[-1]["values"][-1]) + assert res[1] is None + + def test_get_entity_limit(self, connect, collection, args): + ''' + target: test.get_entity_by_id + method: add entity, and get, limit > 1000 + expected: entity returned + ''' + if args["handler"] == "HTTP": + pytest.skip("skip in http mode") + + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + with pytest.raises(Exception) as e: + res = connect.get_entity_by_id(collection, ids) + + def test_get_entity_same_ids(self, connect, id_collection): + ''' + target: test.get_entity_by_id, with the same ids + method: add entity, and get one id + expected: entity returned equals insert + ''' + ids = [1 for i in range(default_nb)] + res_ids = connect.bulk_insert(id_collection, default_entities, ids) + connect.flush([id_collection]) + get_ids = [ids[0]] + res = connect.get_entity_by_id(id_collection, get_ids) + assert len(res) == 1 + assert_equal_vector(res[0].get(default_float_vec_field_name), default_entities[-1]["values"][0]) + + def test_get_entity_params_same_ids(self, connect, id_collection): + ''' + target: test.get_entity_by_id, with the same ids + method: add entity, and get entity with the same ids + expected: entity returned equals insert + ''' + ids = [1] + res_ids = connect.bulk_insert(id_collection, default_entity, ids) + connect.flush([id_collection]) + get_ids = [1, 1] + res = connect.get_entity_by_id(id_collection, get_ids) + assert len(res) == len(get_ids) + for i in range(len(get_ids)): + logging.getLogger().info(i) + assert_equal_vector(res[i].get(default_float_vec_field_name), default_entity[-1]["values"][0]) + + def test_get_entities_params_same_ids(self, connect, collection): + ''' + target: test.get_entity_by_id, with the same ids + method: add entities, and get entity with the same ids + expected: entity returned equals insert + ''' + res_ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + get_ids = [res_ids[0], res_ids[0]] + res = connect.get_entity_by_id(collection, get_ids) + assert len(res) == len(get_ids) + for i in range(len(get_ids)): + assert_equal_vector(res[i].get(default_float_vec_field_name), default_entities[-1]["values"][0]) + + """ + ****************************************************************** + The following cases are used to test `get_entity_by_id` function, with different metric type + ****************************************************************** + """ + + def test_get_entity_parts_ids_binary(self, connect, binary_collection): + ''' + target: test.get_entity_by_id, some ids in jac_collection, some ids not + method: add entity, and get + expected: entity returned equals insert + ''' + ids = connect.bulk_insert(binary_collection, default_binary_entities) + connect.flush([binary_collection]) + get_ids = [ids[0], 1, ids[-1]] + res = connect.get_entity_by_id(binary_collection, get_ids) + assert_equal_vector(res[0].get("binary_vector"), default_binary_entities[-1]["values"][0]) + assert_equal_vector(res[-1].get("binary_vector"), default_binary_entities[-1]["values"][-1]) + assert res[1] is None + + """ + ****************************************************************** + The following cases are used to test `get_entity_by_id` function, with tags + ****************************************************************** + """ + + def test_get_entities_tag(self, connect, collection, get_pos): + ''' + target: test.get_entity_by_id + method: add entities with tag, get + expected: entity returned + ''' + connect.create_partition(collection, default_tag) + ids = connect.bulk_insert(collection, default_entities, partition_tag = default_tag) + connect.flush([collection]) + get_ids = ids[:get_pos] + res = connect.get_entity_by_id(collection, get_ids) + for i in range(get_pos): + assert_equal_vector(res[i].get(default_float_vec_field_name), default_entities[-1]["values"][i]) + + def test_get_entities_tag_default(self, connect, collection, get_pos): + ''' + target: test.get_entity_by_id + method: add entities with default tag, get + expected: entity returned + ''' + connect.create_partition(collection, default_tag) + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + get_ids = ids[:get_pos] + res = connect.get_entity_by_id(collection, get_ids) + for i in range(get_pos): + assert_equal_vector(res[i].get(default_float_vec_field_name), default_entities[-1]["values"][i]) + + def test_get_entities_tags_default(self, connect, collection, get_pos): + ''' + target: test.get_entity_by_id + method: create partitions, add entities with default tag, get + expected: entity returned + ''' + tag_new = "tag_new" + connect.create_partition(collection, default_tag) + connect.create_partition(collection, tag_new) + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + get_ids = ids[:get_pos] + res = connect.get_entity_by_id(collection, get_ids) + for i in range(get_pos): + assert_equal_vector(res[i].get(default_float_vec_field_name), default_entities[-1]["values"][i]) + + def test_get_entities_tags_A(self, connect, collection, get_pos): + ''' + target: test.get_entity_by_id + method: create partitions, add entities with default tag, get + expected: entity returned + ''' + tag_new = "tag_new" + connect.create_partition(collection, default_tag) + connect.create_partition(collection, tag_new) + ids = connect.bulk_insert(collection, default_entities, partition_tag = default_tag) + connect.flush([collection]) + get_ids = ids[:get_pos] + res = connect.get_entity_by_id(collection, get_ids) + for i in range(get_pos): + assert_equal_vector(res[i].get(default_float_vec_field_name), default_entities[-1]["values"][i]) + + def test_get_entities_tags_B(self, connect, collection, get_pos): + ''' + target: test.get_entity_by_id + method: create partitions, add entities with default tag, get + expected: entity returned + ''' + tag_new = "tag_new" + connect.create_partition(collection, default_tag) + connect.create_partition(collection, tag_new) + new_entities = gen_entities(default_nb + 1) + ids = connect.bulk_insert(collection, default_entities, partition_tag = default_tag) + ids_new = connect.bulk_insert(collection, new_entities, partition_tag = tag_new) + connect.flush([collection]) + get_ids = ids[:get_pos] + get_ids.extend(ids_new[:get_pos]) + res = connect.get_entity_by_id(collection, get_ids) + for i in range(get_pos): + assert_equal_vector(res[i].get(default_float_vec_field_name), default_entities[-1]["values"][i]) + for i in range(get_pos, get_pos * 2): + assert_equal_vector(res[i].get(default_float_vec_field_name), new_entities[-1]["values"][i - get_pos]) + + @pytest.mark.level(2) + def test_get_entities_indexed_tag(self, connect, collection, get_simple_index, get_pos): + ''' + target: test.get_entity_by_id + method: add entities with tag, get + expected: entity returned + ''' + connect.create_partition(collection, default_tag) + ids = connect.bulk_insert(collection, default_entities, partition_tag = default_tag) + connect.flush([collection]) + connect.create_index(collection, default_float_vec_field_name, get_simple_index) + get_ids = ids[:get_pos] + res = connect.get_entity_by_id(collection, get_ids) + for i in range(get_pos): + assert_equal_vector(res[i].get(default_float_vec_field_name), default_entities[-1]["values"][i]) + + """ + ****************************************************************** + The following cases are used to test `get_entity_by_id` function, with fields params + ****************************************************************** + """ + + def test_get_entity_field(self, connect, collection, get_pos): + ''' + target: test.get_entity_by_id, get one + method: add entity, and get + expected: entity returned equals insert + ''' + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + get_ids = [ids[get_pos]] + fields = ["int64"] + res = connect.get_entity_by_id(collection, get_ids, fields = fields) + # assert fields + res = res.dict() + assert res[0]["field"] == fields[0] + assert res[0]["values"] == [default_entities[0]["values"][get_pos]] + assert res[0]["type"] == DataType.INT64 + + def test_get_entity_fields(self, connect, collection, get_pos): + ''' + target: test.get_entity_by_id, get one + method: add entity, and get + expected: entity returned equals insert + ''' + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + get_ids = [ids[get_pos]] + fields = ["int64", "float", default_float_vec_field_name] + res = connect.get_entity_by_id(collection, get_ids, fields = fields) + # assert fields + res = res.dict() + assert len(res) == len(fields) + for field in res: + if field["field"] == fields[0]: + assert field["values"] == [default_entities[0]["values"][get_pos]] + elif field["field"] == fields[1]: + assert field["values"] == [default_entities[1]["values"][get_pos]] + else: + assert_equal_vector(field["values"][0], default_entities[-1]["values"][get_pos]) + + # TODO: assert exception + def test_get_entity_field_not_match(self, connect, collection, get_pos): + ''' + target: test.get_entity_by_id, get one + method: add entity, and get + expected: entity returned equals insert + ''' + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + get_ids = [ids[get_pos]] + fields = ["int1288"] + with pytest.raises(Exception) as e: + res = connect.get_entity_by_id(collection, get_ids, fields = fields) + + # TODO: assert exception + def test_get_entity_fields_not_match(self, connect, collection, get_pos): + ''' + target: test.get_entity_by_id, get one + method: add entity, and get + expected: entity returned equals insert + ''' + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + get_ids = [ids[get_pos]] + fields = ["int1288"] + with pytest.raises(Exception) as e: + res = connect.get_entity_by_id(collection, get_ids, fields = fields) + + def test_get_entity_id_not_exised(self, connect, collection): + ''' + target: test get entity, params entity_id not existed + method: add entity and get + expected: empty result + ''' + ids = connect.bulk_insert(collection, default_entity) + connect.flush([collection]) + res = connect.get_entity_by_id(collection, [1]) + assert res[0] is None + + def test_get_entity_collection_not_existed(self, connect, collection): + ''' + target: test get entity, params collection_name not existed + method: add entity and get + expected: error raised + ''' + ids = connect.bulk_insert(collection, default_entity) + connect.flush([collection]) + collection_new = gen_unique_str() + with pytest.raises(Exception) as e: + res = connect.get_entity_by_id(collection_new, [ids[0]]) + + """ + ****************************************************************** + The following cases are used to test `get_entity_by_id` function, after deleted + ****************************************************************** + """ + + def test_get_entity_after_delete(self, connect, collection, get_pos): + ''' + target: test.get_entity_by_id + method: add entities, and delete, get entity by the given id + expected: empty result + ''' + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + delete_ids = [ids[get_pos]] + status = connect.delete_entity_by_id(collection, delete_ids) + connect.flush([collection]) + get_ids = [ids[get_pos]] + res = connect.get_entity_by_id(collection, get_ids) + assert res[0] is None + + def test_get_entities_after_delete(self, connect, collection, get_pos): + ''' + target: test.get_entity_by_id + method: add entities, and delete, get entity by the given id + expected: empty result + ''' + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + delete_ids = ids[:get_pos] + status = connect.delete_entity_by_id(collection, delete_ids) + connect.flush([collection]) + get_ids = delete_ids + res = connect.get_entity_by_id(collection, get_ids) + for i in range(get_pos): + assert res[i] is None + + def test_get_entities_after_delete_compact(self, connect, collection, get_pos): + ''' + target: test.get_entity_by_id + method: add entities, and delete, get entity by the given id + expected: empty result + ''' + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + delete_ids = ids[:get_pos] + status = connect.delete_entity_by_id(collection, delete_ids) + connect.flush([collection]) + connect.compact(collection) + get_ids = ids[:get_pos] + res = connect.get_entity_by_id(collection, get_ids) + for i in range(get_pos): + assert res[i] is None + + def test_get_entities_indexed_batch(self, connect, collection, get_simple_index, get_pos): + ''' + target: test.get_entity_by_id + method: add entities batch, create index, get + expected: entity returned + ''' + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + connect.create_index(collection, default_float_vec_field_name, get_simple_index) + get_ids = ids[:get_pos] + res = connect.get_entity_by_id(collection, get_ids) + for i in range(get_pos): + assert_equal_vector(res[i].get(default_float_vec_field_name), default_entities[-1]["values"][i]) + + @pytest.mark.level(2) + def test_get_entities_indexed_single(self, connect, collection, get_simple_index, get_pos): + ''' + target: test.get_entity_by_id + method: add entities 1 entity/per request, create index, get + expected: entity returned + ''' + ids = [] + for i in range(default_nb): + ids.append(connect.bulk_insert(collection, default_entity)[0]) + connect.flush([collection]) + connect.create_index(collection, default_float_vec_field_name, get_simple_index) + get_ids = ids[:get_pos] + res = connect.get_entity_by_id(collection, get_ids) + for i in range(get_pos): + assert_equal_vector(res[i].get(default_float_vec_field_name), default_entity[-1]["values"][0]) + + def test_get_entities_with_deleted_ids(self, connect, id_collection): + ''' + target: test.get_entity_by_id + method: add entities ids, and delete part, get entity include the deleted id + expected: + ''' + ids = [i for i in range(default_nb)] + res_ids = connect.bulk_insert(id_collection, default_entities, ids) + connect.flush([id_collection]) + status = connect.delete_entity_by_id(id_collection, [res_ids[1]]) + connect.flush([id_collection]) + get_ids = res_ids[:2] + res = connect.get_entity_by_id(id_collection, get_ids) + assert len(res) == len(get_ids) + assert_equal_vector(res[0].get(default_float_vec_field_name), default_entities[-1]["values"][0]) + assert res[1] is None + + # TODO: unable to set config + def _test_get_entities_after_delete_disable_autoflush(self, connect, collection, get_pos): + ''' + target: test.get_entity_by_id + method: disable autoflush, add entities, and delete, get entity by the given id + expected: empty result + ''' + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + delete_ids = ids[:get_pos] + try: + disable_flush(connect) + status = connect.delete_entity_by_id(collection, delete_ids) + get_ids = ids[:get_pos] + res = connect.get_entity_by_id(collection, get_ids) + for i in range(get_pos): + assert_equal_vector(res[i].get(default_float_vec_field_name), default_entities[-1]["values"][i]) + finally: + enable_flush(connect) + + def test_get_entities_after_delete_same_ids(self, connect, id_collection): + ''' + target: test.get_entity_by_id + method: add entities with the same ids, and delete, get entity by the given id + expected: empty result + ''' + ids = [i for i in range(default_nb)] + ids[0] = 1 + res_ids = connect.bulk_insert(id_collection, default_entities, ids) + connect.flush([id_collection]) + status = connect.delete_entity_by_id(id_collection, [1]) + connect.flush([id_collection]) + get_ids = [1] + res = connect.get_entity_by_id(id_collection, get_ids) + assert res[0] is None + + def test_get_entity_after_delete_with_partition(self, connect, collection, get_pos): + ''' + target: test.get_entity_by_id + method: add entities into partition, and delete, get entity by the given id + expected: get one entity + ''' + connect.create_partition(collection, default_tag) + ids = connect.bulk_insert(collection, default_entities, partition_tag = default_tag) + connect.flush([collection]) + status = connect.delete_entity_by_id(collection, [ids[get_pos]]) + connect.flush([collection]) + res = connect.get_entity_by_id(collection, [ids[get_pos]]) + assert res[0] is None + + def test_get_entity_by_id_multithreads(self, connect, collection): + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + get_id = ids[100:200] + + def get(): + res = connect.get_entity_by_id(collection, get_id) + assert len(res) == len(get_id) + for i in range(len(res)): + assert_equal_vector(res[i].get(default_float_vec_field_name), default_entities[-1]["values"][100 + i]) + + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + future_results = {executor.submit( + get): i for i in range(10)} + for future in concurrent.futures.as_completed(future_results): + future.result() + + @pytest.mark.level(2) + def test_get_entity_by_id_insert_multi_threads(self, connect, collection): + ''' + target: test.get_entity_by_id + method: thread do insert and get + expected: + ''' + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + get_id = ids[:1000] + + def insert(): + # logging.getLogger().info(current_thread().getName() + " insert") + step = 1000 + for i in range(default_nb // step): + group_entities = gen_entities(step, False) + connect.bulk_insert(collection, group_entities) + connect.flush([collection]) + + def get(): + # logging.getLogger().info(current_thread().getName() + " get") + res = connect.get_entity_by_id(collection, get_id) + assert len(res) == len(get_id) + for i in range(len(res)): + assert_equal_vector(res[i].get(default_float_vec_field_name), default_entities[-1]["values"][i]) + + with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor: + for i in range(20): + fun = random.choices([get, insert])[0] + future = executor.submit(fun) + future.result() + + @pytest.mark.level(2) + def test_get_entity_by_id_insert_multi_threads_2(self, connect, collection): + ''' + target: test.get_entity_by_id + method: thread do insert and get + expected: + ''' + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + def get(group_ids, group_entities): + # logging.getLogger().info(current_thread().getName() + " get") + res = connect.get_entity_by_id(collection, group_ids) + assert len(res) == len(group_ids) + for i in range(len(res)): + assert_equal_vector(res[i].get(default_float_vec_field_name), group_entities[-1]["values"][i]) + + def insert(group_vectors): + # logging.getLogger().info(current_thread().getName() + " insert") + for group_vector in group_vectors: + group_entities = [ + {"name": "int64", "type": DataType.INT64, "values": [i for i in range(step)]}, + {"name": "float", "type": DataType.FLOAT, "values": [float(i) for i in range(step)]}, + {"name": default_float_vec_field_name, "type": DataType.FLOAT_VECTOR, "values": group_vector} + ] + group_ids = connect.bulk_insert(collection, group_entities) + connect.flush([collection]) + executor.submit(get, group_ids, group_entities) + + step = 100 + vectors = gen_vectors(default_nb, default_dim, False) + group_vectors = [vectors[i:i + step] for i in range(0, len(vectors), step)] + task = executor.submit(insert, group_vectors) + task.result() + + +class TestGetInvalid(object): + """ + Test get entities with invalid params + """ + + @pytest.fixture( + scope="function", + params=gen_invalid_strs() + ) + def get_collection_name(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_invalid_strs() + ) + def get_field_name(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_invalid_ints() + ) + def get_entity_id(self, request): + yield request.param + + @pytest.mark.level(2) + def test_insert_ids_invalid(self, connect, collection, get_entity_id): + ''' + target: test insert, with using customize ids, which are not int64 + method: create collection and insert entities in it + expected: raise an exception + ''' + entity_id = get_entity_id + ids = [entity_id for _ in range(default_nb)] + with pytest.raises(Exception): + connect.get_entity_by_id(collection, ids) + + @pytest.mark.level(2) + def test_insert_parts_ids_invalid(self, connect, collection, get_entity_id): + ''' + target: test insert, with using customize ids, which are not int64 + method: create collection and insert entities in it + expected: raise an exception + ''' + entity_id = get_entity_id + ids = [i for i in range(default_nb)] + ids[-1] = entity_id + with pytest.raises(Exception): + connect.get_entity_by_id(collection, ids) + + @pytest.mark.level(2) + def test_get_entities_with_invalid_collection_name(self, connect, get_collection_name): + collection_name = get_collection_name + ids = [1] + with pytest.raises(Exception): + res = connect.get_entity_by_id(collection_name, ids) + + @pytest.mark.level(2) + def test_get_entities_with_invalid_field_name(self, connect, collection, get_field_name): + field_name = get_field_name + ids = [1] + fields = [field_name] + with pytest.raises(Exception): + res = connect.get_entity_by_id(collection, ids, fields = fields) diff --git a/tests/python_test/entity/test_insert.py b/tests/python_test/entity/test_insert.py new file mode 100644 index 0000000000..dc587a8984 --- /dev/null +++ b/tests/python_test/entity/test_insert.py @@ -0,0 +1,1090 @@ +import logging +import time +import pdb +import copy +import threading +from multiprocessing import Pool, Process +import pytest +from milvus import DataType +from utils import * +from constants import * + +ADD_TIMEOUT = 60 +uid = "test_insert" +field_name = default_float_vec_field_name +binary_field_name = default_binary_vec_field_name +default_single_query = { + "bool": { + "must": [ + {"vector": {field_name: {"topk": 10, "query": gen_vectors(1, default_dim), "metric_type": "L2", + "params": {"nprobe": 10}}}} + ] + } +} + + +class TestInsertBase: + """ + ****************************************************************** + The following cases are used to test `insert` function + ****************************************************************** + """ + + @pytest.fixture( + scope="function", + params=gen_simple_index() + ) + def get_simple_index(self, request, connect): + if str(connect._cmd("mode")) == "CPU": + if request.param["index_type"] in index_cpu_not_support(): + pytest.skip("CPU not support index_type: ivf_sq8h") + return request.param + + @pytest.fixture( + scope="function", + params=gen_single_filter_fields() + ) + def get_filter_field(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_single_vector_fields() + ) + def get_vector_field(self, request): + yield request.param + + def test_insert_with_empty_entity(self, connect, collection): + ''' + target: test insert with empty entity list + method: set empty entity list as insert method params + expected: raises a ParamError exception + ''' + entities = [] + with pytest.raises(ParamError) as e: + status, ids = connect.insert(collection, entities) + + def test_insert_with_None(self, connect, collection): + ''' + target: test insert with None + method: set None as insert method params + expected: raises a ParamError + ''' + entity = None + with pytest.raises(Exception) as e: + status, ids = connect.insert(collection, entity) + + @pytest.mark.timeout(ADD_TIMEOUT) + def test_insert_collection_not_existed(self, connect): + ''' + target: test insert, with collection not existed + method: insert entity into a random named collection + expected: raise a BaseException + ''' + collection_name = gen_unique_str(uid) + with pytest.raises(Exception) as e: + connect.insert(collection_name, default_entities) + + @pytest.mark.level(2) + def test_insert_without_connect(self, dis_connect, collection): + ''' + target: test insert entities without connection + method: create collection and insert entities in it, check if inserted successfully + expected: raise exception + ''' + with pytest.raises(Exception) as e: + ids = dis_connect.insert(collection, default_entities) + + @pytest.mark.timeout(ADD_TIMEOUT) + def test_insert_drop_collection(self, connect, collection): + ''' + target: test delete collection after insert entities + method: insert entities and drop collection + expected: has_collection false + ''' + ids = connect.insert(collection, default_entity_row) + assert len(ids) == 1 + connect.drop_collection(collection) + assert connect.has_collection(collection) == False + + @pytest.mark.timeout(ADD_TIMEOUT) + def test_insert_flush_drop_collection(self, connect, collection): + ''' + target: test drop collection after insert entities for a while + method: insert entities, sleep, and delete collection + expected: has_collection false + ''' + ids = connect.insert(collection, default_entity_row) + assert len(ids) == 1 + connect.flush([collection]) + connect.drop_collection(collection) + assert connect.has_collection(collection) == False + + @pytest.mark.timeout(ADD_TIMEOUT) + def test_insert_create_index(self, connect, collection, get_simple_index): + ''' + target: test build index insert after entities + method: insert entities and build index + expected: no error raised + ''' + ids = connect.insert(collection, default_entities) + assert len(ids) == default_nb + connect.flush([collection]) + connect.create_index(collection, field_name, get_simple_index) + index = connect.describe_index(collection, field_name) + assert index == get_simple_index + # fields = info["fields"] + # for field in fields: + # if field["name"] == field_name: + # assert field["indexes"][0] == get_simple_index + + @pytest.mark.timeout(ADD_TIMEOUT) + def test_insert_after_create_index(self, connect, collection, get_simple_index): + ''' + target: test build index insert after vector + method: insert entities and build index + expected: no error raised + ''' + connect.create_index(collection, field_name, get_simple_index) + ids = connect.insert(collection, default_entities) + assert len(ids) == default_nb + index = connect.describe_index(collection, field_name) + assert index == get_simple_index + + # TODO + @pytest.mark.timeout(ADD_TIMEOUT) + def test_insert_search(self, connect, collection): + ''' + target: test search entity after insert entity after a while + method: insert entity, sleep, and search collection + expected: no error raised + ''' + ids = connect.insert(collection, default_entities) + connect.flush([collection]) + connect.load_collection(collection) + res = connect.search(collection, default_single_query) + assert res + + @pytest.mark.skip("No segment_row_limit") + def test_insert_segment_row_count(self, connect, collection): + nb = default_segment_row_limit + 1 + res_ids = connect.insert(collection, gen_entities(nb)) + connect.flush([collection]) + assert len(res_ids) == nb + stats = connect.get_collection_stats(collection) + assert len(stats['partitions'][0]['segments']) == 2 + for segment in stats['partitions'][0]['segments']: + assert segment['row_count'] in [default_segment_row_limit, 1] + + @pytest.fixture( + scope="function", + params=[ + 1, + 2000 + ], + ) + def insert_count(self, request): + yield request.param + + @pytest.mark.timeout(ADD_TIMEOUT) + def test_insert_ids(self, connect, id_collection, insert_count): + ''' + target: test insert entities in collection, use customize ids + method: create collection and insert entities in it, check the ids returned and the collection length after entities inserted + expected: the length of ids and the collection row count + ''' + nb = insert_count + ids = [i for i in range(nb)] + res_ids = connect.insert(id_collection, gen_entities(nb), ids) + connect.flush([id_collection]) + assert len(res_ids) == nb + assert res_ids == ids + stats = connect.get_collection_stats(id_collection) + assert stats["row_count"] == nb + + @pytest.mark.timeout(ADD_TIMEOUT) + def test_insert_the_same_ids(self, connect, id_collection, insert_count): + ''' + target: test insert vectors in collection, use customize the same ids + method: create collection and insert vectors in it, check the ids returned and the collection length after vectors inserted + expected: the length of ids and the collection row count + ''' + nb = insert_count + ids = [1 for i in range(nb)] + res_ids = connect.insert(id_collection, gen_entities(nb), ids) + connect.flush([id_collection]) + assert len(res_ids) == nb + assert res_ids == ids + stats = connect.get_collection_stats(id_collection) + assert stats["row_count"] == nb + + @pytest.mark.timeout(ADD_TIMEOUT) + def test_insert_ids_fields(self, connect, get_filter_field, get_vector_field): + ''' + target: test create normal collection with different fields, insert entities into id with ids + method: create collection with diff fields: metric/field_type/..., insert, and count + expected: row count correct + ''' + nb = 5 + filter_field = get_filter_field + vector_field = get_vector_field + collection_name = gen_unique_str("test_collection") + fields = { + "fields": [filter_field, vector_field], + "auto_id": True + } + connect.create_collection(collection_name, fields) + ids = [i for i in range(nb)] + entities = gen_entities_by_fields(fields["fields"], nb, dim) + res_ids = connect.insert(collection_name, entities, ids) + assert res_ids == ids + connect.flush([collection_name]) + stats = connect.get_collection_stats(id_collection) + assert stats["row_count"] == nb + + @pytest.mark.timeout(ADD_TIMEOUT) + def test_insert_ids_not_match(self, connect, id_collection, insert_count): + ''' + target: test insert entities in collection without ids + method: create id_collection and insert entities without + expected: exception raised + ''' + nb = insert_count + with pytest.raises(Exception) as e: + connect.insert(id_collection, gen_entities(nb)) + + # TODO + @pytest.mark.timeout(ADD_TIMEOUT) + def test_insert_twice_ids_no_ids(self, connect, collection): + ''' + target: check the result of insert, with params ids and no ids + method: test insert vectors twice, use customize ids first, and then use no ids + expected: BaseException raised + ''' + ids = [i for i in range(default_nb)] + res_ids = connect.insert(id_collection, default_entities, ids) + with pytest.raises(Exception) as e: + res_ids_new = connect.insert(id_collection, default_entities) + + # TODO: assert exception && enable + @pytest.mark.level(2) + @pytest.mark.timeout(ADD_TIMEOUT) + def test_insert_twice_not_ids_ids(self, connect, id_collection): + ''' + target: check the result of insert, with params ids and no ids + method: test insert vectors twice, use not ids first, and then use customize ids + expected: error raised + ''' + with pytest.raises(Exception) as e: + res_ids = connect.insert(id_collection, default_entities) + + @pytest.mark.timeout(ADD_TIMEOUT) + def test_insert_ids_length_not_match_batch(self, connect, id_collection): + ''' + target: test insert vectors in collection, use customize ids, len(ids) != len(vectors) + method: create collection and insert vectors in it + expected: raise an exception + ''' + ids = [i for i in range(1, default_nb)] + logging.getLogger().info(len(ids)) + with pytest.raises(Exception) as e: + res_ids = connect.insert(id_collection, default_entities, ids) + + @pytest.mark.timeout(ADD_TIMEOUT) + def test_insert_ids_length_not_match_single(self, connect, id_collection): + ''' + target: test insert vectors in collection, use customize ids, len(ids) != len(vectors) + method: create collection and insert vectors in it + expected: raise an exception + ''' + ids = [i for i in range(1, default_nb)] + logging.getLogger().info(len(ids)) + with pytest.raises(Exception) as e: + res_ids = connect.insert(id_collection, default_entity, ids) + + @pytest.mark.timeout(ADD_TIMEOUT) + def test_insert_partition(self, connect, collection): + ''' + target: test insert entities in collection created before + method: create collection and insert entities in it, with the partition_tag param + expected: the collection row count equals to nq + ''' + connect.create_partition(collection, default_tag) + ids = connect.insert(collection, default_entities, partition_tag=default_tag) + assert len(ids) == default_nb + assert connect.has_partition(collection, default_tag) + connect.flush([collection_name]) + stats = connect.get_collection_stats(id_collection) + assert stats["row_count"] == default_nb + + # TODO + @pytest.mark.timeout(ADD_TIMEOUT) + def test_insert_partition_with_ids(self, connect, id_collection): + ''' + target: test insert entities in collection created before, insert with ids + method: create collection and insert entities in it, with the partition_tag param + expected: the collection row count equals to nq + ''' + connect.create_partition(id_collection, default_tag) + ids = [i for i in range(default_nb)] + res_ids = connect.insert(id_collection, gen_entities(default_nb, _id=False), partition_tag=default_tag) + assert res_ids == ids + + @pytest.mark.timeout(ADD_TIMEOUT) + def test_insert_default_partition(self, connect, collection): + ''' + target: test insert entities into default partition + method: create partition and insert info collection without tag params + expected: the collection row count equals to nb + ''' + default_tag = "_default" + with pytest.raises(Exception) as e: + connect.create_partition(collection, default_tag) + + @pytest.mark.timeout(ADD_TIMEOUT) + def test_insert_partition_not_existed(self, connect, collection): + ''' + target: test insert entities in collection created before + method: create collection and insert entities in it, with the not existed partition_tag param + expected: error raised + ''' + tag = gen_unique_str() + with pytest.raises(Exception) as e: + ids = connect.insert(collection, default_entities, partition_tag=tag) + + @pytest.mark.timeout(ADD_TIMEOUT) + def test_insert_partition_repeatedly(self, connect, collection): + ''' + target: test insert entities in collection created before + method: create collection and insert entities in it repeatly, with the partition_tag param + expected: the collection row count equals to nq + ''' + connect.create_partition(collection, default_tag) + ids = connect.insert(collection, default_entities, partition_tag=default_tag) + ids = connect.insert(collection, default_entities, partition_tag=default_tag) + connect.flush([collection]) + res = connect.get_collection_stats(collection) + assert res["row_count"] == 2 * default_nb + + def test_insert_dim_not_matched(self, connect, collection): + ''' + target: test insert entities, the vector dimension is not equal to the collection dimension + method: the entities dimension is half of the collection dimension, check the status + expected: error raised + ''' + vectors = gen_vectors(default_nb, int(default_dim) // 2) + insert_entities = copy.deepcopy(default_entities) + insert_entities[-1][default_float_vec_field_name] = vectors + with pytest.raises(Exception) as e: + ids = connect.insert(collection, insert_entities) + + def test_insert_with_field_name_not_match(self, connect, collection): + ''' + target: test insert entities, with the entity field name updated + method: update entity field name + expected: error raised + ''' + tmp_entity = update_field_name(copy.deepcopy(default_entity), "int64", "int64new") + with pytest.raises(Exception): + connect.insert(collection, tmp_entity) + + @pytest.mark.level(2) + def test_insert_with_field_type_not_match(self, connect, collection): + ''' + target: test insert entities, with the entity field type updated + method: update entity field type + expected: error raised + ''' + tmp_entity = update_field_type(copy.deepcopy(default_entity), "int64", DataType.FLOAT) + with pytest.raises(Exception): + connect.insert(collection, tmp_entity) + + @pytest.mark.level(2) + def test_insert_with_field_value_not_match(self, connect, collection): + ''' + target: test insert entities, with the entity field value updated + method: update entity field value + expected: error raised + ''' + tmp_entity = update_field_value(copy.deepcopy(default_entity), DataType.FLOAT, 's') + with pytest.raises(Exception): + connect.insert(collection, tmp_entity) + + def test_insert_with_field_more(self, connect, collection): + ''' + target: test insert entities, with more fields than collection schema + method: add entity field + expected: error raised + ''' + tmp_entity = add_field(copy.deepcopy(default_entity)) + with pytest.raises(Exception): + connect.insert(collection, tmp_entity) + + def test_insert_with_field_vector_more(self, connect, collection): + ''' + target: test insert entities, with more fields than collection schema + method: add entity vector field + expected: error raised + ''' + tmp_entity = add_vector_field(default_nb, default_dim) + with pytest.raises(Exception): + connect.insert(collection, tmp_entity) + + def test_insert_with_field_less(self, connect, collection): + ''' + target: test insert entities, with less fields than collection schema + method: remove entity field + expected: error raised + ''' + tmp_entity = remove_field(copy.deepcopy(default_entity)) + with pytest.raises(Exception): + connect.insert(collection, tmp_entity) + + def test_insert_with_field_vector_less(self, connect, collection): + ''' + target: test insert entities, with less fields than collection schema + method: remove entity vector field + expected: error raised + ''' + tmp_entity = remove_vector_field(copy.deepcopy(default_entity)) + with pytest.raises(Exception): + connect.insert(collection, tmp_entity) + + def test_insert_with_no_field_vector_value(self, connect, collection): + ''' + target: test insert entities, with no vector field value + method: remove entity values of vector field + expected: error raised + ''' + tmp_entity = copy.deepcopy(default_entity) + del tmp_entity[-1]["values"] + with pytest.raises(Exception): + connect.insert(collection, tmp_entity) + + def test_insert_with_no_field_vector_type(self, connect, collection): + ''' + target: test insert entities, with no vector field type + method: remove entity vector field + expected: error raised + ''' + tmp_entity = copy.deepcopy(default_entity) + del tmp_entity[-1]["type"] + with pytest.raises(Exception): + connect.insert(collection, tmp_entity) + + def test_insert_with_no_field_vector_name(self, connect, collection): + ''' + target: test insert entities, with no vector field name + method: remove entity vector field + expected: error raised + ''' + tmp_entity = copy.deepcopy(default_entity) + del tmp_entity[-1]["name"] + with pytest.raises(Exception): + connect.insert(collection, tmp_entity) + + @pytest.mark.level(2) + @pytest.mark.timeout(30) + def test_collection_insert_rows_count_multi_threading(self, args, collection): + ''' + target: test collection rows_count is correct or not with multi threading + method: create collection and insert entities in it(idmap), + assert the value returned by count_entities method is equal to length of entities + expected: the count is equal to the length of entities + ''' + if args["handler"] == "HTTP": + pytest.skip("Skip test in http mode") + thread_num = 8 + threads = [] + milvus = get_milvus(host=args["ip"], port=args["port"], handler=args["handler"], try_connect=False) + + def insert(thread_i): + logging.getLogger().info("In thread-%d" % thread_i) + res_ids = milvus.insert(collection, default_entities) + milvus.flush([collection]) + + for i in range(thread_num): + x = threading.Thread(target=insert, args=(i,)) + threads.append(x) + x.start() + for th in threads: + th.join() + res_count = milvus.count_entities(collection) + assert res_count == thread_num * default_nb + + # TODO: unable to set config + @pytest.mark.level(2) + def _test_insert_disable_auto_flush(self, connect, collection): + ''' + target: test insert entities, with disable autoflush + method: disable autoflush and insert, get entity + expected: the count is equal to 0 + ''' + delete_nums = 500 + disable_flush(connect) + ids = connect.insert(collection, default_entities) + res = connect.get_entity_by_id(collection, ids[:delete_nums]) + assert len(res) == delete_nums + assert res[0] is None + + +class TestInsertBinary: + @pytest.fixture( + scope="function", + params=gen_binary_index() + ) + def get_binary_index(self, request): + request.param["metric_type"] = "JACCARD" + return request.param + + def test_insert_binary_entities(self, connect, binary_collection): + ''' + target: test insert entities in binary collection + method: create collection and insert binary entities in it + expected: the collection row count equals to nb + ''' + ids = connect.insert(binary_collection, default_binary_entities) + assert len(ids) == default_nb + connect.flush() + stats = connect.get_collection_stats(binary_collection) + assert stats["row_count"] == default_nb + + def test_insert_binary_partition(self, connect, binary_collection): + ''' + target: test insert entities and create partition tag + method: create collection and insert binary entities in it, with the partition_tag param + expected: the collection row count equals to nb + ''' + connect.create_partition(binary_collection, default_tag) + ids = connect.insert(binary_collection, default_binary_entities, partition_tag=default_tag) + assert len(ids) == default_nb + assert connect.has_partition(binary_collection, default_tag) + stats = connect.get_collection_stats(binary_collection) + assert stats["row_count"] == default_nb + + def test_insert_binary_multi_times(self, connect, binary_collection): + ''' + target: test insert entities multi times and final flush + method: create collection and insert binary entity multi and final flush + expected: the collection row count equals to nb + ''' + for i in range(default_nb): + ids = connect.insert(binary_collection, default_binary_entity) + assert len(ids) == 1 + connect.flush([binary_collection]) + stats = connect.get_collection_stats(binary_collection) + assert stats["row_count"] == default_nb + + def test_insert_binary_after_create_index(self, connect, binary_collection, get_binary_index): + ''' + target: test insert binary entities after build index + method: build index and insert entities + expected: no error raised + ''' + connect.create_index(binary_collection, binary_field_name, get_binary_index) + ids = connect.insert(binary_collection, default_binary_entities) + assert len(ids) == default_nb + connect.flush([binary_collection]) + index = connect.describe_index(binary_collection, binary_field_name) + assert index == get_binary_index + + @pytest.mark.timeout(ADD_TIMEOUT) + def test_insert_binary_create_index(self, connect, binary_collection, get_binary_index): + ''' + target: test build index insert after vector + method: insert vector and build index + expected: no error raised + ''' + ids = connect.insert(binary_collection, default_binary_entities) + assert len(ids) == default_nb + connect.flush([binary_collection]) + connect.create_index(binary_collection, binary_field_name, get_binary_index) + index = connect.describe_index(binary_collection, binary_field_name) + assert index == get_binary_index + + def test_insert_binary_search(self, connect, binary_collection): + ''' + target: test search vector after insert vector after a while + method: insert vector, sleep, and search collection + expected: no error raised + ''' + ids = connect.insert(binary_collection, default_binary_entities) + connect.flush([binary_collection]) + query, vecs = gen_query_vectors(binary_field_name, default_binary_entities, default_top_k, 1, metric_type="JACCARD") + connect.load_collection(binary_collection) + res = connect.search(binary_collection, query) + logging.getLogger().debug(res) + assert res + + +class TestInsertAsync: + @pytest.fixture(scope="function", autouse=True) + def skip_http_check(self, args): + if args["handler"] == "HTTP": + pytest.skip("skip in http mode") + + @pytest.fixture( + scope="function", + params=[ + 1, + 1000 + ], + ) + def insert_count(self, request): + yield request.param + + def check_status(self, result): + logging.getLogger().info("In callback check status") + assert not result + + def check_result(self, result): + logging.getLogger().info("In callback check status") + assert result + + def test_insert_async(self, connect, collection, insert_count): + ''' + target: test insert vectors with different length of vectors + method: set different vectors as insert method params + expected: length of ids is equal to the length of vectors + ''' + nb = insert_count + future = connect.insert(collection, gen_entities(nb), _async=True) + ids = future.result() + connect.flush([collection]) + assert len(ids) == nb + + @pytest.mark.level(2) + def test_insert_async_false(self, connect, collection, insert_count): + ''' + target: test insert vectors with different length of vectors + method: set different vectors as insert method params + expected: length of ids is equal to the length of vectors + ''' + nb = insert_count + ids = connect.insert(collection, gen_entities(nb), _async=False) + # ids = future.result() + connect.flush([collection]) + assert len(ids) == nb + + def test_insert_async_callback(self, connect, collection, insert_count): + ''' + target: test insert vectors with different length of vectors + method: set different vectors as insert method params + expected: length of ids is equal to the length of vectors + ''' + nb = insert_count + future = connect.insert(collection, gen_entities(nb), _async=True, _callback=self.check_status) + future.done() + + @pytest.mark.level(2) + def test_insert_async_long(self, connect, collection): + ''' + target: test insert vectors with different length of vectors + method: set different vectors as insert method params + expected: length of ids is equal to the length of vectors + ''' + nb = 50000 + future = connect.insert(collection, gen_entities(nb), _async=True, _callback=self.check_result) + result = future.result() + assert len(result) == nb + connect.flush([collection]) + stats = connect.get_collection_stats(collection) + logging.getLogger().info(stats) + assert stats["row_count"] == nb + + @pytest.mark.level(2) + def test_insert_async_callback_timeout(self, connect, collection): + ''' + target: test insert vectors with different length of vectors + method: set different vectors as insert method params + expected: length of ids is equal to the length of vectors + ''' + nb = 100000 + future = connect.insert(collection, gen_entities(nb), _async=True, _callback=self.check_status, timeout=1) + with pytest.raises(Exception) as e: + result = future.result() + stats = connect.get_collection_stats(collection) + assert stats["row_count"] == 0 + + def test_insert_async_invalid_params(self, connect): + ''' + target: test insert vectors with different length of vectors + method: set different vectors as insert method params + expected: length of ids is equal to the length of vectors + ''' + collection_new = gen_unique_str() + future = connect.insert(collection_new, default_entities, _async=True) + with pytest.raises(Exception) as e: + result = future.result() + + def test_insert_async_invalid_params_raise_exception(self, connect, collection): + ''' + target: test insert vectors with different length of vectors + method: set different vectors as insert method params + expected: length of ids is equal to the length of vectors + ''' + entities = [] + future = connect.insert(collection, entities, _async=True) + with pytest.raises(Exception) as e: + future.result() + + +class TestInsertMultiCollections: + """ + ****************************************************************** + The following cases are used to test `insert` function + ****************************************************************** + """ + + @pytest.fixture( + scope="function", + params=gen_simple_index() + ) + def get_simple_index(self, request, connect): + logging.getLogger().info(request.param) + if str(connect._cmd("mode")) == "CPU": + if request.param["index_type"] in index_cpu_not_support(): + pytest.skip("sq8h not support in CPU mode") + return request.param + + def test_insert_entity_multi_collections(self, connect): + ''' + target: test insert entities + method: create 10 collections and insert entities into them in turn + expected: row count + ''' + collection_num = 10 + collection_list = [] + for i in range(collection_num): + collection_name = gen_unique_str(uid) + collection_list.append(collection_name) + connect.create_collection(collection_name, default_fields) + ids = connect.insert(collection_name, default_entities) + connect.flush([collection_name]) + assert len(ids) == default_nb + stats = connect.get_collection_stats(collection_name) + assert stats["row_count"] == default_nb + + @pytest.mark.timeout(ADD_TIMEOUT) + def test_drop_collection_insert_entity_another(self, connect, collection): + ''' + target: test insert vector to collection_1 after collection_2 deleted + method: delete collection_2 and insert vector to collection_1 + expected: row count equals the length of entities inserted + ''' + collection_name = gen_unique_str(uid) + connect.create_collection(collection_name, default_fields) + connect.drop_collection(collection) + ids = connect.insert(collection_name, default_entity) + connect.flush([collection_name]) + assert len(ids) == 1 + + @pytest.mark.timeout(ADD_TIMEOUT) + def test_create_index_insert_entity_another(self, connect, collection, get_simple_index): + ''' + target: test insert vector to collection_2 after build index for collection_1 + method: build index and insert vector + expected: status ok + ''' + collection_name = gen_unique_str(uid) + connect.create_collection(collection_name, default_fields) + connect.create_index(collection, field_name, get_simple_index) + ids = connect.insert(collection_name, default_entity) + assert len(ids) == 1 + # connect.drop_collection(collection_name) + + @pytest.mark.timeout(ADD_TIMEOUT) + def test_insert_entity_create_index_another(self, connect, collection, get_simple_index): + ''' + target: test insert vector to collection_2 after build index for collection_1 + method: build index and insert vector + expected: status ok + ''' + collection_name = gen_unique_str(uid) + connect.create_collection(collection_name, default_fields) + ids = connect.insert(collection, default_entity) + connect.create_index(collection_name, field_name, get_simple_index) + index = connect.describe_index(collection_name, field_name) + assert index == get_simple_index + stats = connect.get_collection_stats(collection) + assert stats["row_count"] == 1 + + @pytest.mark.timeout(ADD_TIMEOUT) + def test_insert_entity_sleep_create_index_another(self, connect, collection, get_simple_index): + ''' + target: test insert vector to collection_2 after build index for collection_1 for a while + method: build index and insert vector + expected: status ok + ''' + collection_name = gen_unique_str(uid) + connect.create_collection(collection_name, default_fields) + ids = connect.insert(collection, default_entity) + connect.flush([collection]) + connect.create_index(collection_name, field_name, get_simple_index) + stats = connect.get_collection_stats(collection) + assert stats["row_count"] == 1 + + @pytest.mark.timeout(ADD_TIMEOUT) + def test_search_entity_insert_vector_another(self, connect, collection): + ''' + target: test insert entity to collection_1 after search collection_2 + method: search collection and insert entity + expected: status ok + ''' + collection_name = gen_unique_str(uid) + connect.create_collection(collection_name, default_fields) + res = connect.search(collection, default_single_query) + logging.getLogger().debug(res) + ids = connect.insert(collection_name, default_entity) + connect.flush() + stats = connect.get_collection_stats(collection_name) + assert stats["row_count"] == 1 + + @pytest.mark.timeout(ADD_TIMEOUT) + def test_insert_entity_search_entity_another(self, connect, collection): + ''' + target: test insert entity to collection_1 after search collection_2 + method: search collection and insert entity + expected: status ok + ''' + collection_name = gen_unique_str(uid) + connect.create_collection(collection_name, default_fields) + ids = connect.insert(collection, default_entity) + result = connect.search(collection_name, default_single_query) + + @pytest.mark.timeout(ADD_TIMEOUT) + def test_insert_entity_sleep_search_entity_another(self, connect, collection): + ''' + target: test insert entity to collection_1 after search collection_2 a while + method: search collection, sleep, and insert entity + expected: status ok + ''' + collection_name = gen_unique_str(uid) + connect.create_collection(collection_name, default_fields) + ids = connect.insert(collection, default_entity) + connect.flush([collection]) + result = connect.search(collection_name, default_single_query) + + @pytest.mark.timeout(ADD_TIMEOUT) + def _test_insert_entity_during_release_collection(self, connect, collection): + ''' + target: test insert entity during release + method: release collection async, then do insert operation + expected: insert ok + ''' + for i in range(10): + connect.insert(collection, default_entities) + connect.flush([collection]) + connect.load_collection(collection) + def release(collection): + connect.release_collection(collection) + t = threading.Thread(target=release, (collection, )) + t.start() + ids = connect.insert(collection, default_entities) + assert len(ids) == default_nb + + +class TestInsertInvalid(object): + """ + Test inserting vectors with invalid collection names + """ + + @pytest.fixture( + scope="function", + params=gen_invalid_strs() + ) + def get_collection_name(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_invalid_strs() + ) + def get_tag_name(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_invalid_strs() + ) + def get_field_name(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_invalid_strs() + ) + def get_field_type(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_invalid_strs() + ) + def get_field_int_value(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_invalid_ints() + ) + def get_entity_id(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_invalid_vectors() + ) + def get_field_vectors_value(self, request): + yield request.param + + def test_insert_ids_invalid(self, connect, id_collection, get_entity_id): + ''' + target: test insert, with using customize ids, which are not int64 + method: create collection and insert entities in it + expected: raise an exception + ''' + entity_id = get_entity_id + ids = [entity_id for _ in range(default_nb)] + with pytest.raises(Exception): + connect.insert(id_collection, default_entities, ids) + + def test_insert_with_invalid_collection_name(self, connect, get_collection_name): + collection_name = get_collection_name + with pytest.raises(Exception): + connect.insert(collection_name, default_entity) + + def test_insert_with_invalid_partition_name(self, connect, collection, get_tag_name): + tag_name = get_tag_name + connect.create_partition(collection, default_tag) + if tag_name is not None: + with pytest.raises(Exception): + connect.insert(collection, default_entity, partition_tag=tag_name) + else: + connect.insert(collection, default_entity, partition_tag=tag_name) + + def test_insert_with_invalid_field_name(self, connect, collection, get_field_name): + tmp_entity = update_field_name(copy.deepcopy(default_entity), "int64", get_field_name) + with pytest.raises(Exception): + connect.insert(collection, tmp_entity) + + def test_insert_with_invalid_field_type(self, connect, collection, get_field_type): + field_type = get_field_type + tmp_entity = update_field_type(copy.deepcopy(default_entity), 'float', field_type) + with pytest.raises(Exception): + connect.insert(collection, tmp_entity) + + def test_insert_with_invalid_field_value(self, connect, collection, get_field_int_value): + field_value = get_field_int_value + tmp_entity = update_field_type(copy.deepcopy(default_entity), 'int64', field_value) + with pytest.raises(Exception): + connect.insert(collection, tmp_entity) + + def test_insert_with_invalid_field_entity_value(self, connect, collection, get_field_vectors_value): + tmp_entity = copy.deepcopy(default_entity) + src_vector = tmp_entity[-1]["values"] + src_vector[0][1] = get_field_vectors_value + with pytest.raises(Exception): + connect.insert(collection, tmp_entity) + + +class TestInsertInvalidBinary(object): + """ + Test inserting vectors with invalid collection names + """ + + @pytest.fixture( + scope="function", + params=gen_invalid_strs() + ) + def get_collection_name(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_invalid_strs() + ) + def get_tag_name(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_invalid_strs() + ) + def get_field_name(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_invalid_strs() + ) + def get_field_type(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_invalid_strs() + ) + def get_field_int_value(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_invalid_ints() + ) + def get_entity_id(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_invalid_vectors() + ) + def get_field_vectors_value(self, request): + yield request.param + + @pytest.mark.level(2) + def test_insert_with_invalid_field_name(self, connect, binary_collection, get_field_name): + tmp_entity = update_field_name(copy.deepcopy(default_binary_entity), "int64", get_field_name) + with pytest.raises(Exception): + connect.insert(binary_collection, tmp_entity) + + @pytest.mark.level(2) + def test_insert_with_invalid_field_value(self, connect, binary_collection, get_field_int_value): + tmp_entity = update_field_type(copy.deepcopy(default_binary_entity), 'int64', get_field_int_value) + with pytest.raises(Exception): + connect.insert(binary_collection, tmp_entity) + + @pytest.mark.level(2) + def test_insert_with_invalid_field_entity_value(self, connect, binary_collection, get_field_vectors_value): + tmp_entity = copy.deepcopy(default_binary_entity) + src_vector = tmp_entity[-1]["values"] + src_vector[0][1] = get_field_vectors_value + with pytest.raises(Exception): + connect.insert(binary_collection, tmp_entity) + + @pytest.mark.level(2) + def test_insert_ids_invalid(self, connect, binary_id_collection, get_entity_id): + ''' + target: test insert, with using customize ids, which are not int64 + method: create collection and insert entities in it + expected: raise an exception + ''' + entity_id = get_entity_id + ids = [entity_id for _ in range(default_nb)] + with pytest.raises(Exception): + connect.insert(binary_id_collection, default_binary_entities, ids) + + @pytest.mark.level(2) + def test_insert_with_invalid_field_type(self, connect, binary_collection, get_field_type): + field_type = get_field_type + tmp_entity = update_field_type(copy.deepcopy(default_binary_entity), 'int64', field_type) + with pytest.raises(Exception): + connect.insert(binary_collection, tmp_entity) + + @pytest.mark.level(2) + def test_insert_with_invalid_field_entity_value(self, connect, binary_collection, get_field_vectors_value): + tmp_entity = copy.deepcopy(default_binary_entities) + src_vector = tmp_entity[-1]["values"] + src_vector[1] = get_field_vectors_value + with pytest.raises(Exception): + connect.insert(binary_collection, tmp_entity) diff --git a/tests/python_test/entity/test_list_id_in_segment.py b/tests/python_test/entity/test_list_id_in_segment.py new file mode 100644 index 0000000000..dd95c86951 --- /dev/null +++ b/tests/python_test/entity/test_list_id_in_segment.py @@ -0,0 +1,318 @@ +import time +import random +import pdb +import threading +import logging +from multiprocessing import Pool, Process +import pytest +from utils import * +from constants import * + +uid = "list_id_in_segment" + +def get_segment_id(connect, collection, nb=1, vec_type='float', index_params=None): + if vec_type != "float": + vectors, entities = gen_binary_entities(nb) + else: + entities = gen_entities(nb) + ids = connect.bulk_insert(collection, entities) + connect.flush([collection]) + if index_params: + if vec_type == 'float': + connect.create_index(collection, default_float_vec_field_name, index_params) + else: + connect.create_index(collection, default_binary_vec_field_name, index_params) + stats = connect.get_collection_stats(collection) + return ids, stats["partitions"][0]["segments"][0]["id"] + + +class TestListIdInSegmentBase: + + """ + ****************************************************************** + The following cases are used to test `list_id_in_segment` function + ****************************************************************** + """ + def test_list_id_in_segment_collection_name_None(self, connect, collection): + ''' + target: get vector ids where collection name is None + method: call list_id_in_segment with the collection_name: None + expected: exception raised + ''' + collection_name = None + ids, segment_id = get_segment_id(connect, collection) + with pytest.raises(Exception) as e: + connect.list_id_in_segment(collection_name, segment_id) + + def test_list_id_in_segment_collection_name_not_existed(self, connect, collection): + ''' + target: get vector ids where collection name does not exist + method: call list_id_in_segment with a random collection_name, which is not in db + expected: status not ok + ''' + collection_name = gen_unique_str(uid) + ids, segment_id = get_segment_id(connect, collection) + with pytest.raises(Exception) as e: + vector_ids = connect.list_id_in_segment(collection_name, segment_id) + + @pytest.fixture( + scope="function", + params=gen_invalid_strs() + ) + def get_collection_name(self, request): + yield request.param + + @pytest.mark.level(2) + def test_list_id_in_segment_collection_name_invalid(self, connect, collection, get_collection_name): + ''' + target: get vector ids where collection name is invalid + method: call list_id_in_segment with invalid collection_name + expected: status not ok + ''' + collection_name = get_collection_name + ids, segment_id = get_segment_id(connect, collection) + with pytest.raises(Exception) as e: + connect.list_id_in_segment(collection_name, segment_id) + + def test_list_id_in_segment_name_None(self, connect, collection): + ''' + target: get vector ids where segment name is None + method: call list_id_in_segment with the name: None + expected: exception raised + ''' + ids, segment_id = get_segment_id(connect, collection) + segment = None + with pytest.raises(Exception) as e: + vector_ids = connect.list_id_in_segment(collection, segment) + + def test_list_id_in_segment_name_not_existed(self, connect, collection): + ''' + target: get vector ids where segment name does not exist + method: call list_id_in_segment with a random segment name + expected: status not ok + ''' + ids, seg_id = get_segment_id(connect, collection) + # segment = gen_unique_str(uid) + with pytest.raises(Exception) as e: + vector_ids = connect.list_id_in_segment(collection, seg_id + 10000) + + @pytest.mark.level(2) + def test_list_id_in_segment_without_index_A(self, connect, collection): + ''' + target: get vector ids when there is no index + method: call list_id_in_segment and check if the segment contains vectors + expected: status ok + ''' + nb = 1 + ids, seg_id = get_segment_id(connect, collection, nb=nb) + vector_ids = connect.list_id_in_segment(collection, seg_id) + # vector_ids should match ids + assert len(vector_ids) == nb + assert vector_ids[0] == ids[0] + + @pytest.mark.level(2) + def test_list_id_in_segment_without_index_B(self, connect, collection): + ''' + target: get vector ids when there is no index but with partition + method: create partition, add vectors to it and call list_id_in_segment, check if the segment contains vectors + expected: status ok + ''' + nb = 10 + entities = gen_entities(nb) + connect.create_partition(collection, default_tag) + ids = connect.bulk_insert(collection, entities, partition_tag=default_tag) + connect.flush([collection]) + stats = connect.get_collection_stats(collection) + assert stats["partitions"][1]["tag"] == default_tag + vector_ids = connect.list_id_in_segment(collection, stats["partitions"][1]["segments"][0]["id"]) + # vector_ids should match ids + assert len(vector_ids) == nb + for i in range(nb): + assert vector_ids[i] == ids[i] + + @pytest.fixture( + scope="function", + params=gen_simple_index() + ) + def get_simple_index(self, request, connect): + if str(connect._cmd("mode")) == "CPU": + if request.param["index_type"] in index_cpu_not_support(): + pytest.skip("CPU not support index_type: ivf_sq8h") + return request.param + + @pytest.mark.level(2) + def test_list_id_in_segment_with_index_A(self, connect, collection, get_simple_index): + ''' + target: get vector ids when there is index + method: call list_id_in_segment and check if the segment contains vectors + expected: status ok + ''' + ids, seg_id = get_segment_id(connect, collection, nb=default_nb, index_params=get_simple_index) + try: + connect.list_id_in_segment(collection, seg_id) + except Exception as e: + assert False, str(e) + # TODO: + + @pytest.mark.level(2) + def test_list_id_in_segment_with_index_B(self, connect, collection, get_simple_index): + ''' + target: get vector ids when there is index and with partition + method: create partition, add vectors to it and call list_id_in_segment, check if the segment contains vectors + expected: status ok + ''' + connect.create_partition(collection, default_tag) + ids = connect.bulk_insert(collection, default_entities, partition_tag=default_tag) + connect.flush([collection]) + stats = connect.get_collection_stats(collection) + assert stats["partitions"][1]["tag"] == default_tag + try: + connect.list_id_in_segment(collection, stats["partitions"][1]["segments"][0]["id"]) + except Exception as e: + assert False, str(e) + # vector_ids should match ids + # TODO + + def test_list_id_in_segment_after_delete_vectors(self, connect, collection): + ''' + target: get vector ids after vectors are deleted + method: add vectors and delete a few, call list_id_in_segment + expected: status ok, vector_ids decreased after vectors deleted + ''' + nb = 2 + ids, seg_id = get_segment_id(connect, collection, nb=nb) + delete_ids = [ids[0]] + status = connect.delete_entity_by_id(collection, delete_ids) + connect.flush([collection]) + stats = connect.get_collection_stats(collection) + vector_ids = connect.list_id_in_segment(collection, stats["partitions"][0]["segments"][0]["id"]) + assert len(vector_ids) == 1 + assert vector_ids[0] == ids[1] + + @pytest.mark.level(2) + def test_list_id_in_segment_after_delete_vectors(self, connect, collection): + ''' + target: get vector ids after vectors are deleted + method: add vectors and delete a few, call list_id_in_segment + expected: vector_ids decreased after vectors deleted + ''' + nb = 60 + delete_length = 10 + ids, seg_id = get_segment_id(connect, collection, nb=nb) + delete_ids = ids[:delete_length] + status = connect.delete_entity_by_id(collection, delete_ids) + connect.flush([collection]) + stats = connect.get_collection_stats(collection) + vector_ids = connect.list_id_in_segment(collection, stats["partitions"][0]["segments"][0]["id"]) + assert len(vector_ids) == nb - delete_length + assert vector_ids[0] == ids[delete_length] + + @pytest.mark.level(2) + def test_list_id_in_segment_with_index_ip(self, connect, collection, get_simple_index): + ''' + target: get vector ids when there is index + method: call list_id_in_segment and check if the segment contains vectors + expected: ids returned in ids inserted + ''' + get_simple_index["metric_type"] = "IP" + ids, seg_id = get_segment_id(connect, collection, nb=default_nb, index_params=get_simple_index) + vector_ids = connect.list_id_in_segment(collection, seg_id) + # TODO: + segment_row_limit = connect.get_collection_info(collection)["segment_row_limit"] + assert vector_ids[0:segment_row_limit] == ids[0:segment_row_limit] + +class TestListIdInSegmentBinary: + """ + ****************************************************************** + The following cases are used to test `list_id_in_segment` function + ****************************************************************** + """ + @pytest.mark.level(2) + def test_list_id_in_segment_without_index_A(self, connect, binary_collection): + ''' + target: get vector ids when there is no index + method: call list_id_in_segment and check if the segment contains vectors + expected: status ok + ''' + nb = 10 + vectors, entities = gen_binary_entities(nb) + ids = connect.bulk_insert(binary_collection, entities) + connect.flush([binary_collection]) + stats = connect.get_collection_stats(binary_collection) + vector_ids = connect.list_id_in_segment(binary_collection, stats["partitions"][0]["segments"][0]["id"]) + # vector_ids should match ids + assert len(vector_ids) == nb + for i in range(nb): + assert vector_ids[i] == ids[i] + + @pytest.mark.level(2) + def test_list_id_in_segment_without_index_B(self, connect, binary_collection): + ''' + target: get vector ids when there is no index but with partition + method: create partition, add vectors to it and call list_id_in_segment, check if the segment contains vectors + expected: status ok + ''' + connect.create_partition(binary_collection, default_tag) + nb = 10 + vectors, entities = gen_binary_entities(nb) + ids = connect.bulk_insert(binary_collection, entities, partition_tag=default_tag) + connect.flush([binary_collection]) + stats = connect.get_collection_stats(binary_collection) + vector_ids = connect.list_id_in_segment(binary_collection, stats["partitions"][1]["segments"][0]["id"]) + # vector_ids should match ids + assert len(vector_ids) == nb + for i in range(nb): + assert vector_ids[i] == ids[i] + + @pytest.fixture( + scope="function", + params=gen_binary_index() + ) + def get_jaccard_index(self, request, connect): + logging.getLogger().info(request.param) + if request.param["index_type"] in binary_support(): + request.param["metric_type"] = "JACCARD" + return request.param + else: + pytest.skip("not support") + + def test_list_id_in_segment_with_index_A(self, connect, binary_collection, get_jaccard_index): + ''' + target: get vector ids when there is index + method: call list_id_in_segment and check if the segment contains vectors + expected: status ok + ''' + ids, seg_id = get_segment_id(connect, binary_collection, nb=default_nb, index_params=get_jaccard_index, vec_type='binary') + vector_ids = connect.list_id_in_segment(binary_collection, seg_id) + # TODO: + + def test_list_id_in_segment_with_index_B(self, connect, binary_collection, get_jaccard_index): + ''' + target: get vector ids when there is index and with partition + method: create partition, add vectors to it and call list_id_in_segment, check if the segment contains vectors + expected: status ok + ''' + connect.create_partition(binary_collection, default_tag) + ids = connect.bulk_insert(binary_collection, default_binary_entities, partition_tag=default_tag) + connect.flush([binary_collection]) + stats = connect.get_collection_stats(binary_collection) + assert stats["partitions"][1]["tag"] == default_tag + vector_ids = connect.list_id_in_segment(binary_collection, stats["partitions"][1]["segments"][0]["id"]) + # vector_ids should match ids + # TODO + + def test_list_id_in_segment_after_delete_vectors(self, connect, binary_collection, get_jaccard_index): + ''' + target: get vector ids after vectors are deleted + method: add vectors and delete a few, call list_id_in_segment + expected: status ok, vector_ids decreased after vectors deleted + ''' + nb = 2 + ids, seg_id = get_segment_id(connect, binary_collection, nb=nb, vec_type='binary', index_params=get_jaccard_index) + delete_ids = [ids[0]] + status = connect.delete_entity_by_id(binary_collection, delete_ids) + connect.flush([binary_collection]) + stats = connect.get_collection_stats(binary_collection) + vector_ids = connect.list_id_in_segment(binary_collection, stats["partitions"][0]["segments"][0]["id"]) + assert len(vector_ids) == 1 + assert vector_ids[0] == ids[1] diff --git a/tests/python_test/entity/test_search.py b/tests/python_test/entity/test_search.py new file mode 100644 index 0000000000..69b131e139 --- /dev/null +++ b/tests/python_test/entity/test_search.py @@ -0,0 +1,1782 @@ +import time +import pdb +import copy +import logging +from multiprocessing import Pool, Process +import pytest +import numpy as np + +from milvus import DataType +from utils import * +from constants import * + +uid = "test_search" +nq = 1 +epsilon = 0.001 +field_name = default_float_vec_field_name +binary_field_name = default_binary_vec_field_name +search_param = {"nprobe": 1} + +entity = gen_entities(1, is_normal=True) +entities = gen_entities(default_nb, is_normal=True) +raw_vectors, binary_entities = gen_binary_entities(default_nb) +default_query, default_query_vecs = gen_query_vectors(field_name, entities, default_top_k, nq) +default_binary_query, default_binary_query_vecs = gen_query_vectors(binary_field_name, binary_entities, default_top_k, + nq) + + +def init_data(connect, collection, nb=1200, partition_tags=None, auto_id=True): + ''' + Generate entities and add it in collection + ''' + global entities + if nb == 1200: + insert_entities = entities + else: + insert_entities = gen_entities(nb, is_normal=True) + if partition_tags is None: + if auto_id: + ids = connect.insert(collection, insert_entities) + else: + ids = connect.insert(collection, insert_entities, ids=[i for i in range(nb)]) + else: + if auto_id: + ids = connect.insert(collection, insert_entities, partition_tag=partition_tags) + else: + ids = connect.insert(collection, insert_entities, ids=[i for i in range(nb)], partition_tag=partition_tags) + connect.flush([collection]) + return insert_entities, ids + + +def init_binary_data(connect, collection, nb=1200, insert=True, partition_tags=None): + ''' + Generate entities and add it in collection + ''' + ids = [] + global binary_entities + global raw_vectors + if nb == 1200: + insert_entities = binary_entities + insert_raw_vectors = raw_vectors + else: + insert_raw_vectors, insert_entities = gen_binary_entities(nb) + if insert is True: + if partition_tags is None: + ids = connect.insert(collection, insert_entities) + else: + ids = connect.insert(collection, insert_entities, partition_tag=partition_tags) + connect.flush([collection]) + return insert_raw_vectors, insert_entities, ids + + +class TestSearchBase: + """ + generate valid create_index params + """ + + @pytest.fixture( + scope="function", + params=gen_index() + ) + def get_index(self, request, connect): + if str(connect._cmd("mode")) == "CPU": + if request.param["index_type"] in index_cpu_not_support(): + pytest.skip("sq8h not support in CPU mode") + return request.param + + @pytest.fixture( + scope="function", + params=gen_simple_index() + ) + def get_simple_index(self, request, connect): + if str(connect._cmd("mode")) == "CPU": + if request.param["index_type"] in index_cpu_not_support(): + pytest.skip("sq8h not support in CPU mode") + return request.param + + @pytest.fixture( + scope="function", + params=gen_binary_index() + ) + def get_jaccard_index(self, request, connect): + logging.getLogger().info(request.param) + if request.param["index_type"] in binary_support(): + return request.param + else: + pytest.skip("Skip index Temporary") + + @pytest.fixture( + scope="function", + params=gen_binary_index() + ) + def get_hamming_index(self, request, connect): + logging.getLogger().info(request.param) + if request.param["index_type"] in binary_support(): + return request.param + else: + pytest.skip("Skip index Temporary") + + @pytest.fixture( + scope="function", + params=gen_binary_index() + ) + def get_structure_index(self, request, connect): + logging.getLogger().info(request.param) + if request.param["index_type"] == "FLAT": + return request.param + else: + pytest.skip("Skip index Temporary") + + """ + generate top-k params + """ + + @pytest.fixture( + scope="function", + params=[1, 10] + ) + def get_top_k(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=[1, 10, 1100] + ) + def get_nq(self, request): + yield request.param + + def test_search_flat(self, connect, collection, get_top_k, get_nq): + ''' + target: test basic search function, all the search params is correct, change top-k value + method: search with the given vectors, check the result + expected: the length of the result is top_k + ''' + top_k = get_top_k + nq = get_nq + entities, ids = init_data(connect, collection) + query, vecs = gen_query_vectors(field_name, entities, top_k, nq) + if top_k <= max_top_k: + connect.load_collection(collection) + res = connect.search(collection, query) + assert len(res[0]) == top_k + assert res[0]._distances[0] <= epsilon + assert check_id_result(res[0], ids[0]) + else: + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + + def test_search_flat_top_k(self, connect, collection, get_nq): + ''' + target: test basic search function, all the search params is correct, change top-k value + method: search with the given vectors, check the result + expected: the length of the result is top_k + ''' + top_k = 16385 + nq = get_nq + entities, ids = init_data(connect, collection) + query, vecs = gen_query_vectors(field_name, entities, top_k, nq) + if top_k <= max_top_k: + connect.load_collection(collection) + res = connect.search(collection, query) + assert len(res[0]) == top_k + assert res[0]._distances[0] <= epsilon + assert check_id_result(res[0], ids[0]) + else: + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + + def test_search_field(self, connect, collection, get_top_k, get_nq): + ''' + target: test basic search function, all the search params is correct, change top-k value + method: search with the given vectors, check the result + expected: the length of the result is top_k + ''' + top_k = get_top_k + nq = get_nq + entities, ids = init_data(connect, collection) + query, vecs = gen_query_vectors(field_name, entities, top_k, nq) + if top_k <= max_top_k: + connect.load_collection(collection) + res = connect.search(collection, query, fields=["float_vector"]) + assert len(res[0]) == top_k + assert res[0]._distances[0] <= epsilon + assert check_id_result(res[0], ids[0]) + res = connect.search(collection, query, fields=["float"]) + for i in range(nq): + assert entities[1]["values"][:nq][i] in [r.entity.get('float') for r in res[i]] + else: + with pytest.raises(Exception): + connect.search(collection, query) + + @pytest.mark.skip("delete_entity_by_id not ready") + def test_search_after_delete(self, connect, collection, get_top_k, get_nq): + ''' + target: test basic search function before and after deletion, all the search params is + correct, change top-k value. + check issue #4200 + method: search with the given vectors, check the result + expected: the deleted entities do not exist in the result. + ''' + top_k = get_top_k + nq = get_nq + + entities, ids = init_data(connect, collection, nb=10000) + first_int64_value = entities[0]["values"][0] + first_vector = entities[2]["values"][0] + + search_param = get_search_param("FLAT") + query, vecs = gen_query_vectors(field_name, entities, top_k, nq, search_params=search_param) + vecs[:] = [] + vecs.append(first_vector) + + res = None + if top_k > max_top_k: + with pytest.raises(Exception): + connect.search(collection, query, fields=['int64']) + pytest.skip("top_k value is larger than max_topp_k") + else: + res = connect.search(collection, query, fields=['int64']) + assert len(res) == 1 + assert len(res[0]) >= top_k + assert res[0][0].id == ids[0] + assert res[0][0].entity.get("int64") == first_int64_value + assert res[0]._distances[0] < epsilon + assert check_id_result(res[0], ids[0]) + + connect.delete_entity_by_id(collection, ids[:1]) + connect.flush([collection]) + + res2 = connect.search(collection, query, fields=['int64']) + assert len(res2) == 1 + assert len(res2[0]) >= top_k + assert res2[0][0].id != ids[0] + if top_k > 1: + assert res2[0][0].id == res[0][1].id + assert res2[0][0].entity.get("int64") == res[0][1].entity.get("int64") + + # TODO: + @pytest.mark.level(2) + def test_search_after_index(self, connect, collection, get_simple_index, get_top_k, get_nq): + ''' + target: test basic search function, all the search params is correct, test all index params, and build + method: search with the given vectors, check the result + expected: the length of the result is top_k + ''' + top_k = get_top_k + nq = get_nq + + index_type = get_simple_index["index_type"] + if index_type in skip_pq(): + pytest.skip("Skip PQ") + entities, ids = init_data(connect, collection) + connect.create_index(collection, field_name, get_simple_index) + search_param = get_search_param(index_type) + query, vecs = gen_query_vectors(field_name, entities, top_k, nq, search_params=search_param) + if top_k > max_top_k: + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + else: + connect.load_collection(collection) + res = connect.search(collection, query) + assert len(res) == nq + assert len(res[0]) >= top_k + assert res[0]._distances[0] < epsilon + assert check_id_result(res[0], ids[0]) + + def test_search_after_index_different_metric_type(self, connect, collection, get_simple_index): + ''' + target: test search with different metric_type + method: build index with L2, and search using IP + expected: search ok + ''' + search_metric_type = "IP" + index_type = get_simple_index["index_type"] + entities, ids = init_data(connect, collection) + connect.create_index(collection, field_name, get_simple_index) + search_param = get_search_param(index_type) + query, vecs = gen_query_vectors(field_name, entities, default_top_k, nq, metric_type=search_metric_type, + search_params=search_param) + connect.load_collection(collection) + res = connect.search(collection, query) + assert len(res) == nq + assert len(res[0]) == default_top_k + assert res[0]._distances[0] > res[0]._distances[default_top_k - 1] + + @pytest.mark.level(2) + def test_search_index_empty_partition(self, connect, collection, get_simple_index, get_top_k, get_nq): + ''' + target: test basic search function, all the search params is correct, test all index params, and build + method: add vectors into collection, search with the given vectors, check the result + expected: the length of the result is top_k, search collection with partition tag return empty + ''' + top_k = get_top_k + nq = get_nq + + index_type = get_simple_index["index_type"] + if index_type in skip_pq(): + pytest.skip("Skip PQ") + connect.create_partition(collection, default_tag) + entities, ids = init_data(connect, collection) + connect.create_index(collection, field_name, get_simple_index) + search_param = get_search_param(index_type) + query, vecs = gen_query_vectors(field_name, entities, top_k, nq, search_params=search_param) + if top_k > max_top_k: + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + else: + connect.load_collection(collection) + res = connect.search(collection, query) + assert len(res) == nq + assert len(res[0]) >= top_k + assert res[0]._distances[0] < epsilon + assert check_id_result(res[0], ids[0]) + connect.release(collection) + connect.load_partitions(collection, [default_tag]) + res = connect.search(collection, query, partition_tags=[default_tag]) + assert len(res[0]) == 0 + + @pytest.mark.level(2) + def test_search_index_partition(self, connect, collection, get_simple_index, get_top_k, get_nq): + ''' + target: test basic search function, all the search params is correct, test all index params, and build + method: search with the given vectors, check the result + expected: the length of the result is top_k + ''' + top_k = get_top_k + nq = get_nq + + index_type = get_simple_index["index_type"] + if index_type in skip_pq(): + pytest.skip("Skip PQ") + connect.create_partition(collection, default_tag) + entities, ids = init_data(connect, collection, partition_tags=default_tag) + connect.create_index(collection, field_name, get_simple_index) + search_param = get_search_param(index_type) + query, vecs = gen_query_vectors(field_name, entities, top_k, nq, search_params=search_param) + for tags in [[default_tag], [default_tag, "new_tag"]]: + if top_k > max_top_k: + with pytest.raises(Exception) as e: + res = connect.search(collection, query, partition_tags=tags) + else: + connect.load_partitions(collection, tags) + res = connect.search(collection, query, partition_tags=tags) + assert len(res) == nq + assert len(res[0]) >= top_k + assert res[0]._distances[0] < epsilon + assert check_id_result(res[0], ids[0]) + + @pytest.mark.level(2) + def test_search_index_partition_not_existed(self, connect, collection, get_top_k, get_nq): + ''' + target: test basic search function, all the search params is correct, test all index params, and build + method: search with the given vectors and tag (tag name not existed in collection), check the result + expected: error raised + ''' + top_k = get_top_k + nq = get_nq + entities, ids = init_data(connect, collection) + connect.create_index(collection, field_name, get_simple_index) + query, vecs = gen_query_vectors(field_name, entities, top_k, nq) + if top_k > max_top_k: + with pytest.raises(Exception) as e: + res = connect.search(collection, query, partition_tags=["new_tag"]) + else: + connect.load_partitions(collection, ["new_tag"]) + res = connect.search(collection, query, partition_tags=["new_tag"]) + assert len(res) == nq + assert len(res[0]) == 0 + + @pytest.mark.level(2) + def test_search_index_partitions(self, connect, collection, get_simple_index, get_top_k): + ''' + target: test basic search function, all the search params is correct, test all index params, and build + method: search collection with the given vectors and tags, check the result + expected: the length of the result is top_k + ''' + top_k = get_top_k + nq = 2 + new_tag = "new_tag" + index_type = get_simple_index["index_type"] + if index_type in skip_pq(): + pytest.skip("Skip PQ") + connect.create_partition(collection, default_tag) + connect.create_partition(collection, new_tag) + entities, ids = init_data(connect, collection, partition_tags=default_tag) + new_entities, new_ids = init_data(connect, collection, nb=6001, partition_tags=new_tag) + connect.create_index(collection, field_name, get_simple_index) + search_param = get_search_param(index_type) + query, vecs = gen_query_vectors(field_name, entities, top_k, nq, search_params=search_param) + if top_k > max_top_k: + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + else: + connect.load_collection(collection) + res = connect.search(collection, query) + assert check_id_result(res[0], ids[0]) + assert not check_id_result(res[1], new_ids[0]) + assert res[0]._distances[0] < epsilon + assert res[1]._distances[0] < epsilon + res = connect.search(collection, query, partition_tags=[new_tag]) + assert res[0]._distances[0] > epsilon + assert res[1]._distances[0] > epsilon + connect.release_collection(collection) + + @pytest.mark.level(2) + def test_search_index_partitions_B(self, connect, collection, get_simple_index, get_top_k): + ''' + target: test basic search function, all the search params is correct, test all index params, and build + method: search collection with the given vectors and tags, check the result + expected: the length of the result is top_k + ''' + top_k = get_top_k + nq = 2 + tag = "tag" + new_tag = "new_tag" + index_type = get_simple_index["index_type"] + if index_type in skip_pq(): + pytest.skip("Skip PQ") + connect.create_partition(collection, tag) + connect.create_partition(collection, new_tag) + entities, ids = init_data(connect, collection, partition_tags=tag) + new_entities, new_ids = init_data(connect, collection, nb=6001, partition_tags=new_tag) + connect.create_index(collection, field_name, get_simple_index) + search_param = get_search_param(index_type) + query, vecs = gen_query_vectors(field_name, new_entities, top_k, nq, search_params=search_param) + if top_k > max_top_k: + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + else: + connect.load_collection(collection) + res = connect.search(collection, query, partition_tags=["(.*)tag"]) + assert not check_id_result(res[0], ids[0]) + assert res[0]._distances[0] < epsilon + assert res[1]._distances[0] < epsilon + res = connect.search(collection, query, partition_tags=["new(.*)"]) + assert res[0]._distances[0] < epsilon + assert res[1]._distances[0] < epsilon + connect.release_collection(collection) + + # + # test for ip metric + # + @pytest.mark.level(2) + def test_search_ip_flat(self, connect, collection, get_simple_index, get_top_k, get_nq): + ''' + target: test basic search function, all the search params is correct, change top-k value + method: search with the given vectors, check the result + expected: the length of the result is top_k + ''' + top_k = get_top_k + nq = get_nq + entities, ids = init_data(connect, collection) + query, vecs = gen_query_vectors(field_name, entities, top_k, nq, metric_type="IP") + connect.load_collection(collection) + res = connect.search(collection, query) + assert len(res[0]) == top_k + assert res[0]._distances[0] >= 1 - gen_inaccuracy(res[0]._distances[0]) + assert check_id_result(res[0], ids[0]) + + @pytest.mark.level(2) + def test_search_ip_after_index(self, connect, collection, get_simple_index, get_top_k, get_nq): + ''' + target: test basic search function, all the search params is correct, test all index params, and build + method: search with the given vectors, check the result + expected: the length of the result is top_k + ''' + top_k = get_top_k + nq = get_nq + + index_type = get_simple_index["index_type"] + if index_type in skip_pq(): + pytest.skip("Skip PQ") + entities, ids = init_data(connect, collection) + get_simple_index["metric_type"] = "IP" + connect.create_index(collection, field_name, get_simple_index) + search_param = get_search_param(index_type) + query, vecs = gen_query_vectors(field_name, entities, top_k, nq, metric_type="IP", search_params=search_param) + connect.load_collection(collection) + res = connect.search(collection, query) + assert len(res) == nq + assert len(res[0]) >= top_k + assert check_id_result(res[0], ids[0]) + assert res[0]._distances[0] >= 1 - gen_inaccuracy(res[0]._distances[0]) + + @pytest.mark.level(2) + def test_search_ip_index_empty_partition(self, connect, collection, get_simple_index, get_top_k, get_nq): + ''' + target: test basic search function, all the search params is correct, test all index params, and build + method: add vectors into collection, search with the given vectors, check the result + expected: the length of the result is top_k, search collection with partition tag return empty + ''' + top_k = get_top_k + nq = get_nq + metric_type = "IP" + index_type = get_simple_index["index_type"] + if index_type in skip_pq(): + pytest.skip("Skip PQ") + connect.create_partition(collection, default_tag) + entities, ids = init_data(connect, collection) + get_simple_index["metric_type"] = metric_type + connect.create_index(collection, field_name, get_simple_index) + search_param = get_search_param(index_type) + query, vecs = gen_query_vectors(field_name, entities, top_k, nq, metric_type=metric_type, + search_params=search_param) + if top_k > max_top_k: + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + else: + connect.load_collection(collection) + res = connect.search(collection, query) + assert len(res) == nq + assert len(res[0]) >= top_k + assert res[0]._distances[0] >= 1 - gen_inaccuracy(res[0]._distances[0]) + assert check_id_result(res[0], ids[0]) + res = connect.search(collection, query, partition_tags=[default_tag]) + assert len(res[0]) == 0 + + @pytest.mark.level(2) + def test_search_ip_index_partitions(self, connect, collection, get_simple_index, get_top_k): + ''' + target: test basic search function, all the search params is correct, test all index params, and build + method: search collection with the given vectors and tags, check the result + expected: the length of the result is top_k + ''' + top_k = get_top_k + nq = 2 + metric_type = "IP" + new_tag = "new_tag" + index_type = get_simple_index["index_type"] + if index_type in skip_pq(): + pytest.skip("Skip PQ") + connect.create_partition(collection, default_tag) + connect.create_partition(collection, new_tag) + entities, ids = init_data(connect, collection, partition_tags=default_tag) + new_entities, new_ids = init_data(connect, collection, nb=6001, partition_tags=new_tag) + get_simple_index["metric_type"] = metric_type + connect.create_index(collection, field_name, get_simple_index) + search_param = get_search_param(index_type) + query, vecs = gen_query_vectors(field_name, entities, top_k, nq, metric_type="IP", search_params=search_param) + connect.load_collection(collection) + res = connect.search(collection, query) + assert check_id_result(res[0], ids[0]) + assert not check_id_result(res[1], new_ids[0]) + assert res[0]._distances[0] >= 1 - gen_inaccuracy(res[0]._distances[0]) + assert res[1]._distances[0] >= 1 - gen_inaccuracy(res[1]._distances[0]) + res = connect.search(collection, query, partition_tags=["new_tag"]) + assert res[0]._distances[0] < 1 - gen_inaccuracy(res[0]._distances[0]) + # TODO: + # assert res[1]._distances[0] >= 1 - gen_inaccuracy(res[1]._distances[0]) + + @pytest.mark.level(2) + def test_search_without_connect(self, dis_connect, collection): + ''' + target: test search vectors without connection + method: use dis connected instance, call search method and check if search successfully + expected: raise exception + ''' + with pytest.raises(Exception) as e: + res = dis_connect.search(collection, default_query) + + def test_search_collection_not_existed(self, connect): + ''' + target: search collection not existed + method: search with the random collection_name, which is not in db + expected: status not ok + ''' + collection_name = gen_unique_str(uid) + with pytest.raises(Exception) as e: + res = connect.search(collection_name, default_query) + + def test_search_distance_l2(self, connect, collection): + ''' + target: search collection, and check the result: distance + method: compare the return distance value with value computed with Euclidean + expected: the return distance equals to the computed value + ''' + nq = 2 + search_param = {"nprobe": 1} + entities, ids = init_data(connect, collection, nb=nq) + query, vecs = gen_query_vectors(field_name, entities, default_top_k, nq, rand_vector=True, + search_params=search_param) + inside_query, inside_vecs = gen_query_vectors(field_name, entities, default_top_k, nq, + search_params=search_param) + distance_0 = l2(vecs[0], inside_vecs[0]) + distance_1 = l2(vecs[0], inside_vecs[1]) + connect.load_collection(collection) + res = connect.search(collection, query) + assert abs(np.sqrt(res[0]._distances[0]) - min(distance_0, distance_1)) <= gen_inaccuracy(res[0]._distances[0]) + + # TODO + def test_search_distance_l2_after_index(self, connect, id_collection, get_simple_index): + ''' + target: search collection, and check the result: distance + method: compare the return distance value with value computed with Inner product + expected: the return distance equals to the computed value + ''' + index_type = get_simple_index["index_type"] + nq = 2 + entities, ids = init_data(connect, id_collection, auto_id=False) + connect.create_index(id_collection, field_name, get_simple_index) + search_param = get_search_param(index_type) + query, vecs = gen_query_vectors(field_name, entities, default_top_k, nq, rand_vector=True, + search_params=search_param) + inside_vecs = entities[-1]["values"] + min_distance = 1.0 + min_id = None + for i in range(default_nb): + tmp_dis = l2(vecs[0], inside_vecs[i]) + if min_distance > tmp_dis: + min_distance = tmp_dis + min_id = ids[i] + connect.load_collection(id_collection) + res = connect.search(id_collection, query) + tmp_epsilon = epsilon + check_id_result(res[0], min_id) + # if index_type in ["ANNOY", "IVF_PQ"]: + # tmp_epsilon = 0.1 + # TODO: + # assert abs(np.sqrt(res[0]._distances[0]) - min_distance) <= tmp_epsilon + + @pytest.mark.level(2) + def test_search_distance_ip(self, connect, collection): + ''' + target: search collection, and check the result: distance + method: compare the return distance value with value computed with Inner product + expected: the return distance equals to the computed value + ''' + nq = 2 + metirc_type = "IP" + search_param = {"nprobe": 1} + entities, ids = init_data(connect, collection, nb=nq) + query, vecs = gen_query_vectors(field_name, entities, default_top_k, nq, rand_vector=True, + metric_type=metirc_type, + search_params=search_param) + inside_query, inside_vecs = gen_query_vectors(field_name, entities, default_top_k, nq, + search_params=search_param) + distance_0 = ip(vecs[0], inside_vecs[0]) + distance_1 = ip(vecs[0], inside_vecs[1]) + connect.load_collection(collection) + res = connect.search(collection, query) + assert abs(res[0]._distances[0] - max(distance_0, distance_1)) <= epsilon + + def test_search_distance_ip_after_index(self, connect, id_collection, get_simple_index): + ''' + target: search collection, and check the result: distance + method: compare the return distance value with value computed with Inner product + expected: the return distance equals to the computed value + ''' + index_type = get_simple_index["index_type"] + nq = 2 + metirc_type = "IP" + entities, ids = init_data(connect, id_collection, auto_id=False) + get_simple_index["metric_type"] = metirc_type + connect.create_index(id_collection, field_name, get_simple_index) + search_param = get_search_param(index_type) + query, vecs = gen_query_vectors(field_name, entities, default_top_k, nq, rand_vector=True, + metric_type=metirc_type, + search_params=search_param) + inside_vecs = entities[-1]["values"] + max_distance = 0 + max_id = None + for i in range(default_nb): + tmp_dis = ip(vecs[0], inside_vecs[i]) + if max_distance < tmp_dis: + max_distance = tmp_dis + max_id = ids[i] + connect.load_collection(id_collection) + res = connect.search(id_collection, query) + tmp_epsilon = epsilon + check_id_result(res[0], max_id) + # if index_type in ["ANNOY", "IVF_PQ"]: + # tmp_epsilon = 0.1 + # TODO: + # assert abs(res[0]._distances[0] - max_distance) <= tmp_epsilon + + def test_search_distance_jaccard_flat_index(self, connect, binary_collection): + ''' + target: search binary_collection, and check the result: distance + method: compare the return distance value with value computed with L2 + expected: the return distance equals to the computed value + ''' + nq = 1 + int_vectors, entities, ids = init_binary_data(connect, binary_collection, nb=2) + query_int_vectors, query_entities, tmp_ids = init_binary_data(connect, binary_collection, nb=1, insert=False) + distance_0 = jaccard(query_int_vectors[0], int_vectors[0]) + distance_1 = jaccard(query_int_vectors[0], int_vectors[1]) + query, vecs = gen_query_vectors(binary_field_name, query_entities, default_top_k, nq, metric_type="JACCARD") + connect.load_collection(binary_collection) + res = connect.search(binary_collection, query) + assert abs(res[0]._distances[0] - min(distance_0, distance_1)) <= epsilon + + @pytest.mark.level(2) + def test_search_binary_flat_with_L2(self, connect, binary_collection): + ''' + target: search binary_collection, and check the result: distance + method: compare the return distance value with value computed with L2 + expected: the return distance equals to the computed value + ''' + nq = 1 + int_vectors, entities, ids = init_binary_data(connect, binary_collection, nb=2) + query_int_vectors, query_entities, tmp_ids = init_binary_data(connect, binary_collection, nb=1, insert=False) + query, vecs = gen_query_vectors(binary_field_name, query_entities, default_top_k, nq, metric_type="L2") + with pytest.raises(Exception) as e: + connect.search(binary_collection, query) + + @pytest.mark.level(2) + def test_search_distance_hamming_flat_index(self, connect, binary_collection): + ''' + target: search binary_collection, and check the result: distance + method: compare the return distance value with value computed with Inner product + expected: the return distance equals to the computed value + ''' + nq = 1 + int_vectors, entities, ids = init_binary_data(connect, binary_collection, nb=2) + query_int_vectors, query_entities, tmp_ids = init_binary_data(connect, binary_collection, nb=1, insert=False) + distance_0 = hamming(query_int_vectors[0], int_vectors[0]) + distance_1 = hamming(query_int_vectors[0], int_vectors[1]) + query, vecs = gen_query_vectors(binary_field_name, query_entities, default_top_k, nq, metric_type="HAMMING") + connect.load_collection(binary_collection) + res = connect.search(binary_collection, query) + assert abs(res[0][0].distance - min(distance_0, distance_1).astype(float)) <= epsilon + + # TODO + @pytest.mark.level(2) + def test_search_distance_substructure_flat_index(self, connect, binary_collection): + ''' + target: search binary_collection, and check the result: distance + method: search with new random binary entities and SUBSTRUCTURE metric type + expected: the return distance equals to the computed value + ''' + nq = 1 + int_vectors, entities, ids = init_binary_data(connect, binary_collection, nb=2) + query_int_vectors, query_entities, tmp_ids = init_binary_data(connect, binary_collection, nb=1, insert=False) + distance_0 = substructure(query_int_vectors[0], int_vectors[0]) + distance_1 = substructure(query_int_vectors[0], int_vectors[1]) + query, vecs = gen_query_vectors(binary_field_name, query_entities, default_top_k, nq, + metric_type="SUBSTRUCTURE") + connect.load_collection(binary_collection) + res = connect.search(binary_collection, query) + assert len(res[0]) == 0 + + @pytest.mark.level(2) + def test_search_distance_substructure_flat_index_B(self, connect, binary_collection): + ''' + target: search binary_collection, and check the result: distance + method: search with entities that related to inserted entities + expected: the return distance equals to the computed value + ''' + top_k = 3 + int_vectors, entities, ids = init_binary_data(connect, binary_collection, nb=2) + query_int_vectors, query_vecs = gen_binary_sub_vectors(int_vectors, 2) + query, vecs = gen_query_vectors(binary_field_name, entities, top_k, nq, metric_type="SUBSTRUCTURE", + replace_vecs=query_vecs) + connect.load_collection(binary_collection) + res = connect.search(binary_collection, query) + assert res[0][0].distance <= epsilon + assert res[0][0].id == ids[0] + assert res[1][0].distance <= epsilon + assert res[1][0].id == ids[1] + + @pytest.mark.level(2) + def test_search_distance_superstructure_flat_index(self, connect, binary_collection): + ''' + target: search binary_collection, and check the result: distance + method: compare the return distance value with value computed with Inner product + expected: the return distance equals to the computed value + ''' + nq = 1 + int_vectors, entities, ids = init_binary_data(connect, binary_collection, nb=2) + query_int_vectors, query_entities, tmp_ids = init_binary_data(connect, binary_collection, nb=1, insert=False) + distance_0 = superstructure(query_int_vectors[0], int_vectors[0]) + distance_1 = superstructure(query_int_vectors[0], int_vectors[1]) + query, vecs = gen_query_vectors(binary_field_name, query_entities, default_top_k, nq, + metric_type="SUPERSTRUCTURE") + connect.load_collection(binary_collection) + res = connect.search(binary_collection, query) + assert len(res[0]) == 0 + + @pytest.mark.level(2) + def test_search_distance_superstructure_flat_index_B(self, connect, binary_collection): + ''' + target: search binary_collection, and check the result: distance + method: compare the return distance value with value computed with SUPER + expected: the return distance equals to the computed value + ''' + top_k = 3 + int_vectors, entities, ids = init_binary_data(connect, binary_collection, nb=2) + query_int_vectors, query_vecs = gen_binary_super_vectors(int_vectors, 2) + query, vecs = gen_query_vectors(binary_field_name, entities, top_k, nq, metric_type="SUPERSTRUCTURE", + replace_vecs=query_vecs) + connect.load_collection(binary_collection) + res = connect.search(binary_collection, query) + assert len(res[0]) == 2 + assert len(res[1]) == 2 + assert res[0][0].id in ids + assert res[0][0].distance <= epsilon + assert res[1][0].id in ids + assert res[1][0].distance <= epsilon + + @pytest.mark.level(2) + def test_search_distance_tanimoto_flat_index(self, connect, binary_collection): + ''' + target: search binary_collection, and check the result: distance + method: compare the return distance value with value computed with Inner product + expected: the return distance equals to the computed value + ''' + nq = 1 + int_vectors, entities, ids = init_binary_data(connect, binary_collection, nb=2) + query_int_vectors, query_entities, tmp_ids = init_binary_data(connect, binary_collection, nb=1, insert=False) + distance_0 = tanimoto(query_int_vectors[0], int_vectors[0]) + distance_1 = tanimoto(query_int_vectors[0], int_vectors[1]) + query, vecs = gen_query_vectors(binary_field_name, query_entities, default_top_k, nq, metric_type="TANIMOTO") + connect.load_collection(binary_collection) + res = connect.search(binary_collection, query) + assert abs(res[0][0].distance - min(distance_0, distance_1)) <= epsilon + + @pytest.mark.level(2) + @pytest.mark.timeout(30) + def test_search_concurrent_multithreads(self, connect, args): + ''' + target: test concurrent search with multiprocessess + method: search with 10 processes, each process uses dependent connection + expected: status ok and the returned vectors should be query_records + ''' + nb = 100 + top_k = 10 + threads_num = 4 + threads = [] + collection = gen_unique_str(uid) + uri = "tcp://%s:%s" % (args["ip"], args["port"]) + # create collection + milvus = get_milvus(args["ip"], args["port"], handler=args["handler"]) + milvus.create_collection(collection, default_fields) + entities, ids = init_data(milvus, collection) + connect.load_collection(collection) + + def search(milvus): + res = milvus.search(collection, default_query) + assert len(res) == 1 + assert res[0]._entities[0].id in ids + assert res[0]._distances[0] < epsilon + + for i in range(threads_num): + milvus = get_milvus(args["ip"], args["port"], handler=args["handler"]) + t = TestThread(target=search, args=(milvus,)) + threads.append(t) + t.start() + time.sleep(0.2) + for t in threads: + t.join() + + @pytest.mark.level(2) + @pytest.mark.timeout(30) + def test_search_concurrent_multithreads_single_connection(self, connect, args): + ''' + target: test concurrent search with multiprocessess + method: search with 10 processes, each process uses dependent connection + expected: status ok and the returned vectors should be query_records + ''' + nb = 100 + top_k = 10 + threads_num = 4 + threads = [] + collection = gen_unique_str(uid) + uri = "tcp://%s:%s" % (args["ip"], args["port"]) + # create collection + milvus = get_milvus(args["ip"], args["port"], handler=args["handler"]) + milvus.create_collection(collection, default_fields) + entities, ids = init_data(milvus, collection) + connect.load_collection(collection) + + def search(milvus): + res = milvus.search(collection, default_query) + assert len(res) == 1 + assert res[0]._entities[0].id in ids + assert res[0]._distances[0] < epsilon + + for i in range(threads_num): + t = TestThread(target=search, args=(milvus,)) + threads.append(t) + t.start() + time.sleep(0.2) + for t in threads: + t.join() + + @pytest.mark.level(2) + def test_search_multi_collections(self, connect, args): + ''' + target: test search multi collections of L2 + method: add vectors into 10 collections, and search + expected: search status ok, the length of result + ''' + num = 10 + top_k = 10 + nq = 20 + for i in range(num): + collection = gen_unique_str(uid + str(i)) + connect.create_collection(collection, default_fields) + entities, ids = init_data(connect, collection) + assert len(ids) == default_nb + query, vecs = gen_query_vectors(field_name, entities, top_k, nq, search_params=search_param) + connect.load_collection(collection) + res = connect.search(collection, query) + assert len(res) == nq + for i in range(nq): + assert check_id_result(res[i], ids[i]) + assert res[i]._distances[0] < epsilon + assert res[i]._distances[1] > epsilon + + def test_query_entities_with_field_less_than_top_k(self, connect, id_collection): + """ + target: test search with field, and let return entities less than topk + method: insert entities and build ivf_ index, and search with field, n_probe=1 + expected: + """ + entities, ids = init_data(connect, id_collection, auto_id=False) + simple_index = {"index_type": "IVF_FLAT", "params": {"nlist": 200}, "metric_type": "L2"} + connect.create_index(id_collection, field_name, simple_index) + # logging.getLogger().info(connect.get_collection_info(id_collection)) + top_k = 300 + default_query, default_query_vecs = gen_query_vectors(field_name, entities, top_k, nq, + search_params={"nprobe": 1}) + expr = {"must": [gen_default_vector_expr(default_query)]} + query = update_query_expr(default_query, expr=expr) + connect.load_collection(id_collection) + res = connect.search(id_collection, query, fields=["int64"]) + assert len(res) == nq + for r in res[0]: + assert getattr(r.entity, "int64") == getattr(r.entity, "id") + + +class TestSearchDSL(object): + """ + ****************************************************************** + # The following cases are used to build invalid query expr + ****************************************************************** + """ + + def test_query_no_must(self, connect, collection): + ''' + method: build query without must expr + expected: error raised + ''' + # entities, ids = init_data(connect, collection) + query = update_query_expr(default_query, keep_old=False) + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + + def test_query_no_vector_term_only(self, connect, collection): + ''' + method: build query without vector only term + expected: error raised + ''' + # entities, ids = init_data(connect, collection) + expr = { + "must": [gen_default_term_expr] + } + query = update_query_expr(default_query, keep_old=False, expr=expr) + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + + def test_query_no_vector_range_only(self, connect, collection): + ''' + method: build query without vector only range + expected: error raised + ''' + # entities, ids = init_data(connect, collection) + expr = { + "must": [gen_default_range_expr] + } + query = update_query_expr(default_query, keep_old=False, expr=expr) + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + + def test_query_vector_only(self, connect, collection): + entities, ids = init_data(connect, collection) + connect.load_collection(collection) + res = connect.search(collection, default_query) + assert len(res) == nq + assert len(res[0]) == default_top_k + + def test_query_wrong_format(self, connect, collection): + ''' + method: build query without must expr, with wrong expr name + expected: error raised + ''' + # entities, ids = init_data(connect, collection) + expr = { + "must1": [gen_default_term_expr] + } + query = update_query_expr(default_query, keep_old=False, expr=expr) + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + + def test_query_empty(self, connect, collection): + ''' + method: search with empty query + expected: error raised + ''' + query = {} + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + + """ + ****************************************************************** + # The following cases are used to build valid query expr + ****************************************************************** + """ + + @pytest.mark.level(2) + def test_query_term_value_not_in(self, connect, collection): + ''' + method: build query with vector and term expr, with no term can be filtered + expected: filter pass + ''' + entities, ids = init_data(connect, collection) + expr = { + "must": [gen_default_vector_expr(default_query), gen_default_term_expr(values=[100000])]} + query = update_query_expr(default_query, expr=expr) + connect.load_collection(collection) + res = connect.search(collection, query) + assert len(res) == nq + assert len(res[0]) == 0 + # TODO: + + # TODO: + @pytest.mark.level(2) + def test_query_term_value_all_in(self, connect, collection): + ''' + method: build query with vector and term expr, with all term can be filtered + expected: filter pass + ''' + entities, ids = init_data(connect, collection) + expr = {"must": [gen_default_vector_expr(default_query), gen_default_term_expr(values=[1])]} + query = update_query_expr(default_query, expr=expr) + connect.load_collection(collection) + res = connect.search(collection, query) + assert len(res) == nq + assert len(res[0]) == 1 + # TODO: + + # TODO: + @pytest.mark.level(2) + def test_query_term_values_not_in(self, connect, collection): + ''' + method: build query with vector and term expr, with no term can be filtered + expected: filter pass + ''' + entities, ids = init_data(connect, collection) + expr = {"must": [gen_default_vector_expr(default_query), + gen_default_term_expr(values=[i for i in range(100000, 100010)])]} + query = update_query_expr(default_query, expr=expr) + connect.load_collection(collection) + res = connect.search(collection, query) + assert len(res) == nq + assert len(res[0]) == 0 + # TODO: + + def test_query_term_values_all_in(self, connect, collection): + ''' + method: build query with vector and term expr, with all term can be filtered + expected: filter pass + ''' + entities, ids = init_data(connect, collection) + expr = {"must": [gen_default_vector_expr(default_query), gen_default_term_expr()]} + query = update_query_expr(default_query, expr=expr) + connect.load_collection(collection) + res = connect.search(collection, query) + assert len(res) == nq + assert len(res[0]) == default_top_k + limit = default_nb // 2 + for i in range(nq): + for result in res[i]: + logging.getLogger().info(result.id) + assert result.id in ids[:limit] + # TODO: + + def test_query_term_values_parts_in(self, connect, collection): + ''' + method: build query with vector and term expr, with parts of term can be filtered + expected: filter pass + ''' + entities, ids = init_data(connect, collection) + expr = {"must": [gen_default_vector_expr(default_query), + gen_default_term_expr( + values=[i for i in range(default_nb // 2, default_nb + default_nb // 2)])]} + query = update_query_expr(default_query, expr=expr) + connect.load_collection(collection) + res = connect.search(collection, query) + assert len(res) == nq + assert len(res[0]) == default_top_k + # TODO: + + # TODO: + @pytest.mark.level(2) + def test_query_term_values_repeat(self, connect, collection): + ''' + method: build query with vector and term expr, with the same values + expected: filter pass + ''' + entities, ids = init_data(connect, collection) + expr = { + "must": [gen_default_vector_expr(default_query), + gen_default_term_expr(values=[1 for i in range(1, default_nb)])]} + query = update_query_expr(default_query, expr=expr) + connect.load_collection(collection) + res = connect.search(collection, query) + assert len(res) == nq + assert len(res[0]) == 1 + # TODO: + + def test_query_term_value_empty(self, connect, collection): + ''' + method: build query with term value empty + expected: return null + ''' + expr = {"must": [gen_default_vector_expr(default_query), gen_default_term_expr(values=[])]} + query = update_query_expr(default_query, expr=expr) + connect.load_collection(collection) + res = connect.search(collection, query) + assert len(res) == nq + assert len(res[0]) == 0 + + def test_query_complex_dsl(self, connect, collection): + ''' + method: query with complicated dsl + expected: no error raised + ''' + expr = {"must": [ + {"must": [{"should": [gen_default_term_expr(values=[1]), gen_default_range_expr()]}]}, + {"must": [gen_default_vector_expr(default_query)]} + ]} + logging.getLogger().info(expr) + query = update_query_expr(default_query, expr=expr) + logging.getLogger().info(query) + connect.load_collection(collection) + res = connect.search(collection, query) + logging.getLogger().info(res) + + """ + ****************************************************************** + # The following cases are used to build invalid term query expr + ****************************************************************** + """ + + # TODO + @pytest.mark.level(2) + def test_query_term_key_error(self, connect, collection): + ''' + method: build query with term key error + expected: Exception raised + ''' + expr = {"must": [gen_default_vector_expr(default_query), + gen_default_term_expr(keyword="terrm", values=[i for i in range(default_nb // 2)])]} + query = update_query_expr(default_query, expr=expr) + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + + @pytest.fixture( + scope="function", + params=gen_invalid_term() + ) + def get_invalid_term(self, request): + return request.param + + @pytest.mark.level(2) + def test_query_term_wrong_format(self, connect, collection, get_invalid_term): + ''' + method: build query with wrong format term + expected: Exception raised + ''' + entities, ids = init_data(connect, collection) + term = get_invalid_term + expr = {"must": [gen_default_vector_expr(default_query), term]} + query = update_query_expr(default_query, expr=expr) + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + + # TODO + @pytest.mark.level(2) + def test_query_term_field_named_term(self, connect, collection): + ''' + method: build query with field named "term" + expected: error raised + ''' + term_fields = add_field_default(default_fields, field_name="term") + collection_term = gen_unique_str("term") + connect.create_collection(collection_term, term_fields) + term_entities = add_field(entities, field_name="term") + ids = connect.insert(collection_term, term_entities) + assert len(ids) == default_nb + connect.flush([collection_term]) + # count = connect.count_entities(collection_term) + # assert count == default_nb + stats = connect.get_collection_stats(collection_term) + assert stats["row_count"] == default_nb + term_param = {"term": {"term": {"values": [i for i in range(default_nb // 2)]}}} + expr = {"must": [gen_default_vector_expr(default_query), + term_param]} + query = update_query_expr(default_query, expr=expr) + connect.load_collection(collection) + res = connect.search(collection_term, query) + assert len(res) == nq + assert len(res[0]) == default_top_k + connect.drop_collection(collection_term) + + @pytest.mark.level(2) + def test_query_term_one_field_not_existed(self, connect, collection): + ''' + method: build query with two fields term, one of it not existed + expected: exception raised + ''' + entities, ids = init_data(connect, collection) + term = gen_default_term_expr() + term["term"].update({"a": [0]}) + expr = {"must": [gen_default_vector_expr(default_query), term]} + query = update_query_expr(default_query, expr=expr) + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + + """ + ****************************************************************** + # The following cases are used to build valid range query expr + ****************************************************************** + """ + + # TODO + def test_query_range_key_error(self, connect, collection): + ''' + method: build query with range key error + expected: Exception raised + ''' + range = gen_default_range_expr(keyword="ranges") + expr = {"must": [gen_default_vector_expr(default_query), range]} + query = update_query_expr(default_query, expr=expr) + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + + @pytest.fixture( + scope="function", + params=gen_invalid_range() + ) + def get_invalid_range(self, request): + return request.param + + # TODO + @pytest.mark.level(2) + def test_query_range_wrong_format(self, connect, collection, get_invalid_range): + ''' + method: build query with wrong format range + expected: Exception raised + ''' + entities, ids = init_data(connect, collection) + range = get_invalid_range + expr = {"must": [gen_default_vector_expr(default_query), range]} + query = update_query_expr(default_query, expr=expr) + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + + @pytest.mark.level(2) + def test_query_range_string_ranges(self, connect, collection): + ''' + method: build query with invalid ranges + expected: raise Exception + ''' + entities, ids = init_data(connect, collection) + ranges = {"GT": "0", "LT": "1000"} + range = gen_default_range_expr(ranges=ranges) + expr = {"must": [gen_default_vector_expr(default_query), range]} + query = update_query_expr(default_query, expr=expr) + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + + @pytest.mark.level(2) + def test_query_range_invalid_ranges(self, connect, collection): + ''' + method: build query with invalid ranges + expected: 0 + ''' + entities, ids = init_data(connect, collection) + ranges = {"GT": default_nb, "LT": 0} + range = gen_default_range_expr(ranges=ranges) + expr = {"must": [gen_default_vector_expr(default_query), range]} + query = update_query_expr(default_query, expr=expr) + connect.load_collection(collection) + res = connect.search(collection, query) + assert len(res[0]) == 0 + + @pytest.fixture( + scope="function", + params=gen_valid_ranges() + ) + def get_valid_ranges(self, request): + return request.param + + @pytest.mark.level(2) + def test_query_range_valid_ranges(self, connect, collection, get_valid_ranges): + ''' + method: build query with valid ranges + expected: pass + ''' + entities, ids = init_data(connect, collection) + ranges = get_valid_ranges + range = gen_default_range_expr(ranges=ranges) + expr = {"must": [gen_default_vector_expr(default_query), range]} + query = update_query_expr(default_query, expr=expr) + connect.load_collection(collection) + res = connect.search(collection, query) + assert len(res) == nq + assert len(res[0]) == default_top_k + + def test_query_range_one_field_not_existed(self, connect, collection): + ''' + method: build query with two fields ranges, one of fields not existed + expected: exception raised + ''' + entities, ids = init_data(connect, collection) + range = gen_default_range_expr() + range["range"].update({"a": {"GT": 1, "LT": default_nb // 2}}) + expr = {"must": [gen_default_vector_expr(default_query), range]} + query = update_query_expr(default_query, expr=expr) + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + + """ + ************************************************************************ + # The following cases are used to build query expr multi range and term + ************************************************************************ + """ + + # TODO + @pytest.mark.level(2) + def test_query_multi_term_has_common(self, connect, collection): + ''' + method: build query with multi term with same field, and values has common + expected: pass + ''' + entities, ids = init_data(connect, collection) + term_first = gen_default_term_expr() + term_second = gen_default_term_expr(values=[i for i in range(default_nb // 3)]) + expr = {"must": [gen_default_vector_expr(default_query), term_first, term_second]} + query = update_query_expr(default_query, expr=expr) + connect.load_collection(collection) + res = connect.search(collection, query) + assert len(res) == nq + assert len(res[0]) == default_top_k + + # TODO + @pytest.mark.level(2) + def test_query_multi_term_no_common(self, connect, collection): + ''' + method: build query with multi range with same field, and ranges no common + expected: pass + ''' + entities, ids = init_data(connect, collection) + term_first = gen_default_term_expr() + term_second = gen_default_term_expr(values=[i for i in range(default_nb // 2, default_nb + default_nb // 2)]) + expr = {"must": [gen_default_vector_expr(default_query), term_first, term_second]} + query = update_query_expr(default_query, expr=expr) + connect.load_collection(collection) + res = connect.search(collection, query) + assert len(res) == nq + assert len(res[0]) == 0 + + # TODO + def test_query_multi_term_different_fields(self, connect, collection): + ''' + method: build query with multi range with same field, and ranges no common + expected: pass + ''' + entities, ids = init_data(connect, collection) + term_first = gen_default_term_expr() + term_second = gen_default_term_expr(field="float", + values=[float(i) for i in range(default_nb // 2, default_nb)]) + expr = {"must": [gen_default_vector_expr(default_query), term_first, term_second]} + query = update_query_expr(default_query, expr=expr) + connect.load_collection(collection) + res = connect.search(collection, query) + assert len(res) == nq + assert len(res[0]) == 0 + + # TODO + @pytest.mark.level(2) + def test_query_single_term_multi_fields(self, connect, collection): + ''' + method: build query with multi term, different field each term + expected: pass + ''' + entities, ids = init_data(connect, collection) + term_first = {"int64": {"values": [i for i in range(default_nb // 2)]}} + term_second = {"float": {"values": [float(i) for i in range(default_nb // 2, default_nb)]}} + term = update_term_expr({"term": {}}, [term_first, term_second]) + expr = {"must": [gen_default_vector_expr(default_query), term]} + query = update_query_expr(default_query, expr=expr) + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + + # TODO + @pytest.mark.level(2) + def test_query_multi_range_has_common(self, connect, collection): + ''' + method: build query with multi range with same field, and ranges has common + expected: pass + ''' + entities, ids = init_data(connect, collection) + range_one = gen_default_range_expr() + range_two = gen_default_range_expr(ranges={"GT": 1, "LT": default_nb // 3}) + expr = {"must": [gen_default_vector_expr(default_query), range_one, range_two]} + query = update_query_expr(default_query, expr=expr) + connect.load_collection(collection) + res = connect.search(collection, query) + assert len(res) == nq + assert len(res[0]) == default_top_k + + # TODO + @pytest.mark.level(2) + def test_query_multi_range_no_common(self, connect, collection): + ''' + method: build query with multi range with same field, and ranges no common + expected: pass + ''' + entities, ids = init_data(connect, collection) + range_one = gen_default_range_expr() + range_two = gen_default_range_expr(ranges={"GT": default_nb // 2, "LT": default_nb}) + expr = {"must": [gen_default_vector_expr(default_query), range_one, range_two]} + query = update_query_expr(default_query, expr=expr) + connect.load_collection(collection) + res = connect.search(collection, query) + assert len(res) == nq + assert len(res[0]) == 0 + + # TODO + @pytest.mark.level(2) + def test_query_multi_range_different_fields(self, connect, collection): + ''' + method: build query with multi range, different field each range + expected: pass + ''' + entities, ids = init_data(connect, collection) + range_first = gen_default_range_expr() + range_second = gen_default_range_expr(field="float", ranges={"GT": default_nb // 2, "LT": default_nb}) + expr = {"must": [gen_default_vector_expr(default_query), range_first, range_second]} + query = update_query_expr(default_query, expr=expr) + connect.load_collection(collection) + res = connect.search(collection, query) + assert len(res) == nq + assert len(res[0]) == 0 + + # TODO + @pytest.mark.level(2) + def test_query_single_range_multi_fields(self, connect, collection): + ''' + method: build query with multi range, different field each range + expected: pass + ''' + entities, ids = init_data(connect, collection) + range_first = {"int64": {"GT": 0, "LT": default_nb // 2}} + range_second = {"float": {"GT": default_nb / 2, "LT": float(default_nb)}} + range = update_range_expr({"range": {}}, [range_first, range_second]) + expr = {"must": [gen_default_vector_expr(default_query), range]} + query = update_query_expr(default_query, expr=expr) + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + + """ + ****************************************************************** + # The following cases are used to build query expr both term and range + ****************************************************************** + """ + + # TODO + @pytest.mark.level(2) + def test_query_single_term_range_has_common(self, connect, collection): + ''' + method: build query with single term single range + expected: pass + ''' + entities, ids = init_data(connect, collection) + term = gen_default_term_expr() + range = gen_default_range_expr(ranges={"GT": -1, "LT": default_nb // 2}) + expr = {"must": [gen_default_vector_expr(default_query), term, range]} + query = update_query_expr(default_query, expr=expr) + connect.load_collection(collection) + res = connect.search(collection, query) + assert len(res) == nq + assert len(res[0]) == default_top_k + + # TODO + def test_query_single_term_range_no_common(self, connect, collection): + ''' + method: build query with single term single range + expected: pass + ''' + entities, ids = init_data(connect, collection) + term = gen_default_term_expr() + range = gen_default_range_expr(ranges={"GT": default_nb // 2, "LT": default_nb}) + expr = {"must": [gen_default_vector_expr(default_query), term, range]} + query = update_query_expr(default_query, expr=expr) + connect.load_collection(collection) + res = connect.search(collection, query) + assert len(res) == nq + assert len(res[0]) == 0 + + """ + ****************************************************************** + # The following cases are used to build multi vectors query expr + ****************************************************************** + """ + + # TODO + def test_query_multi_vectors_same_field(self, connect, collection): + ''' + method: build query with two vectors same field + expected: error raised + ''' + entities, ids = init_data(connect, collection) + vector1 = default_query + vector2 = gen_query_vectors(field_name, entities, default_top_k, nq=2) + expr = { + "must": [vector1, vector2] + } + query = update_query_expr(default_query, expr=expr) + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + + +class TestSearchDSLBools(object): + """ + ****************************************************************** + # The following cases are used to build invalid query expr + ****************************************************************** + """ + + @pytest.mark.level(2) + def test_query_no_bool(self, connect, collection): + ''' + method: build query without bool expr + expected: error raised + ''' + entities, ids = init_data(connect, collection) + expr = {"bool1": {}} + query = expr + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + + def test_query_should_only_term(self, connect, collection): + ''' + method: build query without must, with should.term instead + expected: error raised + ''' + expr = {"should": gen_default_term_expr} + query = update_query_expr(default_query, keep_old=False, expr=expr) + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + + def test_query_should_only_vector(self, connect, collection): + ''' + method: build query without must, with should.vector instead + expected: error raised + ''' + expr = {"should": default_query["bool"]["must"]} + query = update_query_expr(default_query, keep_old=False, expr=expr) + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + + def test_query_must_not_only_term(self, connect, collection): + ''' + method: build query without must, with must_not.term instead + expected: error raised + ''' + expr = {"must_not": gen_default_term_expr} + query = update_query_expr(default_query, keep_old=False, expr=expr) + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + + def test_query_must_not_vector(self, connect, collection): + ''' + method: build query without must, with must_not.vector instead + expected: error raised + ''' + expr = {"must_not": default_query["bool"]["must"]} + query = update_query_expr(default_query, keep_old=False, expr=expr) + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + + def test_query_must_should(self, connect, collection): + ''' + method: build query must, and with should.term + expected: error raised + ''' + expr = {"should": gen_default_term_expr} + query = update_query_expr(default_query, keep_old=True, expr=expr) + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + + +""" +****************************************************************** +# The following cases are used to test `search` function +# with invalid collection_name, or invalid query expr +****************************************************************** +""" + + +class TestSearchInvalid(object): + """ + Test search collection with invalid collection names + """ + + @pytest.fixture( + scope="function", + params=gen_invalid_strs() + ) + def get_collection_name(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_invalid_strs() + ) + def get_invalid_partition(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_invalid_strs() + ) + def get_invalid_field(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_simple_index() + ) + def get_simple_index(self, request, connect): + if str(connect._cmd("mode")) == "CPU": + if request.param["index_type"] in index_cpu_not_support(): + pytest.skip("sq8h not support in CPU mode") + return request.param + + @pytest.mark.level(2) + def test_search_with_invalid_collection(self, connect, get_collection_name): + collection_name = get_collection_name + with pytest.raises(Exception) as e: + res = connect.search(collection_name, default_query) + + @pytest.mark.level(2) + def test_search_with_invalid_partition(self, connect, collection, get_invalid_partition): + # tag = " " + tag = get_invalid_partition + with pytest.raises(Exception) as e: + res = connect.search(collection, default_query, partition_tags=tag) + + @pytest.mark.level(2) + def test_search_with_invalid_field_name(self, connect, collection, get_invalid_field): + fields = [get_invalid_field] + with pytest.raises(Exception) as e: + res = connect.search(collection, default_query, fields=fields) + + @pytest.mark.level(1) + def test_search_with_not_existed_field(self, connect, collection): + fields = [gen_unique_str("field_name")] + with pytest.raises(Exception) as e: + res = connect.search(collection, default_query, fields=fields) + + """ + Test search collection with invalid query + """ + + @pytest.fixture( + scope="function", + params=gen_invalid_ints() + ) + def get_top_k(self, request): + yield request.param + + @pytest.mark.level(1) + def test_search_with_invalid_top_k(self, connect, collection, get_top_k): + ''' + target: test search function, with the wrong top_k + method: search with top_k + expected: raise an error, and the connection is normal + ''' + top_k = get_top_k + default_query["bool"]["must"][0]["vector"][field_name]["topk"] = top_k + with pytest.raises(Exception) as e: + res = connect.search(collection, default_query) + + """ + Test search collection with invalid search params + """ + + @pytest.fixture( + scope="function", + params=gen_invaild_search_params() + ) + def get_search_params(self, request): + yield request.param + + @pytest.mark.level(2) + def test_search_with_invalid_params(self, connect, collection, get_simple_index, get_search_params): + ''' + target: test search function, with the wrong nprobe + method: search with nprobe + expected: raise an error, and the connection is normal + ''' + search_params = get_search_params + index_type = get_simple_index["index_type"] + if index_type in ["FLAT"]: + pytest.skip("skip in FLAT index") + if index_type != search_params["index_type"]: + pytest.skip("skip if index_type not matched") + entities, ids = init_data(connect, collection) + connect.create_index(collection, field_name, get_simple_index) + query, vecs = gen_query_vectors(field_name, entities, default_top_k, 1, + search_params=search_params["search_params"]) + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + + @pytest.mark.level(2) + def test_search_with_invalid_params_binary(self, connect, binary_collection): + ''' + target: test search function, with the wrong nprobe + method: search with nprobe + expected: raise an error, and the connection is normal + ''' + nq = 1 + index_type = "BIN_IVF_FLAT" + int_vectors, entities, ids = init_binary_data(connect, binary_collection) + query_int_vectors, query_entities, tmp_ids = init_binary_data(connect, binary_collection, nb=1, insert=False) + connect.create_index(binary_collection, binary_field_name, + {"index_type": index_type, "metric_type": "JACCARD", "params": {"nlist": 128}}) + query, vecs = gen_query_vectors(binary_field_name, query_entities, default_top_k, nq, + search_params={"nprobe": 0}, metric_type="JACCARD") + with pytest.raises(Exception) as e: + res = connect.search(binary_collection, query) + + @pytest.mark.level(2) + def test_search_with_empty_params(self, connect, collection, args, get_simple_index): + ''' + target: test search function, with empty search params + method: search with params + expected: raise an error, and the connection is normal + ''' + index_type = get_simple_index["index_type"] + if args["handler"] == "HTTP": + pytest.skip("skip in http mode") + if index_type == "FLAT": + pytest.skip("skip in FLAT index") + entities, ids = init_data(connect, collection) + connect.create_index(collection, field_name, get_simple_index) + query, vecs = gen_query_vectors(field_name, entities, default_top_k, 1, search_params={}) + with pytest.raises(Exception) as e: + res = connect.search(collection, query) + + +def check_id_result(result, id): + limit_in = 5 + ids = [entity.id for entity in result] + if len(result) >= limit_in: + return id in ids[:limit_in] + else: + return id in ids diff --git a/tests/python_test/pytest.ini b/tests/python_test/pytest.ini new file mode 100644 index 0000000000..1a720a2d01 --- /dev/null +++ b/tests/python_test/pytest.ini @@ -0,0 +1,14 @@ +[pytest] +log_format = [%(asctime)s-%(levelname)s-%(name)s]: %(message)s (%(filename)s:%(lineno)s) +log_date_format = %Y-%m-%d %H:%M:%S + +log_cli = true +log_level = 20 + +timeout = 360 + +markers = + level: test level + serial + +#level = 1 diff --git a/tests/python_test/requirements.txt b/tests/python_test/requirements.txt new file mode 100644 index 0000000000..69f16f1c8a --- /dev/null +++ b/tests/python_test/requirements.txt @@ -0,0 +1,12 @@ +numpy>=1.18.0 +pylint==2.5.0 +pytest==4.5.0 +pytest-timeout==1.3.3 +pytest-repeat==0.8.0 +allure-pytest==2.7.0 +pytest-print==0.1.2 +pytest-level==0.1.1 +pytest-xdist==1.23.2 +scikit-learn>=0.19.1 +kubernetes==10.0.1 +pymilvus-test>=0.5.0,<0.6.0 \ No newline at end of file diff --git a/tests/python_test/requirements_cluster.txt b/tests/python_test/requirements_cluster.txt new file mode 100644 index 0000000000..a4f56b4f77 --- /dev/null +++ b/tests/python_test/requirements_cluster.txt @@ -0,0 +1,25 @@ +astroid==2.2.5 +atomicwrites==1.3.0 +attrs==19.1.0 +importlib-metadata==0.15 +isort==4.3.20 +lazy-object-proxy==1.4.1 +mccabe==0.6.1 +more-itertools==7.0.0 +numpy==1.16.3 +pluggy==0.12.0 +py==1.8.0 +pylint==2.5.0 +pytest==4.5.0 +pytest-timeout==1.3.3 +pytest-repeat==0.8.0 +allure-pytest==2.7.0 +pytest-print==0.1.2 +pytest-level==0.1.1 +six==1.12.0 +thrift==0.11.0 +typed-ast==1.3.5 +wcwidth==0.1.7 +wrapt==1.11.1 +zipp==0.5.1 +pymilvus>=0.2.0 diff --git a/tests/python_test/run.sh b/tests/python_test/run.sh new file mode 100644 index 0000000000..cee5b061f5 --- /dev/null +++ b/tests/python_test/run.sh @@ -0,0 +1,4 @@ +#/bin/bash + + +pytest . $@ \ No newline at end of file diff --git a/tests/python_test/stability/test_mysql.py b/tests/python_test/stability/test_mysql.py new file mode 100644 index 0000000000..7853a220f5 --- /dev/null +++ b/tests/python_test/stability/test_mysql.py @@ -0,0 +1,43 @@ +import time +import random +import pdb +import threading +import logging +from multiprocessing import Pool, Process +import pytest +from utils import * + +class TestMysql: + """ + ****************************************************************** + The following cases are used to test mysql failure + ****************************************************************** + """ + @pytest.fixture(scope="function", autouse=True) + def skip_check(self, connect, args): + if args["service_name"].find("shards") != -1: + reason = "Skip restart cases in shards mode" + logging.getLogger().info(reason) + pytest.skip(reason) + + def _test_kill_mysql_during_index(self, connect, collection, args): + big_nb = 20000 + index_param = {"nlist": 1024, "m": 16} + index_type = IndexType.IVF_PQ + vectors = gen_vectors(big_nb, default_dim) + status, ids = connect.bulk_insert(collection, vectors, ids=[i for i in range(big_nb)]) + status = connect.flush([collection]) + assert status.OK() + status, res_count = connect.count_entities(collection) + logging.getLogger().info(res_count) + assert status.OK() + assert res_count == big_nb + logging.getLogger().info("Start create index async") + status = connect.create_index(collection, index_type, index_param, _async=True) + time.sleep(2) + logging.getLogger().info("Start play mysql failure") + # pass + new_connect = get_milvus(args["ip"], args["port"], handler=args["handler"]) + status, res_count = new_connect.count_entities(collection) + assert status.OK() + assert res_count == big_nb diff --git a/tests/python_test/stability/test_restart.py b/tests/python_test/stability/test_restart.py new file mode 100644 index 0000000000..bfdc655b4f --- /dev/null +++ b/tests/python_test/stability/test_restart.py @@ -0,0 +1,315 @@ +import time +import random +import pdb +import threading +import logging +import json +from multiprocessing import Pool, Process +import pytest +from utils import * + + +uid = "wal" +TIMEOUT = 120 +insert_interval_time = 1.5 +big_nb = 100000 +field_name = "float_vector" +big_entities = gen_entities(big_nb) +default_index = {"index_type": "IVF_FLAT", "params": {"nlist": 128}, "metric_type": "L2"} + + +class TestRestartBase: + """ + ****************************************************************** + The following cases are used to test `create_partition` function + ****************************************************************** + """ + @pytest.fixture(scope="module", autouse=True) + def skip_check(self, args): + logging.getLogger().info(args) + if "service_name" not in args or not args["service_name"]: + reason = "Skip if service name not provided" + logging.getLogger().info(reason) + pytest.skip(reason) + if args["service_name"].find("shards") != -1: + reason = "Skip restart cases in shards mode" + logging.getLogger().info(reason) + pytest.skip(reason) + + @pytest.mark.level(2) + def _test_insert_flush(self, connect, collection, args): + ''' + target: return the same row count after server restart + method: call function: create collection, then insert/flush, restart server and assert row count + expected: row count keep the same + ''' + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + res_count = connect.count_entities(collection) + logging.getLogger().info(res_count) + assert res_count == 2 * nb + # restart server + logging.getLogger().info("Start restart server") + assert restart_server(args["service_name"]) + # assert row count again + new_connect = get_milvus(args["ip"], args["port"], handler=args["handler"]) + res_count = new_connect.count_entities(collection) + logging.getLogger().info(res_count) + assert res_count == 2 * nb + + @pytest.mark.level(2) + def _test_insert_during_flushing(self, connect, collection, args): + ''' + target: flushing will recover + method: call function: create collection, then insert/flushing, restart server and assert row count + expected: row count equals 0 + ''' + # disable_autoflush() + ids = connect.bulk_insert(collection, big_entities) + connect.flush([collection], _async=True) + res_count = connect.count_entities(collection) + logging.getLogger().info(res_count) + if res_count < big_nb: + # restart server + assert restart_server(args["service_name"]) + # assert row count again + new_connect = get_milvus(args["ip"], args["port"], handler=args["handler"]) + res_count_2 = new_connect.count_entities(collection) + logging.getLogger().info(res_count_2) + timeout = 300 + start_time = time.time() + while new_connect.count_entities(collection) != big_nb and (time.time() - start_time < timeout): + time.sleep(10) + logging.getLogger().info(new_connect.count_entities(collection)) + res_count_3 = new_connect.count_entities(collection) + logging.getLogger().info(res_count_3) + assert res_count_3 == big_nb + + @pytest.mark.level(2) + def _test_delete_during_flushing(self, connect, collection, args): + ''' + target: flushing will recover + method: call function: create collection, then delete/flushing, restart server and assert row count + expected: row count equals (nb - delete_length) + ''' + # disable_autoflush() + ids = connect.bulk_insert(collection, big_entities) + connect.flush([collection]) + delete_length = 1000 + delete_ids = ids[big_nb//4:big_nb//4+delete_length] + delete_res = connect.delete_entity_by_id(collection, delete_ids) + connect.flush([collection], _async=True) + res_count = connect.count_entities(collection) + logging.getLogger().info(res_count) + # restart server + assert restart_server(args["service_name"]) + # assert row count again + new_connect = get_milvus(args["ip"], args["port"], handler=args["handler"]) + res_count_2 = new_connect.count_entities(collection) + logging.getLogger().info(res_count_2) + timeout = 100 + start_time = time.time() + while new_connect.count_entities(collection) != big_nb - delete_length and (time.time() - start_time < timeout): + time.sleep(10) + logging.getLogger().info(new_connect.count_entities(collection)) + if new_connect.count_entities(collection) == big_nb - delete_length: + time.sleep(10) + res_count_3 = new_connect.count_entities(collection) + logging.getLogger().info(res_count_3) + assert res_count_3 == big_nb - delete_length + + @pytest.mark.level(2) + def _test_during_indexed(self, connect, collection, args): + ''' + target: flushing will recover + method: call function: create collection, then indexed, restart server and assert row count + expected: row count equals nb + ''' + # disable_autoflush() + ids = connect.bulk_insert(collection, big_entities) + connect.flush([collection]) + connect.create_index(collection, field_name, default_index) + res_count = connect.count_entities(collection) + logging.getLogger().info(res_count) + stats = connect.get_collection_stats(collection) + # logging.getLogger().info(stats) + # pdb.set_trace() + # restart server + assert restart_server(args["service_name"]) + # assert row count again + new_connect = get_milvus(args["ip"], args["port"], handler=args["handler"]) + assert new_connect.count_entities(collection) == big_nb + stats = connect.get_collection_stats(collection) + for file in stats["partitions"][0]["segments"][0]["files"]: + if file["field"] == field_name and file["name"] != "_raw": + assert file["data_size"] > 0 + if file["index_type"] != default_index["index_type"]: + assert False + else: + assert True + + @pytest.mark.level(2) + def _test_during_indexing(self, connect, collection, args): + ''' + target: flushing will recover + method: call function: create collection, then indexing, restart server and assert row count + expected: row count equals nb, server contitue to build index after restart + ''' + # disable_autoflush() + loop = 5 + for i in range(loop): + ids = connect.bulk_insert(collection, big_entities) + connect.flush([collection]) + connect.create_index(collection, field_name, default_index, _async=True) + res_count = connect.count_entities(collection) + logging.getLogger().info(res_count) + stats = connect.get_collection_stats(collection) + # logging.getLogger().info(stats) + # restart server + assert restart_server(args["service_name"]) + # assert row count again + new_connect = get_milvus(args["ip"], args["port"], handler=args["handler"]) + res_count_2 = new_connect.count_entities(collection) + logging.getLogger().info(res_count_2) + assert res_count_2 == loop * big_nb + status = new_connect._cmd("status") + assert json.loads(status)["indexing"] == True + # timeout = 100 + # start_time = time.time() + # while time.time() - start_time < timeout: + # time.sleep(5) + # assert new_connect.count_entities(collection) == loop * big_nb + # stats = connect.get_collection_stats(collection) + # assert stats["row_count"] == loop * big_nb + # for file in stats["partitions"][0]["segments"][0]["files"]: + # # logging.getLogger().info(file) + # if file["field"] == field_name and file["name"] != "_raw": + # assert file["data_size"] > 0 + # if file["index_type"] != default_index["index_type"]: + # continue + # for file in stats["partitions"][0]["segments"][0]["files"]: + # if file["field"] == field_name and file["name"] != "_raw": + # assert file["data_size"] > 0 + # if file["index_type"] != default_index["index_type"]: + # assert False + # else: + # assert True + + @pytest.mark.level(2) + def _test_delete_flush_during_compacting(self, connect, collection, args): + ''' + target: verify server work after restart during compaction + method: call function: create collection, then delete/flush/compacting, restart server and assert row count + call `compact` again, compact pass + expected: row count equals (nb - delete_length) + ''' + # disable_autoflush() + ids = connect.bulk_insert(collection, big_entities) + connect.flush([collection]) + delete_length = 1000 + loop = 10 + for i in range(loop): + delete_ids = ids[i*delete_length:(i+1)*delete_length] + delete_res = connect.delete_entity_by_id(collection, delete_ids) + connect.flush([collection]) + connect.compact(collection, _async=True) + res_count = connect.count_entities(collection) + logging.getLogger().info(res_count) + assert res_count == big_nb - delete_length*loop + info = connect.get_collection_stats(collection) + size_old = info["partitions"][0]["segments"][0]["data_size"] + logging.getLogger().info(size_old) + # restart server + assert restart_server(args["service_name"]) + # assert row count again + new_connect = get_milvus(args["ip"], args["port"], handler=args["handler"]) + res_count_2 = new_connect.count_entities(collection) + logging.getLogger().info(res_count_2) + assert res_count_2 == big_nb - delete_length*loop + info = connect.get_collection_stats(collection) + size_before = info["partitions"][0]["segments"][0]["data_size"] + status = connect.compact(collection) + assert status.OK() + info = connect.get_collection_stats(collection) + size_after = info["partitions"][0]["segments"][0]["data_size"] + assert size_before > size_after + + + @pytest.mark.level(2) + def _test_insert_during_flushing_multi_collections(self, connect, args): + ''' + target: flushing will recover + method: call function: create collections, then insert/flushing, restart server and assert row count + expected: row count equals 0 + ''' + # disable_autoflush() + collection_num = 2 + collection_list = [] + for i in range(collection_num): + collection_name = gen_unique_str(uid) + collection_list.append(collection_name) + connect.create_collection(collection_name, default_fields) + ids = connect.bulk_insert(collection_name, big_entities) + connect.flush(collection_list, _async=True) + res_count = connect.count_entities(collection_list[-1]) + logging.getLogger().info(res_count) + if res_count < big_nb: + # restart server + assert restart_server(args["service_name"]) + # assert row count again + new_connect = get_milvus(args["ip"], args["port"], handler=args["handler"]) + res_count_2 = new_connect.count_entities(collection_list[-1]) + logging.getLogger().info(res_count_2) + timeout = 300 + start_time = time.time() + while time.time() - start_time < timeout: + count_list = [] + break_flag = True + for index, name in enumerate(collection_list): + tmp_count = new_connect.count_entities(name) + count_list.append(tmp_count) + logging.getLogger().info(count_list) + if tmp_count != big_nb: + break_flag = False + break + if break_flag == True: + break + time.sleep(10) + for name in collection_list: + assert new_connect.count_entities(name) == big_nb + + @pytest.mark.level(2) + def _test_insert_during_flushing_multi_partitions(self, connect, collection, args): + ''' + target: flushing will recover + method: call function: create collection/partition, then insert/flushing, restart server and assert row count + expected: row count equals 0 + ''' + # disable_autoflush() + partitions_num = 2 + partitions = [] + for i in range(partitions_num): + tag_tmp = gen_unique_str() + partitions.append(tag_tmp) + connect.create_partition(collection, tag_tmp) + ids = connect.bulk_insert(collection, big_entities, partition_tag=tag_tmp) + connect.flush([collection], _async=True) + res_count = connect.count_entities(collection) + logging.getLogger().info(res_count) + if res_count < big_nb: + # restart server + assert restart_server(args["service_name"]) + # assert row count again + new_connect = get_milvus(args["ip"], args["port"], handler=args["handler"]) + res_count_2 = new_connect.count_entities(collection) + logging.getLogger().info(res_count_2) + timeout = 300 + start_time = time.time() + while new_connect.count_entities(collection) != big_nb * 2 and (time.time() - start_time < timeout): + time.sleep(10) + logging.getLogger().info(new_connect.count_entities(collection)) + res_count_3 = new_connect.count_entities(collection) + logging.getLogger().info(res_count_3) + assert res_count_3 == big_nb * 2 \ No newline at end of file diff --git a/tests/python_test/test_compact.py b/tests/python_test/test_compact.py new file mode 100644 index 0000000000..26929e12be --- /dev/null +++ b/tests/python_test/test_compact.py @@ -0,0 +1,722 @@ +import time +import pdb +import threading +import logging +from multiprocessing import Pool, Process +import pytest +from utils import * +from constants import * + +COMPACT_TIMEOUT = 180 +field_name = default_float_vec_field_name +binary_field_name = default_binary_vec_field_name +default_single_query = { + "bool": { + "must": [ + {"vector": {field_name: {"topk": 10, "query": gen_vectors(1, default_dim), "metric_type":"L2", + "params": {"nprobe": 10}}}} + ] + } +} +default_binary_single_query = { + "bool": { + "must": [ + {"vector": {binary_field_name: {"topk": 10, "query": gen_binary_vectors(1, default_dim), + "metric_type":"JACCARD", "params": {"nprobe": 10}}}} + ] + } +} +default_query, default_query_vecs = gen_query_vectors(binary_field_name, default_binary_entities, 1, 2) + + +def ip_query(): + query = copy.deepcopy(default_single_query) + query["bool"]["must"][0]["vector"][field_name].update({"metric_type": "IP"}) + return query + + +class TestCompactBase: + """ + ****************************************************************** + The following cases are used to test `compact` function + ****************************************************************** + """ + @pytest.mark.timeout(COMPACT_TIMEOUT) + def test_compact_collection_name_None(self, connect, collection): + ''' + target: compact collection where collection name is None + method: compact with the collection_name: None + expected: exception raised + ''' + collection_name = None + with pytest.raises(Exception) as e: + status = connect.compact(collection_name) + + @pytest.mark.timeout(COMPACT_TIMEOUT) + def test_compact_collection_name_not_existed(self, connect, collection): + ''' + target: compact collection not existed + method: compact with a random collection_name, which is not in db + expected: exception raised + ''' + collection_name = gen_unique_str("not_existed") + with pytest.raises(Exception) as e: + status = connect.compact(collection_name) + + @pytest.fixture( + scope="function", + params=gen_invalid_strs() + ) + def get_collection_name(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_invalid_ints() + ) + def get_threshold(self, request): + yield request.param + + @pytest.mark.level(2) + @pytest.mark.timeout(COMPACT_TIMEOUT) + def test_compact_collection_name_invalid(self, connect, get_collection_name): + ''' + target: compact collection with invalid name + method: compact with invalid collection_name + expected: exception raised + ''' + collection_name = get_collection_name + with pytest.raises(Exception) as e: + status = connect.compact(collection_name) + # assert not status.OK() + + @pytest.mark.level(2) + @pytest.mark.timeout(COMPACT_TIMEOUT) + def test_compact_threshold_invalid(self, connect, collection, get_threshold): + ''' + target: compact collection with invalid name + method: compact with invalid threshold + expected: exception raised + ''' + threshold = get_threshold + if threshold != None: + with pytest.raises(Exception) as e: + status = connect.compact(collection, threshold) + + @pytest.mark.level(2) + @pytest.mark.timeout(COMPACT_TIMEOUT) + def test_add_entity_and_compact(self, connect, collection): + ''' + target: test add entity and compact + method: add entity and compact collection + expected: data_size before and after Compact + ''' + # vector = gen_single_vector(dim) + ids = connect.bulk_insert(collection, default_entity) + assert len(ids) == 1 + connect.flush([collection]) + # get collection info before compact + info = connect.get_collection_stats(collection) + logging.getLogger().info(info) + size_before = info["partitions"][0]["segments"][0]["data_size"] + status = connect.compact(collection) + assert status.OK() + # get collection info after compact + info = connect.get_collection_stats(collection) + size_after = info["partitions"][0]["segments"][0]["data_size"] + assert(size_before == size_after) + + @pytest.mark.timeout(COMPACT_TIMEOUT) + def test_insert_and_compact(self, connect, collection): + ''' + target: test add entities and compact + method: add entities and compact collection + expected: data_size before and after Compact + ''' + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + # get collection info before compact + info = connect.get_collection_stats(collection) + # assert status.OK() + size_before = info["partitions"][0]["segments"][0]["data_size"] + status = connect.compact(collection) + assert status.OK() + # get collection info after compact + info = connect.get_collection_stats(collection) + # assert status.OK() + size_after = info["partitions"][0]["segments"][0]["data_size"] + assert(size_before == size_after) + + @pytest.mark.timeout(COMPACT_TIMEOUT) + def test_insert_delete_part_and_compact(self, connect, collection): + ''' + target: test add entities, delete part of them and compact + method: add entities, delete a few and compact collection + expected: status ok, data size maybe is smaller after compact + ''' + ids = connect.bulk_insert(collection, default_entities) + assert len(ids) == default_nb + connect.flush([collection]) + delete_ids = [ids[0], ids[-1]] + status = connect.delete_entity_by_id(collection, delete_ids) + assert status.OK() + connect.flush([collection]) + # get collection info before compact + info = connect.get_collection_stats(collection) + logging.getLogger().info(info["partitions"]) + size_before = info["partitions"][0]["data_size"] + logging.getLogger().info(size_before) + status = connect.compact(collection) + assert status.OK() + # get collection info after compact + info = connect.get_collection_stats(collection) + logging.getLogger().info(info["partitions"]) + size_after = info["partitions"][0]["data_size"] + logging.getLogger().info(size_after) + assert(size_before >= size_after) + + @pytest.mark.timeout(COMPACT_TIMEOUT) + def test_insert_delete_part_and_compact_threshold(self, connect, collection): + ''' + target: test add entities, delete part of them and compact + method: add entities, delete a few and compact collection + expected: status ok, data size maybe is smaller after compact + ''' + ids = connect.bulk_insert(collection, default_entities) + assert len(ids) == default_nb + connect.flush([collection]) + delete_ids = [ids[0], ids[-1]] + status = connect.delete_entity_by_id(collection, delete_ids) + assert status.OK() + connect.flush([collection]) + # get collection info before compact + info = connect.get_collection_stats(collection) + logging.getLogger().info(info["partitions"]) + size_before = info["partitions"][0]["data_size"] + logging.getLogger().info(size_before) + status = connect.compact(collection, 0.1) + assert status.OK() + # get collection info after compact + info = connect.get_collection_stats(collection) + logging.getLogger().info(info["partitions"]) + size_after = info["partitions"][0]["data_size"] + logging.getLogger().info(size_after) + assert(size_before >= size_after) + + @pytest.mark.level(2) + @pytest.mark.timeout(COMPACT_TIMEOUT) + def test_insert_delete_all_and_compact(self, connect, collection): + ''' + target: test add entities, delete them and compact + method: add entities, delete all and compact collection + expected: status ok, no data size in collection info because collection is empty + ''' + ids = connect.bulk_insert(collection, default_entities) + assert len(ids) == default_nb + connect.flush([collection]) + status = connect.delete_entity_by_id(collection, ids) + assert status.OK() + connect.flush([collection]) + # get collection info before compact + info = connect.get_collection_stats(collection) + status = connect.compact(collection) + assert status.OK() + # get collection info after compact + info = connect.get_collection_stats(collection) + logging.getLogger().info(info["partitions"]) + assert not info["partitions"][0]["segments"] + + # TODO: enable + @pytest.mark.level(2) + @pytest.mark.timeout(COMPACT_TIMEOUT) + def test_insert_partition_delete_half_and_compact(self, connect, collection): + ''' + target: test add entities into partition, delete them and compact + method: add entities, delete half of entities in partition and compact collection + expected: status ok, data_size less than the older version + ''' + connect.create_partition(collection, default_tag) + assert connect.has_partition(collection, default_tag) + ids = connect.bulk_insert(collection, default_entities, partition_tag=default_tag) + connect.flush([collection]) + info = connect.get_collection_stats(collection) + logging.getLogger().info(info["partitions"]) + delete_ids = ids[:default_nb//2] + status = connect.delete_entity_by_id(collection, delete_ids) + assert status.OK() + connect.flush([collection]) + # get collection info before compact + info = connect.get_collection_stats(collection) + logging.getLogger().info(info["partitions"]) + status = connect.compact(collection) + assert status.OK() + # get collection info after compact + info_after = connect.get_collection_stats(collection) + logging.getLogger().info(info_after["partitions"]) + assert info["partitions"][1]["segments"][0]["data_size"] >= info_after["partitions"][1]["segments"][0]["data_size"] + + @pytest.fixture( + scope="function", + params=gen_simple_index() + ) + def get_simple_index(self, request, connect): + if str(connect._cmd("mode")) == "GPU": + if not request.param["index_type"] not in ivf(): + pytest.skip("Only support index_type: idmap/ivf") + if str(connect._cmd("mode")) == "CPU": + if request.param["index_type"] in index_cpu_not_support(): + pytest.skip("CPU not support index_type: ivf_sq8h") + return request.param + + @pytest.mark.level(2) + def test_compact_after_index_created(self, connect, collection, get_simple_index): + ''' + target: test compact collection after index created + method: add entities, create index, delete part of entities and compact + expected: status ok, index description no change, data size smaller after compact + ''' + count = 10 + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + connect.create_index(collection, field_name, get_simple_index) + connect.flush([collection]) + # get collection info before compact + info = connect.get_collection_stats(collection) + size_before = info["partitions"][0]["segments"][0]["data_size"] + delete_ids = ids[:default_nb//2] + status = connect.delete_entity_by_id(collection, delete_ids) + assert status.OK() + connect.flush([collection]) + status = connect.compact(collection) + assert status.OK() + # get collection info after compact + info = connect.get_collection_stats(collection) + size_after = info["partitions"][0]["segments"][0]["data_size"] + assert(size_before >= size_after) + + @pytest.mark.timeout(COMPACT_TIMEOUT) + def test_add_entity_and_compact_twice(self, connect, collection): + ''' + target: test add entity and compact twice + method: add entity and compact collection twice + expected: status ok, data size no change + ''' + ids = connect.bulk_insert(collection, default_entity) + connect.flush([collection]) + # get collection info before compact + info = connect.get_collection_stats(collection) + size_before = info["partitions"][0]["segments"][0]["data_size"] + status = connect.compact(collection) + assert status.OK() + connect.flush([collection]) + # get collection info after compact + info = connect.get_collection_stats(collection) + size_after = info["partitions"][0]["segments"][0]["data_size"] + assert(size_before == size_after) + status = connect.compact(collection) + assert status.OK() + # get collection info after compact twice + info = connect.get_collection_stats(collection) + size_after_twice = info["partitions"][0]["segments"][0]["data_size"] + assert(size_after == size_after_twice) + + @pytest.mark.timeout(COMPACT_TIMEOUT) + def test_insert_delete_part_and_compact_twice(self, connect, collection): + ''' + target: test add entities, delete part of them and compact twice + method: add entities, delete part and compact collection twice + expected: status ok, data size smaller after first compact, no change after second + ''' + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + delete_ids = [ids[0], ids[-1]] + status = connect.delete_entity_by_id(collection, delete_ids) + assert status.OK() + connect.flush([collection]) + # get collection info before compact + info = connect.get_collection_stats(collection) + size_before = info["partitions"][0]["data_size"] + status = connect.compact(collection) + assert status.OK() + # get collection info after compact + info = connect.get_collection_stats(collection) + size_after = info["partitions"][0]["data_size"] + assert(size_before >= size_after) + status = connect.compact(collection) + assert status.OK() + # get collection info after compact twice + info = connect.get_collection_stats(collection) + size_after_twice = info["partitions"][0]["data_size"] + assert(size_after == size_after_twice) + + @pytest.mark.timeout(COMPACT_TIMEOUT) + def test_compact_multi_collections(self, connect): + ''' + target: test compact works or not with multiple collections + method: create 50 collections, add entities into them and compact in turn + expected: status ok + ''' + nb = 100 + num_collections = 20 + entities = gen_entities(nb) + collection_list = [] + for i in range(num_collections): + collection_name = gen_unique_str("test_compact_multi_collection_%d" % i) + collection_list.append(collection_name) + connect.create_collection(collection_name, default_fields) + for i in range(num_collections): + ids = connect.bulk_insert(collection_list[i], entities) + connect.delete_entity_by_id(collection_list[i], ids[:nb//2]) + status = connect.compact(collection_list[i]) + assert status.OK() + connect.drop_collection(collection_list[i]) + + @pytest.mark.timeout(COMPACT_TIMEOUT) + def test_add_entity_after_compact(self, connect, collection): + ''' + target: test add entity after compact + method: after compact operation, add entity + expected: status ok, entity added + ''' + ids = connect.bulk_insert(collection, default_entities) + assert len(ids) == default_nb + connect.flush([collection]) + # get collection info before compact + info = connect.get_collection_stats(collection) + size_before = info["partitions"][0]["segments"][0]["data_size"] + status = connect.compact(collection) + assert status.OK() + # get collection info after compact + info = connect.get_collection_stats(collection) + size_after = info["partitions"][0]["segments"][0]["data_size"] + assert(size_before == size_after) + ids = connect.bulk_insert(collection, default_entity) + connect.flush([collection]) + res = connect.count_entities(collection) + assert res == default_nb+1 + + @pytest.mark.timeout(COMPACT_TIMEOUT) + def test_index_creation_after_compact(self, connect, collection, get_simple_index): + ''' + target: test index creation after compact + method: after compact operation, create index + expected: status ok, index description no change + ''' + ids = connect.bulk_insert(collection, default_entities) + connect.flush([collection]) + status = connect.delete_entity_by_id(collection, ids[:10]) + assert status.OK() + connect.flush([collection]) + status = connect.compact(collection) + assert status.OK() + status = connect.create_index(collection, field_name, get_simple_index) + assert status.OK() + # status, result = connect.get_index_info(collection) + + @pytest.mark.timeout(COMPACT_TIMEOUT) + def test_delete_entities_after_compact(self, connect, collection): + ''' + target: test delete entities after compact + method: after compact operation, delete entities + expected: status ok, entities deleted + ''' + ids = connect.bulk_insert(collection, default_entities) + assert len(ids) == default_nb + connect.flush([collection]) + status = connect.compact(collection) + assert status.OK() + connect.flush([collection]) + status = connect.delete_entity_by_id(collection, ids) + assert status.OK() + connect.flush([collection]) + assert connect.count_entities(collection) == 0 + + @pytest.mark.timeout(COMPACT_TIMEOUT) + def test_search_after_compact(self, connect, collection): + ''' + target: test search after compact + method: after compact operation, search vector + expected: status ok + ''' + ids = connect.bulk_insert(collection, default_entities) + assert len(ids) == default_nb + connect.flush([collection]) + status = connect.compact(collection) + assert status.OK() + query = copy.deepcopy(default_single_query) + query["bool"]["must"][0]["vector"][field_name]["query"] = [default_entity[-1]["values"][0], + default_entities[-1]["values"][0], + default_entities[-1]["values"][-1]] + res = connect.search(collection, query) + logging.getLogger().debug(res) + assert len(res) == len(query["bool"]["must"][0]["vector"][field_name]["query"]) + assert res[0]._distances[0] > epsilon + assert res[1]._distances[0] < epsilon + assert res[2]._distances[0] < epsilon + + +class TestCompactBinary: + """ + ****************************************************************** + The following cases are used to test `compact` function + ****************************************************************** + """ + @pytest.mark.timeout(COMPACT_TIMEOUT) + def test_add_entity_and_compact(self, connect, binary_collection): + ''' + target: test add binary vector and compact + method: add vector and compact collection + expected: status ok, vector added + ''' + ids = connect.bulk_insert(binary_collection, default_binary_entity) + assert len(ids) == 1 + connect.flush([binary_collection]) + # get collection info before compact + info = connect.get_collection_stats(binary_collection) + size_before = info["partitions"][0]["segments"][0]["data_size"] + status = connect.compact(binary_collection) + assert status.OK() + # get collection info after compact + info = connect.get_collection_stats(binary_collection) + size_after = info["partitions"][0]["segments"][0]["data_size"] + assert(size_before == size_after) + + @pytest.mark.timeout(COMPACT_TIMEOUT) + def test_insert_and_compact(self, connect, binary_collection): + ''' + target: test add entities with binary vector and compact + method: add entities and compact collection + expected: status ok, entities added + ''' + ids = connect.bulk_insert(binary_collection, default_binary_entities) + assert len(ids) == default_nb + connect.flush([binary_collection]) + # get collection info before compact + info = connect.get_collection_stats(binary_collection) + size_before = info["partitions"][0]["segments"][0]["data_size"] + status = connect.compact(binary_collection) + assert status.OK() + # get collection info after compact + info = connect.get_collection_stats(binary_collection) + size_after = info["partitions"][0]["segments"][0]["data_size"] + assert(size_before == size_after) + + @pytest.mark.timeout(COMPACT_TIMEOUT) + def test_insert_delete_part_and_compact(self, connect, binary_collection): + ''' + target: test add entities, delete part of them and compact + method: add entities, delete a few and compact collection + expected: status ok, data size is smaller after compact + ''' + ids = connect.bulk_insert(binary_collection, default_binary_entities) + assert len(ids) == default_nb + connect.flush([binary_collection]) + delete_ids = [ids[0], ids[-1]] + status = connect.delete_entity_by_id(binary_collection, delete_ids) + assert status.OK() + connect.flush([binary_collection]) + # get collection info before compact + info = connect.get_collection_stats(binary_collection) + logging.getLogger().info(info["partitions"]) + size_before = info["partitions"][0]["data_size"] + logging.getLogger().info(size_before) + status = connect.compact(binary_collection) + assert status.OK() + # get collection info after compact + info = connect.get_collection_stats(binary_collection) + logging.getLogger().info(info["partitions"]) + size_after = info["partitions"][0]["data_size"] + logging.getLogger().info(size_after) + assert(size_before >= size_after) + + @pytest.mark.level(2) + @pytest.mark.timeout(COMPACT_TIMEOUT) + def test_insert_delete_all_and_compact(self, connect, binary_collection): + ''' + target: test add entities, delete them and compact + method: add entities, delete all and compact collection + expected: status ok, no data size in collection info because collection is empty + ''' + ids = connect.bulk_insert(binary_collection, default_binary_entities) + assert len(ids) == default_nb + connect.flush([binary_collection]) + status = connect.delete_entity_by_id(binary_collection, ids) + assert status.OK() + connect.flush([binary_collection]) + # get collection info before compact + info = connect.get_collection_stats(binary_collection) + status = connect.compact(binary_collection) + assert status.OK() + # get collection info after compact + info = connect.get_collection_stats(binary_collection) + assert status.OK() + logging.getLogger().info(info["partitions"]) + assert not info["partitions"][0]["segments"] + + @pytest.mark.timeout(COMPACT_TIMEOUT) + def test_add_entity_and_compact_twice(self, connect, binary_collection): + ''' + target: test add entity and compact twice + method: add entity and compact collection twice + expected: status ok + ''' + ids = connect.bulk_insert(binary_collection, default_binary_entity) + assert len(ids) == 1 + connect.flush([binary_collection]) + # get collection info before compact + info = connect.get_collection_stats(binary_collection) + size_before = info["partitions"][0]["segments"][0]["data_size"] + status = connect.compact(binary_collection) + assert status.OK() + # get collection info after compact + info = connect.get_collection_stats(binary_collection) + size_after = info["partitions"][0]["segments"][0]["data_size"] + assert(size_before == size_after) + status = connect.compact(binary_collection) + assert status.OK() + # get collection info after compact twice + info = connect.get_collection_stats(binary_collection) + size_after_twice = info["partitions"][0]["segments"][0]["data_size"] + assert(size_after == size_after_twice) + + @pytest.mark.timeout(COMPACT_TIMEOUT) + def test_insert_delete_part_and_compact_twice(self, connect, binary_collection): + ''' + target: test add entities, delete part of them and compact twice + method: add entities, delete part and compact collection twice + expected: status ok, data size smaller after first compact, no change after second + ''' + ids = connect.bulk_insert(binary_collection, default_binary_entities) + assert len(ids) == default_nb + connect.flush([binary_collection]) + delete_ids = [ids[0], ids[-1]] + status = connect.delete_entity_by_id(binary_collection, delete_ids) + assert status.OK() + connect.flush([binary_collection]) + # get collection info before compact + info = connect.get_collection_stats(binary_collection) + size_before = info["partitions"][0]["data_size"] + status = connect.compact(binary_collection) + assert status.OK() + # get collection info after compact + info = connect.get_collection_stats(binary_collection) + size_after = info["partitions"][0]["data_size"] + assert(size_before >= size_after) + status = connect.compact(binary_collection) + assert status.OK() + # get collection info after compact twice + info = connect.get_collection_stats(binary_collection) + size_after_twice = info["partitions"][0]["data_size"] + assert(size_after == size_after_twice) + + @pytest.mark.timeout(COMPACT_TIMEOUT) + def test_compact_multi_collections(self, connect): + ''' + target: test compact works or not with multiple collections + method: create 10 collections, add entities into them and compact in turn + expected: status ok + ''' + nq = 100 + num_collections = 10 + tmp, entities = gen_binary_entities(nq) + collection_list = [] + for i in range(num_collections): + collection_name = gen_unique_str("test_compact_multi_collection_%d" % i) + collection_list.append(collection_name) + connect.create_collection(collection_name, default_binary_fields) + for i in range(num_collections): + ids = connect.bulk_insert(collection_list[i], entities) + assert len(ids) == nq + status = connect.delete_entity_by_id(collection_list[i], [ids[0], ids[-1]]) + assert status.OK() + connect.flush([collection_list[i]]) + status = connect.compact(collection_list[i]) + assert status.OK() + status = connect.drop_collection(collection_list[i]) + assert status.OK() + + @pytest.mark.level(2) + @pytest.mark.timeout(COMPACT_TIMEOUT) + def test_add_entity_after_compact(self, connect, binary_collection): + ''' + target: test add entity after compact + method: after compact operation, add entity + expected: status ok, entity added + ''' + ids = connect.bulk_insert(binary_collection, default_binary_entities) + connect.flush([binary_collection]) + # get collection info before compact + info = connect.get_collection_stats(binary_collection) + size_before = info["partitions"][0]["segments"][0]["data_size"] + status = connect.compact(binary_collection) + assert status.OK() + # get collection info after compact + info = connect.get_collection_stats(binary_collection) + size_after = info["partitions"][0]["segments"][0]["data_size"] + assert(size_before == size_after) + ids = connect.bulk_insert(binary_collection, default_binary_entity) + connect.flush([binary_collection]) + res = connect.count_entities(binary_collection) + assert res == default_nb + 1 + + @pytest.mark.timeout(COMPACT_TIMEOUT) + def test_delete_entities_after_compact(self, connect, binary_collection): + ''' + target: test delete entities after compact + method: after compact operation, delete entities + expected: status ok, entities deleted + ''' + ids = connect.bulk_insert(binary_collection, default_binary_entities) + connect.flush([binary_collection]) + status = connect.compact(binary_collection) + assert status.OK() + connect.flush([binary_collection]) + status = connect.delete_entity_by_id(binary_collection, ids) + assert status.OK() + connect.flush([binary_collection]) + res = connect.count_entities(binary_collection) + assert res == 0 + + @pytest.mark.level(2) + @pytest.mark.timeout(COMPACT_TIMEOUT) + def test_search_after_compact(self, connect, binary_collection): + ''' + target: test search after compact + method: after compact operation, search vector + expected: status ok + ''' + ids = connect.bulk_insert(binary_collection, default_binary_entities) + assert len(ids) == default_nb + connect.flush([binary_collection]) + status = connect.compact(binary_collection) + assert status.OK() + query_vecs = [default_raw_binary_vectors[0]] + distance = jaccard(query_vecs[0], default_raw_binary_vectors[0]) + query = copy.deepcopy(default_binary_single_query) + query["bool"]["must"][0]["vector"][binary_field_name]["query"] = [default_binary_entities[-1]["values"][0], + default_binary_entities[-1]["values"][-1]] + + res = connect.search(binary_collection, query) + assert abs(res[0]._distances[0]-distance) <= epsilon + + @pytest.mark.timeout(COMPACT_TIMEOUT) + def test_search_after_compact_ip(self, connect, collection): + ''' + target: test search after compact + method: after compact operation, search vector + expected: status ok + ''' + ids = connect.bulk_insert(collection, default_entities) + assert len(ids) == default_nb + connect.flush([collection]) + status = connect.compact(collection) + query = ip_query() + query["bool"]["must"][0]["vector"][field_name]["query"] = [default_entity[-1]["values"][0], + default_entities[-1]["values"][0], + default_entities[-1]["values"][-1]] + res = connect.search(collection, query) + logging.getLogger().info(res) + assert len(res) == len(query["bool"]["must"][0]["vector"][field_name]["query"]) + assert res[0]._distances[0] < 1 - epsilon + assert res[1]._distances[0] > 1 - epsilon + assert res[2]._distances[0] > 1 - epsilon diff --git a/tests/python_test/test_config.py b/tests/python_test/test_config.py new file mode 100644 index 0000000000..664553a0a3 --- /dev/null +++ b/tests/python_test/test_config.py @@ -0,0 +1,1402 @@ +import time +import random +import pdb +import threading +import logging +from multiprocessing import Pool, Process +import pytest +from utils import * +import ujson + +CONFIG_TIMEOUT = 80 + +class TestCacheConfig: + """ + ****************************************************************** + The following cases are used to test `get_config` function + ****************************************************************** + """ + @pytest.fixture(scope="function", autouse=True) + def skip_http_check(self, args): + if args["handler"] == "HTTP": + pytest.skip("skip in http mode") + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def reset_configs(self, connect): + ''' + reset configs so the tests are stable + ''' + relpy = connect.set_config("cache.cache_size", '4GB') + config_value = connect.get_config("cache.cache_size") + assert config_value == '4GB' + #relpy = connect.set_config("cache", "insert_buffer_size", '2GB') + #config_value = connect.get_config("cache", "insert_buffer_size") + #assert config_value == '1073741824' + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_cache_size_invalid_parent_key(self, connect, collection): + ''' + target: get invalid parent key + method: call get_config without parent_key: cache + expected: status not ok + ''' + invalid_configs = ["Cache_config", "cache config", "cache_Config", "cacheconfig"] + for config in invalid_configs: + with pytest.raises(Exception) as e: + config_value = connect.get_config(config+str(".cache_size")) + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_cache_size_invalid_child_key(self, connect, collection): + ''' + target: get invalid child key + method: call get_config without child_key: cache_size + expected: status not ok + ''' + invalid_configs = ["Cpu_cache_size", "cpu cache_size", "cpucachecapacity"] + for config in invalid_configs: + with pytest.raises(Exception) as e: + config_value = connect.get_config("cache."+config) + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_cache_size_valid(self, connect, collection): + ''' + target: get cache_size + method: call get_config correctly + expected: status ok + ''' + config_value = connect.get_config("cache.cache_size") + assert config_value + + @pytest.mark.level(2) + def test_get_insert_buffer_size_invalid_parent_key(self, connect, collection): + ''' + target: get invalid parent key + method: call get_config without parent_key: cache + expected: status not ok + ''' + invalid_configs = ["Cache_config", "cache config", "cache_Config", "cacheconfig"] + for config in invalid_configs: + with pytest.raises(Exception) as e: + config_value = connect.get_config(config+".insert_buffer_size") + + @pytest.mark.level(2) + def test_get_insert_buffer_size_invalid_child_key(self, connect, collection): + ''' + target: get invalid child key + method: call get_config without child_key: insert_buffer_size + expected: status not ok + ''' + invalid_configs = ["Insert_buffer size", "insert buffer_size", "insertbuffersize"] + for config in invalid_configs: + with pytest.raises(Exception) as e: + config_value = connect.get_config("cache."+config) + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_insert_buffer_size_valid(self, connect, collection): + ''' + target: get insert_buffer_size + method: call get_config correctly + expected: status ok + ''' + config_value = connect.get_config("cache.insert_buffer_size") + assert config_value + + @pytest.mark.level(2) + def test_get_preload_collection_invalid_child_key(self, connect, collection): + ''' + target: get invalid child key + method: call get_config without child_key: preload_collection + expected: status not ok + ''' + invalid_configs = ["preloadtable", "preload collection "] + for config in invalid_configs: + with pytest.raises(Exception) as e: + config_value = connect.get_config("cache."+config) + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_preload_collection_valid(self, connect, collection): + ''' + target: get preload_collection + method: call get_config correctly + expected: status ok + ''' + config_value = connect.get_config("cache.preload_collection") + assert config_value == '' + + """ + ****************************************************************** + The following cases are used to test `set_config` function + ****************************************************************** + """ + def get_memory_available(self, connect): + info = connect._cmd("get_system_info") + mem_info = ujson.loads(info) + mem_total = int(mem_info["memory_total"]) + mem_used = int(mem_info["memory_used"]) + logging.getLogger().info(mem_total) + logging.getLogger().info(mem_used) + mem_available = mem_total - mem_used + return int(mem_available / 1024 / 1024 / 1024) + + def get_memory_total(self, connect): + info = connect._cmd("get_system_info") + mem_info = ujson.loads(info) + mem_total = int(mem_info["memory_total"]) + return int(mem_total / 1024 / 1024 / 1024) + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_cache_size_invalid_parent_key(self, connect, collection): + ''' + target: set invalid parent key + method: call set_config without parent_key: cache + expected: status not ok + ''' + self.reset_configs(connect) + invalid_configs = ["Cache_config", "cache config", "cache_Config", "cacheconfig"] + for config in invalid_configs: + with pytest.raises(Exception) as e: + relpy = connect.set_config(config+".cache_size", '4294967296') + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_cache_invalid_child_key(self, connect, collection): + ''' + target: set invalid child key + method: call set_config with invalid child_key + expected: status not ok + ''' + self.reset_configs(connect) + invalid_configs = ["abc", 1] + for config in invalid_configs: + with pytest.raises(Exception) as e: + relpy = connect.set_config("cache."+config, '4294967296') + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_cache_size_valid(self, connect, collection): + ''' + target: set cache_size + method: call set_config correctly + expected: status ok, set successfully + ''' + self.reset_configs(connect) + relpy = connect.set_config("cache.cache_size", '2147483648') + config_value = connect.get_config("cache.cache_size") + assert config_value == '2GB' + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.level(2) + def test_set_cache_size_valid_multiple_times(self, connect, collection): + ''' + target: set cache_size + method: call set_config correctly and repeatedly + expected: status ok + ''' + self.reset_configs(connect) + for i in range(20): + relpy = connect.set_config("cache.cache_size", '4294967296') + config_value = connect.get_config("cache.cache_size") + assert config_value == '4294967296' + for i in range(20): + relpy = connect.set_config("cache.cache_size", '2147483648') + config_value = connect.get_config("cache.cache_size") + assert config_value == '2147483648' + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.level(2) + def test_set_insert_buffer_size_invalid_parent_key(self, connect, collection): + ''' + target: set invalid parent key + method: call set_config without parent_key: cache + expected: status not ok + ''' + self.reset_configs(connect) + invalid_configs = ["Cache_config", "cache config", "cache_Config", "cacheconfig"] + for config in invalid_configs: + with pytest.raises(Exception) as e: + relpy = connect.set_config(config+".insert_buffer_size", '1073741824') + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_insert_buffer_size_valid(self, connect, collection): + ''' + target: set insert_buffer_size + method: call get_config correctly + expected: status ok, set successfully + ''' + self.reset_configs(connect) + relpy = connect.set_config("cache.insert_buffer_size", '2GB') + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.level(2) + def test_set_insert_buffer_size_valid_multiple_times(self, connect, collection): + ''' + target: set insert_buffer_size + method: call get_config correctly and repeatedly + expected: status ok + ''' + self.reset_configs(connect) + for i in range(20): + with pytest.raises(Exception) as e: + relpy = connect.set_config("cache.insert_buffer_size", '1GB') + for i in range(20): + with pytest.raises(Exception) as e: + relpy = connect.set_config("cache.insert_buffer_size", '2GB') + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_cache_out_of_memory_value_A(self, connect, collection): + ''' + target: set cache_size / insert_buffer_size to be out-of-memory + method: call set_config with child values bigger than current system memory + expected: status not ok (cache_size + insert_buffer_size < system memory) + ''' + self.reset_configs(connect) + mem_total = self.get_memory_total(connect) + logging.getLogger().info(mem_total) + with pytest.raises(Exception) as e: + relpy = connect.set_config("cache.cache_size", str(int(mem_total + 1)+'')) + + + +class TestGPUConfig: + """ + ****************************************************************** + The following cases are used to test `get_config` function + ****************************************************************** + """ + @pytest.fixture(scope="function", autouse=True) + def skip_http_check(self, args): + if args["handler"] == "HTTP": + pytest.skip("skip in http mode") + + @pytest.mark.level(2) + def test_get_gpu_search_threshold_invalid_parent_key(self, connect, collection): + ''' + target: get invalid parent key + method: call get_config without parent_key: gpu + expected: status not ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + invalid_configs = ["Engine_config", "engine config"] + for config in invalid_configs: + with pytest.raises(Exception) as e: + config_value = connect.get_config(config+".gpu_search_threshold") + + @pytest.mark.level(2) + def test_get_gpu_search_threshold_invalid_child_key(self, connect, collection): + ''' + target: get invalid child key + method: call get_config without child_key: gpu_search_threshold + expected: status not ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + invalid_configs = ["Gpu_search threshold", "gpusearchthreshold"] + for config in invalid_configs: + with pytest.raises(Exception) as e: + config_value = connect.get_config("gpu."+config) + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_gpu_search_threshold_valid(self, connect, collection): + ''' + target: get gpu_search_threshold + method: call get_config correctly + expected: status ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + config_value = connect.get_config("gpu.gpu_search_threshold") + assert config_value + + """ + ****************************************************************** + The following cases are used to test `set_config` function + ****************************************************************** + """ + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_gpu_invalid_child_key(self, connect, collection): + ''' + target: set invalid child key + method: call set_config with invalid child_key + expected: status not ok + ''' + invalid_configs = ["abc", 1] + for config in invalid_configs: + with pytest.raises(Exception) as e: + relpy = connect.set_config("gpu."+config, 1000) + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_gpu_search_threshold_invalid_parent_key(self, connect, collection): + ''' + target: set invalid parent key + method: call set_config without parent_key: gpu + expected: status not ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + invalid_configs = ["Engine_config", "engine config"] + for config in invalid_configs: + with pytest.raises(Exception) as e: + relpy = connect.set_config(config+".gpu_search_threshold", 1000) + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_gpu_search_threshold_valid(self, connect, collection): + ''' + target: set gpu_search_threshold + method: call set_config correctly + expected: status ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + relpy = connect.set_config("gpu.gpu_search_threshold", 2000) + config_value = connect.get_config("gpu.gpu_search_threshold") + assert config_value == '2000' + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_gpu_invalid_values(self, connect, collection): + ''' + target: set gpu + method: call set_config with invalid child values + expected: status not ok + ''' + for i in [-1, "1000\n", "1000\t", "1000.0", 1000.35]: + with pytest.raises(Exception) as e: + relpy = connect.set_config("gpu.use_blas_threshold", i) + if str(connect._cmd("mode")) == "GPU": + with pytest.raises(Exception) as e: + relpy = connect.set_config("gpu.gpu_search_threshold", i) + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def reset_configs(self, connect): + ''' + reset configs so the tests are stable + ''' + relpy = connect.set_config("gpu.cache_size", 1) + config_value = connect.get_config("gpu.cache_size") + assert config_value == '1' + + #follows can not be changed + #relpy = connect.set_config("gpu", "enable", "true") + #config_value = connect.get_config("gpu", "enable") + #assert config_value == "true" + #relpy = connect.set_config("gpu", "search_devices", "gpu0") + #config_value = connect.get_config("gpu", "search_devices") + #assert config_value == 'gpu0' + #relpy = connect.set_config("gpu", "build_index_devices", "gpu0") + #config_value = connect.get_config("gpu", "build_index_devices") + #assert config_value == 'gpu0' + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_gpu_enable_invalid_parent_key(self, connect, collection): + ''' + target: get invalid parent key + method: call get_config without parent_key: gpu + expected: status not ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + invalid_configs = ["Gpu_resource_config", "gpu resource config", \ + "gpu_resource"] + for config in invalid_configs: + with pytest.raises(Exception) as e: + config_value = connect.get_config(config+".enable") + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_gpu_enable_invalid_child_key(self, connect, collection): + ''' + target: get invalid child key + method: call get_config without child_key: enable + expected: status not ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + invalid_configs = ["Enab_le", "enab_le ", "disable", "true"] + for config in invalid_configs: + with pytest.raises(Exception) as e: + config_value = connect.get_config("gpu."+config) + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_gpu_enable_valid(self, connect, collection): + ''' + target: get enable status + method: call get_config correctly + expected: status ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + config_value = connect.get_config("gpu.enable") + assert config_value == "true" or config_value == "false" + + @pytest.mark.level(2) + def test_get_cache_size_invalid_parent_key(self, connect, collection): + ''' + target: get invalid parent key + method: call get_config without parent_key: gpu + expected: status not ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + invalid_configs = ["Gpu_resource_config", "gpu resource config", \ + "gpu_resource"] + for config in invalid_configs: + with pytest.raises(Exception) as e: + config_value = connect.get_config(config+".cache_size") + + @pytest.mark.level(2) + def test_get_cache_size_invalid_child_key(self, connect, collection): + ''' + target: get invalid child key + method: call get_config without child_key: cache_size + expected: status not ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + invalid_configs = ["Cache_capacity", "cachecapacity"] + for config in invalid_configs: + with pytest.raises(Exception) as e: + config_value = connect.get_config("gpu."+config) + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_cache_size_valid(self, connect, collection): + ''' + target: get cache_size + method: call get_config correctly + expected: status ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + config_value = connect.get_config("gpu.cache_size") + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_search_devices_invalid_parent_key(self, connect, collection): + ''' + target: get invalid parent key + method: call get_config without parent_key: gpu + expected: status not ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + invalid_configs = ["Gpu_resource_config", "gpu resource config", \ + "gpu_resource"] + for config in invalid_configs: + with pytest.raises(Exception) as e: + config_value = connect.get_config(config+".search_devices") + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_search_devices_invalid_child_key(self, connect, collection): + ''' + target: get invalid child key + method: call get_config without child_key: search_devices + expected: status not ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + invalid_configs = ["Search_resources"] + for config in invalid_configs: + with pytest.raises(Exception) as e: + config_value = connect.get_config("gpu."+config) + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_search_devices_valid(self, connect, collection): + ''' + target: get search_devices + method: call get_config correctly + expected: status ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + config_value = connect.get_config("gpu.search_devices") + logging.getLogger().info(config_value) + + @pytest.mark.level(2) + def test_get_build_index_devices_invalid_parent_key(self, connect, collection): + ''' + target: get invalid parent key + method: call get_config without parent_key: gpu + expected: status not ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + invalid_configs = ["Gpu_resource_config", "gpu resource config", \ + "gpu_resource"] + for config in invalid_configs: + with pytest.raises(Exception) as e: + config_value = connect.get_config(config+".build_index_devices") + + @pytest.mark.level(2) + def test_get_build_index_devices_invalid_child_key(self, connect, collection): + ''' + target: get invalid child key + method: call get_config without child_key: build_index_devices + expected: status not ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + invalid_configs = ["Build_index_resources"] + for config in invalid_configs: + with pytest.raises(Exception) as e: + config_value = connect.get_config("gpu."+config) + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_build_index_devices_valid(self, connect, collection): + ''' + target: get build_index_devices + method: call get_config correctly + expected: status ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + config_value = connect.get_config("gpu.build_index_devices") + logging.getLogger().info(config_value) + assert config_value + + """ + ****************************************************************** + The following cases are used to test `set_config` function + ****************************************************************** + """ + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_gpu_enable_invalid_parent_key(self, connect, collection): + ''' + target: set invalid parent key + method: call set_config without parent_key: gpu + expected: status not ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + invalid_configs = ["Gpu_resource_config", "gpu resource config", \ + "gpu_resource"] + for config in invalid_configs: + with pytest.raises(Exception) as e: + relpy = connect.set_config(config+".enable", "true") + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_gpu_invalid_child_key(self, connect, collection): + ''' + target: set invalid child key + method: call set_config with invalid child_key + expected: status not ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + invalid_configs = ["Gpu_resource_config", "gpu resource config", \ + "gpu_resource"] + for config in invalid_configs: + with pytest.raises(Exception) as e: + relpy = connect.set_config("gpu."+config, "true") + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_gpu_enable_invalid_values(self, connect, collection): + ''' + target: set "enable" param + method: call set_config with invalid child values + expected: status not ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + for i in [-1, -2, 100]: + with pytest.raises(Exception) as e: + relpy = connect.set_config("gpu.enable", i) + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_gpu_enable_valid(self, connect, collection): + ''' + target: set "enable" param + method: call set_config correctly + expected: status ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + valid_configs = ["off", "False", "0", "nO", "on", "True", 1, "yES"] + for config in valid_configs: + with pytest.raises(Exception) as e: + relpy = connect.set_config("gpu.enable", config) + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_cache_size_invalid_parent_key(self, connect, collection): + ''' + target: set invalid parent key + method: call set_config without parent_key: gpu + expected: status not ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + invalid_configs = ["Gpu_resource_config", "gpu resource config", \ + "gpu_resource"] + for config in invalid_configs: + with pytest.raises(Exception) as e: + relpy = connect.set_config(config+".cache_size", 2) + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_cache_size_valid(self, connect, collection): + ''' + target: set cache_size + method: call set_config correctly + expected: status ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + relpy = connect.set_config("gpu.cache_size", 2) + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_cache_size_invalid_values(self, connect, collection): + ''' + target: set cache_size + method: call set_config with invalid child values + expected: status not ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + self.reset_configs(connect) + for i in [-1, "1\n", "1\t"]: + logging.getLogger().info(i) + with pytest.raises(Exception) as e: + relpy = connect.set_config("gpu", "cache_size", i) + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_search_devices_invalid_parent_key(self, connect, collection): + ''' + target: set invalid parent key + method: call set_config without parent_key: gpu + expected: status not ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + invalid_configs = ["Gpu_resource_config", "gpu resource config", \ + "gpu_resource"] + for config in invalid_configs: + with pytest.raises(Exception) as e: + relpy = connect.set_config(config, "search_devices", "gpu0") + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_search_devices_valid(self, connect, collection): + ''' + target: set search_devices + method: call set_config correctly + expected: status ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + with pytest.raises(Exception) as e: + relpy = connect.set_config("gpu", "search_devices", "gpu0") + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_search_devices_invalid_values(self, connect, collection): + ''' + target: set search_devices + method: call set_config with invalid child values + expected: status not ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + for i in [-1, "10", "gpu-1", "gpu0, gpu1", "gpu22,gpu44","gpu10000","gpu 0","-gpu0"]: + with pytest.raises(Exception) as e: + relpy = connect.set_config("gpu", "search_devices", i) + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_build_index_devices_invalid_parent_key(self, connect, collection): + ''' + target: set invalid parent key + method: call set_config without parent_key: gpu + expected: status not ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + invalid_configs = ["Gpu_resource_config", "gpu resource config", \ + "gpu_resource"] + for config in invalid_configs: + with pytest.raises(Exception) as e: + relpy = connect.set_config(config, "build_index_devices", "gpu0") + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_build_index_devices_valid(self, connect, collection): + ''' + target: set build_index_devices + method: call set_config correctly + expected: status ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + with pytest.raises(Exception) as e: + relpy = connect.set_config("gpu", "build_index_devices", "gpu0") + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_build_index_devices_invalid_values(self, connect, collection): + ''' + target: set build_index_devices + method: call set_config with invalid child values + expected: status not ok + ''' + if str(connect._cmd("mode")) == "CPU": + pytest.skip("Only support GPU mode") + for i in [-1, "10", "gpu-1", "gpu0, gpu1", "gpu22,gpu44","gpu10000","gpu 0","-gpu0"]: + with pytest.raises(Exception) as e: + relpy = connect.set_config("gpu", "build_index_devices", i) + self.reset_configs(connect) + + +class TestNetworkConfig: + """ + ****************************************************************** + The following cases are used to test `get_config` function + ****************************************************************** + """ + @pytest.fixture(scope="function", autouse=True) + def skip_http_check(self, args): + if args["handler"] == "HTTP": + pytest.skip("skip in http mode") + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_address_invalid_child_key(self, connect, collection): + ''' + target: get invalid child key + method: call get_config without child_key: address + expected: status not ok + ''' + invalid_configs = ["Address", "addresses", "address "] + for config in invalid_configs: + with pytest.raises(Exception) as e: + config_value = connect.get_config("network."+config) + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_address_valid(self, connect, collection): + ''' + target: get address + method: call get_config correctly + expected: status ok + ''' + config_value = connect.get_config("network.bind.address") + + @pytest.mark.level(2) + def test_get_port_invalid_child_key(self, connect, collection): + ''' + target: get invalid child key + method: call get_config without child_key: port + expected: status not ok + ''' + invalid_configs = ["Port", "PORT", "port "] + for config in invalid_configs: + with pytest.raises(Exception) as e: + config_value = connect.get_config("network."+config) + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_port_valid(self, connect, collection): + ''' + target: get port + method: call get_config correctly + expected: status ok + ''' + config_value = connect.get_config("network.http.port") + assert config_value + + @pytest.mark.level(2) + def test_get_http_port_invalid_child_key(self, connect, collection): + ''' + target: get invalid child key + method: call get_config without child_key: http.port + expected: status not ok + ''' + invalid_configs = ["webport", "Web_port", "http port "] + for config in invalid_configs: + with pytest.raises(Exception) as e: + config_value = connect.get_config("network."+config) + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_http_port_valid(self, connect, collection): + ''' + target: get http.port + method: call get_config correctly + expected: status ok + ''' + config_value = connect.get_config("network.http.port") + assert config_value + + """ + ****************************************************************** + The following cases are used to test `set_config` function + ****************************************************************** + """ + def gen_valid_timezones(self): + timezones = [] + for i in range(0, 13): + timezones.append("UTC+" + str(i)) + timezones.append("UTC-" + str(i)) + timezones.extend(["UTC+13", "UTC+14"]) + return timezones + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_network_invalid_child_key(self, connect, collection): + ''' + target: set invalid child key + method: call set_config with invalid child_key + expected: status not ok + ''' + with pytest.raises(Exception) as e: + relpy = connect.set_config("network.child_key", 19530) + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_address_valid(self, connect, collection): + ''' + target: set address + method: call set_config correctly + expected: status ok, set successfully + ''' + relpy = connect.set_config("network.bind.address", '0.0.0.0') + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + def test_set_port_valid(self, connect, collection): + ''' + target: set port + method: call set_config correctly + expected: status ok, set successfully + ''' + for valid_port in [1025, 65534, 12345, "19530"]: + relpy = connect.set_config("network.http.port", valid_port) + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + def test_set_port_invalid(self, connect, collection): + ''' + target: set port + method: call set_config with port number out of range(1024, 65535) + expected: status not ok + ''' + for invalid_port in [1024, 65535, "0", "True", "100000"]: + logging.getLogger().info(invalid_port) + with pytest.raises(Exception) as e: + relpy = connect.set_config("network.http.port", invalid_port) + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + def test_set_http_port_valid(self, connect, collection): + ''' + target: set http.port + method: call set_config correctly + expected: status ok, set successfully + ''' + for valid_http_port in [1025, 65534, "12345", 19121]: + relpy = connect.set_config("network.http.port", valid_http_port) + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + def test_set_http_port_invalid(self, connect, collection): + ''' + target: set http.port + method: call set_config with http.port number out of range(1024, 65535) + expected: status not ok + ''' + for invalid_http_port in [1024, 65535, "0", "True", "1000000"]: + with pytest.raises(Exception) as e: + relpy = connect.set_config("network.http.port", invalid_http_port) + + +class TestGeneralConfig: + """ + ****************************************************************** + The following cases are used to test `get_config` function + ****************************************************************** + """ + @pytest.fixture(scope="function", autouse=True) + def skip_http_check(self, args): + if args["handler"] == "HTTP": + pytest.skip("skip in http mode") + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_meta_uri_invalid_child_key(self, connect, collection): + ''' + target: get invalid child key + method: call get_config without child_key: meta_uri + expected: status not ok + ''' + invalid_configs = ["backend_Url", "backend-url", "meta uri "] + for config in invalid_configs: + with pytest.raises(Exception) as e: + config_value = connect.get_config("general."+config) + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_meta_uri_valid(self, connect, collection): + ''' + target: get meta_uri + method: call get_config correctly + expected: status ok + ''' + config_value = connect.get_config("general.meta_uri") + assert config_value + + @pytest.mark.level(2) + def test_get_timezone_invalid_child_key(self, connect, collection): + ''' + target: get invalid child key + method: call get_config without child_key: timezone + expected: status not ok + ''' + invalid_configs = ["time", "time_zone "] + for config in invalid_configs: + with pytest.raises(Exception) as e: + config_value = connect.get_config("general."+config) + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_timezone_valid(self, connect, collection): + ''' + target: get timezone + method: call get_config correctly + expected: status ok + ''' + config_value = connect.get_config("general.timezone") + assert "UTC" in config_value + + """ + ****************************************************************** + The following cases are used to test `set_config` function + ****************************************************************** + """ + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + def test_set_timezone_invalid(self, connect, collection): + ''' + target: set timezone + method: call set_config with invalid timezone + expected: status not ok + ''' + for invalid_timezone in ["utc++8", "UTC++8"]: + logging.getLogger().info(invalid_timezone) + with pytest.raises(Exception) as e: + relpy = connect.set_config("general.timezone", invalid_timezone) + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_general_invalid_child_key(self, connect, collection): + ''' + target: set invalid child key + method: call set_config with invalid child_key + expected: status not ok + ''' + with pytest.raises(Exception) as e: + relpy = connect.set_config("general.child_key", 1) + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_meta_uri_valid(self, connect, collection): + ''' + target: set meta_uri + method: call set_config correctly + expected: status ok, set successfully + ''' + relpy = connect.set_config("general.meta_uri", 'sqlite://:@:/') + + +class TestStorageConfig: + """ + ****************************************************************** + The following cases are used to test `get_config` function + ****************************************************************** + """ + @pytest.fixture(scope="function", autouse=True) + def skip_http_check(self, args): + if args["handler"] == "HTTP": + pytest.skip("skip in http mode") + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_path_invalid_child_key(self, connect, collection): + ''' + target: get invalid child key + method: call get_config without child_key: path + expected: status not ok + ''' + invalid_configs = ["Primary_path", "primarypath", "pa_th "] + for config in invalid_configs: + with pytest.raises(Exception) as e: + config_value = connect.get_config("storage."+config) + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_path_valid(self, connect, collection): + ''' + target: get path + method: call get_config correctly + expected: status ok + ''' + config_value = connect.get_config("storage.path") + assert config_value + + @pytest.mark.level(2) + def test_get_auto_flush_interval_invalid_child_key(self, connect, collection): + ''' + target: get invalid child key + method: call get_config without child_key: auto_flush_interval + expected: status not ok + ''' + invalid_configs = ["autoFlushInterval", "auto_flush", "auto_flush interval "] + for config in invalid_configs: + with pytest.raises(Exception) as e: + config_value = connect.get_config("storage."+config) + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_auto_flush_interval_valid(self, connect, collection): + ''' + target: get auto_flush_interval + method: call get_config correctly + expected: status ok + ''' + config_value = connect.get_config("storage.auto_flush_interval") + + """ + ****************************************************************** + The following cases are used to test `set_config` function + ****************************************************************** + """ + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_storage_invalid_child_key(self, connect, collection): + ''' + target: set invalid child key + method: call set_config with invalid child_key + expected: status not ok + ''' + with pytest.raises(Exception) as e: + relpy = connect.set_config("storage.child_key", "") + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_path_valid(self, connect, collection): + ''' + target: set path + method: call set_config correctly + expected: status ok, set successfully + ''' + relpy = connect.set_config("storage.path", '/var/lib/milvus') + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + def test_set_auto_flush_interval_valid(self, connect, collection): + ''' + target: set auto_flush_interval + method: call set_config correctly + expected: status ok, set successfully + ''' + for valid_auto_flush_interval in [2, 1]: + logging.getLogger().info(valid_auto_flush_interval) + relpy = connect.set_config("storage.auto_flush_interval", valid_auto_flush_interval) + config_value = connect.get_config("storage.auto_flush_interval") + assert config_value == str(valid_auto_flush_interval) + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + def test_set_auto_flush_interval_invalid(self, connect, collection): + ''' + target: set auto_flush_interval + method: call set_config with invalid auto_flush_interval + expected: status not ok + ''' + for invalid_auto_flush_interval in [-1, "1.5", "invalid", "1+2"]: + with pytest.raises(Exception) as e: + relpy = connect.set_config("storage.auto_flush_interval", invalid_auto_flush_interval) + + +class TestMetricConfig: + """ + ****************************************************************** + The following cases are used to test `get_config` function + ****************************************************************** + """ + @pytest.fixture(scope="function", autouse=True) + def skip_http_check(self, args): + if args["handler"] == "HTTP": + pytest.skip("skip in http mode") + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_enable_invalid_child_key(self, connect, collection): + ''' + target: get invalid child key + method: call get_config without child_key: enable + expected: status not ok + ''' + invalid_configs = ["enablemonitor", "Enable_monitor", "en able "] + for config in invalid_configs: + with pytest.raises(Exception) as e: + config_value = connect.get_config("metric."+config) + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_enable_valid(self, connect, collection): + ''' + target: get enable + method: call get_config correctly + expected: status ok + ''' + config_value = connect.get_config("metric.enable") + assert config_value + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_address_invalid_child_key(self, connect, collection): + ''' + target: get invalid child key + method: call get_config without child_key: address + expected: status not ok + ''' + invalid_configs = ["Add ress", "addresses", "add ress "] + for config in invalid_configs: + with pytest.raises(Exception) as e: + config_value = connect.get_config("metric."+config) + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_address_valid(self, connect, collection): + ''' + target: get address + method: call get_config correctly + expected: status ok + ''' + config_value = connect.get_config("metric.address") + assert config_value + + @pytest.mark.level(2) + def test_get_port_invalid_child_key(self, connect, collection): + ''' + target: get invalid child key + method: call get_config without child_key: port + expected: status not ok + ''' + invalid_configs = ["Po_rt", "PO_RT", "po_rt "] + for config in invalid_configs: + with pytest.raises(Exception) as e: + config_value = connect.get_config("metric."+config) + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_port_valid(self, connect, collection): + ''' + target: get port + method: call get_config correctly + expected: status ok + ''' + config_value = connect.get_config("metric.port") + assert config_value + + """ + ****************************************************************** + The following cases are used to test `set_config` function + ****************************************************************** + """ + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_metric_invalid_child_key(self, connect, collection): + ''' + target: set invalid child key + method: call set_config with invalid child_key + expected: status not ok + ''' + with pytest.raises(Exception) as e: + relpy = connect.set_config("metric.child_key", 19530) + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + def test_set_enable_valid(self, connect, collection): + ''' + target: set enable + method: call set_config correctly + expected: status ok, set successfully + ''' + for valid_enable in ["false", "true"]: + relpy = connect.set_config("metric.enable", valid_enable) + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_address_valid(self, connect, collection): + ''' + target: set address + method: call set_config correctly + expected: status ok, set successfully + ''' + relpy = connect.set_config("metric.address", '127.0.0.1') + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + def test_set_port_valid(self, connect, collection): + ''' + target: set port + method: call set_config correctly + expected: status ok, set successfully + ''' + for valid_port in [1025, 65534, "19530", "9091"]: + relpy = connect.set_config("metric.port", valid_port) + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + def test_set_port_invalid(self, connect, collection): + ''' + target: set port + method: call set_config with port number out of range(1024, 65535), or same as http.port number + expected: status not ok + ''' + for invalid_port in [1024, 65535, "0", "True", "100000"]: + with pytest.raises(Exception) as e: + relpy = connect.set_config("metric.port", invalid_port) + + +class TestWALConfig: + """ + ****************************************************************** + The following cases are used to test `get_config` function + ****************************************************************** + """ + @pytest.fixture(scope="function", autouse=True) + def skip_http_check(self, args): + if args["handler"] == "HTTP": + pytest.skip("skip in http mode") + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_enable_invalid_child_key(self, connect, collection): + ''' + target: get invalid child key + method: call get_config without child_key: enable + expected: status not ok + ''' + invalid_configs = ["enabled", "Enab_le", "enable_"] + for config in invalid_configs: + with pytest.raises(Exception) as e: + config_value = connect.get_config("wal."+config) + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_enable_valid(self, connect, collection): + ''' + target: get enable + method: call get_config correctly + expected: status ok + ''' + config_value = connect.get_config("wal.enable") + assert config_value + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_recovery_error_ignore_invalid_child_key(self, connect, collection): + ''' + target: get invalid child key + method: call get_config without child_key: recovery_error_ignore + expected: status not ok + ''' + invalid_configs = ["recovery-error-ignore", "Recovery error_ignore", "recoveryxerror_ignore "] + for config in invalid_configs: + with pytest.raises(Exception) as e: + config_value = connect.get_config("wal."+config) + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_recovery_error_ignore_valid(self, connect, collection): + ''' + target: get recovery_error_ignore + method: call get_config correctly + expected: status ok + ''' + config_value = connect.get_config("wal.recovery_error_ignore") + assert config_value + + @pytest.mark.level(2) + def test_get_buffer_size_invalid_child_key(self, connect, collection): + ''' + target: get invalid child key + method: call get_config without child_key: buffer_size + expected: status not ok + ''' + invalid_configs = ["buffersize", "Buffer size", "buffer size "] + for config in invalid_configs: + with pytest.raises(Exception) as e: + config_value = connect.get_config("wal."+config) + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_buffer_size_valid(self, connect, collection): + ''' + target: get buffer_size + method: call get_config correctly + expected: status ok + ''' + config_value = connect.get_config("wal.buffer_size") + assert config_value + + @pytest.mark.level(2) + def test_get_wal_path_invalid_child_key(self, connect, collection): + ''' + target: get invalid child key + method: call get_config without child_key: wal_path + expected: status not ok + ''' + invalid_configs = ["wal", "Wal_path", "wal_path "] + for config in invalid_configs: + with pytest.raises(Exception) as e: + config_value = connect.get_config("wal."+config) + + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_get_wal_path_valid(self, connect, collection): + ''' + target: get wal_path + method: call get_config correctly + expected: status ok + ''' + config_value = connect.get_config("wal.path") + assert config_value + + """ + ****************************************************************** + The following cases are used to test `set_config` function + ****************************************************************** + """ + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_wal_invalid_child_key(self, connect, collection): + ''' + target: set invalid child key + method: call set_config with invalid child_key + expected: status not ok + ''' + with pytest.raises(Exception) as e: + relpy = connect.set_config("wal.child_key", 256) + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + def test_set_enable_valid(self, connect, collection): + ''' + target: set enable + method: call set_config correctly + expected: status ok, set successfully + ''' + for valid_enable in ["false", "true"]: + relpy = connect.set_config("wal.enable", valid_enable) + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + def test_set_recovery_error_ignore_valid(self, connect, collection): + ''' + target: set recovery_error_ignore + method: call set_config correctly + expected: status ok, set successfully + ''' + for valid_recovery_error_ignore in ["false", "true"]: + relpy = connect.set_config("wal.recovery_error_ignore", valid_recovery_error_ignore) + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + def test_set_buffer_size_valid_A(self, connect, collection): + ''' + target: set buffer_size + method: call set_config correctly + expected: status ok, set successfully + ''' + for valid_buffer_size in ["64MB", "128MB", "4096MB", "1000MB", "256MB"]: + relpy = connect.set_config("wal.buffer_size", valid_buffer_size) + + @pytest.mark.skip(reason="overwrite config file is not supported in ci yet.") + @pytest.mark.timeout(CONFIG_TIMEOUT) + def test_set_wal_path_valid(self, connect, collection, args): + ''' + target: set wal_path + method: call set_config correctly + expected: status ok, set successfully + ''' + relpy = connect.set_config("wal.path", "/var/lib/milvus/wal") + diff --git a/tests/python_test/test_connect.py b/tests/python_test/test_connect.py new file mode 100644 index 0000000000..9d4dc5a1b8 --- /dev/null +++ b/tests/python_test/test_connect.py @@ -0,0 +1,232 @@ +import pytest +import pdb +import threading +from multiprocessing import Process +import concurrent.futures +from utils import * + +CONNECT_TIMEOUT = 12 + + +class TestConnect: + + def local_ip(self, args): + ''' + check if ip is localhost or not + ''' + if not args["ip"] or args["ip"] == 'localhost' or args["ip"] == "127.0.0.1": + return True + else: + return False + + # TODO: remove + def _test_disconnect(self, connect): + ''' + target: test disconnect + method: disconnect a connected client + expected: connect failed after disconnected + ''' + res = connect.close() + with pytest.raises(Exception) as e: + res = connect.() + + # TODO: remove + def _test_disconnect_repeatedly(self, dis_connect, args): + ''' + target: test disconnect repeatedly + method: disconnect a connected client, disconnect again + expected: raise an error after disconnected + ''' + with pytest.raises(Exception) as e: + connect.close() + + def test_connect_correct_ip_port(self, args): + ''' + target: test connect with correct ip and port value + method: set correct ip and port + expected: connected is True + ''' + milvus = get_milvus(args["ip"], args["port"], handler=args["handler"]) + # assert milvus.connected() + + # TODO: Currently we test with remote IP, localhost testing need to add + def _test_connect_ip_localhost(self, args): + ''' + target: test connect with ip value: localhost + method: set host localhost + expected: connected is True + ''' + milvus = get_milvus(args["ip"], args["port"], args["handler"]) + # milvus.connect(host='localhost', port=args["port"]) + # assert milvus.connected() + + @pytest.mark.timeout(CONNECT_TIMEOUT) + def test_connect_wrong_ip_null(self, args): + ''' + target: test connect with wrong ip value + method: set host null + expected: not use default ip, connected is False + ''' + ip = "" + with pytest.raises(Exception) as e: + milvus = get_milvus(ip, args["port"], args["handler"]) + # assert not milvus.connected() + + def test_connect_uri(self, args): + ''' + target: test connect with correct uri + method: uri format and value are both correct + expected: connected is True + ''' + uri_value = "tcp://%s:%s" % (args["ip"], args["port"]) + milvus = get_milvus(args["ip"], args["port"], uri=uri_value, handler=args["handler"]) + # assert milvus.connected() + + def test_connect_uri_null(self, args): + ''' + target: test connect with null uri + method: uri set null + expected: connected is True + ''' + uri_value = "" + if self.local_ip(args): + milvus = get_milvus(None, None, uri=uri_value, handler=args["handler"]) + # assert milvus.connected() + else: + with pytest.raises(Exception) as e: + milvus = get_milvus(None, None, uri=uri_value, handler=args["handler"]) + # assert not milvus.connected() + + def test_connect_with_multiprocess(self, args): + ''' + target: test uri connect with multiprocess + method: set correct uri, test with multiprocessing connecting + expected: all connection is connected + ''' + processes = [] + def connect(): + milvus = get_milvus(args["ip"], args["port"], handler=args["handler"]) + assert milvus + with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor: + future_results = {executor.submit( + connect): i for i in range(100)} + for future in concurrent.futures.as_completed(future_results): + future.result() + + def test_connect_repeatedly(self, args): + ''' + target: test connect repeatedly + method: connect again + expected: status.code is 0, and status.message shows have connected already + ''' + uri_value = "tcp://%s:%s" % (args["ip"], args["port"]) + milvus = Milvus(uri=uri_value, handler=args["handler"]) + # milvus.connect(uri=uri_value, timeout=5) + # milvus.connect(uri=uri_value, timeout=5) + milvus = Milvus(uri=uri_value, handler=args["handler"]) + # assert milvus.connected() + + def _test_add_vector_and_disconnect_concurrently(self): + ''' + Target: test disconnect in the middle of add vectors + Method: + a. use coroutine or multi-processing, to simulate network crashing + b. data_set not too large incase disconnection happens when data is underd-preparing + c. data_set not too small incase disconnection happens when data has already been transferred + d. make sure disconnection happens when data is in-transport + Expected: Failure, count_entities == 0 + + ''' + pass + + def _test_search_vector_and_disconnect_concurrently(self): + ''' + Target: Test disconnect in the middle of search vectors(with large nq and topk)multiple times, and search/add vectors still work + Method: + a. coroutine or multi-processing, to simulate network crashing + b. connect, search and disconnect, repeating many times + c. connect and search, add vectors + Expected: Successfully searched back, successfully added + + ''' + pass + + def _test_thread_safe_with_one_connection_shared_in_multi_threads(self): + ''' + Target: test 1 connection thread safe + Method: 1 connection shared in multi-threads, all adding vectors, or other things + Expected: Functional as one thread + + ''' + pass + + +class TestConnectIPInvalid(object): + """ + Test connect server with invalid ip + """ + @pytest.fixture( + scope="function", + params=gen_invalid_ips() + ) + def get_invalid_ip(self, request): + yield request.param + + @pytest.mark.level(2) + @pytest.mark.timeout(CONNECT_TIMEOUT) + def test_connect_with_invalid_ip(self, args, get_invalid_ip): + ip = get_invalid_ip + with pytest.raises(Exception) as e: + milvus = get_milvus(ip, args["port"], args["handler"]) + # assert not milvus.connected() + + +class TestConnectPortInvalid(object): + """ + Test connect server with invalid ip + """ + + @pytest.fixture( + scope="function", + params=gen_invalid_ints() + ) + def get_invalid_port(self, request): + yield request.param + + @pytest.mark.level(2) + @pytest.mark.timeout(CONNECT_TIMEOUT) + def test_connect_with_invalid_port(self, args, get_invalid_port): + ''' + target: test ip:port connect with invalid port value + method: set port in gen_invalid_ports + expected: connected is False + ''' + port = get_invalid_port + with pytest.raises(Exception) as e: + milvus = get_milvus(args["ip"], port, args["handler"]) + # assert not milvus.connected() + + +class TestConnectURIInvalid(object): + """ + Test connect server with invalid uri + """ + @pytest.fixture( + scope="function", + params=gen_invalid_uris() + ) + def get_invalid_uri(self, request): + yield request.param + + @pytest.mark.level(2) + @pytest.mark.timeout(CONNECT_TIMEOUT) + def test_connect_with_invalid_uri(self, get_invalid_uri, args): + ''' + target: test uri connect with invalid uri value + method: set port in gen_invalid_uris + expected: connected is False + ''' + uri_value = get_invalid_uri + with pytest.raises(Exception) as e: + milvus = get_milvus(uri=uri_value, handler=args["handler"]) + # assert not milvus.connected() diff --git a/tests/python_test/test_flush.py b/tests/python_test/test_flush.py new file mode 100644 index 0000000000..a6d831ac9d --- /dev/null +++ b/tests/python_test/test_flush.py @@ -0,0 +1,353 @@ +import time +import pdb +import threading +import logging +from multiprocessing import Pool, Process +import pytest +from utils import * +from constants import * + +DELETE_TIMEOUT = 60 +default_single_query = { + "bool": { + "must": [ + {"vector": {default_float_vec_field_name: {"topk": 10, "query": gen_vectors(1, default_dim), + "metric_type": "L2", "params": {"nprobe": 10}}}} + ] + } +} + + +class TestFlushBase: + """ + ****************************************************************** + The following cases are used to test `flush` function + ****************************************************************** + """ + + @pytest.fixture( + scope="function", + params=gen_simple_index() + ) + def get_simple_index(self, request, connect): + if str(connect._cmd("mode")[1]) == "GPU": + if request.param["index_type"] not in ivf(): + pytest.skip("Only support index_type: idmap/flat") + return request.param + + @pytest.fixture( + scope="function", + params=gen_single_filter_fields() + ) + def get_filter_field(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_single_vector_fields() + ) + def get_vector_field(self, request): + yield request.param + + def test_flush_collection_not_existed(self, connect, collection): + ''' + target: test flush, params collection_name not existed + method: flush, with collection not existed + expected: error raised + ''' + collection_new = gen_unique_str("test_flush_1") + with pytest.raises(Exception) as e: + connect.flush([collection_new]) + + def test_flush_empty_collection(self, connect, collection): + ''' + method: flush collection with no vectors + expected: no error raised + ''' + connect.flush([collection]) + ids = connect.insert(collection, default_entities) + assert len(ids) == default_nb + # status = connect.delete_entity_by_id(collection, ids) + # assert status.OK() + # connect.flush([collection]) + res = connect.get_collection_stats(collection) + # print(res) + # assert res == default_nb + # with pytest.raises(Exception) as e: + # connect.flush([collection]) + + def test_add_partition_flush(self, connect, id_collection): + ''' + method: add entities into partition in collection, flush serveral times + expected: the length of ids and the collection row count + ''' + connect.create_partition(id_collection, default_tag) + ids = [i for i in range(default_nb)] + ids = connect.insert(id_collection, default_entities, ids) + connect.flush([id_collection]) + res_count = connect.get_collection_stats(id_collection) + assert res_count["row_count"] == default_nb + ids = connect.insert(id_collection, default_entities, ids, partition_tag=default_tag) + assert len(ids) == default_nb + connect.flush([id_collection]) + res_count = connect.get_collection_stats(id_collection) + assert res_count["row_count"] == default_nb * 2 + + def test_add_partitions_flush(self, connect, id_collection): + ''' + method: add entities into partitions in collection, flush one + expected: the length of ids and the collection row count + ''' + tag_new = gen_unique_str() + connect.create_partition(id_collection, default_tag) + connect.create_partition(id_collection, tag_new) + ids = [i for i in range(default_nb)] + ids = connect.insert(id_collection, default_entities, ids, partition_tag=default_tag) + connect.flush([id_collection]) + ids = connect.insert(id_collection, default_entities, ids, partition_tag=tag_new) + connect.flush([id_collection]) + res = connect.get_collection_stats(id_collection) + assert res["row_count"] == 2 * default_nb + + def test_add_collections_flush(self, connect, id_collection): + ''' + method: add entities into collections, flush one + expected: the length of ids and the collection row count + ''' + collection_new = gen_unique_str() + default_fields = gen_default_fields(False) + connect.create_collection(collection_new, default_fields) + connect.create_partition(id_collection, default_tag) + connect.create_partition(collection_new, default_tag) + ids = [i for i in range(default_nb)] + # ids = connect.insert(id_collection, default_entities, ids, partition_tag=default_tag) + # ids = connect.insert(collection_new, default_entities, ids, partition_tag=default_tag) + connect.insert(id_collection, default_entities, ids, partition_tag=default_tag) + connect.insert(collection_new, default_entities, ids, partition_tag=default_tag) + connect.flush([id_collection]) + connect.flush([collection_new]) + res = connect.get_collection_stats(id_collection) + assert res["row_count"] == default_nb + res = connect.get_collection_stats(collection_new) + assert res["row_count"] == default_nb + + def test_add_collections_fields_flush(self, connect, id_collection, get_filter_field, get_vector_field): + ''' + method: create collection with different fields, and add entities into collections, flush one + expected: the length of ids and the collection row count + ''' + nb_new = 5 + filter_field = get_filter_field + vector_field = get_vector_field + collection_new = gen_unique_str("test_flush") + fields = { + "fields": [filter_field, vector_field], + "segment_row_limit": default_segment_row_limit, + "auto_id": False + } + connect.create_collection(collection_new, fields) + connect.create_partition(id_collection, default_tag) + connect.create_partition(collection_new, default_tag) + entities_new = gen_entities_by_fields(fields["fields"], nb_new, default_dim) + ids = [i for i in range(default_nb)] + ids_new = [i for i in range(nb_new)] + # ids = connect.insert(id_collection, default_entities, ids, partition_tag=default_tag) + # ids = connect.insert(collection_new, entities_new, ids_new, partition_tag=default_tag) + connect.insert(id_collection, default_entities, ids, partition_tag=default_tag) + connect.insert(collection_new, entities_new, ids_new, partition_tag=default_tag) + connect.flush([id_collection]) + connect.flush([collection_new]) + res = connect.get_collection_stats(id_collection) + assert res["row_count"] == default_nb + res = connect.get_collection_stats(collection_new) + assert res["row_count"] == nb_new + + def test_add_flush_multiable_times(self, connect, collection): + ''' + method: add entities, flush serveral times + expected: no error raised + ''' + ids = connect.insert(collection, default_entities) + for i in range(10): + connect.flush([collection]) + res = connect.get_collection_stats(collection) + assert res["row_count"] == len(ids) + # query_vecs = [vectors[0], vectors[1], vectors[-1]] + res = connect.search(collection, default_single_query) + logging.getLogger().debug(res) + assert res + + def test_add_flush_auto(self, connect, id_collection): + ''' + method: add entities + expected: no error raised + ''' + ids = [i for i in range(default_nb)] + ids = connect.insert(id_collection, default_entities, ids) + timeout = 20 + start_time = time.time() + while (time.time() - start_time < timeout): + time.sleep(1) + res = connect.get_collection_stats(id_collection) + if res["row_count"] == default_nb: + break + if time.time() - start_time > timeout: + assert False + + @pytest.fixture( + scope="function", + params=[ + 1, + 100 + ], + ) + def same_ids(self, request): + yield request.param + + def test_add_flush_same_ids(self, connect, id_collection, same_ids): + ''' + method: add entities, with same ids, count(same ids) < 15, > 15 + expected: the length of ids and the collection row count + ''' + ids = [i for i in range(default_nb)] + for i, item in enumerate(ids): + if item <= same_ids: + ids[i] = 0 + ids = connect.insert(id_collection, default_entities, ids) + connect.flush([id_collection]) + res = connect.get_collection_stats(id_collection) + assert res["row_count"] == default_nb + + def test_delete_flush_multiable_times(self, connect, collection): + ''' + method: delete entities, flush serveral times + expected: no error raised + ''' + ids = connect.insert(collection, default_entities) + status = connect.delete_entity_by_id(collection, [ids[-1]]) + assert status.OK() + for i in range(10): + connect.flush([collection]) + # query_vecs = [vectors[0], vectors[1], vectors[-1]] + res = connect.search(collection, default_single_query) + logging.getLogger().debug(res) + assert res + + # TODO: unable to set config + @pytest.mark.level(2) + def _test_collection_count_during_flush(self, connect, collection, args): + ''' + method: flush collection at background, call `get_collection_stats` + expected: no timeout + ''' + ids = [] + for i in range(5): + tmp_ids = connect.insert(collection, default_entities) + connect.flush([collection]) + ids.extend(tmp_ids) + disable_flush(connect) + status = connect.delete_entity_by_id(collection, ids) + + def flush(): + milvus = get_milvus(args["ip"], args["port"], handler=args["handler"]) + logging.error("start flush") + milvus.flush([collection]) + logging.error("end flush") + + p = TestThread(target=flush, args=()) + p.start() + time.sleep(0.2) + logging.error("start count") + res = connect.get_collection_stats(collection, timeout=10) + p.join() + res = connect.get_collection_stats(collection) + assert res["row_count"] == 0 + + @pytest.mark.level(2) + def test_delete_flush_during_search(self, connect, collection, args): + ''' + method: search at background, call `delete and flush` + expected: no timeout + ''' + ids = [] + loops = 5 + for i in range(loops): + tmp_ids = connect.insert(collection, default_entities) + connect.flush([collection]) + ids.extend(tmp_ids) + nq = 10000 + query, query_vecs = gen_query_vectors(default_float_vec_field_name, default_entities, default_top_k, nq) + time.sleep(0.1) + future = connect.search(collection, query, _async=True) + delete_ids = [ids[0], ids[-1]] + status = connect.delete_entity_by_id(collection, delete_ids) + connect.flush([collection]) + res = future.result() + res_count = connect.get_collection_stats(collection, timeout=120) + assert res_count["row_count"] == loops * default_nb - len(delete_ids) + + +class TestFlushAsync: + @pytest.fixture(scope="function", autouse=True) + def skip_http_check(self, args): + if args["handler"] == "HTTP": + pytest.skip("skip in http mode") + + """ + ****************************************************************** + The following cases are used to test `flush` function + ****************************************************************** + """ + + def check_status(self): + logging.getLogger().info("In callback check status") + + def test_flush_empty_collection(self, connect, collection): + ''' + method: flush collection with no vectors + expected: status ok + ''' + future = connect.flush([collection], _async=True) + status = future.result() + + def test_flush_async_long(self, connect, collection): + ids = connect.insert(collection, default_entities) + future = connect.flush([collection], _async=True) + status = future.result() + + def test_flush_async_long_drop_collection(self, connect, collection): + for i in range(5): + ids = connect.insert(collection, default_entities) + future = connect.flush([collection], _async=True) + logging.getLogger().info("DROP") + connect.drop_collection(collection) + + def test_flush_async(self, connect, collection): + connect.insert(collection, default_entities) + logging.getLogger().info("before") + future = connect.flush([collection], _async=True, _callback=self.check_status) + logging.getLogger().info("after") + future.done() + status = future.result() + + +class TestCollectionNameInvalid(object): + """ + Test adding vectors with invalid collection names + """ + + @pytest.fixture( + scope="function", + # params=gen_invalid_collection_names() + params=gen_invalid_strs() + ) + def get_invalid_collection_name(self, request): + yield request.param + + @pytest.mark.level(2) + def test_flush_with_invalid_collection_name(self, connect, get_invalid_collection_name): + collection_name = get_invalid_collection_name + if collection_name is None or not collection_name: + pytest.skip("while collection_name is None, then flush all collections") + with pytest.raises(Exception) as e: + connect.flush(collection_name) diff --git a/tests/python_test/test_index.py b/tests/python_test/test_index.py new file mode 100644 index 0000000000..833457a1b4 --- /dev/null +++ b/tests/python_test/test_index.py @@ -0,0 +1,832 @@ +import logging +import time +import pdb +import threading +from multiprocessing import Pool, Process +import numpy +import pytest +import sklearn.preprocessing +from utils import * +from constants import * + +uid = "test_index" +BUILD_TIMEOUT = 300 +field_name = default_float_vec_field_name +binary_field_name = default_binary_vec_field_name +query, query_vecs = gen_query_vectors(field_name, default_entities, default_top_k, 1) +default_index = {"index_type": "IVF_FLAT", "params": {"nlist": 128}, "metric_type": "L2"} + + +class TestIndexBase: + @pytest.fixture( + scope="function", + params=gen_simple_index() + ) + def get_simple_index(self, request, connect): + logging.getLogger().info(request.param) + if str(connect._cmd("mode")) == "CPU": + if request.param["index_type"] in index_cpu_not_support(): + pytest.skip("sq8h not support in CPU mode") + return request.param + + @pytest.fixture( + scope="function", + params=[ + 1, + 10, + 1111 + ], + ) + def get_nq(self, request): + yield request.param + + """ + ****************************************************************** + The following cases are used to test `create_index` function + ****************************************************************** + """ + + @pytest.mark.timeout(BUILD_TIMEOUT) + def test_create_index(self, connect, collection, get_simple_index): + ''' + target: test create index interface + method: create collection and add entities in it, create index + expected: return search success + ''' + ids = connect.insert(collection, default_entities) + connect.create_index(collection, field_name, get_simple_index) + index = connect.describe_index(collection, field_name) + assert index == get_simple_index + + def test_create_index_on_field_not_existed(self, connect, collection, get_simple_index): + ''' + target: test create index interface + method: create collection and add entities in it, create index on field not existed + expected: error raised + ''' + tmp_field_name = gen_unique_str() + ids = connect.insert(collection, default_entities) + with pytest.raises(Exception) as e: + connect.create_index(collection, tmp_field_name, get_simple_index) + + @pytest.mark.level(2) + def test_create_index_on_field(self, connect, collection, get_simple_index): + ''' + target: test create index interface + method: create collection and add entities in it, create index on other field + expected: error raised + ''' + tmp_field_name = "int64" + ids = connect.insert(collection, default_entities) + with pytest.raises(Exception) as e: + connect.create_index(collection, tmp_field_name, get_simple_index) + + @pytest.mark.timeout(BUILD_TIMEOUT) + def test_create_index_no_vectors(self, connect, collection, get_simple_index): + ''' + target: test create index interface + method: create collection and add entities in it, create index + expected: return search success + ''' + connect.create_index(collection, field_name, get_simple_index) + index = connect.describe_index(collection, field_name) + assert index == get_simple_index + + @pytest.mark.timeout(BUILD_TIMEOUT) + def test_create_index_partition(self, connect, collection, get_simple_index): + ''' + target: test create index interface + method: create collection, create partition, and add entities in it, create index + expected: return search success + ''' + connect.create_partition(collection, default_tag) + ids = connect.insert(collection, default_entities, partition_tag=default_tag) + connect.flush([collection]) + connect.create_index(collection, field_name, get_simple_index) + index = connect.describe_index(collection, field_name) + assert index == get_simple_index + + @pytest.mark.timeout(BUILD_TIMEOUT) + def test_create_index_partition_flush(self, connect, collection, get_simple_index): + ''' + target: test create index interface + method: create collection, create partition, and add entities in it, create index + expected: return search success + ''' + connect.create_partition(collection, default_tag) + ids = connect.insert(collection, default_entities, partition_tag=default_tag) + connect.flush() + connect.create_index(collection, field_name, get_simple_index) + index = connect.describe_index(collection, field_name) + assert index == get_simple_index + + def test_create_index_without_connect(self, dis_connect, collection): + ''' + target: test create index without connection + method: create collection and add entities in it, check if added successfully + expected: raise exception + ''' + with pytest.raises(Exception) as e: + dis_connect.create_index(collection, field_name, get_simple_index) + + @pytest.mark.timeout(BUILD_TIMEOUT) + def test_create_index_search_with_query_vectors(self, connect, collection, get_simple_index, get_nq): + ''' + target: test create index interface, search with more query vectors + method: create collection and add entities in it, create index + expected: return search success + ''' + ids = connect.insert(collection, default_entities) + connect.create_index(collection, field_name, get_simple_index) + logging.getLogger().info(connect.describe_index(collection, field_name)) + nq = get_nq + index_type = get_simple_index["index_type"] + search_param = get_search_param(index_type) + query, vecs = gen_query_vectors(field_name, default_entities, default_top_k, nq, search_params=search_param) + connect.load_collection(collection) + res = connect.search(collection, query) + assert len(res) == nq + + @pytest.mark.timeout(BUILD_TIMEOUT) + @pytest.mark.level(2) + def test_create_index_multithread(self, connect, collection, args): + ''' + target: test create index interface with multiprocess + method: create collection and add entities in it, create index + expected: return search success + ''' + connect.insert(collection, default_entities) + + def build(connect): + connect.create_index(collection, field_name, default_index) + index = connect.describe_index(collection, field_name) + assert index == default_index + + threads_num = 8 + threads = [] + for i in range(threads_num): + m = get_milvus(host=args["ip"], port=args["port"], handler=args["handler"]) + t = TestThread(target=build, args=(m,)) + threads.append(t) + t.start() + time.sleep(0.2) + for t in threads: + t.join() + + def test_create_index_collection_not_existed(self, connect): + ''' + target: test create index interface when collection name not existed + method: create collection and add entities in it, create index + , make sure the collection name not in index + expected: create index failed + ''' + collection_name = gen_unique_str(uid) + with pytest.raises(Exception) as e: + connect.create_index(collection_name, field_name, default_index) + + @pytest.mark.level(2) + @pytest.mark.timeout(BUILD_TIMEOUT) + def test_create_index_insert_flush(self, connect, collection, get_simple_index): + ''' + target: test create index + method: create collection and create index, add entities in it + expected: create index ok, and count correct + ''' + connect.create_index(collection, field_name, get_simple_index) + ids = connect.insert(collection, default_entities) + connect.flush([collection]) + stats = connect.get_collection_stats(collection) + assert stats["row_count"] == default_nb + index = connect.describe_index(collection, field_name) + assert index == get_simple_index + + @pytest.mark.level(2) + @pytest.mark.timeout(BUILD_TIMEOUT) + def test_create_same_index_repeatedly(self, connect, collection, get_simple_index): + ''' + target: check if index can be created repeatedly, with the same create_index params + method: create index after index have been built + expected: return code success, and search ok + ''' + connect.create_index(collection, field_name, get_simple_index) + connect.create_index(collection, field_name, get_simple_index) + index = connect.describe_index(collection, field_name) + assert index == get_simple_index + + # TODO: + @pytest.mark.level(2) + @pytest.mark.timeout(BUILD_TIMEOUT) + def test_create_different_index_repeatedly(self, connect, collection): + ''' + target: check if index can be created repeatedly, with the different create_index params + method: create another index with different index_params after index have been built + expected: return code 0, and describe index result equals with the second index params + ''' + ids = connect.insert(collection, default_entities) + indexs = [default_index, {"metric_type":"L2", "index_type": "FLAT", "params":{"nlist": 1024}}] + for index in indexs: + connect.create_index(collection, field_name, index) + index = connect.describe_index(collection, field_name) + assert index == indexs[-1] + + @pytest.mark.timeout(BUILD_TIMEOUT) + def test_create_index_ip(self, connect, collection, get_simple_index): + ''' + target: test create index interface + method: create collection and add entities in it, create index + expected: return search success + ''' + ids = connect.insert(collection, default_entities) + get_simple_index["metric_type"] = "IP" + connect.create_index(collection, field_name, get_simple_index) + index = connect.describe_index(collection, field_name) + assert index == get_simple_index + + @pytest.mark.timeout(BUILD_TIMEOUT) + def test_create_index_no_vectors_ip(self, connect, collection, get_simple_index): + ''' + target: test create index interface + method: create collection and add entities in it, create index + expected: return search success + ''' + get_simple_index["metric_type"] = "IP" + connect.create_index(collection, field_name, get_simple_index) + index = connect.describe_index(collection, field_name) + assert index == get_simple_index + + @pytest.mark.timeout(BUILD_TIMEOUT) + def test_create_index_partition_ip(self, connect, collection, get_simple_index): + ''' + target: test create index interface + method: create collection, create partition, and add entities in it, create index + expected: return search success + ''' + connect.create_partition(collection, default_tag) + ids = connect.insert(collection, default_entities, partition_tag=default_tag) + connect.flush([collection]) + get_simple_index["metric_type"] = "IP" + connect.create_index(collection, field_name, get_simple_index) + index = connect.describe_index(collection, field_name) + assert index == get_simple_index + + @pytest.mark.timeout(BUILD_TIMEOUT) + def test_create_index_partition_flush_ip(self, connect, collection, get_simple_index): + ''' + target: test create index interface + method: create collection, create partition, and add entities in it, create index + expected: return search success + ''' + connect.create_partition(collection, default_tag) + ids = connect.insert(collection, default_entities, partition_tag=default_tag) + connect.flush() + get_simple_index["metric_type"] = "IP" + connect.create_index(collection, field_name, get_simple_index) + index = connect.describe_index(collection, field_name) + assert index == get_simple_index + + @pytest.mark.timeout(BUILD_TIMEOUT) + def test_create_index_search_with_query_vectors_ip(self, connect, collection, get_simple_index, get_nq): + ''' + target: test create index interface, search with more query vectors + method: create collection and add entities in it, create index + expected: return search success + ''' + metric_type = "IP" + ids = connect.insert(collection, default_entities) + get_simple_index["metric_type"] = metric_type + connect.create_index(collection, field_name, get_simple_index) + logging.getLogger().info(connect.describe_index(collection)) + nq = get_nq + index_type = get_simple_index["index_type"] + search_param = get_search_param(index_type) + query, vecs = gen_query_vectors(field_name, default_entities, default_top_k, nq, metric_type=metric_type, search_params=search_param) + res = connect.search(collection, query) + assert len(res) == nq + + @pytest.mark.timeout(BUILD_TIMEOUT) + @pytest.mark.level(2) + def test_create_index_multithread_ip(self, connect, collection, args): + ''' + target: test create index interface with multiprocess + method: create collection and add entities in it, create index + expected: return search success + ''' + connect.insert(collection, default_entities) + + def build(connect): + default_index["metric_type"] = "IP" + connect.create_index(collection, field_name, default_index) + index = connect.describe_index(collection, field_name) + assert index == default_index + + threads_num = 8 + threads = [] + for i in range(threads_num): + m = get_milvus(host=args["ip"], port=args["port"], handler=args["handler"]) + t = TestThread(target=build, args=(m,)) + threads.append(t) + t.start() + time.sleep(0.2) + for t in threads: + t.join() + + def test_create_index_collection_not_existed_ip(self, connect, collection): + ''' + target: test create index interface when collection name not existed + method: create collection and add entities in it, create index + , make sure the collection name not in index + expected: return code not equals to 0, create index failed + ''' + collection_name = gen_unique_str(uid) + default_index["metric_type"] = "IP" + with pytest.raises(Exception) as e: + connect.create_index(collection_name, field_name, default_index) + + @pytest.mark.timeout(BUILD_TIMEOUT) + def test_create_index_no_vectors_insert_ip(self, connect, collection): + ''' + target: test create index interface when there is no vectors in collection, and does not affect the subsequent process + method: create collection and add no vectors in it, and then create index, add entities in it + expected: return code equals to 0 + ''' + default_index["metric_type"] = "IP" + connect.create_index(collection, field_name, default_index) + ids = connect.insert(collection, default_entities) + connect.flush([collection]) + stats = connect.get_collection_stats(collection) + assert stats["row_count"] == default_nb + index = connect.describe_index(collection, field_name) + assert index == default_index + + @pytest.mark.level(2) + @pytest.mark.timeout(BUILD_TIMEOUT) + def test_create_same_index_repeatedly_ip(self, connect, collection): + ''' + target: check if index can be created repeatedly, with the same create_index params + method: create index after index have been built + expected: return code success, and search ok + ''' + default_index["metric_type"] = "IP" + connect.create_index(collection, field_name, default_index) + connect.create_index(collection, field_name, default_index) + index = connect.describe_index(collection, field_name) + assert index == default_index + + # TODO: + @pytest.mark.level(2) + @pytest.mark.timeout(BUILD_TIMEOUT) + def test_create_different_index_repeatedly_ip(self, connect, collection): + ''' + target: check if index can be created repeatedly, with the different create_index params + method: create another index with different index_params after index have been built + expected: return code 0, and describe index result equals with the second index params + ''' + ids = connect.insert(collection, default_entities) + stats = connect.get_collection_stats(collection) + assert stats["row_count"] == default_nb + default_index["metric_type"] = "IP" + indexs = [default_index, {"index_type": "FLAT", "params": {"nlist": 1024}, "metric_type": "IP"}] + for index in indexs: + connect.create_index(collection, field_name, index) + index = connect.describe_index(collection, field_name) + assert index == indexs[-1] + + """ + ****************************************************************** + The following cases are used to test `drop_index` function + ****************************************************************** + """ + def test_drop_index(self, connect, collection, get_simple_index): + ''' + target: test drop index interface + method: create collection and add entities in it, create index, call drop index + expected: return code 0, and default index param + ''' + # ids = connect.insert(collection, entities) + connect.create_index(collection, field_name, get_simple_index) + connect.drop_index(collection, field_name) + index = connect.describe_index(collection, field_name) + assert not index + + @pytest.mark.level(2) + def test_drop_index_repeatedly(self, connect, collection, get_simple_index): + ''' + target: test drop index repeatedly + method: create index, call drop index, and drop again + expected: return code 0 + ''' + connect.create_index(collection, field_name, get_simple_index) + connect.drop_index(collection, field_name) + connect.drop_index(collection, field_name) + index = connect.describe_index(collection, field_name) + assert not index + + @pytest.mark.level(2) + def test_drop_index_without_connect(self, dis_connect, collection): + ''' + target: test drop index without connection + method: drop index, and check if drop successfully + expected: raise exception + ''' + with pytest.raises(Exception) as e: + dis_connect.drop_index(collection, field_name) + + def test_drop_index_collection_not_existed(self, connect): + ''' + target: test drop index interface when collection name not existed + method: create collection and add entities in it, create index + , make sure the collection name not in index, and then drop it + expected: return code not equals to 0, drop index failed + ''' + collection_name = gen_unique_str(uid) + with pytest.raises(Exception) as e: + connect.drop_index(collection_name, field_name) + + def test_drop_index_collection_not_create(self, connect, collection): + ''' + target: test drop index interface when index not created + method: create collection and add entities in it, create index + expected: return code not equals to 0, drop index failed + ''' + # no create index + connect.drop_index(collection, field_name) + + @pytest.mark.level(2) + def test_create_drop_index_repeatedly(self, connect, collection, get_simple_index): + ''' + target: test create / drop index repeatedly, use the same index params + method: create index, drop index, four times + expected: return code 0 + ''' + for i in range(4): + connect.create_index(collection, field_name, get_simple_index) + connect.drop_index(collection, field_name) + + def test_drop_index_ip(self, connect, collection, get_simple_index): + ''' + target: test drop index interface + method: create collection and add entities in it, create index, call drop index + expected: return code 0, and default index param + ''' + # ids = connect.insert(collection, entities) + get_simple_index["metric_type"] = "IP" + connect.create_index(collection, field_name, get_simple_index) + connect.drop_index(collection, field_name) + index = connect.describe_index(collection, field_name) + assert not index + + @pytest.mark.level(2) + def test_drop_index_repeatedly_ip(self, connect, collection, get_simple_index): + ''' + target: test drop index repeatedly + method: create index, call drop index, and drop again + expected: return code 0 + ''' + get_simple_index["metric_type"] = "IP" + connect.create_index(collection, field_name, get_simple_index) + connect.drop_index(collection, field_name) + connect.drop_index(collection, field_name) + index = connect.describe_index(collection, field_name) + assert not index + + @pytest.mark.level(2) + def test_drop_index_without_connect_ip(self, dis_connect, collection): + ''' + target: test drop index without connection + method: drop index, and check if drop successfully + expected: raise exception + ''' + with pytest.raises(Exception) as e: + dis_connect.drop_index(collection, field_name) + + def test_drop_index_collection_not_create_ip(self, connect, collection): + ''' + target: test drop index interface when index not created + method: create collection and add entities in it, create index + expected: return code not equals to 0, drop index failed + ''' + # ids = connect.insert(collection, entities) + # no create index + connect.drop_index(collection, field_name) + + @pytest.mark.level(2) + def test_create_drop_index_repeatedly_ip(self, connect, collection, get_simple_index): + ''' + target: test create / drop index repeatedly, use the same index params + method: create index, drop index, four times + expected: return code 0 + ''' + get_simple_index["metric_type"] = "IP" + for i in range(4): + connect.create_index(collection, field_name, get_simple_index) + connect.drop_index(collection, field_name) + + +class TestIndexBinary: + @pytest.fixture( + scope="function", + params=gen_simple_index() + ) + def get_simple_index(self, request, connect): + if str(connect._cmd("mode")) == "CPU": + if request.param["index_type"] in index_cpu_not_support(): + pytest.skip("sq8h not support in CPU mode") + return request.param + + @pytest.fixture( + scope="function", + params=gen_binary_index() + ) + def get_jaccard_index(self, request, connect): + if request.param["index_type"] in binary_support(): + request.param["metric_type"] = "JACCARD" + return request.param + else: + pytest.skip("Skip index") + + @pytest.fixture( + scope="function", + params=gen_binary_index() + ) + def get_l2_index(self, request, connect): + request.param["metric_type"] = "L2" + return request.param + + @pytest.fixture( + scope="function", + params=[ + 1, + 10, + 1111 + ], + ) + def get_nq(self, request): + yield request.param + + """ + ****************************************************************** + The following cases are used to test `create_index` function + ****************************************************************** + """ + + @pytest.mark.timeout(BUILD_TIMEOUT) + def test_create_index(self, connect, binary_collection, get_jaccard_index): + ''' + target: test create index interface + method: create collection and add entities in it, create index + expected: return search success + ''' + ids = connect.insert(binary_collection, default_binary_entities) + connect.create_index(binary_collection, binary_field_name, get_jaccard_index) + binary_index = connect.describe_index(binary_collection, binary_field_name) + assert binary_index == get_jaccard_index + + @pytest.mark.timeout(BUILD_TIMEOUT) + def test_create_index_partition(self, connect, binary_collection, get_jaccard_index): + ''' + target: test create index interface + method: create collection, create partition, and add entities in it, create index + expected: return search success + ''' + connect.create_partition(binary_collection, default_tag) + ids = connect.insert(binary_collection, default_binary_entities, partition_tag=default_tag) + connect.create_index(binary_collection, binary_field_name, get_jaccard_index) + binary_index = connect.describe_index(binary_collection, binary_field_name) + assert binary_index == get_jaccard_index + + @pytest.mark.timeout(BUILD_TIMEOUT) + def test_create_index_search_with_query_vectors(self, connect, binary_collection, get_jaccard_index, get_nq): + ''' + target: test create index interface, search with more query vectors + method: create collection and add entities in it, create index + expected: return search success + ''' + nq = get_nq + ids = connect.insert(binary_collection, default_binary_entities) + connect.create_index(binary_collection, binary_field_name, get_jaccard_index) + query, vecs = gen_query_vectors(binary_field_name, default_binary_entities, default_top_k, nq, metric_type="JACCARD") + search_param = get_search_param(get_jaccard_index["index_type"], metric_type="JACCARD") + logging.getLogger().info(search_param) + connect.load_collection(binary_collection) + res = connect.search(binary_collection, query, search_params=search_param) + assert len(res) == nq + + @pytest.mark.timeout(BUILD_TIMEOUT) + def test_create_index_invalid_metric_type_binary(self, connect, binary_collection, get_l2_index): + ''' + target: test create index interface with invalid metric type + method: add entitys into binary connection, flash, create index with L2 metric type. + expected: return create_index failure + ''' + # insert 6000 vectors + ids = connect.insert(binary_collection, default_binary_entities) + connect.flush([binary_collection]) + if get_l2_index["index_type"] == "BIN_FLAT": + connect.create_index(binary_collection, binary_field_name, get_l2_index) + binary_index = connect.describe_index(binary_collection, binary_field_name) + assert binary_index == get_l2_index + else: + with pytest.raises(Exception) as e: + res = connect.create_index(binary_collection, binary_field_name, get_l2_index) + + """ + ****************************************************************** + The following cases are used to test `describe_index` function + *************************************************************** + """ + @pytest.mark.skip("repeat with test_create_index binary") + def test_get_index_info(self, connect, binary_collection, get_jaccard_index): + ''' + target: test describe index interface + method: create collection and add entities in it, create index, call describe index + expected: return code 0, and index instructure + ''' + ids = connect.insert(binary_collection, default_binary_entities) + connect.flush([binary_collection]) + connect.create_index(binary_collection, binary_field_name, get_jaccard_index) + stats = connect.get_collection_stats(binary_collection) + assert stats["row_count"] == default_nb + for partition in stats["partitions"]: + segments = partition["segments"] + if segments: + for segment in segments: + for file in segment["files"]: + if "index_type" in file: + assert file["index_type"] == get_jaccard_index["index_type"] + + @pytest.mark.skip("repeat with test_create_index_partition binary") + def test_get_index_info_partition(self, connect, binary_collection, get_jaccard_index): + ''' + target: test describe index interface + method: create collection, create partition and add entities in it, create index, call describe index + expected: return code 0, and index instructure + ''' + connect.create_partition(binary_collection, default_tag) + ids = connect.insert(binary_collection, default_binary_entities, partition_tag=default_tag) + connect.flush([binary_collection]) + connect.create_index(binary_collection, binary_field_name, get_jaccard_index) + stats = connect.get_collection_stats(binary_collection) + logging.getLogger().info(stats) + assert stats["row_count"] == default_nb + assert len(stats["partitions"]) == 2 + for partition in stats["partitions"]: + segments = partition["segments"] + if segments: + for segment in segments: + for file in segment["files"]: + if "index_type" in file: + assert file["index_type"] == get_jaccard_index["index_type"] + + """ + ****************************************************************** + The following cases are used to test `drop_index` function + ****************************************************************** + """ + def test_drop_index(self, connect, binary_collection, get_jaccard_index): + ''' + target: test drop index interface + method: create collection and add entities in it, create index, call drop index + expected: return code 0, and default index param + ''' + connect.create_index(binary_collection, binary_field_name, get_jaccard_index) + stats = connect.get_collection_stats(binary_collection) + logging.getLogger().info(stats) + connect.drop_index(binary_collection, binary_field_name) + binary_index = connect.describe_index(binary_collection, binary_field_name) + assert not binary_index + + def test_drop_index_partition(self, connect, binary_collection, get_jaccard_index): + ''' + target: test drop index interface + method: create collection, create partition and add entities in it, create index on collection, call drop collection index + expected: return code 0, and default index param + ''' + connect.create_partition(binary_collection, default_tag) + ids = connect.insert(binary_collection, default_binary_entities, partition_tag=default_tag) + connect.flush([binary_collection]) + connect.create_index(binary_collection, binary_field_name, get_jaccard_index) + connect.drop_index(binary_collection, binary_field_name) + binary_index = connect.describe_index(binary_collection, binary_field_name) + assert not binary_index + + +class TestIndexInvalid(object): + """ + Test create / describe / drop index interfaces with invalid collection names + """ + + @pytest.fixture( + scope="function", + params=gen_invalid_strs() + ) + def get_collection_name(self, request): + yield request.param + + @pytest.mark.level(1) + def test_create_index_with_invalid_collection_name(self, connect, get_collection_name): + collection_name = get_collection_name + with pytest.raises(Exception) as e: + connect.create_index(collection_name, field_name, default_index) + + @pytest.mark.level(1) + def test_drop_index_with_invalid_collection_name(self, connect, get_collection_name): + collection_name = get_collection_name + with pytest.raises(Exception) as e: + connect.drop_index(collection_name) + + @pytest.fixture( + scope="function", + params=gen_invalid_index() + ) + def get_index(self, request): + yield request.param + + @pytest.mark.level(2) + def test_create_index_with_invalid_index_params(self, connect, collection, get_index): + logging.getLogger().info(get_index) + with pytest.raises(Exception) as e: + connect.create_index(collection, field_name, get_index) + + +class TestIndexAsync: + @pytest.fixture(scope="function", autouse=True) + def skip_http_check(self, args): + if args["handler"] == "HTTP": + pytest.skip("skip in http mode") + + """ + ****************************************************************** + The following cases are used to test `create_index` function + ****************************************************************** + """ + + @pytest.fixture( + scope="function", + params=gen_simple_index() + ) + def get_simple_index(self, request, connect): + if str(connect._cmd("mode")) == "CPU": + if request.param["index_type"] in index_cpu_not_support(): + pytest.skip("sq8h not support in CPU mode") + return request.param + + def check_result(self, res): + logging.getLogger().info("In callback check search result") + logging.getLogger().info(res) + + """ + ****************************************************************** + The following cases are used to test `create_index` function + ****************************************************************** + """ + + @pytest.mark.timeout(BUILD_TIMEOUT) + def test_create_index(self, connect, collection, get_simple_index): + ''' + target: test create index interface + method: create collection and add entities in it, create index + expected: return search success + ''' + ids = connect.insert(collection, default_entities) + logging.getLogger().info("start index") + future = connect.create_index(collection, field_name, get_simple_index, _async=True) + logging.getLogger().info("before result") + res = future.result() + # TODO: + logging.getLogger().info(res) + + @pytest.mark.timeout(BUILD_TIMEOUT) + def test_create_index_drop(self, connect, collection, get_simple_index): + ''' + target: test create index interface + method: create collection and add entities in it, create index + expected: return search success + ''' + ids = connect.insert(collection, default_entities) + logging.getLogger().info("start index") + future = connect.create_index(collection, field_name, get_simple_index, _async=True) + logging.getLogger().info("DROP") + connect.drop_collection(collection) + + @pytest.mark.level(2) + def test_create_index_with_invalid_collection_name(self, connect): + collection_name = " " + future = connect.create_index(collection_name, field_name, default_index, _async=True) + with pytest.raises(Exception) as e: + res = future.result() + + @pytest.mark.timeout(BUILD_TIMEOUT) + def test_create_index_callback(self, connect, collection, get_simple_index): + ''' + target: test create index interface + method: create collection and add entities in it, create index + expected: return search success + ''' + ids = connect.insert(collection, default_entities) + logging.getLogger().info("start index") + future = connect.create_index(collection, field_name, get_simple_index, _async=True, + _callback=self.check_result) + logging.getLogger().info("before result") + res = future.result() + # TODO: + logging.getLogger().info(res) diff --git a/tests/python_test/test_mix.py b/tests/python_test/test_mix.py new file mode 100644 index 0000000000..2712b89d5b --- /dev/null +++ b/tests/python_test/test_mix.py @@ -0,0 +1,161 @@ +import pdb +import copy +import pytest +import threading +import datetime +import logging +from time import sleep +from multiprocessing import Process +import sklearn.preprocessing +from utils import * + +index_file_size = 10 +vectors = gen_vectors(10000, default_dim) +vectors = sklearn.preprocessing.normalize(vectors, axis=1, norm='l2') +vectors = vectors.tolist() +top_k = 1 +nprobe = 1 +epsilon = 0.001 +nlist = 128 +# index_params = {'index_type': IndexType.IVFLAT, 'nlist': 16384} + + +class TestMixBase: + # disable + def _test_search_during_createIndex(self, args): + loops = 10000 + collection = gen_unique_str() + query_vecs = [vectors[0], vectors[1]] + uri = "tcp://%s:%s" % (args["ip"], args["port"]) + id_0 = 0; id_1 = 0 + milvus_instance = get_milvus(args["handler"]) + # milvus_instance.connect(uri=uri) + milvus_instance.create_collection({'collection_name': collection, + 'dimension': default_dim, + 'index_file_size': index_file_size, + 'metric_type': "L2"}) + for i in range(10): + status, ids = milvus_instance.bulk_insert(collection, vectors) + # logging.getLogger().info(ids) + if i == 0: + id_0 = ids[0]; id_1 = ids[1] + # def create_index(milvus_instance): + # logging.getLogger().info("In create index") + # status = milvus_instance.create_index(collection, index_params) + # logging.getLogger().info(status) + # status, result = milvus_instance.get_index_info(collection) + # logging.getLogger().info(result) + def insert(milvus_instance): + logging.getLogger().info("In add vectors") + status, ids = milvus_instance.bulk_insert(collection, vectors) + logging.getLogger().info(status) + def search(milvus_instance): + logging.getLogger().info("In search vectors") + for i in range(loops): + status, result = milvus_instance.search(collection, top_k, nprobe, query_vecs) + logging.getLogger().info(status) + assert result[0][0].id == id_0 + assert result[1][0].id == id_1 + milvus_instance = get_milvus(args["handler"]) + # milvus_instance.connect(uri=uri) + p_search = Process(target=search, args=(milvus_instance, )) + p_search.start() + milvus_instance = get_milvus(args["handler"]) + # milvus_instance.connect(uri=uri) + p_create = Process(target=insert, args=(milvus_instance, )) + p_create.start() + p_create.join() + + @pytest.mark.level(2) + def _test_mix_multi_collections(self, connect): + ''' + target: test functions with multiple collections of different metric_types and index_types + method: create 60 collections which 30 are L2 and the other are IP, add vectors into them + and test describe index and search + expected: status ok + ''' + nq = 10000 + collection_list = [] + idx = [] + index_param = {'nlist': nlist} + + #create collection and add vectors + for i in range(30): + collection_name = gen_unique_str('test_mix_multi_collections') + collection_list.append(collection_name) + param = {'collection_name': collection_name, + 'dimension': default_dim, + 'index_file_size': index_file_size, + 'metric_type': MetricType.L2} + connect.create_collection(param) + status, ids = connect.bulk_insert(collection_name=collection_name, records=vectors) + idx.append(ids[0]) + idx.append(ids[10]) + idx.append(ids[20]) + assert status.OK() + for i in range(30): + collection_name = gen_unique_str('test_mix_multi_collections') + collection_list.append(collection_name) + param = {'collection_name': collection_name, + 'dimension': default_dim, + 'index_file_size': index_file_size, + 'metric_type': MetricType.IP} + connect.create_collection(param) + status, ids = connect.bulk_insert(collection_name=collection_name, records=vectors) + assert status.OK() + status = connect.flush([collection_name]) + assert status.OK() + idx.append(ids[0]) + idx.append(ids[10]) + idx.append(ids[20]) + assert status.OK() + for i in range(10): + status = connect.create_index(collection_list[i], IndexType.FLAT, index_param) + assert status.OK() + status = connect.create_index(collection_list[30 + i], IndexType.FLAT, index_param) + assert status.OK() + status = connect.create_index(collection_list[10 + i], IndexType.IVFLAT, index_param) + assert status.OK() + status = connect.create_index(collection_list[40 + i], IndexType.IVFLAT, index_param) + assert status.OK() + status = connect.create_index(collection_list[20 + i], IndexType.IVF_SQ8, index_param) + assert status.OK() + status = connect.create_index(collection_list[50 + i], IndexType.IVF_SQ8, index_param) + assert status.OK() + + #describe index + for i in range(10): + status, result = connect.get_index_info(collection_list[i]) + assert result._index_type == IndexType.FLAT + status, result = connect.get_index_info(collection_list[10 + i]) + assert result._index_type == IndexType.IVFLAT + status, result = connect.get_index_info(collection_list[20 + i]) + assert result._index_type == IndexType.IVF_SQ8 + status, result = connect.get_index_info(collection_list[30 + i]) + assert result._index_type == IndexType.FLAT + status, result = connect.get_index_info(collection_list[40 + i]) + assert result._index_type == IndexType.IVFLAT + status, result = connect.get_index_info(collection_list[50 + i]) + assert result._index_type == IndexType.IVF_SQ8 + + #search + query_vecs = [vectors[0], vectors[10], vectors[20]] + for i in range(60): + collection = collection_list[i] + status, result = connect.search(collection, top_k, query_records=query_vecs, params={"nprobe": 1}) + assert status.OK() + assert len(result) == len(query_vecs) + logging.getLogger().info(i) + for j in range(len(query_vecs)): + assert len(result[j]) == top_k + for j in range(len(query_vecs)): + if not check_result(result[j], idx[3 * i + j]): + logging.getLogger().info(result[j]._id_list) + logging.getLogger().info(idx[3 * i + j]) + assert check_result(result[j], idx[3 * i + j]) + +def check_result(result, id): + if len(result) >= 5: + return id in [result[0].id, result[1].id, result[2].id, result[3].id, result[4].id] + else: + return id in (i.id for i in result) diff --git a/tests/python_test/test_partition.py b/tests/python_test/test_partition.py new file mode 100644 index 0000000000..0f104b4012 --- /dev/null +++ b/tests/python_test/test_partition.py @@ -0,0 +1,425 @@ +import time +import random +import pdb +import threading +import logging +from multiprocessing import Pool, Process +import pytest +from utils import * +from constants import * + +TIMEOUT = 120 + + +class TestCreateBase: + """ + ****************************************************************** + The following cases are used to test `create_partition` function + ****************************************************************** + """ + def test_create_partition_a(self, connect, collection): + ''' + target: test create partition, check status returned + method: call function: create_partition + expected: status ok + ''' + connect.create_partition(collection, default_tag) + + # TODO: enable + @pytest.mark.level(2) + @pytest.mark.timeout(600) + def test_create_partition_limit(self, connect, collection, args): + ''' + target: test create partitions, check status returned + method: call function: create_partition for 4097 times + expected: exception raised + ''' + threads_num = 8 + threads = [] + if args["handler"] == "HTTP": + pytest.skip("skip in http mode") + + def create(connect, threads_num): + for i in range(max_partition_num // threads_num): + tag_tmp = gen_unique_str() + connect.create_partition(collection, tag_tmp) + + for i in range(threads_num): + m = get_milvus(host=args["ip"], port=args["port"], handler=args["handler"]) + t = threading.Thread(target=create, args=(m, threads_num, )) + threads.append(t) + t.start() + for t in threads: + t.join() + tag_tmp = gen_unique_str() + with pytest.raises(Exception) as e: + connect.create_partition(collection, tag_tmp) + + def test_create_partition_repeat(self, connect, collection): + ''' + target: test create partition, check status returned + method: call function: create_partition + expected: status ok + ''' + connect.create_partition(collection, default_tag) + with pytest.raises(Exception) as e: + connect.create_partition(collection, default_tag) + assert compare_list_elements(connect.list_partitions(collection), [default_tag, '_default']) + + def test_create_partition_collection_not_existed(self, connect): + ''' + target: test create partition, its owner collection name not existed in db, check status returned + method: call function: create_partition + expected: status not ok + ''' + collection_name = gen_unique_str() + with pytest.raises(Exception) as e: + connect.create_partition(collection_name, default_tag) + + def test_create_partition_tag_name_None(self, connect, collection): + ''' + target: test create partition, tag name set None, check status returned + method: call function: create_partition + expected: status ok + ''' + tag_name = None + with pytest.raises(Exception) as e: + connect.create_partition(collection, tag_name) + + def test_create_different_partition_tags(self, connect, collection): + ''' + target: test create partition twice with different names + method: call function: create_partition, and again + expected: status ok + ''' + connect.create_partition(collection, default_tag) + tag_name = gen_unique_str() + connect.create_partition(collection, tag_name) + assert compare_list_elements(connect.list_partitions(collection), [default_tag, tag_name, '_default']) + + def test_create_partition_insert_default(self, connect, id_collection): + ''' + target: test create partition, and insert vectors, check status returned + method: call function: create_partition + expected: status ok + ''' + connect.create_partition(id_collection, default_tag) + ids = [i for i in range(default_nb)] + insert_ids = connect.insert(id_collection, default_entities, ids) + assert len(insert_ids) == len(ids) + + def test_create_partition_insert_with_tag(self, connect, id_collection): + ''' + target: test create partition, and insert vectors, check status returned + method: call function: create_partition + expected: status ok + ''' + connect.create_partition(id_collection, default_tag) + ids = [i for i in range(default_nb)] + insert_ids = connect.insert(id_collection, default_entities, ids, partition_tag=default_tag) + assert len(insert_ids) == len(ids) + + def test_create_partition_insert_with_tag_not_existed(self, connect, collection): + ''' + target: test create partition, and insert vectors, check status returned + method: call function: create_partition + expected: status not ok + ''' + tag_new = "tag_new" + connect.create_partition(collection, default_tag) + ids = [i for i in range(default_nb)] + with pytest.raises(Exception) as e: + connect.insert(collection, default_entities, ids, partition_tag=tag_new) + + @pytest.mark.skip("get_collection_stats") + def test_create_partition_insert_same_tags(self, connect, id_collection): + ''' + target: test create partition, and insert vectors, check status returned + method: call function: create_partition + expected: status ok + ''' + connect.create_partition(id_collection, default_tag) + ids = [i for i in range(default_nb)] + insert_ids = connect.insert(id_collection, default_entities, ids, partition_tag=default_tag) + ids = [(i+default_nb) for i in range(default_nb)] + new_insert_ids = connect.insert(id_collection, default_entities, ids, partition_tag=default_tag) + connect.flush([id_collection]) + res = connect.get_collection_stats(id_collection) + assert res["row_count"] == default_nb * 2 + + @pytest.mark.skip("get_collection_stats") + @pytest.mark.level(2) + def test_create_partition_insert_same_tags_two_collections(self, connect, collection): + ''' + target: test create two partitions, and insert vectors with the same tag to each collection, check status returned + method: call function: create_partition + expected: status ok, collection length is correct + ''' + connect.create_partition(collection, default_tag) + collection_new = gen_unique_str() + connect.create_collection(collection_new, default_fields) + connect.create_partition(collection_new, default_tag) + ids = connect.insert(collection, default_entities, partition_tag=default_tag) + ids = connect.insert(collection_new, default_entities, partition_tag=default_tag) + connect.flush([collection, collection_new]) + res = connect.get_collection_stats(collection) + assert res["row_count"] == default_nb + res = connect.get_collection_stats(collection_new) + assert res["row_count"] == default_nb + + +class TestShowBase: + + """ + ****************************************************************** + The following cases are used to test `list_partitions` function + ****************************************************************** + """ + def test_list_partitions(self, connect, collection): + ''' + target: test show partitions, check status and partitions returned + method: create partition first, then call function: list_partitions + expected: status ok, partition correct + ''' + connect.create_partition(collection, default_tag) + assert compare_list_elements(connect.list_partitions(collection), [default_tag, '_default']) + + def test_list_partitions_no_partition(self, connect, collection): + ''' + target: test show partitions with collection name, check status and partitions returned + method: call function: list_partitions + expected: status ok, partitions correct + ''' + res = connect.list_partitions(collection) + assert compare_list_elements(res, ['_default']) + + def test_show_multi_partitions(self, connect, collection): + ''' + target: test show partitions, check status and partitions returned + method: create partitions first, then call function: list_partitions + expected: status ok, partitions correct + ''' + tag_new = gen_unique_str() + connect.create_partition(collection, default_tag) + connect.create_partition(collection, tag_new) + res = connect.list_partitions(collection) + assert compare_list_elements(res, [default_tag, tag_new, '_default']) + + +class TestHasBase: + + """ + ****************************************************************** + The following cases are used to test `has_partition` function + ****************************************************************** + """ + @pytest.fixture( + scope="function", + params=gen_invalid_strs() + ) + def get_tag_name(self, request): + yield request.param + + def test_has_partition_a(self, connect, collection): + ''' + target: test has_partition, check status and result + method: create partition first, then call function: has_partition + expected: status ok, result true + ''' + connect.create_partition(collection, default_tag) + res = connect.has_partition(collection, default_tag) + logging.getLogger().info(res) + assert res + + def test_has_partition_multi_partitions(self, connect, collection): + ''' + target: test has_partition, check status and result + method: create partition first, then call function: has_partition + expected: status ok, result true + ''' + for tag_name in [default_tag, "tag_new", "tag_new_new"]: + connect.create_partition(collection, tag_name) + for tag_name in [default_tag, "tag_new", "tag_new_new"]: + res = connect.has_partition(collection, tag_name) + assert res + + def test_has_partition_tag_not_existed(self, connect, collection): + ''' + target: test has_partition, check status and result + method: then call function: has_partition, with tag not existed + expected: status ok, result empty + ''' + res = connect.has_partition(collection, default_tag) + logging.getLogger().info(res) + assert not res + + def test_has_partition_collection_not_existed(self, connect, collection): + ''' + target: test has_partition, check status and result + method: then call function: has_partition, with collection not existed + expected: status not ok + ''' + with pytest.raises(Exception) as e: + connect.has_partition("not_existed_collection", default_tag) + + @pytest.mark.level(2) + def test_has_partition_with_invalid_tag_name(self, connect, collection, get_tag_name): + ''' + target: test has partition, with invalid tag name, check status returned + method: call function: has_partition + expected: status ok + ''' + tag_name = get_tag_name + connect.create_partition(collection, default_tag) + with pytest.raises(Exception) as e: + connect.has_partition(collection, tag_name) + + +class TestDropBase: + + """ + ****************************************************************** + The following cases are used to test `drop_partition` function + ****************************************************************** + """ + def test_drop_partition_a(self, connect, collection): + ''' + target: test drop partition, check status and partition if existed + method: create partitions first, then call function: drop_partition + expected: status ok, no partitions in db + ''' + connect.create_partition(collection, default_tag) + res1 = connect.list_partitions(collection) + assert default_tag in res1 + connect.drop_partition(collection, default_tag) + res2 = connect.list_partitions(collection) + assert default_tag not in res2 + + def test_drop_partition_tag_not_existed(self, connect, collection): + ''' + target: test drop partition, but tag not existed + method: create partitions first, then call function: drop_partition + expected: status not ok + ''' + connect.create_partition(collection, default_tag) + new_tag = "new_tag" + with pytest.raises(Exception) as e: + connect.drop_partition(collection, new_tag) + + def test_drop_partition_tag_not_existed_A(self, connect, collection): + ''' + target: test drop partition, but collection not existed + method: create partitions first, then call function: drop_partition + expected: status not ok + ''' + connect.create_partition(collection, default_tag) + new_collection = gen_unique_str() + with pytest.raises(Exception) as e: + connect.drop_partition(new_collection, default_tag) + + @pytest.mark.level(2) + def test_drop_partition_repeatedly(self, connect, collection): + ''' + target: test drop partition twice, check status and partition if existed + method: create partitions first, then call function: drop_partition + expected: status not ok, no partitions in db + ''' + connect.create_partition(collection, default_tag) + connect.drop_partition(collection, default_tag) + time.sleep(2) + with pytest.raises(Exception) as e: + connect.drop_partition(collection, default_tag) + tag_list = connect.list_partitions(collection) + assert default_tag not in tag_list + + def test_drop_partition_create(self, connect, collection): + ''' + target: test drop partition, and create again, check status + method: create partitions first, then call function: drop_partition, create_partition + expected: status not ok, partition in db + ''' + connect.create_partition(collection, default_tag) + assert compare_list_elements(connect.list_partitions(collection), [default_tag, '_default']) + connect.drop_partition(collection, default_tag) + assert compare_list_elements(connect.list_partitions(collection), ['_default']) + time.sleep(2) + connect.create_partition(collection, default_tag) + assert compare_list_elements(connect.list_partitions(collection), [default_tag, '_default']) + + +class TestNameInvalid(object): + @pytest.fixture( + scope="function", + params=gen_invalid_strs() + ) + def get_tag_name(self, request): + yield request.param + + @pytest.fixture( + scope="function", + params=gen_invalid_strs() + ) + def get_collection_name(self, request): + yield request.param + + @pytest.mark.level(2) + def test_drop_partition_with_invalid_collection_name(self, connect, collection, get_collection_name): + ''' + target: test drop partition, with invalid collection name, check status returned + method: call function: drop_partition + expected: status not ok + ''' + collection_name = get_collection_name + connect.create_partition(collection, default_tag) + with pytest.raises(Exception) as e: + connect.drop_partition(collection_name, default_tag) + + @pytest.mark.level(2) + def test_drop_partition_with_invalid_tag_name(self, connect, collection, get_tag_name): + ''' + target: test drop partition, with invalid tag name, check status returned + method: call function: drop_partition + expected: status not ok + ''' + tag_name = get_tag_name + connect.create_partition(collection, default_tag) + with pytest.raises(Exception) as e: + connect.drop_partition(collection, tag_name) + + @pytest.mark.level(2) + def test_list_partitions_with_invalid_collection_name(self, connect, collection, get_collection_name): + ''' + target: test show partitions, with invalid collection name, check status returned + method: call function: list_partitions + expected: status not ok + ''' + collection_name = get_collection_name + connect.create_partition(collection, default_tag) + with pytest.raises(Exception) as e: + connect.list_partitions(collection_name) + + +class TestNewCase(object): + + def test_drop_default_partition_a(self, connect, collection): + ''' + target: test drop partition of default, check status returned + method: call function: drop_partition + expected: status not ok + ''' + with pytest.raises(Exception) as e: + connect.drop_partition(collection, partition_tag='_default') + list_partition = connect.list_partitions(collection) + assert '_default' in list_partition + + def test_drop_default_partition_b(self, connect, collection): + ''' + target: test drop partition of default, check status returned + method: call function: drop_partition + expected: status not ok + ''' + connect.create_partition(collection, default_tag) + with pytest.raises(Exception) as e: + connect.drop_partition(collection, partition_tag='_default') + list_partition = connect.list_partitions(collection) + assert '_default' in list_partition diff --git a/tests/python_test/test_ping.py b/tests/python_test/test_ping.py new file mode 100644 index 0000000000..46ae4c8986 --- /dev/null +++ b/tests/python_test/test_ping.py @@ -0,0 +1,129 @@ +import logging +import pytest + +__version__ = '0.11.1' + + +class TestPing: + def test_server_version(self, connect): + ''' + target: test get the server version + method: call the server_version method after connected + expected: version should be the milvus version + ''' + res = connect.server_version() + assert res == __version__ + + def test_server_status(self, connect): + ''' + target: test get the server status + method: call the server_status method after connected + expected: status returned should be ok + ''' + msg = connect.server_status() + assert msg + + def test_server_cmd_with_params_version(self, connect): + ''' + target: test cmd: version + method: cmd = "version" ... + expected: when cmd = 'version', return version of server; + ''' + cmd = "version" + msg = connect._cmd(cmd) + logging.getLogger().info(msg) + assert msg == __version__ + + def test_server_cmd_with_params_others(self, connect): + ''' + target: test cmd: lalala + method: cmd = "lalala" ... + expected: when cmd = 'version', return version of server; + ''' + cmd = "rm -rf test" + msg = connect._cmd(cmd) + + def test_connected(self, connect): + # assert connect.connected() + assert connect + + +class TestPingWithTimeout: + def test_server_version_legal_timeout(self, connect): + ''' + target: test get the server version with legal timeout + method: call the server_version method after connected with altering timeout + expected: version should be the milvus version + ''' + res = connect.server_version(20) + assert res == __version__ + + def test_server_version_negative_timeout(self, connect): + ''' + target: test get the server version with negative timeout + method: call the server_version method after connected with altering timeout + expected: when timeout is illegal raises an error; + ''' + with pytest.raises(Exception) as e: + res = connect.server_version(-1) + + def test_server_cmd_with_params_version_with_legal_timeout(self, connect): + ''' + target: test cmd: version and timeout + method: cmd = "version" , timeout=10 + expected: when cmd = 'version', return version of server; + ''' + cmd = "version" + msg = connect._cmd(cmd, 10) + logging.getLogger().info(msg) + assert msg == __version__ + + def test_server_cmd_with_params_version_with_illegal_timeout(self, connect): + ''' + target: test cmd: version and timeout + method: cmd = "version" , timeout=-1 + expected: when timeout is illegal raises an error; + ''' + with pytest.raises(Exception) as e: + res = connect.server_version(-1) + + def test_server_cmd_with_params_others_with_illegal_timeout(self, connect): + ''' + target: test cmd: lalala, timeout = -1 + method: cmd = "lalala", timeout = -1 + expected: when timeout is illegal raises an error; + ''' + cmd = "rm -rf test" + with pytest.raises(Exception) as e: + res = connect.server_version(-1) + + +class TestPingDisconnect: + def test_server_version(self, dis_connect): + ''' + target: test get the server version, after disconnect + method: call the server_version method after connected + expected: version should not be the pymilvus version + ''' + with pytest.raises(Exception) as e: + res = dis_connect.server_version() + + def test_server_status(self, dis_connect): + ''' + target: test get the server status, after disconnect + method: call the server_status method after connected + expected: status returned should be not ok + ''' + with pytest.raises(Exception) as e: + res = dis_connect.server_status() + + @pytest.mark.level(2) + def test_server_version_with_timeout(self, dis_connect): + ''' + target: test get the server status with timeout settings after disconnect + method: call the server_status method after connected + expected: status returned should be not ok + ''' + status = None + with pytest.raises(Exception) as e: + res = connect.server_status(100) diff --git a/tests/python_test/utils.py b/tests/python_test/utils.py new file mode 100644 index 0000000000..7462e43a49 --- /dev/null +++ b/tests/python_test/utils.py @@ -0,0 +1,1001 @@ +import os +import sys +import random +import pdb +import string +import struct +import logging +import threading +import time +import copy +import numpy as np +from sklearn import preprocessing +from milvus import Milvus, DataType + +port = 19530 +epsilon = 0.000001 +namespace = "milvus" + +default_flush_interval = 1 +big_flush_interval = 1000 +default_drop_interval = 3 +default_dim = 128 +default_nb = 1200 +default_top_k = 10 +max_top_k = 16384 +max_partition_num = 256 +default_segment_row_limit = 1000 +default_server_segment_row_limit = 1024 * 512 +default_float_vec_field_name = "float_vector" +default_binary_vec_field_name = "binary_vector" +default_partition_name = "_default" +default_tag = "1970_01_01" + +# TODO: +# TODO: disable RHNSW_SQ/PQ in 0.11.0 +all_index_types = [ + "FLAT", + "IVF_FLAT", + "IVF_SQ8", + "IVF_SQ8_HYBRID", + "IVF_PQ", + "HNSW", + # "NSG", + "ANNOY", + "RHNSW_PQ", + "RHNSW_SQ", + "BIN_FLAT", + "BIN_IVF_FLAT" +] + +default_index_params = [ + {"nlist": 128}, + {"nlist": 128}, + {"nlist": 128}, + {"nlist": 128}, + {"nlist": 128, "m": 16}, + {"M": 48, "efConstruction": 500}, + # {"search_length": 50, "out_degree": 40, "candidate_pool_size": 100, "knng": 50}, + {"n_trees": 50}, + {"M": 48, "efConstruction": 500, "PQM": 64}, + {"M": 48, "efConstruction": 500}, + {"nlist": 128}, + {"nlist": 128} +] + + +def index_cpu_not_support(): + return ["IVF_SQ8_HYBRID"] + + +def binary_support(): + return ["BIN_FLAT", "BIN_IVF_FLAT"] + + +def delete_support(): + return ["FLAT", "IVF_FLAT", "IVF_SQ8", "IVF_SQ8_HYBRID", "IVF_PQ"] + + +def ivf(): + return ["FLAT", "IVF_FLAT", "IVF_SQ8", "IVF_SQ8_HYBRID", "IVF_PQ"] + + +def skip_pq(): + return ["IVF_PQ", "RHNSW_PQ", "RHNSW_SQ"] + + +def binary_metrics(): + return ["JACCARD", "HAMMING", "TANIMOTO", "SUBSTRUCTURE", "SUPERSTRUCTURE"] + + +def structure_metrics(): + return ["SUBSTRUCTURE", "SUPERSTRUCTURE"] + + +def l2(x, y): + return np.linalg.norm(np.array(x) - np.array(y)) + + +def ip(x, y): + return np.inner(np.array(x), np.array(y)) + + +def jaccard(x, y): + x = np.asarray(x, np.bool) + y = np.asarray(y, np.bool) + return 1 - np.double(np.bitwise_and(x, y).sum()) / np.double(np.bitwise_or(x, y).sum()) + + +def hamming(x, y): + x = np.asarray(x, np.bool) + y = np.asarray(y, np.bool) + return np.bitwise_xor(x, y).sum() + + +def tanimoto(x, y): + x = np.asarray(x, np.bool) + y = np.asarray(y, np.bool) + return -np.log2(np.double(np.bitwise_and(x, y).sum()) / np.double(np.bitwise_or(x, y).sum())) + + +def substructure(x, y): + x = np.asarray(x, np.bool) + y = np.asarray(y, np.bool) + return 1 - np.double(np.bitwise_and(x, y).sum()) / np.count_nonzero(y) + + +def superstructure(x, y): + x = np.asarray(x, np.bool) + y = np.asarray(y, np.bool) + return 1 - np.double(np.bitwise_and(x, y).sum()) / np.count_nonzero(x) + + +def get_milvus(host, port, uri=None, handler=None, **kwargs): + if handler is None: + handler = "GRPC" + try_connect = kwargs.get("try_connect", True) + if uri is not None: + milvus = Milvus(uri=uri, handler=handler, try_connect=try_connect) + else: + milvus = Milvus(host=host, port=port, handler=handler, try_connect=try_connect) + return milvus + + +def reset_build_index_threshold(connect): + connect.set_config("engine", "build_index_threshold", 1024) + + +def disable_flush(connect): + connect.set_config("storage", "auto_flush_interval", big_flush_interval) + + +def enable_flush(connect): + # reset auto_flush_interval=1 + connect.set_config("storage", "auto_flush_interval", default_flush_interval) + config_value = connect.get_config("storage", "auto_flush_interval") + assert config_value == str(default_flush_interval) + + +def gen_inaccuracy(num): + return num / 255.0 + + +def gen_vectors(num, dim, is_normal=True): + vectors = [[random.random() for _ in range(dim)] for _ in range(num)] + vectors = preprocessing.normalize(vectors, axis=1, norm='l2') + return vectors.tolist() + + +# def gen_vectors(num, dim, seed=np.random.RandomState(1234), is_normal=False): +# xb = seed.rand(num, dim).astype("float32") +# xb = preprocessing.normalize(xb, axis=1, norm='l2') +# return xb.tolist() + + +def gen_binary_vectors(num, dim): + raw_vectors = [] + binary_vectors = [] + for i in range(num): + raw_vector = [random.randint(0, 1) for i in range(dim)] + raw_vectors.append(raw_vector) + binary_vectors.append(bytes(np.packbits(raw_vector, axis=-1).tolist())) + return raw_vectors, binary_vectors + + +def gen_binary_sub_vectors(vectors, length): + raw_vectors = [] + binary_vectors = [] + dim = len(vectors[0]) + for i in range(length): + raw_vector = [0 for i in range(dim)] + vector = vectors[i] + for index, j in enumerate(vector): + if j == 1: + raw_vector[index] = 1 + raw_vectors.append(raw_vector) + binary_vectors.append(bytes(np.packbits(raw_vector, axis=-1).tolist())) + return raw_vectors, binary_vectors + + +def gen_binary_super_vectors(vectors, length): + raw_vectors = [] + binary_vectors = [] + dim = len(vectors[0]) + for i in range(length): + cnt_1 = np.count_nonzero(vectors[i]) + # raw_vector = [0 for i in range(dim)] ??? + raw_vector = [1 for i in range(dim)] + raw_vectors.append(raw_vector) + binary_vectors.append(bytes(np.packbits(raw_vector, axis=-1).tolist())) + return raw_vectors, binary_vectors + + +def gen_int_attr(row_num): + return [random.randint(0, 255) for _ in range(row_num)] + + +def gen_float_attr(row_num): + return [random.uniform(0, 255) for _ in range(row_num)] + + +def gen_unique_str(str_value=None): + prefix = "".join(random.choice(string.ascii_letters + string.digits) for _ in range(8)) + return "test_" + prefix if str_value is None else str_value + "_" + prefix + + +def gen_single_filter_fields(): + fields = [] + for data_type in DataType: + if data_type in [DataType.INT32, DataType.INT64, DataType.FLOAT, DataType.DOUBLE]: + fields.append({"name": data_type.name, "type": data_type}) + return fields + + +def gen_single_vector_fields(): + fields = [] + for data_type in [DataType.FLOAT_VECTOR, DataType.BINARY_VECTOR]: + field = {"name": data_type.name, "type": data_type, "params": {"dim": default_dim}} + fields.append(field) + return fields + + +def gen_default_fields(auto_id=True): + default_fields = { + "fields": [ + {"name": "int64", "type": DataType.INT64}, + {"name": "float", "type": DataType.FLOAT}, + {"name": default_float_vec_field_name, "type": DataType.FLOAT_VECTOR, "params": {"dim": default_dim}}, + ], + "segment_row_limit": default_segment_row_limit, + "auto_id": auto_id + } + return default_fields + + +def gen_binary_default_fields(auto_id=True): + default_fields = { + "fields": [ + {"name": "int64", "type": DataType.INT64}, + {"name": "float", "type": DataType.FLOAT}, + {"name": default_binary_vec_field_name, "type": DataType.BINARY_VECTOR, "params": {"dim": default_dim}} + ], + "segment_row_limit": default_segment_row_limit, + "auto_id": auto_id + } + return default_fields + + +def gen_entities(nb, is_normal=False): + vectors = gen_vectors(nb, default_dim, is_normal) + entities = [ + {"name": "int64", "type": DataType.INT64, "values": [i for i in range(nb)]}, + {"name": "float", "type": DataType.FLOAT, "values": [float(i) for i in range(nb)]}, + {"name": default_float_vec_field_name, "type": DataType.FLOAT_VECTOR, "values": vectors} + ] + return entities + + +def gen_entities_new(nb, is_normal=False): + vectors = gen_vectors(nb, default_dim, is_normal) + entities = [ + {"name": "int64", "values": [i for i in range(nb)]}, + {"name": "float", "values": [float(i) for i in range(nb)]}, + {"name": default_float_vec_field_name, "values": vectors} + ] + return entities + + +def gen_entities_rows(nb, is_normal=False, _id=True): + vectors = gen_vectors(nb, default_dim, is_normal) + entities = [] + if not _id: + for i in range(nb): + entity = { + "_id": i, + "int64": i, + "float": float(i), + default_float_vec_field_name: vectors[i] + } + entities.append(entity) + else: + for i in range(nb): + entity = { + "int64": i, + "float": float(i), + default_float_vec_field_name: vectors[i] + } + entities.append(entity) + return entities + + +def gen_binary_entities(nb): + raw_vectors, vectors = gen_binary_vectors(nb, default_dim) + entities = [ + {"name": "int64", "type": DataType.INT64, "values": [i for i in range(nb)]}, + {"name": "float", "type": DataType.FLOAT, "values": [float(i) for i in range(nb)]}, + {"name": default_binary_vec_field_name, "type": DataType.BINARY_VECTOR, "values": vectors} + ] + return raw_vectors, entities + + +def gen_binary_entities_new(nb): + raw_vectors, vectors = gen_binary_vectors(nb, default_dim) + entities = [ + {"name": "int64", "values": [i for i in range(nb)]}, + {"name": "float", "values": [float(i) for i in range(nb)]}, + {"name": default_binary_vec_field_name, "values": vectors} + ] + return raw_vectors, entities + + +def gen_binary_entities_rows(nb, _id=True): + raw_vectors, vectors = gen_binary_vectors(nb, default_dim) + entities = [] + if not _id: + for i in range(nb): + entity = { + "_id": i, + "int64": i, + "float": float(i), + default_binary_vec_field_name: vectors[i] + } + entities.append(entity) + else: + for i in range(nb): + entity = { + "int64": i, + "float": float(i), + default_binary_vec_field_name: vectors[i] + } + entities.append(entity) + return raw_vectors, entities + + +def gen_entities_by_fields(fields, nb, dim): + entities = [] + for field in fields: + if field["type"] in [DataType.INT32, DataType.INT64]: + field_value = [1 for i in range(nb)] + elif field["type"] in [DataType.FLOAT, DataType.DOUBLE]: + field_value = [3.0 for i in range(nb)] + elif field["type"] == DataType.BINARY_VECTOR: + field_value = gen_binary_vectors(nb, dim)[1] + elif field["type"] == DataType.FLOAT_VECTOR: + field_value = gen_vectors(nb, dim) + field.update({"values": field_value}) + entities.append(field) + return entities + + +def assert_equal_entity(a, b): + pass + + +def gen_query_vectors(field_name, entities, top_k, nq, search_params={"nprobe": 10}, rand_vector=False, + metric_type="L2", replace_vecs=None): + if rand_vector is True: + dimension = len(entities[-1]["values"][0]) + query_vectors = gen_vectors(nq, dimension) + else: + query_vectors = entities[-1]["values"][:nq] + if replace_vecs: + query_vectors = replace_vecs + must_param = {"vector": {field_name: {"topk": top_k, "query": query_vectors, "params": search_params}}} + must_param["vector"][field_name]["metric_type"] = metric_type + query = { + "bool": { + "must": [must_param] + } + } + return query, query_vectors + + +def update_query_expr(src_query, keep_old=True, expr=None): + tmp_query = copy.deepcopy(src_query) + if expr is not None: + tmp_query["bool"].update(expr) + if keep_old is not True: + tmp_query["bool"].pop("must") + return tmp_query + + +def gen_default_vector_expr(default_query): + return default_query["bool"]["must"][0] + + +def gen_default_term_expr(keyword="term", field="int64", values=None): + if values is None: + values = [i for i in range(default_nb // 2)] + expr = {keyword: {field: {"values": values}}} + return expr + + +def update_term_expr(src_term, terms): + tmp_term = copy.deepcopy(src_term) + for term in terms: + tmp_term["term"].update(term) + return tmp_term + + +def gen_default_range_expr(keyword="range", field="int64", ranges=None): + if ranges is None: + ranges = {"GT": 1, "LT": default_nb // 2} + expr = {keyword: {field: ranges}} + return expr + + +def update_range_expr(src_range, ranges): + tmp_range = copy.deepcopy(src_range) + for range in ranges: + tmp_range["range"].update(range) + return tmp_range + + +def gen_invalid_range(): + range = [ + {"range": 1}, + {"range": {}}, + {"range": []}, + {"range": {"range": {"int64": {"GT": 0, "LT": default_nb // 2}}}} + ] + return range + + +def gen_valid_ranges(): + ranges = [ + {"GT": 0, "LT": default_nb // 2}, + {"GT": default_nb // 2, "LT": default_nb * 2}, + {"GT": 0}, + {"LT": default_nb}, + {"GT": -1, "LT": default_top_k}, + ] + return ranges + + +def gen_invalid_term(): + terms = [ + {"term": 1}, + {"term": []}, + {"term": {}}, + {"term": {"term": {"int64": {"values": [i for i in range(default_nb // 2)]}}}} + ] + return terms + + +def add_field_default(default_fields, type=DataType.INT64, field_name=None): + tmp_fields = copy.deepcopy(default_fields) + if field_name is None: + field_name = gen_unique_str() + field = { + "name": field_name, + "type": type + } + tmp_fields["fields"].append(field) + return tmp_fields + + +def add_field(entities, field_name=None): + nb = len(entities[0]["values"]) + tmp_entities = copy.deepcopy(entities) + if field_name is None: + field_name = gen_unique_str() + field = { + "name": field_name, + "type": DataType.INT64, + "values": [i for i in range(nb)] + } + tmp_entities.append(field) + return tmp_entities + + +def add_vector_field(entities, is_normal=False): + nb = len(entities[0]["values"]) + vectors = gen_vectors(nb, default_dim, is_normal) + field = { + "name": gen_unique_str(), + "type": DataType.FLOAT_VECTOR, + "values": vectors + } + entities.append(field) + return entities + + +# def update_fields_metric_type(fields, metric_type): +# tmp_fields = copy.deepcopy(fields) +# if metric_type in ["L2", "IP"]: +# tmp_fields["fields"][-1]["type"] = DataType.FLOAT_VECTOR +# else: +# tmp_fields["fields"][-1]["type"] = DataType.BINARY_VECTOR +# tmp_fields["fields"][-1]["params"]["metric_type"] = metric_type +# return tmp_fields + + +def remove_field(entities): + del entities[0] + return entities + + +def remove_vector_field(entities): + del entities[-1] + return entities + + +def update_field_name(entities, old_name, new_name): + tmp_entities = copy.deepcopy(entities) + for item in tmp_entities: + if item["name"] == old_name: + item["name"] = new_name + return tmp_entities + + +def update_field_type(entities, old_name, new_name): + tmp_entities = copy.deepcopy(entities) + for item in tmp_entities: + if item["name"] == old_name: + item["type"] = new_name + return tmp_entities + + +def update_field_value(entities, old_type, new_value): + tmp_entities = copy.deepcopy(entities) + for item in tmp_entities: + if item["type"] == old_type: + for index, value in enumerate(item["values"]): + item["values"][index] = new_value + return tmp_entities + + +def update_field_name_row(entities, old_name, new_name): + tmp_entities = copy.deepcopy(entities) + for item in tmp_entities: + if old_name in item: + item[new_name] = item[old_name] + item.pop(old_name) + else: + raise Exception("Field %s not in field" % old_name) + return tmp_entities + + +def update_field_type_row(entities, old_name, new_name): + tmp_entities = copy.deepcopy(entities) + for item in tmp_entities: + if old_name in item: + item["type"] = new_name + return tmp_entities + + +def add_vector_field(nb, dimension=default_dim): + field_name = gen_unique_str() + field = { + "name": field_name, + "type": DataType.FLOAT_VECTOR, + "values": gen_vectors(nb, dimension) + } + return field_name + + +def gen_segment_row_limits(): + sizes = [ + 1024, + 4096 + ] + return sizes + + +def gen_invalid_ips(): + ips = [ + # "255.0.0.0", + # "255.255.0.0", + # "255.255.255.0", + # "255.255.255.255", + "127.0.0", + # "123.0.0.2", + "12-s", + " ", + "12 s", + "BB。A", + " siede ", + "(mn)", + "中文", + "a".join("a" for _ in range(256)) + ] + return ips + + +def gen_invalid_uris(): + ip = None + uris = [ + " ", + "中文", + # invalid protocol + # "tc://%s:%s" % (ip, port), + # "tcp%s:%s" % (ip, port), + + # # invalid port + # "tcp://%s:100000" % ip, + # "tcp://%s: " % ip, + # "tcp://%s:19540" % ip, + # "tcp://%s:-1" % ip, + # "tcp://%s:string" % ip, + + # invalid ip + "tcp:// :19530", + # "tcp://123.0.0.1:%s" % port, + "tcp://127.0.0:19530", + # "tcp://255.0.0.0:%s" % port, + # "tcp://255.255.0.0:%s" % port, + # "tcp://255.255.255.0:%s" % port, + # "tcp://255.255.255.255:%s" % port, + "tcp://\n:19530", + ] + return uris + + +def gen_invalid_strs(): + strings = [ + 1, + [1], + None, + "12-s", + # " ", + # "", + # None, + "12 s", + "(mn)", + "中文", + "a".join("a" for i in range(256)) + ] + return strings + + +def gen_invalid_field_types(): + field_types = [ + # 1, + "=c", + # 0, + None, + "", + "a".join("a" for i in range(256)) + ] + return field_types + + +def gen_invalid_metric_types(): + metric_types = [ + 1, + "=c", + 0, + None, + "", + "a".join("a" for i in range(256)) + ] + return metric_types + + +# TODO: +def gen_invalid_ints(): + int_values = [ + # 1.0, + None, + [1, 2, 3], + " ", + "", + -1, + "String", + "=c", + "中文", + "a".join("a" for i in range(256)) + ] + return int_values + + +def gen_invalid_params(): + params = [ + 9999999999, + -1, + # None, + [1, 2, 3], + " ", + "", + "String", + "中文" + ] + return params + + +def gen_invalid_vectors(): + invalid_vectors = [ + "1*2", + [], + [1], + [1, 2], + [" "], + ['a'], + [None], + None, + (1, 2), + {"a": 1}, + " ", + "", + "String", + " siede ", + "中文", + "a".join("a" for i in range(256)) + ] + return invalid_vectors + + +def gen_invaild_search_params(): + invalid_search_key = 100 + search_params = [] + for index_type in all_index_types: + if index_type == "FLAT": + continue + search_params.append({"index_type": index_type, "search_params": {"invalid_key": invalid_search_key}}) + if index_type in delete_support(): + for nprobe in gen_invalid_params(): + ivf_search_params = {"index_type": index_type, "search_params": {"nprobe": nprobe}} + search_params.append(ivf_search_params) + elif index_type in ["HNSW", "RHNSW_PQ", "RHNSW_SQ"]: + for ef in gen_invalid_params(): + hnsw_search_param = {"index_type": index_type, "search_params": {"ef": ef}} + search_params.append(hnsw_search_param) + elif index_type == "NSG": + for search_length in gen_invalid_params(): + nsg_search_param = {"index_type": index_type, "search_params": {"search_length": search_length}} + search_params.append(nsg_search_param) + search_params.append({"index_type": index_type, "search_params": {"invalid_key": 100}}) + elif index_type == "ANNOY": + for search_k in gen_invalid_params(): + if isinstance(search_k, int): + continue + annoy_search_param = {"index_type": index_type, "search_params": {"search_k": search_k}} + search_params.append(annoy_search_param) + return search_params + + +def gen_invalid_index(): + index_params = [] + for index_type in gen_invalid_strs(): + index_param = {"index_type": index_type, "params": {"nlist": 1024}} + index_params.append(index_param) + for nlist in gen_invalid_params(): + index_param = {"index_type": "IVF_FLAT", "params": {"nlist": nlist}} + index_params.append(index_param) + for M in gen_invalid_params(): + index_param = {"index_type": "HNSW", "params": {"M": M, "efConstruction": 100}} + index_param = {"index_type": "RHNSW_PQ", "params": {"M": M, "efConstruction": 100}} + index_param = {"index_type": "RHNSW_SQ", "params": {"M": M, "efConstruction": 100}} + index_params.append(index_param) + for efConstruction in gen_invalid_params(): + index_param = {"index_type": "HNSW", "params": {"M": 16, "efConstruction": efConstruction}} + index_param = {"index_type": "RHNSW_PQ", "params": {"M": 16, "efConstruction": efConstruction}} + index_param = {"index_type": "RHNSW_SQ", "params": {"M": 16, "efConstruction": efConstruction}} + index_params.append(index_param) + for search_length in gen_invalid_params(): + index_param = {"index_type": "NSG", + "params": {"search_length": search_length, "out_degree": 40, "candidate_pool_size": 50, + "knng": 100}} + index_params.append(index_param) + for out_degree in gen_invalid_params(): + index_param = {"index_type": "NSG", + "params": {"search_length": 100, "out_degree": out_degree, "candidate_pool_size": 50, + "knng": 100}} + index_params.append(index_param) + for candidate_pool_size in gen_invalid_params(): + index_param = {"index_type": "NSG", "params": {"search_length": 100, "out_degree": 40, + "candidate_pool_size": candidate_pool_size, + "knng": 100}} + index_params.append(index_param) + index_params.append({"index_type": "IVF_FLAT", "params": {"invalid_key": 1024}}) + index_params.append({"index_type": "HNSW", "params": {"invalid_key": 16, "efConstruction": 100}}) + index_params.append({"index_type": "RHNSW_PQ", "params": {"invalid_key": 16, "efConstruction": 100}}) + index_params.append({"index_type": "RHNSW_SQ", "params": {"invalid_key": 16, "efConstruction": 100}}) + index_params.append({"index_type": "NSG", + "params": {"invalid_key": 100, "out_degree": 40, "candidate_pool_size": 300, + "knng": 100}}) + for invalid_n_trees in gen_invalid_params(): + index_params.append({"index_type": "ANNOY", "params": {"n_trees": invalid_n_trees}}) + + return index_params + + +def gen_index(): + nlists = [1, 1024, 16384] + pq_ms = [128, 64, 32, 16, 8, 4] + Ms = [5, 24, 48] + efConstructions = [100, 300, 500] + search_lengths = [10, 100, 300] + out_degrees = [5, 40, 300] + candidate_pool_sizes = [50, 100, 300] + knngs = [5, 100, 300] + + index_params = [] + for index_type in all_index_types: + if index_type in ["FLAT", "BIN_FLAT", "BIN_IVF_FLAT"]: + index_params.append({"index_type": index_type, "index_param": {"nlist": 1024}}) + elif index_type in ["IVF_FLAT", "IVF_SQ8", "IVF_SQ8_HYBRID"]: + ivf_params = [{"index_type": index_type, "index_param": {"nlist": nlist}} \ + for nlist in nlists] + index_params.extend(ivf_params) + elif index_type == "IVF_PQ": + IVFPQ_params = [{"index_type": index_type, "index_param": {"nlist": nlist, "m": m}} \ + for nlist in nlists \ + for m in pq_ms] + index_params.extend(IVFPQ_params) + elif index_type in ["HNSW", "RHNSW_SQ", "RHNSW_PQ"]: + hnsw_params = [{"index_type": index_type, "index_param": {"M": M, "efConstruction": efConstruction}} \ + for M in Ms \ + for efConstruction in efConstructions] + index_params.extend(hnsw_params) + elif index_type == "NSG": + nsg_params = [{"index_type": index_type, + "index_param": {"search_length": search_length, "out_degree": out_degree, + "candidate_pool_size": candidate_pool_size, "knng": knng}} \ + for search_length in search_lengths \ + for out_degree in out_degrees \ + for candidate_pool_size in candidate_pool_sizes \ + for knng in knngs] + index_params.extend(nsg_params) + + return index_params + + +def gen_simple_index(): + index_params = [] + for i in range(len(all_index_types)): + if all_index_types[i] in binary_support(): + continue + dic = {"index_type": all_index_types[i], "metric_type": "L2"} + dic.update({"params": default_index_params[i]}) + index_params.append(dic) + return index_params + + +def gen_binary_index(): + index_params = [] + for i in range(len(all_index_types)): + if all_index_types[i] in binary_support(): + dic = {"index_type": all_index_types[i]} + dic.update({"params": default_index_params[i]}) + index_params.append(dic) + return index_params + + +def get_search_param(index_type, metric_type="L2"): + search_params = {"metric_type": metric_type} + if index_type in ivf() or index_type in binary_support(): + search_params.update({"nprobe": 64}) + elif index_type in ["HNSW", "RHNSW_SQ", "RHNSW_PQ"]: + search_params.update({"ef": 64}) + elif index_type == "NSG": + search_params.update({"search_length": 100}) + elif index_type == "ANNOY": + search_params.update({"search_k": 1000}) + else: + logging.getLogger().error("Invalid index_type.") + raise Exception("Invalid index_type.") + return search_params + + +def assert_equal_vector(v1, v2): + if len(v1) != len(v2): + assert False + for i in range(len(v1)): + assert abs(v1[i] - v2[i]) < epsilon + + +def restart_server(helm_release_name): + res = True + timeout = 120 + from kubernetes import client, config + client.rest.logger.setLevel(logging.WARNING) + + # service_name = "%s.%s.svc.cluster.local" % (helm_release_name, namespace) + config.load_kube_config() + v1 = client.CoreV1Api() + pod_name = None + # config_map_names = v1.list_namespaced_config_map(namespace, pretty='true') + # body = {"replicas": 0} + pods = v1.list_namespaced_pod(namespace) + for i in pods.items: + if i.metadata.name.find(helm_release_name) != -1 and i.metadata.name.find("mysql") == -1: + pod_name = i.metadata.name + break + # v1.patch_namespaced_config_map(config_map_name, namespace, body, pretty='true') + # status_res = v1.read_namespaced_service_status(helm_release_name, namespace, pretty='true') + logging.getLogger().debug("Pod name: %s" % pod_name) + if pod_name is not None: + try: + v1.delete_namespaced_pod(pod_name, namespace) + except Exception as e: + logging.error(str(e)) + logging.error("Exception when calling CoreV1Api->delete_namespaced_pod") + res = False + return res + logging.error("Sleep 10s after pod deleted") + time.sleep(10) + # check if restart successfully + pods = v1.list_namespaced_pod(namespace) + for i in pods.items: + pod_name_tmp = i.metadata.name + logging.error(pod_name_tmp) + if pod_name_tmp == pod_name: + continue + elif pod_name_tmp.find(helm_release_name) == -1 or pod_name_tmp.find("mysql") != -1: + continue + else: + status_res = v1.read_namespaced_pod_status(pod_name_tmp, namespace, pretty='true') + logging.error(status_res.status.phase) + start_time = time.time() + ready_break = False + while time.time() - start_time <= timeout: + logging.error(time.time()) + status_res = v1.read_namespaced_pod_status(pod_name_tmp, namespace, pretty='true') + if status_res.status.phase == "Running": + logging.error("Already running") + ready_break = True + time.sleep(10) + break + else: + time.sleep(1) + if time.time() - start_time > timeout: + logging.error("Restart pod: %s timeout" % pod_name_tmp) + res = False + return res + if ready_break: + break + else: + raise Exception("Pod: %s not found" % pod_name) + follow = True + pretty = True + previous = True # bool | Return previous terminated container logs. Defaults to false. (optional) + since_seconds = 56 # int | A relative time in seconds before the current time from which to show logs. If this value precedes the time a pod was started, only logs since the pod start will be returned. If this value is in the future, no logs will be returned. Only one of sinceSeconds or sinceTime may be specified. (optional) + timestamps = True # bool | If true, add an RFC3339 or RFC3339Nano timestamp at the beginning of every line of log output. Defaults to false. (optional) + container = "milvus" + # start_time = time.time() + # while time.time() - start_time <= timeout: + # try: + # api_response = v1.read_namespaced_pod_log(pod_name_tmp, namespace, container=container, follow=follow, + # pretty=pretty, previous=previous, since_seconds=since_seconds, + # timestamps=timestamps) + # logging.error(api_response) + # return res + # except Exception as e: + # logging.error("Exception when calling CoreV1Api->read_namespaced_pod_log: %s\n" % e) + # # waiting for server start + # time.sleep(5) + # # res = False + # # return res + # if time.time() - start_time > timeout: + # logging.error("Restart pod: %s timeout" % pod_name_tmp) + # res = False + return res + + +def compare_list_elements(_first, _second): + if not isinstance(_first, list) or not isinstance(_second, list) or len(_first) != len(_second): + return False + + for ele in _first: + if ele not in _second: + return False + + return True + + +class TestThread(threading.Thread): + def __init__(self, target, args=()): + threading.Thread.__init__(self, target=target, args=args) + + def run(self): + self.exc = None + try: + super(TestThread, self).run() + except BaseException as e: + self.exc = e + + def join(self): + super(TestThread, self).join() + if self.exc: + raise self.exc