test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-02 03:48:37 +08:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

[test]Create index before load for deploy test (#19876)

Browse Source 
Signed-off-by: zhuwenxing <wenxing.zhu@zilliz.com>

Signed-off-by: zhuwenxing <wenxing.zhu@zilliz.com>
This commit is contained in:
zhuwenxing 2022-10-19 10:19:26 +08:00 committed by GitHub
parent db354c3564
commit 2e7b2cea1b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 24 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
tests/python_client/deploy/scripts/action_after_upgrade.py
View File

@ -57,8 +57,8 @@ def task_3(data_size, host):
def task_4(data_size, host):
"""
task_4:
before upgrade: create collection, load, insert data without flush
after upgrade: get collection, load with multi replicas, search, insert data with flush, load with multi replicas and search
before upgrade: create collection, insert data, flush, and create index
after upgrade: get collection, load with multi replicas, search, insert data, load with multi replicas and search
"""
prefix = "task_4_"
connections.connect(host=host, port=19530, timeout=60)
@ -73,12 +73,13 @@ def task_5(data_size, host):
"""
task_5_:
before upgrade: create collection and insert data without flush
after upgrade: get collection, load with multi replicas, search, insert data with flush, load with multi replicas and search
after upgrade: get collection, create index, load with multi replicas, search, insert data with flush, load with multi replicas and search
"""
prefix = "task_5_"
connections.connect(host=host, port=19530, timeout=60)
col_list = get_collections(prefix, check=True)
assert len(col_list) == len(all_index_types)
create_index(prefix)
load_and_search(prefix, replicas=NUM_REPLICAS)
create_collections_and_insert_data(prefix, flush=True, count=data_size)
load_and_search(prefix, replicas=NUM_REPLICAS)

6
tests/python_client/deploy/scripts/action_before_reinstall.py
View File

@ -5,12 +5,13 @@ from utils import *
def task_1(data_size, host):
"""
task_1:
before reinstall: create collection and insert data, load and search
before reinstall: create collection create index and insert data, load and search
after reinstall: get collection, load, search, create index, load, and search
"""
prefix = "task_1_"
connections.connect(host=host, port=19530, timeout=60)
get_collections(prefix)
create_index(prefix)
load_and_search(prefix)
create_collections_and_insert_data(prefix,data_size)
load_and_search(prefix)
@ -19,12 +20,13 @@ def task_1(data_size, host):
def task_2(data_size, host):
"""
task_2:
before reinstall: create collection, insert data and create index,load and search
before reinstall: create collection, create index, insert data and create index,load and search
after reinstall: get collection, load, search, insert data, create index, load, and search
"""
prefix = "task_2_"
connections.connect(host=host, port=19530, timeout=60)
get_collections(prefix)
create_index(prefix)
load_and_search(prefix)
create_collections_and_insert_data(prefix, data_size)
create_index(prefix)

2
tests/python_client/deploy/scripts/action_before_upgrade.py
View File

@ -62,7 +62,7 @@ def task_5(data_size, host):
"""
task_5_:
before upgrade: create collection and insert data without flush
after upgrade: get collection, load with multi replicas, search, insert data with flush, load with multi replicas and search
after upgrade: get collection, create index, load with multi replicas, search, insert data with flush, load with multi replicas and search
"""
prefix = "task_5_"
connections.connect(host=host, port=19530, timeout=60)

30
tests/python_client/deploy/testcases/test_action_first_deployment.py
View File

@ -60,11 +60,10 @@ class TestActionFirstDeployment(TestDeployBase):
@pytest.mark.parametrize("is_compacted", ["is_compacted", "not_compacted"])
@pytest.mark.parametrize("is_deleted", ["is_deleted"])
@pytest.mark.parametrize("is_string_indexed", ["is_string_indexed", "not_string_indexed"])
@pytest.mark.parametrize("is_vector_indexed", ["is_vector_indexed", "not_vector_indexed"])
@pytest.mark.parametrize("segment_status", ["only_growing", "all"])
@pytest.mark.parametrize("index_type", ["HNSW", "BIN_IVF_FLAT"]) #"IVF_FLAT", "HNSW", "BIN_IVF_FLAT"
def test_task_all(self, index_type, is_compacted,
segment_status, is_vector_indexed, is_string_indexed, replica_number, is_deleted, data_size):
segment_status, is_string_indexed, replica_number, is_deleted, data_size):
"""
before reinstall: create collection and insert data, load and search
"""
@ -83,7 +82,7 @@ class TestActionFirstDeployment(TestDeployBase):
pytest.skip("skip test, not enough nodes")
log.info(f"collection name: {name}, replica_number: {replica_number}, is_compacted: {is_compacted},"
f"is_deleted: {is_deleted}, is_vector_indexed: {is_vector_indexed}, is_string_indexed: {is_string_indexed},"
f"is_deleted: {is_deleted}, is_string_indexed: {is_string_indexed},"
f"segment_status: {segment_status}, index_type: {index_type}")
is_binary = True if "BIN" in index_type else False
@ -101,6 +100,16 @@ class TestActionFirstDeployment(TestDeployBase):
# init collection and insert with small size data without flush to get growing segment
collection_w = self.init_collection_general(insert_data=True, is_binary=is_binary, nb=3000,
is_flush=False, is_index=True, name=name)[0]
# params for creating index
if is_binary:
default_index_field = ct.default_binary_vec_field_name
else:
default_index_field = ct.default_float_vec_field_name
# create index for vector
default_index_param = gen_index_param(index_type)
collection_w.create_index(default_index_field, default_index_param)
# load for growing segment
if replica_number >= 1:
try:
@ -143,18 +152,7 @@ class TestActionFirstDeployment(TestDeployBase):
# delete data for sealed segment and before index
delete_expr = f"{ct.default_int64_field_name} in {[i for i in range(10,20)]}"
if is_deleted == "is_deleted":
collection_w.delete(expr=delete_expr)
# params for creating index
if is_binary:
default_index_field = ct.default_binary_vec_field_name
else:
default_index_field = ct.default_float_vec_field_name
# create index for vector
if is_vector_indexed == "is_vector_indexed":
default_index_param = gen_index_param(index_type)
collection_w.create_index(default_index_field, default_index_param)
collection_w.delete(expr=delete_expr)
# create index for string
if is_string_indexed == "is_string_indexed":
default_string_index_params = {}
@ -162,7 +160,7 @@ class TestActionFirstDeployment(TestDeployBase):
collection_w.create_index(
default_string_field_name, default_string_index_params, index_name=default_string_index_name)
# delete data for sealed segment and afer index
# delete data for sealed segment and after index
delete_expr = f"{ct.default_int64_field_name} in {[i for i in range(20,30)]}"
if is_deleted == "is_deleted":
collection_w.delete(expr=delete_expr)

10
tests/python_client/deploy/testcases/test_action_second_deployment.py
View File

@ -64,21 +64,13 @@ class TestActionSecondDeployment(TestDeployBase):
else:
default_index_field = ct.default_float_vec_field_name
vector_index_type = "IVF_FLAT"
is_vector_indexed = False
is_string_indexed = False
indexed_fields = [index.field_name for index in collection_w.indexes]
binary_vector_index_types = [index.params["index_type"] for index in collection_w.indexes if index.field_name == type_field_map.get("BINARY_VECTOR", "")]
float_vector_index_types = [index.params["index_type"] for index in collection_w.indexes if index.field_name == type_field_map.get("FLOAT_VECTOR", "")]
string_index_types = [index.params["index_type"] for index in collection_w.indexes if index.field_name == type_field_map.get("VARCHAR", "")]
index_names = [index.index_name for index in collection_w.indexes] # used to drop index
vector_index_types = binary_vector_index_types + float_vector_index_types
if len(vector_index_types) > 0:
is_vector_indexed = True
vector_index_type = vector_index_types[0]
if len(string_index_types) > 0:
is_string_indexed = True
try:
replicas, _ = collection_w.get_replicas(enable_traceback=False)