test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-02 20:09:57 +08:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

[test] Update some index related cases (#19031)

Browse Source 
Signed-off-by: ThreadDao <yufen.zong@zilliz.com>

Signed-off-by: ThreadDao <yufen.zong@zilliz.com>
This commit is contained in:
ThreadDao 2022-09-06 09:03:14 +08:00 committed by GitHub
parent 2bafdf8c53
commit 495b214dd0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 47 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
tests/python_client/base/collection_wrapper.py
View File

@ -16,6 +16,7 @@ from common.common_func import param_info
TIMEOUT = 20 TIMEOUT = 20
INDEX_NAME = "_default_idx" INDEX_NAME = "_default_idx"
# keep small timeout for stability tests # keep small timeout for stability tests
# TIMEOUT = 5 # TIMEOUT = 5
@ -26,7 +27,8 @@ class ApiCollectionWrapper:
def __init__(self, active_trace=False): def __init__(self, active_trace=False):
self.active_trace = active_trace self.active_trace = active_trace
def init_collection(self, name, schema=None, using="default", shards_num=2, check_task=None, check_items=None, active_trace=False, **kwargs): def init_collection(self, name, schema=None, using="default", shards_num=2, check_task=None, check_items=None,
active_trace=False, **kwargs):
self.active_trace = active_trace self.active_trace = active_trace
consistency_level = kwargs.get("consistency_level", CONSISTENCY_STRONG) consistency_level = kwargs.get("consistency_level", CONSISTENCY_STRONG)
kwargs.update({"consistency_level": consistency_level}) kwargs.update({"consistency_level": consistency_level})
@ -92,7 +94,7 @@ class ApiCollectionWrapper:
def load(self, partition_names=None, replica_number=NaN, timeout=None, check_task=None, check_items=None, **kwargs): def load(self, partition_names=None, replica_number=NaN, timeout=None, check_task=None, check_items=None, **kwargs):
timeout = TIMEOUT if timeout is None else timeout timeout = TIMEOUT if timeout is None else timeout
replica_number = param_info.param_replica_num if replica_number is NaN else replica_number replica_number = param_info.param_replica_num if replica_number is NaN else replica_number
func_name = sys._getframe().f_code.co_name func_name = sys._getframe().f_code.co_name
res, check = api_request([self.collection.load, partition_names, replica_number, timeout], **kwargs) res, check = api_request([self.collection.load, partition_names, replica_number, timeout], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, check_result = ResponseChecker(res, func_name, check_task, check_items, check,
@ -233,8 +235,8 @@ class ApiCollectionWrapper:
@trace() @trace()
def create_index(self, field_name, index_params, index_name=None, check_task=None, check_items=None, **kwargs): def create_index(self, field_name, index_params, index_name=None, check_task=None, check_items=None, **kwargs):
timeout = kwargs.get("timeout", TIMEOUT * 2) timeout = kwargs.get("timeout", TIMEOUT * 2)
index_name = INDEX_NAME if index_name is None else index_name index_name = INDEX_NAME if index_name is None else index_name
index_name = kwargs.get("index_name", index_name) index_name = kwargs.get("index_name", index_name)
kwargs.update({"timeout": timeout, "index_name": index_name}) kwargs.update({"timeout": timeout, "index_name": index_name})
func_name = sys._getframe().f_code.co_name func_name = sys._getframe().f_code.co_name
@ -245,8 +247,8 @@ class ApiCollectionWrapper:
@trace() @trace()
def has_index(self, index_name=None, check_task=None, check_items=None, **kwargs): def has_index(self, index_name=None, check_task=None, check_items=None, **kwargs):
index_name = INDEX_NAME if index_name is None else index_name index_name = INDEX_NAME if index_name is None else index_name
index_name = kwargs.get("index_name", index_name) index_name = kwargs.get("index_name", index_name)
kwargs.update({"index_name": index_name}) kwargs.update({"index_name": index_name})
func_name = sys._getframe().f_code.co_name func_name = sys._getframe().f_code.co_name
@ -257,10 +259,10 @@ class ApiCollectionWrapper:
@trace() @trace()
def drop_index(self, index_name=None, check_task=None, check_items=None, **kwargs): def drop_index(self, index_name=None, check_task=None, check_items=None, **kwargs):
timeout = kwargs.get("timeout", TIMEOUT) timeout = kwargs.get("timeout", TIMEOUT)
index_name = INDEX_NAME if index_name is None else index_name index_name = INDEX_NAME if index_name is None else index_name
index_name = kwargs.get("index_name", index_name) index_name = kwargs.get("index_name", index_name)
kwargs.update({"timeout": timeout, "index_name": index_name}) kwargs.update({"timeout": timeout, "index_name": index_name})
func_name = sys._getframe().f_code.co_name func_name = sys._getframe().f_code.co_name
res, check = api_request([self.collection.drop_index], **kwargs) res, check = api_request([self.collection.drop_index], **kwargs)
check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run() check_result = ResponseChecker(res, func_name, check_task, check_items, check, **kwargs).run()

18
tests/python_client/base/index_wrapper.py
View File

@ -5,20 +5,20 @@ sys.path.append("..")
from check.func_check import ResponseChecker from check.func_check import ResponseChecker
from utils.api_request import api_request from utils.api_request import api_request
TIMEOUT = 20 TIMEOUT = 20
INDEX_NAME = "_default_idx" INDEX_NAME = "_default_idx"
class ApiIndexWrapper: class ApiIndexWrapper:
index = None index = None
def init_index(self, collection, field_name, index_params, index_name=None, check_task=None, check_items=None, **kwargs): def init_index(self, collection, field_name, index_params, index_name=None, check_task=None, check_items=None,
**kwargs):
timeout = kwargs.get("timeout", TIMEOUT * 2) timeout = kwargs.get("timeout", TIMEOUT * 2)
index_name = INDEX_NAME if index_name is None else index_name index_name = INDEX_NAME if index_name is None else index_name
index_name = kwargs.get("index_name", index_name) index_name = kwargs.get("index_name", index_name)
kwargs.update({"timeout": timeout, "index_name": index_name}) kwargs.update({"timeout": timeout, "index_name": index_name})
""" In order to distinguish the same name of index """ """ In order to distinguish the same name of index """
func_name = sys._getframe().f_code.co_name func_name = sys._getframe().f_code.co_name
res, is_succ = api_request([Index, collection, field_name, index_params], **kwargs) res, is_succ = api_request([Index, collection, field_name, index_params], **kwargs)
@ -28,9 +28,9 @@ class ApiIndexWrapper:
index_params=index_params, **kwargs).run() index_params=index_params, **kwargs).run()
return res, check_result return res, check_result
def drop(self, index_name=None ,check_task=None, check_items=None, **kwargs): def drop(self, index_name=None, check_task=None, check_items=None, **kwargs):
timeout = kwargs.get("timeout", TIMEOUT) timeout = kwargs.get("timeout", TIMEOUT)
index_name = INDEX_NAME if index_name is None else index_name index_name = INDEX_NAME if index_name is None else index_name
index_name = kwargs.get("index_name", index_name) index_name = kwargs.get("index_name", index_name)
kwargs.update({"timeout": timeout, "index_name": index_name}) kwargs.update({"timeout": timeout, "index_name": index_name})
@ -49,4 +49,4 @@ class ApiIndexWrapper:
@property @property
def field_name(self): def field_name(self):
return self.index.field_name return self.index.field_name

21
tests/python_client/testcases/test_index.py
View File

@ -987,17 +987,17 @@ class TestNewIndexBase(TestcaseBase):
""" """
collection_w = self.init_collection_wrap(cf.gen_unique_str(prefix)) collection_w = self.init_collection_wrap(cf.gen_unique_str(prefix))
nums = 20 nums = 20
tmp_nb = 5000 tmp_nb = 5000
for i in range (nums) : for i in range(nums):
df = cf.gen_default_dataframe_data(nb=tmp_nb, start=i * tmp_nb) df = cf.gen_default_dataframe_data(nb=tmp_nb, start=i * tmp_nb)
insert_res, _ = collection_w.insert(df) insert_res, _ = collection_w.insert(df)
assert collection_w.num_entities ==(i+1) * tmp_nb assert collection_w.num_entities == (i + 1) * tmp_nb
collection_w.create_index(ct.default_float_vec_field_name, default_index_params) collection_w.create_index(ct.default_float_vec_field_name, default_index_params)
collection_w.load() collection_w.load()
vectors = [[random.random() for _ in range(default_dim)] for _ in range(default_nq)] vectors = [[random.random() for _ in range(default_dim)] for _ in range(default_nq)]
search_res, _ = collection_w.search(vectors, default_search_field, default_search_params, default_limit) search_res, _ = collection_w.search(vectors, default_search_field, default_search_params, default_limit)
assert len(search_res[0]) == ct.default_limit assert len(search_res[0]) == ct.default_limit
class TestNewIndexBinary(TestcaseBase): class TestNewIndexBinary(TestcaseBase):
@ -1229,9 +1229,18 @@ class TestNewIndexAsync(TestcaseBase):
collection_w.insert(data=data) collection_w.insert(data=data)
res, _ = collection_w.create_index(ct.default_float_vec_field_name, default_index_params, res, _ = collection_w.create_index(ct.default_float_vec_field_name, default_index_params,
index_name=ct.default_index_name, _async=_async) index_name=ct.default_index_name, _async=_async)
# load and search
collection_w.load()
vectors_s = [[random.random() for _ in range(ct.default_dim)] for _ in range(ct.default_nq)]
search_res, _ = collection_w.search(vectors_s[:ct.default_nq], ct.default_float_vec_field_name,
ct.default_search_params, ct.default_limit)
assert len(search_res) == ct.default_nq
assert len(search_res[0]) == ct.default_limit
if _async: if _async:
res.done() res.done()
assert len(collection_w.indexes) == 1 assert collection_w.indexes[0].params == default_index_params
collection_w.drop_index(index_name=ct.default_index_name) collection_w.drop_index(index_name=ct.default_index_name)
assert len(collection_w.indexes) == 0 assert len(collection_w.indexes) == 0

16
tests/python_client/testcases/test_utility.py
View File

@ -1725,6 +1725,14 @@ class TestUtilityAdvanced(TestcaseBase):
@pytest.mark.tags(CaseLabel.L1) @pytest.mark.tags(CaseLabel.L1)
def test_handoff_query_search(self): def test_handoff_query_search(self):
"""
target: test query search after handoff
method: 1.load collection
2.insert, query and search
3.flush collection and triggere handoff
4. search with handoff indexed segments
expected: Search ids before and after handoff are different, because search from growing and search from index
"""
collection_w = self.init_collection_wrap(name=cf.gen_unique_str(prefix), shards_num=1) collection_w = self.init_collection_wrap(name=cf.gen_unique_str(prefix), shards_num=1)
collection_w.create_index(default_field_name, default_index_params) collection_w.create_index(default_field_name, default_index_params)
collection_w.load() collection_w.load()
@ -1743,7 +1751,7 @@ class TestUtilityAdvanced(TestcaseBase):
start = time.time() start = time.time()
while True: while True:
time.sleep(0.5) time.sleep(2)
segment_infos, _ = self.utility_wrap.get_query_segment_info(collection_w.name) segment_infos, _ = self.utility_wrap.get_query_segment_info(collection_w.name)
# handoff done # handoff done
if len(segment_infos) == 1 and segment_infos[0].state == SegmentState.Sealed: if len(segment_infos) == 1 and segment_infos[0].state == SegmentState.Sealed:
@ -1758,9 +1766,9 @@ class TestUtilityAdvanced(TestcaseBase):
ct.default_float_vec_field_name, ct.default_float_vec_field_name,
ct.default_search_params, ct.default_limit) ct.default_search_params, ct.default_limit)
# the ids between twice search is different because of index building # the ids between twice search is different because of index building
log.debug(search_res_before[0].ids) # log.debug(search_res_before[0].ids)
log.debug(search_res_after[0].ids) # log.debug(search_res_after[0].ids)
# assert search_res_before[0].ids != search_res_after[0].ids assert search_res_before[0].ids != search_res_after[0].ids
# assert search result includes the nq-vector before or after handoff # assert search result includes the nq-vector before or after handoff
assert search_res_after[0].ids[0] == 0 assert search_res_after[0].ids[0] == 0