diff --git a/tests/milvus-java-test/src/main/java/com/TestSearchByIds.java b/tests/milvus-java-test/src/main/java/com/TestSearchByIds.java index 94a8cb2498..0391268dd2 100644 --- a/tests/milvus-java-test/src/main/java/com/TestSearchByIds.java +++ b/tests/milvus-java-test/src/main/java/com/TestSearchByIds.java @@ -14,6 +14,7 @@ public class TestSearchByIds { int n_list = 1024; int default_n_list = 16384; int nb = 10000; + int small_nb = 10; int n_probe = 20; int top_k = 10; int nq = 5; @@ -22,6 +23,7 @@ public class TestSearchByIds { IndexType defaultIndexType = IndexType.FLAT; List default_ids = Utils.toListIds(1111); List> vectors = Utils.genVectors(nb, dimension, true); + List> small_vectors = Utils.genVectors(small_nb, dimension, true); List vectorsBinary = Utils.genBinaryVectors(nb, dimension); String indexParam = Utils.setIndexParam(n_list); public String searchParamStr = Utils.setSearchParam(n_probe); @@ -62,6 +64,22 @@ public class TestSearchByIds { List> res_search = client.searchByIds(searchParam).getQueryResultsList(); assert (client.searchByIds(searchParam).getResponse().ok()); Assert.assertEquals(res_search.get(0).size(), 0); + } + + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_search_count_lt_top_k(MilvusClient client, String collectionName) { + int top_k = 100; + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(small_vectors).build(); + InsertResponse res_insert = client.insert(insertParam); + client.flush(collectionName); + SearchByIdsParam searchParam = new SearchByIdsParam.Builder(collectionName) + .withParamsInJson(searchParamStr) + .withTopK(top_k) + .withIDs(Utils.toListIds(res_insert.getVectorIds().get(0))) + .build(); + List> res_search = client.searchByIds(searchParam).getQueryResultsList(); + // reason: "Failed to query by id in collection L2_FmVKbqSZaN, result doesn\'t match id count" + assert (!client.searchByIds(searchParam).getResponse().ok()); // Assert.assertEquals(res_search.size(), default_ids.size()); // Assert.assertEquals(res_search.get(0).get(0).getVectorId(), -1); } diff --git a/tests/milvus-java-test/src/main/java/com/TestSearchVectors.java b/tests/milvus-java-test/src/main/java/com/TestSearchVectors.java index 750296c094..1e4c2d5f1e 100644 --- a/tests/milvus-java-test/src/main/java/com/TestSearchVectors.java +++ b/tests/milvus-java-test/src/main/java/com/TestSearchVectors.java @@ -15,6 +15,7 @@ public class TestSearchVectors { int n_list = 1024; int default_n_list = 16384; int nb = 10000; + int small_nb = 10; int n_probe = 20; int top_k = 10; int nq = 5; @@ -22,6 +23,7 @@ public class TestSearchVectors { IndexType indexType = IndexType.IVF_SQ8; IndexType defaultIndexType = IndexType.FLAT; List> vectors = Utils.genVectors(nb, dimension, true); + List> small_vectors = Utils.genVectors(small_nb, dimension, true); List vectorsBinary = Utils.genBinaryVectors(nb, dimension); List> queryVectors = vectors.subList(0, nq); List queryVectorsBinary = vectorsBinary.subList(0, nq); @@ -190,6 +192,21 @@ public class TestSearchVectors { assert (!res_search.getResponse().ok()); } + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) + public void test_search_count_lt_top_k(MilvusClient client, String collectionName) { + int top_k_new = 100; + InsertParam insertParam = new InsertParam.Builder(collectionName).withFloatVectors(small_vectors).build(); + client.insert(insertParam); + client.flush(collectionName); + SearchParam searchParam = new SearchParam.Builder(collectionName) + .withFloatVectors(queryVectors) + .withParamsInJson(searchParamStr) + .withTopK(top_k_new).build(); + List> res_search = client.search(searchParam).getQueryResultsList(); + Assert.assertEquals(res_search.size(), nq); + Assert.assertEquals(res_search.get(0).size(), small_vectors.size()); + } + @Test(dataProvider = "Collection", dataProviderClass = MainClass.class) public void test_search_invalid_top_k(MilvusClient client, String collectionName) { int top_k_new = 0; diff --git a/tests/milvus_benchmark/.gitignore b/tests/milvus_benchmark/.gitignore index 70af07bba8..3688f39a9a 100644 --- a/tests/milvus_benchmark/.gitignore +++ b/tests/milvus_benchmark/.gitignore @@ -5,4 +5,6 @@ logs/ *idmap*.txt __pycache__/ venv -.idea \ No newline at end of file +.idea + +suites/ diff --git a/tests/milvus_benchmark/ci/jenkinsfile/deploy_test.groovy b/tests/milvus_benchmark/ci/jenkinsfile/deploy_test.groovy index f85ec8364b..f31e2215b0 100644 --- a/tests/milvus_benchmark/ci/jenkinsfile/deploy_test.groovy +++ b/tests/milvus_benchmark/ci/jenkinsfile/deploy_test.groovy @@ -1,17 +1,18 @@ -timeout(time: 1440, unit: 'MINUTES') { +timeout(time: 4000, unit: 'MINUTES') { try { dir ("milvus-helm") { - sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' - sh 'helm repo update' + // sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' + // sh 'helm repo update' checkout([$class: 'GitSCM', branches: [[name: "${HELM_BRANCH}"]], userRemoteConfigs: [[url: "${HELM_URL}", name: 'origin', refspec: "+refs/heads/${HELM_BRANCH}:refs/remotes/origin/${HELM_BRANCH}"]]]) } dir ("milvus_benchmark") { print "Git clone url: ${TEST_URL}:${TEST_BRANCH}" checkout([$class: 'GitSCM', branches: [[name: "${TEST_BRANCH}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "${TEST_URL}", name: 'origin', refspec: "+refs/heads/${TEST_BRANCH}:refs/remotes/origin/${TEST_BRANCH}"]]]) print "Install requirements" - sh "python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com" + // sh "python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com" + sh "python3 -m pip install -r requirements.txt" sh "python3 -m pip install git+${TEST_LIB_URL}" - sh "python3 main.py --hostname=${params.SERVER_HOST} --image-tag=${IMAGE_TAG} --image-type=${params.IMAGE_TYPE} --suite=suites/${params.SUITE}" + sh "python3 main.py --image-version=${params.IMAGE_VERSION} --schedule-conf=scheduler/${params.CONFIG_FILE}" } } catch (exc) { echo 'Deploy Test Failed !' diff --git a/tests/milvus_benchmark/ci/jenkinsfile/publishDailyImages.groovy b/tests/milvus_benchmark/ci/jenkinsfile/publishDailyImages.groovy new file mode 100644 index 0000000000..07214a95f8 --- /dev/null +++ b/tests/milvus_benchmark/ci/jenkinsfile/publishDailyImages.groovy @@ -0,0 +1,46 @@ +timeout(time: 15, unit: 'MINUTES') { + def imageName = "milvus/engine:${DOCKER_VERSION}" + def remoteImageName = "milvusdb/daily-build:${REMOTE_DOCKER_VERSION}" + def localDockerRegistryImage = "${params.LOCAL_DOKCER_REGISTRY_URL}/${imageName}" + def remoteDockerRegistryImage = "${params.REMOTE_DOKCER_REGISTRY_URL}/${remoteImageName}" + try { + deleteImages("${localDockerRegistryImage}", true) + + def pullSourceImageStatus = sh(returnStatus: true, script: "docker pull ${localDockerRegistryImage}") + + if (pullSourceImageStatus == 0) { + def renameImageStatus = sh(returnStatus: true, script: "docker tag ${localDockerRegistryImage} ${remoteImageName} && docker rmi ${localDockerRegistryImage}") + def sourceImage = docker.image("${remoteImageName}") + docker.withRegistry("https://${params.REMOTE_DOKCER_REGISTRY_URL}", "${params.REMOTE_DOCKER_CREDENTIALS_ID}") { + sourceImage.push() + sourceImage.push("${REMOTE_DOCKER_LATEST_VERSION}") + } + } else { + echo "\"${localDockerRegistryImage}\" image does not exist !" + } + } catch (exc) { + throw exc + } finally { + deleteImages("${localDockerRegistryImage}", true) + deleteImages("${remoteDockerRegistryImage}", true) + } +} + +boolean deleteImages(String imageName, boolean force) { + def imageNameStr = imageName.trim() + def isExistImage = sh(returnStatus: true, script: "docker inspect --type=image ${imageNameStr} 2>&1 > /dev/null") + if (isExistImage == 0) { + def deleteImageStatus = 0 + if (force) { + def imageID = sh(returnStdout: true, script: "docker inspect --type=image --format \"{{.ID}}\" ${imageNameStr}") + deleteImageStatus = sh(returnStatus: true, script: "docker rmi -f ${imageID}") + } else { + deleteImageStatus = sh(returnStatus: true, script: "docker rmi ${imageNameStr}") + } + + if (deleteImageStatus != 0) { + return false + } + } + return true +} diff --git a/tests/milvus_benchmark/ci/main_jenkinsfile b/tests/milvus_benchmark/ci/main_jenkinsfile index 08b987eb4a..6431b2f09c 100644 --- a/tests/milvus_benchmark/ci/main_jenkinsfile +++ b/tests/milvus_benchmark/ci/main_jenkinsfile @@ -6,17 +6,14 @@ pipeline { } parameters{ - choice choices: ['gpu', 'cpu'], description: 'cpu or gpu version', name: 'IMAGE_TYPE' string defaultValue: 'master', description: 'server image version', name: 'IMAGE_VERSION', trim: true - choice choices: ['poseidon', 'eros', 'apollo', 'athena'], description: 'server host', name: 'SERVER_HOST' - string defaultValue: 'gpu_search_performance_sift1m.yaml', description: 'test suite config yaml', name: 'SUITE', trim: true + string defaultValue: '080_data.json', description: 'test suite config yaml', name: 'CONFIG_FILE', trim: true string defaultValue: '09509e53-9125-4f5d-9ce8-42855987ad67', description: 'git credentials', name: 'GIT_USER', trim: true } environment { - IMAGE_TAG = "${params.IMAGE_VERSION}-${params.IMAGE_TYPE}-ubuntu18.04-release" HELM_URL = "https://github.com/milvus-io/milvus-helm.git" - HELM_BRANCH = "0.6.0" + HELM_BRANCH = "master" TEST_URL = "git@192.168.1.105:Test/milvus_benchmark.git" TEST_BRANCH = "master" TEST_LIB_URL = "http://192.168.1.105:6060/Test/milvus_metrics.git" @@ -37,8 +34,8 @@ pipeline { componet: test spec: containers: - - name: milvus-testframework - image: registry.zilliz.com/milvus/milvus-test:v0.2 + - name: milvus-test-env + image: registry.zilliz.com/milvus/milvus-test-env:v0.2 command: - cat tty: true @@ -79,10 +76,25 @@ pipeline { } stages { + stage("Publish Daily Docker images") { + steps { + container('milvus-test-env') { + script { + boolean isNightlyTest = isTimeTriggeredBuild() + if (isNightlyTest) { + build job: 'milvus-publish-daily-docker', wait: false + } else { + echo "Skip publish daily docker images ..." + } + } + } + } + } + stage("Deploy Test") { steps { gitlabCommitStatus(name: 'Deploy Test') { - container('milvus-testframework') { + container('milvus-test-env') { script { print "In Deploy Test Stage" load "${env.WORKSPACE}/ci/jenkinsfile/deploy_test.groovy" @@ -95,7 +107,7 @@ pipeline { stage ("Cleanup Env") { steps { gitlabCommitStatus(name: 'Cleanup Env') { - container('milvus-testframework') { + container('milvus-test-env') { script { load "${env.WORKSPACE}/ci/jenkinsfile/cleanup.groovy" } @@ -106,7 +118,7 @@ pipeline { } post { always { - container('milvus-testframework') { + container('milvus-test-env') { script { load "${env.WORKSPACE}/ci/jenkinsfile/cleanup.groovy" } @@ -131,3 +143,10 @@ pipeline { } } } + +boolean isTimeTriggeredBuild() { + if (currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause').size() != 0) { + return true + } + return false +} diff --git a/tests/milvus_benchmark/ci/publish_jenkinsfile b/tests/milvus_benchmark/ci/publish_jenkinsfile new file mode 100644 index 0000000000..353ed82f9d --- /dev/null +++ b/tests/milvus_benchmark/ci/publish_jenkinsfile @@ -0,0 +1,103 @@ +pipeline { + agent none + + options { + timestamps() + } + + parameters{ + string defaultValue: 'registry.zilliz.com', description: 'Local Docker registry URL', name: 'LOCAL_DOKCER_REGISTRY_URL', trim: true + string defaultValue: 'registry-1.docker.io', description: 'Remote Docker registry URL', name: 'REMOTE_DOKCER_REGISTRY_URL', trim: true + string defaultValue: 'milvus-docker-access-token', description: 'Remote Docker credentials id', name: 'REMOTE_DOCKER_CREDENTIALS_ID', trim: true + } + + environment { + DAILY_BUILD_VERSION = VersionNumber([ + versionNumberString : '${BUILD_DATE_FORMATTED, "yyyyMMdd"}' + ]); + } + + stages { + stage('Push Daily Docker Images') { + matrix { + agent none + axes { + axis { + name 'OS_NAME' + values 'ubuntu18.04', 'centos7' + } + + axis { + name 'CPU_ARCH' + values 'amd64' + } + + axis { + name 'BINARY_VERSION' + values 'gpu', 'cpu' + } + } + + stages { + stage("Publish Docker Images") { + environment { + DOCKER_VERSION = "master-${BINARY_VERSION}-${OS_NAME}-release" + REMOTE_DOCKER_VERSION = "${OS_NAME}-${BINARY_VERSION}-${DAILY_BUILD_VERSION}" + REMOTE_DOCKER_LATEST_VERSION = "${OS_NAME}-${BINARY_VERSION}-latest" + } + + agent { + kubernetes { + label "${OS_NAME}-${BINARY_VERSION}-publish-${env.BUILD_NUMBER}" + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: publish + componet: docker +spec: + containers: + - name: publish-images + image: registry.zilliz.com/library/docker:v1.0.0 + securityContext: + privileged: true + command: + - cat + tty: true + resources: + limits: + memory: "4Gi" + cpu: "1.0" + requests: + memory: "2Gi" + cpu: "0.5" + volumeMounts: + - name: docker-sock + mountPath: /var/run/docker.sock + volumes: + - name: docker-sock + hostPath: + path: /var/run/docker.sock + """ + } + } + + stages { + stage('Publish') { + steps { + container('publish-images') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/publishDailyImages.groovy" + } + } + } + } + } + } + } + } + } + } +} diff --git a/tests/milvus_benchmark/ci/scripts/yaml_processor.py b/tests/milvus_benchmark/ci/scripts/yaml_processor.py new file mode 100755 index 0000000000..0e6d7dbbf4 --- /dev/null +++ b/tests/milvus_benchmark/ci/scripts/yaml_processor.py @@ -0,0 +1,536 @@ +#!/usr/bin/env python3 + +import sys +import argparse +from argparse import Namespace +import os, shutil +import getopt +from ruamel.yaml import YAML, yaml_object +from ruamel.yaml.comments import CommentedSeq, CommentedMap +from ruamel.yaml.tokens import CommentToken + +## +yaml = YAML(typ="rt") +## format yaml file +yaml.indent(mapping=2, sequence=4, offset=2) + + +############################################ +# Comment operation +# +############################################ +def _extract_comment(_comment): + """ + remove '#' at start of comment + """ + # if _comment is empty, do nothing + if not _comment: + return _comment + + # str_ = _comment.lstrip(" ") + str_ = _comment.strip() + str_ = str_.lstrip("#") + + return str_ + + +def _add_eol_comment(element, *args, **kwargs): + """ + add_eol_comment + args --> (comment, key) + """ + if element is None or \ + (not isinstance(element, CommentedMap) and + not isinstance(element, CommentedSeq)) or \ + args[0] is None or \ + len(args[0]) == 0: + return + + comment = args[0] + # comment is empty, do nothing + if not comment: + return + + key = args[1] + try: + element.yaml_add_eol_comment(*args, **kwargs) + except Exception: + element.ca.items.pop(key, None) + element.yaml_add_eol_comment(*args, **kwargs) + + +def _map_comment(_element, _key): + origin_comment = "" + token = _element.ca.items.get(_key, None) + if token is not None: + try: + origin_comment = token[2].value + except Exception: + try: + # comment is below element, add profix "#\n" + col = _element.lc.col + 2 + space_list = [" " for i in range(col)] + space_str = "".join(space_list) + + origin_comment = "\n" + "".join([space_str + t.value for t in token[3]]) + except Exception: + pass + + return origin_comment + + +def _seq_comment(_element, _index): + # get target comment + _comment = "" + token = _element.ca.items.get(_index, None) + if token is not None: + _comment = token[0].value + + return _comment + + +def _start_comment(_element): + _comment = "" + cmt = _element.ca.comment + try: + _comment = cmt[1][0].value + except Exception: + pass + + return _comment + + +def _comment_counter(_comment): + """ + + counter comment tips and split into list + """ + + x = lambda l: l.strip().strip("#").strip() + + _counter = [] + if _comment.startswith("\n"): + _counter.append("") + _counter.append(x(_comment[1:])) + + return _counter + elif _comment.startswith("#\n"): + _counter.append("") + _counter.append(x(_comment[2:])) + else: + index = _comment.find("\n") + _counter.append(x(_comment[:index])) + _counter.append(x(_comment[index + 1:])) + + return _counter + + +def _obtain_comment(_m_comment, _t_comment): + if not _m_comment or not _t_comment: + return _m_comment or _t_comment + + _m_counter = _comment_counter(_m_comment) + _t_counter = _comment_counter(_t_comment) + + if not _m_counter[0] and not _t_counter[1]: + comment = _t_comment + _m_comment + elif not _m_counter[1] and not _t_counter[0]: + comment = _m_comment + _t_comment + elif _t_counter[0] and _t_counter[1]: + comment = _t_comment + elif not _t_counter[0] and not _t_counter[1]: + comment = _m_comment + elif not _m_counter[0] and not _m_counter[1]: + comment = _t_comment + else: + if _t_counter[0]: + comment = _m_comment.replace(_m_counter[0], _t_counter[0], 1) + else: + comment = _m_comment.replace(_m_counter[1], _t_counter[1], 1) + + i = comment.find("\n\n") + while i >= 0: + comment = comment.replace("\n\n\n", "\n\n", 1) + i = comment.find("\n\n\n") + + return comment + + +############################################ +# Utils +# +############################################ +def _get_update_par(_args): + _dict = _args.__dict__ + + # file path + _in_file = _dict.get("f", None) or _dict.get("file", None) + # tips + _tips = _dict.get('tips', None) or "Input \"-h\" for more information" + # update + _u = _dict.get("u", None) or _dict.get("update", None) + # apppend + _a = _dict.get('a', None) or _dict.get('append', None) + # out stream group + _i = _dict.get("i", None) or _dict.get("inplace", None) + _o = _dict.get("o", None) or _dict.get("out_file", None) + + return _in_file, _u, _a, _i, _o, _tips + + +############################################ +# Element operation +# +############################################ +def update_map_element(element, key, value, comment, _type): + """ + element: + key: + value: + comment: + _type: value type. + """ + if element is None or not isinstance(element, CommentedMap): + print("Only key-value update support") + sys.exit(1) + + origin_comment = _map_comment(element, key) + + sub_element = element.get(key, None) + if isinstance(sub_element, CommentedMap) or isinstance(sub_element, CommentedSeq): + print("Only support update a single value") + + element.update({key: value}) + + comment = _obtain_comment(origin_comment, comment) + _add_eol_comment(element, _extract_comment(comment), key) + + +def update_seq_element(element, value, comment, _type): + if element is None or not isinstance(element, CommentedSeq): + print("Param `-a` only use to append yaml list") + sys.exit(1) + element.append(str(value)) + + comment = _obtain_comment("", comment) + _add_eol_comment(element, _extract_comment(comment), len(element) - 1) + + +def run_update(code, keys, value, comment, _app): + key_list = keys.split(".") + + space_str = ":\n " + key_str = "{}".format(key_list[0]) + for key in key_list[1:]: + key_str = key_str + space_str + key + space_str = space_str + " " + if not _app: + yaml_str = """{}: {}""".format(key_str, value) + else: + yaml_str = "{}{}- {}".format(key_str, space_str, value) + + if comment: + yaml_str = "{} # {}".format(yaml_str, comment) + + mcode = yaml.load(yaml_str) + + _merge(code, mcode) + + +def _update(code, _update, _app, _tips): + if not _update: + return code + + _update_list = [l.strip() for l in _update.split(",")] + for l in _update_list: + try: + variant, comment = l.split("#") + except ValueError: + variant = l + comment = None + + try: + keys, value = variant.split("=") + run_update(code, keys, value, comment, _app) + except ValueError: + print("Invalid format. print command \"--help\" get more info.") + sys.exit(1) + + return code + + +def _backup(in_file_p): + backup_p = in_file_p + ".bak" + + if os.path.exists(backup_p): + os.remove(backup_p) + + if not os.path.exists(in_file_p): + print("File {} not exists.".format(in_file_p)) + sys.exit(1) + + shutil.copyfile(in_file_p, backup_p) # 复制文件 + + +def _recovery(in_file_p): + backup_p = in_file_p + ".bak" + + if not os.path.exists(in_file_p): + print("File {} not exists.".format(in_file_p)) + sys.exit(1) + elif not os.path.exists(backup_p): + print("Backup file not exists") + sys.exit(0) + + os.remove(in_file_p) + + os.rename(backup_p, in_file_p) + + +# master merge target +def _merge(master, target): + if type(master) != type(target): + print("yaml format not match:\n") + yaml.dump(master, sys.stdout) + print("\n&&\n") + yaml.dump(target, sys.stdout) + + sys.exit(1) + + ## item is a sequence + if isinstance(target, CommentedSeq): + for index in range(len(target)): + # get target comment + target_comment = _seq_comment(target, index) + + master_index = len(master) + + target_item = target[index] + + if isinstance(target_item, CommentedMap): + merge_flag = False + for idx in range(len(master)): + if isinstance(master[idx], CommentedMap): + if master[idx].keys() == target_item.keys(): + _merge(master[idx], target_item) + # nonlocal merge_flag + master_index = idx + merge_flag = True + break + + if merge_flag is False: + master.append(target_item) + elif target_item not in master: + master.append(target[index]) + else: + # merge(master[index], target[index]) + pass + + # # remove enter signal in previous item + previous_comment = _seq_comment(master, master_index - 1) + _add_eol_comment(master, _extract_comment(previous_comment), master_index - 1) + + origin_comment = _seq_comment(master, master_index) + comment = _obtain_comment(origin_comment, target_comment) + if len(comment) > 0: + _add_eol_comment(master, _extract_comment(comment) + "\n\n", len(master) - 1) + + ## item is a map + elif isinstance(target, CommentedMap): + for item in target: + if item == "flag": + print("") + origin_comment = _map_comment(master, item) + target_comment = _map_comment(target, item) + + # get origin start comment + origin_start_comment = _start_comment(master) + + # get target start comment + target_start_comment = _start_comment(target) + + m = master.get(item, default=None) + if m is None or \ + (not (isinstance(m, CommentedMap) or + isinstance(m, CommentedSeq))): + master.update({item: target[item]}) + + else: + _merge(master[item], target[item]) + + comment = _obtain_comment(origin_comment, target_comment) + if len(comment) > 0: + _add_eol_comment(master, _extract_comment(comment), item) + + start_comment = _obtain_comment(origin_start_comment, target_start_comment) + if len(start_comment) > 0: + master.yaml_set_start_comment(_extract_comment(start_comment)) + + +def _save(_code, _file): + with open(_file, 'w') as wf: + yaml.dump(_code, wf) + + +def _load(_file): + with open(_file, 'r') as rf: + code = yaml.load(rf) + return code + + +############################################ +# sub parser process operation +# +############################################ +def merge_yaml(_args): + _dict = _args.__dict__ + + _m_file = _dict.get("merge_file", None) + _in_file, _u, _a, _i, _o, _tips = _get_update_par(_args) + + if not (_in_file and _m_file): + print(_tips) + sys.exit(1) + + code = _load(_in_file) + mcode = _load(_m_file) + + _merge(code, mcode) + + _update(code, _u, _a, _tips) + + if _i: + _backup(_in_file) + _save(code, _in_file) + elif _o: + _save(code, _o) + else: + print(_tips) + sys.exit(1) + + +def update_yaml(_args): + _in_file, _u, _a, _i, _o, _tips = _get_update_par(_args) + + if not _in_file or not _u: + print(_tips) + sys.exit(1) + + code = _load(_in_file) + + if _i and _o: + print(_tips) + sys.exit(1) + + _update(code, _u, _a, _tips) + + if _i: + _backup(_in_file) + _save(code, _in_file) + elif _o: + _save(code, _o) + + +def reset(_args): + _dict = _args.__dict__ + _f = _dict.get('f', None) or _dict.get('file', None) + + if _f: + _recovery(_f) + else: + _t = _dict.get('tips', None) or "Input \"-h\" for more information" + print(_t) + + +############################################ +# Cli operation +# +############################################ +def _set_merge_parser(_parsers): + """ + config merge parser + """ + + merge_parser = _parsers.add_parser("merge", help="merge with another yaml file") + + _set_merge_parser_arg(merge_parser) + _set_update_parser_arg(merge_parser) + + merge_parser.set_defaults( + function=merge_yaml, + tips=merge_parser.format_help() + ) + + +def _set_merge_parser_arg(_parser): + """ + config parser argument for merging + """ + + _parser.add_argument("-m", "--merge-file", help="indicate merge yaml file") + + +def _set_update_parser(_parsers): + """ + config merge parser + """ + + update_parser = _parsers.add_parser("update", help="update with another yaml file") + _set_update_parser_arg(update_parser) + + update_parser.set_defaults( + function=update_yaml, + tips=update_parser.format_help() + ) + + +def _set_update_parser_arg(_parser): + """ + config parser argument for updating + """ + + _parser.add_argument("-f", "--file", help="source yaml file") + _parser.add_argument('-u', '--update', help="update with args, instance as \"a.b.c=d# d comment\"") + _parser.add_argument('-a', '--append', action="store_true", help="append to a seq") + + group = _parser.add_mutually_exclusive_group() + group.add_argument("-o", "--out-file", help="indicate output yaml file") + group.add_argument("-i", "--inplace", action="store_true", help="indicate whether result store in origin file") + + +def _set_reset_parser(_parsers): + """ + config merge parser + """ + + reset_parser = _parsers.add_parser("reset", help="reset yaml file") + + # indicate yaml file + reset_parser.add_argument('-f', '--file', help="indicate input yaml file") + + reset_parser.set_defaults( + function=reset, + tips=reset_parser.format_help() + ) + + +def main(): + parser = argparse.ArgumentParser() + sub_parsers = parser.add_subparsers() + + # set merge command + _set_merge_parser(sub_parsers) + + # set update command + _set_update_parser(sub_parsers) + + # set reset command + _set_reset_parser(sub_parsers) + + # parse argument and run func + args = parser.parse_args() + args.function(args) + + +if __name__ == '__main__': + main() diff --git a/tests/milvus_benchmark/client.py b/tests/milvus_benchmark/client.py index 62e3263ecd..b6670bb6bd 100644 --- a/tests/milvus_benchmark/client.py +++ b/tests/milvus_benchmark/client.py @@ -18,8 +18,11 @@ INDEX_MAP = { "ivf_sq8": IndexType.IVF_SQ8, "nsg": IndexType.RNSG, "ivf_sq8h": IndexType.IVF_SQ8H, - "ivf_pq": IndexType.IVF_PQ + "ivf_pq": IndexType.IVF_PQ, + "hnsw": IndexType.HNSW, + "annoy": IndexType.ANNOY } +epsilon = 0.1 def time_wrapper(func): """ @@ -35,24 +38,23 @@ def time_wrapper(func): class MilvusClient(object): - def __init__(self, table_name=None, ip=None, port=None, timeout=60): - self._milvus = Milvus() - self._table_name = table_name + def __init__(self, collection_name=None, ip=None, port=None, timeout=60): + self._collection_name = collection_name try: i = 1 start_time = time.time() if not ip: - self._milvus.connect( + self._milvus = Milvus( host = SERVER_HOST_DEFAULT, port = SERVER_PORT_DEFAULT) else: # retry connect for remote server while time.time() < start_time + timeout: try: - self._milvus.connect( + self._milvus = Milvus( host = ip, port = port) - if self._milvus.connected() is True: + if self._milvus.server_status(): logger.debug("Try connect times: %d, %s" % (i, round(time.time() - start_time, 2))) break except Exception as e: @@ -63,16 +65,23 @@ class MilvusClient(object): raise e def __str__(self): - return 'Milvus table %s' % self._table_name + return 'Milvus collection %s' % self._collection_name def check_status(self, status): if not status.OK(): logger.error(status.message) - # raise Exception("Status not ok") + raise Exception("Status not ok") - def create_table(self, table_name, dimension, index_file_size, metric_type): - if not self._table_name: - self._table_name = table_name + def check_result_ids(self, result): + for index, item in enumerate(result): + if item[0].distance >= epsilon: + logger.error(index) + logger.error(item[0].distance) + raise Exception("Distance wrong") + + def create_collection(self, collection_name, dimension, index_file_size, metric_type): + if not self._collection_name: + self._collection_name = collection_name if metric_type == "l2": metric_type = MetricType.L2 elif metric_type == "ip": @@ -81,61 +90,82 @@ class MilvusClient(object): metric_type = MetricType.JACCARD elif metric_type == "hamming": metric_type = MetricType.HAMMING + elif metric_type == "sub": + metric_type = MetricType.SUBSTRUCTURE + elif metric_type == "super": + metric_type = MetricType.SUPERSTRUCTURE else: logger.error("Not supported metric_type: %s" % metric_type) - create_param = {'table_name': table_name, + create_param = {'collection_name': collection_name, 'dimension': dimension, 'index_file_size': index_file_size, "metric_type": metric_type} - status = self._milvus.create_table(create_param) + status = self._milvus.create_collection(create_param) self.check_status(status) @time_wrapper def insert(self, X, ids=None): - status, result = self._milvus.add_vectors(self._table_name, X, ids) + status, result = self._milvus.add_vectors(self._collection_name, X, ids) self.check_status(status) return status, result @time_wrapper - def create_index(self, index_type, nlist): - index_params = { - "index_type": INDEX_MAP[index_type], - "nlist": nlist, - } - logger.info("Building index start, table_name: %s, index_params: %s" % (self._table_name, json.dumps(index_params))) - status = self._milvus.create_index(self._table_name, index=index_params) + def delete_vectors(self, ids): + status = self._milvus.delete_by_id(self._collection_name, ids) + self.check_status(status) + + @time_wrapper + def flush(self): + status = self._milvus.flush([self._collection_name]) + self.check_status(status) + + @time_wrapper + def compact(self): + status = self._milvus.compact(self._collection_name) + self.check_status(status) + + @time_wrapper + def create_index(self, index_type, index_param=None): + index_type = INDEX_MAP[index_type] + logger.info("Building index start, collection_name: %s, index_type: %s" % (self._collection_name, index_type)) + if index_param: + logger.info(index_param) + status = self._milvus.create_index(self._collection_name, index_type, index_param) self.check_status(status) def describe_index(self): - status, result = self._milvus.describe_index(self._table_name) + status, result = self._milvus.describe_index(self._collection_name) + self.check_status(status) index_type = None for k, v in INDEX_MAP.items(): if result._index_type == v: index_type = k break - nlist = result._nlist - res = { - "index_type": index_type, - "nlist": nlist - } - return res + return {"index_type": index_type, "index_param": result._params} def drop_index(self): - logger.info("Drop index: %s" % self._table_name) - return self._milvus.drop_index(self._table_name) + logger.info("Drop index: %s" % self._collection_name) + return self._milvus.drop_index(self._collection_name) @time_wrapper - def query(self, X, top_k, nprobe): - status, result = self._milvus.search_vectors(self._table_name, top_k, nprobe, X) + def query(self, X, top_k, search_param=None): + status, result = self._milvus.search_vectors(self._collection_name, top_k, query_records=X, params=search_param) self.check_status(status) return result - def count(self): - return self._milvus.get_table_row_count(self._table_name)[1] + @time_wrapper + def query_ids(self, top_k, ids, search_param=None): + status, result = self._milvus.search_by_ids(self._collection_name, ids, top_k, params=search_param) + self.check_result_ids(result) + return result - def delete(self, timeout=60): - logger.info("Start delete table: %s" % self._table_name) - self._milvus.delete_table(self._table_name) + def count(self): + return self._milvus.count_collection(self._collection_name)[1] + + def delete(self, timeout=120): + timeout = int(timeout) + logger.info("Start delete collection: %s" % self._collection_name) + self._milvus.drop_collection(self._collection_name) i = 0 while i < timeout: if self.count(): @@ -145,24 +175,26 @@ class MilvusClient(object): else: break if i >= timeout: - logger.error("Delete table timeout") + logger.error("Delete collection timeout") def describe(self): - return self._milvus.describe_table(self._table_name) + return self._milvus.describe_collection(self._collection_name) - def show_tables(self): - return self._milvus.show_tables() + def show_collections(self): + return self._milvus.show_collections() - def exists_table(self, table_name=None): - if table_name is None: - table_name = self._table_name - status, res = self._milvus.has_table(table_name) - self.check_status(status) + def exists_collection(self, collection_name=None): + if collection_name is None: + collection_name = self._collection_name + status, res = self._milvus.has_collection(collection_name) + # self.check_status(status) return res @time_wrapper - def preload_table(self): - return self._milvus.preload_table(self._table_name, timeout=3000) + def preload_collection(self): + status = self._milvus.preload_collection(self._collection_name, timeout=3000) + self.check_status(status) + return status def get_server_version(self): status, res = self._milvus.server_version() @@ -192,20 +224,20 @@ class MilvusClient(object): return res -def fit(table_name, X): +def fit(collection_name, X): milvus = Milvus() milvus.connect(host = SERVER_HOST_DEFAULT, port = SERVER_PORT_DEFAULT) start = time.time() - status, ids = milvus.add_vectors(table_name, X) + status, ids = milvus.add_vectors(collection_name, X) end = time.time() logger(status, round(end - start, 2)) -def fit_concurrent(table_name, process_num, vectors): +def fit_concurrent(collection_name, process_num, vectors): processes = [] for i in range(process_num): - p = Process(target=fit, args=(table_name, vectors, )) + p = Process(target=fit, args=(collection_name, vectors, )) processes.append(p) p.start() for p in processes: @@ -216,12 +248,12 @@ if __name__ == "__main__": import numpy import sklearn.preprocessing - # table_name = "tset_test" - # # table_name = "test_tset1" - # m = MilvusClient(table_name) + # collection_name = "tset_test" + # # collection_name = "test_tset1" + # m = MilvusClient(collection_name) # m.delete() # time.sleep(2) - # m.create_table(table_name, 128, 20, "ip") + # m.create_collection(collection_name, 128, 20, "ip") # print(m.describe()) # print(m.count()) diff --git a/tests/milvus_benchmark/docker_runner.py b/tests/milvus_benchmark/docker_runner.py index 78c9550df7..406f3524d2 100644 --- a/tests/milvus_benchmark/docker_runner.py +++ b/tests/milvus_benchmark/docker_runner.py @@ -33,47 +33,47 @@ class DockerRunner(Runner): continue for index, param in enumerate(run_params): logger.info("Definition param: %s" % str(param)) - table_name = param["table_name"] + collection_name = param["collection_name"] volume_name = param["db_path_prefix"] - print(table_name) - (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) + print(collection_name) + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) for k, v in param.items(): if k.startswith("server."): # Update server config utils.modify_config(k, v, type="server", db_slave=None) container = utils.run_server(self.image, test_type="remote", volume_name=volume_name, db_slave=None) time.sleep(2) - milvus = MilvusClient(table_name) - # Check has table or not - if milvus.exists_table(): + milvus = MilvusClient(collection_name) + # Check has collection or not + if milvus.exists_collection(): milvus.delete() time.sleep(10) - milvus.create_table(table_name, dimension, index_file_size, metric_type) + milvus.create_collection(collection_name, dimension, index_file_size, metric_type) # debug # milvus.create_index("ivf_sq8", 16384) - res = self.do_insert(milvus, table_name, data_type, dimension, table_size, param["ni_per"]) + res = self.do_insert(milvus, collection_name, data_type, dimension, collection_size, param["ni_per"]) logger.info(res) # wait for file merge - time.sleep(table_size * dimension / 5000000) + time.sleep(collection_size * dimension / 5000000) # Clear up utils.remove_container(container) elif op_type == "query": for index, param in enumerate(run_params): logger.info("Definition param: %s" % str(param)) - table_name = param["dataset"] + collection_name = param["dataset"] volume_name = param["db_path_prefix"] - (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) for k, v in param.items(): if k.startswith("server."): utils.modify_config(k, v, type="server") container = utils.run_server(self.image, test_type="remote", volume_name=volume_name, db_slave=None) time.sleep(2) - milvus = MilvusClient(table_name) - logger.debug(milvus.show_tables()) - # Check has table or not - if not milvus.exists_table(): - logger.warning("Table %s not existed, continue exec next params ..." % table_name) + milvus = MilvusClient(collection_name) + logger.debug(milvus.show_collections()) + # Check has collection or not + if not milvus.exists_collection(): + logger.warning("Table %s not existed, continue exec next params ..." % collection_name) continue # parse index info index_types = param["index.index_types"] @@ -90,17 +90,17 @@ class DockerRunner(Runner): logger.info(result) logger.info(milvus.count()) # preload index - milvus.preload_table() + milvus.preload_collection() logger.info("Start warm up query") - res = self.do_query(milvus, table_name, [1], [1], 1, 1) + res = self.do_query(milvus, collection_name, [1], [1], 1, 1) logger.info("End warm up query") # Run query test for nprobe in nprobes: logger.info("index_type: %s, nlist: %s, metric_type: %s, nprobe: %s" % (index_type, nlist, metric_type, nprobe)) - res = self.do_query(milvus, table_name, top_ks, nqs, nprobe, run_count) + res = self.do_query(milvus, collection_name, top_ks, nqs, nprobe, run_count) headers = ["Nq/Top-k"] headers.extend([str(top_k) for top_k in top_ks]) - utils.print_table(headers, nqs, res) + utils.print_collection(headers, nqs, res) utils.remove_container(container) elif run_type == "insert_performance": @@ -114,28 +114,28 @@ class DockerRunner(Runner): continue for index, param in enumerate(run_params): logger.info("Definition param: %s" % str(param)) - table_name = param["table_name"] + collection_name = param["collection_name"] volume_name = param["db_path_prefix"] - print(table_name) - (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) + print(collection_name) + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) for k, v in param.items(): if k.startswith("server."): # Update server config utils.modify_config(k, v, type="server", db_slave=None) container = utils.run_server(self.image, test_type="remote", volume_name=volume_name, db_slave=None) time.sleep(2) - milvus = MilvusClient(table_name) - # Check has table or not - if milvus.exists_table(): + milvus = MilvusClient(collection_name) + # Check has collection or not + if milvus.exists_collection(): milvus.delete() time.sleep(10) - milvus.create_table(table_name, dimension, index_file_size, metric_type) + milvus.create_collection(collection_name, dimension, index_file_size, metric_type) # debug # milvus.create_index("ivf_sq8", 16384) - res = self.do_insert(milvus, table_name, data_type, dimension, table_size, param["ni_per"]) + res = self.do_insert(milvus, collection_name, data_type, dimension, collection_size, param["ni_per"]) logger.info(res) # wait for file merge - time.sleep(table_size * dimension / 5000000) + time.sleep(collection_size * dimension / 5000000) # Clear up utils.remove_container(container) @@ -147,19 +147,19 @@ class DockerRunner(Runner): container = None for index, param in enumerate(run_params): logger.info("Definition param: %s" % str(param)) - table_name = param["dataset"] + collection_name = param["dataset"] volume_name = param["db_path_prefix"] - (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) for k, v in param.items(): if k.startswith("server."): utils.modify_config(k, v, type="server") container = utils.run_server(self.image, test_type="remote", volume_name=volume_name, db_slave=None) time.sleep(2) - milvus = MilvusClient(table_name) - logger.debug(milvus.show_tables()) - # Check has table or not - if not milvus.exists_table(): - logger.warning("Table %s not existed, continue exec next params ..." % table_name) + milvus = MilvusClient(collection_name) + logger.debug(milvus.show_collections()) + # Check has collection or not + if not milvus.exists_collection(): + logger.warning("Table %s not existed, continue exec next params ..." % collection_name) continue # parse index info index_types = param["index.index_types"] @@ -176,17 +176,17 @@ class DockerRunner(Runner): logger.info(result) logger.info(milvus.count()) # preload index - milvus.preload_table() + milvus.preload_collection() logger.info("Start warm up query") - res = self.do_query(milvus, table_name, [1], [1], 1, 1) + res = self.do_query(milvus, collection_name, [1], [1], 1, 1) logger.info("End warm up query") # Run query test for nprobe in nprobes: logger.info("index_type: %s, nlist: %s, metric_type: %s, nprobe: %s" % (index_type, nlist, metric_type, nprobe)) - res = self.do_query(milvus, table_name, top_ks, nqs, nprobe, run_count) + res = self.do_query(milvus, collection_name, top_ks, nqs, nprobe, run_count) headers = ["Nq/Top-k"] headers.extend([str(top_k) for top_k in top_ks]) - utils.print_table(headers, nqs, res) + utils.print_collection(headers, nqs, res) utils.remove_container(container) elif run_type == "accuracy": @@ -212,21 +212,21 @@ class DockerRunner(Runner): for index, param in enumerate(run_params): logger.info("Definition param: %s" % str(param)) - table_name = param["dataset"] + collection_name = param["dataset"] sift_acc = False if "sift_acc" in param: sift_acc = param["sift_acc"] - (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) for k, v in param.items(): if k.startswith("server."): utils.modify_config(k, v, type="server") volume_name = param["db_path_prefix"] container = utils.run_server(self.image, test_type="remote", volume_name=volume_name, db_slave=None) time.sleep(2) - milvus = MilvusClient(table_name) - # Check has table or not - if not milvus.exists_table(): - logger.warning("Table %s not existed, continue exec next params ..." % table_name) + milvus = MilvusClient(collection_name) + # Check has collection or not + if not milvus.exists_collection(): + logger.warning("Table %s not existed, continue exec next params ..." % collection_name) continue # parse index info @@ -236,7 +236,7 @@ class DockerRunner(Runner): top_ks, nqs, nprobes = parser.search_params_parser(param) if sift_acc is True: # preload groundtruth data - true_ids_all = self.get_groundtruth_ids(table_size) + true_ids_all = self.get_groundtruth_ids(collection_size) acc_dict = {} for index_type in index_types: for nlist in nlists: @@ -244,7 +244,7 @@ class DockerRunner(Runner): logger.info(result) milvus.create_index(index_type, nlist) # preload index - milvus.preload_table() + milvus.preload_collection() # Run query test for nprobe in nprobes: logger.info("index_type: %s, nlist: %s, metric_type: %s, nprobe: %s" % (index_type, nlist, metric_type, nprobe)) @@ -252,17 +252,17 @@ class DockerRunner(Runner): for nq in nqs: result_ids = [] id_prefix = "%s_index_%s_nlist_%s_metric_type_%s_nprobe_%s_top_k_%s_nq_%s" % \ - (table_name, index_type, nlist, metric_type, nprobe, top_k, nq) + (collection_name, index_type, nlist, metric_type, nprobe, top_k, nq) if sift_acc is False: - self.do_query_acc(milvus, table_name, top_k, nq, nprobe, id_prefix) + self.do_query_acc(milvus, collection_name, top_k, nq, nprobe, id_prefix) if index_type != "flat": # Compute accuracy base_name = "%s_index_flat_nlist_%s_metric_type_%s_nprobe_%s_top_k_%s_nq_%s" % \ - (table_name, nlist, metric_type, nprobe, top_k, nq) + (collection_name, nlist, metric_type, nprobe, top_k, nq) avg_acc = self.compute_accuracy(base_name, id_prefix) logger.info("Query: <%s> accuracy: %s" % (id_prefix, avg_acc)) else: - result_ids, result_distances = self.do_query_ids(milvus, table_name, top_k, nq, nprobe) + result_ids, result_distances = self.do_query_ids(milvus, collection_name, top_k, nq, nprobe) debug_file_ids = "0.5.3_result_ids" debug_file_distances = "0.5.3_result_distances" with open(debug_file_ids, "w+") as fd: @@ -276,10 +276,10 @@ class DockerRunner(Runner): fd.write("%s\n" % str(true_item)) acc_value = self.get_recall_value(true_ids_all[:nq, :top_k].tolist(), result_ids) logger.info("Query: <%s> accuracy: %s" % (id_prefix, acc_value)) - # # print accuracy table - # headers = [table_name] + # # print accuracy collection + # headers = [collection_name] # headers.extend([str(top_k) for top_k in top_ks]) - # utils.print_table(headers, nqs, res) + # utils.print_collection(headers, nqs, res) # remove container, and run next definition logger.info("remove container, and run next definition") @@ -295,10 +295,10 @@ class DockerRunner(Runner): container = None for index, param in enumerate(run_params): logger.info("Definition param: %s" % str(param)) - table_name = param["dataset"] + collection_name = param["dataset"] index_type = param["index_type"] volume_name = param["db_path_prefix"] - (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) # set default test time if "during_time" not in param: @@ -317,10 +317,10 @@ class DockerRunner(Runner): container = utils.run_server(self.image, test_type="remote", volume_name=volume_name, db_slave=None) time.sleep(2) - milvus = MilvusClient(table_name) - # Check has table or not - if not milvus.exists_table(): - logger.warning("Table %s not existed, continue exec next params ..." % table_name) + milvus = MilvusClient(collection_name) + # Check has collection or not + if not milvus.exists_collection(): + logger.warning("Table %s not existed, continue exec next params ..." % collection_name) continue start_time = time.time() @@ -331,22 +331,22 @@ class DockerRunner(Runner): processes = [] # do query # for i in range(query_process_num): - # milvus_instance = MilvusClient(table_name) + # milvus_instance = MilvusClient(collection_name) # top_k = random.choice([x for x in range(1, 100)]) # nq = random.choice([x for x in range(1, 100)]) # nprobe = random.choice([x for x in range(1, 1000)]) # # logger.info("index_type: %s, nlist: %s, metric_type: %s, nprobe: %s" % (index_type, nlist, metric_type, nprobe)) - # p = Process(target=self.do_query, args=(milvus_instance, table_name, [top_k], [nq], [nprobe], run_count, )) + # p = Process(target=self.do_query, args=(milvus_instance, collection_name, [top_k], [nq], [nprobe], run_count, )) # processes.append(p) # p.start() # time.sleep(0.1) # for p in processes: # p.join() - milvus_instance = MilvusClient(table_name) + milvus_instance = MilvusClient(collection_name) top_ks = random.sample([x for x in range(1, 100)], 3) nqs = random.sample([x for x in range(1, 1000)], 3) nprobe = random.choice([x for x in range(1, 500)]) - res = self.do_query(milvus, table_name, top_ks, nqs, nprobe, run_count) + res = self.do_query(milvus, collection_name, top_ks, nqs, nprobe, run_count) if i % 10 == 0: status, res = milvus_instance.insert(insert_vectors, ids=[x for x in range(len(insert_vectors))]) if not status.OK(): diff --git a/tests/milvus_benchmark/k8s_runner.py b/tests/milvus_benchmark/k8s_runner.py index a8d35e63ea..1afc04c9f7 100644 --- a/tests/milvus_benchmark/k8s_runner.py +++ b/tests/milvus_benchmark/k8s_runner.py @@ -5,10 +5,11 @@ import time import re import random import traceback +import json from multiprocessing import Process import numpy as np +from yaml import full_load, dump from client import MilvusClient -import utils import parser from runner import Runner from milvus_metrics.api import report @@ -23,6 +24,7 @@ INSERT_INTERVAL = 50000 timestamp = int(time.time()) default_path = "/var/lib/milvus" + class K8sRunner(Runner): """run docker mode""" def __init__(self): @@ -33,41 +35,17 @@ class K8sRunner(Runner): self.hostname = None self.env_value = None - def init_env(self, server_config, args): - self.hostname = args.hostname - # update server_config - helm_path = os.path.join(os.getcwd(), "../milvus-helm/milvus") - server_config_file = helm_path+"/ci/config/sqlite/%s/server_config.yaml" % (args.image_type) - if not os.path.exists(server_config_file): - raise Exception("File %s not existed" % server_config_file) - if server_config: - logger.debug("Update server config") - utils.update_server_config(server_config_file, server_config) - # update log_config - log_config_file = helm_path+"/config/log_config.conf" - if not os.path.exists(log_config_file): - raise Exception("File %s not existed" % log_config_file) - src_log_config_file = helm_path+"/config/log_config.conf.src" - if not os.path.exists(src_log_config_file): - # copy - os.system("cp %s %s" % (log_config_file, src_log_config_file)) - else: - # reset - os.system("cp %s %s" % (src_log_config_file, log_config_file)) - if "db_config.primary_path" in server_config: - os.system("sed -i 's#%s#%s#g' %s" % (default_path, server_config["db_config.primary_path"], log_config_file)) - - # with open(log_config_file, "r+") as fd: - # for line in fd.readlines(): - # fd.write(re.sub(r'^%s' % default_path, server_config["db_config.primary_path"], line)) + def init_env(self, server_config, server_host, image_type, image_tag): + self.hostname = server_host # update values + helm_path = os.path.join(os.getcwd(), "../milvus-helm") values_file_path = helm_path+"/values.yaml" if not os.path.exists(values_file_path): raise Exception("File %s not existed" % values_file_path) - utils.update_values(values_file_path, args.hostname) + utils.update_values(values_file_path, server_host, server_config) try: logger.debug("Start install server") - self.host, self.ip = utils.helm_install_server(helm_path, args.image_tag, args.image_type, self.name, namespace) + self.host, self.ip = utils.helm_install_server(helm_path, image_tag, image_type, self.name, namespace) except Exception as e: logger.error("Helm install server failed: %s" % str(e)) logger.error(traceback.format_exc()) @@ -83,9 +61,9 @@ class K8sRunner(Runner): def clean_up(self): logger.debug(self.name) - utils.helm_del_server(self.name) + utils.helm_del_server(self.name, namespace) - def report_wrapper(self, milvus_instance, env_value, hostname, table_info, index_info, search_params): + def report_wrapper(self, milvus_instance, env_value, hostname, collection_info, index_info, search_params): metric = Metric() metric.set_run_id(timestamp) metric.env = Env(env_value) @@ -95,45 +73,51 @@ class K8sRunner(Runner): server_mode = milvus_instance.get_server_mode() commit = milvus_instance.get_server_commit() metric.server = Server(version=server_version, mode=server_mode, build_commit=commit) - metric.table = table_info + metric.collection = collection_info metric.index = index_info metric.search = search_params return metric - def run(self, run_type, table): + def run(self, run_type, collection): logger.debug(run_type) - logger.debug(table) - table_name = table["table_name"] - milvus_instance = MilvusClient(table_name=table_name, ip=self.ip) + logger.debug(collection) + collection_name = collection["collection_name"] + milvus_instance = MilvusClient(collection_name=collection_name, ip=self.ip) self.env_value = milvus_instance.get_server_config() + + # ugly implemention + self.env_value.pop("logs") + if run_type == "insert_performance": - (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) - ni_per = table["ni_per"] - build_index = table["build_index"] - if milvus_instance.exists_table(): + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + ni_per = collection["ni_per"] + build_index = collection["build_index"] + if milvus_instance.exists_collection(): milvus_instance.delete() time.sleep(10) index_info = {} search_params = {} - milvus_instance.create_table(table_name, dimension, index_file_size, metric_type) + milvus_instance.create_collection(collection_name, dimension, index_file_size, metric_type) if build_index is True: - index_type = table["index_type"] - nlist = table["nlist"] + index_type = collection["index_type"] + index_param = collection["index_param"] index_info = { "index_type": index_type, - "index_nlist": nlist + "index_param": index_param } - milvus_instance.create_index(index_type, nlist) - res = self.do_insert(milvus_instance, table_name, data_type, dimension, table_size, ni_per) + milvus_instance.create_index(index_type, index_param) + logger.debug(milvus_instance.describe_index()) + res = self.do_insert(milvus_instance, collection_name, data_type, dimension, collection_size, ni_per) logger.info(res) - table_info = { + milvus_instance.flush() + collection_info = { "dimension": dimension, "metric_type": metric_type, - "dataset_name": table_name + "dataset_name": collection_name } - metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, table_info, index_info, search_params) + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, search_params) metric.metrics = { - "type": "insert_performance", + "type": run_type, "value": { "total_time": res["total_time"], "qps": res["qps"], @@ -141,24 +125,56 @@ class K8sRunner(Runner): } } report(metric) - logger.debug("Wait for file merge") - time.sleep(120) + if build_index is True: + logger.debug("Start build index for last file") + milvus_instance.create_index(index_type, index_param) + logger.debug(milvus_instance.describe_index()) - elif run_type == "build_performance": - (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) - index_type = table["index_type"] - nlist = table["nlist"] - table_info = { + if run_type == "insert_flush_performance": + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + ni_per = collection["ni_per"] + if milvus_instance.exists_collection(): + milvus_instance.delete() + time.sleep(10) + index_info = {} + search_params = {} + milvus_instance.create_collection(collection_name, dimension, index_file_size, metric_type) + res = self.do_insert(milvus_instance, collection_name, data_type, dimension, collection_size, ni_per) + logger.info(res) + logger.debug(milvus_instance.count()) + start_time = time.time() + milvus_instance.flush() + end_time = time.time() + logger.debug(milvus_instance.count()) + collection_info = { "dimension": dimension, "metric_type": metric_type, - "dataset_name": table_name + "dataset_name": collection_name + } + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, search_params) + metric.metrics = { + "type": run_type, + "value": { + "flush_time": round(end_time - start_time, 1) + } + } + report(metric) + + elif run_type == "build_performance": + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + index_type = collection["index_type"] + index_param = collection["index_param"] + collection_info = { + "dimension": dimension, + "metric_type": metric_type, + "dataset_name": collection_name } index_info = { "index_type": index_type, - "index_nlist": nlist + "index_param": index_param } - if not milvus_instance.exists_table(): - logger.error("Table name: %s not existed" % table_name) + if not milvus_instance.exists_collection(): + logger.error("Table name: %s not existed" % collection_name) return search_params = {} start_time = time.time() @@ -166,11 +182,12 @@ class K8sRunner(Runner): logger.debug("Drop index") milvus_instance.drop_index() start_mem_usage = milvus_instance.get_mem_info()["memory_used"] - milvus_instance.create_index(index_type, nlist) + milvus_instance.create_index(index_type, index_param) logger.debug(milvus_instance.describe_index()) + logger.debug(milvus_instance.count()) end_time = time.time() end_mem_usage = milvus_instance.get_mem_info()["memory_used"] - metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, table_info, index_info, search_params) + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, search_params) metric.metrics = { "type": "build_performance", "value": { @@ -182,47 +199,90 @@ class K8sRunner(Runner): } report(metric) - elif run_type == "search_performance": - (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) - run_count = table["run_count"] - search_params = table["search_params"] - table_info = { + elif run_type == "delete_performance": + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + ni_per = collection["ni_per"] + search_params = {} + collection_info = { "dimension": dimension, "metric_type": metric_type, - "dataset_name": table_name + "dataset_name": collection_name } - if not milvus_instance.exists_table(): - logger.error("Table name: %s not existed" % table_name) + if not milvus_instance.exists_collection(): + logger.error("Table name: %s not existed" % collection_name) return - logger.info(milvus_instance.count()) - result = milvus_instance.describe_index() - index_info = { - "index_type": result["index_type"], - "index_nlist": result["nlist"] - } + length = milvus_instance.count() + logger.info(length) + index_info = milvus_instance.describe_index() logger.info(index_info) - nprobes = search_params["nprobes"] - top_ks = search_params["top_ks"] - nqs = search_params["nqs"] - milvus_instance.preload_table() + ids = [i for i in range(length)] + loops = int(length / ni_per) + milvus_instance.preload_collection() + start_mem_usage = milvus_instance.get_mem_info()["memory_used"] + start_time = time.time() + for i in range(loops): + delete_ids = ids[i*ni_per : i*ni_per+ni_per] + logger.debug("Delete %d - %d" % (delete_ids[0], delete_ids[-1])) + milvus_instance.delete_vectors(delete_ids) + milvus_instance.flush() + logger.debug("Table row counts: %d" % milvus_instance.count()) + logger.debug("Table row counts: %d" % milvus_instance.count()) + milvus_instance.flush() + end_time = time.time() + end_mem_usage = milvus_instance.get_mem_info()["memory_used"] + logger.debug("Table row counts: %d" % milvus_instance.count()) + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, search_params) + metric.metrics = { + "type": "delete_performance", + "value": { + "delete_time": round(end_time - start_time, 1), + "start_mem_usage": start_mem_usage, + "end_mem_usage": end_mem_usage, + "diff_mem": end_mem_usage - start_mem_usage + } + } + report(metric) + + elif run_type == "search_performance": + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + run_count = collection["run_count"] + top_ks = collection["top_ks"] + nqs = collection["nqs"] + search_params = collection["search_params"] + collection_info = { + "dimension": dimension, + "metric_type": metric_type, + "dataset_name": collection_name + } + # fro debugging + # time.sleep(3600) + if not milvus_instance.exists_collection(): + logger.error("Table name: %s not existed" % collection_name) + return + + logger.info(milvus_instance.count()) + index_info = milvus_instance.describe_index() + logger.info(index_info) + milvus_instance.preload_collection() logger.info("Start warm up query") - res = self.do_query(milvus_instance, table_name, [1], [1], 1, 2) + res = self.do_query(milvus_instance, collection_name, [1], [1], 2, search_param=search_params[0]) logger.info("End warm up query") - for nprobe in nprobes: - logger.info("Search nprobe: %s" % nprobe) - res = self.do_query(milvus_instance, table_name, top_ks, nqs, nprobe, run_count) + for search_param in search_params: + logger.info("Search param: %s" % json.dumps(search_param)) + res = self.do_query(milvus_instance, collection_name, top_ks, nqs, run_count, search_param) headers = ["Nq/Top-k"] headers.extend([str(top_k) for top_k in top_ks]) + logger.info("Search param: %s" % json.dumps(search_param)) utils.print_table(headers, nqs, res) for index_nq, nq in enumerate(nqs): for index_top_k, top_k in enumerate(top_ks): - search_param = { - "nprobe": nprobe, + search_param_group = { "nq": nq, - "topk": top_k + "topk": top_k, + "search_param": search_param } search_time = res[index_nq][index_top_k] - metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, table_info, index_info, search_param) + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, search_param_group) metric.metrics = { "type": "search_performance", "value": { @@ -231,44 +291,93 @@ class K8sRunner(Runner): } report(metric) - # for sift/deep datasets - elif run_type == "accuracy": - (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) - search_params = table["search_params"] - table_info = { + elif run_type == "search_ids_stability": + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + search_params = collection["search_params"] + during_time = collection["during_time"] + ids_length = collection["ids_length"] + ids = collection["ids"] + collection_info = { "dimension": dimension, "metric_type": metric_type, - "dataset_name": table_name + "dataset_name": collection_name } - if not milvus_instance.exists_table(): - logger.error("Table name: %s not existed" % table_name) + if not milvus_instance.exists_collection(): + logger.error("Table name: %s not existed" % collection_name) return logger.info(milvus_instance.count()) - result = milvus_instance.describe_index() - index_info = { - "index_type": result["index_type"], - "index_nlist": result["nlist"] - } + index_info = milvus_instance.describe_index() logger.info(index_info) - nprobes = search_params["nprobes"] - top_ks = search_params["top_ks"] - nqs = search_params["nqs"] - milvus_instance.preload_table() - true_ids_all = self.get_groundtruth_ids(table_size) - for nprobe in nprobes: - logger.info("Search nprobe: %s" % nprobe) + g_top_k = int(collection["top_ks"].split("-")[1]) + l_top_k = int(collection["top_ks"].split("-")[0]) + g_id = int(ids.split("-")[1]) + l_id = int(ids.split("-")[0]) + g_id_length = int(ids_length.split("-")[1]) + l_id_length = int(ids_length.split("-")[0]) + + milvus_instance.preload_collection() + start_mem_usage = milvus_instance.get_mem_info()["memory_used"] + logger.debug(start_mem_usage) + start_time = time.time() + while time.time() < start_time + during_time * 60: + search_param = {} + top_k = random.randint(l_top_k, g_top_k) + ids_num = random.randint(l_id_length, g_id_length) + ids_param = [random.randint(l_id_length, g_id_length) for _ in range(ids_num)] + for k, v in search_params.items(): + search_param[k] = random.randint(int(v.split("-")[0]), int(v.split("-")[1])) + logger.debug("Query top-k: %d, ids_num: %d, param: %s" % (top_k, ids_num, json.dumps(search_param))) + result = milvus_instance.query_ids(top_k, ids_param, search_param=search_param) + end_mem_usage = milvus_instance.get_mem_info()["memory_used"] + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, {}) + metric.metrics = { + "type": "search_ids_stability", + "value": { + "during_time": during_time, + "start_mem_usage": start_mem_usage, + "end_mem_usage": end_mem_usage, + "diff_mem": end_mem_usage - start_mem_usage + } + } + report(metric) + + # for sift/deep datasets + # TODO: enable + elif run_type == "accuracy": + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + search_params = collection["search_params"] + # mapping to search param list + search_params = self.generate_combinations(search_params) + + top_ks = collection["top_ks"] + nqs = collection["nqs"] + collection_info = { + "dimension": dimension, + "metric_type": metric_type, + "dataset_name": collection_name + } + if not milvus_instance.exists_collection(): + logger.error("Table name: %s not existed" % collection_name) + return + logger.info(milvus_instance.count()) + index_info = milvus_instance.describe_index() + logger.info(index_info) + milvus_instance.preload_collection() + true_ids_all = self.get_groundtruth_ids(collection_size) + for search_param in search_params: for top_k in top_ks: for nq in nqs: total = 0 - search_param = { - "nprobe": nprobe, + search_param_group = { "nq": nq, - "topk": top_k + "topk": top_k, + "search_param": search_param } - result_ids, result_distances = self.do_query_ids(milvus_instance, table_name, top_k, nq, nprobe) + logger.info("Query params: %s" % json.dumps(search_param_group)) + result_ids, result_distances = self.do_query_ids(milvus_instance, collection_name, top_k, nq, search_param=search_param) acc_value = self.get_recall_value(true_ids_all[:nq, :top_k].tolist(), result_ids) logger.info("Query accuracy: %s" % acc_value) - metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, table_info, index_info, search_param) + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, search_param_group) metric.metrics = { "type": "accuracy", "value": { @@ -278,29 +387,33 @@ class K8sRunner(Runner): report(metric) elif run_type == "ann_accuracy": - hdf5_source_file = table["source_file"] - table_name = table["table_name"] - index_file_sizes = table["index_file_sizes"] - index_types = table["index_types"] - nlists = table["nlists"] - search_params = table["search_params"] - nprobes = search_params["nprobes"] - top_ks = search_params["top_ks"] - nqs = search_params["nqs"] - data_type, dimension, metric_type = parser.parse_ann_table_name(table_name) - table_info = { + hdf5_source_file = collection["source_file"] + collection_name = collection["collection_name"] + index_file_sizes = collection["index_file_sizes"] + index_types = collection["index_types"] + index_params = collection["index_params"] + top_ks = collection["top_ks"] + nqs = collection["nqs"] + search_params = collection["search_params"] + # mapping to search param list + search_params = self.generate_combinations(search_params) + # mapping to index param list + index_params = self.generate_combinations(index_params) + + data_type, dimension, metric_type = parser.parse_ann_collection_name(collection_name) + collection_info = { "dimension": dimension, "metric_type": metric_type, - "dataset_name": table_name + "dataset_name": collection_name } dataset = utils.get_dataset(hdf5_source_file) - if milvus_instance.exists_table(table_name): - logger.info("Re-create table: %s" % table_name) - milvus_instance.delete(table_name) + if milvus_instance.exists_collection(collection_name): + logger.info("Re-create collection: %s" % collection_name) + milvus_instance.delete() time.sleep(DELETE_INTERVAL_TIME) true_ids = np.array(dataset["neighbors"]) for index_file_size in index_file_sizes: - milvus_instance.create_table(table_name, dimension, index_file_size, metric_type) + milvus_instance.create_collection(collection_name, dimension, index_file_size, metric_type) logger.info(milvus_instance.describe()) insert_vectors = self.normalize(metric_type, np.array(dataset["train"])) # Insert batch once @@ -315,38 +428,41 @@ class K8sRunner(Runner): milvus_instance.insert(tmp_vectors.tolist(), ids=[i for i in range(start, end)]) else: milvus_instance.insert(tmp_vectors, ids=[i for i in range(start, end)]) - time.sleep(20) - logger.info("Table: %s, row count: %s" % (table_name, milvus_instance.count())) + milvus_instance.flush() + logger.info("Table: %s, row count: %s" % (collection_name, milvus_instance.count())) if milvus_instance.count() != len(insert_vectors): logger.error("Table row count is not equal to insert vectors") return for index_type in index_types: - for nlist in nlists: - milvus_instance.create_index(index_type, nlist) - # logger.info(milvus_instance.describe_index()) - logger.info("Start preload table: %s, index_type: %s, nlist: %s" % (table_name, index_type, nlist)) - milvus_instance.preload_table() + for index_param in index_params: + logger.debug("Building index with param: %s" % json.dumps(index_param)) + milvus_instance.create_index(index_type, index_param=index_param) + logger.info(milvus_instance.describe_index()) + logger.info("Start preload collection: %s" % collection_name) + milvus_instance.preload_collection() index_info = { "index_type": index_type, - "index_nlist": nlist + "index_param": index_param } - for nprobe in nprobes: + logger.debug(index_info) + for search_param in search_params: for nq in nqs: query_vectors = self.normalize(metric_type, np.array(dataset["test"][:nq])) for top_k in top_ks: - search_params = { + search_param_group = { "nq": len(query_vectors), - "nprobe": nprobe, - "topk": top_k + "topk": top_k, + "search_param": search_param } + logger.debug(search_param_group) if not isinstance(query_vectors, list): - result = milvus_instance.query(query_vectors.tolist(), top_k, nprobe) + result = milvus_instance.query(query_vectors.tolist(), top_k, search_param=search_param) else: - result = milvus_instance.query(query_vectors, top_k, nprobe) + result = milvus_instance.query(query_vectors, top_k, search_param=search_param) result_ids = result.id_array acc_value = self.get_recall_value(true_ids[:nq, :top_k].tolist(), result_ids) logger.info("Query ann_accuracy: %s" % acc_value) - metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, table_info, index_info, search_params) + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, search_param_group) metric.metrics = { "type": "ann_accuracy", "value": { @@ -354,50 +470,44 @@ class K8sRunner(Runner): } } report(metric) - milvus_instance.delete() elif run_type == "search_stability": - (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) - search_params = table["search_params"] - during_time = table["during_time"] - table_info = { + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + search_params = collection["search_params"] + during_time = collection["during_time"] + collection_info = { "dimension": dimension, "metric_type": metric_type, - "dataset_name": table_name + "dataset_name": collection_name } - if not milvus_instance.exists_table(): - logger.error("Table name: %s not existed" % table_name) + if not milvus_instance.exists_collection(): + logger.error("Table name: %s not existed" % collection_name) return logger.info(milvus_instance.count()) - result = milvus_instance.describe_index() - index_info = { - "index_type": result["index_type"], - "index_nlist": result["nlist"] - } - search_param = {} + index_info = milvus_instance.describe_index() logger.info(index_info) - g_nprobe = int(search_params["nprobes"].split("-")[1]) - g_top_k = int(search_params["top_ks"].split("-")[1]) - g_nq = int(search_params["nqs"].split("-")[1]) - l_nprobe = int(search_params["nprobes"].split("-")[0]) - l_top_k = int(search_params["top_ks"].split("-")[0]) - l_nq = int(search_params["nqs"].split("-")[0]) - milvus_instance.preload_table() + g_top_k = int(collection["top_ks"].split("-")[1]) + g_nq = int(collection["nqs"].split("-")[1]) + l_top_k = int(collection["top_ks"].split("-")[0]) + l_nq = int(collection["nqs"].split("-")[0]) + milvus_instance.preload_collection() start_mem_usage = milvus_instance.get_mem_info()["memory_used"] logger.debug(start_mem_usage) - logger.info("Start warm up query") - res = self.do_query(milvus_instance, table_name, [1], [1], 1, 2) - logger.info("End warm up query") + start_row_count = milvus_instance.count() + logger.debug(milvus_instance.describe_index()) + logger.info(start_row_count) start_time = time.time() while time.time() < start_time + during_time * 60: + search_param = {} top_k = random.randint(l_top_k, g_top_k) nq = random.randint(l_nq, g_nq) - nprobe = random.randint(l_nprobe, g_nprobe) + for k, v in search_params.items(): + search_param[k] = random.randint(int(v.split("-")[0]), int(v.split("-")[1])) query_vectors = [[random.random() for _ in range(dimension)] for _ in range(nq)] - logger.debug("Query nprobe:%d, nq:%d, top-k:%d" % (nprobe, nq, top_k)) - result = milvus_instance.query(query_vectors, top_k, nprobe) + logger.debug("Query nq: %d, top-k: %d, param: %s" % (nq, top_k, json.dumps(search_param))) + result = milvus_instance.query(query_vectors, top_k, search_param=search_param) end_mem_usage = milvus_instance.get_mem_info()["memory_used"] - metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, table_info, index_info, search_param) + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, {}) metric.metrics = { "type": "search_stability", "value": { @@ -410,56 +520,58 @@ class K8sRunner(Runner): report(metric) elif run_type == "stability": - (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) - search_params = table["search_params"] - insert_xb = table["insert_xb"] - insert_interval = table["insert_interval"] - during_time = table["during_time"] - table_info = { + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + search_params = collection["search_params"] + insert_xb = collection["insert_xb"] + insert_interval = collection["insert_interval"] + delete_xb = collection["delete_xb"] + during_time = collection["during_time"] + collection_info = { "dimension": dimension, "metric_type": metric_type, - "dataset_name": table_name + "dataset_name": collection_name } - if not milvus_instance.exists_table(): - logger.error("Table name: %s not existed" % table_name) + if not milvus_instance.exists_collection(): + logger.error("Table name: %s not existed" % collection_name) return logger.info(milvus_instance.count()) - result = milvus_instance.describe_index() - index_info = { - "index_type": result["index_type"], - "index_nlist": result["nlist"] - } - search_param = {} + index_info = milvus_instance.describe_index() logger.info(index_info) - g_nprobe = int(search_params["nprobes"].split("-")[1]) - g_top_k = int(search_params["top_ks"].split("-")[1]) - g_nq = int(search_params["nqs"].split("-")[1]) - l_nprobe = int(search_params["nprobes"].split("-")[0]) - l_top_k = int(search_params["top_ks"].split("-")[0]) - l_nq = int(search_params["nqs"].split("-")[0]) - milvus_instance.preload_table() - logger.info("Start warm up query") - res = self.do_query(milvus_instance, table_name, [1], [1], 1, 2) - logger.info("End warm up query") + g_top_k = int(collection["top_ks"].split("-")[1]) + g_nq = int(collection["nqs"].split("-")[1]) + l_top_k = int(collection["top_ks"].split("-")[0]) + l_nq = int(collection["nqs"].split("-")[0]) + milvus_instance.preload_collection() start_mem_usage = milvus_instance.get_mem_info()["memory_used"] start_row_count = milvus_instance.count() + logger.debug(milvus_instance.describe_index()) + logger.info(start_row_count) start_time = time.time() i = 0 + ids = [] + insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(insert_xb)] + query_vectors = [[random.random() for _ in range(dimension)] for _ in range(10000)] while time.time() < start_time + during_time * 60: i = i + 1 for j in range(insert_interval): top_k = random.randint(l_top_k, g_top_k) nq = random.randint(l_nq, g_nq) - nprobe = random.randint(l_nprobe, g_nprobe) - query_vectors = [[random.random() for _ in range(dimension)] for _ in range(nq)] - logger.debug("Query nprobe:%d, nq:%d, top-k:%d" % (nprobe, nq, top_k)) - result = milvus_instance.query(query_vectors, top_k, nprobe) - insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(insert_xb)] - status, res = milvus_instance.insert(insert_vectors, ids=[x for x in range(len(insert_vectors))]) + search_param = {} + for k, v in search_params.items(): + search_param[k] = random.randint(int(v.split("-")[0]), int(v.split("-")[1])) + logger.debug("Query nq: %d, top-k: %d, param: %s" % (nq, top_k, json.dumps(search_param))) + result = milvus_instance.query(query_vectors[0:nq], top_k, search_param=search_param) + count = milvus_instance.count() + insert_ids = [(count+x) for x in range(len(insert_vectors))] + ids.extend(insert_ids) + status, res = milvus_instance.insert(insert_vectors, ids=insert_ids) logger.debug("%d, row_count: %d" % (i, milvus_instance.count())) + milvus_instance.delete_vectors(ids[-delete_xb:]) + milvus_instance.flush() + milvus_instance.compact() end_mem_usage = milvus_instance.get_mem_info()["memory_used"] end_row_count = milvus_instance.count() - metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, table_info, index_info, search_param) + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, collection_info, index_info, {}) metric.metrics = { "type": "stability", "value": { @@ -470,4 +582,9 @@ class K8sRunner(Runner): "row_count_increments": end_row_count - start_row_count } } - report(metric) \ No newline at end of file + report(metric) + + else: + logger.warning("Run type not defined") + return + logger.debug("Test finished") diff --git a/tests/milvus_benchmark/local_runner.py b/tests/milvus_benchmark/local_runner.py index c7822fb7bc..a79f4abfbc 100644 --- a/tests/milvus_benchmark/local_runner.py +++ b/tests/milvus_benchmark/local_runner.py @@ -3,13 +3,17 @@ import logging import pdb import time import random +import json from multiprocessing import Process import numpy as np +import concurrent.futures from client import MilvusClient import utils import parser from runner import Runner +DELETE_INTERVAL_TIME = 5 +INSERT_INTERVAL = 50000 logger = logging.getLogger("milvus_benchmark.local_runner") @@ -20,200 +24,307 @@ class LocalRunner(Runner): self.ip = ip self.port = port - def run(self, definition, run_type=None): - if run_type == "performance": - for op_type, op_value in definition.items(): - run_count = op_value["run_count"] - run_params = op_value["params"] + def run(self, run_type, collection): + logger.debug(run_type) + logger.debug(collection) + collection_name = collection["collection_name"] + milvus_instance = MilvusClient(collection_name=collection_name, ip=self.ip, port=self.port) + logger.info(milvus_instance.show_collections()) + env_value = milvus_instance.get_server_config() + logger.debug(env_value) - if op_type == "insert": - for index, param in enumerate(run_params): - table_name = param["table_name"] - (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) - milvus = MilvusClient(table_name, ip=self.ip, port=self.port) - # Check has table or not - if milvus.exists_table(): - milvus.delete() - time.sleep(10) - milvus.create_table(table_name, dimension, index_file_size, metric_type) - res = self.do_insert(milvus, table_name, data_type, dimension, table_size, param["ni_per"]) - logger.info(res) + if run_type in ["insert_performance", "insert_flush_performance"]: + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + ni_per = collection["ni_per"] + build_index = collection["build_index"] + if milvus_instance.exists_collection(): + milvus_instance.delete() + time.sleep(10) + milvus_instance.create_collection(collection_name, dimension, index_file_size, metric_type) + if build_index is True: + index_type = collection["index_type"] + index_param = collection["index_param"] + milvus_instance.create_index(index_type, index_param) + logger.debug(milvus_instance.describe_index()) + res = self.do_insert(milvus_instance, collection_name, data_type, dimension, collection_size, ni_per) + milvus_instance.flush() + logger.debug("Table row counts: %d" % milvus_instance.count()) + if build_index is True: + logger.debug("Start build index for last file") + milvus_instance.create_index(index_type, index_param) + logger.debug(milvus_instance.describe_index()) - elif op_type == "query": - for index, param in enumerate(run_params): - logger.info("Definition param: %s" % str(param)) - table_name = param["dataset"] - (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) + elif run_type == "delete_performance": + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + ni_per = collection["ni_per"] + if not milvus_instance.exists_collection(): + logger.error(milvus_instance.show_collections()) + logger.warning("Table: %s not found" % collection_name) + return + length = milvus_instance.count() + ids = [i for i in range(length)] + loops = int(length / ni_per) + for i in range(loops): + delete_ids = ids[i*ni_per : i*ni_per+ni_per] + logger.debug("Delete %d - %d" % (delete_ids[0], delete_ids[-1])) + milvus_instance.delete_vectors(delete_ids) + milvus_instance.flush() + logger.debug("Table row counts: %d" % milvus_instance.count()) + logger.debug("Table row counts: %d" % milvus_instance.count()) + milvus_instance.flush() + logger.debug("Table row counts: %d" % milvus_instance.count()) - milvus = MilvusClient(table_name, ip=self.ip, port=self.port) - logger.info(milvus.describe()) - logger.info(milvus.describe_index()) - logger.info(milvus.count()) - logger.info(milvus.show_tables()) - # parse index info - index_types = param["index.index_types"] - nlists = param["index.nlists"] - # parse top-k, nq, nprobe - top_ks, nqs, nprobes = parser.search_params_parser(param) - # milvus.drop_index() + elif run_type == "build_performance": + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + index_type = collection["index_type"] + index_param = collection["index_param"] + if not milvus_instance.exists_collection(): + logger.error("Table name: %s not existed" % collection_name) + return + search_params = {} + start_time = time.time() + # drop index + logger.debug("Drop index") + milvus_instance.drop_index() + start_mem_usage = milvus_instance.get_mem_info()["memory_used"] + milvus_instance.create_index(index_type, index_param) + logger.debug(milvus_instance.describe_index()) + logger.debug("Table row counts: %d" % milvus_instance.count()) + end_time = time.time() + end_mem_usage = milvus_instance.get_mem_info()["memory_used"] + logger.debug("Diff memory: %s, current memory usage: %s, build time: %s" % ((end_mem_usage - start_mem_usage), end_mem_usage, round(end_time - start_time, 1))) - for index_type in index_types: - for nlist in nlists: - # milvus.create_index(index_type, nlist) - # preload index - logger.info("Start preloading table") - milvus.preload_table() - logger.info("End preloading table") - # Run query test - logger.info("Start warm up query") - res = self.do_query(milvus, table_name, [1], [1], 1, 2) - logger.info("End warm up query") - for nprobe in nprobes: - logger.info("index_type: %s, nlist: %s, metric_type: %s, nprobe: %s" % (index_type, nlist, metric_type, nprobe)) - res = self.do_query(milvus, table_name, top_ks, nqs, nprobe, run_count) - headers = ["nq/topk"] - headers.extend([str(top_k) for top_k in top_ks]) - utils.print_table(headers, nqs, res) + elif run_type == "search_performance": + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + run_count = collection["run_count"] + top_ks = collection["top_ks"] + nqs = collection["nqs"] + search_params = collection["search_params"] + # for debugging + # time.sleep(3600) + if not milvus_instance.exists_collection(): + logger.error("Table name: %s not existed" % collection_name) + return + logger.info(milvus_instance.count()) + result = milvus_instance.describe_index() + logger.info(result) + milvus_instance.preload_collection() + mem_usage = milvus_instance.get_mem_info()["memory_used"] + logger.info(mem_usage) + for search_param in search_params: + logger.info("Search param: %s" % json.dumps(search_param)) + res = self.do_query(milvus_instance, collection_name, top_ks, nqs, run_count, search_param) + headers = ["Nq/Top-k"] + headers.extend([str(top_k) for top_k in top_ks]) + logger.info("Search param: %s" % json.dumps(search_param)) + utils.print_table(headers, nqs, res) + mem_usage = milvus_instance.get_mem_info()["memory_used"] + logger.info(mem_usage) - elif run_type == "accuracy": - for op_type, op_value in definition.items(): - if op_type != "query": - logger.warning("invalid operation: %s in accuracy test, only support query operation" % op_type) - break - run_count = op_value["run_count"] - run_params = op_value["params"] - for index, param in enumerate(run_params): - logger.info("Definition param: %s" % str(param)) - table_name = param["dataset"] - sift_acc = False - if "sift_acc" in param: - sift_acc = param["sift_acc"] - (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) + elif run_type == "search_ids_stability": + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + search_params = collection["search_params"] + during_time = collection["during_time"] + ids_length = collection["ids_length"] + ids = collection["ids"] + logger.info(milvus_instance.count()) + index_info = milvus_instance.describe_index() + logger.info(index_info) + g_top_k = int(collection["top_ks"].split("-")[1]) + l_top_k = int(collection["top_ks"].split("-")[0]) + g_id = int(ids.split("-")[1]) + l_id = int(ids.split("-")[0]) + g_id_length = int(ids_length.split("-")[1]) + l_id_length = int(ids_length.split("-")[0]) - milvus = MilvusClient(table_name, ip=self.ip, port=self.port) - logger.debug(milvus.show_tables()) - # Check has table or not - if not milvus.exists_table(): - logger.warning("Table %s not existed, continue exec next params ..." % table_name) - continue + milvus_instance.preload_collection() + start_mem_usage = milvus_instance.get_mem_info()["memory_used"] + logger.debug(start_mem_usage) + start_time = time.time() + while time.time() < start_time + during_time * 60: + search_param = {} + top_k = random.randint(l_top_k, g_top_k) + ids_num = random.randint(l_id_length, g_id_length) + l_ids = random.randint(l_id, g_id-ids_num) + # ids_param = [random.randint(l_id_length, g_id_length) for _ in range(ids_num)] + ids_param = [id for id in range(l_ids, l_ids+ids_num)] + for k, v in search_params.items(): + search_param[k] = random.randint(int(v.split("-")[0]), int(v.split("-")[1])) + logger.debug("Query top-k: %d, ids_num: %d, param: %s" % (top_k, ids_num, json.dumps(search_param))) + result = milvus_instance.query_ids(top_k, ids_param, search_param=search_param) + end_mem_usage = milvus_instance.get_mem_info()["memory_used"] + metrics = { + "during_time": during_time, + "start_mem_usage": start_mem_usage, + "end_mem_usage": end_mem_usage, + "diff_mem": end_mem_usage - start_mem_usage, + } + logger.info(metrics) - # parse index info - index_types = param["index.index_types"] - nlists = param["index.nlists"] - # parse top-k, nq, nprobe - top_ks, nqs, nprobes = parser.search_params_parser(param) + elif run_type == "search_performance_concurrents": + data_type, dimension, metric_type = parser.parse_ann_collection_name(collection_name) + hdf5_source_file = collection["source_file"] + use_single_connection = collection["use_single_connection"] + concurrents = collection["concurrents"] + top_ks = collection["top_ks"] + nqs = collection["nqs"] + search_params = self.generate_combinations(collection["search_params"]) + if not milvus_instance.exists_collection(): + logger.error("Table name: %s not existed" % collection_name) + return + logger.info(milvus_instance.count()) + result = milvus_instance.describe_index() + logger.info(result) + milvus_instance.preload_collection() + dataset = utils.get_dataset(hdf5_source_file) + for concurrent_num in concurrents: + top_k = top_ks[0] + for nq in nqs: + mem_usage = milvus_instance.get_mem_info()["memory_used"] + logger.info(mem_usage) + query_vectors = self.normalize(metric_type, np.array(dataset["test"][:nq])) + logger.debug(search_params) + for search_param in search_params: + logger.info("Search param: %s" % json.dumps(search_param)) + total_time = 0.0 + if use_single_connection is True: + connections = [MilvusClient(collection_name=collection_name, ip=self.ip, port=self.port)] + with concurrent.futures.ThreadPoolExecutor(max_workers=concurrent_num) as executor: + future_results = {executor.submit( + self.do_query_qps, connections[0], query_vectors, top_k, search_param=search_param) : index for index in range(concurrent_num)} + else: + connections = [MilvusClient(collection_name=collection_name, ip=self.ip, port=self.port) for i in range(concurrent_num)] + with concurrent.futures.ThreadPoolExecutor(max_workers=concurrent_num) as executor: + future_results = {executor.submit( + self.do_query_qps, connections[index], query_vectors, top_k, search_param=search_param) : index for index in range(concurrent_num)} + for future in concurrent.futures.as_completed(future_results): + total_time = total_time + future.result() + qps_value = total_time / concurrent_num + logger.debug("QPS value: %f, total_time: %f, request_nums: %f" % (qps_value, total_time, concurrent_num)) + mem_usage = milvus_instance.get_mem_info()["memory_used"] + logger.info(mem_usage) - if sift_acc is True: - # preload groundtruth data - true_ids_all = self.get_groundtruth_ids(table_size) + elif run_type == "ann_accuracy": + hdf5_source_file = collection["source_file"] + collection_name = collection["collection_name"] + index_file_sizes = collection["index_file_sizes"] + index_types = collection["index_types"] + index_params = collection["index_params"] + top_ks = collection["top_ks"] + nqs = collection["nqs"] + search_params = collection["search_params"] + # mapping to search param list + search_params = self.generate_combinations(search_params) + # mapping to index param list + index_params = self.generate_combinations(index_params) - acc_dict = {} - for index_type in index_types: - for nlist in nlists: - result = milvus.describe_index() - logger.info(result) - # milvus.drop_index() - milvus.create_index(index_type, nlist) - # preload index - milvus.preload_table() - # Run query test - for nprobe in nprobes: - logger.info("index_type: %s, nlist: %s, metric_type: %s, nprobe: %s" % (index_type, nlist, metric_type, nprobe)) + data_type, dimension, metric_type = parser.parse_ann_collection_name(collection_name) + dataset = utils.get_dataset(hdf5_source_file) + if milvus_instance.exists_collection(collection_name): + logger.info("Re-create collection: %s" % collection_name) + milvus_instance.delete() + time.sleep(DELETE_INTERVAL_TIME) + true_ids = np.array(dataset["neighbors"]) + for index_file_size in index_file_sizes: + milvus_instance.create_collection(collection_name, dimension, index_file_size, metric_type) + logger.info(milvus_instance.describe()) + insert_vectors = self.normalize(metric_type, np.array(dataset["train"])) + logger.debug(len(insert_vectors)) + # Insert batch once + # milvus_instance.insert(insert_vectors) + loops = len(insert_vectors) // INSERT_INTERVAL + 1 + for i in range(loops): + start = i*INSERT_INTERVAL + end = min((i+1)*INSERT_INTERVAL, len(insert_vectors)) + tmp_vectors = insert_vectors[start:end] + if start < end: + if not isinstance(tmp_vectors, list): + milvus_instance.insert(tmp_vectors.tolist(), ids=[i for i in range(start, end)]) + else: + milvus_instance.insert(tmp_vectors, ids=[i for i in range(start, end)]) + milvus_instance.flush() + logger.info("Table: %s, row count: %s" % (collection_name, milvus_instance.count())) + if milvus_instance.count() != len(insert_vectors): + logger.error("Table row count is not equal to insert vectors") + return + for index_type in index_types: + for index_param in index_params: + logger.debug("Building index with param: %s" % json.dumps(index_param)) + milvus_instance.create_index(index_type, index_param=index_param) + logger.info(milvus_instance.describe_index()) + logger.info("Start preload collection: %s" % collection_name) + milvus_instance.preload_collection() + for search_param in search_params: + for nq in nqs: + query_vectors = self.normalize(metric_type, np.array(dataset["test"][:nq])) for top_k in top_ks: - for nq in nqs: - result_ids = [] - id_prefix = "%s_index_%s_nlist_%s_metric_type_%s_nprobe_%s_top_k_%s_nq_%s" % \ - (table_name, index_type, nlist, metric_type, nprobe, top_k, nq) - if sift_acc is False: - self.do_query_acc(milvus, table_name, top_k, nq, nprobe, id_prefix) - if index_type != "flat": - # Compute accuracy - base_name = "%s_index_flat_nlist_%s_metric_type_%s_nprobe_%s_top_k_%s_nq_%s" % \ - (table_name, nlist, metric_type, nprobe, top_k, nq) - avg_acc = self.compute_accuracy(base_name, id_prefix) - logger.info("Query: <%s> accuracy: %s" % (id_prefix, avg_acc)) - else: - result_ids, result_distances = self.do_query_ids(milvus, table_name, top_k, nq, nprobe) - debug_file_ids = "0.5.3_result_ids" - debug_file_distances = "0.5.3_result_distances" - with open(debug_file_ids, "w+") as fd: - total = 0 - for index, item in enumerate(result_ids): - true_item = true_ids_all[:nq, :top_k].tolist()[index] - tmp = set(item).intersection(set(true_item)) - total = total + len(tmp) - fd.write("query: N-%d, intersection: %d, total: %d\n" % (index, len(tmp), total)) - fd.write("%s\n" % str(item)) - fd.write("%s\n" % str(true_item)) - acc_value = self.get_recall_value(true_ids_all[:nq, :top_k].tolist(), result_ids) - logger.info("Query: <%s> accuracy: %s" % (id_prefix, acc_value)) - # # print accuracy table - # headers = [table_name] - # headers.extend([str(top_k) for top_k in top_ks]) - # utils.print_table(headers, nqs, res) + logger.debug("Search nq: %d, top-k: %d, search_param: %s" % (nq, top_k, json.dumps(search_param))) + if not isinstance(query_vectors, list): + result = milvus_instance.query(query_vectors.tolist(), top_k, search_param=search_param) + else: + result = milvus_instance.query(query_vectors, top_k, search_param=search_param) + result_ids = result.id_array + acc_value = self.get_recall_value(true_ids[:nq, :top_k].tolist(), result_ids) + logger.info("Query ann_accuracy: %s" % acc_value) + elif run_type == "stability": - for op_type, op_value in definition.items(): - if op_type != "query": - logger.warning("invalid operation: %s in accuracy test, only support query operation" % op_type) - break - run_count = op_value["run_count"] - run_params = op_value["params"] - nq = 100000 + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) + search_params = collection["search_params"] + insert_xb = collection["insert_xb"] + insert_interval = collection["insert_interval"] + delete_xb = collection["delete_xb"] + # flush_interval = collection["flush_interval"] + # compact_interval = collection["compact_interval"] + during_time = collection["during_time"] + if not milvus_instance.exists_collection(): + logger.error(milvus_instance.show_collections()) + logger.error("Table name: %s not existed" % collection_name) + return + g_top_k = int(collection["top_ks"].split("-")[1]) + g_nq = int(collection["nqs"].split("-")[1]) + l_top_k = int(collection["top_ks"].split("-")[0]) + l_nq = int(collection["nqs"].split("-")[0]) + milvus_instance.preload_collection() + start_mem_usage = milvus_instance.get_mem_info()["memory_used"] + start_row_count = milvus_instance.count() + logger.debug(milvus_instance.describe_index()) + logger.info(start_row_count) + start_time = time.time() + i = 0 + ids = [] + insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(insert_xb)] + query_vectors = [[random.random() for _ in range(dimension)] for _ in range(10000)] + while time.time() < start_time + during_time * 60: + i = i + 1 + for j in range(insert_interval): + top_k = random.randint(l_top_k, g_top_k) + nq = random.randint(l_nq, g_nq) + search_param = {} + for k, v in search_params.items(): + search_param[k] = random.randint(int(v.split("-")[0]), int(v.split("-")[1])) + logger.debug("Query nq: %d, top-k: %d, param: %s" % (nq, top_k, json.dumps(search_param))) + result = milvus_instance.query(query_vectors[0:nq], top_k, search_param=search_param) + count = milvus_instance.count() + insert_ids = [(count+x) for x in range(len(insert_vectors))] + ids.extend(insert_ids) + status, res = milvus_instance.insert(insert_vectors, ids=insert_ids) + logger.debug("%d, row_count: %d" % (i, milvus_instance.count())) + milvus_instance.delete_vectors(ids[-delete_xb:]) + milvus_instance.flush() + milvus_instance.compact() + end_mem_usage = milvus_instance.get_mem_info()["memory_used"] + end_row_count = milvus_instance.count() + metrics = { + "during_time": during_time, + "start_mem_usage": start_mem_usage, + "end_mem_usage": end_mem_usage, + "diff_mem": end_mem_usage - start_mem_usage, + "row_count_increments": end_row_count - start_row_count + } + logger.info(metrics) - for index, param in enumerate(run_params): - logger.info("Definition param: %s" % str(param)) - table_name = param["dataset"] - index_type = param["index_type"] - (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) - - # set default test time - if "during_time" not in param: - during_time = 100 # seconds - else: - during_time = int(param["during_time"]) * 60 - # set default query process num - if "query_process_num" not in param: - query_process_num = 10 - else: - query_process_num = int(param["query_process_num"]) - milvus = MilvusClient(table_name, ip=self.ip, port=self.port) - logger.debug(milvus.show_tables()) - logger.debug(milvus.describe_index()) - logger.debug(milvus.count()) - # Check has table or not - if not milvus.exists_table(): - logger.warning("Table %s not existed, continue exec next params ..." % table_name) - continue - - start_time = time.time() - insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(nq)] - i = 0 - while time.time() < start_time + during_time: - # processes = [] - # # do query - # for i in range(query_process_num): - # milvus_instance = MilvusClient(table_name) - # top_k = random.choice([x for x in range(1, 100)]) - # nq = random.choice([x for x in range(1, 1000)]) - # nprobe = random.choice([x for x in range(1, 500)]) - # logger.info(nprobe) - # p = Process(target=self.do_query, args=(milvus_instance, table_name, [top_k], [nq], 64, run_count, )) - # processes.append(p) - # p.start() - # time.sleep(0.1) - # for p in processes: - # p.join() - i = i + 1 - milvus_instance = MilvusClient(table_name, ip=self.ip, port=self.port) - top_ks = random.sample([x for x in range(1, 100)], 1) - nqs = random.sample([x for x in range(1, 200)], 2) - nprobe = random.choice([x for x in range(1, 100)]) - res = self.do_query(milvus_instance, table_name, top_ks, nqs, nprobe, run_count) - # milvus_instance = MilvusClient(table_name) - status, res = milvus_instance.insert(insert_vectors, ids=[x for x in range(len(insert_vectors))]) - if not status.OK(): - logger.error(status.message) - logger.debug(milvus.count()) - res = self.do_query(milvus_instance, table_name, top_ks, nqs, nprobe, run_count) - # status = milvus_instance.create_index(index_type, 16384) + else: + logger.warning("Run type not defined") + return + logger.debug("Test finished") diff --git a/tests/milvus_benchmark/main.py b/tests/milvus_benchmark/main.py index 0af4a9ee81..7738829785 100644 --- a/tests/milvus_benchmark/main.py +++ b/tests/milvus_benchmark/main.py @@ -5,26 +5,19 @@ import pdb import argparse import logging import traceback +from multiprocessing import Process +from queue import Queue from logging import handlers - from yaml import full_load, dump -from parser import operations_parser from local_runner import LocalRunner from docker_runner import DockerRunner from k8s_runner import K8sRunner +import parser DEFAULT_IMAGE = "milvusdb/milvus:latest" LOG_FOLDER = "logs" +NAMESPACE = "milvus" -# formatter = logging.Formatter('[%(asctime)s] [%(levelname)-4s] [%(pathname)s:%(lineno)d] %(message)s') -# if not os.path.exists(LOG_FOLDER): -# os.system('mkdir -p %s' % LOG_FOLDER) -# fileTimeHandler = handlers.TimedRotatingFileHandler(os.path.join(LOG_FOLDER, 'milvus_benchmark'), "D", 1, 10) -# fileTimeHandler.suffix = "%Y%m%d.log" -# fileTimeHandler.setFormatter(formatter) -# logging.basicConfig(level=logging.DEBUG) -# fileTimeHandler.setFormatter(formatter) -# logger.addHandler(fileTimeHandler) logging.basicConfig(format='%(asctime)s,%(msecs)d %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s', datefmt='%Y-%m-%d:%H:%M:%S', level=logging.DEBUG) @@ -42,96 +35,36 @@ def positive_int(s): return i -# # link random_data if not exists -# def init_env(): -# if not os.path.islink(BINARY_DATA_FOLDER): -# try: -# os.symlink(SRC_BINARY_DATA_FOLDER, BINARY_DATA_FOLDER) -# except Exception as e: -# logger.error("Create link failed: %s" % str(e)) -# sys.exit() +def get_image_tag(image_version, image_type): + return "%s-%s-centos7-release" % (image_version, image_type) + # return "%s-%s-centos7-release" % ("0.7.1", image_type) + # return "%s-%s-centos7-release" % ("PR-2159", image_type) -def main(): - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument( - "--hostname", - default="eros", - help="server host name") - parser.add_argument( - "--image-tag", - default="", - help="image tag") - parser.add_argument( - "--image-type", - default="", - help="image type") - # parser.add_argument( - # "--run-count", - # default=1, - # type=positive_int, - # help="run times for each test") - # # performance / stability / accuracy test - # parser.add_argument( - # "--run-type", - # default="search_performance", - # help="run type, default performance") - parser.add_argument( - '--suite', - metavar='FILE', - help='load test suite from FILE', - default='suites/suite.yaml') - parser.add_argument( - '--local', - action='store_true', - help='use local milvus server') - parser.add_argument( - '--host', - help='server host ip param for local mode', - default='127.0.0.1') - parser.add_argument( - '--port', - help='server port param for local mode', - default='19530') +def queue_worker(queue): + while not queue.empty(): + q = queue.get() + suite = q["suite"] + server_host = q["server_host"] + image_type = q["image_type"] + image_tag = q["image_tag"] - args = parser.parse_args() - - # Get all benchmark test suites - if args.suite: - with open(args.suite) as f: + with open(suite) as f: suite_dict = full_load(f) f.close() - # With definition order - run_type, run_params = operations_parser(suite_dict) + logger.debug(suite_dict) - # init_env() - # run_params = {"run_count": args.run_count} - - if args.image_tag: - namespace = "milvus" - logger.debug(args) - # for docker mode - if args.local: - logger.error("Local mode and docker mode are incompatible") - sys.exit(-1) - # Docker pull image - # if not utils.pull_image(args.image): - # raise Exception('Image %s pull failed' % image) - # TODO: Check milvus server port is available - # logger.info("Init: remove all containers created with image: %s" % args.image) - # utils.remove_all_containers(args.image) - # runner = DockerRunner(args) - tables = run_params["tables"] - for table in tables: + run_type, run_params = parser.operations_parser(suite_dict) + collections = run_params["collections"] + for collection in collections: # run tests - server_config = table["server"] + server_config = collection["server"] logger.debug(server_config) runner = K8sRunner() - if runner.init_env(server_config, args): + if runner.init_env(server_config, server_host, image_type, image_tag): logger.debug("Start run tests") try: - runner.run(run_type, table) + runner.run(run_type, collection) except Exception as e: logger.error(str(e)) logger.error(traceback.format_exc()) @@ -139,24 +72,101 @@ def main(): runner.clean_up() else: logger.error("Runner init failed") - # for operation_type in operations: - # logger.info("Start run test, test type: %s" % operation_type) - # run_params["params"] = operations[operation_type] - # runner.run({operation_type: run_params}, run_type=args.run_type) - # logger.info("Run params: %s" % str(run_params)) + logger.debug("All task finished in queue: %s" % server_host) - if args.local: + +def main(): + arg_parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + # helm mode with scheduler + arg_parser.add_argument( + "--image-version", + default="", + help="image version") + arg_parser.add_argument( + "--schedule-conf", + metavar='FILE', + default='', + help="load test schedule from FILE") + + # local mode + arg_parser.add_argument( + '--local', + action='store_true', + help='use local milvus server') + arg_parser.add_argument( + '--host', + help='server host ip param for local mode', + default='127.0.0.1') + arg_parser.add_argument( + '--port', + help='server port param for local mode', + default='19530') + arg_parser.add_argument( + '--suite', + metavar='FILE', + help='load test suite from FILE', + default='') + + args = arg_parser.parse_args() + + if args.schedule_conf: + if args.local: + raise Exception("Helm mode with scheduler and other mode are incompatible") + if not args.image_version: + raise Exception("Image version not given") + image_version = args.image_version + with open(args.schedule_conf) as f: + schedule_config = full_load(f) + f.close() + queues = [] + server_names = set() + for item in schedule_config: + server_host = item["server"] + suite_params = item["suite_params"] + server_names.add(server_host) + q = Queue() + for suite_param in suite_params: + suite = "suites/"+suite_param["suite"] + image_type = suite_param["image_type"] + image_tag = get_image_tag(image_version, image_type) + q.put({ + "suite": suite, + "server_host": server_host, + "image_tag": image_tag, + "image_type": image_type + }) + queues.append(q) + logger.debug(server_names) + thread_num = len(server_names) + processes = [] + + for i in range(thread_num): + x = Process(target=queue_worker, args=(queues[i], )) + processes.append(x) + x.start() + time.sleep(5) + for x in processes: + x.join() + + elif args.local: # for local mode host = args.host port = args.port - + suite = args.suite + with open(suite) as f: + suite_dict = full_load(f) + f.close() + logger.debug(suite_dict) + run_type, run_params = parser.operations_parser(suite_dict) + collections = run_params["collections"] + if len(collections) > 1: + raise Exception("Multi collections not supported in Local Mode") + collection = collections[0] runner = LocalRunner(host, port) - for operation_type in operations: - logger.info("Start run local mode test, test type: %s" % operation_type) - run_params["params"] = operations[operation_type] - runner.run({operation_type: run_params}, run_type=args.run_type) - logger.info("Run params: %s" % str(run_params)) + logger.info("Start run local mode test, test type: %s" % run_type) + runner.run(run_type, collection) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/tests/milvus_benchmark/parser.py b/tests/milvus_benchmark/parser.py index 4727071e9b..1615e72267 100644 --- a/tests/milvus_benchmark/parser.py +++ b/tests/milvus_benchmark/parser.py @@ -12,29 +12,29 @@ def operations_parser(operations): return (run_type, run_params) -def table_parser(table_name): - tmp = table_name.split("_") +def collection_parser(collection_name): + tmp = collection_name.split("_") # if len(tmp) != 5: # return None data_type = tmp[0] - table_size_unit = tmp[1][-1] - table_size = tmp[1][0:-1] - if table_size_unit == "m": - table_size = int(table_size) * 1000000 - elif table_size_unit == "b": - table_size = int(table_size) * 1000000000 + collection_size_unit = tmp[1][-1] + collection_size = tmp[1][0:-1] + if collection_size_unit == "m": + collection_size = int(collection_size) * 1000000 + elif collection_size_unit == "b": + collection_size = int(collection_size) * 1000000000 index_file_size = int(tmp[2]) dimension = int(tmp[3]) metric_type = str(tmp[4]) - return (data_type, table_size, index_file_size, dimension, metric_type) + return (data_type, collection_size, index_file_size, dimension, metric_type) -def parse_ann_table_name(table_name): - data_type = table_name.split("_")[0] - dimension = int(table_name.split("_")[1]) - metric = table_name.split("_")[-1] - # metric = table_name.attrs['distance'] - # dimension = len(table_name["train"][0]) +def parse_ann_collection_name(collection_name): + data_type = collection_name.split("_")[0] + dimension = int(collection_name.split("_")[1]) + metric = collection_name.split("_")[-1] + # metric = collection_name.attrs['distance'] + # dimension = len(collection_name["train"][0]) if metric == "euclidean": metric_type = "l2" elif metric == "angular": diff --git a/tests/milvus_benchmark/requirements.txt b/tests/milvus_benchmark/requirements.txt index cd459f296d..a9ab1ee045 100644 --- a/tests/milvus_benchmark/requirements.txt +++ b/tests/milvus_benchmark/requirements.txt @@ -1,5 +1,5 @@ -numpy==1.16.3 pymilvus-test>=0.2.0 +scipy==1.3.1 scikit-learn==0.19.1 h5py==2.7.1 # influxdb==5.2.2 @@ -8,3 +8,4 @@ tableprint==0.8.0 ansicolors==1.1.8 scipy==1.3.1 kubernetes==10.0.1 +# rq==1.2.0 diff --git a/tests/milvus_benchmark/runner.py b/tests/milvus_benchmark/runner.py index 8d1cc61778..3be7d712f9 100644 --- a/tests/milvus_benchmark/runner.py +++ b/tests/milvus_benchmark/runner.py @@ -4,6 +4,7 @@ import pdb import time import random from multiprocessing import Process +from itertools import product import numpy as np import sklearn.preprocessing from client import MilvusClient @@ -25,6 +26,7 @@ SIFT_SRC_DATA_DIR = '/test/milvus/raw_data/sift1b/' DEEP_SRC_DATA_DIR = '/test/milvus/raw_data/deep1b/' JACCARD_SRC_DATA_DIR = '/test/milvus/raw_data/jaccard/' HAMMING_SRC_DATA_DIR = '/test/milvus/raw_data/jaccard/' +STRUCTURE_SRC_DATA_DIR = '/test/milvus/raw_data/jaccard/' SIFT_SRC_GROUNDTRUTH_DATA_DIR = SIFT_SRC_DATA_DIR + 'gnd' WARM_TOP_K = 1 @@ -59,6 +61,8 @@ def gen_file_name(idx, dimension, data_type): fname = JACCARD_SRC_DATA_DIR+fname elif data_type == "hamming": fname = HAMMING_SRC_DATA_DIR+fname + elif data_type == "sub" or data_type == "super": + fname = STRUCTURE_SRC_DATA_DIR+fname return fname @@ -76,6 +80,8 @@ def get_vectors_from_binary(nq, dimension, data_type): file_name = JACCARD_SRC_DATA_DIR+'query.npy' elif data_type == "hamming": file_name = HAMMING_SRC_DATA_DIR+'query.npy' + elif data_type == "sub" or data_type == "super": + file_name = STRUCTURE_SRC_DATA_DIR+'query.npy' data = np.load(file_name) vectors = data[0:nq].tolist() return vectors @@ -92,7 +98,7 @@ class Runner(object): X = X.astype(np.float32) elif metric_type == "l2": X = X.astype(np.float32) - elif metric_type == "jaccard" or metric_type == "hamming": + elif metric_type in ["jaccard", "hamming", "sub", "super"]: tmp = [] for index, item in enumerate(X): new_vector = bytes(np.packbits(item, axis=-1).tolist()) @@ -100,11 +106,26 @@ class Runner(object): X = tmp return X - def do_insert(self, milvus, table_name, data_type, dimension, size, ni): + def generate_combinations(self, args): + if isinstance(args, list): + args = [el if isinstance(el, list) else [el] for el in args] + return [list(x) for x in product(*args)] + elif isinstance(args, dict): + flat = [] + for k, v in args.items(): + if isinstance(v, list): + flat.append([(k, el) for el in v]) + else: + flat.append([(k, v)]) + return [dict(x) for x in product(*flat)] + else: + raise TypeError("No args handling exists for %s" % type(args).__name__) + + def do_insert(self, milvus, collection_name, data_type, dimension, size, ni): ''' @params: mivlus: server connect instance - dimension: table dimensionn + dimension: collection dimensionn # index_file_size: size trigger file merge size: row count of vectors to be insert ni: row count of vectors to be insert each time @@ -127,12 +148,12 @@ class Runner(object): vectors_per_file = 10000 elif data_type == "sift": vectors_per_file = SIFT_VECTORS_PER_FILE - elif data_type == "jaccard" or data_type == "hamming": + elif data_type in ["jaccard", "hamming", "sub", "super"]: vectors_per_file = JACCARD_VECTORS_PER_FILE else: raise Exception("data_type: %s not supported" % data_type) if size % vectors_per_file or ni > vectors_per_file: - raise Exception("Not invalid table size or ni") + raise Exception("Not invalid collection size or ni") file_num = size // vectors_per_file for i in range(file_num): file_name = gen_file_name(i, dimension, data_type) @@ -150,6 +171,8 @@ class Runner(object): logger.info("Start id: %s, end id: %s" % (start_id, end_id)) ids = [k for k in range(start_id, end_id)] status, ids = milvus.insert(vectors, ids=ids) + # milvus.flush() + logger.debug(milvus.count()) ni_end_time = time.time() total_time = total_time + ni_end_time - ni_start_time @@ -160,9 +183,9 @@ class Runner(object): bi_res["ni_time"] = ni_time return bi_res - def do_query(self, milvus, table_name, top_ks, nqs, nprobe, run_count=1): + def do_query(self, milvus, collection_name, top_ks, nqs, run_count=1, search_param=None): bi_res = [] - (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) base_query_vectors = get_vectors_from_binary(MAX_NQ, dimension, data_type) for nq in nqs: tmp_res = [] @@ -174,7 +197,7 @@ class Runner(object): for i in range(run_count): logger.info("Start run query, run %d of %s" % (i+1, run_count)) start_time = time.time() - query_res = milvus.query(vectors, top_k, nprobe) + query_res = milvus.query(vectors, top_k, search_param=search_param) interval_time = time.time() - start_time if (i == 0) or (min_query_time > interval_time): min_query_time = interval_time @@ -183,12 +206,18 @@ class Runner(object): bi_res.append(tmp_res) return bi_res - def do_query_ids(self, milvus, table_name, top_k, nq, nprobe): - (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) + def do_query_qps(self, milvus, query_vectors, top_k, search_param): + start_time = time.time() + result = milvus.query(query_vectors, top_k, search_param) + end_time = time.time() + return end_time - start_time + + def do_query_ids(self, milvus, collection_name, top_k, nq, search_param=None): + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) base_query_vectors = get_vectors_from_binary(MAX_NQ, dimension, data_type) vectors = base_query_vectors[0:nq] logger.info("Start query, query params: top-k: {}, nq: {}, actually length of vectors: {}".format(top_k, nq, len(vectors))) - query_res = milvus.query(vectors, top_k, nprobe) + query_res = milvus.query(vectors, top_k, search_param=search_param) result_ids = [] result_distances = [] for result in query_res: @@ -201,12 +230,12 @@ class Runner(object): result_distances.append(tmp_distance) return result_ids, result_distances - def do_query_acc(self, milvus, table_name, top_k, nq, nprobe, id_store_name): - (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) + def do_query_acc(self, milvus, collection_name, top_k, nq, id_store_name, search_param=None): + (data_type, collection_size, index_file_size, dimension, metric_type) = parser.collection_parser(collection_name) base_query_vectors = get_vectors_from_binary(MAX_NQ, dimension, data_type) vectors = base_query_vectors[0:nq] logger.info("Start query, query params: top-k: {}, nq: {}, actually length of vectors: {}".format(top_k, nq, len(vectors))) - query_res = milvus.query(vectors, top_k, nprobe) + query_res = milvus.query(vectors, top_k, search_param=None) # if file existed, cover it if os.path.isfile(id_store_name): os.remove(id_store_name) @@ -250,10 +279,10 @@ class Runner(object): Implementation based on: https://github.com/facebookresearch/faiss/blob/master/benchs/datasets.py """ - def get_groundtruth_ids(self, table_size): - fname = GROUNDTRUTH_MAP[str(table_size)] + def get_groundtruth_ids(self, collection_size): + fname = GROUNDTRUTH_MAP[str(collection_size)] fname = SIFT_SRC_GROUNDTRUTH_DATA_DIR + "/" + fname a = np.fromfile(fname, dtype='int32') d = a[0] true_ids = a.reshape(-1, d + 1)[:, 1:].copy() - return true_ids \ No newline at end of file + return true_ids diff --git a/tests/milvus_benchmark/scheduler/070.json b/tests/milvus_benchmark/scheduler/070.json new file mode 100644 index 0000000000..6b76ddc4cd --- /dev/null +++ b/tests/milvus_benchmark/scheduler/070.json @@ -0,0 +1,53 @@ +[ + { + "server": "apollo", + "suite_params": [ + { + "suite": "cpu_accuracy_ann.yaml", + "image_type": "cpu" + } + ] + }, + { + "server": "poseidon", + "suite_params": [ + { + "suite": "gpu_search_performance.yaml", + "image_type": "gpu" + }, + { + "suite": "cpu_search_performance.yaml", + "image_type": "cpu" + }, + { + "suite": "insert_performance.yaml", + "image_type": "gpu" + }, + { + "suite": "gpu_accuracy.yaml", + "image_type": "gpu" + } + ] + }, + { + "server": "eros", + "suite_params": [ + { + "suite": "gpu_accuracy_ann.yaml", + "image_type": "gpu" + }, + { + "suite": "gpu_search_stability.yaml", + "image_type": "gpu" + }, + { + "suite": "gpu_build_performance.yaml", + "image_type": "gpu" + }, + { + "suite": "cpu_build_performance.yaml", + "image_type": "cpu" + } + ] + } +] \ No newline at end of file diff --git a/tests/milvus_benchmark/scheduler/070_cpu.json b/tests/milvus_benchmark/scheduler/070_cpu.json new file mode 100644 index 0000000000..4b7fdc6fb4 --- /dev/null +++ b/tests/milvus_benchmark/scheduler/070_cpu.json @@ -0,0 +1,29 @@ +[ + { + "server": "apollo", + "suite_params": [ + { + "suite": "cpu_accuracy_ann.yaml", + "image_type": "cpu" + } + ] + }, + { + "server": "poseidon", + "suite_params": [ + { + "suite": "cpu_search_performance.yaml", + "image_type": "cpu" + } + ] + }, + { + "server": "eros", + "suite_params": [ + { + "suite": "cpu_build_performance.yaml", + "image_type": "cpu" + } + ] + } +] diff --git a/tests/milvus_benchmark/scheduler/070_data.json b/tests/milvus_benchmark/scheduler/070_data.json new file mode 100644 index 0000000000..db930c72d3 --- /dev/null +++ b/tests/milvus_benchmark/scheduler/070_data.json @@ -0,0 +1,58 @@ +[ + { + "server": "athena", + "suite_params": [ + { + "suite": "gpu_accuracy.yaml", + "image_type": "gpu" + } + ] + }, + { + "server": "poseidon", + "suite_params": [ + { + "suite": "070_gpu_search.yaml", + "image_type": "gpu" + }, + { + "suite": "070_cpu_search.yaml", + "image_type": "cpu" + }, + { + "suite": "070_gpu_build.yaml", + "image_type": "gpu" + }, + { + "suite": "cpu_accuracy.yaml", + "image_type": "cpu" + } + ] + }, + { + "server": "apollo", + "suite_params": [ + { + "suite": "cpu_accuracy_ann.yaml", + "image_type": "cpu" + }, + { + "suite": "070_cpu_search_stability.yaml", + "image_type": "cpu" + } + ] + }, + { + "server": "eros", + "suite_params": [ + { + "suite": "070_gpu_stability.yaml", + "image_type": "gpu" + }, + { + "suite": "gpu_accuracy_ann.yaml", + "image_type": "gpu" + } + ] + } +] diff --git a/tests/milvus_benchmark/scheduler/070_data_eros.json b/tests/milvus_benchmark/scheduler/070_data_eros.json new file mode 100644 index 0000000000..ad94e2c676 --- /dev/null +++ b/tests/milvus_benchmark/scheduler/070_data_eros.json @@ -0,0 +1,11 @@ +[ + { + "server": "poseidon", + "suite_params": [ + { + "suite": "gpu_accuracy_ann.yaml", + "image_type": "gpu" + } + ] + } +] diff --git a/tests/milvus_benchmark/scheduler/080_data.json b/tests/milvus_benchmark/scheduler/080_data.json new file mode 100644 index 0000000000..3c375aee62 --- /dev/null +++ b/tests/milvus_benchmark/scheduler/080_data.json @@ -0,0 +1,63 @@ +[ + { + "server": "athena", + "suite_params": [ + { + "suite": "080_gpu_accuracy.yaml", + "image_type": "gpu" + } + ] + }, + { + "server": "poseidon", + "suite_params": [ + { + "suite": "080_gpu_search.yaml", + "image_type": "gpu" + }, + { + "suite": "080_cpu_search.yaml", + "image_type": "cpu" + }, + { + "suite": "080_gpu_build.yaml", + "image_type": "gpu" + }, + { + "suite": "080_cpu_accuracy.yaml", + "image_type": "cpu" + }, + { + "suite": "080_cpu_build.yaml", + "image_type": "cpu" + } + ] + }, + { + "server": "apollo", + "suite_params": [ + { + "suite": "cpu_accuracy_ann.yaml", + "image_type": "cpu" + }, + { + "suite": "080_cpu_search_stability.yaml", + "image_type": "cpu" + } + ] + }, + { + "server": "eros", + "suite_params": [ + { + "suite": "gpu_accuracy_ann.yaml", + "image_type": "gpu" + }, + { + "suite": "080_gpu_stability.yaml", + "image_type": "gpu" + } + ] + } + +] diff --git a/tests/milvus_benchmark/scheduler/acc.json b/tests/milvus_benchmark/scheduler/acc.json new file mode 100644 index 0000000000..e4d47bd947 --- /dev/null +++ b/tests/milvus_benchmark/scheduler/acc.json @@ -0,0 +1,15 @@ +[ + { + "server": "poseidon", + "suite_params": [ + { + "suite": "crud_add.yaml", + "image_type": "gpu" + }, + { + "suite": "gpu_accuracy_sift1m.yaml", + "image_type": "gpu" + } + ] + } +] diff --git a/tests/milvus_benchmark/scheduler/ann_acc.json b/tests/milvus_benchmark/scheduler/ann_acc.json new file mode 100644 index 0000000000..32cc93604c --- /dev/null +++ b/tests/milvus_benchmark/scheduler/ann_acc.json @@ -0,0 +1,11 @@ +[ + { + "server": "eros", + "suite_params": [ + { + "suite": "070_ann.yaml", + "image_type": "gpu" + } + ] + } +] diff --git a/tests/milvus_benchmark/scheduler/apollo.json b/tests/milvus_benchmark/scheduler/apollo.json new file mode 100644 index 0000000000..6d2c529c1c --- /dev/null +++ b/tests/milvus_benchmark/scheduler/apollo.json @@ -0,0 +1,15 @@ +[ + { + "server": "apollo", + "suite_params": [ + { + "suite": "070_insert_10m.yaml", + "image_type": "cpu" + }, + { + "suite": "070_cpu_build.yaml", + "image_type": "cpu" + } + ] + } +] diff --git a/tests/milvus_benchmark/scheduler/athena.json b/tests/milvus_benchmark/scheduler/athena.json new file mode 100644 index 0000000000..c9b3b47eb8 --- /dev/null +++ b/tests/milvus_benchmark/scheduler/athena.json @@ -0,0 +1,11 @@ +[ + { + "server": "athena", + "suite_params": [ + { + "suite": "add_flush_performance.yaml", + "image_type": "cpu" + } + ] + } +] diff --git a/tests/milvus_benchmark/scheduler/build.json b/tests/milvus_benchmark/scheduler/build.json new file mode 100644 index 0000000000..d5961c2ed4 --- /dev/null +++ b/tests/milvus_benchmark/scheduler/build.json @@ -0,0 +1,11 @@ +[ + { + "server": "athena", + "suite_params": [ + { + "suite": "debug_build.yaml", + "image_type": "gpu" + } + ] + } +] diff --git a/tests/milvus_benchmark/scheduler/clean.json b/tests/milvus_benchmark/scheduler/clean.json new file mode 100644 index 0000000000..be9cbbe231 --- /dev/null +++ b/tests/milvus_benchmark/scheduler/clean.json @@ -0,0 +1,11 @@ +[ + { + "server": "poseidon", + "suite_params": [ + { + "suite": "clean.yaml", + "image_type": "cpu" + } + ] + } +] diff --git a/tests/milvus_benchmark/scheduler/crud_add.json b/tests/milvus_benchmark/scheduler/crud_add.json new file mode 100644 index 0000000000..485bf23e66 --- /dev/null +++ b/tests/milvus_benchmark/scheduler/crud_add.json @@ -0,0 +1,11 @@ +[ + { + "server": "poseidon", + "suite_params": [ + { + "suite": "crud_add.yaml", + "image_type": "cpu" + } + ] + } +] diff --git a/tests/milvus_benchmark/scheduler/crud_build.json b/tests/milvus_benchmark/scheduler/crud_build.json new file mode 100644 index 0000000000..69efcc81be --- /dev/null +++ b/tests/milvus_benchmark/scheduler/crud_build.json @@ -0,0 +1,12 @@ +[ + { + "server": "poseidon", + "suite_params": [ + { + "suite": "070_gpu_build.yaml", + "image_type": "gpu" + } + ] + } +] + diff --git a/tests/milvus_benchmark/scheduler/crud_flush.json b/tests/milvus_benchmark/scheduler/crud_flush.json new file mode 100644 index 0000000000..44c52eadeb --- /dev/null +++ b/tests/milvus_benchmark/scheduler/crud_flush.json @@ -0,0 +1,11 @@ +[ + { + "server": "apollo", + "suite_params": [ + { + "suite": "crud_add_flush.yaml", + "image_type": "cpu" + } + ] + } +] diff --git a/tests/milvus_benchmark/scheduler/crud_search.json b/tests/milvus_benchmark/scheduler/crud_search.json new file mode 100644 index 0000000000..ccb95a7d28 --- /dev/null +++ b/tests/milvus_benchmark/scheduler/crud_search.json @@ -0,0 +1,11 @@ +[ + { + "server": "athena", + "suite_params": [ + { + "suite": "crud_search.yaml", + "image_type": "gpu" + } + ] + } +] diff --git a/tests/milvus_benchmark/scheduler/crud_stability.json b/tests/milvus_benchmark/scheduler/crud_stability.json new file mode 100644 index 0000000000..74decfae4e --- /dev/null +++ b/tests/milvus_benchmark/scheduler/crud_stability.json @@ -0,0 +1,11 @@ +[ + { + "server": "eros", + "suite_params": [ + { + "suite": "070_stability.yaml", + "image_type": "gpu" + } + ] + } +] diff --git a/tests/milvus_benchmark/scheduler/debug.json b/tests/milvus_benchmark/scheduler/debug.json new file mode 100644 index 0000000000..27ed229fd9 --- /dev/null +++ b/tests/milvus_benchmark/scheduler/debug.json @@ -0,0 +1,11 @@ +[ + { + "server": "poseidon", + "suite_params": [ + { + "suite": "debug.yaml", + "image_type": "gpu" + } + ] + } +] diff --git a/tests/milvus_benchmark/scheduler/default_config.json b/tests/milvus_benchmark/scheduler/default_config.json new file mode 100644 index 0000000000..6b76ddc4cd --- /dev/null +++ b/tests/milvus_benchmark/scheduler/default_config.json @@ -0,0 +1,53 @@ +[ + { + "server": "apollo", + "suite_params": [ + { + "suite": "cpu_accuracy_ann.yaml", + "image_type": "cpu" + } + ] + }, + { + "server": "poseidon", + "suite_params": [ + { + "suite": "gpu_search_performance.yaml", + "image_type": "gpu" + }, + { + "suite": "cpu_search_performance.yaml", + "image_type": "cpu" + }, + { + "suite": "insert_performance.yaml", + "image_type": "gpu" + }, + { + "suite": "gpu_accuracy.yaml", + "image_type": "gpu" + } + ] + }, + { + "server": "eros", + "suite_params": [ + { + "suite": "gpu_accuracy_ann.yaml", + "image_type": "gpu" + }, + { + "suite": "gpu_search_stability.yaml", + "image_type": "gpu" + }, + { + "suite": "gpu_build_performance.yaml", + "image_type": "gpu" + }, + { + "suite": "cpu_build_performance.yaml", + "image_type": "cpu" + } + ] + } +] \ No newline at end of file diff --git a/tests/milvus_benchmark/scheduler/eros.json b/tests/milvus_benchmark/scheduler/eros.json new file mode 100644 index 0000000000..58ccc78a33 --- /dev/null +++ b/tests/milvus_benchmark/scheduler/eros.json @@ -0,0 +1,11 @@ +[ + { + "server": "eros", + "suite_params": [ + { + "suite": "080_gpu_search_id.yaml", + "image_type": "gpu" + } + ] + } +] diff --git a/tests/milvus_benchmark/scheduler/file_size.json b/tests/milvus_benchmark/scheduler/file_size.json new file mode 100644 index 0000000000..f8da372e46 --- /dev/null +++ b/tests/milvus_benchmark/scheduler/file_size.json @@ -0,0 +1,11 @@ +[ + { + "server": "athena", + "suite_params": [ + { + "suite": "file_size.yaml", + "image_type": "cpu" + } + ] + } + ] diff --git a/tests/milvus_benchmark/scheduler/hnsw.json b/tests/milvus_benchmark/scheduler/hnsw.json new file mode 100644 index 0000000000..cdd8bd6db1 --- /dev/null +++ b/tests/milvus_benchmark/scheduler/hnsw.json @@ -0,0 +1,11 @@ +[ + { + "server": "poseidon", + "suite_params": [ + { + "suite": "cpu_accuracy_ann_crud_debug.yaml", + "image_type": "cpu" + } + ] + } +] diff --git a/tests/milvus_benchmark/scheduler/insert.json b/tests/milvus_benchmark/scheduler/insert.json new file mode 100644 index 0000000000..c6c23a376e --- /dev/null +++ b/tests/milvus_benchmark/scheduler/insert.json @@ -0,0 +1,11 @@ +[ + { + "server": "poseidon", + "suite_params": [ + { + "suite": "insert_performance.yaml", + "image_type": "gpu" + } + ] + } +] diff --git a/tests/milvus_benchmark/scheduler/jaccard.json b/tests/milvus_benchmark/scheduler/jaccard.json new file mode 100644 index 0000000000..9c73c4f0b5 --- /dev/null +++ b/tests/milvus_benchmark/scheduler/jaccard.json @@ -0,0 +1,11 @@ +[ + { + "server": "poseidon", + "suite_params": [ + { + "suite": "cpu_search_performance_jaccard.yaml", + "image_type": "cpu" + } + ] + } +] diff --git a/tests/milvus_benchmark/scheduler/poseidon.json b/tests/milvus_benchmark/scheduler/poseidon.json new file mode 100644 index 0000000000..1429c77a57 --- /dev/null +++ b/tests/milvus_benchmark/scheduler/poseidon.json @@ -0,0 +1,27 @@ +[ + { + "server": "poseidon", + "suite_params": [ + { + "suite": "080_gpu_search.yaml", + "image_type": "gpu" + }, + { + "suite": "080_cpu_search.yaml", + "image_type": "cpu" + }, + { + "suite": "080_gpu_build.yaml", + "image_type": "gpu" + }, + { + "suite": "080_cpu_accuracy.yaml", + "image_type": "cpu" + }, + { + "suite": "080_cpu_build.yaml", + "image_type": "cpu" + } + ] + } +] diff --git a/tests/milvus_benchmark/scheduler/search.json b/tests/milvus_benchmark/scheduler/search.json new file mode 100644 index 0000000000..8fb2fef45a --- /dev/null +++ b/tests/milvus_benchmark/scheduler/search.json @@ -0,0 +1,11 @@ +[ + { + "server": "eros", + "suite_params": [ + { + "suite": "debug.yaml", + "image_type": "cpu" + } + ] + } +] diff --git a/tests/milvus_benchmark/scheduler/search_performance.json b/tests/milvus_benchmark/scheduler/search_performance.json new file mode 100644 index 0000000000..826818e08f --- /dev/null +++ b/tests/milvus_benchmark/scheduler/search_performance.json @@ -0,0 +1,11 @@ +[ + { + "server": "athena", + "suite_params": [ + { + "suite": "070_gpu_search.yaml", + "image_type": "gpu" + } + ] + } +] diff --git a/tests/milvus_benchmark/scheduler/sift1b.json b/tests/milvus_benchmark/scheduler/sift1b.json new file mode 100644 index 0000000000..d79802f278 --- /dev/null +++ b/tests/milvus_benchmark/scheduler/sift1b.json @@ -0,0 +1,11 @@ +[ + { + "server": "athena", + "suite_params": [ + { + "suite": "insert_performance_sift1b.yaml", + "image_type": "gpu" + } + ] + } +] diff --git a/tests/milvus_benchmark/scheduler/stability.json b/tests/milvus_benchmark/scheduler/stability.json new file mode 100644 index 0000000000..497a161200 --- /dev/null +++ b/tests/milvus_benchmark/scheduler/stability.json @@ -0,0 +1,11 @@ +[ + { + "server": "eros", + "suite_params": [ + { + "suite": "gpu_search_stability.yaml", + "image_type": "gpu" + } + ] + } +] diff --git a/tests/milvus_benchmark/suites/accuracy_random50m.yaml b/tests/milvus_benchmark/suites/accuracy_random50m.yaml deleted file mode 100644 index d84c9e301c..0000000000 --- a/tests/milvus_benchmark/suites/accuracy_random50m.yaml +++ /dev/null @@ -1,57 +0,0 @@ - -accuracy: - # interface: search_vectors - query: - [ - { - "dataset": "random_50m_1024_512_ip", - "index.index_types": ["flat", "ivf_flat"], - "index.nlists": [16384], - "index.metric_types": ["ip"], - "nprobes": [8, 64, 128, 512, 2048], - "top_ks": [64], - "nqs": [1000], - "server.use_blas_threshold": 1100, - "server.use_gpu_threshold": 100, - "server.cpu_cache_capacity": 120, - "server.gpu_cache_capacity": 6, - "server.enable_gpu": True, - "server.resources": ["gpu0", "gpu1"], - "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip_ivf", - "sift_acc": false - }, - { - "dataset": "random_50m_1024_512_ip", - "index.index_types": ["flat", "ivf_sq8"], - "index.nlists": [16384], - "index.metric_types": ["ip"], - "nprobes": [8, 64, 128, 512, 2048], - "top_ks": [64], - "nqs": [1000], - "server.use_blas_threshold": 1100, - "server.use_gpu_threshold": 100, - "server.cpu_cache_capacity": 120, - "server.gpu_cache_capacity": 6, - "server.enable_gpu": True, - "server.resources": ["gpu0", "gpu1"], - "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip_sq8", - "sift_acc": false - }, - { - "dataset": "random_50m_1024_512_ip", - "index.index_types": ["flat", "ivf_sq8h"], - "index.nlists": [16384], - "index.metric_types": ["ip"], - "nprobes": [8, 64, 128, 512, 2048], - "top_ks": [64], - "nqs": [1000], - "server.use_blas_threshold": 1100, - "server.use_gpu_threshold": 100, - "server.cpu_cache_capacity": 120, - "server.gpu_cache_capacity": 6, - "server.enable_gpu": True, - "server.resources": ["gpu0", "gpu1"], - "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip", - "sift_acc": false - }, - ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/accuracy_sift10m.yaml b/tests/milvus_benchmark/suites/accuracy_sift10m.yaml deleted file mode 100644 index 35074868f1..0000000000 --- a/tests/milvus_benchmark/suites/accuracy_sift10m.yaml +++ /dev/null @@ -1,75 +0,0 @@ -accuracy: - - # interface: search_vectors - query: - # dataset: table name you have already created - # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity .. - [ - # { - # "dataset": "sift_10m_1024_128_l2", - # "index.index_types": ["ivf_sq8h"], - # "index.nlists": [16384], - # "index.metric_types": ["l2"], - # "nprobes": [1, 8, 16, 32, 64, 128, 256], - # "top_ks": [64], - # "nqs": [1000], - # "server.cpu_cache_capacity": 50, - # "server.use_gpu_threshold": 1001, - # "server.gpu_cache_capacity": 6, - # "server.resources": ["cpu", "gpu0"], - # "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_sq8h", - # "sift_acc": true - # }, - # { - # "dataset": "sift_10m_1024_128_l2", - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "index.metric_types": ["l2"], - # "nprobes": [1, 8, 16, 32, 64, 128, 256], - # "top_ks": [64], - # "nqs": [1000], - # "server.cpu_cache_capacity": 50, - # "server.resources": ["cpu", "gpu0"], - # "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_sq8", - # "sift_acc": true - # }, - { - "dataset": "sift_10m_1024_128_l2", - "index.index_types": ["ivf_pq"], - "index.nlists": [16384], - "index.metric_types": ["l2"], - "nprobes": [1, 8, 16, 32, 64, 128, 256], - "top_ks": [64], - "nqs": [1000], - "server.cpu_cache_capacity": 50, - "server.resources": ["cpu", "gpu0"], - "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_sq8", - "sift_acc": true - }, - # { - # "dataset": "sift_10m_1024_128_l2", - # "index.index_types": ["ivf_flat"], - # "index.nlists": [16384], - # "index.metric_types": ["l2"], - # "nprobes": [1, 8, 16, 32, 64, 128, 256], - # "top_ks": [64], - # "nqs": [1000], - # "server.cpu_cache_capacity": 100, - # "server.resources": ["cpu", "gpu0"], - # "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_ivf", - # "sift_acc": true - # }, - # { - # "dataset": "sift_10m_1024_128_l2", - # "index.index_types": ["nsg"], - # "index.nlists": [16384], - # "index.metric_types": ["l2"], - # "nprobes": [1], - # "top_ks": [64], - # "nqs": [1000], - # # "server.cpu_cache_capacity": 100, - # # "server.resources": ["cpu", "gpu0"], - # "db_path_prefix": "/test/milvus/db_data_gpu/sift_10m_1024_128_l2_nsg", - # "sift_acc": true - # }, - ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/accuracy_sift1b.yaml b/tests/milvus_benchmark/suites/accuracy_sift1b.yaml deleted file mode 100644 index 7ad6e85da3..0000000000 --- a/tests/milvus_benchmark/suites/accuracy_sift1b.yaml +++ /dev/null @@ -1,73 +0,0 @@ - -accuracy: - # interface: search_vectors - query: - [ - # { - # "dataset": "sift_1b_2048_128_l2", - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "index.metric_types": ["l2"], - # # "nprobes": [1, 8, 32, 64, 128], - # "nprobes": [1, 8, 32, 64, 128, 256], - # "top_ks": [64], - # "nqs": [1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 1001, - # "server.cpu_cache_capacity": 150, - # "server.gpu_cache_capacity": 6, - # "server.resources": ["gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data_gpu/sift_1b_2048_128_l2", - # "sift_acc": true - # }, - { - "dataset": "sift_1b_2048_128_l2", - "index.index_types": ["ivf_pq"], - "index.nlists": [16384], - "index.metric_types": ["l2"], - "nprobes": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512], - "top_ks": [64], - "nqs": [1000], - "server.use_blas_threshold": 1100, - "server.use_gpu_threshold": 10001, - "server.cpu_cache_capacity": 150, - "server.gpu_cache_capacity": 6, - "server.resources": ["gpu0", "gpu1"], - "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2_pq", - "sift_acc": true - }, - - # { - # "dataset": "sift_1b_2048_128_l2", - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "index.metric_types": ["l2"], - # "nprobes": [1, 8, 32, 64, 128], - # "top_ks": [64], - # "nqs": [1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 150, - # "server.gpu_cache_capacity": 6, - # "server.resources": ["gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2", - # "sift_acc": true - # }, - # { - # "dataset": "sift_1b_2048_128_l2", - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "index.metric_types": ["l2"], - # "nprobes": [1, 8, 32, 64, 128], - # "top_ks": [64], - # "nqs": [1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 150, - # "server.gpu_cache_capacity": 6, - # "server.resources": ["cpu", "gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2", - # "sift_acc": true - # }, - - ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/accuracy_sift500m.yaml b/tests/milvus_benchmark/suites/accuracy_sift500m.yaml deleted file mode 100644 index 5977fbbc89..0000000000 --- a/tests/milvus_benchmark/suites/accuracy_sift500m.yaml +++ /dev/null @@ -1,21 +0,0 @@ -accuracy: - - # interface: search_vectors - query: - # dataset: table name you have already created - # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity .. - [ - { - "dataset": "sift_500m_1024_128_l2", - "index.index_types": ["ivf_sq8h"], - "index.nlists": [16384], - "index.metric_types": ["l2"], - "nprobes": [1, 8, 16, 32, 64, 128, 256], - "top_ks": [64], - "nqs": [1000], - "server.cpu_cache_capacity": 200, - "server.resources": ["cpu", "gpu0"], - "db_path_prefix": "/test/milvus/db_data/sift_500m_1024_128_l2", - "sift_acc": true - }, - ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/accuracy_sift50m.yaml b/tests/milvus_benchmark/suites/accuracy_sift50m.yaml deleted file mode 100644 index 1f4dadbb00..0000000000 --- a/tests/milvus_benchmark/suites/accuracy_sift50m.yaml +++ /dev/null @@ -1,76 +0,0 @@ -accuracy: - - # interface: search_vectors - query: - # dataset: table name you have already created - # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity .. - [ - # { - # "dataset": "sift_50m_1024_128_l2", - # "index.index_types": ["ivf_sq8h"], - # "index.nlists": [16384], - # "index.metric_types": ["l2"], - # "nprobes": [1, 8, 16, 32, 64, 128, 256], - # "top_ks": [64], - # "nqs": [1000], - # "server.cpu_cache_capacity": 50, - # "server.use_gpu_threshold": 1001, - # "server.gpu_cache_capacity": 6, - # "server.resources": ["cpu", "gpu0"], - # "db_path_prefix": "/test/milvus/db_data/sift_50m_1024_128_l2_sq8h", - # "sift_acc": true - # }, - { - "dataset": "sift_50m_1024_128_l2", - "index.index_types": ["ivf_sq8"], - "index.nlists": [16384], - "index.metric_types": ["l2"], - "nprobes": [1, 8, 16, 32, 64, 128, 256], - "top_ks": [64], - "nqs": [1000], - "server.cpu_cache_capacity": 50, - "server.resources": ["cpu", "gpu0"], - "db_path_prefix": "/test/milvus/db_data/sift_50m_1024_128_l2_sq8", - "sift_acc": true - }, - # { - # "dataset": "sift_50m_1024_128_l2", - # "index.index_types": ["ivf_flat"], - # "index.nlists": [16384], - # "index.metric_types": ["l2"], - # "nprobes": [1, 8, 16, 32, 64, 128, 256], - # "top_ks": [64], - # "nqs": [1000], - # "server.cpu_cache_capacity": 100, - # "server.resources": ["cpu", "gpu0"], - # "db_path_prefix": "/test/milvus/db_data/sift_50m_1024_128_l2_ivf", - # "sift_acc": true - # }, - # { - # "dataset": "sift_50m_1024_128_l2", - # "index.index_types": ["ivf_flat"], - # "index.nlists": [16384], - # "index.metric_types": ["l2"], - # "nprobes": [1, 8, 16, 32, 64, 128, 256], - # "top_ks": [64], - # "nqs": [1000], - # "server.cpu_cache_capacity": 100, - # "server.resources": ["cpu", "gpu0"], - # "db_path_prefix": "/test/milvus/db_data_gpu/sift_50m_1024_128_l2_ivf", - # "sift_acc": true - # }, - # { - # "dataset": "sift_50m_1024_128_l2", - # "index.index_types": ["nsg"], - # "index.nlists": [16384], - # "index.metric_types": ["l2"], - # "nprobes": [1], - # "top_ks": [64], - # "nqs": [1000], - # "server.cpu_cache_capacity": 100, - # "server.resources": ["gpu0"], - # "db_path_prefix": "/test/milvus/db_data_gpu/sift_50m_1024_128_l2_nsg", - # "sift_acc": true - # }, - - ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/cpu_accuracy.yaml b/tests/milvus_benchmark/suites/cpu_accuracy.yaml deleted file mode 100644 index 2d71da9315..0000000000 --- a/tests/milvus_benchmark/suites/cpu_accuracy.yaml +++ /dev/null @@ -1,119 +0,0 @@ -accuracy: - tables: - # sift1b - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8 - cache_config.cpu_cache_capacity: 150 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_1b_2048_128_l2 - search_params: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - top_ks: [64] - nqs: [1000] - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_pq - cache_config.cpu_cache_capacity: 150 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_1b_2048_128_l2 - search_params: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - top_ks: [64] - nqs: [1000] - - # sift50m - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_ivf - cache_config.cpu_cache_capacity: 60 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_50m_1024_128_l2 - search_params: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - top_ks: [64] - nqs: [1000] - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8 - cache_config.cpu_cache_capacity: 30 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_50m_1024_128_l2 - search_params: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - top_ks: [64] - nqs: [1000] - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_pq - cache_config.cpu_cache_capacity: 30 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_50m_1024_128_l2 - search_params: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - top_ks: [64] - nqs: [1000] - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_nsg - cache_config.cpu_cache_capacity: 100 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_50m_1024_128_l2 - search_params: - nprobes: [1] - top_ks: [64] - nqs: [1000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/cpu_accuracy_ann.yaml b/tests/milvus_benchmark/suites/cpu_accuracy_ann.yaml index 99553f0c9b..21cc7995f5 100644 --- a/tests/milvus_benchmark/suites/cpu_accuracy_ann.yaml +++ b/tests/milvus_benchmark/suites/cpu_accuracy_ann.yaml @@ -1,5 +1,78 @@ ann_accuracy: - tables: + collections: + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_file_sizes: [1024] + index_types: ['flat', 'ivf_flat', 'ivf_sq8'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_file_sizes: [1024] + index_types: ['ivf_pq'] + index_params: + nlist: [16384] + m: [32] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_file_sizes: [1024] + index_types: ['annoy'] + index_params: + n_trees: [8, 32] + top_ks: [10] + nqs: [10000] + search_params: + search_k: [50, 100, 500, 1000] + - server: cache_config.cpu_cache_capacity: 16 @@ -14,42 +87,23 @@ ann_accuracy: - gpu0 - gpu1 source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 - table_name: sift_128_euclidean - index_file_sizes: [1024] - index_types: ['flat', 'ivf_flat', 'ivf_sq8', 'ivf_pq', 'nsg'] - nlists: [16384] + collection_name: sift_128_euclidean + index_file_sizes: [256] + index_types: ['hnsw'] + index_params: + M: [16] + efConstruction: [500] + top_ks: [10] + nqs: [10000] search_params: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - top_ks: [10] - nqs: [10000] + ef: [16, 32, 64, 128, 256, 512] + - server: cache_config.cpu_cache_capacity: 16 engine_config.use_blas_threshold: 1100 engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - source_file: /test/milvus/ann_hdf5/gist-960-euclidean.hdf5 - table_name: gist_960_euclidean - index_file_sizes: [1024] - index_types: ['flat', 'ivf_flat', 'ivf_sq8', 'ivf_pq', 'nsg'] - nlists: [16384] - search_params: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - top_ks: [10] - nqs: [10000] - - - server: - cache_config.cpu_cache_capacity: 16 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: true + gpu_resource_config.enable: false gpu_resource_config.cache_capacity: 4 gpu_resource_config.search_resources: - gpu0 @@ -58,11 +112,62 @@ ann_accuracy: - gpu0 - gpu1 source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 - table_name: glove_200_angular + collection_name: glove_200_angular index_file_sizes: [1024] - index_types: ['flat', 'ivf_flat', 'ivf_sq8', 'ivf_pq', 'nsg'] - nlists: [16384] + index_types: ['flat', 'ivf_flat', 'ivf_sq8'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] search_params: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - top_ks: [10] - nqs: [10000] \ No newline at end of file + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + index_file_sizes: [1024] + index_types: ['ivf_pq'] + index_params: + nlist: [16384] + m: [20] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + index_file_sizes: [256] + index_types: ['hnsw'] + index_params: + M: [36] + efConstruction: [500] + top_ks: [10] + nqs: [10000] + search_params: + ef: [10, 16, 32, 64, 128, 256, 512] diff --git a/tests/milvus_benchmark/suites/cpu_build_performance.yaml b/tests/milvus_benchmark/suites/cpu_build_performance.yaml deleted file mode 100644 index 98f7014375..0000000000 --- a/tests/milvus_benchmark/suites/cpu_build_performance.yaml +++ /dev/null @@ -1,36 +0,0 @@ -build_performance: - tables: - # - - # server: - # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_4096 - # cache_config.cpu_cache_capacity: 32 - # engine_config.use_blas_threshold: 1100 - # engine_config.gpu_search_threshold: 1 - # gpu_resource_config.enable: false - # gpu_resource_config.cache_capacity: 6 - # gpu_resource_config.search_resources: - # - gpu0 - # - gpu1 - # gpu_resource_config.build_index_resources: - # - gpu0 - # - gpu1 - # table_name: sift_50m_1024_128_l2 - # index_type: ivf_sq8 - # nlist: 4096 - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_8192 - cache_config.cpu_cache_capacity: 32 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 6 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_50m_1024_128_l2 - index_type: ivf_sq8 - nlist: 8192 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/cpu_search_performance.yaml b/tests/milvus_benchmark/suites/cpu_search_performance.yaml deleted file mode 100644 index d971579756..0000000000 --- a/tests/milvus_benchmark/suites/cpu_search_performance.yaml +++ /dev/null @@ -1,169 +0,0 @@ -search_performance: - tables: - # sift_50m - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_ivf - cache_config.cpu_cache_capacity: 32 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 200 - gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - gpu_resource_config.build_index_resources: - - gpu0 - table_name: sift_50m_1024_128_l2 - run_count: 2 - search_params: - nprobes: [8, 32] - top_ks: [1, 10, 100, 1000] - nqs: [1, 10, 100, 200, 500, 1000] - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8 - cache_config.cpu_cache_capacity: 16 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 200 - gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - gpu_resource_config.build_index_resources: - - gpu0 - table_name: sift_50m_1024_128_l2 - run_count: 2 - search_params: - nprobes: [8, 32] - top_ks: [1, 10, 100, 1000] - nqs: [1, 10, 100, 200, 500, 1000] - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_pq - cache_config.cpu_cache_capacity: 32 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 200 - gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - gpu_resource_config.build_index_resources: - - gpu0 - table_name: sift_50m_1024_128_l2 - run_count: 2 - search_params: - nprobes: [8, 32] - top_ks: [1, 10, 100, 1000] - nqs: [1, 10, 100, 200, 500, 1000] - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_nsg - cache_config.cpu_cache_capacity: 50 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 200 - gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - gpu_resource_config.build_index_resources: - - gpu0 - table_name: sift_50m_1024_128_l2 - run_count: 2 - search_params: - nprobes: [8] - top_ks: [1, 10, 100, 1000] - nqs: [1, 10, 100, 200, 500, 1000] - - # random_50m - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_ivf - cache_config.cpu_cache_capacity: 110 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 200 - gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - gpu_resource_config.build_index_resources: - - gpu0 - table_name: random_50m_1024_512_ip - run_count: 2 - search_params: - nprobes: [8, 32] - top_ks: [1, 10, 100, 1000] - nqs: [1, 10, 100, 200, 500, 1000] - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8 - cache_config.cpu_cache_capacity: 30 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 200 - gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - gpu_resource_config.build_index_resources: - - gpu0 - table_name: random_50m_1024_512_ip - run_count: 2 - search_params: - nprobes: [8, 32] - top_ks: [1, 10, 100, 1000] - nqs: [1, 10, 100, 200, 500, 1000] - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_nsg - cache_config.cpu_cache_capacity: 200 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 200 - gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - gpu_resource_config.build_index_resources: - - gpu0 - table_name: random_50m_1024_512_ip - run_count: 2 - search_params: - nprobes: [8] - top_ks: [1, 10, 100, 1000] - nqs: [1, 10, 100, 200, 500, 1000] - - # sift_1b - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8 - cache_config.cpu_cache_capacity: 150 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 200 - gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - gpu_resource_config.build_index_resources: - - gpu0 - table_name: sift_1b_2048_128_l2 - run_count: 2 - search_params: - nprobes: [8, 32] - top_ks: [1, 10, 100, 1000] - nqs: [1, 10, 100, 200, 500, 1000] - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_pq - cache_config.cpu_cache_capacity: 150 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 200 - gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - gpu_resource_config.build_index_resources: - - gpu0 - table_name: sift_1b_2048_128_l2 - run_count: 2 - search_params: - nprobes: [8, 32] - top_ks: [1, 10, 100, 1000] - nqs: [1, 10, 100, 200, 500, 1000] diff --git a/tests/milvus_benchmark/suites/cpu_search_stability.yaml b/tests/milvus_benchmark/suites/cpu_search_stability.yaml deleted file mode 100644 index 85df288ad0..0000000000 --- a/tests/milvus_benchmark/suites/cpu_search_stability.yaml +++ /dev/null @@ -1,20 +0,0 @@ -search_stability: - tables: - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_pq - cache_config.cpu_cache_capacity: 50 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 100 - gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - gpu_resource_config.build_index_resources: - - gpu0 - table_name: sift_50m_1024_128_l2 - during_time: 240 - search_params: - nprobes: 1-200 - top_ks: 1-200 - nqs: 1-200 diff --git a/tests/milvus_benchmark/suites/cpu_stability_sift50m.yaml b/tests/milvus_benchmark/suites/cpu_stability_sift50m.yaml deleted file mode 100644 index fd79a49d7c..0000000000 --- a/tests/milvus_benchmark/suites/cpu_stability_sift50m.yaml +++ /dev/null @@ -1,27 +0,0 @@ -stability: - tables: - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_8192_stability - cache_config.cpu_cache_capacity: 64 - cache_config.cache_insert_data: true - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 100 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_50m_1024_128_l2 - during_time: 480 - search_params: - nprobes: 1-200 - top_ks: 1-200 - nqs: 1-200 - # length of insert vectors - insert_xb: 100000 - # insert after search 4 times - insert_interval: 4 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_accuracy.yaml b/tests/milvus_benchmark/suites/gpu_accuracy.yaml deleted file mode 100644 index 2791851b1b..0000000000 --- a/tests/milvus_benchmark/suites/gpu_accuracy.yaml +++ /dev/null @@ -1,157 +0,0 @@ -accuracy: - tables: - # sift1b - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8 - cache_config.cpu_cache_capacity: 150 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_1b_2048_128_l2 - search_params: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - top_ks: [64] - nqs: [1000] - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8h - cache_config.cpu_cache_capacity: 150 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_1b_2048_128_l2 - search_params: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - top_ks: [64] - nqs: [1000] - # - - # server: - # db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_pq - # cache_config.cpu_cache_capacity: 150 - # engine_config.use_blas_threshold: 1100 - # engine_config.gpu_search_threshold: 1 - # gpu_resource_config.enable: true - # gpu_resource_config.cache_capacity: 4 - # gpu_resource_config.search_resources: - # - gpu0 - # - gpu1 - # gpu_resource_config.build_index_resources: - # - gpu0 - # - gpu1 - # table_name: sift_1b_2048_128_l2 - # search_params: - # nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - # top_ks: [64] - # nqs: [1000] - - # sift50m - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_ivf - cache_config.cpu_cache_capacity: 60 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_50m_1024_128_l2 - search_params: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - top_ks: [64] - nqs: [1000] - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8 - cache_config.cpu_cache_capacity: 30 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_50m_1024_128_l2 - search_params: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - top_ks: [64] - nqs: [1000] - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8h - cache_config.cpu_cache_capacity: 30 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_50m_1024_128_l2 - search_params: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - top_ks: [64] - nqs: [1000] - # - - # server: - # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_pq - # cache_config.cpu_cache_capacity: 30 - # engine_config.use_blas_threshold: 1100 - # engine_config.gpu_search_threshold: 1 - # gpu_resource_config.enable: true - # gpu_resource_config.cache_capacity: 4 - # gpu_resource_config.search_resources: - # - gpu0 - # - gpu1 - # gpu_resource_config.build_index_resources: - # - gpu0 - # - gpu1 - # table_name: sift_50m_1024_128_l2 - # search_params: - # nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - # top_ks: [64] - # nqs: [1000] - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_nsg - cache_config.cpu_cache_capacity: 100 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_50m_1024_128_l2 - search_params: - nprobes: [1] - top_ks: [64] - nqs: [1000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_accuracy_ann.yaml b/tests/milvus_benchmark/suites/gpu_accuracy_ann.yaml index 00cdb388fb..1b7bcf248a 100644 --- a/tests/milvus_benchmark/suites/gpu_accuracy_ann.yaml +++ b/tests/milvus_benchmark/suites/gpu_accuracy_ann.yaml @@ -1,5 +1,5 @@ ann_accuracy: - tables: + collections: - server: cache_config.cpu_cache_capacity: 16 @@ -14,14 +14,16 @@ ann_accuracy: - gpu0 - gpu1 source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 - table_name: sift_128_euclidean + collection_name: sift_128_euclidean index_file_sizes: [1024] - index_types: ['flat', 'ivf_flat', 'ivf_sq8', 'ivf_sq8h', 'ivf_pq', 'nsg'] - nlists: [16384] + index_types: ['flat', 'ivf_flat', 'ivf_sq8', 'ivf_sq8h'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] search_params: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - top_ks: [10] - nqs: [10000] + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + - server: cache_config.cpu_cache_capacity: 16 @@ -35,15 +37,67 @@ ann_accuracy: gpu_resource_config.build_index_resources: - gpu0 - gpu1 - source_file: /test/milvus/ann_hdf5/gist-960-euclidean.hdf5 - table_name: gist_960_euclidean + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean index_file_sizes: [1024] - index_types: ['flat', 'ivf_flat', 'ivf_sq8', 'ivf_sq8h', 'ivf_pq', 'nsg'] - nlists: [16384] + index_types: ['ivf_pq'] + index_params: + nlist: [16384] + m: [32] + top_ks: [10] + nqs: [10000] search_params: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - top_ks: [10] - nqs: [10000] + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + collection_name: sift_128_euclidean + index_file_sizes: [256] + index_types: ['hnsw'] + index_params: + M: [16] + efConstruction: [500] + top_ks: [10] + nqs: [10000] + search_params: + ef: [16, 32, 64, 128, 256, 512] + + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + collection_name: glove_200_angular + index_file_sizes: [1024] + index_types: ['flat', 'ivf_flat', 'ivf_sq8', 'ivf_sq8h'] + index_params: + nlist: [16384] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + - server: cache_config.cpu_cache_capacity: 16 @@ -58,11 +112,61 @@ ann_accuracy: - gpu0 - gpu1 source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 - table_name: glove_200_angular - index_file_sizes: [1024] - index_types: ['flat', 'ivf_flat', 'ivf_sq8', 'ivf_sq8h', 'nsg'] - nlists: [16384] + collection_name: glove_200_angular + index_file_sizes: [256] + index_types: ['hnsw'] + index_params: + M: [36] + efConstruction: [500] + top_ks: [10] + nqs: [10000] search_params: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - top_ks: [10] - nqs: [10000] \ No newline at end of file + ef: [10, 16, 32, 64, 128, 256, 512] + + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/kosarak-27983-jaccard.hdf5 + collection_name: kosarak_27984_jaccard + index_file_sizes: [1024] + index_types: ['flat', 'ivf_flat'] + index_params: + nlist: [2048] + top_ks: [10] + nqs: [10000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-256-hamming.hdf5 + collection_name: sift_256_hamming + index_file_sizes: [1024] + index_types: ['flat', 'ivf_flat'] + index_params: + nlist: [2048] + top_ks: [100] + nqs: [1000] + search_params: + nprobe: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] diff --git a/tests/milvus_benchmark/suites/gpu_accuracy_ann_debug.yaml b/tests/milvus_benchmark/suites/gpu_accuracy_ann_debug.yaml deleted file mode 100644 index b9d2e22b3d..0000000000 --- a/tests/milvus_benchmark/suites/gpu_accuracy_ann_debug.yaml +++ /dev/null @@ -1,47 +0,0 @@ -ann_accuracy: - tables: - - - server: - cache_config.cpu_cache_capacity: 16 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - source_file: /test/milvus/ann_hdf5/kosarak-27983-jaccard.hdf5 - table_name: kosarak_27984_jaccard - index_file_sizes: [1024] - index_types: ['flat', 'ivf_flat'] - nlists: [2048] - search_params: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - top_ks: [10] - nqs: [10000] - - - - server: - cache_config.cpu_cache_capacity: 16 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - source_file: /test/milvus/ann_hdf5/sift-256-hamming.hdf5 - table_name: sift_256_hamming - index_file_sizes: [1024] - index_types: ['flat', 'ivf_flat'] - nlists: [2048] - search_params: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - top_ks: [100] - nqs: [1000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_accuracy_sift1b.yaml b/tests/milvus_benchmark/suites/gpu_accuracy_sift1b.yaml deleted file mode 100644 index 80d9e69bd1..0000000000 --- a/tests/milvus_benchmark/suites/gpu_accuracy_sift1b.yaml +++ /dev/null @@ -1,59 +0,0 @@ -accuracy: - tables: - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8 - cache_config.cpu_cache_capacity: 150 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_1b_2048_128_l2 - search_params: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - top_ks: [64] - nqs: [1000] - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8h - cache_config.cpu_cache_capacity: 150 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_1b_2048_128_l2 - search_params: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - top_ks: [64] - nqs: [1000] - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_pq - cache_config.cpu_cache_capacity: 150 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_1b_2048_128_l2 - search_params: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - top_ks: [64] - nqs: [1000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_accuracy_sift1m.yaml b/tests/milvus_benchmark/suites/gpu_accuracy_sift1m.yaml deleted file mode 100644 index 6982fa877d..0000000000 --- a/tests/milvus_benchmark/suites/gpu_accuracy_sift1m.yaml +++ /dev/null @@ -1,40 +0,0 @@ -accuracy: - tables: - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_1m_1024_128_l2_ivf - cache_config.cpu_cache_capacity: 16 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_1m_1024_128_l2 - search_params: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - top_ks: [64] - nqs: [1000] - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_1m_1024_128_l2_sq8 - cache_config.cpu_cache_capacity: 16 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_1m_1024_128_l2 - search_params: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - top_ks: [64] - nqs: [1000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_accuracy_sift50m.yaml b/tests/milvus_benchmark/suites/gpu_accuracy_sift50m.yaml deleted file mode 100644 index e065753011..0000000000 --- a/tests/milvus_benchmark/suites/gpu_accuracy_sift50m.yaml +++ /dev/null @@ -1,80 +0,0 @@ -accuracy: - tables: - # - - # server: - # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8 - # cache_config.cpu_cache_capacity: 30 - # engine_config.use_blas_threshold: 1100 - # engine_config.gpu_search_threshold: 1 - # gpu_resource_config.enable: true - # gpu_resource_config.cache_capacity: 4 - # gpu_resource_config.search_resources: - # - gpu0 - # - gpu1 - # gpu_resource_config.build_index_resources: - # - gpu0 - # - gpu1 - # table_name: sift_50m_1024_128_l2 - # search_params: - # nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - # top_ks: [64] - # nqs: [1000] - # - - # server: - # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_4096 - # cache_config.cpu_cache_capacity: 30 - # engine_config.use_blas_threshold: 1100 - # engine_config.gpu_search_threshold: 1 - # gpu_resource_config.enable: true - # gpu_resource_config.cache_capacity: 4 - # gpu_resource_config.search_resources: - # - gpu0 - # - gpu1 - # gpu_resource_config.build_index_resources: - # - gpu0 - # - gpu1 - # table_name: sift_50m_1024_128_l2 - # search_params: - # nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - # top_ks: [64] - # nqs: [1000] - - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_8192 - cache_config.cpu_cache_capacity: 30 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_50m_1024_128_l2 - search_params: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192] - top_ks: [64] - nqs: [1000] - - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_4096 - cache_config.cpu_cache_capacity: 30 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_50m_1024_128_l2 - search_params: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096] - top_ks: [64] - nqs: [1000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_build_performance.yaml b/tests/milvus_benchmark/suites/gpu_build_performance.yaml deleted file mode 100644 index 306d65d609..0000000000 --- a/tests/milvus_benchmark/suites/gpu_build_performance.yaml +++ /dev/null @@ -1,36 +0,0 @@ -build_performance: - tables: - # - - # server: - # db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8h - # cache_config.cpu_cache_capacity: 16 - # engine_config.use_blas_threshold: 1100 - # engine_config.gpu_search_threshold: 1 - # gpu_resource_config.enable: true - # gpu_resource_config.cache_capacity: 4 - # gpu_resource_config.search_resources: - # - gpu0 - # - gpu1 - # gpu_resource_config.build_index_resources: - # - gpu0 - # - gpu1 - # table_name: random_50m_1024_512_ip - # index_type: ivf_sq8h - # nlist: 16384 - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_4096 - cache_config.cpu_cache_capacity: 32 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 6 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_50m_1024_128_l2 - index_type: ivf_sq8 - nlist: 4096 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_build_performance_hamming50m.yaml b/tests/milvus_benchmark/suites/gpu_build_performance_hamming50m.yaml deleted file mode 100644 index 89eaf8eb1f..0000000000 --- a/tests/milvus_benchmark/suites/gpu_build_performance_hamming50m.yaml +++ /dev/null @@ -1,36 +0,0 @@ -build_performance: - tables: - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/hamming_50m_128_512_hamming_ivf - cache_config.cpu_cache_capacity: 16 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: hamming_50m_128_512_hamming - index_type: ivf_flat - nlist: 2048 - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/hamming_50m_128_512_hamming_flat - cache_config.cpu_cache_capacity: 16 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: hamming_50m_128_512_hamming - index_type: flat - nlist: 2048 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_build_performance_jaccard50m.yaml b/tests/milvus_benchmark/suites/gpu_build_performance_jaccard50m.yaml deleted file mode 100644 index 15adc2d718..0000000000 --- a/tests/milvus_benchmark/suites/gpu_build_performance_jaccard50m.yaml +++ /dev/null @@ -1,36 +0,0 @@ -build_performance: - tables: - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/jaccard_50m_128_512_jaccard_ivf - cache_config.cpu_cache_capacity: 16 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: jaccard_50m_128_512_jaccard - index_type: ivf_flat - nlist: 2048 - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/jaccard_50m_128_512_jaccard_flat - cache_config.cpu_cache_capacity: 16 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: jaccard_50m_128_512_jaccard - index_type: flat - nlist: 2048 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_search_performance.yaml b/tests/milvus_benchmark/suites/gpu_search_performance.yaml deleted file mode 100644 index 161719ba57..0000000000 --- a/tests/milvus_benchmark/suites/gpu_search_performance.yaml +++ /dev/null @@ -1,247 +0,0 @@ -search_performance: - tables: - # sift_50m - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_ivf - cache_config.cpu_cache_capacity: 32 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 200 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_50m_1024_128_l2 - run_count: 2 - search_params: - nprobes: [8, 32] - top_ks: [1, 10, 100, 1000] - nqs: [1, 10, 100, 200, 500, 1000] - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8 - cache_config.cpu_cache_capacity: 16 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 200 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_50m_1024_128_l2 - run_count: 2 - search_params: - nprobes: [8, 32] - top_ks: [1, 10, 100, 1000] - nqs: [1, 10, 100, 200, 500, 1000] - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8h - cache_config.cpu_cache_capacity: 16 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 200 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_50m_1024_128_l2 - run_count: 2 - search_params: - nprobes: [8, 32] - top_ks: [1, 10, 100, 1000] - nqs: [1, 10, 100, 200, 500, 1000] - # - - # server: - # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_pq - # cache_config.cpu_cache_capacity: 32 - # engine_config.use_blas_threshold: 1100 - # engine_config.gpu_search_threshold: 200 - # gpu_resource_config.enable: true - # gpu_resource_config.cache_capacity: 4 - # gpu_resource_config.search_resources: - # - gpu0 - # - gpu1 - # gpu_resource_config.build_index_resources: - # - gpu0 - # - gpu1 - # table_name: sift_50m_1024_128_l2 - # run_count: 2 - # search_params: - # nprobes: [8, 32] - # top_ks: [1, 10, 100, 1000] - # nqs: [1, 10, 100, 200, 500, 1000] - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_nsg - cache_config.cpu_cache_capacity: 50 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 200 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_50m_1024_128_l2 - run_count: 2 - search_params: - nprobes: [8] - top_ks: [1, 10, 100, 1000] - nqs: [1, 10, 100, 200, 500, 1000] - - # random_50m - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_ivf - cache_config.cpu_cache_capacity: 110 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 200 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: random_50m_1024_512_ip - run_count: 2 - search_params: - nprobes: [8, 32] - top_ks: [1, 10, 100, 1000] - nqs: [1, 10, 100, 200, 500, 1000] - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8 - cache_config.cpu_cache_capacity: 30 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 200 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: random_50m_1024_512_ip - run_count: 2 - search_params: - nprobes: [8, 32] - top_ks: [1, 10, 100, 1000] - nqs: [1, 10, 100, 200, 500, 1000] - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8h - cache_config.cpu_cache_capacity: 30 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 200 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: random_50m_1024_512_ip - run_count: 2 - search_params: - nprobes: [8, 32] - top_ks: [1, 10, 100, 1000] - nqs: [1, 10, 100, 200, 500, 1000] - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_nsg - cache_config.cpu_cache_capacity: 200 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 200 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 6 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: random_50m_1024_512_ip - run_count: 2 - search_params: - nprobes: [8] - top_ks: [1, 10, 100, 1000] - nqs: [1, 10, 100, 200, 500, 1000] - - # sift_1b - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8 - cache_config.cpu_cache_capacity: 150 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 200 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_1b_2048_128_l2 - run_count: 2 - search_params: - nprobes: [8, 32] - top_ks: [1, 10, 100, 1000] - nqs: [1, 10, 100, 200, 500, 1000] - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8h - cache_config.cpu_cache_capacity: 150 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 200 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_1b_2048_128_l2 - run_count: 2 - search_params: - nprobes: [8, 32] - top_ks: [1, 10, 100, 1000] - nqs: [1, 10, 100, 200, 500, 1000] - # - - # server: - # db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_pq - # cache_config.cpu_cache_capacity: 150 - # engine_config.use_blas_threshold: 1100 - # engine_config.gpu_search_threshold: 200 - # gpu_resource_config.enable: true - # gpu_resource_config.cache_capacity: 4 - # gpu_resource_config.search_resources: - # - gpu0 - # - gpu1 - # gpu_resource_config.build_index_resources: - # - gpu0 - # - gpu1 - # table_name: sift_1b_2048_128_l2 - # run_count: 2 - # search_params: - # nprobes: [8, 32] - # top_ks: [1, 10, 100, 1000] - # nqs: [1, 10, 100, 200, 500, 1000] diff --git a/tests/milvus_benchmark/suites/gpu_search_performance_hamming50m.yaml b/tests/milvus_benchmark/suites/gpu_search_performance_hamming50m.yaml deleted file mode 100644 index b212c4eda1..0000000000 --- a/tests/milvus_benchmark/suites/gpu_search_performance_hamming50m.yaml +++ /dev/null @@ -1,22 +0,0 @@ -search_performance: - tables: - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/hamming_50m_128_512_hamming_ivf - cache_config.cpu_cache_capacity: 32 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: hamming_50m_128_512_hamming - run_count: 1 - search_params: - nprobes: [8, 32] - top_ks: [1, 16, 64, 128, 256, 512, 1000] - nqs: [1, 10, 100, 200, 500, 1000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_search_performance_jaccard50m.yaml b/tests/milvus_benchmark/suites/gpu_search_performance_jaccard50m.yaml deleted file mode 100644 index e0d6957e3b..0000000000 --- a/tests/milvus_benchmark/suites/gpu_search_performance_jaccard50m.yaml +++ /dev/null @@ -1,22 +0,0 @@ -search_performance: - tables: - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/jaccard_50m_128_512_jaccard_ivf - cache_config.cpu_cache_capacity: 32 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: jaccard_50m_128_512_jaccard - run_count: 1 - search_params: - nprobes: [8, 32] - top_ks: [1, 16, 64, 128, 256, 512, 1000] - nqs: [1, 10, 100, 200, 500, 1000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_search_performance_random50m.yaml b/tests/milvus_benchmark/suites/gpu_search_performance_random50m.yaml deleted file mode 100644 index 534a82d501..0000000000 --- a/tests/milvus_benchmark/suites/gpu_search_performance_random50m.yaml +++ /dev/null @@ -1,82 +0,0 @@ -search_performance: - tables: - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/random_50m_2048_512_ip_sq8 - cache_config.cpu_cache_capacity: 110 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 200 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: random_50m_2048_512_ip - run_count: 2 - search_params: - nprobes: [8, 32] - top_ks: [1, 10, 100, 1000] - nqs: [1, 10, 100, 200, 500, 1000] - # - - # server: - # db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8 - # cache_config.cpu_cache_capacity: 30 - # engine_config.use_blas_threshold: 1100 - # engine_config.gpu_search_threshold: 200 - # gpu_resource_config.enable: true - # gpu_resource_config.cache_capacity: 4 - # gpu_resource_config.search_resources: - # - gpu0 - # - gpu1 - # gpu_resource_config.build_index_resources: - # - gpu0 - # - gpu1 - # table_name: random_50m_2048_512_ip - # run_count: 2 - # search_params: - # nprobes: [8, 32] - # top_ks: [1, 10, 100, 1000] - # nqs: [1, 10, 100, 200, 500, 1000] - # - - # server: - # db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8h - # cache_config.cpu_cache_capacity: 30 - # engine_config.use_blas_threshold: 1100 - # engine_config.gpu_search_threshold: 200 - # gpu_resource_config.enable: true - # gpu_resource_config.cache_capacity: 4 - # gpu_resource_config.search_resources: - # - gpu0 - # - gpu1 - # gpu_resource_config.build_index_resources: - # - gpu0 - # - gpu1 - # table_name: random_50m_1024_512_ip - # run_count: 2 - # search_params: - # nprobes: [8, 32] - # top_ks: [1, 10, 100, 1000] - # nqs: [1, 10, 100, 200, 500, 1000] - # - - # server: - # db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_nsg - # cache_config.cpu_cache_capacity: 200 - # engine_config.use_blas_threshold: 1100 - # engine_config.gpu_search_threshold: 200 - # gpu_resource_config.enable: true - # gpu_resource_config.cache_capacity: 4 - # gpu_resource_config.search_resources: - # - gpu0 - # - gpu1 - # gpu_resource_config.build_index_resources: - # - gpu0 - # - gpu1 - # table_name: random_50m_1024_512_ip - # run_count: 2 - # search_params: - # nprobes: [8] - # top_ks: [1, 10, 100, 1000] - # nqs: [1, 10, 100, 200, 500, 1000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_search_performance_sift1b.yaml b/tests/milvus_benchmark/suites/gpu_search_performance_sift1b.yaml deleted file mode 100644 index 88aac71806..0000000000 --- a/tests/milvus_benchmark/suites/gpu_search_performance_sift1b.yaml +++ /dev/null @@ -1,62 +0,0 @@ -search_performance: - tables: - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8 - cache_config.cpu_cache_capacity: 150 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 200 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_1b_2048_128_l2 - run_count: 2 - search_params: - nprobes: [8, 32] - top_ks: [1, 10, 100, 1000] - nqs: [1, 10, 100, 200, 500, 1000] - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8h - cache_config.cpu_cache_capacity: 150 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 200 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_1b_2048_128_l2 - run_count: 2 - search_params: - nprobes: [8, 32] - top_ks: [1, 10, 100, 1000] - nqs: [1, 10, 100, 200, 500, 1000] - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_pq - cache_config.cpu_cache_capacity: 150 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 200 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_1b_2048_128_l2 - run_count: 2 - search_params: - nprobes: [8, 32] - top_ks: [1, 10, 100, 1000] - nqs: [1, 10, 100, 200, 500, 1000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_search_performance_sift1m.yaml b/tests/milvus_benchmark/suites/gpu_search_performance_sift1m.yaml deleted file mode 100644 index 1b2b8485c7..0000000000 --- a/tests/milvus_benchmark/suites/gpu_search_performance_sift1m.yaml +++ /dev/null @@ -1,42 +0,0 @@ -search_performance: - tables: - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_1m_1024_128_l2_ivf - cache_config.cpu_cache_capacity: 8 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_1m_1024_128_l2 - run_count: 1 - search_params: - nprobes: [8, 32] - top_ks: [1, 16, 64, 128, 256, 512, 1000] - nqs: [1, 10, 100, 200, 500, 1000] - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_1m_1024_128_l2_sq8 - cache_config.cpu_cache_capacity: 8 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_1m_1024_128_l2 - run_count: 1 - search_params: - nprobes: [8, 32] - top_ks: [1, 16, 64, 128, 256, 512, 1000] - nqs: [1, 10, 100, 200, 500, 1000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_search_performance_sift50m.yaml b/tests/milvus_benchmark/suites/gpu_search_performance_sift50m.yaml deleted file mode 100644 index 75d3e5f110..0000000000 --- a/tests/milvus_benchmark/suites/gpu_search_performance_sift50m.yaml +++ /dev/null @@ -1,146 +0,0 @@ -search_performance: - tables: - # - - # server: - # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_ivf - # cache_config.cpu_cache_capacity: 32 - # engine_config.use_blas_threshold: 1100 - # engine_config.gpu_search_threshold: 200 - # gpu_resource_config.enable: true - # gpu_resource_config.cache_capacity: 4 - # gpu_resource_config.search_resources: - # - gpu0 - # - gpu1 - # gpu_resource_config.build_index_resources: - # - gpu0 - # - gpu1 - # table_name: sift_50m_1024_128_l2 - # run_count: 2 - # search_params: - # nprobes: [8, 32] - # top_ks: [1, 10, 100, 1000] - # nqs: [1, 10, 100, 200, 500, 1000] - # - - # server: - # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8 - # cache_config.cpu_cache_capacity: 16 - # engine_config.use_blas_threshold: 1100 - # engine_config.gpu_search_threshold: 200 - # gpu_resource_config.enable: true - # gpu_resource_config.cache_capacity: 4 - # gpu_resource_config.search_resources: - # - gpu0 - # - gpu1 - # gpu_resource_config.build_index_resources: - # - gpu0 - # - gpu1 - # table_name: sift_50m_1024_128_l2 - # run_count: 2 - # search_params: - # nprobes: [8, 32] - # top_ks: [1, 10, 100, 1000] - # nqs: [1, 10, 100, 200, 500, 1000] - # - - # server: - # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8h - # cache_config.cpu_cache_capacity: 16 - # engine_config.use_blas_threshold: 1100 - # engine_config.gpu_search_threshold: 200 - # gpu_resource_config.enable: true - # gpu_resource_config.cache_capacity: 4 - # gpu_resource_config.search_resources: - # - gpu0 - # - gpu1 - # gpu_resource_config.build_index_resources: - # - gpu0 - # - gpu1 - # table_name: sift_50m_1024_128_l2 - # run_count: 2 - # search_params: - # nprobes: [8, 32] - # top_ks: [1, 10, 100, 1000] - # nqs: [1, 10, 100, 200, 500, 1000] - - # git issue num: #626 - # - - # server: - # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_pq - # cache_config.cpu_cache_capacity: 32 - # engine_config.use_blas_threshold: 1100 - # engine_config.gpu_search_threshold: 200 - # gpu_resource_config.enable: true - # gpu_resource_config.cache_capacity: 4 - # gpu_resource_config.search_resources: - # - gpu0 - # - gpu1 - # gpu_resource_config.build_index_resources: - # - gpu0 - # - gpu1 - # table_name: sift_50m_1024_128_l2 - # run_count: 2 - # search_params: - # nprobes: [8, 32] - # top_ks: [1, 10, 100, 1000] - # nqs: [1, 10, 100, 200, 500, 1000] - - # - - # server: - # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_nsg - # cache_config.cpu_cache_capacity: 50 - # engine_config.use_blas_threshold: 1100 - # engine_config.gpu_search_threshold: 200 - # gpu_resource_config.enable: true - # gpu_resource_config.cache_capacity: 4 - # gpu_resource_config.search_resources: - # - gpu0 - # - gpu1 - # gpu_resource_config.build_index_resources: - # - gpu0 - # - gpu1 - # table_name: sift_50m_1024_128_l2 - # run_count: 2 - # search_params: - # nprobes: [8] - # top_ks: [1, 10, 100, 1000] - # nqs: [1, 10, 100, 200, 500, 1000] - - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_8192 - cache_config.cpu_cache_capacity: 16 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 200 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_50m_1024_128_l2 - run_count: 2 - search_params: - nprobes: [8, 32] - top_ks: [1, 10, 100, 1000] - nqs: [1, 10, 100, 200, 500, 1000] - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_4096 - cache_config.cpu_cache_capacity: 16 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 200 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_50m_1024_128_l2 - run_count: 2 - search_params: - nprobes: [8, 32] - top_ks: [1, 10, 100, 1000] - nqs: [1, 10, 100, 200, 500, 1000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_search_stability.yaml b/tests/milvus_benchmark/suites/gpu_search_stability.yaml deleted file mode 100644 index 306537138c..0000000000 --- a/tests/milvus_benchmark/suites/gpu_search_stability.yaml +++ /dev/null @@ -1,23 +0,0 @@ -search_stability: - tables: - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8 - cache_config.cpu_cache_capacity: 50 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 100 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - - gpu2 - - gpu3 - gpu_resource_config.build_index_resources: - - gpu0 - table_name: sift_50m_1024_128_l2 - during_time: 240 - search_params: - nprobes: 1-200 - top_ks: 1-200 - nqs: 1-200 diff --git a/tests/milvus_benchmark/suites/gpu_stability_sift1m.yaml b/tests/milvus_benchmark/suites/gpu_stability_sift1m.yaml deleted file mode 100644 index 9d4c75fb99..0000000000 --- a/tests/milvus_benchmark/suites/gpu_stability_sift1m.yaml +++ /dev/null @@ -1,26 +0,0 @@ -stability: - tables: - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_1m_1024_128_l2_ivf_stability - cache_config.cpu_cache_capacity: 64 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 100 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_1m_1024_128_l2 - during_time: 10 - search_params: - nprobes: 1-200 - top_ks: 1-200 - nqs: 1-200 - # length of insert vectors - insert_xb: 10000 - # insert after search 3 times - insert_interval: 3 diff --git a/tests/milvus_benchmark/suites/gpu_stability_sift50m.yaml b/tests/milvus_benchmark/suites/gpu_stability_sift50m.yaml deleted file mode 100644 index 6625d4cd62..0000000000 --- a/tests/milvus_benchmark/suites/gpu_stability_sift50m.yaml +++ /dev/null @@ -1,27 +0,0 @@ -stability: - tables: - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8h_stability - cache_config.cpu_cache_capacity: 64 - cache_config.cache_insert_data: true - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 100 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_50m_1024_128_l2 - during_time: 480 - search_params: - nprobes: 1-200 - top_ks: 1-200 - nqs: 1-200 - # length of insert vectors - insert_xb: 100000 - # insert after search 4 times - insert_interval: 4 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/insert.yaml b/tests/milvus_benchmark/suites/insert.yaml deleted file mode 100644 index 767ad481c5..0000000000 --- a/tests/milvus_benchmark/suites/insert.yaml +++ /dev/null @@ -1,32 +0,0 @@ -performance: - - # interface: add_vectors - insert: - # index_type: flat/ivf_flat/ivf_sq8/mix_nsg - [ - # debug - # data_type / data_size / index_file_size / dimension - # data_type: random / ann_sift - # data_size: 10m / 1b - # { - # "table_name": "random_1m_1024_4096_ip", - # "ni_per": 10000, - # "server.cpu_cache_capacity": 16, - # "server.resources": ["gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data/random_1m_1024_4096_ip" - # }, - # { - # "table_name": "random_1m_1024_16384_ip", - # "ni_per": 10000, - # "server.cpu_cache_capacity": 16, - # "server.resources": ["gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data/random_1m_1024_16384_ip" - # }, - { - "table_name": "random_50m_1024_512_ip", - "ni_per": 100000, - # "server.cpu_cache_capacity": 16, - # "server.resources": ["gpu0", "gpu1"], - "db_path_prefix": "/test/milvus/db_data_cpu/random_50m_1024_512_ip_ivf" - }, - ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/insert_performance.yaml b/tests/milvus_benchmark/suites/insert_performance.yaml deleted file mode 100644 index f1d487850d..0000000000 --- a/tests/milvus_benchmark/suites/insert_performance.yaml +++ /dev/null @@ -1,19 +0,0 @@ -insert_performance: - tables: - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_5m_512_128_l2_ivf - cache_config.cpu_cache_capacity: 8 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - gpu_resource_config.build_index_resources: - - gpu0 - table_name: sift_5m_512_128_l2 - ni_per: 100000 - build_index: false - # index_type: ivf_flat - # nlist: 16384 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/insert_performance_deep1b.yaml b/tests/milvus_benchmark/suites/insert_performance_deep1b.yaml deleted file mode 100644 index 4acbb84099..0000000000 --- a/tests/milvus_benchmark/suites/insert_performance_deep1b.yaml +++ /dev/null @@ -1,87 +0,0 @@ -insert_performance: - tables: - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/deep_1b_1024_96_ip_ivf - cache_config.cpu_cache_capacity: 8 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - gpu_resource_config.build_index_resources: - - gpu0 - table_name: deep_1b_1024_96_ip - ni_per: 100000 - build_index: false - # index_type: ivf_flat - # nlist: 16384 - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/deep_1b_1024_96_ip_sq8 - cache_config.cpu_cache_capacity: 8 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - gpu_resource_config.build_index_resources: - - gpu0 - table_name: deep_1b_1024_96_ip - ni_per: 100000 - build_index: false - # index_type: ivf_sq8 - # nlist: 16384 - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/deep_1b_1024_96_ip_sq8h - cache_config.cpu_cache_capacity: 8 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - gpu_resource_config.build_index_resources: - - gpu0 - table_name: deep_1b_1024_96_ip - ni_per: 100000 - build_index: false - # index_type: ivf_sq8h - # nlist: 16384 - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/deep_1b_1024_96_ip_pq - cache_config.cpu_cache_capacity: 8 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - gpu_resource_config.build_index_resources: - - gpu0 - table_name: deep_1b_1024_96_ip - ni_per: 100000 - build_index: false - # index_type: ivf_pq - # nlist: 16384 - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/deep_1b_1024_96_ip_nsg - cache_config.cpu_cache_capacity: 8 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - gpu_resource_config.build_index_resources: - - gpu0 - table_name: deep_1b_1024_96_ip - ni_per: 100000 - build_index: false - # index_type: nsg - # nlist: 16384 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/insert_performance_hamming50m.yaml b/tests/milvus_benchmark/suites/insert_performance_hamming50m.yaml deleted file mode 100644 index 22b749048c..0000000000 --- a/tests/milvus_benchmark/suites/insert_performance_hamming50m.yaml +++ /dev/null @@ -1,36 +0,0 @@ -insert_performance: - tables: - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/hamming_50m_128_512_hamming_ivf - cache_config.cpu_cache_capacity: 8 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - gpu_resource_config.build_index_resources: - - gpu0 - table_name: hamming_50m_128_512_hamming - ni_per: 100000 - build_index: false - # index_type: ivf_flat - # nlist: 16384 - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/hamming_50m_128_512_hamming_flat - cache_config.cpu_cache_capacity: 8 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - gpu_resource_config.build_index_resources: - - gpu0 - table_name: hamming_50m_128_512_hamming - ni_per: 100000 - build_index: false - # index_type: ivf_flat - # nlist: 16384 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/insert_performance_jaccard50m.yaml b/tests/milvus_benchmark/suites/insert_performance_jaccard50m.yaml deleted file mode 100644 index abea53814c..0000000000 --- a/tests/milvus_benchmark/suites/insert_performance_jaccard50m.yaml +++ /dev/null @@ -1,36 +0,0 @@ -insert_performance: - tables: - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/jaccard_50m_128_512_jaccard_ivf - cache_config.cpu_cache_capacity: 8 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - gpu_resource_config.build_index_resources: - - gpu0 - table_name: jaccard_50m_128_512_jaccard - ni_per: 100000 - build_index: false - # index_type: ivf_flat - # nlist: 16384 - # - - # server: - # db_config.primary_path: /test/milvus/db_data_gpu/jaccard_50m_128_512_jaccard_flat - # cache_config.cpu_cache_capacity: 8 - # engine_config.use_blas_threshold: 1100 - # engine_config.gpu_search_threshold: 1 - # gpu_resource_config.enable: false - # gpu_resource_config.cache_capacity: 4 - # gpu_resource_config.search_resources: - # - gpu0 - # gpu_resource_config.build_index_resources: - # - gpu0 - # table_name: jaccard_50m_128_512_jaccard - # ni_per: 100000 - # build_index: false - # # index_type: ivf_flat - # # nlist: 16384 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/insert_performance_sift50m.yaml b/tests/milvus_benchmark/suites/insert_performance_sift50m.yaml deleted file mode 100644 index e5804a4c89..0000000000 --- a/tests/milvus_benchmark/suites/insert_performance_sift50m.yaml +++ /dev/null @@ -1,87 +0,0 @@ -insert_performance: - tables: - # - - # server: - # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_ip_ivf - # cache_config.cpu_cache_capacity: 8 - # engine_config.use_blas_threshold: 1100 - # engine_config.gpu_search_threshold: 1 - # gpu_resource_config.enable: false - # gpu_resource_config.cache_capacity: 4 - # gpu_resource_config.search_resources: - # - gpu0 - # gpu_resource_config.build_index_resources: - # - gpu0 - # table_name: sift_50m_1024_128_ip - # ni_per: 100000 - # build_index: false - # # index_type: ivf_flat - # # nlist: 16384 - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_4096 - cache_config.cpu_cache_capacity: 8 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - gpu_resource_config.build_index_resources: - - gpu0 - table_name: sift_50m_1024_128_l2 - ni_per: 100000 - build_index: false - # index_type: ivf_sq8 - # nlist: 16384 - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_8192 - cache_config.cpu_cache_capacity: 8 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: false - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - gpu_resource_config.build_index_resources: - - gpu0 - table_name: sift_50m_1024_128_l2 - ni_per: 100000 - build_index: false - # index_type: ivf_sq8h - # nlist: 16384 - # - - # server: - # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_ip_pq - # cache_config.cpu_cache_capacity: 8 - # engine_config.use_blas_threshold: 1100 - # engine_config.gpu_search_threshold: 1 - # gpu_resource_config.enable: false - # gpu_resource_config.cache_capacity: 4 - # gpu_resource_config.search_resources: - # - gpu0 - # gpu_resource_config.build_index_resources: - # - gpu0 - # table_name: sift_50m_1024_128_ip - # ni_per: 100000 - # build_index: false - # # index_type: ivf_pq - # # nlist: 16384 - # - - # server: - # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_ip_nsg - # cache_config.cpu_cache_capacity: 8 - # engine_config.use_blas_threshold: 1100 - # engine_config.gpu_search_threshold: 1 - # gpu_resource_config.enable: false - # gpu_resource_config.cache_capacity: 4 - # gpu_resource_config.search_resources: - # - gpu0 - # gpu_resource_config.build_index_resources: - # - gpu0 - # table_name: sift_50m_1024_128_ip - # ni_per: 100000 - # build_index: false - # # index_type: nsg - # # nlist: 16384 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/performance_random1m.yaml b/tests/milvus_benchmark/suites/performance_random1m.yaml deleted file mode 100644 index 606f721567..0000000000 --- a/tests/milvus_benchmark/suites/performance_random1m.yaml +++ /dev/null @@ -1,35 +0,0 @@ -performance: - - # interface: search_vectors - query: - # dataset: table name you have already created - # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity .. - [ - # { - # "dataset": "random_1m_1024_4096_ip", - # "index.index_types": ["ivf_sq8h"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 800, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 1100, - # "server.cpu_cache_capacity": 50, - # "server.resources": ["cpu", "gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data/random_1m_1024_4096_ip_sq8h" - # }, - { - "dataset": "random_2m_1024_512_ip", - "index.index_types": ["mix_nsg"], - "index.nlists": [16384], - "nprobes": [8], - "top_ks": [1], - "nqs": [1], - "server.use_blas_threshold": 1100, - "server.use_gpu_threshold": 1100, - "server.cpu_cache_capacity": 50, - "server.resources": ["gpu0", "gpu1"], - "server.enable_gpu": True, - "db_path_prefix": "/test/milvus/db_data/random_1m_1024_4096_ip_sq8" - }, - ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/performance_random50m.yaml b/tests/milvus_benchmark/suites/performance_random50m.yaml deleted file mode 100644 index 20cb78c099..0000000000 --- a/tests/milvus_benchmark/suites/performance_random50m.yaml +++ /dev/null @@ -1,177 +0,0 @@ -performance: - - # interface: search_vectors - query: - # dataset: table name you have already created - # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity .. - [ - # { - # "dataset": "random_50m_2048_512_ip", - # "index.index_types": ["ivf_sq8h"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 800, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 50, - # "server.resources": ["cpu", "gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data/random_50m_2048_512_ip_sq8h" - # }, - # { - # "dataset": "random_50m_2048_512_ip", - # "index.index_types": ["ivf_sq8h"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 800, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 1000, - # "server.cpu_cache_capacity": 50, - # "server.resources": ["cpu", "gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data/random_50m_2048_512_ip_sq8h" - # }, - # { - # "dataset": "random_50m_2048_512_ip", - # "index.index_types": ["ivf_sq8h"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 800, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 10000, - # "server.cpu_cache_capacity": 50, - # "server.resources": ["gpu0", "gpu1"], - # "server.enable_gpu": True, - # "db_path_prefix": "/test/milvus/db_data/random_50m_2048_512_ip_sq8h" - # }, - # { - # "dataset": "random_50m_2048_512_ip", - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 800, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 50, - # "server.resources": ["gpu0", "gpu1"], - # "server.enable_gpu": True, - # "db_path_prefix": "/test/milvus/db_data/random_50m_2048_512_ip_sq8" - # }, - { - "dataset": "random_50m_2048_512_ip", - "index.index_types": ["ivf_sq8"], - "index.nlists": [16384], - "nprobes": [8], - "top_ks": [64], - "nqs": [1, 10, 100], - "server.use_blas_threshold": 1100, - "server.use_gpu_threshold": 100, - "server.cpu_cache_capacity": 50, - "server.enable_gpu": True, - "server.resources": ["gpu0", "gpu1"], - "db_path_prefix": "/test/milvus/db_data_cpu/random_50m_2048_512_ip_sq8" - }, - - # { - # "dataset": "random_50m_1024_512_ip", - # "index.index_types": ["ivf_sq8h"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 800, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 110, - # "server.resources": ["gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip" - # }, - # { - # "dataset": "random_50m_1024_512_ip", - # "index.index_types": ["ivf_sq8h"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 800, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 1000, - # "server.cpu_cache_capacity": 110, - # "server.resources": ["gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip" - # }, - # { - # "dataset": "random_50m_1024_512_ip", - # "index.index_types": ["ivf_sq8h"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 800, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 10000, - # "server.cpu_cache_capacity": 110, - # "server.resources": ["gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip" - # }, - # { - # "dataset": "random_50m_1024_512_ip", - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 800, 1000], - # "server.use_blas_threshold": 1100, - # "server.cpu_cache_capacity": 110, - # "server.resources": ["gpu0", "gpu1"], - # "server.enable_gpu": True, - # "db_path_prefix": "/test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8" - # }, - # { - # "dataset": "random_50m_1024_512_ip", - # "index.index_types": ["ivf_pq"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 800, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 110, - # "server.gpu_cache_capacity": 6, - # "server.index_build_device": "gpu0", - # "server.resources": ["gpu0", "gpu1"], - # "server.enable_gpu": True, - # "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip_pq" - # }, - # { - # "dataset": "random_50m_1024_512_ip", - # "index.index_types": ["ivf_flat"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 800, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 110, - # "server.gpu_cache_capacity": 6, - # "server.index_build_device": "gpu0", - # "server.resources": ["gpu0", "gpu1"], - # "server.enable_gpu": True, - # "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip_ivf" - # }, - # { - # "dataset": "random_50m_1024_512_ip", - # "index.index_types": ["nsg"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 800, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 110, - # "server.gpu_cache_capacity": 6, - # "server.index_build_device": "gpu0", - # "server.resources": ["gpu0", "gpu1"], - # "server.enable_gpu": True, - # "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip_nsg" - # } - ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/performance_sift10m.yaml b/tests/milvus_benchmark/suites/performance_sift10m.yaml deleted file mode 100644 index e98d54bc3a..0000000000 --- a/tests/milvus_benchmark/suites/performance_sift10m.yaml +++ /dev/null @@ -1,83 +0,0 @@ -performance: - - # interface: search_vectors - query: - # dataset: table name you have already created - # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity .. - [ - # { - # "dataset": "sift_10m_1024_128_l2", - # "index.index_types": ["ivf_flat"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 200, 500, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 50, - # "server.gpu_cache_capacity": 6, - # "server.resources": ["gpu0"], - # "server.enable_gpu": False, - # "db_path_prefix": "/test/milvus/db_data_cpu/sift_10m_1024_128_l2_ivf" - # }, - # { - # "dataset": "sift_10m_1024_128_l2", - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 800, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 50, - # "server.gpu_cache_capacity": 6, - # "server.resources": ["gpu0"], - # "server.enable_gpu": False, - # "db_path_prefix": "/test/milvus/db_data_cpu/sift_10m_1024_128_l2_sq8" - # }, - # { - # "dataset": "sift_10m_1024_128_l2", - # "index.index_types": ["ivf_sq8h"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 800, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 50, - # "server.gpu_cache_capacity": 6, - # "server.resources": ["gpu0"], - # "server.enable_gpu": False, - # "db_path_prefix": "/test/milvus/db_data_cpu/sift_10m_1024_128_l2_sq8h" - # }, - { - "dataset": "sift_10m_1024_128_l2", - "index.index_types": ["ivf_pq"], - "index.nlists": [16384], - "nprobes": [8, 32], - "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], - "nqs": [1, 10, 100, 500, 800, 1000], - "server.use_blas_threshold": 1100, - "server.use_gpu_threshold": 100, - "server.cpu_cache_capacity": 50, - "server.gpu_cache_capacity": 6, - "server.resources": ["gpu0"], - "server.enable_gpu": False, - "db_path_prefix": "/test/milvus/db_data_cpu/sift_10m_1024_128_l2_pq" - }, - # { - # "dataset": "sift_10m_1024_128_l2", - # "index.index_types": ["nsg"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 200, 500, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 50, - # "server.gpu_cache_capacity": 6, - # "server.resources": ["gpu0"], - # "server.enable_gpu": False, - # "db_path_prefix": "/test/milvus/db_data_cpu/sift_10m_1024_128_l2_nsg" - # }, - ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/performance_sift1b.yaml b/tests/milvus_benchmark/suites/performance_sift1b.yaml deleted file mode 100644 index 505bf726af..0000000000 --- a/tests/milvus_benchmark/suites/performance_sift1b.yaml +++ /dev/null @@ -1,123 +0,0 @@ -performance: - - # interface: search_vectors - query: - # dataset: table name you have already created - # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity .. - [ - # { - # "dataset": "sift_1b_2048_128_l2", - # "index.index_types": ["ivf_sq8h"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 200, 500, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 10000, - # "server.cpu_cache_capacity": 150, - # "server.gpu_cache_capacity": 6, - # "server.resources": ["gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2_sq8h" - # }, - # { - # "dataset": "sift_1b_2048_128_l2", - # "index.index_types": ["ivf_sq8h"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 200, 500, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 150, - # "server.gpu_cache_capacity": 6, - # "server.resources": ["gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2_sq8h" - # }, - # { - # "dataset": "sift_1b_2048_128_l2", - # "index.index_types": ["ivf_sq8h"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 200, 500, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 150, - # "server.gpu_cache_capacity": 6, - # "server.resources": ["cpu", "gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2_sq8h" - # }, - # { - # "dataset": "sift_1b_2048_128_l2", - # "index.index_types": ["ivf_sq8h"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 200, 500, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 1000, - # "server.cpu_cache_capacity": 150, - # "server.gpu_cache_capacity": 6, - # "server.resources": ["gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2_sq8h" - # }, - # { - # "dataset": "sift_1b_2048_128_l2", - # "index.index_types": ["ivf_sq8h"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 200, 500, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 10000, - # "server.cpu_cache_capacity": 150, - # "server.gpu_cache_capacity": 6, - # "server.resources": ["gpu0", "gpu1"], - # "server.enable_gpu": False, - # "db_path_prefix": "/test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8h" - # }, - { - "dataset": "sift_1b_2048_128_l2", - "index.index_types": ["ivf_pq"], - "index.nlists": [16384], - "nprobes": [8, 32], - "top_ks": [1, 16, 64, 128, 256, 512, 1000], - "nqs": [1, 10, 100, 200, 500, 1000], - "server.use_blas_threshold": 1100, - "server.use_gpu_threshold": 10000, - "server.cpu_cache_capacity": 150, - "server.gpu_cache_capacity": 6, - "server.resources": ["gpu0", "gpu1"], - "server.enable_gpu": False, - "db_path_prefix": "/test/milvus/db_data_gpu/sift_1b_2048_128_l2_pq" - }, - # { - # "dataset": "sift_1b_2048_128_l2", - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 200, 500, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 150, - # "server.gpu_cache_capacity": 6, - # "server.resources": ["gpu0", "gpu1"], - # "server.enable_gpu": False, - # "db_path_prefix": "/test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8" - # }, - # { - # "dataset": "sift_1b_2048_128_l2", - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 200, 500, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 150, - # "server.gpu_cache_capacity": 6, - # "server.resources": ["cpu", "gpu0"], - # "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2" - # }, - ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/performance_sift1m.yaml b/tests/milvus_benchmark/suites/performance_sift1m.yaml deleted file mode 100644 index 5c434b7d3f..0000000000 --- a/tests/milvus_benchmark/suites/performance_sift1m.yaml +++ /dev/null @@ -1,23 +0,0 @@ -performance: - - # interface: search_vectors - query: - # dataset: table name you have already created - # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity .. - [ - { - "dataset": "sift_1m_1024_128_l2", - "index.index_types": ["mix_nsg"], - "index.nlists": [16384], - "nprobes": [8, 32], - "top_ks": [1, 16, 64, 128, 256, 512, 1000], - "nqs": [1, 10, 100, 200, 500, 1000], - "server.use_blas_threshold": 1100, - "server.use_gpu_threshold": 100, - "server.cpu_cache_capacity": 50, - "server.gpu_cache_capacity": 6, - # "server.resources": ["gpu0", "gpu1"], - "server.enable_gpu": True, - "db_path_prefix": "/test/milvus/db_data/sift_1m_1024_128_l2_nsg" - }, - ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/performance_sift500m.yaml b/tests/milvus_benchmark/suites/performance_sift500m.yaml deleted file mode 100644 index 99ab7a3204..0000000000 --- a/tests/milvus_benchmark/suites/performance_sift500m.yaml +++ /dev/null @@ -1,50 +0,0 @@ -performance: - - # interface: search_vectors - query: - # dataset: table name you have already created - # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity .. - [ - # { - # "dataset": "sift_500m_1024_128_l2", - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 800, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 150, - # "server.gpu_cache_capacity": 6, - # "server.resources": ["gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data/sift_500m_1024_128_l2_sq8" - # }, - # { - # "dataset": "sift_500m_1024_128_l2", - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 800, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 150, - # "server.gpu_cache_capacity": 6, - # "server.resources": ["gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data/sift_500m_1024_128_l2_sq8" - # }, - { - "dataset": "sift_500m_1024_128_l2", - "index.index_types": ["ivf_sq8h"], - "index.nlists": [16384], - "nprobes": [8, 32], - "top_ks": [1, 100, 200, 1000], - "nqs": [1, 10, 100, 500, 1000], - "server.use_blas_threshold": 1100, - "server.use_gpu_threshold": 100, - "server.cpu_cache_capacity": 150, - "server.gpu_cache_capacity": 6, - "server.resources": ["gpu0", "gpu1"], - "db_path_prefix": "/test/milvus/db_data/sift_500m_1024_128_l2" - }, - ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/performance_sift50m.yaml b/tests/milvus_benchmark/suites/performance_sift50m.yaml deleted file mode 100644 index 85fece5b56..0000000000 --- a/tests/milvus_benchmark/suites/performance_sift50m.yaml +++ /dev/null @@ -1,82 +0,0 @@ -performance: - - # interface: search_vectors - query: - # dataset: table name you have already created - # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity .. - [ - { - "dataset": "sift_50m_1024_128_l2", - "index.index_types": ["nsg"], - "index.nlists": [16384], - "nprobes": [8], - "top_ks": [64], - "nqs": [1, 10, 100], - "server.use_blas_threshold": 1100, - "server.use_gpu_threshold": 100, - "server.cpu_cache_capacity": 50, - "server.gpu_cache_capacity": 6, - "server.resources": ["gpu0", "gpu1"], - "server.enable_gpu": True, - "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_nsg" - }, - # { - # "dataset": "sift_50m_1024_128_l2", - # "index.index_types": ["ivf_flat"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 200, 500, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 50, - # "server.gpu_cache_capacity": 6, - # "server.resources": ["gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_ivf" - # }, - # { - # "dataset": "sift_50m_1024_128_l2", - # "index.index_types": ["ivf_sq8h"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 200, 500, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 50, - # "server.gpu_cache_capacity": 3, - # "server.resources": ["gpu0", "gpu1"], - # "server.enable_gpu": True, - # "db_path_prefix": "/test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8h" - # }, - # { - # "dataset": "sift_50m_1024_128_l2", - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 800, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 50, - # "server.gpu_cache_capacity": 6, - # "server.resources": ["gpu0", "gpu1"], - # "server.enable_gpu": True, - # "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_sq8" - # }, - # { - # "dataset": "sift_50m_1024_128_l2", - # "index.index_types": ["ivf_pq"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 200, 500, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 50, - # "server.gpu_cache_capacity": 6, - # "server.resources": ["gpu0", "gpu1"], - # "server.enable_gpu": True, - # "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_pq" - # }, - ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/search_debug.yaml b/tests/milvus_benchmark/suites/search_debug.yaml deleted file mode 100644 index 63960827ff..0000000000 --- a/tests/milvus_benchmark/suites/search_debug.yaml +++ /dev/null @@ -1,22 +0,0 @@ -search_performance: - tables: - - - server: - db_config.primary_path: /test/milvus/db_data_gpu/sift_1m_1024_128_l2_ivf - cache_config.cpu_cache_capacity: 8 - engine_config.use_blas_threshold: 1100 - engine_config.gpu_search_threshold: 1 - gpu_resource_config.enable: true - gpu_resource_config.cache_capacity: 4 - gpu_resource_config.search_resources: - - gpu0 - - gpu1 - gpu_resource_config.build_index_resources: - - gpu0 - - gpu1 - table_name: sift_1m_1024_128_l2 - run_count: 3 - search_params: - nprobes: [8, 32] - top_ks: [1, 16] - nqs: [1, 10] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/suites.yaml b/tests/milvus_benchmark/suites/suites.yaml deleted file mode 100644 index 475020aa8e..0000000000 --- a/tests/milvus_benchmark/suites/suites.yaml +++ /dev/null @@ -1,161 +0,0 @@ -performance: - - # interface: search_vectors - query: - # dataset: table name you have already created - # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity .. - [ - # { - # "dataset": "random_50m_2048_512_ip", - # "index.index_types": ["ivf_sq8h"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 800, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 50, - # "server.resources": ["cpu", "gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data/random_50m_2048_512_ip_sq8h" - # }, - # { - # "dataset": "random_50m_2048_512_ip", - # "index.index_types": ["ivf_sq8h"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 800, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 1000, - # "server.cpu_cache_capacity": 50, - # "server.resources": ["cpu", "gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data/random_50m_2048_512_ip_sq8h" - # }, - # { - # "dataset": "random_50m_2048_512_ip", - # "index.index_types": ["ivf_sq8h"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 800, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 10000, - # "server.cpu_cache_capacity": 50, - # "server.resources": ["gpu0", "gpu1"], - # "server.enable_gpu": True, - # "db_path_prefix": "/test/milvus/db_data/random_50m_2048_512_ip_sq8h" - # }, - # { - # "dataset": "random_50m_2048_512_ip", - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 800, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 50, - # "server.resources": ["gpu0", "gpu1"], - # "server.enable_gpu": True, - # "db_path_prefix": "/test/milvus/db_data/random_50m_2048_512_ip_sq8" - # }, - # { - # "dataset": "random_50m_2048_512_ip", - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 800, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 50, - # "server.enable_gpu": True, - # "server.resources": ["gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data_cpu/random_50m_2048_512_ip_sq8" - # }, - - # { - # "dataset": "random_50m_1024_512_ip", - # "index.index_types": ["ivf_sq8h"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 800, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 110, - # "server.resources": ["gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip" - # }, - # { - # "dataset": "random_50m_1024_512_ip", - # "index.index_types": ["ivf_sq8h"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 800, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 1000, - # "server.cpu_cache_capacity": 110, - # "server.resources": ["gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip" - # }, - # { - # "dataset": "random_50m_1024_512_ip", - # "index.index_types": ["ivf_sq8h"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 800, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 10000, - # "server.cpu_cache_capacity": 110, - # "server.resources": ["gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip" - # }, - # { - # "dataset": "random_50m_1024_512_ip", - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 800, 1000], - # "server.use_blas_threshold": 1100, - # "server.cpu_cache_capacity": 110, - # "server.resources": ["gpu0", "gpu1"], - # "server.enable_gpu": True, - # "db_path_prefix": "/test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8" - # }, - { - "dataset": "random_50m_1024_512_ip", - "index.index_types": ["ivf_pq"], - "index.nlists": [16384], - "nprobes": [8, 32], - "top_ks": [1, 4, 16, 64, 128, 256, 512, 1000], - "nqs": [1, 10, 100, 500, 800, 1000], - "server.use_blas_threshold": 1100, - "server.use_gpu_threshold": 100, - "server.cpu_cache_capacity": 110, - "server.gpu_cache_capacity": 6, - "server.index_build_device": "gpu0", - "server.resources": ["gpu0", "gpu1"], - "server.enable_gpu": True, - "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip_pq" - }, - # { - # "dataset": "random_50m_1024_512_ip", - # "index.index_types": ["ivf_flat"], - # "index.nlists": [16384], - # "nprobes": [8], - # "top_ks": [8], - # "nqs": [500], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 110, - # "server.gpu_cache_capacity": 6, - # "server.index_build_device": "gpu0", - # "server.resources": ["gpu0", "gpu1"], - # "server.enable_gpu": True, - # "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip_ivf" - # } - ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/suites_accuracy.yaml b/tests/milvus_benchmark/suites/suites_accuracy.yaml deleted file mode 100644 index 3dd91d2c02..0000000000 --- a/tests/milvus_benchmark/suites/suites_accuracy.yaml +++ /dev/null @@ -1,51 +0,0 @@ - -accuracy: - # interface: search_vectors - query: - [ - # { - # "dataset": "random_20m_1024_512_ip", - # # index info - # "index.index_types": ["flat", "ivf_sq8"], - # "index.nlists": [16384], - # "index.metric_types": ["ip"], - # "nprobes": [1, 16, 64], - # "top_ks": [64], - # "nqs": [100], - # "server.cpu_cache_capacity": 100, - # "server.resources": ["gpu0"], - # "db_path_prefix": "/test/milvus/db_data/random_20m_1024_512_ip", - # }, - # { - # "dataset": "sift_1m_1024_128_l2", - # "index.index_types": ["flat", "ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [1, 32, 128, 256, 512], - # "nqs": 10, - # "top_ks": 10, - # "server.use_blas_threshold": 1100, - # "server.cpu_cache_capacity": 16, - # }, - # { - # "dataset": "sift_10m_1024_128_l2", - # "index.index_types": ["flat", "ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [1, 32, 128, 256, 512], - # "nqs": 10, - # "top_ks": 10, - # "server.use_blas_threshold": 1100, - # "server.cpu_cache_capacity": 32, - # }, - # { - # "dataset": "sift_50m_1024_128_l2", - # "index.index_types": ["flat", "ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [1, 32, 128, 256, 512], - # "nqs": 10, - # "top_ks": 10, - # "server.use_blas_threshold": 1100, - # "server.cpu_cache_capacity": 64, - # } - - - ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/suites_performance.yaml b/tests/milvus_benchmark/suites/suites_performance.yaml deleted file mode 100644 index 50100f4646..0000000000 --- a/tests/milvus_benchmark/suites/suites_performance.yaml +++ /dev/null @@ -1,57 +0,0 @@ -performance: - - # interface: add_vectors - insert: - # index_type: flat/ivf_flat/ivf_sq8/mix_nsg - [ - # debug - # data_type / data_size / index_file_size / dimension - # data_type: random / ann_sift - # data_size: 10m / 1b - # { - # "table_name": "random_50m_1024_512_ip", - # "ni_per": 100000, - # "server.cpu_cache_capacity": 16, - # "server.resources": ["gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data" - # }, - # { - # "table_name": "random_2m_1024_512_ip", - # "ni_per": 100000, - # "processes": 5, # multiprocessing - # "server.cpu_cache_capacity": 16, - # "server.resources": ["gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data/random_5m_1024_512_ip" - # }, - # { - # "table_name": "sift_1m_50_128_l2", - # "ni_per": 100000, - # "processes": 5, # multiprocessing - # # "server.cpu_cache_capacity": 16, - # "db_path_prefix": "/test/milvus/db_data" - # }, - # { - # "table_name": "sift_1m_256_128_l2", - # "ni_per": 100000, - # "processes": 5, # multiprocessing - # # "server.cpu_cache_capacity": 16, - # "db_path_prefix": "/test/milvus/db_data" - # } - # { - # "table_name": "sift_50m_1024_128_l2", - # "ni_per": 100000, - # "processes": 5, # multiprocessing - # # "server.cpu_cache_capacity": 16, - # }, - # { - # "table_name": "sift_100m_1024_128_l2", - # "ni_per": 100000, - # "processes": 5, # multiprocessing - # }, - # { - # "table_name": "sift_1b_2048_128_l2", - # "ni_per": 100000, - # "processes": 5, # multiprocessing - # "server.cpu_cache_capacity": 16, - # } - ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/suites_stability.yaml b/tests/milvus_benchmark/suites/suites_stability.yaml deleted file mode 100644 index 427f48d924..0000000000 --- a/tests/milvus_benchmark/suites/suites_stability.yaml +++ /dev/null @@ -1,31 +0,0 @@ - -stability: - # interface: search_vectors / add_vectors mix operation - query: - [ - # { - # "dataset": "random_50m_1024_512_ip", - # "index_type": "ivf_sq8", - # "query_process_num": 10, - # "during_time": 960, - # "server.cpu_cache_capacity": 100, - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.resources": ["gpu0", "gpu1"], - # "server.enable_gpu": false, - # "db_path_prefix": "/test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8_stability" - # }, - - { - "dataset": "random_50m_1024_512_ip", - "index_type": "ivf_sq8", - "query_process_num": 10, - "during_time": 960, - "server.cpu_cache_capacity": 100, - "server.use_blas_threshold": 1100, - "server.use_gpu_threshold": 100, - "server.resources": ["gpu0", "gpu1"], - "server.enable_gpu": false, - "db_path_prefix": "/test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8_stability" - }, - ] \ No newline at end of file diff --git a/tests/milvus_benchmark/utils.py b/tests/milvus_benchmark/utils.py index 76bf04a873..3c6a43c991 100644 --- a/tests/milvus_benchmark/utils.py +++ b/tests/milvus_benchmark/utils.py @@ -22,14 +22,12 @@ import h5py from yaml import full_load, dump import tableprint as tp from pprint import pprint -from kubernetes import client, config logger = logging.getLogger("milvus_benchmark.utils") -config.load_kube_config() MULTI_DB_SLAVE_PATH = "/opt/milvus/data2;/opt/milvus/data3" - +REGISTRY_URL = "registry.zilliz.com/milvus/engine" def get_unique_name(): return "benchmark-test-"+"".join(random.choice(string.ascii_letters + string.digits) for _ in range(8)).lower() @@ -118,38 +116,88 @@ def update_server_config(file_path, server_config): # update values.yaml -def update_values(file_path, hostname): +def update_values(file_path, hostname, server_config): + from kubernetes import client, config + client.rest.logger.setLevel(logging.WARNING) + if not os.path.isfile(file_path): raise Exception('File: %s not found' % file_path) + #　bak values.yaml + file_name = os.path.basename(file_path) + bak_file_name = file_name+".bak" + file_parent_path = os.path.dirname(file_path) + bak_file_path = file_parent_path+'/'+bak_file_name + if os.path.exists(bak_file_path): + os.system("cp %s %s" % (bak_file_path, file_path)) + else: + os.system("cp %s %s" % (file_path, bak_file_path)) with open(file_path) as f: values_dict = full_load(f) f.close() - if values_dict['engine']['nodeSelector']: - logger.warning("nodeSelector has been set: %s" % str(values_dict['engine']['nodeSelector'])) - return + + for k, v in server_config.items(): + if k.find("primary_path") != -1: + values_dict["primaryPath"] = v + values_dict['wal']['path'] = v+"/wal" + elif k.find("use_blas_threshold") != -1: + values_dict['useBLASThreshold'] = int(v) + elif k.find("gpu_search_threshold") != -1: + values_dict['gpuSearchThreshold'] = int(v) + elif k.find("cpu_cache_capacity") != -1: + values_dict['cpuCacheCapacity'] = int(v) + elif k.find("cache_insert_data") != -1: + values_dict['cacheInsertData'] = v + elif k.find("insert_buffer_size") != -1: + values_dict['insertBufferSize'] = v + elif k.find("gpu_resource_config.enable") != -1: + values_dict['gpu']['enabled'] = v + elif k.find("gpu_resource_config.cache_capacity") != -1: + values_dict['gpu']['cacheCapacity'] = int(v) + elif k.find("build_index_resources") != -1: + values_dict['gpu']['buildIndexResources'] = v + elif k.find("search_resources") != -1: + values_dict['gpu']['searchResources'] = v + # wal + elif k.find("auto_flush_interval") != -1: + values_dict['autoFlushInterval'] = v + elif k.find("wal_enable") != -1: + values_dict['wal']['enabled'] = v + + # if values_dict['nodeSelector']: + # logger.warning("nodeSelector has been set: %s" % str(values_dict['engine']['nodeSelector'])) + # return + values_dict["wal"]["ignoreErrorLog"] = True + # enable monitor + values_dict["metrics"]["enabled"] = True + values_dict["metrics"]["address"] = "192.168.1.237" + values_dict["metrics"]["port"] = 9091 # update values.yaml with the given host - # set limit/request cpus in resources + values_dict['nodeSelector'] = {'kubernetes.io/hostname': hostname} + # Using sqlite + values_dict["mysql"]["enabled"] = False + + config.load_kube_config() v1 = client.CoreV1Api() - node = v1.read_node(hostname) - cpus = node.status.allocatable.get("cpu") + # node = v1.read_node(hostname) + cpus = v1.read_node(hostname).status.allocatable.get("cpu") # DEBUG - values_dict['engine']['resources'] = { + # set limit/request cpus in resources + values_dict['resources'] = { "limits": { - "cpu": str(int(cpus)-1)+".0" + "cpu": str(int(cpus))+".0" }, "requests": { - "cpu": str(int(cpus)-2)+".0" + "cpu": str(int(cpus)-1)+".0" } } - values_dict['engine']['nodeSelector'] = {'kubernetes.io/hostname': hostname} - values_dict['engine']['tolerations'].append({ + values_dict['tolerations'] = [{ "key": "worker", "operator": "Equal", "value": "performance", "effect": "NoSchedule" - }) + }] # add extra volumes - values_dict['extraVolumes'].append({ + values_dict['extraVolumes'] = [{ 'name': 'test', 'flexVolume': { 'driver': "fstab/cifs", @@ -162,11 +210,12 @@ def update_values(file_path, hostname): 'mountOptions': "vers=1.0" } } - }) - values_dict['extraVolumeMounts'].append({ + }] + values_dict['extraVolumeMounts'] = [{ 'name': 'test', 'mountPath': '/test' - }) + }] + logger.debug(values_dict) with open(file_path, 'w') as f: dump(values_dict, f, default_flow_style=False) f.close() @@ -174,22 +223,27 @@ def update_values(file_path, hostname): # deploy server def helm_install_server(helm_path, image_tag, image_type, name, namespace): - timeout = 180 - install_cmd = "helm install --wait --timeout %d \ - --set engine.image.tag=%s \ - --set expose.type=clusterIP \ - --name %s \ - -f ci/db_backend/sqlite_%s_values.yaml \ + from kubernetes import client, config + client.rest.logger.setLevel(logging.WARNING) + + timeout = 300 + install_cmd = "helm install --wait --timeout %ds \ + --set image.repository=%s \ + --set image.tag=%s \ + --set image.pullPolicy=Always \ + --set service.type=ClusterIP \ -f ci/filebeat/values.yaml \ --namespace %s \ - --version 0.0 ." % (timeout, image_tag, name, image_type, namespace) + %s ." % (timeout, REGISTRY_URL, image_tag, namespace, name) logger.debug(install_cmd) if os.system("cd %s && %s" % (helm_path, install_cmd)): logger.error("Helm install failed") return None time.sleep(5) + config.load_kube_config() v1 = client.CoreV1Api() - host = "%s-milvus-engine.%s.svc.cluster.local" % (name, namespace) + host = "%s.%s.svc.cluster.local" % (name, namespace) + logger.debug(host) pod_name = None pod_id = None pods = v1.list_namespaced_pod(namespace) @@ -203,8 +257,10 @@ def helm_install_server(helm_path, image_tag, image_type, name, namespace): # delete server -def helm_del_server(name): - del_cmd = "helm del --purge %s" % name +def helm_del_server(name, namespace): + # del_cmd = "helm uninstall -n milvus benchmark-test-gzelwvgk" + # os.system(del_cmd) + del_cmd = "helm uninstall -n milvus %s" % name logger.debug(del_cmd) if os.system(del_cmd): logger.error("Helm delete name:%s failed" % name) @@ -342,4 +398,4 @@ def helm_del_server(name): if __name__ == '__main__': # print(pull_image('branch-0.3.1-debug')) - stop_server() \ No newline at end of file + stop_server() diff --git a/tests/milvus_python_test/test_add_vectors.py b/tests/milvus_python_test/test_add_vectors.py index 6307418d09..ec00f487b3 100644 --- a/tests/milvus_python_test/test_add_vectors.py +++ b/tests/milvus_python_test/test_add_vectors.py @@ -678,6 +678,11 @@ class TestAddBase: assert status.OK() class TestAddAsync: + @pytest.fixture(scope="function", autouse=True) + def skip_http_check(self, args): + if args["handler"] == "HTTP": + pytest.skip("skip in http mode") + @pytest.fixture( scope="function", params=[ diff --git a/tests/milvus_python_test/test_flush.py b/tests/milvus_python_test/test_flush.py index c32b9ad397..5bc257a822 100644 --- a/tests/milvus_python_test/test_flush.py +++ b/tests/milvus_python_test/test_flush.py @@ -234,6 +234,11 @@ class TestFlushBase: class TestFlushAsync: + @pytest.fixture(scope="function", autouse=True) + def skip_http_check(self, args): + if args["handler"] == "HTTP": + pytest.skip("skip in http mode") + """ ****************************************************************** The following cases are used to test `flush` function diff --git a/tests/milvus_python_test/test_index.py b/tests/milvus_python_test/test_index.py index 3949f38a4d..73f966b467 100644 --- a/tests/milvus_python_test/test_index.py +++ b/tests/milvus_python_test/test_index.py @@ -1808,6 +1808,11 @@ class TestCreateIndexParamsInvalid(object): assert result._index_type == IndexType.FLAT class TestIndexAsync: + @pytest.fixture(scope="function", autouse=True) + def skip_http_check(self, args): + if args["handler"] == "HTTP": + pytest.skip("skip in http mode") + """ ****************************************************************** The following cases are used to test `create_index` function diff --git a/tests/milvus_python_test/test_ping.py b/tests/milvus_python_test/test_ping.py index f1addb7ef1..92ab12c868 100644 --- a/tests/milvus_python_test/test_ping.py +++ b/tests/milvus_python_test/test_ping.py @@ -1,7 +1,7 @@ import logging import pytest -__version__ = '0.8.0' +__version__ = '0.9.0' class TestPing: