test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-01 11:29:48 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

fix conflict in SchedInst

Browse Source
This commit is contained in:
fishpenguin 2019-11-19 19:30:53 +08:00
commit 30dcfa6719
22 changed files with 142 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -31,6 +31,8 @@ Please mark all change in change log and use the ticket from JIRA.
- \#322 - Add option to enable / disable prometheus
- \#358 - Add more information in build.sh and install.md
- \#255 - Add ivfsq8 test report detailed version
- \#404 - Add virtual method Init() in Pass abstract class
- \#409 - Add a Fallback pass in optimizer
## Task

6
ci/jenkins/step/deploySingle2Dev.groovy
View File

@ -3,11 +3,7 @@ sh 'helm repo update'
dir ('milvus-helm') {
checkout([$class: 'GitSCM', branches: [[name: "0.6.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.6.0:refs/remotes/origin/0.6.0"]]])
dir ("milvus") {
if ("${env.BINRARY_VERSION}" == "gpu") {
sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f gpu_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
} else {
sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/filebeat/values.yaml --namespace milvus ."
}
sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/sqlite_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
}
}

6
ci/jenkins/step/singleDevNightlyTest.groovy
View File

@ -13,11 +13,7 @@ timeout(time: 90, unit: 'MINUTES') {
}
dir ("milvus-helm") {
dir ("milvus") {
if ("${env.BINRARY_VERSION}" == "gpu") {
sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f gpu_values.yaml -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
} else {
sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
}
sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
}
}
dir ("tests/milvus_python_test") {

6
ci/jenkins/step/singleDevTest.groovy
View File

@ -14,11 +14,7 @@ timeout(time: 60, unit: 'MINUTES') {
// }
// dir ("milvus-helm") {
// dir ("milvus") {
// if ("${env.BINRARY_VERSION}" == "gpu") {
// sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f gpu_values.yaml -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
// } else {
// sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
// }
// sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ."
// }
// }
// dir ("tests/milvus_python_test") {

1
core/src/scheduler/SchedInst.cpp
View File

@ -91,6 +91,7 @@ load_simple_config() {
void
StartSchedulerService() {
load_simple_config();
OptimizerInst::GetInstance()->Init();
ResMgrInst::GetInstance()->Start();
SchedInst::GetInstance()->Start();
JobMgrInst::GetInstance()->Start();

3
core/src/scheduler/SchedInst.h
View File

@ -23,6 +23,7 @@
#include "Scheduler.h"
#include "Utils.h"
#include "optimizer/BuildIndexPass.h"
#include "optimizer/FallbackPass.h"
#include "optimizer/HybridPass.h"
#include "optimizer/LargeSQ8HPass.h"
#include "optimizer/OnlyCPUPass.h"
@ -103,12 +104,14 @@ class OptimizerInst {
pass_list.push_back(std::make_shared<HybridPass>());
#ifdef MILVUS_CPU_VERSION
pass_list.push_back(std::make_shared<OnlyCPUPass>());
<<<<<<< HEAD
#else
server::Config& config = server::Config::GetInstance();
std::vector<int32_t> build_resources;
config.GetGpuResourceConfigBuildIndexResources(build_resources);
pass_list.push_back(std::make_shared<BuildIndexPass>(build_resources));
#endif
// pass_list.push_back(std::make_shared<FallbackPass>());
instance = std::make_shared<Optimizer>(pass_list);
}
}

4
core/src/scheduler/optimizer/BuildIndexPass.cpp
View File

@ -26,6 +26,10 @@ namespace scheduler {
BuildIndexPass::BuildIndexPass(std::vector<int32_t>& build_gpu_ids) : build_gpu_ids_(build_gpu_ids) {
}
void
BuildIndexPass::Init() {
}
bool
BuildIndexPass::Run(const TaskPtr& task) {
if (task->Type() != TaskType::BuildIndexTask)

3
core/src/scheduler/optimizer/BuildIndexPass.h
View File

@ -37,6 +37,9 @@ class BuildIndexPass : public Pass {
explicit BuildIndexPass(std::vector<int32_t>& build_gpu_id);
public:
void
Init() override;
bool
Run(const TaskPtr& task) override;

43
core/src/scheduler/optimizer/FallbackPass.cpp Normal file
View File

@ -0,0 +1,43 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "scheduler/optimizer/FallbackPass.h"
#include "scheduler/SchedInst.h"
#include "scheduler/tasklabel/SpecResLabel.h"
namespace milvus {
namespace scheduler {
void
FallbackPass::Init() {
}
bool
FallbackPass::Run(const TaskPtr& task) {
auto task_type = task->Type();
if (task_type != TaskType::SearchTask && task_type != TaskType::BuildIndexTask) {
return false;
}
// NEVER be empty
auto cpu = ResMgrInst::GetInstance()->GetCpuResources()[0];
auto label = std::make_shared<SpecResLabel>(cpu);
task->label() = label;
return true;
}
} // namespace scheduler
} // namespace milvus

40
core/src/scheduler/optimizer/FallbackPass.h Normal file
View File

@ -0,0 +1,40 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <bits/stdc++.h>
#include <memory>
#include "Pass.h"
namespace milvus {
namespace scheduler {
class FallbackPass : public Pass {
public:
FallbackPass() = default;
public:
void
Init() override;
bool
Run(const TaskPtr& task) override;
};
} // namespace scheduler
} // namespace milvus

4
core/src/scheduler/optimizer/HybridPass.cpp
View File

@ -23,6 +23,10 @@
namespace milvus {
namespace scheduler {
void
HybridPass::Init() {
}
bool
HybridPass::Run(const TaskPtr& task) {
// TODO: future, Index::IVFSQ8H, if nq < threshold set cpu, else set gpu

3
core/src/scheduler/optimizer/HybridPass.h
View File

@ -37,6 +37,9 @@ class HybridPass : public Pass {
HybridPass() = default;
public:
void
Init() override;
bool
Run(const TaskPtr& task) override;
};

3
core/src/scheduler/optimizer/LargeSQ8HPass.cpp
View File

@ -27,7 +27,8 @@
namespace milvus {
namespace scheduler {
LargeSQ8HPass::LargeSQ8HPass() {
void
LargeSQ8HPass::Init() {
server::Config& config = server::Config::GetInstance();
Status s = config.GetEngineConfigGpuSearchThreshold(threshold_);
if (!s.ok()) {

5
core/src/scheduler/optimizer/LargeSQ8HPass.h
View File

@ -35,9 +35,12 @@ namespace scheduler {
class LargeSQ8HPass : public Pass {
public:
LargeSQ8HPass();
LargeSQ8HPass() = default;
public:
void
Init() override;
bool
Run(const TaskPtr& task) override;

4
core/src/scheduler/optimizer/OnlyCPUPass.cpp
View File

@ -24,6 +24,10 @@
namespace milvus {
namespace scheduler {
void
OnlyCPUPass::Init() {
}
bool
OnlyCPUPass::Run(const TaskPtr& task) {
if (task->Type() != TaskType::SearchTask)

3
core/src/scheduler/optimizer/OnlyCPUPass.h
View File

@ -37,6 +37,9 @@ class OnlyCPUPass : public Pass {
OnlyCPUPass() = default;
public:
void
Init() override;
bool
Run(const TaskPtr& task) override;
};

12
core/src/scheduler/optimizer/Optimizer.cpp
View File

@ -20,12 +20,12 @@
namespace milvus {
namespace scheduler {
// void
// Optimizer::Init() {
// for (auto& pass : pass_list_) {
// pass->Init();
// }
// }
void
Optimizer::Init() {
for (auto& pass : pass_list_) {
pass->Init();
}
}
bool
Optimizer::Run(const TaskPtr& task) {

4
core/src/scheduler/optimizer/Optimizer.h
View File

@ -38,8 +38,8 @@ class Optimizer {
explicit Optimizer(std::vector<PassPtr> pass_list) : pass_list_(std::move(pass_list)) {
}
// void
// Init();
void
Init();
bool
Run(const TaskPtr& task);

5
core/src/scheduler/optimizer/Pass.h
View File

@ -34,9 +34,8 @@ namespace scheduler {
class Pass {
public:
// virtual void
// Init() {
// }
virtual void
Init() = 0;
virtual bool
Run(const TaskPtr& task) = 0;

12
docs/test_report/milvus_ivfsq8_test_report_detailed_version.md
View File

@ -28,13 +28,13 @@ Memory: 503GB
Docker version: 18.09
Nvidia Driver version: 430.34
NVIDIA Driver version: 430.34
Milvus version: 0.5.3
SDK interface: Python 3.6.8
Pymilvus version: 0.2.5
pymilvus version: 0.2.5
@ -66,7 +66,7 @@ For details on this dataset, you can check : http://corpus-texmex.irisa.fr/ .
> Note: In the query test of recall, we will test the following parameters with different values:
>
> nq - grouped by: [1, 5, 10, 200, 400, 600, 800, 1000],
> nq - grouped by: [10, 200, 400, 600, 800, 1000],
>
> topk - grouped by: [1, 10, 100]
@ -93,7 +93,7 @@ Milvus configuration
- gpu_cache_capacity: 6
- use_blas_threshold: 1100
You can check the definition of Milvus configuration on https://milvus.io/docs/en/reference/milvus_config/.
The definitions of Milvus configuration are on https://milvus.io/docs/en/reference/milvus_config/.
Test method
@ -177,8 +177,6 @@ search_resources: gpu0, gpu1
| nq/topk | topk=1 | topk=10 | topk=100 |
| :-----: | :----: | :-----: | :------: |
| nq=1 | 1.000 | 0.800 | 0.790 |
| nq=5 | 0.800 | 0.820 | 0.908 |
| nq=10 | 0.900 | 0.910 | 0.939 |
| nq=200 | 0.955 | 0.941 | 0.929 |
| nq=400 | 0.958 | 0.944 | 0.932 |
@ -194,8 +192,6 @@ search_resources: cpu, gpu0
| nq/topk | topk=1 | topk=10 | topk=100 |
| :-----: | :----: | :-----: | :------: |
| nq=1 | 1.000 | 0.800 | 0.790 |
| nq=5 | 0.800 | 0.820 | 0.908 |
| nq=10 | 0.900 | 0.910 | 0.939 |
| nq=200 | 0.955 | 0.941 | 0.929 |
| nq=400 | 0.958 | 0.944 | 0.932 |

16
docs/test_report/milvus_ivfsq8_test_report_detailed_version_cn.md
View File

@ -2,7 +2,7 @@
## 概述
本文描述了ivfsq8索引在milvus单机部署方式下的测试报告。
本文描述了ivfsq8索引在milvus单机部署方式下的测试结果。
@ -28,13 +28,13 @@ GPU1: GeForce GTX 1080
Docker版本: 18.09
Nvidia Driver版本: 430.34
NVIDIA Driver版本: 430.34
Milvus版本: 0.5.3
SDK接口: Python 3.6.8
Pymilvus版本: 0.2.5
pymilvus版本: 0.2.5
@ -66,7 +66,7 @@ Pymilvus版本: 0.2.5
> 备注：在向量准确性测试中，我们会测试下面参数不同的取值来观察结果：
>
> 被查询向量的数量nq将按照 [1, 5, 10, 200, 400, 600, 800, 1000]的数量分组，
> 被查询向量的数量nq将按照 [10, 200, 400, 600, 800, 1000]的数量分组，
>
> 单条查询中最相似的K个结果topk将按照[1, 10, 100]的数量分组。
@ -93,7 +93,7 @@ Milvus设置：
- gpu_cache_capacity: 6
- use_blas_threshold: 1100
你可以在 https://milvus.io/docs/en/reference/milvus_config/上查询Milvus设置的详细定义。
Milvus设置的详细定义可以参考 https://milvus.io/docs/en/reference/milvus_config/ 。
测试方法
@ -177,8 +177,6 @@ search_resources: gpu0, gpu1
| nq/topk | topk=1 | topk=10 | topk=100 |
| :-----: | :----: | :-----: | :------: |
| nq=1 | 1.000 | 0.800 | 0.790 |
| nq=5 | 0.800 | 0.820 | 0.908 |
| nq=10 | 0.900 | 0.910 | 0.939 |
| nq=200 | 0.955 | 0.941 | 0.929 |
| nq=400 | 0.958 | 0.944 | 0.932 |
@ -194,8 +192,6 @@ search_resources: cpu, gpu0
| nq/topk | topk=1 | topk=10 | topk=100 |
| :-----: | :----: | :-----: | :------: |
| nq=1 | 1.000 | 0.800 | 0.790 |
| nq=5 | 0.800 | 0.820 | 0.908 |
| nq=10 | 0.900 | 0.910 | 0.939 |
| nq=200 | 0.955 | 0.941 | 0.929 |
| nq=400 | 0.958 | 0.944 | 0.932 |
@ -207,5 +203,5 @@ search_resources: cpu, gpu0
**总结**
​ 随着nq的增大，召回率逐渐稳定至93%以上。
随着nq的增大，召回率逐渐稳定至93%以上。

3
install.md
View File

@ -3,6 +3,9 @@
## Software requirements
- Ubuntu 18.04 or higher
If your operating system is not Ubuntu 18.04 or higher, we recommend you to pull a [docker image of Ubuntu 18.04](https://docs.docker.com/install/linux/docker-ce/ubuntu/) as your compilation environment.
- CMake 3.12 or higher
##### For GPU version, you will also need: