test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-05 05:18:52 +08:00
Code Issues Actions 14 Packages Projects Releases Wiki Activity

Merge branch '0.5.1' into 0.5.1-yzb

Browse Source 
Former-commit-id: 1e1a78a79db59c10c3ede8fd3187aea41c582c59
This commit is contained in:
Heisenberg 2019-10-31 14:43:11 +08:00
commit 90e961649c
9 changed files with 272 additions and 186 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -5,11 +5,13 @@ Please mark all change in change log and use the ticket from JIRA.
# Milvus 0.5.1 (TODO)
## Bug
- \#134 - JFrog cache error
## Feature
- \#90 - The server start error messages could be improved to enhance user experience
- \#104 - test_scheduler core dump
- \#115 - Using new structure for tasktable
- \#139 - New config opion use_gpu_threshold
## Improvement
- \#64 - Improvement dump function in scheduler

6
core/CMakeLists.txt
View File

@ -32,10 +32,14 @@ string(REGEX REPLACE "\n" "" BUILD_TIME ${BUILD_TIME})
message(STATUS "Build time = ${BUILD_TIME}")
MACRO (GET_GIT_BRANCH_NAME GIT_BRANCH_NAME)
execute_process(COMMAND "git" symbolic-ref --short HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
execute_process(COMMAND "git" rev-parse --abbrev-ref HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
if(GIT_BRANCH_NAME STREQUAL "")
execute_process(COMMAND "git" symbolic-ref --short -q HEAD OUTPUT_VARIABLE ${GIT_BRANCH_NAME})
endif()
ENDMACRO (GET_GIT_BRANCH_NAME)
GET_GIT_BRANCH_NAME(GIT_BRANCH_NAME)
message(STATUS "GIT_BRANCH_NAME = ${GIT_BRANCH_NAME}")
if(NOT GIT_BRANCH_NAME STREQUAL "")
string(REGEX REPLACE "\n" "" GIT_BRANCH_NAME ${GIT_BRANCH_NAME})
endif()

11
core/cmake/ThirdPartyPackages.cmake
View File

@ -108,6 +108,13 @@ if (UNIX)
if (UBUNTU_FOUND)
set(CMAKE_OS_NAME "ubuntu" CACHE STRING "Operating system name" FORCE)
set(DEBIAN_FOUND FALSE)
find_program(LSB_RELEASE_EXEC lsb_release)
execute_process(COMMAND ${LSB_RELEASE_EXEC} -rs
OUTPUT_VARIABLE LSB_RELEASE_ID_SHORT
OUTPUT_STRIP_TRAILING_WHITESPACE
)
STRING(REGEX REPLACE "\\." "_" UBUNTU_VERSION ${LSB_RELEASE_ID_SHORT})
endif (UBUNTU_FOUND)
endif (UBUNTU_EXTRA)
endif (DEBIAN_FOUND)
@ -130,7 +137,11 @@ if(USE_JFROG_CACHE STREQUAL "ON")
if (NOT DEFINED JFROG_ARTFACTORY_URL)
message(FATAL_ERROR "JFROG_ARTFACTORY_URL is not set")
endif ()
if (UBUNTU_FOUND)
set(JFROG_ARTFACTORY_CACHE_URL "${JFROG_ARTFACTORY_URL}/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${UBUNTU_VERSION}/${MILVUS_BUILD_ARCH}/${BUILD_TYPE}")
else ()
set(JFROG_ARTFACTORY_CACHE_URL "${JFROG_ARTFACTORY_URL}/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${MILVUS_BUILD_ARCH}/${BUILD_TYPE}")
endif ()
if (DEFINED ENV{JFROG_USER_NAME})
set(JFROG_USER_NAME "$ENV{JFROG_USER_NAME}")
endif ()

1
core/conf/server_config.template
View File

@ -36,6 +36,7 @@ cache_config:
engine_config:
use_blas_threshold: 20 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times
# if nq >= use_blas_threshold, use OpenBlas, slower with stable response times
use_gpu_threshold: 1000
resource_config:
search_resources: # define the GPUs used for search computation, must be in format: gpux

3
core/src/index/cmake/ThirdPartyPackagesCore.cmake
View File

@ -245,7 +245,8 @@ if(CUSTOMIZATION)
# set(FAISS_MD5 "072db398351cca6e88f52d743bbb9fa0") # commit-id 3a2344d04744166af41ef1a74449d68a315bfe17 branch-0.2.1
# set(FAISS_MD5 "c89ea8e655f5cdf58f42486f13614714") # commit-id 9c28a1cbb88f41fa03b03d7204106201ad33276b branch-0.2.1
# set(FAISS_MD5 "87fdd86351ffcaf3f80dc26ade63c44b") # commit-id 841a156e67e8e22cd8088e1b58c00afbf2efc30b branch-0.2.1
set(FAISS_MD5 "f3b2ce3364c3fa7febd3aa7fdd0fe380") # commit-id 694e03458e6b69ce8a62502f71f69a614af5af8f branch-0.3.0
# set(FAISS_MD5 "f3b2ce3364c3fa7febd3aa7fdd0fe380") # commit-id 694e03458e6b69ce8a62502f71f69a614af5af8f branch-0.3.0
set(FAISS_MD5 "bb30722c22390ce5f6759ccb216c1b2a") # commit-id d324db297475286afe107847c7fb7a0f9dc7e90e branch-0.3.0
endif()
else()
set(FAISS_SOURCE_URL "https://github.com/milvus-io/faiss/archive/1.6.0.tar.gz")

12
core/src/scheduler/optimizer/LargeSQ8HPass.cpp
View File

@ -21,11 +21,20 @@
#include "scheduler/Utils.h"
#include "scheduler/task/SearchTask.h"
#include "scheduler/tasklabel/SpecResLabel.h"
#include "server/Config.h"
#include "utils/Log.h"
namespace milvus {
namespace scheduler {
LargeSQ8HPass::LargeSQ8HPass() {
server::Config& config = server::Config::GetInstance();
Status s = config.GetEngineConfigUseGpuThreshold(threshold_);
if (!s.ok()) {
threshold_ = std::numeric_limits<int32_t>::max();
}
}
bool
LargeSQ8HPass::Run(const TaskPtr& task) {
if (task->Type() != TaskType::SearchTask) {
@ -40,7 +49,8 @@ LargeSQ8HPass::Run(const TaskPtr& task) {
auto search_job = std::static_pointer_cast<SearchJob>(search_task->job_.lock());
// TODO: future, Index::IVFSQ8H, if nq < threshold set cpu, else set gpu
if (search_job->nq() < 100) {
if (search_job->nq() < threshold_) {
return false;
}

6
core/src/scheduler/optimizer/LargeSQ8HPass.h
View File

@ -18,6 +18,7 @@
#include <condition_variable>
#include <deque>
#include <limits>
#include <list>
#include <memory>
#include <mutex>
@ -34,11 +35,14 @@ namespace scheduler {
class LargeSQ8HPass : public Pass {
public:
LargeSQ8HPass() = default;
LargeSQ8HPass();
public:
bool
Run(const TaskPtr& task) override;
private:
int32_t threshold_ = std::numeric_limits<int32_t>::max();
};
using LargeSQ8HPassPtr = std::shared_ptr<LargeSQ8HPass>;

45
core/src/server/Config.cpp
View File

@ -193,6 +193,12 @@ Config::ValidateConfig() {
return s;
}
int32_t engine_use_gpu_threshold;
s = GetEngineConfigUseGpuThreshold(engine_use_gpu_threshold);
if (!s.ok()) {
return s;
}
/* resource config */
std::string resource_mode;
s = GetResourceConfigMode(resource_mode);
@ -324,6 +330,11 @@ Config::ResetDefaultConfig() {
return s;
}
s = SetEngineConfigUseGpuThreshold(CONFIG_ENGINE_USE_GPU_THRESHOLD_DEFAULT);
if (!s.ok()) {
return s;
}
/* resource config */
s = SetResourceConfigMode(CONFIG_RESOURCE_MODE_DEFAULT);
if (!s.ok()) {
@ -656,6 +667,16 @@ Config::CheckEngineConfigOmpThreadNum(const std::string& value) {
return Status::OK();
}
Status
Config::CheckEngineConfigUseGpuThreshold(const std::string& value) {
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid gpu threshold: " + value +
". Possible reason: engine_config.use_gpu_threshold is not a positive integer.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
}
Status
Config::CheckResourceConfigMode(const std::string& value) {
if (value != "simple") {
@ -951,6 +972,19 @@ Config::GetEngineConfigOmpThreadNum(int32_t& value) {
return Status::OK();
}
Status
Config::GetEngineConfigUseGpuThreshold(int32_t& value) {
std::string str =
GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_USE_GPU_THRESHOLD, CONFIG_ENGINE_USE_GPU_THRESHOLD_DEFAULT);
Status s = CheckEngineConfigUseGpuThreshold(str);
if (!s.ok()) {
return s;
}
value = std::stoi(str);
return Status::OK();
}
Status
Config::GetResourceConfigMode(std::string& value) {
value = GetConfigStr(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, CONFIG_RESOURCE_MODE_DEFAULT);
@ -1203,6 +1237,17 @@ Config::SetEngineConfigOmpThreadNum(const std::string& value) {
return Status::OK();
}
Status
Config::SetEngineConfigUseGpuThreshold(const std::string& value) {
Status s = CheckEngineConfigUseGpuThreshold(value);
if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_USE_GPU_THRESHOLD, value);
return Status::OK();
}
/* resource config */
Status
Config::SetResourceConfigMode(const std::string& value) {

8
core/src/server/Config.h
View File

@ -84,6 +84,8 @@ static const char* CONFIG_ENGINE_USE_BLAS_THRESHOLD = "use_blas_threshold";
static const char* CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT = "20";
static const char* CONFIG_ENGINE_OMP_THREAD_NUM = "omp_thread_num";
static const char* CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0";
static const char* CONFIG_ENGINE_USE_GPU_THRESHOLD = "use_gpu_threshold";
static const char* CONFIG_ENGINE_USE_GPU_THRESHOLD_DEFAULT = "1000";
/* resource config */
static const char* CONFIG_RESOURCE = "resource_config";
@ -166,6 +168,8 @@ class Config {
CheckEngineConfigUseBlasThreshold(const std::string& value);
Status
CheckEngineConfigOmpThreadNum(const std::string& value);
Status
CheckEngineConfigUseGpuThreshold(const std::string& value);
/* resource config */
Status
@ -230,6 +234,8 @@ class Config {
GetEngineConfigUseBlasThreshold(int32_t& value);
Status
GetEngineConfigOmpThreadNum(int32_t& value);
Status
GetEngineConfigUseGpuThreshold(int32_t& value);
/* resource config */
Status
@ -289,6 +295,8 @@ class Config {
SetEngineConfigUseBlasThreshold(const std::string& value);
Status
SetEngineConfigOmpThreadNum(const std::string& value);
Status
SetEngineConfigUseGpuThreshold(const std::string& value);
/* resource config */
Status