diff --git a/CHANGELOG.md b/CHANGELOG.md index 161225e636..5f913fce70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,13 @@ Please mark all change in change log and use the issue from GitHub - \#715 - Milvus crash when searching and building index simultaneously using SQ8H - \#744 - Don't return partition table for show_tables - \#770 - Server unittest run failed on low-end server +- \#831 - Judge branch error in CommonUtil.cpp ## Feature +- \#216 - Add CLI to get server info - \#343 - Add Opentracing - \#665 - Support get/set config via CLI +- \#766 - If partition tag is similar, wrong partition is searched - \#771 - Add server build commit info interface - \#759 - Put C++ sdk out of milvus/core diff --git a/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml b/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml index 47735937b6..cfbae6d9ce 100644 --- a/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml @@ -25,7 +25,7 @@ spec: resources: limits: memory: "12Gi" - cpu: "4.0" + cpu: "6.0" requests: memory: "8Gi" cpu: "4.0" diff --git a/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml b/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml index f36569407a..4140d7a5ff 100644 --- a/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml @@ -24,7 +24,7 @@ spec: resources: limits: memory: "12Gi" - cpu: "4.0" + cpu: "6.0" requests: memory: "8Gi" cpu: "4.0" diff --git a/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml b/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml index 55abdff5a3..4a7b970468 100644 --- a/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml @@ -25,7 +25,7 @@ spec: resources: limits: memory: "12Gi" - cpu: "4.0" + cpu: "6.0" nvidia.com/gpu: 1 requests: memory: "8Gi" diff --git a/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml b/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml index 43a4152342..a739a344e0 100644 --- a/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml @@ -24,7 +24,7 @@ spec: resources: limits: memory: "12Gi" - cpu: "4.0" + cpu: "6.0" nvidia.com/gpu: 1 requests: memory: "8Gi" diff --git a/ci/jenkins/step/singleDevNightlyTest.groovy b/ci/jenkins/step/singleDevNightlyTest.groovy index 74a8768cba..823cc25660 100644 --- a/ci/jenkins/step/singleDevNightlyTest.groovy +++ b/ci/jenkins/step/singleDevNightlyTest.groovy @@ -1,6 +1,6 @@ timeout(time: 90, unit: 'MINUTES') { dir ("tests/milvus_python_test") { - sh 'python3 -m pip install -r requirements.txt' + sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com' sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.HELM_RELEASE_NAME}-engine.milvus.svc.cluster.local" } // mysql database backend test diff --git a/ci/jenkins/step/singleDevTest.groovy b/ci/jenkins/step/singleDevTest.groovy index 291b4470e5..0ad9e369bf 100644 --- a/ci/jenkins/step/singleDevTest.groovy +++ b/ci/jenkins/step/singleDevTest.groovy @@ -1,6 +1,6 @@ timeout(time: 60, unit: 'MINUTES') { dir ("tests/milvus_python_test") { - sh 'python3 -m pip install -r requirements.txt' + sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com' sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.HELM_RELEASE_NAME}-engine.milvus.svc.cluster.local" } diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index b440edff02..125677355b 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -19,13 +19,12 @@ FAISS_ROOT="" PRIVILEGES="OFF" CUSTOMIZATION="OFF" # default use origin faiss BUILD_COVERAGE="OFF" -USE_JFROG_CACHE="OFF" RUN_CPPLINT="OFF" GPU_VERSION="OFF" WITH_MKL="OFF" CUDA_COMPILER=/usr/local/cuda/bin/nvcc -while getopts "o:t:b:f:pgxulcjmh" arg +while getopts "o:t:b:f:pgxulcmh" arg do case $arg in o) @@ -59,9 +58,6 @@ do c) BUILD_COVERAGE="ON" ;; - j) - USE_JFROG_CACHE="ON" - ;; m) WITH_MKL="ON" ;; @@ -79,12 +75,11 @@ parameter: -u: building unit test options(default: OFF) -l: run cpplint, clang-format and clang-tidy(default: OFF) -c: code coverage(default: OFF) --j: use jfrog cache build directory(default: OFF) -m: build with MKL(default: OFF) -h: help usage: -./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-p] [-g] [-x] [-u] [-l] [-c] [-j] [-m] [-h] +./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-p] [-g] [-x] [-u] [-l] [-c] [-m] [-h] " exit 0 ;; @@ -112,7 +107,6 @@ CMAKE_CMD="cmake \ -DCUSTOMIZATION=${CUSTOMIZATION} \ -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \ -DBUILD_COVERAGE=${BUILD_COVERAGE} \ --DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ -DFAISS_ROOT=${FAISS_ROOT} \ -DFAISS_WITH_MKL=${WITH_MKL} \ -DArrow_SOURCE=AUTO \ diff --git a/core/src/metrics/SystemInfo.cpp b/core/src/metrics/SystemInfo.cpp index 1421cc1c95..90d670b453 100644 --- a/core/src/metrics/SystemInfo.cpp +++ b/core/src/metrics/SystemInfo.cpp @@ -16,17 +16,14 @@ // under the License. #include "metrics/SystemInfo.h" +#include "thirdparty/nlohmann/json.hpp" #include "utils/Log.h" #include -#include -#include -#include +#include +#include #include -#include -#include -#include -#include +#include #ifdef MILVUS_GPU_VERSION #include @@ -350,5 +347,25 @@ SystemInfo::Octets() { return res; } +void +SystemInfo::GetSysInfoJsonStr(std::string& result) { + std::map sys_info_map; + + sys_info_map["memory_total"] = std::to_string(GetPhysicalMemory()); + sys_info_map["memory_used"] = std::to_string(GetProcessUsedMemory()); + + std::vector gpu_mem_total = GPUMemoryTotal(); + std::vector gpu_mem_used = GPUMemoryUsed(); + for (size_t i = 0; i < gpu_mem_total.size(); i++) { + std::string key_total = "gpu" + std::to_string(i) + "_memory_total"; + std::string key_used = "gpu" + std::to_string(i) + "_memory_used"; + sys_info_map[key_total] = std::to_string(gpu_mem_total[i]); + sys_info_map[key_used] = std::to_string(gpu_mem_used[i]); + } + + nlohmann::json sys_info_json(sys_info_map); + result = sys_info_json.dump(); +} + } // namespace server } // namespace milvus diff --git a/core/src/metrics/SystemInfo.h b/core/src/metrics/SystemInfo.h index 0176475232..03cada2033 100644 --- a/core/src/metrics/SystemInfo.h +++ b/core/src/metrics/SystemInfo.h @@ -17,15 +17,8 @@ #pragma once -#include -#include -#include -#include -#include -#include -#include #include -#include +#include #include #include @@ -127,6 +120,9 @@ class SystemInfo { GPUTemperature(); std::vector CPUTemperature(); + + void + GetSysInfoJsonStr(std::string& result); }; } // namespace server diff --git a/core/src/server/delivery/request/CmdRequest.cpp b/core/src/server/delivery/request/CmdRequest.cpp index 4263f8715c..e4e8c9cc50 100644 --- a/core/src/server/delivery/request/CmdRequest.cpp +++ b/core/src/server/delivery/request/CmdRequest.cpp @@ -16,6 +16,7 @@ // under the License. #include "server/delivery/request/CmdRequest.h" +#include "metrics/SystemInfo.h" #include "scheduler/SchedInst.h" #include "utils/Log.h" #include "utils/TimeRecorder.h" @@ -52,6 +53,9 @@ CmdRequest::OnExecute() { #else result_ = "CPU"; #endif + } else if (cmd_ == "get_system_info") { + server::SystemInfo& sys_info_inst = server::SystemInfo::GetInstance(); + sys_info_inst.GetSysInfoJsonStr(result_); } else if (cmd_ == "build_commit_id") { result_ = LAST_COMMIT_ID; } else if (cmd_.substr(0, 10) == "set_config" || cmd_.substr(0, 10) == "get_config") { diff --git a/core/src/utils/CommonUtil.cpp b/core/src/utils/CommonUtil.cpp index cdfae8f1e5..7064764590 100644 --- a/core/src/utils/CommonUtil.cpp +++ b/core/src/utils/CommonUtil.cpp @@ -190,7 +190,7 @@ CommonUtil::GetExePath() { buf[cnt] = '\0'; std::string exe_path = buf; - if (exe_path.rfind('/') != exe_path.length()) { + if (exe_path.rfind('/') != exe_path.length() - 1) { std::string sub_str = exe_path.substr(0, exe_path.rfind('/')); return sub_str + "/"; } diff --git a/core/src/utils/StringHelpFunctions.cpp b/core/src/utils/StringHelpFunctions.cpp index af5b2e3b4d..0fa9ddfc1c 100644 --- a/core/src/utils/StringHelpFunctions.cpp +++ b/core/src/utils/StringHelpFunctions.cpp @@ -147,7 +147,7 @@ StringHelpFunctions::IsRegexMatch(const std::string& target_str, const std::stri // regex match std::regex pattern(pattern_str); std::smatch results; - if (std::regex_search(target_str, results, pattern)) { + if (std::regex_match(target_str, results, pattern)) { return true; } else { return false; diff --git a/sdk/cmake/ThirdPartyPackages.cmake b/sdk/cmake/ThirdPartyPackages.cmake index 4a82dda2cb..a2cacdd842 100644 --- a/sdk/cmake/ThirdPartyPackages.cmake +++ b/sdk/cmake/ThirdPartyPackages.cmake @@ -215,67 +215,26 @@ macro(build_grpc) set(GRPC_PROTOBUF_STATIC_LIB "${GRPC_PROTOBUF_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}protobuf${CMAKE_STATIC_LIBRARY_SUFFIX}") set(GRPC_PROTOC_STATIC_LIB "${GRPC_PROTOBUF_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}protoc${CMAKE_STATIC_LIBRARY_SUFFIX}") - if (USE_JFROG_CACHE STREQUAL "ON") - set(GRPC_CACHE_PACKAGE_NAME "grpc_${GRPC_MD5}.tar.gz") - set(GRPC_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${GRPC_CACHE_PACKAGE_NAME}") - set(GRPC_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${GRPC_CACHE_PACKAGE_NAME}") + externalproject_add(grpc_ep + URL + ${GRPC_SOURCE_URL} + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "" + BUILD_IN_SOURCE + 1 + BUILD_COMMAND + ${MAKE} ${MAKE_BUILD_ARGS} prefix=${GRPC_PREFIX} + INSTALL_COMMAND + ${MAKE} install prefix=${GRPC_PREFIX} + BUILD_BYPRODUCTS + ${GRPC_STATIC_LIB} + ${GRPC++_STATIC_LIB} + ${GRPCPP_CHANNELZ_STATIC_LIB} + ${GRPC_PROTOBUF_STATIC_LIB} + ${GRPC_PROTOC_STATIC_LIB}) - execute_process(COMMAND wget -q --method HEAD ${GRPC_CACHE_URL} RESULT_VARIABLE return_code) - message(STATUS "Check the remote file ${GRPC_CACHE_URL}. return code = ${return_code}") - if (NOT return_code EQUAL 0) - externalproject_add(grpc_ep - URL - ${GRPC_SOURCE_URL} - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "" - BUILD_IN_SOURCE - 1 - BUILD_COMMAND - ${MAKE} ${MAKE_BUILD_ARGS} prefix=${GRPC_PREFIX} - INSTALL_COMMAND - ${MAKE} install prefix=${GRPC_PREFIX} - BUILD_BYPRODUCTS - ${GRPC_STATIC_LIB} - ${GRPC++_STATIC_LIB} - ${GRPCPP_CHANNELZ_STATIC_LIB} - ${GRPC_PROTOBUF_STATIC_LIB} - ${GRPC_PROTOC_STATIC_LIB}) - - ExternalProject_Add_StepDependencies(grpc_ep build zlib_ep) - - ExternalProject_Create_Cache(grpc_ep ${GRPC_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GRPC_CACHE_URL}) - else () - file(DOWNLOAD ${GRPC_CACHE_URL} ${GRPC_CACHE_PACKAGE_PATH} STATUS status) - list(GET status 0 status_code) - message(STATUS "DOWNLOADING FROM ${GRPC_CACHE_URL} TO ${GRPC_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") - if (status_code EQUAL 0) - ExternalProject_Use_Cache(grpc_ep ${GRPC_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif () - endif () - else () - externalproject_add(grpc_ep - URL - ${GRPC_SOURCE_URL} - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "" - BUILD_IN_SOURCE - 1 - BUILD_COMMAND - ${MAKE} ${MAKE_BUILD_ARGS} prefix=${GRPC_PREFIX} - INSTALL_COMMAND - ${MAKE} install prefix=${GRPC_PREFIX} - BUILD_BYPRODUCTS - ${GRPC_STATIC_LIB} - ${GRPC++_STATIC_LIB} - ${GRPCPP_CHANNELZ_STATIC_LIB} - ${GRPC_PROTOBUF_STATIC_LIB} - ${GRPC_PROTOC_STATIC_LIB}) - - ExternalProject_Add_StepDependencies(grpc_ep build zlib_ep) - - endif () + ExternalProject_Add_StepDependencies(grpc_ep build zlib_ep) file(MAKE_DIRECTORY "${GRPC_INCLUDE_DIR}") @@ -338,48 +297,17 @@ macro(build_zlib) set(ZLIB_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX}" -DBUILD_SHARED_LIBS=OFF) - if (USE_JFROG_CACHE STREQUAL "ON") - set(ZLIB_CACHE_PACKAGE_NAME "zlib_${ZLIB_MD5}.tar.gz") - set(ZLIB_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${ZLIB_CACHE_PACKAGE_NAME}") - set(ZLIB_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${ZLIB_CACHE_PACKAGE_NAME}") - - execute_process(COMMAND wget -q --method HEAD ${ZLIB_CACHE_URL} RESULT_VARIABLE return_code) - message(STATUS "Check the remote file ${ZLIB_CACHE_URL}. return code = ${return_code}") - if (NOT return_code EQUAL 0) - externalproject_add(zlib_ep - URL - ${ZLIB_SOURCE_URL} - ${EP_LOG_OPTIONS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - BUILD_BYPRODUCTS - "${ZLIB_STATIC_LIB}" - CMAKE_ARGS - ${ZLIB_CMAKE_ARGS}) - - ExternalProject_Create_Cache(zlib_ep ${ZLIB_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/zlib_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ZLIB_CACHE_URL}) - else () - file(DOWNLOAD ${ZLIB_CACHE_URL} ${ZLIB_CACHE_PACKAGE_PATH} STATUS status) - list(GET status 0 status_code) - message(STATUS "DOWNLOADING FROM ${ZLIB_CACHE_URL} TO ${ZLIB_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") - if (status_code EQUAL 0) - ExternalProject_Use_Cache(zlib_ep ${ZLIB_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) - endif () - endif () - else () - externalproject_add(zlib_ep - URL - ${ZLIB_SOURCE_URL} - ${EP_LOG_OPTIONS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - BUILD_BYPRODUCTS - "${ZLIB_STATIC_LIB}" - CMAKE_ARGS - ${ZLIB_CMAKE_ARGS}) - endif () + externalproject_add(zlib_ep + URL + ${ZLIB_SOURCE_URL} + ${EP_LOG_OPTIONS} + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + BUILD_BYPRODUCTS + "${ZLIB_STATIC_LIB}" + CMAKE_ARGS + ${ZLIB_CMAKE_ARGS}) file(MAKE_DIRECTORY "${ZLIB_INCLUDE_DIR}") add_library(zlib STATIC IMPORTED) diff --git a/tests/milvus_python_test/requirements.txt b/tests/milvus_python_test/requirements.txt index 016c8dedfc..df3191ccd6 100644 --- a/tests/milvus_python_test/requirements.txt +++ b/tests/milvus_python_test/requirements.txt @@ -1,14 +1,4 @@ -astroid==2.2.5 -atomicwrites==1.3.0 -attrs==19.1.0 -importlib-metadata==0.15 -isort==4.3.20 -lazy-object-proxy==1.4.1 -mccabe==0.6.1 -more-itertools==7.0.0 numpy==1.16.3 -pluggy==0.12.0 -py==1.8.0 pylint==2.3.1 pytest==4.5.0 pytest-timeout==1.3.3 @@ -16,10 +6,5 @@ pytest-repeat==0.8.0 allure-pytest==2.7.0 pytest-print==0.1.2 pytest-level==0.1.1 -six==1.12.0 -typed-ast==1.3.5 -wcwidth==0.1.7 -wrapt==1.11.1 -zipp==0.5.1 scikit-learn>=0.19.1 pymilvus-test>=0.2.0 diff --git a/tests/milvus_python_test/test_search_vectors.py b/tests/milvus_python_test/test_search_vectors.py index 464a28efea..76bc81b384 100644 --- a/tests/milvus_python_test/test_search_vectors.py +++ b/tests/milvus_python_test/test_search_vectors.py @@ -449,7 +449,8 @@ class TestSearchBase: def get_invalid_range(self, request): yield request.param - def test_search_invalid_query_ranges(self, connect, table, get_invalid_range): + # disable + def _test_search_invalid_query_ranges(self, connect, table, get_invalid_range): ''' target: search table with query ranges method: search with the same query ranges @@ -477,7 +478,8 @@ class TestSearchBase: def get_valid_range_no_result(self, request): yield request.param - def test_search_valid_query_ranges_no_result(self, connect, table, get_valid_range_no_result): + # disable + def _test_search_valid_query_ranges_no_result(self, connect, table, get_valid_range_no_result): ''' target: search table with normal query ranges, but no data in db method: search with query ranges (low, low) @@ -505,7 +507,8 @@ class TestSearchBase: def get_valid_range(self, request): yield request.param - def test_search_valid_query_ranges(self, connect, table, get_valid_range): + # disable + def _test_search_valid_query_ranges(self, connect, table, get_valid_range): ''' target: search table with normal query ranges, but no data in db method: search with query ranges (low, normal) @@ -878,8 +881,9 @@ class TestSearchParamsInvalid(object): def get_query_ranges(self, request): yield request.param + # disable @pytest.mark.level(1) - def test_search_flat_with_invalid_query_range(self, connect, table, get_query_ranges): + def _test_search_flat_with_invalid_query_range(self, connect, table, get_query_ranges): ''' target: test search fuction, with the wrong query_range method: search with query_range @@ -893,9 +897,9 @@ class TestSearchParamsInvalid(object): with pytest.raises(Exception) as e: status, result = connect.search_vectors(table, 1, nprobe, query_vecs, query_ranges=query_ranges) - + # disable @pytest.mark.level(2) - def test_search_flat_with_invalid_query_range_ip(self, connect, ip_table, get_query_ranges): + def _test_search_flat_with_invalid_query_range_ip(self, connect, ip_table, get_query_ranges): ''' target: test search fuction, with the wrong query_range method: search with query_range