Merge remote-tracking branch 'upstream/master'

2024-12-02 20:09:57 +08:00 · 2019-12-26 17:15:11 +08:00 · 2019-12-26 17:15:11 +08:00 · 1b1cc27c8c
commit 1b1cc27c8c
parent 4c3fd87139 b140d8259b
16 changed files with 85 additions and 154 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -8,10 +8,13 @@ Please mark all change in change log and use the issue from GitHub
 - \#715 - Milvus crash when searching and building index simultaneously using SQ8H
 - \#744 - Don't return partition table for show_tables
 - \#770 - Server unittest run failed on low-end server
+- \#831 - Judge branch error in CommonUtil.cpp

 ## Feature
+- \#216 - Add CLI to get server info
 - \#343 - Add Opentracing
 - \#665 - Support get/set config via CLI
+- \#766 - If partition tag is similar, wrong partition is searched
 - \#771 - Add server build commit info interface
 - \#759 - Put C++ sdk out of milvus/core

--- a/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml
+++ b/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml
@ -25,7 +25,7 @@ spec:
    resources:
      limits:
        memory: "12Gi"
-        cpu: "4.0"
+        cpu: "6.0"
      requests:
        memory: "8Gi"
        cpu: "4.0"
--- a/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml
+++ b/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml
@ -24,7 +24,7 @@ spec:
    resources:
      limits:
        memory: "12Gi"
-        cpu: "4.0"
+        cpu: "6.0"
      requests:
        memory: "8Gi"
        cpu: "4.0"
--- a/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml
+++ b/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml
@ -25,7 +25,7 @@ spec:
    resources:
      limits:
        memory: "12Gi"
-        cpu: "4.0"
+        cpu: "6.0"
        nvidia.com/gpu: 1
      requests:
        memory: "8Gi"
--- a/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml
+++ b/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml
@ -24,7 +24,7 @@ spec:
    resources:
      limits:
        memory: "12Gi"
-        cpu: "4.0"
+        cpu: "6.0"
        nvidia.com/gpu: 1
      requests:
        memory: "8Gi"
--- a/ci/jenkins/step/singleDevNightlyTest.groovy
+++ b/ci/jenkins/step/singleDevNightlyTest.groovy
@ -1,6 +1,6 @@
 timeout(time: 90, unit: 'MINUTES') {
    dir ("tests/milvus_python_test") {
-        sh 'python3 -m pip install -r requirements.txt'
+        sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com'
        sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.HELM_RELEASE_NAME}-engine.milvus.svc.cluster.local"
    }
    // mysql database backend test
--- a/ci/jenkins/step/singleDevTest.groovy
+++ b/ci/jenkins/step/singleDevTest.groovy
@ -1,6 +1,6 @@
 timeout(time: 60, unit: 'MINUTES') {
    dir ("tests/milvus_python_test") {
-        sh 'python3 -m pip install -r requirements.txt'
+        sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com'
        sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.HELM_RELEASE_NAME}-engine.milvus.svc.cluster.local"
    }

--- a/ci/scripts/build.sh
+++ b/ci/scripts/build.sh
@ -19,13 +19,12 @@ FAISS_ROOT=""
 PRIVILEGES="OFF"
 CUSTOMIZATION="OFF" # default use origin faiss
 BUILD_COVERAGE="OFF"
-USE_JFROG_CACHE="OFF"
 RUN_CPPLINT="OFF"
 GPU_VERSION="OFF"
 WITH_MKL="OFF"
 CUDA_COMPILER=/usr/local/cuda/bin/nvcc

-while getopts "o:t:b:f:pgxulcjmh" arg
+while getopts "o:t:b:f:pgxulcmh" arg
 do
        case $arg in
             o)
@ -59,9 +58,6 @@ do
             c)
                BUILD_COVERAGE="ON"
                ;;
-             j)
-                USE_JFROG_CACHE="ON"
-                ;;
             m)
                WITH_MKL="ON"
                ;;
@ -79,12 +75,11 @@ parameter:
 -u: building unit test options(default: OFF)
 -l: run cpplint, clang-format and clang-tidy(default: OFF)
 -c: code coverage(default: OFF)
-j: use jfrog cache build directory(default: OFF)
 -m: build with MKL(default: OFF)
 -h: help

 usage:
-./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-p] [-g] [-x] [-u] [-l] [-c] [-j] [-m] [-h]
+./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-p] [-g] [-x] [-u] [-l] [-c] [-m] [-h]
                "
                exit 0
                ;;
@ -112,7 +107,6 @@ CMAKE_CMD="cmake \
 -DCUSTOMIZATION=${CUSTOMIZATION} \
 -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
 -DBUILD_COVERAGE=${BUILD_COVERAGE} \
-DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
 -DFAISS_ROOT=${FAISS_ROOT} \
 -DFAISS_WITH_MKL=${WITH_MKL} \
 -DArrow_SOURCE=AUTO \
--- a/core/src/metrics/SystemInfo.cpp
+++ b/core/src/metrics/SystemInfo.cpp
@ -16,17 +16,14 @@
 // under the License.

 #include "metrics/SystemInfo.h"
+#include "thirdparty/nlohmann/json.hpp"
 #include "utils/Log.h"

 #include <dirent.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <sys/types.h>
+#include <sys/sysinfo.h>
+#include <sys/times.h>
 #include <unistd.h>
-#include <fstream>
-#include <iostream>
-#include <string>
-#include <utility>
+#include <map>

 #ifdef MILVUS_GPU_VERSION
 #include <nvml.h>
@ -350,5 +347,25 @@ SystemInfo::Octets() {
    return res;
 }

+void
+SystemInfo::GetSysInfoJsonStr(std::string& result) {
+    std::map<std::string, std::string> sys_info_map;
+
+    sys_info_map["memory_total"] = std::to_string(GetPhysicalMemory());
+    sys_info_map["memory_used"] = std::to_string(GetProcessUsedMemory());
+
+    std::vector<uint64_t> gpu_mem_total = GPUMemoryTotal();
+    std::vector<uint64_t> gpu_mem_used = GPUMemoryUsed();
+    for (size_t i = 0; i < gpu_mem_total.size(); i++) {
+        std::string key_total = "gpu" + std::to_string(i) + "_memory_total";
+        std::string key_used = "gpu" + std::to_string(i) + "_memory_used";
+        sys_info_map[key_total] = std::to_string(gpu_mem_total[i]);
+        sys_info_map[key_used] = std::to_string(gpu_mem_used[i]);
+    }
+
+    nlohmann::json sys_info_json(sys_info_map);
+    result = sys_info_json.dump();
+}
+
 }  // namespace server
 }  // namespace milvus
--- a/core/src/metrics/SystemInfo.h
+++ b/core/src/metrics/SystemInfo.h
@ -17,15 +17,8 @@

 #pragma once

-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/sysinfo.h>
-#include <sys/times.h>
-#include <sys/types.h>
-#include <sys/vtimes.h>
 #include <chrono>
-#include <unordered_map>
+#include <string>
 #include <utility>
 #include <vector>

@ -127,6 +120,9 @@ class SystemInfo {
    GPUTemperature();
    std::vector<float>
    CPUTemperature();
+
+    void
+    GetSysInfoJsonStr(std::string& result);
 };

 }  // namespace server
--- a/core/src/server/delivery/request/CmdRequest.cpp
+++ b/core/src/server/delivery/request/CmdRequest.cpp
@ -16,6 +16,7 @@
 // under the License.

 #include "server/delivery/request/CmdRequest.h"
+#include "metrics/SystemInfo.h"
 #include "scheduler/SchedInst.h"
 #include "utils/Log.h"
 #include "utils/TimeRecorder.h"
@ -52,6 +53,9 @@ CmdRequest::OnExecute() {
 #else
        result_ = "CPU";
 #endif
+    } else if (cmd_ == "get_system_info") {
+        server::SystemInfo& sys_info_inst = server::SystemInfo::GetInstance();
+        sys_info_inst.GetSysInfoJsonStr(result_);
    } else if (cmd_ == "build_commit_id") {
        result_ = LAST_COMMIT_ID;
    } else if (cmd_.substr(0, 10) == "set_config" || cmd_.substr(0, 10) == "get_config") {
--- a/core/src/utils/CommonUtil.cpp
+++ b/core/src/utils/CommonUtil.cpp
@ -190,7 +190,7 @@ CommonUtil::GetExePath() {
    buf[cnt] = '\0';

    std::string exe_path = buf;
-    if (exe_path.rfind('/') != exe_path.length()) {
+    if (exe_path.rfind('/') != exe_path.length() - 1) {
        std::string sub_str = exe_path.substr(0, exe_path.rfind('/'));
        return sub_str + "/";
    }
--- a/core/src/utils/StringHelpFunctions.cpp
+++ b/core/src/utils/StringHelpFunctions.cpp
@ -147,7 +147,7 @@ StringHelpFunctions::IsRegexMatch(const std::string& target_str, const std::stri
    // regex match
    std::regex pattern(pattern_str);
    std::smatch results;
-    if (std::regex_search(target_str, results, pattern)) {
+    if (std::regex_match(target_str, results, pattern)) {
        return true;
    } else {
        return false;
--- a/sdk/cmake/ThirdPartyPackages.cmake
+++ b/sdk/cmake/ThirdPartyPackages.cmake
@ -215,67 +215,26 @@ macro(build_grpc)
    set(GRPC_PROTOBUF_STATIC_LIB "${GRPC_PROTOBUF_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}protobuf${CMAKE_STATIC_LIBRARY_SUFFIX}")
    set(GRPC_PROTOC_STATIC_LIB "${GRPC_PROTOBUF_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}protoc${CMAKE_STATIC_LIBRARY_SUFFIX}")

-    if (USE_JFROG_CACHE STREQUAL "ON")
-        set(GRPC_CACHE_PACKAGE_NAME "grpc_${GRPC_MD5}.tar.gz")
-        set(GRPC_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${GRPC_CACHE_PACKAGE_NAME}")
-        set(GRPC_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${GRPC_CACHE_PACKAGE_NAME}")
+    externalproject_add(grpc_ep
+            URL
+            ${GRPC_SOURCE_URL}
+            ${EP_LOG_OPTIONS}
+            CONFIGURE_COMMAND
+            ""
+            BUILD_IN_SOURCE
+            1
+            BUILD_COMMAND
+            ${MAKE} ${MAKE_BUILD_ARGS} prefix=${GRPC_PREFIX}
+            INSTALL_COMMAND
+            ${MAKE} install prefix=${GRPC_PREFIX}
+            BUILD_BYPRODUCTS
+            ${GRPC_STATIC_LIB}
+            ${GRPC++_STATIC_LIB}
+            ${GRPCPP_CHANNELZ_STATIC_LIB}
+            ${GRPC_PROTOBUF_STATIC_LIB}
+            ${GRPC_PROTOC_STATIC_LIB})

-        execute_process(COMMAND wget -q --method HEAD ${GRPC_CACHE_URL} RESULT_VARIABLE return_code)
-        message(STATUS "Check the remote file ${GRPC_CACHE_URL}. return code = ${return_code}")
-        if (NOT return_code EQUAL 0)
-            externalproject_add(grpc_ep
-                    URL
-                    ${GRPC_SOURCE_URL}
-                    ${EP_LOG_OPTIONS}
-                    CONFIGURE_COMMAND
-                    ""
-                    BUILD_IN_SOURCE
-                    1
-                    BUILD_COMMAND
-                    ${MAKE} ${MAKE_BUILD_ARGS} prefix=${GRPC_PREFIX}
-                    INSTALL_COMMAND
-                    ${MAKE} install prefix=${GRPC_PREFIX}
-                    BUILD_BYPRODUCTS
-                    ${GRPC_STATIC_LIB}
-                    ${GRPC++_STATIC_LIB}
-                    ${GRPCPP_CHANNELZ_STATIC_LIB}
-                    ${GRPC_PROTOBUF_STATIC_LIB}
-                    ${GRPC_PROTOC_STATIC_LIB})
-
-            ExternalProject_Add_StepDependencies(grpc_ep build zlib_ep)
-
-            ExternalProject_Create_Cache(grpc_ep ${GRPC_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GRPC_CACHE_URL})
-        else ()
-            file(DOWNLOAD ${GRPC_CACHE_URL} ${GRPC_CACHE_PACKAGE_PATH} STATUS status)
-            list(GET status 0 status_code)
-            message(STATUS "DOWNLOADING FROM ${GRPC_CACHE_URL} TO ${GRPC_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
-            if (status_code EQUAL 0)
-                ExternalProject_Use_Cache(grpc_ep ${GRPC_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
-            endif ()
-        endif ()
-    else ()
-        externalproject_add(grpc_ep
-                URL
-                ${GRPC_SOURCE_URL}
-                ${EP_LOG_OPTIONS}
-                CONFIGURE_COMMAND
-                ""
-                BUILD_IN_SOURCE
-                1
-                BUILD_COMMAND
-                ${MAKE} ${MAKE_BUILD_ARGS} prefix=${GRPC_PREFIX}
-                INSTALL_COMMAND
-                ${MAKE} install prefix=${GRPC_PREFIX}
-                BUILD_BYPRODUCTS
-                ${GRPC_STATIC_LIB}
-                ${GRPC++_STATIC_LIB}
-                ${GRPCPP_CHANNELZ_STATIC_LIB}
-                ${GRPC_PROTOBUF_STATIC_LIB}
-                ${GRPC_PROTOC_STATIC_LIB})
-
-        ExternalProject_Add_StepDependencies(grpc_ep build zlib_ep)
-
-    endif ()
+    ExternalProject_Add_StepDependencies(grpc_ep build zlib_ep)

    file(MAKE_DIRECTORY "${GRPC_INCLUDE_DIR}")

@ -338,48 +297,17 @@ macro(build_zlib)
    set(ZLIB_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX}"
            -DBUILD_SHARED_LIBS=OFF)

-    if (USE_JFROG_CACHE STREQUAL "ON")
-        set(ZLIB_CACHE_PACKAGE_NAME "zlib_${ZLIB_MD5}.tar.gz")
-        set(ZLIB_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${ZLIB_CACHE_PACKAGE_NAME}")
-        set(ZLIB_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${ZLIB_CACHE_PACKAGE_NAME}")
-
-        execute_process(COMMAND wget -q --method HEAD ${ZLIB_CACHE_URL} RESULT_VARIABLE return_code)
-        message(STATUS "Check the remote file ${ZLIB_CACHE_URL}. return code = ${return_code}")
-        if (NOT return_code EQUAL 0)
-            externalproject_add(zlib_ep
-                    URL
-                    ${ZLIB_SOURCE_URL}
-                    ${EP_LOG_OPTIONS}
-                    BUILD_COMMAND
-                    ${MAKE}
-                    ${MAKE_BUILD_ARGS}
-                    BUILD_BYPRODUCTS
-                    "${ZLIB_STATIC_LIB}"
-                    CMAKE_ARGS
-                    ${ZLIB_CMAKE_ARGS})
-
-            ExternalProject_Create_Cache(zlib_ep ${ZLIB_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/zlib_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ZLIB_CACHE_URL})
-        else ()
-            file(DOWNLOAD ${ZLIB_CACHE_URL} ${ZLIB_CACHE_PACKAGE_PATH} STATUS status)
-            list(GET status 0 status_code)
-            message(STATUS "DOWNLOADING FROM ${ZLIB_CACHE_URL} TO ${ZLIB_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
-            if (status_code EQUAL 0)
-                ExternalProject_Use_Cache(zlib_ep ${ZLIB_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
-            endif ()
-        endif ()
-    else ()
-        externalproject_add(zlib_ep
-                URL
-                ${ZLIB_SOURCE_URL}
-                ${EP_LOG_OPTIONS}
-                BUILD_COMMAND
-                ${MAKE}
-                ${MAKE_BUILD_ARGS}
-                BUILD_BYPRODUCTS
-                "${ZLIB_STATIC_LIB}"
-                CMAKE_ARGS
-                ${ZLIB_CMAKE_ARGS})
-    endif ()
+    externalproject_add(zlib_ep
+            URL
+            ${ZLIB_SOURCE_URL}
+            ${EP_LOG_OPTIONS}
+            BUILD_COMMAND
+            ${MAKE}
+            ${MAKE_BUILD_ARGS}
+            BUILD_BYPRODUCTS
+            "${ZLIB_STATIC_LIB}"
+            CMAKE_ARGS
+            ${ZLIB_CMAKE_ARGS})

    file(MAKE_DIRECTORY "${ZLIB_INCLUDE_DIR}")
    add_library(zlib STATIC IMPORTED)
--- a/tests/milvus_python_test/requirements.txt
+++ b/tests/milvus_python_test/requirements.txt
@ -1,14 +1,4 @@
-astroid==2.2.5
-atomicwrites==1.3.0
-attrs==19.1.0
-importlib-metadata==0.15
-isort==4.3.20
-lazy-object-proxy==1.4.1
-mccabe==0.6.1
-more-itertools==7.0.0
 numpy==1.16.3
-pluggy==0.12.0
-py==1.8.0
 pylint==2.3.1
 pytest==4.5.0
 pytest-timeout==1.3.3
@ -16,10 +6,5 @@ pytest-repeat==0.8.0
 allure-pytest==2.7.0
 pytest-print==0.1.2
 pytest-level==0.1.1
-six==1.12.0
-typed-ast==1.3.5
-wcwidth==0.1.7
-wrapt==1.11.1
-zipp==0.5.1
 scikit-learn>=0.19.1
 pymilvus-test>=0.2.0
--- a/tests/milvus_python_test/test_search_vectors.py
+++ b/tests/milvus_python_test/test_search_vectors.py
@ -449,7 +449,8 @@ class TestSearchBase:
    def get_invalid_range(self, request):
        yield request.param

-    def test_search_invalid_query_ranges(self, connect, table, get_invalid_range):
+    # disable
+    def _test_search_invalid_query_ranges(self, connect, table, get_invalid_range):
        '''
        target: search table with query ranges
        method: search with the same query ranges
@ -477,7 +478,8 @@ class TestSearchBase:
    def get_valid_range_no_result(self, request):
        yield request.param

-    def test_search_valid_query_ranges_no_result(self, connect, table, get_valid_range_no_result):
+    # disable
+    def _test_search_valid_query_ranges_no_result(self, connect, table, get_valid_range_no_result):
        '''
        target: search table with normal query ranges, but no data in db
        method: search with query ranges (low, low)
@ -505,7 +507,8 @@ class TestSearchBase:
    def get_valid_range(self, request):
        yield request.param

-    def test_search_valid_query_ranges(self, connect, table, get_valid_range):
+    # disable
+    def _test_search_valid_query_ranges(self, connect, table, get_valid_range):
        '''
        target: search table with normal query ranges, but no data in db
        method: search with query ranges (low, normal)
@ -878,8 +881,9 @@ class TestSearchParamsInvalid(object):
    def get_query_ranges(self, request):
        yield request.param

+    # disable
    @pytest.mark.level(1)
-    def test_search_flat_with_invalid_query_range(self, connect, table, get_query_ranges):
+    def _test_search_flat_with_invalid_query_range(self, connect, table, get_query_ranges):
        '''
        target: test search fuction, with the wrong query_range
        method: search with query_range
@ -893,9 +897,9 @@ class TestSearchParamsInvalid(object):
        with pytest.raises(Exception) as e:
            status, result = connect.search_vectors(table, 1, nprobe, query_vecs, query_ranges=query_ranges)

-
+    # disable
    @pytest.mark.level(2)
-    def test_search_flat_with_invalid_query_range_ip(self, connect, ip_table, get_query_ranges):
+    def _test_search_flat_with_invalid_query_range_ip(self, connect, ip_table, get_query_ranges):
        '''
        target: test search fuction, with the wrong query_range
        method: search with query_range