Merge remote-tracking branch 'upstream/master'

CHANGELOG.md

@@ -8,10 +8,13 @@ Please mark all change in change log and use the issue from GitHub
 - \#715 - Milvus crash when searching and building index simultaneously using SQ8H
 - \#744 - Don't return partition table for show_tables
 - \#770 - Server unittest run failed on low-end server
+- \#831 - Judge branch error in CommonUtil.cpp
 
 ## Feature
+- \#216 - Add CLI to get server info
 - \#343 - Add Opentracing
 - \#665 - Support get/set config via CLI
+- \#766 - If partition tag is similar, wrong partition is searched
 - \#771 - Add server build commit info interface
 - \#759 - Put C++ sdk out of milvus/core
 

ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml

@@ -25,7 +25,7 @@ spec:
     resources:
       limits:
         memory: "12Gi"
-        cpu: "4.0"
+        cpu: "6.0"
       requests:
         memory: "8Gi"
         cpu: "4.0"

ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml

@@ -24,7 +24,7 @@ spec:
     resources:
       limits:
         memory: "12Gi"
-        cpu: "4.0"
+        cpu: "6.0"
       requests:
         memory: "8Gi"
         cpu: "4.0"

ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml

@@ -25,7 +25,7 @@ spec:
     resources:
       limits:
         memory: "12Gi"
-        cpu: "4.0"
+        cpu: "6.0"
         nvidia.com/gpu: 1
       requests:
         memory: "8Gi"

ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml

@@ -24,7 +24,7 @@ spec:
     resources:
       limits:
         memory: "12Gi"
-        cpu: "4.0"
+        cpu: "6.0"
         nvidia.com/gpu: 1
       requests:
         memory: "8Gi"

ci/jenkins/step/singleDevNightlyTest.groovy

@@ -1,6 +1,6 @@
 timeout(time: 90, unit: 'MINUTES') {
     dir ("tests/milvus_python_test") {
-        sh 'python3 -m pip install -r requirements.txt'
+        sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com'
         sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.HELM_RELEASE_NAME}-engine.milvus.svc.cluster.local"
     }
     // mysql database backend test

ci/jenkins/step/singleDevTest.groovy

@@ -1,6 +1,6 @@
 timeout(time: 60, unit: 'MINUTES') {
     dir ("tests/milvus_python_test") {
-        sh 'python3 -m pip install -r requirements.txt'
+        sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com'
         sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.HELM_RELEASE_NAME}-engine.milvus.svc.cluster.local"
     }
 

ci/scripts/build.sh

@@ -19,13 +19,12 @@ FAISS_ROOT=""
 PRIVILEGES="OFF"
 CUSTOMIZATION="OFF" # default use origin faiss
 BUILD_COVERAGE="OFF"
-USE_JFROG_CACHE="OFF"
 RUN_CPPLINT="OFF"
 GPU_VERSION="OFF"
 WITH_MKL="OFF"
 CUDA_COMPILER=/usr/local/cuda/bin/nvcc
 
-while getopts "o:t:b:f:pgxulcjmh" arg
+while getopts "o:t:b:f:pgxulcmh" arg
 do
         case $arg in
              o)
@@ -59,9 +58,6 @@ do
              c)
                 BUILD_COVERAGE="ON"
                 ;;
-             j)
-                USE_JFROG_CACHE="ON"
-                ;;
              m)
                 WITH_MKL="ON"
                 ;;
@@ -79,12 +75,11 @@ parameter:
 -u: building unit test options(default: OFF)
 -l: run cpplint, clang-format and clang-tidy(default: OFF)
 -c: code coverage(default: OFF)
--j: use jfrog cache build directory(default: OFF)
 -m: build with MKL(default: OFF)
 -h: help
 
 usage:
-./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-p] [-g] [-x] [-u] [-l] [-c] [-j] [-m] [-h]
+./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-p] [-g] [-x] [-u] [-l] [-c] [-m] [-h]
                 "
                 exit 0
                 ;;
@@ -112,7 +107,6 @@ CMAKE_CMD="cmake \
 -DCUSTOMIZATION=${CUSTOMIZATION} \
 -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
 -DBUILD_COVERAGE=${BUILD_COVERAGE} \
--DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
 -DFAISS_ROOT=${FAISS_ROOT} \
 -DFAISS_WITH_MKL=${WITH_MKL} \
 -DArrow_SOURCE=AUTO \

core/src/metrics/SystemInfo.cpp

@@ -16,17 +16,14 @@
 // under the License.
 
 #include "metrics/SystemInfo.h"
+#include "thirdparty/nlohmann/json.hpp"
 #include "utils/Log.h"
 
 #include <dirent.h>
-#include <stdio.h>
+#include <sys/sysinfo.h>
-#include <stdlib.h>
+#include <sys/times.h>
-#include <sys/types.h>
 #include <unistd.h>
-#include <fstream>
+#include <map>
-#include <iostream>
-#include <string>
-#include <utility>
 
 #ifdef MILVUS_GPU_VERSION
 #include <nvml.h>
@@ -350,5 +347,25 @@ SystemInfo::Octets() {
     return res;
 }
 
+void
+SystemInfo::GetSysInfoJsonStr(std::string& result) {
+    std::map<std::string, std::string> sys_info_map;
+
+    sys_info_map["memory_total"] = std::to_string(GetPhysicalMemory());
+    sys_info_map["memory_used"] = std::to_string(GetProcessUsedMemory());
+
+    std::vector<uint64_t> gpu_mem_total = GPUMemoryTotal();
+    std::vector<uint64_t> gpu_mem_used = GPUMemoryUsed();
+    for (size_t i = 0; i < gpu_mem_total.size(); i++) {
+        std::string key_total = "gpu" + std::to_string(i) + "_memory_total";
+        std::string key_used = "gpu" + std::to_string(i) + "_memory_used";
+        sys_info_map[key_total] = std::to_string(gpu_mem_total[i]);
+        sys_info_map[key_used] = std::to_string(gpu_mem_used[i]);
+    }
+
+    nlohmann::json sys_info_json(sys_info_map);
+    result = sys_info_json.dump();
+}
+
 }  // namespace server
 }  // namespace milvus

core/src/metrics/SystemInfo.h

@@ -17,15 +17,8 @@
 
 #pragma once
 
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/sysinfo.h>
-#include <sys/times.h>
-#include <sys/types.h>
-#include <sys/vtimes.h>
 #include <chrono>
-#include <unordered_map>
+#include <string>
 #include <utility>
 #include <vector>
 
@@ -127,6 +120,9 @@ class SystemInfo {
     GPUTemperature();
     std::vector<float>
     CPUTemperature();
+
+    void
+    GetSysInfoJsonStr(std::string& result);
 };
 
 }  // namespace server

core/src/server/delivery/request/CmdRequest.cpp

@@ -16,6 +16,7 @@
 // under the License.
 
 #include "server/delivery/request/CmdRequest.h"
+#include "metrics/SystemInfo.h"
 #include "scheduler/SchedInst.h"
 #include "utils/Log.h"
 #include "utils/TimeRecorder.h"
@@ -52,6 +53,9 @@ CmdRequest::OnExecute() {
 #else
         result_ = "CPU";
 #endif
+    } else if (cmd_ == "get_system_info") {
+        server::SystemInfo& sys_info_inst = server::SystemInfo::GetInstance();
+        sys_info_inst.GetSysInfoJsonStr(result_);
     } else if (cmd_ == "build_commit_id") {
         result_ = LAST_COMMIT_ID;
     } else if (cmd_.substr(0, 10) == "set_config" || cmd_.substr(0, 10) == "get_config") {

core/src/utils/CommonUtil.cpp

@@ -190,7 +190,7 @@ CommonUtil::GetExePath() {
     buf[cnt] = '\0';
 
     std::string exe_path = buf;
-    if (exe_path.rfind('/') != exe_path.length()) {
+    if (exe_path.rfind('/') != exe_path.length() - 1) {
         std::string sub_str = exe_path.substr(0, exe_path.rfind('/'));
         return sub_str + "/";
     }

core/src/utils/StringHelpFunctions.cpp

@@ -147,7 +147,7 @@ StringHelpFunctions::IsRegexMatch(const std::string& target_str, const std::stri
     // regex match
     std::regex pattern(pattern_str);
     std::smatch results;
-    if (std::regex_search(target_str, results, pattern)) {
+    if (std::regex_match(target_str, results, pattern)) {
         return true;
     } else {
         return false;

sdk/cmake/ThirdPartyPackages.cmake

@@ -215,67 +215,26 @@ macro(build_grpc)
     set(GRPC_PROTOBUF_STATIC_LIB "${GRPC_PROTOBUF_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}protobuf${CMAKE_STATIC_LIBRARY_SUFFIX}")
     set(GRPC_PROTOC_STATIC_LIB "${GRPC_PROTOBUF_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}protoc${CMAKE_STATIC_LIBRARY_SUFFIX}")
 
-    if (USE_JFROG_CACHE STREQUAL "ON")
+    externalproject_add(grpc_ep
-        set(GRPC_CACHE_PACKAGE_NAME "grpc_${GRPC_MD5}.tar.gz")
+            URL
-        set(GRPC_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${GRPC_CACHE_PACKAGE_NAME}")
+            ${GRPC_SOURCE_URL}
-        set(GRPC_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${GRPC_CACHE_PACKAGE_NAME}")
+            ${EP_LOG_OPTIONS}
+            CONFIGURE_COMMAND
+            ""
+            BUILD_IN_SOURCE
+            1
+            BUILD_COMMAND
+            ${MAKE} ${MAKE_BUILD_ARGS} prefix=${GRPC_PREFIX}
+            INSTALL_COMMAND
+            ${MAKE} install prefix=${GRPC_PREFIX}
+            BUILD_BYPRODUCTS
+            ${GRPC_STATIC_LIB}
+            ${GRPC++_STATIC_LIB}
+            ${GRPCPP_CHANNELZ_STATIC_LIB}
+            ${GRPC_PROTOBUF_STATIC_LIB}
+            ${GRPC_PROTOC_STATIC_LIB})
 
-        execute_process(COMMAND wget -q --method HEAD ${GRPC_CACHE_URL} RESULT_VARIABLE return_code)
+    ExternalProject_Add_StepDependencies(grpc_ep build zlib_ep)
-        message(STATUS "Check the remote file ${GRPC_CACHE_URL}. return code = ${return_code}")
-        if (NOT return_code EQUAL 0)
-            externalproject_add(grpc_ep
-                    URL
-                    ${GRPC_SOURCE_URL}
-                    ${EP_LOG_OPTIONS}
-                    CONFIGURE_COMMAND
-                    ""
-                    BUILD_IN_SOURCE
-                    1
-                    BUILD_COMMAND
-                    ${MAKE} ${MAKE_BUILD_ARGS} prefix=${GRPC_PREFIX}
-                    INSTALL_COMMAND
-                    ${MAKE} install prefix=${GRPC_PREFIX}
-                    BUILD_BYPRODUCTS
-                    ${GRPC_STATIC_LIB}
-                    ${GRPC++_STATIC_LIB}
-                    ${GRPCPP_CHANNELZ_STATIC_LIB}
-                    ${GRPC_PROTOBUF_STATIC_LIB}
-                    ${GRPC_PROTOC_STATIC_LIB})
-
-            ExternalProject_Add_StepDependencies(grpc_ep build zlib_ep)
-
-            ExternalProject_Create_Cache(grpc_ep ${GRPC_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${GRPC_CACHE_URL})
-        else ()
-            file(DOWNLOAD ${GRPC_CACHE_URL} ${GRPC_CACHE_PACKAGE_PATH} STATUS status)
-            list(GET status 0 status_code)
-            message(STATUS "DOWNLOADING FROM ${GRPC_CACHE_URL} TO ${GRPC_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
-            if (status_code EQUAL 0)
-                ExternalProject_Use_Cache(grpc_ep ${GRPC_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
-            endif ()
-        endif ()
-    else ()
-        externalproject_add(grpc_ep
-                URL
-                ${GRPC_SOURCE_URL}
-                ${EP_LOG_OPTIONS}
-                CONFIGURE_COMMAND
-                ""
-                BUILD_IN_SOURCE
-                1
-                BUILD_COMMAND
-                ${MAKE} ${MAKE_BUILD_ARGS} prefix=${GRPC_PREFIX}
-                INSTALL_COMMAND
-                ${MAKE} install prefix=${GRPC_PREFIX}
-                BUILD_BYPRODUCTS
-                ${GRPC_STATIC_LIB}
-                ${GRPC++_STATIC_LIB}
-                ${GRPCPP_CHANNELZ_STATIC_LIB}
-                ${GRPC_PROTOBUF_STATIC_LIB}
-                ${GRPC_PROTOC_STATIC_LIB})
-
-        ExternalProject_Add_StepDependencies(grpc_ep build zlib_ep)
-
-    endif ()
 
     file(MAKE_DIRECTORY "${GRPC_INCLUDE_DIR}")
 
@@ -338,48 +297,17 @@ macro(build_zlib)
     set(ZLIB_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX}"
             -DBUILD_SHARED_LIBS=OFF)
 
-    if (USE_JFROG_CACHE STREQUAL "ON")
+    externalproject_add(zlib_ep
-        set(ZLIB_CACHE_PACKAGE_NAME "zlib_${ZLIB_MD5}.tar.gz")
+            URL
-        set(ZLIB_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${ZLIB_CACHE_PACKAGE_NAME}")
+            ${ZLIB_SOURCE_URL}
-        set(ZLIB_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${ZLIB_CACHE_PACKAGE_NAME}")
+            ${EP_LOG_OPTIONS}
-
+            BUILD_COMMAND
-        execute_process(COMMAND wget -q --method HEAD ${ZLIB_CACHE_URL} RESULT_VARIABLE return_code)
+            ${MAKE}
-        message(STATUS "Check the remote file ${ZLIB_CACHE_URL}. return code = ${return_code}")
+            ${MAKE_BUILD_ARGS}
-        if (NOT return_code EQUAL 0)
+            BUILD_BYPRODUCTS
-            externalproject_add(zlib_ep
+            "${ZLIB_STATIC_LIB}"
-                    URL
+            CMAKE_ARGS
-                    ${ZLIB_SOURCE_URL}
+            ${ZLIB_CMAKE_ARGS})
-                    ${EP_LOG_OPTIONS}
-                    BUILD_COMMAND
-                    ${MAKE}
-                    ${MAKE_BUILD_ARGS}
-                    BUILD_BYPRODUCTS
-                    "${ZLIB_STATIC_LIB}"
-                    CMAKE_ARGS
-                    ${ZLIB_CMAKE_ARGS})
-
-            ExternalProject_Create_Cache(zlib_ep ${ZLIB_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/zlib_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ZLIB_CACHE_URL})
-        else ()
-            file(DOWNLOAD ${ZLIB_CACHE_URL} ${ZLIB_CACHE_PACKAGE_PATH} STATUS status)
-            list(GET status 0 status_code)
-            message(STATUS "DOWNLOADING FROM ${ZLIB_CACHE_URL} TO ${ZLIB_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
-            if (status_code EQUAL 0)
-                ExternalProject_Use_Cache(zlib_ep ${ZLIB_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
-            endif ()
-        endif ()
-    else ()
-        externalproject_add(zlib_ep
-                URL
-                ${ZLIB_SOURCE_URL}
-                ${EP_LOG_OPTIONS}
-                BUILD_COMMAND
-                ${MAKE}
-                ${MAKE_BUILD_ARGS}
-                BUILD_BYPRODUCTS
-                "${ZLIB_STATIC_LIB}"
-                CMAKE_ARGS
-                ${ZLIB_CMAKE_ARGS})
-    endif ()
 
     file(MAKE_DIRECTORY "${ZLIB_INCLUDE_DIR}")
     add_library(zlib STATIC IMPORTED)

tests/milvus_python_test/requirements.txt

@@ -1,14 +1,4 @@
-astroid==2.2.5
-atomicwrites==1.3.0
-attrs==19.1.0
-importlib-metadata==0.15
-isort==4.3.20
-lazy-object-proxy==1.4.1
-mccabe==0.6.1
-more-itertools==7.0.0
 numpy==1.16.3
-pluggy==0.12.0
-py==1.8.0
 pylint==2.3.1
 pytest==4.5.0
 pytest-timeout==1.3.3
@@ -16,10 +6,5 @@ pytest-repeat==0.8.0
 allure-pytest==2.7.0
 pytest-print==0.1.2
 pytest-level==0.1.1
-six==1.12.0
-typed-ast==1.3.5
-wcwidth==0.1.7
-wrapt==1.11.1
-zipp==0.5.1
 scikit-learn>=0.19.1
 pymilvus-test>=0.2.0

tests/milvus_python_test/test_search_vectors.py

@@ -449,7 +449,8 @@ class TestSearchBase:
     def get_invalid_range(self, request):
         yield request.param
 
-    def test_search_invalid_query_ranges(self, connect, table, get_invalid_range):
+    # disable
+    def _test_search_invalid_query_ranges(self, connect, table, get_invalid_range):
         '''
         target: search table with query ranges
         method: search with the same query ranges
@@ -477,7 +478,8 @@ class TestSearchBase:
     def get_valid_range_no_result(self, request):
         yield request.param
 
-    def test_search_valid_query_ranges_no_result(self, connect, table, get_valid_range_no_result):
+    # disable
+    def _test_search_valid_query_ranges_no_result(self, connect, table, get_valid_range_no_result):
         '''
         target: search table with normal query ranges, but no data in db
         method: search with query ranges (low, low)
@@ -505,7 +507,8 @@ class TestSearchBase:
     def get_valid_range(self, request):
         yield request.param
 
-    def test_search_valid_query_ranges(self, connect, table, get_valid_range):
+    # disable
+    def _test_search_valid_query_ranges(self, connect, table, get_valid_range):
         '''
         target: search table with normal query ranges, but no data in db
         method: search with query ranges (low, normal)
@@ -878,8 +881,9 @@ class TestSearchParamsInvalid(object):
     def get_query_ranges(self, request):
         yield request.param
 
+    # disable
     @pytest.mark.level(1)
-    def test_search_flat_with_invalid_query_range(self, connect, table, get_query_ranges):
+    def _test_search_flat_with_invalid_query_range(self, connect, table, get_query_ranges):
         '''
         target: test search fuction, with the wrong query_range
         method: search with query_range
@@ -893,9 +897,9 @@ class TestSearchParamsInvalid(object):
         with pytest.raises(Exception) as e:
             status, result = connect.search_vectors(table, 1, nprobe, query_vecs, query_ranges=query_ranges)
 
-
+    # disable
     @pytest.mark.level(2)
-    def test_search_flat_with_invalid_query_range_ip(self, connect, ip_table, get_query_ranges):
+    def _test_search_flat_with_invalid_query_range_ip(self, connect, ip_table, get_query_ranges):
         '''
         target: test search fuction, with the wrong query_range
         method: search with query_range