From f8875429b191fa713676bc42de9d23d0794291ee Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 17 Jul 2019 20:24:09 +0800 Subject: [PATCH] MS-248 support Milvus profiling Former-commit-id: 1507ffd4cafbd0005a8edb161169f127f1532c04 --- cpp/CMakeLists.txt | 5 + cpp/build.sh | 8 +- cpp/cmake/DefineOptions.cmake | 5 + cpp/cmake/ThirdPartyPackages.cmake | 111 +++++++++++++++++- cpp/src/CMakeLists.txt | 1 + .../sdk/examples/simple/src/ClientTest.cpp | 4 +- cpp/src/server/RequestTask.cpp | 40 +++++++ cpp/thirdparty/versions.txt | 2 + 8 files changed, 173 insertions(+), 3 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 3a0e8900a6..0bbb0ef891 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -141,6 +141,11 @@ endif() configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.conf) +message("MILVUS_ENABLE_PROFILING = ${MILVUS_ENABLE_PROFILING}") +if (MILVUS_ENABLE_PROFILING STREQUAL "ON") + ADD_DEFINITIONS(-DMILVUS_ENABLE_PROFILING) +endif() + #install install(DIRECTORY scripts/ DESTINATION scripts diff --git a/cpp/build.sh b/cpp/build.sh index 80be1d7ddb..52ac97d22a 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -7,8 +7,9 @@ INSTALL_PREFIX=$(pwd)/milvus MAKE_CLEAN="OFF" BUILD_COVERAGE="OFF" DB_PATH="/opt/milvus" +PROFILING="OFF" -while getopts "p:d:t:uhlrc" arg +while getopts "p:d:t:uhlrcg" arg do case $arg in t) @@ -36,6 +37,9 @@ do c) BUILD_COVERAGE="ON" ;; + g) + PROFILING="ON" + ;; h) # help echo " @@ -47,6 +51,7 @@ parameter: -l: build license version(default: OFF) -r: remove previous build directory(default: OFF) -c: code coverage(default: OFF) +-g: profiling(default: OFF) usage: ./build.sh -t \${BUILD_TYPE} [-u] [-h] [-g] [-r] [-c] @@ -77,6 +82,7 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then -DCMAKE_LICENSE_CHECK=${LICENSE_CHECK} \ -DBUILD_COVERAGE=${BUILD_COVERAGE} \ -DMILVUS_DB_PATH=${DB_PATH} \ + -DMILVUS_ENABLE_PROFILING=${PROFILING} \ $@ ../" echo ${CMAKE_CMD} diff --git a/cpp/cmake/DefineOptions.cmake b/cpp/cmake/DefineOptions.cmake index 147663d0db..5a3358d4b3 100644 --- a/cpp/cmake/DefineOptions.cmake +++ b/cpp/cmake/DefineOptions.cmake @@ -111,6 +111,11 @@ define_option(MILVUS_WITH_ZSTD "Build with zstd compression" ${MILVUS_WITH_ZSTD_ define_option(MILVUS_WITH_AWS "Build with AWS SDK" ON) +if (MILVUS_ENABLE_PROFILING STREQUAL "ON") + define_option(MILVUS_WITH_LIBUNWIND "Build with libunwind" ON) + define_option(MILVUS_WITH_GPERFTOOLS "Build with gperftools" ON) +endif() + #---------------------------------------------------------------------- if(MSVC) set_option_category("MSVC") diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index 9aa3f62124..f7e1725995 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -37,7 +37,9 @@ set(MILVUS_THIRDPARTY_DEPENDENCIES yaml-cpp ZLIB ZSTD - AWS) + AWS + libunwind + gperftools) message(STATUS "Using ${MILVUS_DEPENDENCY_SOURCE} approach to find dependencies") @@ -89,6 +91,10 @@ macro(build_dependency DEPENDENCY_NAME) build_zstd() elseif("${DEPENDENCY_NAME}" STREQUAL "AWS") build_aws() + elseif("${DEPENDENCY_NAME}" STREQUAL "libunwind") + build_libunwind() + elseif("${DEPENDENCY_NAME}" STREQUAL "gperftools") + build_gperftools() else() message(FATAL_ERROR "Unknown thirdparty dependency to build: ${DEPENDENCY_NAME}") endif () @@ -347,6 +353,21 @@ if(DEFINED ENV{MILVUS_AWS_URL}) else() set(AWS_SOURCE_URL "https://github.com/aws/aws-sdk-cpp/archive/${AWS_VERSION}.tar.gz") endif() + +if(DEFINED ENV{MILVUS_LIBUNWIND_URL}) + set(LIBUNWIND_SOURCE_URL "$ENV{MILVUS_LIBUNWIND_URL}") +else() + set(LIBUNWIND_SOURCE_URL + "https://github.com/libunwind/libunwind/releases/download/v${LIBUNWIND_VERSION}/libunwind-${LIBUNWIND_VERSION}.tar.gz") +endif() + +if(DEFINED ENV{MILVUS_GPERFTOOLS_URL}) + set(GPERFTOOLS_SOURCE_URL "$ENV{MILVUS_GPERFTOOLS_URL}") +else() + set(GPERFTOOLS_SOURCE_URL + "https://github.com/gperftools/gperftools/releases/download/gperftools-${GPERFTOOLS_VERSION}/gperftools-${GPERFTOOLS_VERSION}.tar.gz") +endif() + # ---------------------------------------------------------------------- # ARROW @@ -1896,3 +1917,91 @@ if(MILVUS_WITH_AWS) include_directories(SYSTEM ${AWS_CPP_SDK_CORE_INCLUDE_DIR}) endif() + +# ---------------------------------------------------------------------- +# libunwind + +macro(build_libunwind) + message(STATUS "Building libunwind-${LIBUNWIND_VERSION} from source") + set(LIBUNWIND_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/libunwind_ep-prefix/src/libunwind_ep/install") + set(LIBUNWIND_INCLUDE_DIR "${LIBUNWIND_PREFIX}/include") + set(LIBUNWIND_SHARED_LIB "${LIBUNWIND_PREFIX}/lib/libunwind${CMAKE_SHARED_LIBRARY_SUFFIX}") + set(LIBUNWIND_CONFIGURE_ARGS "--prefix=${LIBUNWIND_PREFIX}") + + externalproject_add(libunwind_ep + URL + ${LIBUNWIND_SOURCE_URL} + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "./configure" + ${LIBUNWIND_CONFIGURE_ARGS} + BUILD_COMMAND + ${MAKE} ${MAKE_BUILD_ARGS} + BUILD_IN_SOURCE + 1 + INSTALL_COMMAND + ${MAKE} install + BUILD_BYPRODUCTS + ${LIBUNWIND_SHARED_LIB}) + + file(MAKE_DIRECTORY "${LIBUNWIND_INCLUDE_DIR}") + + add_library(libunwind SHARED IMPORTED) + set_target_properties(libunwind + PROPERTIES IMPORTED_LOCATION "${LIBUNWIND_SHARED_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${LIBUNWIND_INCLUDE_DIR}") + + add_dependencies(libunwind libunwind_ep) +endmacro() + +if(MILVUS_WITH_LIBUNWIND) + resolve_dependency(libunwind) + + # TODO: Don't use global includes but rather target_include_directories + get_target_property(LIBUNWIND_INCLUDE_DIR libunwind INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM ${LIBUNWIND_INCLUDE_DIR}) +endif() + +# ---------------------------------------------------------------------- +# gperftools + +macro(build_gperftools) + message(STATUS "Building gperftools-${GPERFTOOLS_VERSION} from source") + set(GPERFTOOLS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gperftools_ep-prefix/src/gperftools_ep") + set(GPERFTOOLS_INCLUDE_DIR "${GPERFTOOLS_PREFIX}/include") + set(GPERFTOOLS_STATIC_LIB "${GPERFTOOLS_PREFIX}/lib/libprofiler${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(GPERFTOOLS_CONFIGURE_ARGS "--prefix=${GPERFTOOLS_PREFIX}") + + externalproject_add(gperftools_ep + URL + ${GPERFTOOLS_SOURCE_URL} + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "./configure" + ${GPERFTOOLS_CONFIGURE_ARGS} + BUILD_COMMAND + ${MAKE} ${MAKE_BUILD_ARGS} + BUILD_IN_SOURCE + 1 + INSTALL_COMMAND + ${MAKE} install + BUILD_BYPRODUCTS + ${GPERFTOOLS_STATIC_LIB}) + + file(MAKE_DIRECTORY "${GPERFTOOLS_INCLUDE_DIR}") + + add_library(gperftools SHARED IMPORTED) + set_target_properties(gperftools + PROPERTIES IMPORTED_LOCATION "${GPERFTOOLS_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${GPERFTOOLS_INCLUDE_DIR}") + + add_dependencies(gperftools gperftools_ep) +endmacro() + +if(MILVUS_WITH_GPERFTOOLS) + resolve_dependency(gperftools) + + # TODO: Don't use global includes but rather target_include_directories + get_target_property(GPERFTOOLS_INCLUDE_DIR gperftools INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM ${GPERFTOOLS_INCLUDE_DIR}) +endif() diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 00e2e1e87d..c48db0b762 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -85,6 +85,7 @@ set(third_party_libs zlib zstd mysqlpp + profiler ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so ) if (MEGASEARCH_WITH_ARROW STREQUAL "ON") diff --git a/cpp/src/sdk/examples/simple/src/ClientTest.cpp b/cpp/src/sdk/examples/simple/src/ClientTest.cpp index 495e6ae861..ffec003cd9 100644 --- a/cpp/src/sdk/examples/simple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/simple/src/ClientTest.cpp @@ -148,7 +148,7 @@ namespace { std::cout << "The top 1 result is wrong: " << result_id << " vs. " << search_id << std::endl; } else { - std::cout << "Check result sucessfully" << std::endl; + std::cout << "No." << index-1 << " Check result successfully" << std::endl; } } BLOCK_SPLITER @@ -234,6 +234,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::vector> search_record_array; {//add vectors for (int i = 0; i < ADD_VECTOR_LOOP; i++) {//add vectors + TimeRecorder recorder("Add vector No." + std::to_string(i)); std::vector record_array; int64_t begin_index = i * BATCH_ROW_COUNT; BuildVectors(begin_index, begin_index + BATCH_ROW_COUNT, record_array); @@ -255,6 +256,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { } {//wait unit build index finish + TimeRecorder recorder("Build index"); std::cout << "Wait until build all index done" << std::endl; Status stat = conn->BuildIndex(TABLE_NAME); std::cout << "BuildIndex function call status: " << stat.ToString() << std::endl; diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index 3f42b68531..adbde642e5 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -11,6 +11,10 @@ #include "DBWrapper.h" #include "version.h" +#ifdef MILVUS_ENABLE_PROFILING +#include "gperftools/profiler.h" +#endif + namespace zilliz { namespace milvus { namespace server { @@ -120,6 +124,18 @@ namespace { } } } + + std::string + GetCurrTimeStr() { + char tm_buf[20] = {0}; + time_t tt; + time(&tt); + tt = tt + 8 * 60 * 60; + tm* t = gmtime(&tt); + sprintf(tm_buf, "%4d%02d%02d_%02d%02d%02d", (t->tm_year+1900), (t->tm_mon+1), (t->tm_mday), + (t->tm_hour), (t->tm_min), (t->tm_sec)); + return tm_buf; + } } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -403,6 +419,14 @@ ServerError AddVectorTask::OnExecute() { rc.Record("check validation"); +#ifdef MILVUS_ENABLE_PROFILING + std::string fname = "/tmp/insert_" + std::to_string(this->record_array_.size()) + "_" + + GetCurrTimeStr() + ".profiling"; + ProfilerStart(fname.c_str()); +#else + std::cout << "CYD - MILVUS_ENABLE_PROFILING is OFF!" << std::endl; +#endif + //step 3: prepare float data std::vector vec_f; ServerError error_code = SERVER_SUCCESS; @@ -428,6 +452,10 @@ ServerError AddVectorTask::OnExecute() { return SetError(SERVER_ILLEGAL_VECTOR_ID, msg); } +#ifdef MILVUS_ENABLE_PROFILING + ProfilerStop(); +#endif + rc.Record("do insert"); rc.Elapse("totally cost"); @@ -504,6 +532,13 @@ ServerError SearchVectorTask::OnExecute() { rc.Record("check validation"); +#ifdef MILVUS_ENABLE_PROFILING + std::string fname = "/tmp/search_nq_" + std::to_string(this->record_array_.size()) + "_" + + "top_" + std::to_string(this->top_k_) + "_" + + GetCurrTimeStr() + ".profiling"; + ProfilerStart(fname.c_str()); +#endif + //step 3: prepare float data std::vector vec_f; ConvertRowRecordToFloatArray(record_array_, table_info.dimension_, vec_f, error_code, error_msg); @@ -556,6 +591,11 @@ ServerError SearchVectorTask::OnExecute() { result_array_.emplace_back(thrift_topk_result); } + +#ifdef MILVUS_ENABLE_PROFILING + ProfilerStop(); +#endif + rc.Record("construct result"); rc.Elapse("totally cost"); diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 311760948d..885ecfab87 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -19,5 +19,7 @@ YAMLCPP_VERSION=0.6.2 ZLIB_VERSION=v1.2.11 ZSTD_VERSION=v1.4.0 AWS_VERSION=1.7.125 +LIBUNWIND_VERSION=1.3.1 +GPERFTOOLS_VERSION=2.7 # vim: set filetype=sh: