test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-02 20:09:57 +08:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

MS-248 support Milvus profiling

Browse Source 
Former-commit-id: 1507ffd4cafbd0005a8edb161169f127f1532c04
This commit is contained in:
yudong.cai 2019-07-17 20:24:09 +08:00
parent 8173f37b10
commit f8875429b1
8 changed files with 173 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
cpp/CMakeLists.txt
View File

@ -141,6 +141,11 @@ endif()
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.conf)
message("MILVUS_ENABLE_PROFILING = ${MILVUS_ENABLE_PROFILING}")
if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
ADD_DEFINITIONS(-DMILVUS_ENABLE_PROFILING)
endif()
#install
install(DIRECTORY scripts/
DESTINATION scripts

8
cpp/build.sh
View File

@ -7,8 +7,9 @@ INSTALL_PREFIX=$(pwd)/milvus
MAKE_CLEAN="OFF"
BUILD_COVERAGE="OFF"
DB_PATH="/opt/milvus"
PROFILING="OFF"
while getopts "p:d:t:uhlrc" arg
while getopts "p:d:t:uhlrcg" arg
do
case $arg in
t)
@ -36,6 +37,9 @@ do
c)
BUILD_COVERAGE="ON"
;;
g)
PROFILING="ON"
;;
h) # help
echo "
@ -47,6 +51,7 @@ parameter:
-l: build license version(default: OFF)
-r: remove previous build directory(default: OFF)
-c: code coverage(default: OFF)
-g: profiling(default: OFF)
usage:
./build.sh -t \${BUILD_TYPE} [-u] [-h] [-g] [-r] [-c]
@ -77,6 +82,7 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then
-DCMAKE_LICENSE_CHECK=${LICENSE_CHECK} \
-DBUILD_COVERAGE=${BUILD_COVERAGE} \
-DMILVUS_DB_PATH=${DB_PATH} \
-DMILVUS_ENABLE_PROFILING=${PROFILING} \
$@ ../"
echo ${CMAKE_CMD}

5
cpp/cmake/DefineOptions.cmake
View File

@ -111,6 +111,11 @@ define_option(MILVUS_WITH_ZSTD "Build with zstd compression" ${MILVUS_WITH_ZSTD_
define_option(MILVUS_WITH_AWS "Build with AWS SDK" ON)
if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
define_option(MILVUS_WITH_LIBUNWIND "Build with libunwind" ON)
define_option(MILVUS_WITH_GPERFTOOLS "Build with gperftools" ON)
endif()
#----------------------------------------------------------------------
if(MSVC)
set_option_category("MSVC")

111
cpp/cmake/ThirdPartyPackages.cmake
View File

@ -37,7 +37,9 @@ set(MILVUS_THIRDPARTY_DEPENDENCIES
yaml-cpp
ZLIB
ZSTD
AWS)
AWS
libunwind
gperftools)
message(STATUS "Using ${MILVUS_DEPENDENCY_SOURCE} approach to find dependencies")
@ -89,6 +91,10 @@ macro(build_dependency DEPENDENCY_NAME)
build_zstd()
elseif("${DEPENDENCY_NAME}" STREQUAL "AWS")
build_aws()
elseif("${DEPENDENCY_NAME}" STREQUAL "libunwind")
build_libunwind()
elseif("${DEPENDENCY_NAME}" STREQUAL "gperftools")
build_gperftools()
else()
message(FATAL_ERROR "Unknown thirdparty dependency to build: ${DEPENDENCY_NAME}")
endif ()
@ -347,6 +353,21 @@ if(DEFINED ENV{MILVUS_AWS_URL})
else()
set(AWS_SOURCE_URL "https://github.com/aws/aws-sdk-cpp/archive/${AWS_VERSION}.tar.gz")
endif()
if(DEFINED ENV{MILVUS_LIBUNWIND_URL})
set(LIBUNWIND_SOURCE_URL "$ENV{MILVUS_LIBUNWIND_URL}")
else()
set(LIBUNWIND_SOURCE_URL
"https://github.com/libunwind/libunwind/releases/download/v${LIBUNWIND_VERSION}/libunwind-${LIBUNWIND_VERSION}.tar.gz")
endif()
if(DEFINED ENV{MILVUS_GPERFTOOLS_URL})
set(GPERFTOOLS_SOURCE_URL "$ENV{MILVUS_GPERFTOOLS_URL}")
else()
set(GPERFTOOLS_SOURCE_URL
"https://github.com/gperftools/gperftools/releases/download/gperftools-${GPERFTOOLS_VERSION}/gperftools-${GPERFTOOLS_VERSION}.tar.gz")
endif()
# ----------------------------------------------------------------------
# ARROW
@ -1896,3 +1917,91 @@ if(MILVUS_WITH_AWS)
include_directories(SYSTEM ${AWS_CPP_SDK_CORE_INCLUDE_DIR})
endif()
# ----------------------------------------------------------------------
# libunwind
macro(build_libunwind)
message(STATUS "Building libunwind-${LIBUNWIND_VERSION} from source")
set(LIBUNWIND_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/libunwind_ep-prefix/src/libunwind_ep/install")
set(LIBUNWIND_INCLUDE_DIR "${LIBUNWIND_PREFIX}/include")
set(LIBUNWIND_SHARED_LIB "${LIBUNWIND_PREFIX}/lib/libunwind${CMAKE_SHARED_LIBRARY_SUFFIX}")
set(LIBUNWIND_CONFIGURE_ARGS "--prefix=${LIBUNWIND_PREFIX}")
externalproject_add(libunwind_ep
URL
${LIBUNWIND_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${LIBUNWIND_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS}
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE} install
BUILD_BYPRODUCTS
${LIBUNWIND_SHARED_LIB})
file(MAKE_DIRECTORY "${LIBUNWIND_INCLUDE_DIR}")
add_library(libunwind SHARED IMPORTED)
set_target_properties(libunwind
PROPERTIES IMPORTED_LOCATION "${LIBUNWIND_SHARED_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${LIBUNWIND_INCLUDE_DIR}")
add_dependencies(libunwind libunwind_ep)
endmacro()
if(MILVUS_WITH_LIBUNWIND)
resolve_dependency(libunwind)
# TODO: Don't use global includes but rather target_include_directories
get_target_property(LIBUNWIND_INCLUDE_DIR libunwind INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM ${LIBUNWIND_INCLUDE_DIR})
endif()
# ----------------------------------------------------------------------
# gperftools
macro(build_gperftools)
message(STATUS "Building gperftools-${GPERFTOOLS_VERSION} from source")
set(GPERFTOOLS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gperftools_ep-prefix/src/gperftools_ep")
set(GPERFTOOLS_INCLUDE_DIR "${GPERFTOOLS_PREFIX}/include")
set(GPERFTOOLS_STATIC_LIB "${GPERFTOOLS_PREFIX}/lib/libprofiler${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(GPERFTOOLS_CONFIGURE_ARGS "--prefix=${GPERFTOOLS_PREFIX}")
externalproject_add(gperftools_ep
URL
${GPERFTOOLS_SOURCE_URL}
${EP_LOG_OPTIONS}
CONFIGURE_COMMAND
"./configure"
${GPERFTOOLS_CONFIGURE_ARGS}
BUILD_COMMAND
${MAKE} ${MAKE_BUILD_ARGS}
BUILD_IN_SOURCE
1
INSTALL_COMMAND
${MAKE} install
BUILD_BYPRODUCTS
${GPERFTOOLS_STATIC_LIB})
file(MAKE_DIRECTORY "${GPERFTOOLS_INCLUDE_DIR}")
add_library(gperftools SHARED IMPORTED)
set_target_properties(gperftools
PROPERTIES IMPORTED_LOCATION "${GPERFTOOLS_STATIC_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${GPERFTOOLS_INCLUDE_DIR}")
add_dependencies(gperftools gperftools_ep)
endmacro()
if(MILVUS_WITH_GPERFTOOLS)
resolve_dependency(gperftools)
# TODO: Don't use global includes but rather target_include_directories
get_target_property(GPERFTOOLS_INCLUDE_DIR gperftools INTERFACE_INCLUDE_DIRECTORIES)
include_directories(SYSTEM ${GPERFTOOLS_INCLUDE_DIR})
endif()

1
cpp/src/CMakeLists.txt
View File

@ -85,6 +85,7 @@ set(third_party_libs
zlib
zstd
mysqlpp
profiler
${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so
)
if (MEGASEARCH_WITH_ARROW STREQUAL "ON")

4
cpp/src/sdk/examples/simple/src/ClientTest.cpp
View File

@ -148,7 +148,7 @@ namespace {
std::cout << "The top 1 result is wrong: " << result_id
<< " vs. " << search_id << std::endl;
} else {
std::cout << "Check result sucessfully" << std::endl;
std::cout << "No." << index-1 << " Check result successfully" << std::endl;
}
}
BLOCK_SPLITER
@ -234,6 +234,7 @@ ClientTest::Test(const std::string& address, const std::string& port) {
std::vector<std::pair<int64_t, RowRecord>> search_record_array;
{//add vectors
for (int i = 0; i < ADD_VECTOR_LOOP; i++) {//add vectors
TimeRecorder recorder("Add vector No." + std::to_string(i));
std::vector<RowRecord> record_array;
int64_t begin_index = i * BATCH_ROW_COUNT;
BuildVectors(begin_index, begin_index + BATCH_ROW_COUNT, record_array);
@ -255,6 +256,7 @@ ClientTest::Test(const std::string& address, const std::string& port) {
}
{//wait unit build index finish
TimeRecorder recorder("Build index");
std::cout << "Wait until build all index done" << std::endl;
Status stat = conn->BuildIndex(TABLE_NAME);
std::cout << "BuildIndex function call status: " << stat.ToString() << std::endl;

40
cpp/src/server/RequestTask.cpp
View File

@ -11,6 +11,10 @@
#include "DBWrapper.h"
#include "version.h"
#ifdef MILVUS_ENABLE_PROFILING
#include "gperftools/profiler.h"
#endif
namespace zilliz {
namespace milvus {
namespace server {
@ -120,6 +124,18 @@ namespace {
}
}
}
std::string
GetCurrTimeStr() {
char tm_buf[20] = {0};
time_t tt;
time(&tt);
tt = tt + 8 * 60 * 60;
tm* t = gmtime(&tt);
sprintf(tm_buf, "%4d%02d%02d_%02d%02d%02d", (t->tm_year+1900), (t->tm_mon+1), (t->tm_mday),
(t->tm_hour), (t->tm_min), (t->tm_sec));
return tm_buf;
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@ -403,6 +419,14 @@ ServerError AddVectorTask::OnExecute() {
rc.Record("check validation");
#ifdef MILVUS_ENABLE_PROFILING
std::string fname = "/tmp/insert_" + std::to_string(this->record_array_.size()) + "_" +
GetCurrTimeStr() + ".profiling";
ProfilerStart(fname.c_str());
#else
std::cout << "CYD - MILVUS_ENABLE_PROFILING is OFF!" << std::endl;
#endif
//step 3: prepare float data
std::vector<float> vec_f;
ServerError error_code = SERVER_SUCCESS;
@ -428,6 +452,10 @@ ServerError AddVectorTask::OnExecute() {
return SetError(SERVER_ILLEGAL_VECTOR_ID, msg);
}
#ifdef MILVUS_ENABLE_PROFILING
ProfilerStop();
#endif
rc.Record("do insert");
rc.Elapse("totally cost");
@ -504,6 +532,13 @@ ServerError SearchVectorTask::OnExecute() {
rc.Record("check validation");
#ifdef MILVUS_ENABLE_PROFILING
std::string fname = "/tmp/search_nq_" + std::to_string(this->record_array_.size()) + "_" +
"top_" + std::to_string(this->top_k_) + "_" +
GetCurrTimeStr() + ".profiling";
ProfilerStart(fname.c_str());
#endif
//step 3: prepare float data
std::vector<float> vec_f;
ConvertRowRecordToFloatArray(record_array_, table_info.dimension_, vec_f, error_code, error_msg);
@ -556,6 +591,11 @@ ServerError SearchVectorTask::OnExecute() {
result_array_.emplace_back(thrift_topk_result);
}
#ifdef MILVUS_ENABLE_PROFILING
ProfilerStop();
#endif
rc.Record("construct result");
rc.Elapse("totally cost");

2
cpp/thirdparty/versions.txt vendored
View File

@ -19,5 +19,7 @@ YAMLCPP_VERSION=0.6.2
ZLIB_VERSION=v1.2.11
ZSTD_VERSION=v1.4.0
AWS_VERSION=1.7.125
LIBUNWIND_VERSION=1.3.1
GPERFTOOLS_VERSION=2.7
# vim: set filetype=sh: