test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-02 20:09:57 +08:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

MS-327 clean code

Browse Source 
Former-commit-id: 9ca0ea6cb89b3fb95e17b0c0d6f647fe0c3ff7c4
This commit is contained in:
starlord 2019-08-08 14:20:00 +08:00
parent b82fa58a98
commit 4d5618f4c3
27 changed files with 7 additions and 2489 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cpp/.gitignore vendored
View File

@ -8,3 +8,4 @@ base.info
output.info
output_new.info
server.info
thirdparty/knowhere/

1
cpp/CHANGELOG.md
View File

@ -7,6 +7,7 @@ Please mark all change in change log and use the ticket from JIRA.
## Bug
## Improvement
- MS-327 - Clean code for milvus
## New Feature

4
cpp/CMakeLists.txt
View File

@ -91,10 +91,6 @@ endif()
if(CMAKE_BUILD_TYPE STREQUAL "Release")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp")
if (CMAKE_LICENSE_CHECK STREQUAL "ON")
set(ENABLE_LICENSE "ON")
add_definitions("-DENABLE_LICENSE")
endif ()
else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp")
endif()

58
cpp/src/CMakeLists.txt
View File

@ -23,16 +23,6 @@ set(db_scheduler_files
${scheduler_task_files}
)
set(license_check_files
license/LicenseLibrary.cpp
license/LicenseCheck.cpp
)
set(license_generator_files
license/LicenseGenerator.cpp
license/LicenseLibrary.cpp
)
set(service_files
thrift/gen-cpp/MilvusService.cpp
thrift/gen-cpp/milvus_constants.cpp
@ -53,9 +43,6 @@ set(engine_files
${knowhere_files}
)
set(get_sys_info_files
license/GetSysInfo.cpp)
set(s3_client_files
storage/s3/S3ClientWrapper.cpp
storage/s3/S3ClientWrapper.h)
@ -138,29 +125,12 @@ if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
)
endif ()
if (ENABLE_LICENSE STREQUAL "ON")
link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs")
link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
set(license_libs
nvidia-ml
crypto
cudart
cublas
)
endif ()
cuda_add_library(milvus_engine STATIC ${engine_files})
target_link_libraries(milvus_engine ${engine_libs} ${third_party_libs})
add_library(metrics STATIC ${metrics_files})
if (ENABLE_LICENSE STREQUAL "ON")
add_library(license_check STATIC ${license_check_files})
target_link_libraries(license_check ${license_libs} ${third_party_libs})
endif ()
set(metrics_lib
easyloggingpp
yaml-cpp
@ -197,30 +167,12 @@ add_executable(milvus_server
${metrics_files}
)
if (ENABLE_LICENSE STREQUAL "ON")
target_link_libraries(milvus_server
${server_libs}
license_check
${third_party_libs}
${knowhere_libs})
else ()
target_link_libraries(milvus_server
${server_libs}
${third_party_libs}
${knowhere_libs}
)
endif()
if (ENABLE_LICENSE STREQUAL "ON")
add_executable(get_sys_info ${get_sys_info_files})
add_executable(license_generator ${license_generator_files})
target_link_libraries(get_sys_info ${license_libs} license_check ${third_party_libs})
target_link_libraries(license_generator ${license_libs} ${third_party_libs})
install(TARGETS get_sys_info DESTINATION bin)
install(TARGETS license_generator DESTINATION bin)
endif ()
target_link_libraries(milvus_server
${server_libs}
${third_party_libs}
${knowhere_libs}
)
install(TARGETS milvus_server DESTINATION bin)

1
cpp/src/db/DBImpl.h
View File

@ -6,7 +6,6 @@
#pragma once
#include "DB.h"
#include "MemManager.h"
#include "Types.h"
#include "utils/ThreadPool.h"
#include "MemManagerAbstract.h"

11
cpp/src/db/Factories.cpp
View File

@ -6,7 +6,6 @@
#include "Factories.h"
#include "DBImpl.h"
#include "MemManager.h"
#include "NewMemManager.h"
#include "Exception.h"
@ -103,16 +102,6 @@ DB* DBFactory::Build(const Options& options) {
MemManagerAbstractPtr MemManagerFactory::Build(const std::shared_ptr<meta::Meta>& meta,
const Options& options) {
if (const char* env = getenv("MILVUS_USE_OLD_MEM_MANAGER")) {
std::string env_str = env;
std::transform(env_str.begin(), env_str.end(), env_str.begin(), ::toupper);
if (env_str == "ON") {
return std::make_shared<MemManager>(meta, options);
}
else {
return std::make_shared<NewMemManager>(meta, options);
}
}
return std::make_shared<NewMemManager>(meta, options);
}

222
cpp/src/db/FaissExecutionEngine.cpp
View File

@ -1,222 +0,0 @@
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#if 0
#include "FaissExecutionEngine.h"
#include "Log.h"
#include "utils/CommonUtil.h"
#include <faiss/AutoTune.h>
#include <faiss/MetaIndexes.h>
#include <faiss/IndexFlat.h>
#include <faiss/index_io.h>
#include <wrapper/Index.h>
#include <wrapper/IndexBuilder.h>
#include <cache/CpuCacheMgr.h>
#include "faiss/IndexIVF.h"
#include "metrics/Metrics.h"
namespace zilliz {
namespace milvus {
namespace engine {
namespace {
std::string GetMetricType() {
server::ServerConfig &config = server::ServerConfig::GetInstance();
server::ConfigNode engine_config = config.GetConfig(server::CONFIG_ENGINE);
return engine_config.GetValue(server::CONFIG_METRICTYPE, "L2");
}
}
std::string IndexStatsHelper::ToString(const std::string &prefix) const {
return "";
}
void IndexStatsHelper::Reset() const {
faiss::indexIVF_stats.reset();
}
std::string FaissIndexIVFStatsHelper::ToString(const std::string &prefix) const {
std::stringstream ss;
ss << prefix;
ss << identifier_ << ":";
ss << " NQ=" << faiss::indexIVF_stats.nq;
ss << " NL=" << faiss::indexIVF_stats.nlist;
ss << " ND=" << faiss::indexIVF_stats.ndis;
ss << " NH=" << faiss::indexIVF_stats.nheap_updates;
ss << " Q=" << faiss::indexIVF_stats.quantization_time;
ss << " S=" << faiss::indexIVF_stats.search_time;
return ss.str();
}
FaissExecutionEngine::FaissExecutionEngine(uint16_t dimension,
const std::string &location,
const std::string &build_index_type,
const std::string &raw_index_type)
: location_(location),
build_index_type_(build_index_type),
raw_index_type_(raw_index_type) {
std::string metric_type = GetMetricType();
faiss::MetricType faiss_metric_type = (metric_type == "L2") ? faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT;
pIndex_.reset(faiss::index_factory(dimension, raw_index_type.c_str(), faiss_metric_type));
}
FaissExecutionEngine::FaissExecutionEngine(std::shared_ptr<faiss::Index> index,
const std::string &location,
const std::string &build_index_type,
const std::string &raw_index_type)
: pIndex_(index),
location_(location),
build_index_type_(build_index_type),
raw_index_type_(raw_index_type) {
}
Status FaissExecutionEngine::AddWithIds(long n, const float *xdata, const long *xids) {
pIndex_->add_with_ids(n, xdata, xids);
return Status::OK();
}
size_t FaissExecutionEngine::Count() const {
return (size_t) (pIndex_->ntotal);
}
size_t FaissExecutionEngine::Size() const {
return (size_t) (Count() * pIndex_->d) * sizeof(float);
}
size_t FaissExecutionEngine::Dimension() const {
return pIndex_->d;
}
size_t FaissExecutionEngine::PhysicalSize() const {
return server::CommonUtil::GetFileSize(location_);
}
Status FaissExecutionEngine::Serialize() {
write_index(pIndex_.get(), location_.c_str());
return Status::OK();
}
Status FaissExecutionEngine::Load(bool to_cache) {
auto index = zilliz::milvus::cache::CpuCacheMgr::GetInstance()->GetIndex(location_);
bool already_in_cache = (index != nullptr);
auto start_time = METRICS_NOW_TIME;
if (!index) {
index = read_index(location_);
ENGINE_LOG_DEBUG << "Disk io from: " << location_;
}
pIndex_ = index->data();
if (!already_in_cache && to_cache) {
Cache();
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
server::Metrics::GetInstance().FaissDiskLoadDurationSecondsHistogramObserve(total_time);
double total_size = (pIndex_->d) * (pIndex_->ntotal) * 4;
server::Metrics::GetInstance().FaissDiskLoadSizeBytesHistogramObserve(total_size);
// server::Metrics::GetInstance().FaissDiskLoadIOSpeedHistogramObserve(total_size/double(total_time));
server::Metrics::GetInstance().FaissDiskLoadIOSpeedGaugeSet(total_size / double(total_time));
}
return Status::OK();
}
Status FaissExecutionEngine::Merge(const std::string &location) {
if (location == location_) {
return Status::Error("Cannot Merge Self");
}
ENGINE_LOG_DEBUG << "Merge raw file: " << location << " to: " << location_;
auto to_merge = zilliz::milvus::cache::CpuCacheMgr::GetInstance()->GetIndex(location);
if (!to_merge) {
to_merge = read_index(location);
}
auto file_index = dynamic_cast<faiss::IndexIDMap *>(to_merge->data().get());
pIndex_->add_with_ids(file_index->ntotal, dynamic_cast<faiss::IndexFlat *>(file_index->index)->xb.data(),
file_index->id_map.data());
return Status::OK();
}
ExecutionEnginePtr
FaissExecutionEngine::BuildIndex(const std::string &location) {
ENGINE_LOG_DEBUG << "Build index file: " << location << " from: " << location_;
auto opd = std::make_shared<Operand>();
opd->d = pIndex_->d;
opd->index_type = build_index_type_;
opd->metric_type = GetMetricType();
IndexBuilderPtr pBuilder = GetIndexBuilder(opd);
auto from_index = dynamic_cast<faiss::IndexIDMap *>(pIndex_.get());
auto index = pBuilder->build_all(from_index->ntotal,
dynamic_cast<faiss::IndexFlat *>(from_index->index)->xb.data(),
from_index->id_map.data());
ExecutionEnginePtr new_ee(new FaissExecutionEngine(index->data(), location, build_index_type_, raw_index_type_));
return new_ee;
}
Status FaissExecutionEngine::Search(long n,
const float *data,
long k,
float *distances,
long *labels) const {
auto start_time = METRICS_NOW_TIME;
std::shared_ptr<faiss::IndexIVF> ivf_index = std::dynamic_pointer_cast<faiss::IndexIVF>(pIndex_);
if (ivf_index) {
std::string stats_prefix = "K=" + std::to_string(k) + ":";
ENGINE_LOG_DEBUG << "Searching index type: " << build_index_type_ << " nProbe: " << nprobe_;
ivf_index->nprobe = nprobe_;
ivf_stats_helper_.Reset();
ivf_index->search(n, data, k, distances, labels);
ENGINE_LOG_INFO << ivf_stats_helper_.ToString(stats_prefix);
} else {
ENGINE_LOG_DEBUG << "Searching raw file";
pIndex_->search(n, data, k, distances, labels);
}
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
server::Metrics::GetInstance().QueryIndexTypePerSecondSet(build_index_type_, double(n) / double(total_time));
return Status::OK();
}
Status FaissExecutionEngine::Cache() {
auto index = std::make_shared<Index>(pIndex_);
cache::DataObjPtr data_obj = std::make_shared<cache::DataObj>(index, PhysicalSize());
zilliz::milvus::cache::CpuCacheMgr::GetInstance()->InsertItem(location_, data_obj);
return Status::OK();
}
Status FaissExecutionEngine::Init() {
if (build_index_type_ == BUILD_INDEX_TYPE_IVF ||
build_index_type_ == BUILD_INDEX_TYPE_IVFSQ8) {
using namespace zilliz::milvus::server;
ServerConfig &config = ServerConfig::GetInstance();
ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE);
nprobe_ = engine_config.GetInt32Value(CONFIG_NPROBE, 1000);
nlist_ = engine_config.GetInt32Value(CONFIG_NLIST, 16384);
} else if (build_index_type_ == BUILD_INDEX_TYPE_IDMAP) { ;
} else {
return Status::Error("Wrong index type: ", build_index_type_);
}
return Status::OK();
}
} // namespace engine
} // namespace milvus
} // namespace zilliz
#endif

98
cpp/src/db/FaissExecutionEngine.h
View File

@ -1,98 +0,0 @@
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#if 0
#include "ExecutionEngine.h"
#include "faiss/Index.h"
#include <memory>
#include <string>
namespace zilliz {
namespace milvus {
namespace engine {
const static std::string BUILD_INDEX_TYPE_IDMAP = "IDMap";
const static std::string BUILD_INDEX_TYPE_IVF = "IVF";
const static std::string BUILD_INDEX_TYPE_IVFSQ8 = "IVFSQ8";
class IndexStatsHelper {
public:
using Ptr = std::shared_ptr<IndexStatsHelper>;
virtual std::string ToString(const std::string &prefix = "") const;
virtual void Reset() const;
virtual ~IndexStatsHelper() {}
};
class FaissIndexIVFStatsHelper : public IndexStatsHelper {
public:
std::string ToString(const std::string &prefix = "") const override;
private:
const std::string identifier_ = BUILD_INDEX_TYPE_IVF;
};
class FaissExecutionEngine : public ExecutionEngine {
public:
FaissExecutionEngine(uint16_t dimension,
const std::string &location,
const std::string &build_index_type,
const std::string &raw_index_type);
FaissExecutionEngine(std::shared_ptr<faiss::Index> index,
const std::string &location,
const std::string &build_index_type,
const std::string &raw_index_type);
Status AddWithIds(long n, const float *xdata, const long *xids) override;
size_t Count() const override;
size_t Size() const override;
size_t Dimension() const override;
size_t PhysicalSize() const override;
Status Serialize() override;
Status Load(bool to_cache) override;
Status Merge(const std::string &location) override;
Status Search(long n,
const float *data,
long k,
float *distances,
long *labels) const override;
ExecutionEnginePtr BuildIndex(const std::string &) override;
Status Cache() override;
Status Init() override;
protected:
FaissIndexIVFStatsHelper ivf_stats_helper_;
std::shared_ptr<faiss::Index> pIndex_;
std::string location_;
std::string build_index_type_;
std::string raw_index_type_;
size_t nprobe_ = 0;
size_t nlist_ = 0;
};
} // namespace engine
} // namespace milvus
} // namespace zilliz
#endif

226
cpp/src/db/MemManager.cpp
View File

@ -1,226 +0,0 @@
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#include "MemManager.h"
#include "Meta.h"
#include "MetaConsts.h"
#include "EngineFactory.h"
#include "metrics/Metrics.h"
#include "Log.h"
#include <iostream>
#include <sstream>
#include <thread>
#include <easylogging++.h>
namespace zilliz {
namespace milvus {
namespace engine {
MemVectors::MemVectors(const std::shared_ptr<meta::Meta> &meta_ptr,
const meta::TableFileSchema &schema, const Options &options)
: meta_(meta_ptr),
options_(options),
schema_(schema),
id_generator_(new SimpleIDGenerator()),
active_engine_(EngineFactory::Build(schema_.dimension_, schema_.location_, (EngineType) schema_.engine_type_)) {
}
Status MemVectors::Add(size_t n_, const float *vectors_, IDNumbers &vector_ids_) {
if (active_engine_ == nullptr) {
return Status::Error("index engine is null");
}
auto start_time = METRICS_NOW_TIME;
id_generator_->GetNextIDNumbers(n_, vector_ids_);
Status status = active_engine_->AddWithIds(n_, vectors_, vector_ids_.data());
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
server::Metrics::GetInstance().AddVectorsPerSecondGaugeSet(static_cast<int>(n_),
static_cast<int>(schema_.dimension_),
total_time);
return status;
}
size_t MemVectors::RowCount() const {
if (active_engine_ == nullptr) {
return 0;
}
return active_engine_->Count();
}
size_t MemVectors::Size() const {
if (active_engine_ == nullptr) {
return 0;
}
return active_engine_->Size();
}
Status MemVectors::Serialize(std::string &table_id) {
if (active_engine_ == nullptr) {
return Status::Error("index engine is null");
}
table_id = schema_.table_id_;
auto size = Size();
auto start_time = METRICS_NOW_TIME;
active_engine_->Serialize();
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
schema_.size_ = size;
server::Metrics::GetInstance().DiskStoreIOSpeedGaugeSet(size / total_time);
schema_.file_type_ = (size >= options_.index_trigger_size) ?
meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW;
auto status = meta_->UpdateTableFile(schema_);
ENGINE_LOG_DEBUG << "New " << ((schema_.file_type_ == meta::TableFileSchema::RAW) ? "raw" : "to_index")
<< " file " << schema_.file_id_ << " of size " << active_engine_->Size() << " bytes";
if(options_.insert_cache_immediately_) {
active_engine_->Cache();
}
return status;
}
MemVectors::~MemVectors() {
if (id_generator_ != nullptr) {
delete id_generator_;
id_generator_ = nullptr;
}
}
/*
* MemManager
*/
MemManager::MemVectorsPtr MemManager::GetMemByTable(
const std::string &table_id) {
auto memIt = mem_id_map_.find(table_id);
if (memIt != mem_id_map_.end()) {
return memIt->second;
}
meta::TableFileSchema table_file;
table_file.table_id_ = table_id;
auto status = meta_->CreateTableFile(table_file);
if (!status.ok()) {
return nullptr;
}
mem_id_map_[table_id] = MemVectorsPtr(new MemVectors(meta_, table_file, options_));
return mem_id_map_[table_id];
}
Status MemManager::InsertVectors(const std::string &table_id_,
size_t n_,
const float *vectors_,
IDNumbers &vector_ids_) {
std::unique_lock<std::mutex> lock(mutex_);
return InsertVectorsNoLock(table_id_, n_, vectors_, vector_ids_);
}
Status MemManager::InsertVectorsNoLock(const std::string &table_id,
size_t n,
const float *vectors,
IDNumbers &vector_ids) {
MemVectorsPtr mem = GetMemByTable(table_id);
if (mem == nullptr) {
return Status::NotFound("Group " + table_id + " not found!");
}
//makesure each file size less than index_trigger_size
if (mem->Size() > options_.index_trigger_size) {
std::unique_lock<std::mutex> lock(serialization_mtx_);
immu_mem_list_.push_back(mem);
mem_id_map_.erase(table_id);
return InsertVectorsNoLock(table_id, n, vectors, vector_ids);
} else {
return mem->Add(n, vectors, vector_ids);
}
}
Status MemManager::ToImmutable() {
std::unique_lock<std::mutex> lock(mutex_);
MemIdMap temp_map;
for (auto &kv: mem_id_map_) {
if (kv.second->RowCount() == 0) {
temp_map.insert(kv);
continue;//empty vector, no need to serialize
}
immu_mem_list_.push_back(kv.second);
}
mem_id_map_.swap(temp_map);
return Status::OK();
}
Status MemManager::Serialize(std::set<std::string> &table_ids) {
ToImmutable();
std::unique_lock<std::mutex> lock(serialization_mtx_);
std::string table_id;
table_ids.clear();
for (auto &mem : immu_mem_list_) {
mem->Serialize(table_id);
table_ids.insert(table_id);
}
immu_mem_list_.clear();
return Status::OK();
}
Status MemManager::EraseMemVector(const std::string &table_id) {
{//erase MemVector from rapid-insert cache
std::unique_lock<std::mutex> lock(mutex_);
mem_id_map_.erase(table_id);
}
{//erase MemVector from serialize cache
std::unique_lock<std::mutex> lock(serialization_mtx_);
MemList temp_list;
for (auto &mem : immu_mem_list_) {
if (mem->TableId() != table_id) {
temp_list.push_back(mem);
}
}
immu_mem_list_.swap(temp_list);
}
return Status::OK();
}
size_t MemManager::GetCurrentMutableMem() {
size_t totalMem = 0;
for (auto &kv : mem_id_map_) {
auto memVector = kv.second;
totalMem += memVector->Size();
}
return totalMem;
}
size_t MemManager::GetCurrentImmutableMem() {
size_t totalMem = 0;
for (auto &memVector : immu_mem_list_) {
totalMem += memVector->Size();
}
return totalMem;
}
size_t MemManager::GetCurrentMem() {
return GetCurrentMutableMem() + GetCurrentImmutableMem();
}
} // namespace engine
} // namespace milvus
} // namespace zilliz

109
cpp/src/db/MemManager.h
View File

@ -1,109 +0,0 @@
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include "ExecutionEngine.h"
#include "IDGenerator.h"
#include "Status.h"
#include "Meta.h"
#include "MemManagerAbstract.h"
#include <map>
#include <string>
#include <ctime>
#include <memory>
#include <mutex>
namespace zilliz {
namespace milvus {
namespace engine {
namespace meta {
class Meta;
}
class MemVectors {
public:
using MetaPtr = meta::Meta::Ptr;
using Ptr = std::shared_ptr<MemVectors>;
explicit MemVectors(const std::shared_ptr<meta::Meta> &,
const meta::TableFileSchema &, const Options &);
Status Add(size_t n_, const float *vectors_, IDNumbers &vector_ids_);
size_t RowCount() const;
size_t Size() const;
Status Serialize(std::string &table_id);
~MemVectors();
const std::string &Location() const { return schema_.location_; }
std::string TableId() const { return schema_.table_id_; }
private:
MemVectors() = delete;
MemVectors(const MemVectors &) = delete;
MemVectors &operator=(const MemVectors &) = delete;
MetaPtr meta_;
Options options_;
meta::TableFileSchema schema_;
IDGenerator *id_generator_;
ExecutionEnginePtr active_engine_;
}; // MemVectors
class MemManager : public MemManagerAbstract {
public:
using MetaPtr = meta::Meta::Ptr;
using MemVectorsPtr = typename MemVectors::Ptr;
using Ptr = std::shared_ptr<MemManager>;
MemManager(const std::shared_ptr<meta::Meta> &meta, const Options &options)
: meta_(meta), options_(options) {}
Status InsertVectors(const std::string &table_id,
size_t n, const float *vectors, IDNumbers &vector_ids) override;
Status Serialize(std::set<std::string> &table_ids) override;
Status EraseMemVector(const std::string &table_id) override;
size_t GetCurrentMutableMem() override;
size_t GetCurrentImmutableMem() override;
size_t GetCurrentMem() override;
private:
MemVectorsPtr GetMemByTable(const std::string &table_id);
Status InsertVectorsNoLock(const std::string &table_id,
size_t n, const float *vectors, IDNumbers &vector_ids);
Status ToImmutable();
using MemIdMap = std::map<std::string, MemVectorsPtr>;
using MemList = std::vector<MemVectorsPtr>;
MemIdMap mem_id_map_;
MemList immu_mem_list_;
MetaPtr meta_;
Options options_;
std::mutex mutex_;
std::mutex serialization_mtx_;
}; // MemManager
} // namespace engine
} // namespace milvus
} // namespace zilliz

69
cpp/src/license/GPUInfoFile.h
View File

@ -1,69 +0,0 @@
///*******************************************************************************
// * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// * Unauthorized copying of this file, via any medium is strictly prohibited.
// * Proprietary and confidential.
// ******************************************************************************/
//#pragma once
//
//#include <boost/serialization/access.hpp>
//#include <string>
//#include <map>
//
//
//class GPUInfoFile {
// public:
// GPUInfoFile() = default;
//
// GPUInfoFile(const int &device_count, const std::map<int, std::string> &uuid_encryption_map)
// : device_count_(device_count), uuid_encryption_map_(uuid_encryption_map) {}
//
// int get_device_count() {
// return device_count_;
// }
// std::map<int, std::string> &get_uuid_encryption_map() {
// return uuid_encryption_map_;
// }
//
//
// public:
// friend class boost::serialization::access;
//
// template<typename Archive>
// void serialize(Archive &ar, const unsigned int version) {
// ar & device_count_;
// ar & uuid_encryption_map_;
// }
//
// public:
// int device_count_ = 0;
// std::map<int, std::string> uuid_encryption_map_;
//};
//
//class SerializedGPUInfoFile {
// public:
// ~SerializedGPUInfoFile() {
// if (gpu_info_file_ != nullptr) {
// delete (gpu_info_file_);
// gpu_info_file_ = nullptr;
// }
// }
//
// void
// set_gpu_info_file(GPUInfoFile *gpu_info_file) {
// gpu_info_file_ = gpu_info_file;
// }
//
// GPUInfoFile *get_gpu_info_file() {
// return gpu_info_file_;
// }
// private:
// friend class boost::serialization::access;
//
// template<typename Archive>
// void serialize(Archive &ar, const unsigned int version) {
// ar & gpu_info_file_;
// }
//
// private:
// GPUInfoFile *gpu_info_file_ = nullptr;
//};

83
cpp/src/license/GetSysInfo.cpp
View File

@ -1,83 +0,0 @@
//
//#include "utils/Log.h"
//#include "LicenseLibrary.h"
//#include "utils/Error.h"
//
//#include <iostream>
//#include <getopt.h>
//#include <memory.h>
//// Not provide path: current work path will be used and system.info.
//using namespace zilliz::milvus;
//
//void
//print_usage(const std::string &app_name) {
// printf("\n Usage: %s [OPTIONS]\n\n", app_name.c_str());
// printf(" Options:\n");
// printf(" -h --help Print this help\n");
// printf(" -s --sysinfo filename Generate system info file as given name\n");
// printf("\n");
//}
//
//int main(int argc, char *argv[]) {
// std::string app_name = argv[0];
// if (argc != 1 && argc != 3) {
// print_usage(app_name);
// return EXIT_FAILURE;
// }
//
// static struct option long_options[] = {{"system_info", required_argument, 0, 's'},
// {"help", no_argument, 0, 'h'},
// {NULL, 0, 0, 0}};
// int value = 0;
// int option_index = 0;
// std::string system_info_filename = "./system.info";
// while ((value = getopt_long(argc, argv, "s:h", long_options, &option_index)) != -1) {
// switch (value) {
// case 's': {
// char *system_info_filename_ptr = strdup(optarg);
// system_info_filename = system_info_filename_ptr;
// free(system_info_filename_ptr);
//// printf("Generate system info file: %s\n", system_info_filename.c_str());
// break;
// }
// case 'h':print_usage(app_name);
// return EXIT_SUCCESS;
// case '?':print_usage(app_name);
// return EXIT_FAILURE;
// default:print_usage(app_name);
// break;
// }
// }
//
// int device_count = 0;
// server::ServerError err = server::LicenseLibrary::GetDeviceCount(device_count);
// if (err != server::SERVER_SUCCESS) return -1;
//
// // 1. Get All GPU UUID
// std::vector<std::string> uuid_array;
// err = server::LicenseLibrary::GetUUID(device_count, uuid_array);
// if (err != server::SERVER_SUCCESS) return -1;
//
// // 2. Get UUID SHA256
// std::vector<std::string> uuid_sha256_array;
// err = server::LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, uuid_sha256_array);
// if (err != server::SERVER_SUCCESS) return -1;
//
// // 3. Generate GPU ID map with GPU UUID
// std::map<int, std::string> uuid_encrption_map;
// for (int i = 0; i < device_count; ++i) {
// uuid_encrption_map[i] = uuid_sha256_array[i];
// }
//
//
// // 4. Generate GPU_info File
// err = server::LicenseLibrary::GPUinfoFileSerialization(system_info_filename,
// device_count,
// uuid_encrption_map);
// if (err != server::SERVER_SUCCESS) return -1;
//
// printf("Generate GPU_info File Success\n");
//
//
// return 0;
//}

131
cpp/src/license/LicenseCheck.cpp
View File

@ -1,131 +0,0 @@
//#include "LicenseCheck.h"
//#include <iostream>
//#include <thread>
//
//#include <boost/archive/binary_oarchive.hpp>
//#include <boost/archive/binary_iarchive.hpp>
////#include <boost/foreach.hpp>
////#include <boost/serialization/vector.hpp>
//#include <boost/filesystem/path.hpp>
//#include <boost/serialization/map.hpp>
//#include <boost/filesystem/operations.hpp>
//#include <boost/thread.hpp>
//#include <boost/date_time/posix_time/posix_time.hpp>
//
//
//namespace zilliz {
//namespace milvus {
//namespace server {
//
//LicenseCheck::LicenseCheck() {
//
//}
//
//LicenseCheck::~LicenseCheck() {
// StopCountingDown();
//}
//
//ServerError
//LicenseCheck::LegalityCheck(const std::string &license_file_path) {
//
// int device_count;
// LicenseLibrary::GetDeviceCount(device_count);
// std::vector<std::string> uuid_array;
// LicenseLibrary::GetUUID(device_count, uuid_array);
//
// std::vector<std::string> sha_array;
// LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, sha_array);
//
// int output_device_count;
// std::map<int, std::string> uuid_encryption_map;
// time_t starting_time;
// time_t end_time;
// ServerError err = LicenseLibrary::LicenseFileDeserialization(license_file_path,
// output_device_count,
// uuid_encryption_map,
// starting_time,
// end_time);
// if(err !=SERVER_SUCCESS)
// {
// std::cout << "License check error: 01" << std::endl;
// return SERVER_UNEXPECTED_ERROR;
// }
// time_t system_time;
// LicenseLibrary::GetSystemTime(system_time);
//
// if (device_count != output_device_count) {
// std::cout << "License check error: 02" << std::endl;
// return SERVER_UNEXPECTED_ERROR;
// }
// for (int i = 0; i < device_count; ++i) {
// if (sha_array[i] != uuid_encryption_map[i]) {
// std::cout << "License check error: 03" << std::endl;
// return SERVER_UNEXPECTED_ERROR;
// }
// }
// if (system_time < starting_time || system_time > end_time) {
// std::cout << "License check error: 04" << std::endl;
// return SERVER_UNEXPECTED_ERROR;
// }
// std::cout << "Legality Check Success" << std::endl;
// return SERVER_SUCCESS;
//}
//
//// Part 2: Timing check license
//
//ServerError
//LicenseCheck::AlterFile(const std::string &license_file_path,
// const boost::system::error_code &ec,
// boost::asio::deadline_timer *pt) {
//
// ServerError err = LicenseCheck::LegalityCheck(license_file_path);
// if(err!=SERVER_SUCCESS) {
// std::cout << "license file check error" << std::endl;
// exit(1);
// }
//
// std::cout << "---runing---" << std::endl;
// pt->expires_at(pt->expires_at() + boost::posix_time::hours(1));
// pt->async_wait(boost::bind(LicenseCheck::AlterFile, license_file_path, boost::asio::placeholders::error, pt));
//
// return SERVER_SUCCESS;
//
//}
//
//ServerError
//LicenseCheck::StartCountingDown(const std::string &license_file_path) {
//
// if (!LicenseLibrary::IsFileExistent(license_file_path)) {
// std::cout << "license file not exist" << std::endl;
// exit(1);
// }
//
// //create a thread to run AlterFile
// if(counting_thread_ == nullptr) {
// counting_thread_ = std::make_shared<std::thread>([&]() {
// boost::asio::deadline_timer t(io_service_, boost::posix_time::hours(1));
// t.async_wait(boost::bind(LicenseCheck::AlterFile, license_file_path, boost::asio::placeholders::error, &t));
// io_service_.run();//this thread will block here
// });
// }
//
// return SERVER_SUCCESS;
//}
//
//ServerError
//LicenseCheck::StopCountingDown() {
// if(!io_service_.stopped()) {
// io_service_.stop();
// }
//
// if(counting_thread_ != nullptr) {
// counting_thread_->join();
// counting_thread_ = nullptr;
// }
//
// return SERVER_SUCCESS;
//}
//
//}
//}
//}

52
cpp/src/license/LicenseCheck.h
View File

@ -1,52 +0,0 @@
//#pragma once
//
//#include "utils/Error.h"
//#include "LicenseLibrary.h"
//
//#include <boost/asio.hpp>
//
//#include <thread>
//#include <memory>
//
//namespace zilliz {
//namespace milvus {
//namespace server {
//
//class LicenseCheck {
//private:
// LicenseCheck();
// ~LicenseCheck();
//
//public:
// static LicenseCheck &
// GetInstance() {
// static LicenseCheck instance;
// return instance;
// };
//
// static ServerError
// LegalityCheck(const std::string &license_file_path);
//
// ServerError
// StartCountingDown(const std::string &license_file_path);
//
// ServerError
// StopCountingDown();
//
//private:
// static ServerError
// AlterFile(const std::string &license_file_path,
// const boost::system::error_code &ec,
// boost::asio::deadline_timer *pt);
//
//private:
// boost::asio::io_service io_service_;
// std::shared_ptr<std::thread> counting_thread_;
//
//};
//
//}
//}
//}
//
//

86
cpp/src/license/LicenseFile.h
View File

@ -1,86 +0,0 @@
///*******************************************************************************
// * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// * Unauthorized copying of this file, via any medium is strictly prohibited.
// * Proprietary and confidential.
// ******************************************************************************/
//#pragma once
//
//
//#include <boost/serialization/access.hpp>
//#include <string>
//#include <map>
//
//
//class LicenseFile {
// public:
// LicenseFile() = default;
//
// LicenseFile(const int &device_count,
// const std::map<int, std::string> &uuid_encryption_map,
// const time_t &starting_time,
// const time_t &end_time)
// : device_count_(device_count),
// uuid_encryption_map_(uuid_encryption_map),
// starting_time_(starting_time),
// end_time_(end_time) {}
//
// int get_device_count() {
// return device_count_;
// }
// std::map<int, std::string> &get_uuid_encryption_map() {
// return uuid_encryption_map_;
// }
// time_t get_starting_time() {
// return starting_time_;
// }
// time_t get_end_time() {
// return end_time_;
// }
//
// public:
// friend class boost::serialization::access;
//
// template<typename Archive>
// void serialize(Archive &ar, const unsigned int version) {
// ar & device_count_;
// ar & uuid_encryption_map_;
// ar & starting_time_;
// ar & end_time_;
// }
//
// public:
// int device_count_ = 0;
// std::map<int, std::string> uuid_encryption_map_;
// time_t starting_time_ = 0;
// time_t end_time_ = 0;
//};
//
//class SerializedLicenseFile {
// public:
// ~SerializedLicenseFile() {
// if (license_file_ != nullptr) {
// delete (license_file_);
// license_file_ = nullptr;
// }
// }
//
// void
// set_license_file(LicenseFile *license_file) {
// license_file_ = license_file;
// }
//
// LicenseFile *get_license_file() {
// return license_file_;
// }
// private:
// friend class boost::serialization::access;
//
// template<typename Archive>
// void serialize(Archive &ar, const unsigned int version) {
// ar & license_file_;
// }
//
// private:
// LicenseFile *license_file_ = nullptr;
//};
//

121
cpp/src/license/LicenseGenerator.cpp
View File

@ -1,121 +0,0 @@
//
//#include <iostream>
//#include <getopt.h>
//#include <memory.h>
//
//#include "utils/Log.h"
//#include "license/LicenseLibrary.h"
//#include "utils/Error.h"
//
//
//using namespace zilliz::milvus;
//// Not provide path: current work path will be used and system.info.
//
//void
//print_usage(const std::string &app_name) {
// printf("\n Usage: %s [OPTIONS]\n\n", app_name.c_str());
// printf(" Options:\n");
// printf(" -h --help Print this help\n");
// printf(" -s --sysinfo filename sysinfo file location\n");
// printf(" -l --license filename Generate license file as given name\n");
// printf(" -b --starting time Set start time (format: YYYY-MM-DD)\n");
// printf(" -e --end time Set end time (format: YYYY-MM-DD)\n");
// printf("\n");
//}
//
//int main(int argc, char *argv[]) {
// std::string app_name = argv[0];
//// if (argc != 1 && argc != 3) {
//// print_usage(app_name);
//// return EXIT_FAILURE;
//// }
// static struct option long_options[] = {{"system_info", required_argument, 0, 's'},
// {"license", optional_argument, 0, 'l'},
// {"help", no_argument, 0, 'h'},
// {"starting_time", required_argument, 0, 'b'},
// {"end_time", required_argument, 0, 'e'},
// {NULL, 0, 0, 0}};
// server::ServerError err;
// int value = 0;
// int option_index = 0;
// std::string system_info_filename = "./system.info";
// std::string license_filename = "./system.license";
// char *string_starting_time = NULL;
// char *string_end_time = NULL;
// time_t starting_time = 0;
// time_t end_time = 0;
// int flag_s = 1;
// int flag_b = 1;
// int flag_e = 1;
// while ((value = getopt_long(argc, argv, "hl:s:b:e:", long_options, NULL)) != -1) {
// switch (value) {
// case 's': {
// flag_s = 0;
// system_info_filename = (std::string) (optarg);
// break;
// }
// case 'b': {
// flag_b = 0;
// string_starting_time = optarg;
// break;
// }
// case 'e': {
// flag_e = 0;
// string_end_time = optarg;
// break;
// }
// case 'l': {
// license_filename = (std::string) (optarg);
// break;
// }
// case 'h':print_usage(app_name);
// return EXIT_SUCCESS;
// case '?':print_usage(app_name);
// return EXIT_FAILURE;
// default:print_usage(app_name);
// break;
// }
//
// }
// if (flag_s) {
// printf("Error: sysinfo file location must be entered\n");
// return 1;
// }
// if (flag_b) {
// printf("Error: start time must be entered\n");
// return 1;
// }
// if (flag_e) {
// printf("Error: end time must be entered\n");
// return 1;
// }
//
// err = server::LicenseLibrary::GetDateTime(string_starting_time, starting_time);
// if (err != server::SERVER_SUCCESS) return -1;
//
// err = server::LicenseLibrary::GetDateTime(string_end_time, end_time);
// if (err != server::SERVER_SUCCESS) return -1;
//
//
// int output_info_device_count = 0;
// std::map<int, std::string> output_info_uuid_encrption_map;
//
//
// err = server::LicenseLibrary::GPUinfoFileDeserialization(system_info_filename,
// output_info_device_count,
// output_info_uuid_encrption_map);
// if (err != server::SERVER_SUCCESS) return -1;
//
//
// err = server::LicenseLibrary::LicenseFileSerialization(license_filename,
// output_info_device_count,
// output_info_uuid_encrption_map,
// starting_time,
// end_time);
// if (err != server::SERVER_SUCCESS) return -1;
//
//
// printf("Generate License File Success\n");
//
// return 0;
//}

345
cpp/src/license/LicenseLibrary.cpp
View File

@ -1,345 +0,0 @@
//////////////////////////////////////////////////////////////////////////////////
//// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
//// Unauthorized copying of this file, via any medium is strictly prohibited.
//// Proprietary and confidential.
//////////////////////////////////////////////////////////////////////////////////
//
//#include "LicenseLibrary.h"
//#include "utils/Log.h"
//#include <cuda_runtime.h>
//#include <nvml.h>
//#include <openssl/md5.h>
//#include <openssl/sha.h>
//
//#include <boost/archive/binary_oarchive.hpp>
//#include <boost/archive/binary_iarchive.hpp>
////#include <boost/foreach.hpp>
////#include <boost/serialization/vector.hpp>
//#include <boost/filesystem/path.hpp>
//#include <boost/serialization/map.hpp>
//#include <boost/filesystem/operations.hpp>
//
//
//namespace zilliz {
//namespace milvus {
//namespace server {
//
//constexpr int LicenseLibrary::sha256_length_;
//
//// Part 0: File check
//bool
//LicenseLibrary::IsFileExistent(const std::string &path) {
//
// boost::system::error_code error;
// auto file_status = boost::filesystem::status(path, error);
// if (error) {
// return false;
// }
//
// if (!boost::filesystem::exists(file_status)) {
// return false;
// }
//
// return !boost::filesystem::is_directory(file_status);
//}
//
//// Part 1: Get GPU Info
//ServerError
//LicenseLibrary::GetDeviceCount(int &device_count) {
// nvmlReturn_t result = nvmlInit();
// if (NVML_SUCCESS != result) {
// printf("Failed to initialize NVML: %s\n", nvmlErrorString(result));
// return SERVER_UNEXPECTED_ERROR;
// }
// cudaError_t error_id = cudaGetDeviceCount(&device_count);
// if (error_id != cudaSuccess) {
// printf("cudaGetDeviceCount returned %d\n-> %s\n", (int) error_id, cudaGetErrorString(error_id));
// printf("Result = FAIL\n");
// return SERVER_UNEXPECTED_ERROR;
// }
// return SERVER_SUCCESS;
//}
//
//ServerError
//LicenseLibrary::GetUUID(int device_count, std::vector<std::string> &uuid_array) {
// if (device_count == 0) {
// printf("There are no available device(s) that support CUDA\n");
// return SERVER_UNEXPECTED_ERROR;
// }
//
// for (int dev = 0; dev < device_count; ++dev) {
// nvmlDevice_t device;
// nvmlReturn_t result = nvmlDeviceGetHandleByIndex(dev, &device);
// if (NVML_SUCCESS != result) {
// printf("Failed to get handle for device %i: %s\n", dev, nvmlErrorString(result));
// return SERVER_UNEXPECTED_ERROR;
// }
//
// char uuid[80];
// unsigned int length = 80;
// nvmlReturn_t err = nvmlDeviceGetUUID(device, uuid, length);
// if (err != NVML_SUCCESS) {
// printf("nvmlDeviceGetUUID error: %d\n", err);
// return SERVER_UNEXPECTED_ERROR;
// }
//
// uuid_array.emplace_back(uuid);
// }
// return SERVER_SUCCESS;
//}
//
//ServerError
//LicenseLibrary::GetUUIDMD5(int device_count,
// std::vector<std::string> &uuid_array,
// std::vector<std::string> &md5_array) {
// MD5_CTX ctx;
// unsigned char outmd[16];
// char temp[2];
// std::string md5;
// for (int dev = 0; dev < device_count; ++dev) {
// md5.clear();
// memset(outmd, 0, sizeof(outmd));
// MD5_Init(&ctx);
// MD5_Update(&ctx, uuid_array[dev].c_str(), uuid_array[dev].size());
// MD5_Final(outmd, &ctx);
// for (int i = 0; i < 16; ++i) {
// std::snprintf(temp, 2, "%02X", outmd[i]);
// md5 += temp;
// }
// md5_array.push_back(md5);
// }
// return SERVER_SUCCESS;
//}
//
//ServerError
//LicenseLibrary::GetUUIDSHA256(const int &device_count,
// std::vector<std::string> &uuid_array,
// std::vector<std::string> &sha_array) {
// SHA256_CTX ctx;
// unsigned char outmd[sha256_length_];
// char temp[2];
// std::string sha;
// for (int dev = 0; dev < device_count; ++dev) {
// sha.clear();
// memset(outmd, 0, sizeof(outmd));
// SHA256_Init(&ctx);
// SHA256_Update(&ctx, uuid_array[dev].c_str(), uuid_array[dev].size());
// SHA256_Final(outmd, &ctx);
// for (int i = 0; i < sha256_length_; ++i) {
// std::snprintf(temp, 2, "%02X", outmd[i]);
// sha += temp;
// }
// sha_array.push_back(sha);
// }
// return SERVER_SUCCESS;
//}
//
//ServerError
//LicenseLibrary::GetSystemTime(time_t &system_time) {
// system_time = time(NULL);
// return SERVER_SUCCESS;
//}
//
//// Part 2: Handle License File
//ServerError
//LicenseLibrary::LicenseFileSerialization(const std::string &path,
// int device_count,
// const std::map<int, std::string> &uuid_encrption_map,
// time_t starting_time,
// time_t end_time) {
//
// std::ofstream file(path);
// boost::archive::binary_oarchive oa(file);
// oa.register_type<LicenseFile>();
//
// SerializedLicenseFile serialized_license_file;
//
// serialized_license_file.set_license_file(new LicenseFile(device_count,
// uuid_encrption_map,
// starting_time,
// end_time));
// oa << serialized_license_file;
//
// file.close();
// return SERVER_SUCCESS;
//}
//
//ServerError
//LicenseLibrary::LicenseFileDeserialization(const std::string &path,
// int &device_count,
// std::map<int, std::string> &uuid_encrption_map,
// time_t &starting_time,
// time_t &end_time) {
// if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST;
// std::ifstream file(path);
// boost::archive::binary_iarchive ia(file);
// ia.register_type<LicenseFile>();
//
// SerializedLicenseFile serialized_license_file;
// ia >> serialized_license_file;
//
// device_count = serialized_license_file.get_license_file()->get_device_count();
// uuid_encrption_map = serialized_license_file.get_license_file()->get_uuid_encryption_map();
// starting_time = serialized_license_file.get_license_file()->get_starting_time();
// end_time = serialized_license_file.get_license_file()->get_end_time();
//
// file.close();
// return SERVER_SUCCESS;
//}
//
////ServerError
////LicenseLibrary::SecretFileSerialization(const std::string &path,
//// const time_t &update_time,
//// const off_t &file_size,
//// const time_t &starting_time,
//// const time_t &end_time,
//// const std::string &file_md5) {
//// std::ofstream file(path);
//// boost::archive::binary_oarchive oa(file);
//// oa.register_type<SecretFile>();
////
//// SerializedSecretFile serialized_secret_file;
////
//// serialized_secret_file.set_secret_file(new SecretFile(update_time, file_size, starting_time, end_time, file_md5));
//// oa << serialized_secret_file;
////
//// file.close();
//// return SERVER_SUCCESS;
////}
////
////ServerError
////LicenseLibrary::SecretFileDeserialization(const std::string &path,
//// time_t &update_time,
//// off_t &file_size,
//// time_t &starting_time,
//// time_t &end_time,
//// std::string &file_md5) {
//// if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST;
////
//// std::ifstream file(path);
//// boost::archive::binary_iarchive ia(file);
//// ia.register_type<SecretFile>();
//// SerializedSecretFile serialized_secret_file;
////
//// ia >> serialized_secret_file;
//// update_time = serialized_secret_file.get_secret_file()->get_update_time();
//// file_size = serialized_secret_file.get_secret_file()->get_file_size();
//// starting_time = serialized_secret_file.get_secret_file()->get_starting_time();
//// end_time = serialized_secret_file.get_secret_file()->get_end_time();
//// file_md5 = serialized_secret_file.get_secret_file()->get_file_md5();
//// file.close();
//// return SERVER_SUCCESS;
////}
//
//
//
//// Part 3: File attribute: UpdateTime Time/ Size/ MD5
//ServerError
//LicenseLibrary::GetFileUpdateTimeAndSize(const std::string &path, time_t &update_time, off_t &file_size) {
//
// if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST;
//
// struct stat buf;
// int err_no = stat(path.c_str(), &buf);
// if (err_no != 0) {
// std::cout << strerror(err_no) << std::endl;
// return SERVER_UNEXPECTED_ERROR;
// }
//
// update_time = buf.st_mtime;
// file_size = buf.st_size;
//
// return SERVER_SUCCESS;
//}
//
//ServerError
//LicenseLibrary::GetFileMD5(const std::string &path, std::string &filemd5) {
//
// if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST;
//
// filemd5.clear();
//
// std::ifstream file(path.c_str(), std::ifstream::binary);
// if (!file) {
// return -1;
// }
//
// MD5_CTX md5Context;
// MD5_Init(&md5Context);
//
// char buf[1024 * 16];
// while (file.good()) {
// file.read(buf, sizeof(buf));
// MD5_Update(&md5Context, buf, file.gcount());
// }
//
// unsigned char result[MD5_DIGEST_LENGTH];
// MD5_Final(result, &md5Context);
//
// char hex[35];
// memset(hex, 0, sizeof(hex));
// for (int i = 0; i < MD5_DIGEST_LENGTH; ++i) {
// sprintf(hex + i * 2, "%02X", result[i]);
// }
// hex[32] = '\0';
// filemd5 = std::string(hex);
//
// return SERVER_SUCCESS;
//}
//// Part 4: GPU Info File Serialization/Deserialization
//ServerError
//LicenseLibrary::GPUinfoFileSerialization(const std::string &path,
// int device_count,
// const std::map<int, std::string> &uuid_encrption_map) {
// std::ofstream file(path);
// boost::archive::binary_oarchive oa(file);
// oa.register_type<GPUInfoFile>();
//
// SerializedGPUInfoFile serialized_gpu_info_file;
//
// serialized_gpu_info_file.set_gpu_info_file(new GPUInfoFile(device_count, uuid_encrption_map));
// oa << serialized_gpu_info_file;
//
// file.close();
// return SERVER_SUCCESS;
//}
//ServerError
//LicenseLibrary::GPUinfoFileDeserialization(const std::string &path,
// int &device_count,
// std::map<int, std::string> &uuid_encrption_map) {
// if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST;
//
// std::ifstream file(path);
// boost::archive::binary_iarchive ia(file);
// ia.register_type<GPUInfoFile>();
//
// SerializedGPUInfoFile serialized_gpu_info_file;
// ia >> serialized_gpu_info_file;
//
// device_count = serialized_gpu_info_file.get_gpu_info_file()->get_device_count();
// uuid_encrption_map = serialized_gpu_info_file.get_gpu_info_file()->get_uuid_encryption_map();
//
// file.close();
// return SERVER_SUCCESS;
//}
//
//ServerError
//LicenseLibrary::GetDateTime(const char *cha, time_t &data_time) {
// tm tm_;
// int year, month, day;
// sscanf(cha, "%d-%d-%d", &year, &month, &day);
// tm_.tm_year = year - 1900;
// tm_.tm_mon = month - 1;
// tm_.tm_mday = day;
// tm_.tm_hour = 0;
// tm_.tm_min = 0;
// tm_.tm_sec = 0;
// tm_.tm_isdst = 0;
// data_time = mktime(&tm_);
// return SERVER_SUCCESS;
//
//}
//
//}
//}
//}

105
cpp/src/license/LicenseLibrary.h
View File

@ -1,105 +0,0 @@
//#pragma once
//
//#include "LicenseFile.h"
//#include "GPUInfoFile.h"
//
//#include "utils/Error.h"
//
//#include <boost/asio.hpp>
//#include <boost/thread.hpp>
//#include <boost/date_time/posix_time/posix_time.hpp>
//
//#include <vector>
//#include <map>
//#include <time.h>
//
//
//namespace zilliz {
//namespace milvus {
//namespace server {
//
//class LicenseLibrary {
// public:
// // Part 0: File check
// static bool
// IsFileExistent(const std::string &path);
//
// // Part 1: Get GPU Info
// static ServerError
// GetDeviceCount(int &device_count);
//
// static ServerError
// GetUUID(int device_count, std::vector<std::string> &uuid_array);
//
// static ServerError
// GetUUIDMD5(int device_count, std::vector<std::string> &uuid_array, std::vector<std::string> &md5_array);
//
//
// static ServerError
// GetUUIDSHA256(const int &device_count,
// std::vector<std::string> &uuid_array,
// std::vector<std::string> &sha_array);
//
// static ServerError
// GetSystemTime(time_t &system_time);
//
// // Part 2: Handle License File
// static ServerError
// LicenseFileSerialization(const std::string &path,
// int device_count,
// const std::map<int, std::string> &uuid_encrption_map,
// time_t starting_time,
// time_t end_time);
//
// static ServerError
// LicenseFileDeserialization(const std::string &path,
// int &device_count,
// std::map<int, std::string> &uuid_encrption_map,
// time_t &starting_time,
// time_t &end_time);
//
//// static ServerError
//// SecretFileSerialization(const std::string &path,
//// const time_t &update_time,
//// const off_t &file_size,
//// const time_t &starting_time,
//// const time_t &end_time,
//// const std::string &file_md5);
////
//// static ServerError
//// SecretFileDeserialization(const std::string &path,
//// time_t &update_time,
//// off_t &file_size,
//// time_t &starting_time,
//// time_t &end_time,
//// std::string &file_md5);
//
// // Part 3: File attribute: UpdateTime Time/ Size/ MD5
// static ServerError
// GetFileUpdateTimeAndSize(const std::string &path, time_t &update_time, off_t &file_size);
//
// static ServerError
// GetFileMD5(const std::string &path, std::string &filemd5);
//
// // Part 4: GPU Info File Serialization/Deserialization
// static ServerError
// GPUinfoFileSerialization(const std::string &path,
// int device_count,
// const std::map<int, std::string> &uuid_encrption_map);
// static ServerError
// GPUinfoFileDeserialization(const std::string &path,
// int &device_count,
// std::map<int, std::string> &uuid_encrption_map);
//
// static ServerError
// GetDateTime(const char *cha, time_t &data_time);
//
//
// private:
// static constexpr int sha256_length_ = 32;
//};
//
//
//}
//}
//}

17
cpp/src/server/Server.cpp
View File

@ -9,7 +9,6 @@
#include "utils/Log.h"
#include "utils/SignalUtil.h"
#include "utils/TimeRecorder.h"
#include "license/LicenseCheck.h"
#include "metrics/Metrics.h"
#include <fcntl.h>
@ -152,19 +151,6 @@ Server::Start() {
ServerConfig &config = ServerConfig::GetInstance();
ConfigNode server_config = config.GetConfig(CONFIG_SERVER);
#ifdef ENABLE_LICENSE
ConfigNode license_config = config.GetConfig(CONFIG_LICENSE);
std::string license_file_path = license_config.GetValue(CONFIG_LICENSE_PATH);
SERVER_LOG_INFO << "License path: " << license_file_path;
if(server::LicenseCheck::LegalityCheck(license_file_path) != SERVER_SUCCESS) {
SERVER_LOG_ERROR << "License check failed";
exit(1);
}
server::LicenseCheck::GetInstance().StartCountingDown(license_file_path);
#endif
// Handle Signal
signal(SIGINT, SignalUtil::HandleSignal);
signal(SIGHUP, SignalUtil::HandleSignal);
@ -216,9 +202,6 @@ Server::Stop() {
StopService();
#ifdef ENABLE_LICENSE
server::LicenseCheck::GetInstance().StopCountingDown();
#endif
std::cout << "Milvus server is closed!" << std::endl;
}

2
cpp/unittest/CMakeLists.txt
View File

@ -41,8 +41,6 @@ set(unittest_libs
add_subdirectory(server)
add_subdirectory(db)
add_subdirectory(index_wrapper)
#add_subdirectory(faiss_wrapper)
#add_subdirectory(license)
add_subdirectory(metrics)
add_subdirectory(storage)
add_subdirectory(utils)

28
cpp/unittest/db/misc_test.cpp
View File

@ -8,7 +8,6 @@
#include <easylogging++.h>
#include <boost/filesystem.hpp>
#include "db/FaissExecutionEngine.h"
#include "db/Exception.h"
#include "db/Status.h"
#include "db/Options.h"
@ -27,33 +26,6 @@ namespace {
}
//TEST(DBMiscTest, ENGINE_API_TEST) {
// //engine api AddWithIdArray
// const uint16_t dim = 512;
// const long n = 10;
// engine::FaissExecutionEngine engine(512, "/tmp/1", "IDMap", "IDMap,Flat");
// std::vector<float> vectors;
// std::vector<long> ids;
// for (long i = 0; i < n; i++) {
// for (uint16_t k = 0; k < dim; k++) {
// vectors.push_back((float) k);
// }
// ids.push_back(i);
// }
//
// auto status = engine.AddWithIdArray(vectors, ids);
// ASSERT_TRUE(status.ok());
//
// auto engine_ptr = engine::EngineFactory::Build(128, "/tmp", engine::EngineType::INVALID);
// ASSERT_EQ(engine_ptr, nullptr);
//
// engine_ptr = engine::EngineFactory::Build(128, "/tmp", engine::EngineType::FAISS_IVFFLAT_GPU);
// ASSERT_NE(engine_ptr, nullptr);
//
// engine_ptr = engine::EngineFactory::Build(128, "/tmp", engine::EngineType::FAISS_IDMAP);
// ASSERT_NE(engine_ptr, nullptr);
//}
TEST(DBMiscTest, EXCEPTION_TEST) {
engine::Exception ex1("");
std::string what = ex1.what();

55
cpp/unittest/faiss_wrapper/CMakeLists.txt
View File

@ -1,55 +0,0 @@
#-------------------------------------------------------------------------------
# Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
# Unauthorized copying of this file, via any medium is strictly prohibited.
# Proprietary and confidential.
#-------------------------------------------------------------------------------
aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src)
aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
set(util_files
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp)
# Make sure that your call to link_directories takes place before your call to the relevant add_executable.
include_directories(/usr/local/cuda/include)
link_directories("/usr/local/cuda/lib64")
set(wrapper_test_src
${unittest_srcs}
${wrapper_src}
${config_files}
${util_files}
${require_files}
wrapper_test.cpp
)
add_executable(wrapper_test ${wrapper_test_src})
set(wrapper_libs
stdc++
boost_system_static
boost_filesystem_static
faiss
cudart
cublas
sqlite
snappy
bz2
z
zstd
lz4
)
if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
set(wrapper_libs ${wrapper_libs} ${MKL_LIBS} ${MKL_LIBS})
else()
set(wrapper_libs ${wrapper_libs}
lapack
openblas)
endif()
target_link_libraries(wrapper_test ${wrapper_libs} ${unittest_libs})
set(topk_test_src
topk_test.cpp
${CMAKE_SOURCE_DIR}/src/wrapper/gpu/Topk.cu)
install(TARGETS wrapper_test DESTINATION bin)

89
cpp/unittest/faiss_wrapper/topk_test.cpp
View File

@ -1,89 +0,0 @@
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include <gtest/gtest.h>
#include "wrapper/gpu/Topk.h"
using namespace zilliz::milvus::engine;
constexpr float threshhold = 0.00001;
template<typename T>
void TopK_check(T *data,
int length,
int k,
T *result) {
std::vector<T> arr(data, data + length);
sort(arr.begin(), arr.end(), std::less<T>());
for (int i = 0; i < k; ++i) {
ASSERT_TRUE(fabs(arr[i] - result[i]) < threshhold);
}
}
TEST(wrapper_topk, Wrapper_Test) {
int length = 100000;
int k = 1000;
float *host_input, *host_output;
int64_t *ids;
host_input = (float *) malloc(length * sizeof(float));
host_output = (float *) malloc(k * sizeof(float));
ids = (int64_t *) malloc(k * sizeof(int64_t));
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_real_distribution<> dis(-1.0, 1.0);
for (int i = 0; i < length; ++i) {
host_input[i] = 1.0 * dis(gen);
}
TopK(host_input, length, k, host_output, ids);
TopK_check(host_input, length, k, host_output);
}
template<typename T>
void TopK_Test(T factor) {
int length = 1000000; // data length
int k = 100;
T *data, *out;
int64_t *idx;
cudaMallocManaged((void **) &data, sizeof(T) * length);
cudaMallocManaged((void **) &out, sizeof(T) * k);
cudaMallocManaged((void **) &idx, sizeof(int64_t) * k);
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_real_distribution<> dis(-1.0, 1.0);
for (int i = 0; i < length; i++) {
data[i] = factor * dis(gen);
}
cudaMemAdvise(data, sizeof(T) * length, cudaMemAdviseSetReadMostly, 0);
cudaMemPrefetchAsync(data, sizeof(T) * length, 0);
gpu::TopK<T>(data, length, k, out, idx, nullptr);
TopK_check<T>(data, length, k, out);
// order_flag = Ordering::kDescending;
// TopK<T>(data, length, k, out, idx, nullptr);
// TopK_check<T>(data, length, k, out);
cudaFree(data);
cudaFree(out);
cudaFree(idx);
}
TEST(topk_test, Wrapper_Test) {
TopK_Test<float>(1.0);
}

141
cpp/unittest/faiss_wrapper/wrapper_test.cpp
View File

@ -1,141 +0,0 @@
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include "wrapper/Operand.h"
#include "wrapper/Index.h"
#include "wrapper/IndexBuilder.h"
#include <gtest/gtest.h>
#include <random>
using namespace zilliz::milvus::engine;
TEST(operand_test, Wrapper_Test) {
using std::cout;
using std::endl;
auto opd = std::make_shared<Operand>();
opd->index_type = "IVF";
opd->preproc = "OPQ";
opd->postproc = "PQ";
opd->metric_type = "L2";
opd->d = 64;
auto opd_str = operand_to_str(opd);
auto new_opd = str_to_operand(opd_str);
// TODO: fix all place where using opd to build index.
assert(new_opd->get_index_type(10000) == opd->get_index_type(10000));
}
TEST(build_test, Wrapper_Test) {
// dimension of the vectors to index
int d = 3;
// make a set of nt training vectors in the unit cube
size_t nt = 10000;
// a reasonable number of cetroids to index nb vectors
int ncentroids = 16;
std::random_device rd;
std::mt19937 gen(rd());
std::vector<float> xb;
std::vector<long> ids;
//prepare train data
std::uniform_real_distribution<> dis_xt(-1.0, 1.0);
std::vector<float> xt(nt * d);
for (size_t i = 0; i < nt * d; i++) {
xt[i] = dis_xt(gen);
}
//train the index
auto opd = std::make_shared<Operand>();
opd->index_type = "IVF";
opd->d = d;
opd->ncent = ncentroids;
IndexBuilderPtr index_builder_1 = GetIndexBuilder(opd);
auto index_1 = index_builder_1->build_all(0, xb, ids, nt, xt);
ASSERT_TRUE(index_1 != nullptr);
// size of the database we plan to index
size_t nb = 100000;
//prepare raw data
xb.resize(nb);
ids.resize(nb);
for (size_t i = 0; i < nb; i++) {
xb[i] = dis_xt(gen);
ids[i] = i;
}
index_1->add_with_ids(nb, xb.data(), ids.data());
//search in first quadrant
int nq = 1, k = 10;
std::vector<float> xq = {0.5, 0.5, 0.5};
float *result_dists = new float[k];
long *result_ids = new long[k];
index_1->search(nq, xq.data(), k, result_dists, result_ids);
for (int i = 0; i < k; i++) {
if (result_ids[i] < 0) {
ASSERT_TRUE(false);
break;
}
long id = result_ids[i];
std::cout << "No." << id << " [" << xb[id * 3] << ", " << xb[id * 3 + 1] << ", "
<< xb[id * 3 + 2] << "] distance = " << result_dists[i] << std::endl;
//makesure result vector is in first quadrant
ASSERT_TRUE(xb[id * 3] > 0.0);
ASSERT_TRUE(xb[id * 3 + 1] > 0.0);
ASSERT_TRUE(xb[id * 3 + 2] > 0.0);
}
delete[] result_dists;
delete[] result_ids;
}
TEST(gpu_build_test, Wrapper_Test) {
using std::vector;
int d = 256;
int nb = 3 * 1000 * 100;
int nq = 100;
vector<float> xb(d * nb);
vector<float> xq(d * nq);
vector<long> ids(nb);
std::random_device rd;
std::mt19937 gen(rd());
std::uniform_real_distribution<> dis_xt(-1.0, 1.0);
for (auto &e : xb) { e = float(dis_xt(gen)); }
for (auto &e : xq) { e = float(dis_xt(gen)); }
for (int i = 0; i < nb; ++i) { ids[i] = i; }
auto opd = std::make_shared<Operand>();
opd->index_type = "IVF";
opd->d = d;
opd->ncent = 256;
IndexBuilderPtr index_builder_1 = GetIndexBuilder(opd);
auto index_1 = index_builder_1->build_all(nb, xb.data(), ids.data());
assert(index_1->ntotal == nb);
assert(index_1->dim == d);
// sanity check: search 5 first vectors of xb
int k = 1;
vector<long> I(5 * k);
vector<float> D(5 * k);
index_1->search(5, xb.data(), k, D.data(), I.data());
for (int i = 0; i < 5; ++i) { assert(i == I[i]); }
}

43
cpp/unittest/license/CMakeLists.txt
View File

@ -1,43 +0,0 @@
#-------------------------------------------------------------------------------
# Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
# Unauthorized copying of this file, via any medium is strictly prohibited.
# Proprietary and confidential.
#-------------------------------------------------------------------------------
aux_source_directory(${MILVUS_ENGINE_SRC}/db db_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src)
include_directories(/usr/local/cuda/include)
link_directories(/usr/local/cuda)
link_directories(/usr/local/cuda/lib64)
link_directories(/usr/lib/x86_64-linux-gnu)
link_directories(/usr/lib/nvidia-415)
link_directories(/usr/local/cuda/targets/x86_64-linux/lib/stubs/)
link_directories(/usr/local/cuda/lib64/stubs/)
set(license_src_files
${MILVUS_ENGINE_SRC}/license/LicenseLibrary.cpp
${MILVUS_ENGINE_SRC}/license/LicenseCheck.cpp
)
set(db_test_src
license_library_tests.cpp
license_check_test.cpp
${license_src_files})
cuda_add_executable(license_test ${db_test_src})
set(db_libs
nvidia-ml
cudart
cublas
boost_system_static
boost_filesystem_static
lz4
crypto
boost_serialization_static
)
target_link_libraries(license_test ${db_libs} ${unittest_libs})

184
cpp/unittest/license/license_check_test.cpp
View File

@ -1,184 +0,0 @@
////
//// Created by zilliz on 19-5-13.
////
//
//#include "utils/Log.h"
//#include "license/LicenseCheck.h"
//#include "utils/Error.h"
//
//#include <gtest/gtest.h>
//#include <iostream>
//
//
//using namespace zilliz::milvus;
//
//TEST(LicenseLibraryTest, CHECK_TEST) {
//
//
// server::ServerError err;
// // 1. Set license file name
// std::string license_file_path("/tmp/milvus/abc.license");
//
// // 2. Legality check
// err = server::LicenseCheck::LegalityCheck(license_file_path);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
//
//}
//
//TEST(LicenseLibraryTest, CHECK_ERROR1_TEST){
//
// server::ServerError err;
// // 1. Set license file name
// std::string license_file_path("/tmp/milvus/abc");
//
// // 2. Legality check
// err = server::LicenseCheck::LegalityCheck(license_file_path);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
//}
//
//TEST(LicenseLibraryTest, CHECK_ERROR2_TEST){
//
// server::ServerError err;
// // 1. Set license file name
// std::string license_file_path("/tmp/milvus/abc.license");
//
// // 2. Define output var
// int device_count;
// std::map<int, std::string> uuid_encryption_map;
// time_t starting_time;
// time_t end_time;
//
// // 3. Read License File
// err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path,
// device_count,
// uuid_encryption_map,
// starting_time,
// end_time);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
//
// // 4. Change device count
// ++device_count;
// err = server::LicenseLibrary::LicenseFileSerialization(license_file_path,
// device_count,
// uuid_encryption_map,
// starting_time,
// end_time);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
//
// // 5. Legality check
// err = server::LicenseCheck::LegalityCheck(license_file_path);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
//}
//
//TEST(LicenseLibraryTest, CHECK_ERROR3_TEST){
//
// server::ServerError err;
// // 1. Set license file name
// std::string license_file_path("/tmp/milvus/abc.license");
//
// // 2. Define output var
// int device_count;
// std::map<int, std::string> uuid_encryption_map;
// time_t starting_time;
// time_t end_time;
//
// // 3. Read License File
// err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path,
// device_count,
// uuid_encryption_map,
// starting_time,
// end_time);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
//
// // 4. Change device count
// if(device_count) uuid_encryption_map[0]+="u";
// err = server::LicenseLibrary::LicenseFileSerialization(license_file_path,
// device_count,
// uuid_encryption_map,
// starting_time,
// end_time);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
//
// // 5. Legality check
// err = server::LicenseCheck::LegalityCheck(license_file_path);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
//}
//
//TEST(LicenseLibraryTest, CHECK_ERROR4_1_TEST){
//
// server::ServerError err;
// // 1. Set license file name
// std::string license_file_path("/tmp/milvus/abc.license");
//
// // 2. Define output var
// int device_count;
// std::map<int, std::string> uuid_encryption_map;
// time_t starting_time;
// time_t end_time;
//
// // 3. Read License File
// err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path,
// device_count,
// uuid_encryption_map,
// starting_time,
// end_time);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
//
// // 4. Change starting time
// time_t system_time;
// err = server::LicenseLibrary::GetSystemTime(system_time);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
//
// system_time+=60*60*24;
//
// err = server::LicenseLibrary::LicenseFileSerialization(license_file_path,
// device_count,
// uuid_encryption_map,
// system_time,
// end_time);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
//
// // 5. Legality check
// err = server::LicenseCheck::LegalityCheck(license_file_path);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
//}
//
//TEST(LicenseLibraryTest, CHECK_ERROR4_2_TEST){
//
// server::ServerError err;
// // 1. Set license file name
// std::string license_file_path("/tmp/milvus/abc.license");
//
// // 2. Define output var
// int device_count;
// std::map<int, std::string> uuid_encryption_map;
// time_t starting_time;
// time_t end_time;
//
// // 3. Read License File
// err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path,
// device_count,
// uuid_encryption_map,
// starting_time,
// end_time);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
//
// // 4. Change end time
// time_t system_time;
// err = server::LicenseLibrary::GetSystemTime(system_time);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
//
// system_time-=100;
//
// err = server::LicenseLibrary::LicenseFileSerialization(license_file_path,
// device_count,
// uuid_encryption_map,
// starting_time,
// system_time);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
//
// // 5. Legality check
// err = server::LicenseCheck::LegalityCheck(license_file_path);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
//}
//

214
cpp/unittest/license/license_library_tests.cpp
View File

@ -1,214 +0,0 @@
///*******************************************************************************
// * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// * Unauthorized copying of this file, via any medium is strictly prohibited.
// * Proprietary and confidential.
// ******************************************************************************/
//
//
//#include "utils/Log.h"
//#include "license/LicenseLibrary.h"
//#include "utils/Error.h"
//
//#include <gtest/gtest.h>
//#include <iostream>
//
//
//using namespace zilliz::milvus;
//
//TEST(LicenseLibraryTest, FILE_EXISTENT_TEST) {
//
// std::string hosts_file = "/etc/hosts";
// ASSERT_EQ(server::LicenseLibrary::IsFileExistent(hosts_file), true);
//
// std::string no_exist_file = "/temp/asdaasd";
// ASSERT_EQ(server::LicenseLibrary::IsFileExistent(no_exist_file), false);
//
// std::string directory = "/tmp";
// ASSERT_EQ(server::LicenseLibrary::IsFileExistent(directory), false);
//}
//
//TEST(LicenseLibraryTest, GPU_INFO_TEST) {
//
// int device_count = 0;
// server::ServerError err = server::LicenseLibrary::GetDeviceCount(device_count);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
// std::cout << "Device Count: " << device_count << std::endl;
//
// std::vector<std::string> uuid_array;
// err = server::LicenseLibrary::GetUUID(device_count, uuid_array);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
// for (long i = 0; i < device_count; ++i) {
// std::cout << "Device Id: " << i << ", UUID: " << uuid_array[i] << std::endl;
// }
//
// std::vector<std::string> uuid_md5_array;
// err = server::LicenseLibrary::GetUUIDMD5(device_count, uuid_array, uuid_md5_array);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
// for (long i = 0; i < device_count; ++i) {
// std::cout << "Device Id: " << i << ", UUID: " << uuid_array[i] << ", UUID_MD5: " << uuid_md5_array[i]
// << std::endl;
// }
//
// std::vector<std::string> uuid_sha256_array;
// err = server::LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, uuid_sha256_array);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
// for (long i = 0; i < device_count; ++i) {
// std::cout << "Device Id: " << i << ", UUID: " << uuid_array[i] << ", UUID_SHA256: "
// << uuid_sha256_array[i] << std::endl;
// }
//
// time_t systemtime;
// err = server::LicenseLibrary::GetSystemTime(systemtime);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
// std::cout << "System Time: " << systemtime << std::endl;
//
//}
//
//TEST(LicenseLibraryTest, LICENSE_FILE_TEST) {
//
// // 0. File check
// std::string test("/tmp/a.test");
// bool is = server::LicenseLibrary::IsFileExistent(test);
// ASSERT_EQ(is, false);
//
// // 1. Get Device Count
// int device_count = 0;
// server::ServerError err = server::LicenseLibrary::GetDeviceCount(device_count);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
//
// // 2. Get All GPU UUID
// std::vector<std::string> uuid_array;
// err = server::LicenseLibrary::GetUUID(device_count, uuid_array);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
//
// // 3. Get UUID SHA256
// std::vector<std::string> uuid_sha256_array;
// err = server::LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, uuid_sha256_array);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
//
// // 4. Generate GPU ID map with GPU UUID
// std::map<int, std::string> uuid_encrption_map;
// for (int i = 0; i < device_count; ++i) {
// uuid_encrption_map[i] = uuid_sha256_array[i];
// }
//
// // 5.GPU_info File
// std::string GPU_info_file_path("/tmp/milvus.info");
//
//
// // 6. Generate GPU_info File
// err = server::LicenseLibrary::GPUinfoFileSerialization(GPU_info_file_path,
// device_count,
// uuid_encrption_map);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
//
// // 7. Define output var
// int output_info_device_count = 0;
// std::map<int, std::string> output_info_uuid_encrption_map;
//
// // 8. Read GPU_info File
// err = server::LicenseLibrary::GPUinfoFileDeserialization(GPU_info_file_path,
// output_info_device_count,
// output_info_uuid_encrption_map);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
//
// ASSERT_EQ(device_count, output_info_device_count);
// for (int i = 0; i < device_count; ++i) {
// ASSERT_EQ(uuid_encrption_map[i], output_info_uuid_encrption_map[i]);
// }
//
// // 9. Set license file name
// std::string license_file_path("/tmp/milvus.license");
//
// // 10. Get System Time/starting_time ans End Time
// time_t sysyem_time;
// err = server::LicenseLibrary::GetSystemTime(sysyem_time);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
//
// // 11.GetDateTime
// time_t starting_time;
// time_t end_time;
// const char *string_starting_time = "2019-05-10";
// const char *string_end_time = "2022-05-10";
// err = server::LicenseLibrary::GetDateTime(string_starting_time, starting_time);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
// err = server::LicenseLibrary::GetDateTime(string_end_time, end_time);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
//
// // 12. Generate License File
// err = server::LicenseLibrary::LicenseFileSerialization(license_file_path,
// device_count,
// uuid_encrption_map,
// starting_time,
// end_time);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
//
// // 13. Define output var
// int output_device_count = 0;
// std::map<int, std::string> output_uuid_encrption_map;
// time_t output_starting_time;
// time_t output_end_time;
//
// // 14. Read License File
// err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path,
// output_device_count,
// output_uuid_encrption_map,
// output_starting_time,
// output_end_time);
// ASSERT_EQ(err, server::SERVER_SUCCESS);
//
// ASSERT_EQ(device_count, output_device_count);
// ASSERT_EQ(starting_time, output_starting_time);
// ASSERT_EQ(end_time, output_end_time);
//
// for (int i = 0; i < device_count; ++i) {
// ASSERT_EQ(uuid_encrption_map[i], output_uuid_encrption_map[i]);
// }
//
//// // 15. Get License File Attribute
//// time_t update_time;
//// off_t file_size;
//// err = server::LicenseLibrary::GetFileUpdateTimeAndSize(license_file_path, update_time, file_size);
//// ASSERT_EQ(err, server::SERVER_SUCCESS);
////
//// // 16. Get License File MD5
//// std::string file_md5;
//// err = server::LicenseLibrary::GetFileMD5(license_file_path, file_md5);
//// ASSERT_EQ(err, server::SERVER_SUCCESS);
//
//
//
//// // 17. Generate Secret File
//// std::string secret_file_path("/tmp/milvus.secret");
//// err = server::LicenseLibrary::SecretFileSerialization(secret_file_path,
//// update_time,
//// file_size,
//// starting_time,
//// end_time,
//// file_md5);
//// ASSERT_EQ(err, server::SERVER_SUCCESS);
//
//// // 18. Define output var
//// time_t output_update_time;
//// off_t output_file_size;
//// time_t output_starting_time;
//// time_t output_end_time;
//// std::string output_file_md5;
//
//// // 19. Read License File
//// err = server::LicenseLibrary::SecretFileDeserialization(secret_file_path,
//// output_update_time,
//// output_file_size,
//// output_starting_time,
//// output_end_time,
//// output_file_md5);
//// ASSERT_EQ(err, server::SERVER_SUCCESS);
////
//// ASSERT_EQ(update_time, output_update_time);
//// ASSERT_EQ(file_size, output_file_size);
//// ASSERT_EQ(starting_time, output_starting_time);
//// ASSERT_EQ(end_time, output_end_time);
//// ASSERT_EQ(file_md5, output_file_md5);
//
//
//}