From 4d5618f4c37f50b5e20ace54ff989a4f7fff41ae Mon Sep 17 00:00:00 2001 From: starlord Date: Thu, 8 Aug 2019 14:20:00 +0800 Subject: [PATCH] MS-327 clean code Former-commit-id: 9ca0ea6cb89b3fb95e17b0c0d6f647fe0c3ff7c4 --- cpp/.gitignore | 1 + cpp/CHANGELOG.md | 1 + cpp/CMakeLists.txt | 4 - cpp/src/CMakeLists.txt | 58 +-- cpp/src/db/DBImpl.h | 1 - cpp/src/db/Factories.cpp | 11 - cpp/src/db/FaissExecutionEngine.cpp | 222 ----------- cpp/src/db/FaissExecutionEngine.h | 98 ----- cpp/src/db/MemManager.cpp | 226 ------------ cpp/src/db/MemManager.h | 109 ------ cpp/src/license/GPUInfoFile.h | 69 ---- cpp/src/license/GetSysInfo.cpp | 83 ----- cpp/src/license/LicenseCheck.cpp | 131 ------- cpp/src/license/LicenseCheck.h | 52 --- cpp/src/license/LicenseFile.h | 86 ----- cpp/src/license/LicenseGenerator.cpp | 121 ------ cpp/src/license/LicenseLibrary.cpp | 345 ------------------ cpp/src/license/LicenseLibrary.h | 105 ------ cpp/src/server/Server.cpp | 17 - cpp/unittest/CMakeLists.txt | 2 - cpp/unittest/db/misc_test.cpp | 28 -- cpp/unittest/faiss_wrapper/CMakeLists.txt | 55 --- cpp/unittest/faiss_wrapper/topk_test.cpp | 89 ----- cpp/unittest/faiss_wrapper/wrapper_test.cpp | 141 ------- cpp/unittest/license/CMakeLists.txt | 43 --- cpp/unittest/license/license_check_test.cpp | 184 ---------- .../license/license_library_tests.cpp | 214 ----------- 27 files changed, 7 insertions(+), 2489 deletions(-) delete mode 100644 cpp/src/db/FaissExecutionEngine.cpp delete mode 100644 cpp/src/db/FaissExecutionEngine.h delete mode 100644 cpp/src/db/MemManager.cpp delete mode 100644 cpp/src/db/MemManager.h delete mode 100644 cpp/src/license/GPUInfoFile.h delete mode 100644 cpp/src/license/GetSysInfo.cpp delete mode 100644 cpp/src/license/LicenseCheck.cpp delete mode 100644 cpp/src/license/LicenseCheck.h delete mode 100644 cpp/src/license/LicenseFile.h delete mode 100644 cpp/src/license/LicenseGenerator.cpp delete mode 100644 cpp/src/license/LicenseLibrary.cpp delete mode 100644 cpp/src/license/LicenseLibrary.h delete mode 100644 cpp/unittest/faiss_wrapper/CMakeLists.txt delete mode 100644 cpp/unittest/faiss_wrapper/topk_test.cpp delete mode 100644 cpp/unittest/faiss_wrapper/wrapper_test.cpp delete mode 100644 cpp/unittest/license/CMakeLists.txt delete mode 100644 cpp/unittest/license/license_check_test.cpp delete mode 100644 cpp/unittest/license/license_library_tests.cpp diff --git a/cpp/.gitignore b/cpp/.gitignore index 6c2602d341..718081be75 100644 --- a/cpp/.gitignore +++ b/cpp/.gitignore @@ -8,3 +8,4 @@ base.info output.info output_new.info server.info +thirdparty/knowhere/ diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index b8d93c8760..32ea1522f5 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -7,6 +7,7 @@ Please mark all change in change log and use the ticket from JIRA. ## Bug ## Improvement +- MS-327 - Clean code for milvus ## New Feature diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 5082774b02..dec7e39135 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -91,10 +91,6 @@ endif() if(CMAKE_BUILD_TYPE STREQUAL "Release") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp") - if (CMAKE_LICENSE_CHECK STREQUAL "ON") - set(ENABLE_LICENSE "ON") - add_definitions("-DENABLE_LICENSE") - endif () else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp") endif() diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 29bb1de3df..2e6ea84485 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -23,16 +23,6 @@ set(db_scheduler_files ${scheduler_task_files} ) -set(license_check_files - license/LicenseLibrary.cpp - license/LicenseCheck.cpp - ) - -set(license_generator_files - license/LicenseGenerator.cpp - license/LicenseLibrary.cpp - ) - set(service_files thrift/gen-cpp/MilvusService.cpp thrift/gen-cpp/milvus_constants.cpp @@ -53,9 +43,6 @@ set(engine_files ${knowhere_files} ) -set(get_sys_info_files - license/GetSysInfo.cpp) - set(s3_client_files storage/s3/S3ClientWrapper.cpp storage/s3/S3ClientWrapper.h) @@ -138,29 +125,12 @@ if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64") ) endif () -if (ENABLE_LICENSE STREQUAL "ON") - link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs") - link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") - set(license_libs - nvidia-ml - crypto - cudart - cublas - ) -endif () - - cuda_add_library(milvus_engine STATIC ${engine_files}) target_link_libraries(milvus_engine ${engine_libs} ${third_party_libs}) add_library(metrics STATIC ${metrics_files}) -if (ENABLE_LICENSE STREQUAL "ON") - add_library(license_check STATIC ${license_check_files}) - target_link_libraries(license_check ${license_libs} ${third_party_libs}) -endif () - set(metrics_lib easyloggingpp yaml-cpp @@ -197,30 +167,12 @@ add_executable(milvus_server ${metrics_files} ) -if (ENABLE_LICENSE STREQUAL "ON") - target_link_libraries(milvus_server - ${server_libs} - license_check - ${third_party_libs} - ${knowhere_libs}) -else () - target_link_libraries(milvus_server - ${server_libs} - ${third_party_libs} - ${knowhere_libs} - ) -endif() -if (ENABLE_LICENSE STREQUAL "ON") - add_executable(get_sys_info ${get_sys_info_files}) - add_executable(license_generator ${license_generator_files}) - - target_link_libraries(get_sys_info ${license_libs} license_check ${third_party_libs}) - target_link_libraries(license_generator ${license_libs} ${third_party_libs}) - - install(TARGETS get_sys_info DESTINATION bin) - install(TARGETS license_generator DESTINATION bin) -endif () +target_link_libraries(milvus_server + ${server_libs} + ${third_party_libs} + ${knowhere_libs} + ) install(TARGETS milvus_server DESTINATION bin) diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 73d28779b7..27b0b66ffa 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -6,7 +6,6 @@ #pragma once #include "DB.h" -#include "MemManager.h" #include "Types.h" #include "utils/ThreadPool.h" #include "MemManagerAbstract.h" diff --git a/cpp/src/db/Factories.cpp b/cpp/src/db/Factories.cpp index 22d4760b9b..05a1ad7338 100644 --- a/cpp/src/db/Factories.cpp +++ b/cpp/src/db/Factories.cpp @@ -6,7 +6,6 @@ #include "Factories.h" #include "DBImpl.h" -#include "MemManager.h" #include "NewMemManager.h" #include "Exception.h" @@ -103,16 +102,6 @@ DB* DBFactory::Build(const Options& options) { MemManagerAbstractPtr MemManagerFactory::Build(const std::shared_ptr& meta, const Options& options) { - if (const char* env = getenv("MILVUS_USE_OLD_MEM_MANAGER")) { - std::string env_str = env; - std::transform(env_str.begin(), env_str.end(), env_str.begin(), ::toupper); - if (env_str == "ON") { - return std::make_shared(meta, options); - } - else { - return std::make_shared(meta, options); - } - } return std::make_shared(meta, options); } diff --git a/cpp/src/db/FaissExecutionEngine.cpp b/cpp/src/db/FaissExecutionEngine.cpp deleted file mode 100644 index 25e1160ee5..0000000000 --- a/cpp/src/db/FaissExecutionEngine.cpp +++ /dev/null @@ -1,222 +0,0 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ -#if 0 -#include "FaissExecutionEngine.h" -#include "Log.h" -#include "utils/CommonUtil.h" - -#include -#include -#include -#include -#include -#include -#include -#include "faiss/IndexIVF.h" -#include "metrics/Metrics.h" - - -namespace zilliz { -namespace milvus { -namespace engine { - -namespace { -std::string GetMetricType() { - server::ServerConfig &config = server::ServerConfig::GetInstance(); - server::ConfigNode engine_config = config.GetConfig(server::CONFIG_ENGINE); - return engine_config.GetValue(server::CONFIG_METRICTYPE, "L2"); -} -} - -std::string IndexStatsHelper::ToString(const std::string &prefix) const { - return ""; -} - -void IndexStatsHelper::Reset() const { - faiss::indexIVF_stats.reset(); -} - -std::string FaissIndexIVFStatsHelper::ToString(const std::string &prefix) const { - std::stringstream ss; - ss << prefix; - ss << identifier_ << ":"; - ss << " NQ=" << faiss::indexIVF_stats.nq; - ss << " NL=" << faiss::indexIVF_stats.nlist; - ss << " ND=" << faiss::indexIVF_stats.ndis; - ss << " NH=" << faiss::indexIVF_stats.nheap_updates; - ss << " Q=" << faiss::indexIVF_stats.quantization_time; - ss << " S=" << faiss::indexIVF_stats.search_time; - return ss.str(); -} - -FaissExecutionEngine::FaissExecutionEngine(uint16_t dimension, - const std::string &location, - const std::string &build_index_type, - const std::string &raw_index_type) - : location_(location), - build_index_type_(build_index_type), - raw_index_type_(raw_index_type) { - - std::string metric_type = GetMetricType(); - faiss::MetricType faiss_metric_type = (metric_type == "L2") ? faiss::METRIC_L2 : faiss::METRIC_INNER_PRODUCT; - pIndex_.reset(faiss::index_factory(dimension, raw_index_type.c_str(), faiss_metric_type)); -} - -FaissExecutionEngine::FaissExecutionEngine(std::shared_ptr index, - const std::string &location, - const std::string &build_index_type, - const std::string &raw_index_type) - : pIndex_(index), - location_(location), - build_index_type_(build_index_type), - raw_index_type_(raw_index_type) { -} - -Status FaissExecutionEngine::AddWithIds(long n, const float *xdata, const long *xids) { - pIndex_->add_with_ids(n, xdata, xids); - return Status::OK(); -} - -size_t FaissExecutionEngine::Count() const { - return (size_t) (pIndex_->ntotal); -} - -size_t FaissExecutionEngine::Size() const { - return (size_t) (Count() * pIndex_->d) * sizeof(float); -} - -size_t FaissExecutionEngine::Dimension() const { - return pIndex_->d; -} - -size_t FaissExecutionEngine::PhysicalSize() const { - return server::CommonUtil::GetFileSize(location_); -} - -Status FaissExecutionEngine::Serialize() { - write_index(pIndex_.get(), location_.c_str()); - return Status::OK(); -} - -Status FaissExecutionEngine::Load(bool to_cache) { - auto index = zilliz::milvus::cache::CpuCacheMgr::GetInstance()->GetIndex(location_); - bool already_in_cache = (index != nullptr); - auto start_time = METRICS_NOW_TIME; - if (!index) { - index = read_index(location_); - ENGINE_LOG_DEBUG << "Disk io from: " << location_; - } - - pIndex_ = index->data(); - if (!already_in_cache && to_cache) { - Cache(); - auto end_time = METRICS_NOW_TIME; - auto total_time = METRICS_MICROSECONDS(start_time, end_time); - - server::Metrics::GetInstance().FaissDiskLoadDurationSecondsHistogramObserve(total_time); - double total_size = (pIndex_->d) * (pIndex_->ntotal) * 4; - - server::Metrics::GetInstance().FaissDiskLoadSizeBytesHistogramObserve(total_size); -// server::Metrics::GetInstance().FaissDiskLoadIOSpeedHistogramObserve(total_size/double(total_time)); - server::Metrics::GetInstance().FaissDiskLoadIOSpeedGaugeSet(total_size / double(total_time)); - } - return Status::OK(); -} - -Status FaissExecutionEngine::Merge(const std::string &location) { - if (location == location_) { - return Status::Error("Cannot Merge Self"); - } - ENGINE_LOG_DEBUG << "Merge raw file: " << location << " to: " << location_; - - auto to_merge = zilliz::milvus::cache::CpuCacheMgr::GetInstance()->GetIndex(location); - if (!to_merge) { - to_merge = read_index(location); - } - auto file_index = dynamic_cast(to_merge->data().get()); - pIndex_->add_with_ids(file_index->ntotal, dynamic_cast(file_index->index)->xb.data(), - file_index->id_map.data()); - return Status::OK(); -} - -ExecutionEnginePtr -FaissExecutionEngine::BuildIndex(const std::string &location) { - ENGINE_LOG_DEBUG << "Build index file: " << location << " from: " << location_; - - auto opd = std::make_shared(); - opd->d = pIndex_->d; - opd->index_type = build_index_type_; - opd->metric_type = GetMetricType(); - IndexBuilderPtr pBuilder = GetIndexBuilder(opd); - - auto from_index = dynamic_cast(pIndex_.get()); - - auto index = pBuilder->build_all(from_index->ntotal, - dynamic_cast(from_index->index)->xb.data(), - from_index->id_map.data()); - - ExecutionEnginePtr new_ee(new FaissExecutionEngine(index->data(), location, build_index_type_, raw_index_type_)); - return new_ee; -} - -Status FaissExecutionEngine::Search(long n, - const float *data, - long k, - float *distances, - long *labels) const { - auto start_time = METRICS_NOW_TIME; - - std::shared_ptr ivf_index = std::dynamic_pointer_cast(pIndex_); - if (ivf_index) { - std::string stats_prefix = "K=" + std::to_string(k) + ":"; - ENGINE_LOG_DEBUG << "Searching index type: " << build_index_type_ << " nProbe: " << nprobe_; - ivf_index->nprobe = nprobe_; - ivf_stats_helper_.Reset(); - ivf_index->search(n, data, k, distances, labels); - ENGINE_LOG_INFO << ivf_stats_helper_.ToString(stats_prefix); - } else { - ENGINE_LOG_DEBUG << "Searching raw file"; - pIndex_->search(n, data, k, distances, labels); - } - - auto end_time = METRICS_NOW_TIME; - auto total_time = METRICS_MICROSECONDS(start_time, end_time); - server::Metrics::GetInstance().QueryIndexTypePerSecondSet(build_index_type_, double(n) / double(total_time)); - return Status::OK(); -} - -Status FaissExecutionEngine::Cache() { - auto index = std::make_shared(pIndex_); - cache::DataObjPtr data_obj = std::make_shared(index, PhysicalSize()); - zilliz::milvus::cache::CpuCacheMgr::GetInstance()->InsertItem(location_, data_obj); - - return Status::OK(); -} - -Status FaissExecutionEngine::Init() { - - if (build_index_type_ == BUILD_INDEX_TYPE_IVF || - build_index_type_ == BUILD_INDEX_TYPE_IVFSQ8) { - - using namespace zilliz::milvus::server; - ServerConfig &config = ServerConfig::GetInstance(); - ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE); - nprobe_ = engine_config.GetInt32Value(CONFIG_NPROBE, 1000); - nlist_ = engine_config.GetInt32Value(CONFIG_NLIST, 16384); - - } else if (build_index_type_ == BUILD_INDEX_TYPE_IDMAP) { ; - } else { - return Status::Error("Wrong index type: ", build_index_type_); - } - - return Status::OK(); -} - - -} // namespace engine -} // namespace milvus -} // namespace zilliz -#endif diff --git a/cpp/src/db/FaissExecutionEngine.h b/cpp/src/db/FaissExecutionEngine.h deleted file mode 100644 index d566c1c9eb..0000000000 --- a/cpp/src/db/FaissExecutionEngine.h +++ /dev/null @@ -1,98 +0,0 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ -#pragma once - -#if 0 -#include "ExecutionEngine.h" -#include "faiss/Index.h" - -#include -#include - - -namespace zilliz { -namespace milvus { -namespace engine { - -const static std::string BUILD_INDEX_TYPE_IDMAP = "IDMap"; -const static std::string BUILD_INDEX_TYPE_IVF = "IVF"; -const static std::string BUILD_INDEX_TYPE_IVFSQ8 = "IVFSQ8"; - -class IndexStatsHelper { - - public: - using Ptr = std::shared_ptr; - virtual std::string ToString(const std::string &prefix = "") const; - virtual void Reset() const; - virtual ~IndexStatsHelper() {} -}; - -class FaissIndexIVFStatsHelper : public IndexStatsHelper { - public: - std::string ToString(const std::string &prefix = "") const override; - - private: - const std::string identifier_ = BUILD_INDEX_TYPE_IVF; -}; - -class FaissExecutionEngine : public ExecutionEngine { - public: - - FaissExecutionEngine(uint16_t dimension, - const std::string &location, - const std::string &build_index_type, - const std::string &raw_index_type); - - FaissExecutionEngine(std::shared_ptr index, - const std::string &location, - const std::string &build_index_type, - const std::string &raw_index_type); - - Status AddWithIds(long n, const float *xdata, const long *xids) override; - - size_t Count() const override; - - size_t Size() const override; - - size_t Dimension() const override; - - size_t PhysicalSize() const override; - - Status Serialize() override; - - Status Load(bool to_cache) override; - - Status Merge(const std::string &location) override; - - Status Search(long n, - const float *data, - long k, - float *distances, - long *labels) const override; - - ExecutionEnginePtr BuildIndex(const std::string &) override; - - Status Cache() override; - - Status Init() override; - - protected: - FaissIndexIVFStatsHelper ivf_stats_helper_; - std::shared_ptr pIndex_; - std::string location_; - - std::string build_index_type_; - std::string raw_index_type_; - - size_t nprobe_ = 0; - size_t nlist_ = 0; -}; - - -} // namespace engine -} // namespace milvus -} // namespace zilliz -#endif diff --git a/cpp/src/db/MemManager.cpp b/cpp/src/db/MemManager.cpp deleted file mode 100644 index dc6c9681b0..0000000000 --- a/cpp/src/db/MemManager.cpp +++ /dev/null @@ -1,226 +0,0 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ -#include "MemManager.h" -#include "Meta.h" -#include "MetaConsts.h" -#include "EngineFactory.h" -#include "metrics/Metrics.h" -#include "Log.h" - -#include -#include -#include -#include - - -namespace zilliz { -namespace milvus { -namespace engine { - -MemVectors::MemVectors(const std::shared_ptr &meta_ptr, - const meta::TableFileSchema &schema, const Options &options) - : meta_(meta_ptr), - options_(options), - schema_(schema), - id_generator_(new SimpleIDGenerator()), - active_engine_(EngineFactory::Build(schema_.dimension_, schema_.location_, (EngineType) schema_.engine_type_)) { -} - - -Status MemVectors::Add(size_t n_, const float *vectors_, IDNumbers &vector_ids_) { - if (active_engine_ == nullptr) { - return Status::Error("index engine is null"); - } - - auto start_time = METRICS_NOW_TIME; - id_generator_->GetNextIDNumbers(n_, vector_ids_); - Status status = active_engine_->AddWithIds(n_, vectors_, vector_ids_.data()); - auto end_time = METRICS_NOW_TIME; - auto total_time = METRICS_MICROSECONDS(start_time, end_time); - server::Metrics::GetInstance().AddVectorsPerSecondGaugeSet(static_cast(n_), - static_cast(schema_.dimension_), - total_time); - - return status; -} - -size_t MemVectors::RowCount() const { - if (active_engine_ == nullptr) { - return 0; - } - - return active_engine_->Count(); -} - -size_t MemVectors::Size() const { - if (active_engine_ == nullptr) { - return 0; - } - - return active_engine_->Size(); -} - -Status MemVectors::Serialize(std::string &table_id) { - if (active_engine_ == nullptr) { - return Status::Error("index engine is null"); - } - - table_id = schema_.table_id_; - auto size = Size(); - auto start_time = METRICS_NOW_TIME; - active_engine_->Serialize(); - auto end_time = METRICS_NOW_TIME; - auto total_time = METRICS_MICROSECONDS(start_time, end_time); - schema_.size_ = size; - - server::Metrics::GetInstance().DiskStoreIOSpeedGaugeSet(size / total_time); - - schema_.file_type_ = (size >= options_.index_trigger_size) ? - meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; - - auto status = meta_->UpdateTableFile(schema_); - - ENGINE_LOG_DEBUG << "New " << ((schema_.file_type_ == meta::TableFileSchema::RAW) ? "raw" : "to_index") - << " file " << schema_.file_id_ << " of size " << active_engine_->Size() << " bytes"; - - if(options_.insert_cache_immediately_) { - active_engine_->Cache(); - } - - return status; -} - -MemVectors::~MemVectors() { - if (id_generator_ != nullptr) { - delete id_generator_; - id_generator_ = nullptr; - } -} - -/* - * MemManager - */ -MemManager::MemVectorsPtr MemManager::GetMemByTable( - const std::string &table_id) { - auto memIt = mem_id_map_.find(table_id); - if (memIt != mem_id_map_.end()) { - return memIt->second; - } - - meta::TableFileSchema table_file; - table_file.table_id_ = table_id; - auto status = meta_->CreateTableFile(table_file); - if (!status.ok()) { - return nullptr; - } - - mem_id_map_[table_id] = MemVectorsPtr(new MemVectors(meta_, table_file, options_)); - return mem_id_map_[table_id]; -} - -Status MemManager::InsertVectors(const std::string &table_id_, - size_t n_, - const float *vectors_, - IDNumbers &vector_ids_) { - - std::unique_lock lock(mutex_); - - return InsertVectorsNoLock(table_id_, n_, vectors_, vector_ids_); -} - -Status MemManager::InsertVectorsNoLock(const std::string &table_id, - size_t n, - const float *vectors, - IDNumbers &vector_ids) { - - MemVectorsPtr mem = GetMemByTable(table_id); - if (mem == nullptr) { - return Status::NotFound("Group " + table_id + " not found!"); - } - - //makesure each file size less than index_trigger_size - if (mem->Size() > options_.index_trigger_size) { - std::unique_lock lock(serialization_mtx_); - immu_mem_list_.push_back(mem); - mem_id_map_.erase(table_id); - return InsertVectorsNoLock(table_id, n, vectors, vector_ids); - } else { - return mem->Add(n, vectors, vector_ids); - } -} - -Status MemManager::ToImmutable() { - std::unique_lock lock(mutex_); - MemIdMap temp_map; - for (auto &kv: mem_id_map_) { - if (kv.second->RowCount() == 0) { - temp_map.insert(kv); - continue;//empty vector, no need to serialize - } - immu_mem_list_.push_back(kv.second); - } - - mem_id_map_.swap(temp_map); - return Status::OK(); -} - -Status MemManager::Serialize(std::set &table_ids) { - ToImmutable(); - std::unique_lock lock(serialization_mtx_); - std::string table_id; - table_ids.clear(); - for (auto &mem : immu_mem_list_) { - mem->Serialize(table_id); - table_ids.insert(table_id); - } - immu_mem_list_.clear(); - return Status::OK(); -} - -Status MemManager::EraseMemVector(const std::string &table_id) { - {//erase MemVector from rapid-insert cache - std::unique_lock lock(mutex_); - mem_id_map_.erase(table_id); - } - - {//erase MemVector from serialize cache - std::unique_lock lock(serialization_mtx_); - MemList temp_list; - for (auto &mem : immu_mem_list_) { - if (mem->TableId() != table_id) { - temp_list.push_back(mem); - } - } - immu_mem_list_.swap(temp_list); - } - - return Status::OK(); -} - -size_t MemManager::GetCurrentMutableMem() { - size_t totalMem = 0; - for (auto &kv : mem_id_map_) { - auto memVector = kv.second; - totalMem += memVector->Size(); - } - return totalMem; -} - -size_t MemManager::GetCurrentImmutableMem() { - size_t totalMem = 0; - for (auto &memVector : immu_mem_list_) { - totalMem += memVector->Size(); - } - return totalMem; -} - -size_t MemManager::GetCurrentMem() { - return GetCurrentMutableMem() + GetCurrentImmutableMem(); -} - -} // namespace engine -} // namespace milvus -} // namespace zilliz diff --git a/cpp/src/db/MemManager.h b/cpp/src/db/MemManager.h deleted file mode 100644 index 607333a3d8..0000000000 --- a/cpp/src/db/MemManager.h +++ /dev/null @@ -1,109 +0,0 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ -#pragma once - -#include "ExecutionEngine.h" -#include "IDGenerator.h" -#include "Status.h" -#include "Meta.h" -#include "MemManagerAbstract.h" - -#include -#include -#include -#include -#include - - - -namespace zilliz { -namespace milvus { -namespace engine { - -namespace meta { -class Meta; -} - -class MemVectors { - public: - using MetaPtr = meta::Meta::Ptr; - using Ptr = std::shared_ptr; - - explicit MemVectors(const std::shared_ptr &, - const meta::TableFileSchema &, const Options &); - - Status Add(size_t n_, const float *vectors_, IDNumbers &vector_ids_); - - size_t RowCount() const; - - size_t Size() const; - - Status Serialize(std::string &table_id); - - ~MemVectors(); - - const std::string &Location() const { return schema_.location_; } - - std::string TableId() const { return schema_.table_id_; } - - private: - MemVectors() = delete; - MemVectors(const MemVectors &) = delete; - MemVectors &operator=(const MemVectors &) = delete; - - MetaPtr meta_; - Options options_; - meta::TableFileSchema schema_; - IDGenerator *id_generator_; - ExecutionEnginePtr active_engine_; - -}; // MemVectors - - - -class MemManager : public MemManagerAbstract { - public: - using MetaPtr = meta::Meta::Ptr; - using MemVectorsPtr = typename MemVectors::Ptr; - using Ptr = std::shared_ptr; - - MemManager(const std::shared_ptr &meta, const Options &options) - : meta_(meta), options_(options) {} - - Status InsertVectors(const std::string &table_id, - size_t n, const float *vectors, IDNumbers &vector_ids) override; - - Status Serialize(std::set &table_ids) override; - - Status EraseMemVector(const std::string &table_id) override; - - size_t GetCurrentMutableMem() override; - - size_t GetCurrentImmutableMem() override; - - size_t GetCurrentMem() override; - - private: - MemVectorsPtr GetMemByTable(const std::string &table_id); - - Status InsertVectorsNoLock(const std::string &table_id, - size_t n, const float *vectors, IDNumbers &vector_ids); - Status ToImmutable(); - - using MemIdMap = std::map; - using MemList = std::vector; - MemIdMap mem_id_map_; - MemList immu_mem_list_; - MetaPtr meta_; - Options options_; - std::mutex mutex_; - std::mutex serialization_mtx_; -}; // MemManager - - -} // namespace engine -} // namespace milvus -} // namespace zilliz diff --git a/cpp/src/license/GPUInfoFile.h b/cpp/src/license/GPUInfoFile.h deleted file mode 100644 index eaad3efefe..0000000000 --- a/cpp/src/license/GPUInfoFile.h +++ /dev/null @@ -1,69 +0,0 @@ -///******************************************************************************* -// * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// * Unauthorized copying of this file, via any medium is strictly prohibited. -// * Proprietary and confidential. -// ******************************************************************************/ -//#pragma once -// -//#include -//#include -//#include -// -// -//class GPUInfoFile { -// public: -// GPUInfoFile() = default; -// -// GPUInfoFile(const int &device_count, const std::map &uuid_encryption_map) -// : device_count_(device_count), uuid_encryption_map_(uuid_encryption_map) {} -// -// int get_device_count() { -// return device_count_; -// } -// std::map &get_uuid_encryption_map() { -// return uuid_encryption_map_; -// } -// -// -// public: -// friend class boost::serialization::access; -// -// template -// void serialize(Archive &ar, const unsigned int version) { -// ar & device_count_; -// ar & uuid_encryption_map_; -// } -// -// public: -// int device_count_ = 0; -// std::map uuid_encryption_map_; -//}; -// -//class SerializedGPUInfoFile { -// public: -// ~SerializedGPUInfoFile() { -// if (gpu_info_file_ != nullptr) { -// delete (gpu_info_file_); -// gpu_info_file_ = nullptr; -// } -// } -// -// void -// set_gpu_info_file(GPUInfoFile *gpu_info_file) { -// gpu_info_file_ = gpu_info_file; -// } -// -// GPUInfoFile *get_gpu_info_file() { -// return gpu_info_file_; -// } -// private: -// friend class boost::serialization::access; -// -// template -// void serialize(Archive &ar, const unsigned int version) { -// ar & gpu_info_file_; -// } -// -// private: -// GPUInfoFile *gpu_info_file_ = nullptr; -//}; diff --git a/cpp/src/license/GetSysInfo.cpp b/cpp/src/license/GetSysInfo.cpp deleted file mode 100644 index 7a30083469..0000000000 --- a/cpp/src/license/GetSysInfo.cpp +++ /dev/null @@ -1,83 +0,0 @@ -// -//#include "utils/Log.h" -//#include "LicenseLibrary.h" -//#include "utils/Error.h" -// -//#include -//#include -//#include -//// Not provide path: current work path will be used and system.info. -//using namespace zilliz::milvus; -// -//void -//print_usage(const std::string &app_name) { -// printf("\n Usage: %s [OPTIONS]\n\n", app_name.c_str()); -// printf(" Options:\n"); -// printf(" -h --help Print this help\n"); -// printf(" -s --sysinfo filename Generate system info file as given name\n"); -// printf("\n"); -//} -// -//int main(int argc, char *argv[]) { -// std::string app_name = argv[0]; -// if (argc != 1 && argc != 3) { -// print_usage(app_name); -// return EXIT_FAILURE; -// } -// -// static struct option long_options[] = {{"system_info", required_argument, 0, 's'}, -// {"help", no_argument, 0, 'h'}, -// {NULL, 0, 0, 0}}; -// int value = 0; -// int option_index = 0; -// std::string system_info_filename = "./system.info"; -// while ((value = getopt_long(argc, argv, "s:h", long_options, &option_index)) != -1) { -// switch (value) { -// case 's': { -// char *system_info_filename_ptr = strdup(optarg); -// system_info_filename = system_info_filename_ptr; -// free(system_info_filename_ptr); -//// printf("Generate system info file: %s\n", system_info_filename.c_str()); -// break; -// } -// case 'h':print_usage(app_name); -// return EXIT_SUCCESS; -// case '?':print_usage(app_name); -// return EXIT_FAILURE; -// default:print_usage(app_name); -// break; -// } -// } -// -// int device_count = 0; -// server::ServerError err = server::LicenseLibrary::GetDeviceCount(device_count); -// if (err != server::SERVER_SUCCESS) return -1; -// -// // 1. Get All GPU UUID -// std::vector uuid_array; -// err = server::LicenseLibrary::GetUUID(device_count, uuid_array); -// if (err != server::SERVER_SUCCESS) return -1; -// -// // 2. Get UUID SHA256 -// std::vector uuid_sha256_array; -// err = server::LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, uuid_sha256_array); -// if (err != server::SERVER_SUCCESS) return -1; -// -// // 3. Generate GPU ID map with GPU UUID -// std::map uuid_encrption_map; -// for (int i = 0; i < device_count; ++i) { -// uuid_encrption_map[i] = uuid_sha256_array[i]; -// } -// -// -// // 4. Generate GPU_info File -// err = server::LicenseLibrary::GPUinfoFileSerialization(system_info_filename, -// device_count, -// uuid_encrption_map); -// if (err != server::SERVER_SUCCESS) return -1; -// -// printf("Generate GPU_info File Success\n"); -// -// -// return 0; -//} \ No newline at end of file diff --git a/cpp/src/license/LicenseCheck.cpp b/cpp/src/license/LicenseCheck.cpp deleted file mode 100644 index 8ea396ef41..0000000000 --- a/cpp/src/license/LicenseCheck.cpp +++ /dev/null @@ -1,131 +0,0 @@ -//#include "LicenseCheck.h" -//#include -//#include -// -//#include -//#include -////#include -////#include -//#include -//#include -//#include -//#include -//#include -// -// -//namespace zilliz { -//namespace milvus { -//namespace server { -// -//LicenseCheck::LicenseCheck() { -// -//} -// -//LicenseCheck::~LicenseCheck() { -// StopCountingDown(); -//} -// -//ServerError -//LicenseCheck::LegalityCheck(const std::string &license_file_path) { -// -// int device_count; -// LicenseLibrary::GetDeviceCount(device_count); -// std::vector uuid_array; -// LicenseLibrary::GetUUID(device_count, uuid_array); -// -// std::vector sha_array; -// LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, sha_array); -// -// int output_device_count; -// std::map uuid_encryption_map; -// time_t starting_time; -// time_t end_time; -// ServerError err = LicenseLibrary::LicenseFileDeserialization(license_file_path, -// output_device_count, -// uuid_encryption_map, -// starting_time, -// end_time); -// if(err !=SERVER_SUCCESS) -// { -// std::cout << "License check error: 01" << std::endl; -// return SERVER_UNEXPECTED_ERROR; -// } -// time_t system_time; -// LicenseLibrary::GetSystemTime(system_time); -// -// if (device_count != output_device_count) { -// std::cout << "License check error: 02" << std::endl; -// return SERVER_UNEXPECTED_ERROR; -// } -// for (int i = 0; i < device_count; ++i) { -// if (sha_array[i] != uuid_encryption_map[i]) { -// std::cout << "License check error: 03" << std::endl; -// return SERVER_UNEXPECTED_ERROR; -// } -// } -// if (system_time < starting_time || system_time > end_time) { -// std::cout << "License check error: 04" << std::endl; -// return SERVER_UNEXPECTED_ERROR; -// } -// std::cout << "Legality Check Success" << std::endl; -// return SERVER_SUCCESS; -//} -// -//// Part 2: Timing check license -// -//ServerError -//LicenseCheck::AlterFile(const std::string &license_file_path, -// const boost::system::error_code &ec, -// boost::asio::deadline_timer *pt) { -// -// ServerError err = LicenseCheck::LegalityCheck(license_file_path); -// if(err!=SERVER_SUCCESS) { -// std::cout << "license file check error" << std::endl; -// exit(1); -// } -// -// std::cout << "---runing---" << std::endl; -// pt->expires_at(pt->expires_at() + boost::posix_time::hours(1)); -// pt->async_wait(boost::bind(LicenseCheck::AlterFile, license_file_path, boost::asio::placeholders::error, pt)); -// -// return SERVER_SUCCESS; -// -//} -// -//ServerError -//LicenseCheck::StartCountingDown(const std::string &license_file_path) { -// -// if (!LicenseLibrary::IsFileExistent(license_file_path)) { -// std::cout << "license file not exist" << std::endl; -// exit(1); -// } -// -// //create a thread to run AlterFile -// if(counting_thread_ == nullptr) { -// counting_thread_ = std::make_shared([&]() { -// boost::asio::deadline_timer t(io_service_, boost::posix_time::hours(1)); -// t.async_wait(boost::bind(LicenseCheck::AlterFile, license_file_path, boost::asio::placeholders::error, &t)); -// io_service_.run();//this thread will block here -// }); -// } -// -// return SERVER_SUCCESS; -//} -// -//ServerError -//LicenseCheck::StopCountingDown() { -// if(!io_service_.stopped()) { -// io_service_.stop(); -// } -// -// if(counting_thread_ != nullptr) { -// counting_thread_->join(); -// counting_thread_ = nullptr; -// } -// -// return SERVER_SUCCESS; -//} -// -//} -//} -//} \ No newline at end of file diff --git a/cpp/src/license/LicenseCheck.h b/cpp/src/license/LicenseCheck.h deleted file mode 100644 index 7b20d014b5..0000000000 --- a/cpp/src/license/LicenseCheck.h +++ /dev/null @@ -1,52 +0,0 @@ -//#pragma once -// -//#include "utils/Error.h" -//#include "LicenseLibrary.h" -// -//#include -// -//#include -//#include -// -//namespace zilliz { -//namespace milvus { -//namespace server { -// -//class LicenseCheck { -//private: -// LicenseCheck(); -// ~LicenseCheck(); -// -//public: -// static LicenseCheck & -// GetInstance() { -// static LicenseCheck instance; -// return instance; -// }; -// -// static ServerError -// LegalityCheck(const std::string &license_file_path); -// -// ServerError -// StartCountingDown(const std::string &license_file_path); -// -// ServerError -// StopCountingDown(); -// -//private: -// static ServerError -// AlterFile(const std::string &license_file_path, -// const boost::system::error_code &ec, -// boost::asio::deadline_timer *pt); -// -//private: -// boost::asio::io_service io_service_; -// std::shared_ptr counting_thread_; -// -//}; -// -//} -//} -//} -// -// diff --git a/cpp/src/license/LicenseFile.h b/cpp/src/license/LicenseFile.h deleted file mode 100644 index deefe19d14..0000000000 --- a/cpp/src/license/LicenseFile.h +++ /dev/null @@ -1,86 +0,0 @@ -///******************************************************************************* -// * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// * Unauthorized copying of this file, via any medium is strictly prohibited. -// * Proprietary and confidential. -// ******************************************************************************/ -//#pragma once -// -// -//#include -//#include -//#include -// -// -//class LicenseFile { -// public: -// LicenseFile() = default; -// -// LicenseFile(const int &device_count, -// const std::map &uuid_encryption_map, -// const time_t &starting_time, -// const time_t &end_time) -// : device_count_(device_count), -// uuid_encryption_map_(uuid_encryption_map), -// starting_time_(starting_time), -// end_time_(end_time) {} -// -// int get_device_count() { -// return device_count_; -// } -// std::map &get_uuid_encryption_map() { -// return uuid_encryption_map_; -// } -// time_t get_starting_time() { -// return starting_time_; -// } -// time_t get_end_time() { -// return end_time_; -// } -// -// public: -// friend class boost::serialization::access; -// -// template -// void serialize(Archive &ar, const unsigned int version) { -// ar & device_count_; -// ar & uuid_encryption_map_; -// ar & starting_time_; -// ar & end_time_; -// } -// -// public: -// int device_count_ = 0; -// std::map uuid_encryption_map_; -// time_t starting_time_ = 0; -// time_t end_time_ = 0; -//}; -// -//class SerializedLicenseFile { -// public: -// ~SerializedLicenseFile() { -// if (license_file_ != nullptr) { -// delete (license_file_); -// license_file_ = nullptr; -// } -// } -// -// void -// set_license_file(LicenseFile *license_file) { -// license_file_ = license_file; -// } -// -// LicenseFile *get_license_file() { -// return license_file_; -// } -// private: -// friend class boost::serialization::access; -// -// template -// void serialize(Archive &ar, const unsigned int version) { -// ar & license_file_; -// } -// -// private: -// LicenseFile *license_file_ = nullptr; -//}; -// diff --git a/cpp/src/license/LicenseGenerator.cpp b/cpp/src/license/LicenseGenerator.cpp deleted file mode 100644 index a5064eb4c3..0000000000 --- a/cpp/src/license/LicenseGenerator.cpp +++ /dev/null @@ -1,121 +0,0 @@ -// -//#include -//#include -//#include -// -//#include "utils/Log.h" -//#include "license/LicenseLibrary.h" -//#include "utils/Error.h" -// -// -//using namespace zilliz::milvus; -//// Not provide path: current work path will be used and system.info. -// -//void -//print_usage(const std::string &app_name) { -// printf("\n Usage: %s [OPTIONS]\n\n", app_name.c_str()); -// printf(" Options:\n"); -// printf(" -h --help Print this help\n"); -// printf(" -s --sysinfo filename sysinfo file location\n"); -// printf(" -l --license filename Generate license file as given name\n"); -// printf(" -b --starting time Set start time (format: YYYY-MM-DD)\n"); -// printf(" -e --end time Set end time (format: YYYY-MM-DD)\n"); -// printf("\n"); -//} -// -//int main(int argc, char *argv[]) { -// std::string app_name = argv[0]; -//// if (argc != 1 && argc != 3) { -//// print_usage(app_name); -//// return EXIT_FAILURE; -//// } -// static struct option long_options[] = {{"system_info", required_argument, 0, 's'}, -// {"license", optional_argument, 0, 'l'}, -// {"help", no_argument, 0, 'h'}, -// {"starting_time", required_argument, 0, 'b'}, -// {"end_time", required_argument, 0, 'e'}, -// {NULL, 0, 0, 0}}; -// server::ServerError err; -// int value = 0; -// int option_index = 0; -// std::string system_info_filename = "./system.info"; -// std::string license_filename = "./system.license"; -// char *string_starting_time = NULL; -// char *string_end_time = NULL; -// time_t starting_time = 0; -// time_t end_time = 0; -// int flag_s = 1; -// int flag_b = 1; -// int flag_e = 1; -// while ((value = getopt_long(argc, argv, "hl:s:b:e:", long_options, NULL)) != -1) { -// switch (value) { -// case 's': { -// flag_s = 0; -// system_info_filename = (std::string) (optarg); -// break; -// } -// case 'b': { -// flag_b = 0; -// string_starting_time = optarg; -// break; -// } -// case 'e': { -// flag_e = 0; -// string_end_time = optarg; -// break; -// } -// case 'l': { -// license_filename = (std::string) (optarg); -// break; -// } -// case 'h':print_usage(app_name); -// return EXIT_SUCCESS; -// case '?':print_usage(app_name); -// return EXIT_FAILURE; -// default:print_usage(app_name); -// break; -// } -// -// } -// if (flag_s) { -// printf("Error: sysinfo file location must be entered\n"); -// return 1; -// } -// if (flag_b) { -// printf("Error: start time must be entered\n"); -// return 1; -// } -// if (flag_e) { -// printf("Error: end time must be entered\n"); -// return 1; -// } -// -// err = server::LicenseLibrary::GetDateTime(string_starting_time, starting_time); -// if (err != server::SERVER_SUCCESS) return -1; -// -// err = server::LicenseLibrary::GetDateTime(string_end_time, end_time); -// if (err != server::SERVER_SUCCESS) return -1; -// -// -// int output_info_device_count = 0; -// std::map output_info_uuid_encrption_map; -// -// -// err = server::LicenseLibrary::GPUinfoFileDeserialization(system_info_filename, -// output_info_device_count, -// output_info_uuid_encrption_map); -// if (err != server::SERVER_SUCCESS) return -1; -// -// -// err = server::LicenseLibrary::LicenseFileSerialization(license_filename, -// output_info_device_count, -// output_info_uuid_encrption_map, -// starting_time, -// end_time); -// if (err != server::SERVER_SUCCESS) return -1; -// -// -// printf("Generate License File Success\n"); -// -// return 0; -//} diff --git a/cpp/src/license/LicenseLibrary.cpp b/cpp/src/license/LicenseLibrary.cpp deleted file mode 100644 index bf377650a6..0000000000 --- a/cpp/src/license/LicenseLibrary.cpp +++ /dev/null @@ -1,345 +0,0 @@ -////////////////////////////////////////////////////////////////////////////////// -//// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -//// Unauthorized copying of this file, via any medium is strictly prohibited. -//// Proprietary and confidential. -////////////////////////////////////////////////////////////////////////////////// -// -//#include "LicenseLibrary.h" -//#include "utils/Log.h" -//#include -//#include -//#include -//#include -// -//#include -//#include -////#include -////#include -//#include -//#include -//#include -// -// -//namespace zilliz { -//namespace milvus { -//namespace server { -// -//constexpr int LicenseLibrary::sha256_length_; -// -//// Part 0: File check -//bool -//LicenseLibrary::IsFileExistent(const std::string &path) { -// -// boost::system::error_code error; -// auto file_status = boost::filesystem::status(path, error); -// if (error) { -// return false; -// } -// -// if (!boost::filesystem::exists(file_status)) { -// return false; -// } -// -// return !boost::filesystem::is_directory(file_status); -//} -// -//// Part 1: Get GPU Info -//ServerError -//LicenseLibrary::GetDeviceCount(int &device_count) { -// nvmlReturn_t result = nvmlInit(); -// if (NVML_SUCCESS != result) { -// printf("Failed to initialize NVML: %s\n", nvmlErrorString(result)); -// return SERVER_UNEXPECTED_ERROR; -// } -// cudaError_t error_id = cudaGetDeviceCount(&device_count); -// if (error_id != cudaSuccess) { -// printf("cudaGetDeviceCount returned %d\n-> %s\n", (int) error_id, cudaGetErrorString(error_id)); -// printf("Result = FAIL\n"); -// return SERVER_UNEXPECTED_ERROR; -// } -// return SERVER_SUCCESS; -//} -// -//ServerError -//LicenseLibrary::GetUUID(int device_count, std::vector &uuid_array) { -// if (device_count == 0) { -// printf("There are no available device(s) that support CUDA\n"); -// return SERVER_UNEXPECTED_ERROR; -// } -// -// for (int dev = 0; dev < device_count; ++dev) { -// nvmlDevice_t device; -// nvmlReturn_t result = nvmlDeviceGetHandleByIndex(dev, &device); -// if (NVML_SUCCESS != result) { -// printf("Failed to get handle for device %i: %s\n", dev, nvmlErrorString(result)); -// return SERVER_UNEXPECTED_ERROR; -// } -// -// char uuid[80]; -// unsigned int length = 80; -// nvmlReturn_t err = nvmlDeviceGetUUID(device, uuid, length); -// if (err != NVML_SUCCESS) { -// printf("nvmlDeviceGetUUID error: %d\n", err); -// return SERVER_UNEXPECTED_ERROR; -// } -// -// uuid_array.emplace_back(uuid); -// } -// return SERVER_SUCCESS; -//} -// -//ServerError -//LicenseLibrary::GetUUIDMD5(int device_count, -// std::vector &uuid_array, -// std::vector &md5_array) { -// MD5_CTX ctx; -// unsigned char outmd[16]; -// char temp[2]; -// std::string md5; -// for (int dev = 0; dev < device_count; ++dev) { -// md5.clear(); -// memset(outmd, 0, sizeof(outmd)); -// MD5_Init(&ctx); -// MD5_Update(&ctx, uuid_array[dev].c_str(), uuid_array[dev].size()); -// MD5_Final(outmd, &ctx); -// for (int i = 0; i < 16; ++i) { -// std::snprintf(temp, 2, "%02X", outmd[i]); -// md5 += temp; -// } -// md5_array.push_back(md5); -// } -// return SERVER_SUCCESS; -//} -// -//ServerError -//LicenseLibrary::GetUUIDSHA256(const int &device_count, -// std::vector &uuid_array, -// std::vector &sha_array) { -// SHA256_CTX ctx; -// unsigned char outmd[sha256_length_]; -// char temp[2]; -// std::string sha; -// for (int dev = 0; dev < device_count; ++dev) { -// sha.clear(); -// memset(outmd, 0, sizeof(outmd)); -// SHA256_Init(&ctx); -// SHA256_Update(&ctx, uuid_array[dev].c_str(), uuid_array[dev].size()); -// SHA256_Final(outmd, &ctx); -// for (int i = 0; i < sha256_length_; ++i) { -// std::snprintf(temp, 2, "%02X", outmd[i]); -// sha += temp; -// } -// sha_array.push_back(sha); -// } -// return SERVER_SUCCESS; -//} -// -//ServerError -//LicenseLibrary::GetSystemTime(time_t &system_time) { -// system_time = time(NULL); -// return SERVER_SUCCESS; -//} -// -//// Part 2: Handle License File -//ServerError -//LicenseLibrary::LicenseFileSerialization(const std::string &path, -// int device_count, -// const std::map &uuid_encrption_map, -// time_t starting_time, -// time_t end_time) { -// -// std::ofstream file(path); -// boost::archive::binary_oarchive oa(file); -// oa.register_type(); -// -// SerializedLicenseFile serialized_license_file; -// -// serialized_license_file.set_license_file(new LicenseFile(device_count, -// uuid_encrption_map, -// starting_time, -// end_time)); -// oa << serialized_license_file; -// -// file.close(); -// return SERVER_SUCCESS; -//} -// -//ServerError -//LicenseLibrary::LicenseFileDeserialization(const std::string &path, -// int &device_count, -// std::map &uuid_encrption_map, -// time_t &starting_time, -// time_t &end_time) { -// if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST; -// std::ifstream file(path); -// boost::archive::binary_iarchive ia(file); -// ia.register_type(); -// -// SerializedLicenseFile serialized_license_file; -// ia >> serialized_license_file; -// -// device_count = serialized_license_file.get_license_file()->get_device_count(); -// uuid_encrption_map = serialized_license_file.get_license_file()->get_uuid_encryption_map(); -// starting_time = serialized_license_file.get_license_file()->get_starting_time(); -// end_time = serialized_license_file.get_license_file()->get_end_time(); -// -// file.close(); -// return SERVER_SUCCESS; -//} -// -////ServerError -////LicenseLibrary::SecretFileSerialization(const std::string &path, -//// const time_t &update_time, -//// const off_t &file_size, -//// const time_t &starting_time, -//// const time_t &end_time, -//// const std::string &file_md5) { -//// std::ofstream file(path); -//// boost::archive::binary_oarchive oa(file); -//// oa.register_type(); -//// -//// SerializedSecretFile serialized_secret_file; -//// -//// serialized_secret_file.set_secret_file(new SecretFile(update_time, file_size, starting_time, end_time, file_md5)); -//// oa << serialized_secret_file; -//// -//// file.close(); -//// return SERVER_SUCCESS; -////} -//// -////ServerError -////LicenseLibrary::SecretFileDeserialization(const std::string &path, -//// time_t &update_time, -//// off_t &file_size, -//// time_t &starting_time, -//// time_t &end_time, -//// std::string &file_md5) { -//// if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST; -//// -//// std::ifstream file(path); -//// boost::archive::binary_iarchive ia(file); -//// ia.register_type(); -//// SerializedSecretFile serialized_secret_file; -//// -//// ia >> serialized_secret_file; -//// update_time = serialized_secret_file.get_secret_file()->get_update_time(); -//// file_size = serialized_secret_file.get_secret_file()->get_file_size(); -//// starting_time = serialized_secret_file.get_secret_file()->get_starting_time(); -//// end_time = serialized_secret_file.get_secret_file()->get_end_time(); -//// file_md5 = serialized_secret_file.get_secret_file()->get_file_md5(); -//// file.close(); -//// return SERVER_SUCCESS; -////} -// -// -// -//// Part 3: File attribute: UpdateTime Time/ Size/ MD5 -//ServerError -//LicenseLibrary::GetFileUpdateTimeAndSize(const std::string &path, time_t &update_time, off_t &file_size) { -// -// if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST; -// -// struct stat buf; -// int err_no = stat(path.c_str(), &buf); -// if (err_no != 0) { -// std::cout << strerror(err_no) << std::endl; -// return SERVER_UNEXPECTED_ERROR; -// } -// -// update_time = buf.st_mtime; -// file_size = buf.st_size; -// -// return SERVER_SUCCESS; -//} -// -//ServerError -//LicenseLibrary::GetFileMD5(const std::string &path, std::string &filemd5) { -// -// if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST; -// -// filemd5.clear(); -// -// std::ifstream file(path.c_str(), std::ifstream::binary); -// if (!file) { -// return -1; -// } -// -// MD5_CTX md5Context; -// MD5_Init(&md5Context); -// -// char buf[1024 * 16]; -// while (file.good()) { -// file.read(buf, sizeof(buf)); -// MD5_Update(&md5Context, buf, file.gcount()); -// } -// -// unsigned char result[MD5_DIGEST_LENGTH]; -// MD5_Final(result, &md5Context); -// -// char hex[35]; -// memset(hex, 0, sizeof(hex)); -// for (int i = 0; i < MD5_DIGEST_LENGTH; ++i) { -// sprintf(hex + i * 2, "%02X", result[i]); -// } -// hex[32] = '\0'; -// filemd5 = std::string(hex); -// -// return SERVER_SUCCESS; -//} -//// Part 4: GPU Info File Serialization/Deserialization -//ServerError -//LicenseLibrary::GPUinfoFileSerialization(const std::string &path, -// int device_count, -// const std::map &uuid_encrption_map) { -// std::ofstream file(path); -// boost::archive::binary_oarchive oa(file); -// oa.register_type(); -// -// SerializedGPUInfoFile serialized_gpu_info_file; -// -// serialized_gpu_info_file.set_gpu_info_file(new GPUInfoFile(device_count, uuid_encrption_map)); -// oa << serialized_gpu_info_file; -// -// file.close(); -// return SERVER_SUCCESS; -//} -//ServerError -//LicenseLibrary::GPUinfoFileDeserialization(const std::string &path, -// int &device_count, -// std::map &uuid_encrption_map) { -// if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST; -// -// std::ifstream file(path); -// boost::archive::binary_iarchive ia(file); -// ia.register_type(); -// -// SerializedGPUInfoFile serialized_gpu_info_file; -// ia >> serialized_gpu_info_file; -// -// device_count = serialized_gpu_info_file.get_gpu_info_file()->get_device_count(); -// uuid_encrption_map = serialized_gpu_info_file.get_gpu_info_file()->get_uuid_encryption_map(); -// -// file.close(); -// return SERVER_SUCCESS; -//} -// -//ServerError -//LicenseLibrary::GetDateTime(const char *cha, time_t &data_time) { -// tm tm_; -// int year, month, day; -// sscanf(cha, "%d-%d-%d", &year, &month, &day); -// tm_.tm_year = year - 1900; -// tm_.tm_mon = month - 1; -// tm_.tm_mday = day; -// tm_.tm_hour = 0; -// tm_.tm_min = 0; -// tm_.tm_sec = 0; -// tm_.tm_isdst = 0; -// data_time = mktime(&tm_); -// return SERVER_SUCCESS; -// -//} -// -//} -//} -//} \ No newline at end of file diff --git a/cpp/src/license/LicenseLibrary.h b/cpp/src/license/LicenseLibrary.h deleted file mode 100644 index 427dda8d47..0000000000 --- a/cpp/src/license/LicenseLibrary.h +++ /dev/null @@ -1,105 +0,0 @@ -//#pragma once -// -//#include "LicenseFile.h" -//#include "GPUInfoFile.h" -// -//#include "utils/Error.h" -// -//#include -//#include -//#include -// -//#include -//#include -//#include -// -// -//namespace zilliz { -//namespace milvus { -//namespace server { -// -//class LicenseLibrary { -// public: -// // Part 0: File check -// static bool -// IsFileExistent(const std::string &path); -// -// // Part 1: Get GPU Info -// static ServerError -// GetDeviceCount(int &device_count); -// -// static ServerError -// GetUUID(int device_count, std::vector &uuid_array); -// -// static ServerError -// GetUUIDMD5(int device_count, std::vector &uuid_array, std::vector &md5_array); -// -// -// static ServerError -// GetUUIDSHA256(const int &device_count, -// std::vector &uuid_array, -// std::vector &sha_array); -// -// static ServerError -// GetSystemTime(time_t &system_time); -// -// // Part 2: Handle License File -// static ServerError -// LicenseFileSerialization(const std::string &path, -// int device_count, -// const std::map &uuid_encrption_map, -// time_t starting_time, -// time_t end_time); -// -// static ServerError -// LicenseFileDeserialization(const std::string &path, -// int &device_count, -// std::map &uuid_encrption_map, -// time_t &starting_time, -// time_t &end_time); -// -//// static ServerError -//// SecretFileSerialization(const std::string &path, -//// const time_t &update_time, -//// const off_t &file_size, -//// const time_t &starting_time, -//// const time_t &end_time, -//// const std::string &file_md5); -//// -//// static ServerError -//// SecretFileDeserialization(const std::string &path, -//// time_t &update_time, -//// off_t &file_size, -//// time_t &starting_time, -//// time_t &end_time, -//// std::string &file_md5); -// -// // Part 3: File attribute: UpdateTime Time/ Size/ MD5 -// static ServerError -// GetFileUpdateTimeAndSize(const std::string &path, time_t &update_time, off_t &file_size); -// -// static ServerError -// GetFileMD5(const std::string &path, std::string &filemd5); -// -// // Part 4: GPU Info File Serialization/Deserialization -// static ServerError -// GPUinfoFileSerialization(const std::string &path, -// int device_count, -// const std::map &uuid_encrption_map); -// static ServerError -// GPUinfoFileDeserialization(const std::string &path, -// int &device_count, -// std::map &uuid_encrption_map); -// -// static ServerError -// GetDateTime(const char *cha, time_t &data_time); -// -// -// private: -// static constexpr int sha256_length_ = 32; -//}; -// -// -//} -//} -//} diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index aeccc1e5be..cb5079f30e 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -9,7 +9,6 @@ #include "utils/Log.h" #include "utils/SignalUtil.h" #include "utils/TimeRecorder.h" -#include "license/LicenseCheck.h" #include "metrics/Metrics.h" #include @@ -152,19 +151,6 @@ Server::Start() { ServerConfig &config = ServerConfig::GetInstance(); ConfigNode server_config = config.GetConfig(CONFIG_SERVER); -#ifdef ENABLE_LICENSE - ConfigNode license_config = config.GetConfig(CONFIG_LICENSE); - std::string license_file_path = license_config.GetValue(CONFIG_LICENSE_PATH); - SERVER_LOG_INFO << "License path: " << license_file_path; - - if(server::LicenseCheck::LegalityCheck(license_file_path) != SERVER_SUCCESS) { - SERVER_LOG_ERROR << "License check failed"; - exit(1); - } - - server::LicenseCheck::GetInstance().StartCountingDown(license_file_path); -#endif - // Handle Signal signal(SIGINT, SignalUtil::HandleSignal); signal(SIGHUP, SignalUtil::HandleSignal); @@ -216,9 +202,6 @@ Server::Stop() { StopService(); -#ifdef ENABLE_LICENSE - server::LicenseCheck::GetInstance().StopCountingDown(); -#endif std::cout << "Milvus server is closed!" << std::endl; } diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index 2b100448f9..458ad7589d 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -41,8 +41,6 @@ set(unittest_libs add_subdirectory(server) add_subdirectory(db) add_subdirectory(index_wrapper) -#add_subdirectory(faiss_wrapper) -#add_subdirectory(license) add_subdirectory(metrics) add_subdirectory(storage) add_subdirectory(utils) \ No newline at end of file diff --git a/cpp/unittest/db/misc_test.cpp b/cpp/unittest/db/misc_test.cpp index 3ec54751ab..d968b69b8e 100644 --- a/cpp/unittest/db/misc_test.cpp +++ b/cpp/unittest/db/misc_test.cpp @@ -8,7 +8,6 @@ #include #include -#include "db/FaissExecutionEngine.h" #include "db/Exception.h" #include "db/Status.h" #include "db/Options.h" @@ -27,33 +26,6 @@ namespace { } -//TEST(DBMiscTest, ENGINE_API_TEST) { -// //engine api AddWithIdArray -// const uint16_t dim = 512; -// const long n = 10; -// engine::FaissExecutionEngine engine(512, "/tmp/1", "IDMap", "IDMap,Flat"); -// std::vector vectors; -// std::vector ids; -// for (long i = 0; i < n; i++) { -// for (uint16_t k = 0; k < dim; k++) { -// vectors.push_back((float) k); -// } -// ids.push_back(i); -// } -// -// auto status = engine.AddWithIdArray(vectors, ids); -// ASSERT_TRUE(status.ok()); -// -// auto engine_ptr = engine::EngineFactory::Build(128, "/tmp", engine::EngineType::INVALID); -// ASSERT_EQ(engine_ptr, nullptr); -// -// engine_ptr = engine::EngineFactory::Build(128, "/tmp", engine::EngineType::FAISS_IVFFLAT_GPU); -// ASSERT_NE(engine_ptr, nullptr); -// -// engine_ptr = engine::EngineFactory::Build(128, "/tmp", engine::EngineType::FAISS_IDMAP); -// ASSERT_NE(engine_ptr, nullptr); -//} - TEST(DBMiscTest, EXCEPTION_TEST) { engine::Exception ex1(""); std::string what = ex1.what(); diff --git a/cpp/unittest/faiss_wrapper/CMakeLists.txt b/cpp/unittest/faiss_wrapper/CMakeLists.txt deleted file mode 100644 index 90d2ae36e5..0000000000 --- a/cpp/unittest/faiss_wrapper/CMakeLists.txt +++ /dev/null @@ -1,55 +0,0 @@ -#------------------------------------------------------------------------------- -# Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -# Unauthorized copying of this file, via any medium is strictly prohibited. -# Proprietary and confidential. -#------------------------------------------------------------------------------- -aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src) -aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) - -set(util_files - ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp) - -# Make sure that your call to link_directories takes place before your call to the relevant add_executable. -include_directories(/usr/local/cuda/include) -link_directories("/usr/local/cuda/lib64") - -set(wrapper_test_src - ${unittest_srcs} - ${wrapper_src} - ${config_files} - ${util_files} - ${require_files} - wrapper_test.cpp - ) - -add_executable(wrapper_test ${wrapper_test_src}) - -set(wrapper_libs - stdc++ - boost_system_static - boost_filesystem_static - faiss - cudart - cublas - sqlite - snappy - bz2 - z - zstd - lz4 - ) -if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON") - set(wrapper_libs ${wrapper_libs} ${MKL_LIBS} ${MKL_LIBS}) -else() - set(wrapper_libs ${wrapper_libs} - lapack - openblas) -endif() - -target_link_libraries(wrapper_test ${wrapper_libs} ${unittest_libs}) - -set(topk_test_src - topk_test.cpp - ${CMAKE_SOURCE_DIR}/src/wrapper/gpu/Topk.cu) - -install(TARGETS wrapper_test DESTINATION bin) diff --git a/cpp/unittest/faiss_wrapper/topk_test.cpp b/cpp/unittest/faiss_wrapper/topk_test.cpp deleted file mode 100644 index 7bcd28a7ff..0000000000 --- a/cpp/unittest/faiss_wrapper/topk_test.cpp +++ /dev/null @@ -1,89 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// - -#include - -#include "wrapper/gpu/Topk.h" - - -using namespace zilliz::milvus::engine; - -constexpr float threshhold = 0.00001; - -template -void TopK_check(T *data, - int length, - int k, - T *result) { - - std::vector arr(data, data + length); - sort(arr.begin(), arr.end(), std::less()); - - for (int i = 0; i < k; ++i) { - ASSERT_TRUE(fabs(arr[i] - result[i]) < threshhold); - } -} - -TEST(wrapper_topk, Wrapper_Test) { - int length = 100000; - int k = 1000; - - float *host_input, *host_output; - int64_t *ids; - - host_input = (float *) malloc(length * sizeof(float)); - host_output = (float *) malloc(k * sizeof(float)); - ids = (int64_t *) malloc(k * sizeof(int64_t)); - - std::random_device rd; - std::mt19937 gen(rd()); - std::uniform_real_distribution<> dis(-1.0, 1.0); - for (int i = 0; i < length; ++i) { - host_input[i] = 1.0 * dis(gen); - } - - TopK(host_input, length, k, host_output, ids); - TopK_check(host_input, length, k, host_output); -} - -template -void TopK_Test(T factor) { - int length = 1000000; // data length - int k = 100; - - T *data, *out; - int64_t *idx; - cudaMallocManaged((void **) &data, sizeof(T) * length); - cudaMallocManaged((void **) &out, sizeof(T) * k); - cudaMallocManaged((void **) &idx, sizeof(int64_t) * k); - - std::random_device rd; - std::mt19937 gen(rd()); - std::uniform_real_distribution<> dis(-1.0, 1.0); - - for (int i = 0; i < length; i++) { - data[i] = factor * dis(gen); - } - - cudaMemAdvise(data, sizeof(T) * length, cudaMemAdviseSetReadMostly, 0); - - cudaMemPrefetchAsync(data, sizeof(T) * length, 0); - - gpu::TopK(data, length, k, out, idx, nullptr); - TopK_check(data, length, k, out); - -// order_flag = Ordering::kDescending; -// TopK(data, length, k, out, idx, nullptr); -// TopK_check(data, length, k, out); - - cudaFree(data); - cudaFree(out); - cudaFree(idx); -} - -TEST(topk_test, Wrapper_Test) { - TopK_Test(1.0); -} diff --git a/cpp/unittest/faiss_wrapper/wrapper_test.cpp b/cpp/unittest/faiss_wrapper/wrapper_test.cpp deleted file mode 100644 index 6f4a651a55..0000000000 --- a/cpp/unittest/faiss_wrapper/wrapper_test.cpp +++ /dev/null @@ -1,141 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// - - - -#include "wrapper/Operand.h" -#include "wrapper/Index.h" -#include "wrapper/IndexBuilder.h" - -#include -#include - -using namespace zilliz::milvus::engine; - - -TEST(operand_test, Wrapper_Test) { - using std::cout; - using std::endl; - - auto opd = std::make_shared(); - opd->index_type = "IVF"; - opd->preproc = "OPQ"; - opd->postproc = "PQ"; - opd->metric_type = "L2"; - opd->d = 64; - - auto opd_str = operand_to_str(opd); - auto new_opd = str_to_operand(opd_str); - - // TODO: fix all place where using opd to build index. - assert(new_opd->get_index_type(10000) == opd->get_index_type(10000)); -} - -TEST(build_test, Wrapper_Test) { - // dimension of the vectors to index - int d = 3; - - // make a set of nt training vectors in the unit cube - size_t nt = 10000; - - // a reasonable number of cetroids to index nb vectors - int ncentroids = 16; - - std::random_device rd; - std::mt19937 gen(rd()); - - std::vector xb; - std::vector ids; - - //prepare train data - std::uniform_real_distribution<> dis_xt(-1.0, 1.0); - std::vector xt(nt * d); - for (size_t i = 0; i < nt * d; i++) { - xt[i] = dis_xt(gen); - } - - //train the index - auto opd = std::make_shared(); - opd->index_type = "IVF"; - opd->d = d; - opd->ncent = ncentroids; - IndexBuilderPtr index_builder_1 = GetIndexBuilder(opd); - auto index_1 = index_builder_1->build_all(0, xb, ids, nt, xt); - ASSERT_TRUE(index_1 != nullptr); - - // size of the database we plan to index - size_t nb = 100000; - - //prepare raw data - xb.resize(nb); - ids.resize(nb); - for (size_t i = 0; i < nb; i++) { - xb[i] = dis_xt(gen); - ids[i] = i; - } - index_1->add_with_ids(nb, xb.data(), ids.data()); - - //search in first quadrant - int nq = 1, k = 10; - std::vector xq = {0.5, 0.5, 0.5}; - float *result_dists = new float[k]; - long *result_ids = new long[k]; - index_1->search(nq, xq.data(), k, result_dists, result_ids); - - for (int i = 0; i < k; i++) { - if (result_ids[i] < 0) { - ASSERT_TRUE(false); - break; - } - - long id = result_ids[i]; - std::cout << "No." << id << " [" << xb[id * 3] << ", " << xb[id * 3 + 1] << ", " - << xb[id * 3 + 2] << "] distance = " << result_dists[i] << std::endl; - - //makesure result vector is in first quadrant - ASSERT_TRUE(xb[id * 3] > 0.0); - ASSERT_TRUE(xb[id * 3 + 1] > 0.0); - ASSERT_TRUE(xb[id * 3 + 2] > 0.0); - } - - delete[] result_dists; - delete[] result_ids; -} - -TEST(gpu_build_test, Wrapper_Test) { - using std::vector; - - int d = 256; - int nb = 3 * 1000 * 100; - int nq = 100; - vector xb(d * nb); - vector xq(d * nq); - vector ids(nb); - - std::random_device rd; - std::mt19937 gen(rd()); - std::uniform_real_distribution<> dis_xt(-1.0, 1.0); - for (auto &e : xb) { e = float(dis_xt(gen)); } - for (auto &e : xq) { e = float(dis_xt(gen)); } - for (int i = 0; i < nb; ++i) { ids[i] = i; } - - auto opd = std::make_shared(); - opd->index_type = "IVF"; - opd->d = d; - opd->ncent = 256; - - IndexBuilderPtr index_builder_1 = GetIndexBuilder(opd); - auto index_1 = index_builder_1->build_all(nb, xb.data(), ids.data()); - assert(index_1->ntotal == nb); - assert(index_1->dim == d); - - // sanity check: search 5 first vectors of xb - int k = 1; - vector I(5 * k); - vector D(5 * k); - index_1->search(5, xb.data(), k, D.data(), I.data()); - for (int i = 0; i < 5; ++i) { assert(i == I[i]); } -} diff --git a/cpp/unittest/license/CMakeLists.txt b/cpp/unittest/license/CMakeLists.txt deleted file mode 100644 index 39d0ede197..0000000000 --- a/cpp/unittest/license/CMakeLists.txt +++ /dev/null @@ -1,43 +0,0 @@ -#------------------------------------------------------------------------------- -# Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -# Unauthorized copying of this file, via any medium is strictly prohibited. -# Proprietary and confidential. -#------------------------------------------------------------------------------- -aux_source_directory(${MILVUS_ENGINE_SRC}/db db_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs) -aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src) - -include_directories(/usr/local/cuda/include) -link_directories(/usr/local/cuda) -link_directories(/usr/local/cuda/lib64) -link_directories(/usr/lib/x86_64-linux-gnu) -link_directories(/usr/lib/nvidia-415) -link_directories(/usr/local/cuda/targets/x86_64-linux/lib/stubs/) -link_directories(/usr/local/cuda/lib64/stubs/) - -set(license_src_files - ${MILVUS_ENGINE_SRC}/license/LicenseLibrary.cpp - ${MILVUS_ENGINE_SRC}/license/LicenseCheck.cpp - ) - - -set(db_test_src - license_library_tests.cpp - license_check_test.cpp - ${license_src_files}) - -cuda_add_executable(license_test ${db_test_src}) - -set(db_libs - nvidia-ml - cudart - cublas - boost_system_static - boost_filesystem_static - lz4 - crypto - boost_serialization_static - ) - -target_link_libraries(license_test ${db_libs} ${unittest_libs}) diff --git a/cpp/unittest/license/license_check_test.cpp b/cpp/unittest/license/license_check_test.cpp deleted file mode 100644 index 0ad719bb60..0000000000 --- a/cpp/unittest/license/license_check_test.cpp +++ /dev/null @@ -1,184 +0,0 @@ -//// -//// Created by zilliz on 19-5-13. -//// -// -//#include "utils/Log.h" -//#include "license/LicenseCheck.h" -//#include "utils/Error.h" -// -//#include -//#include -// -// -//using namespace zilliz::milvus; -// -//TEST(LicenseLibraryTest, CHECK_TEST) { -// -// -// server::ServerError err; -// // 1. Set license file name -// std::string license_file_path("/tmp/milvus/abc.license"); -// -// // 2. Legality check -// err = server::LicenseCheck::LegalityCheck(license_file_path); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -// -//} -// -//TEST(LicenseLibraryTest, CHECK_ERROR1_TEST){ -// -// server::ServerError err; -// // 1. Set license file name -// std::string license_file_path("/tmp/milvus/abc"); -// -// // 2. Legality check -// err = server::LicenseCheck::LegalityCheck(license_file_path); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -//} -// -//TEST(LicenseLibraryTest, CHECK_ERROR2_TEST){ -// -// server::ServerError err; -// // 1. Set license file name -// std::string license_file_path("/tmp/milvus/abc.license"); -// -// // 2. Define output var -// int device_count; -// std::map uuid_encryption_map; -// time_t starting_time; -// time_t end_time; -// -// // 3. Read License File -// err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path, -// device_count, -// uuid_encryption_map, -// starting_time, -// end_time); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -// -// // 4. Change device count -// ++device_count; -// err = server::LicenseLibrary::LicenseFileSerialization(license_file_path, -// device_count, -// uuid_encryption_map, -// starting_time, -// end_time); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -// -// // 5. Legality check -// err = server::LicenseCheck::LegalityCheck(license_file_path); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -//} -// -//TEST(LicenseLibraryTest, CHECK_ERROR3_TEST){ -// -// server::ServerError err; -// // 1. Set license file name -// std::string license_file_path("/tmp/milvus/abc.license"); -// -// // 2. Define output var -// int device_count; -// std::map uuid_encryption_map; -// time_t starting_time; -// time_t end_time; -// -// // 3. Read License File -// err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path, -// device_count, -// uuid_encryption_map, -// starting_time, -// end_time); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -// -// // 4. Change device count -// if(device_count) uuid_encryption_map[0]+="u"; -// err = server::LicenseLibrary::LicenseFileSerialization(license_file_path, -// device_count, -// uuid_encryption_map, -// starting_time, -// end_time); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -// -// // 5. Legality check -// err = server::LicenseCheck::LegalityCheck(license_file_path); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -//} -// -//TEST(LicenseLibraryTest, CHECK_ERROR4_1_TEST){ -// -// server::ServerError err; -// // 1. Set license file name -// std::string license_file_path("/tmp/milvus/abc.license"); -// -// // 2. Define output var -// int device_count; -// std::map uuid_encryption_map; -// time_t starting_time; -// time_t end_time; -// -// // 3. Read License File -// err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path, -// device_count, -// uuid_encryption_map, -// starting_time, -// end_time); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -// -// // 4. Change starting time -// time_t system_time; -// err = server::LicenseLibrary::GetSystemTime(system_time); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -// -// system_time+=60*60*24; -// -// err = server::LicenseLibrary::LicenseFileSerialization(license_file_path, -// device_count, -// uuid_encryption_map, -// system_time, -// end_time); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -// -// // 5. Legality check -// err = server::LicenseCheck::LegalityCheck(license_file_path); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -//} -// -//TEST(LicenseLibraryTest, CHECK_ERROR4_2_TEST){ -// -// server::ServerError err; -// // 1. Set license file name -// std::string license_file_path("/tmp/milvus/abc.license"); -// -// // 2. Define output var -// int device_count; -// std::map uuid_encryption_map; -// time_t starting_time; -// time_t end_time; -// -// // 3. Read License File -// err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path, -// device_count, -// uuid_encryption_map, -// starting_time, -// end_time); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -// -// // 4. Change end time -// time_t system_time; -// err = server::LicenseLibrary::GetSystemTime(system_time); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -// -// system_time-=100; -// -// err = server::LicenseLibrary::LicenseFileSerialization(license_file_path, -// device_count, -// uuid_encryption_map, -// starting_time, -// system_time); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -// -// // 5. Legality check -// err = server::LicenseCheck::LegalityCheck(license_file_path); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -//} -// diff --git a/cpp/unittest/license/license_library_tests.cpp b/cpp/unittest/license/license_library_tests.cpp deleted file mode 100644 index e96a1ecb2b..0000000000 --- a/cpp/unittest/license/license_library_tests.cpp +++ /dev/null @@ -1,214 +0,0 @@ -///******************************************************************************* -// * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// * Unauthorized copying of this file, via any medium is strictly prohibited. -// * Proprietary and confidential. -// ******************************************************************************/ -// -// -//#include "utils/Log.h" -//#include "license/LicenseLibrary.h" -//#include "utils/Error.h" -// -//#include -//#include -// -// -//using namespace zilliz::milvus; -// -//TEST(LicenseLibraryTest, FILE_EXISTENT_TEST) { -// -// std::string hosts_file = "/etc/hosts"; -// ASSERT_EQ(server::LicenseLibrary::IsFileExistent(hosts_file), true); -// -// std::string no_exist_file = "/temp/asdaasd"; -// ASSERT_EQ(server::LicenseLibrary::IsFileExistent(no_exist_file), false); -// -// std::string directory = "/tmp"; -// ASSERT_EQ(server::LicenseLibrary::IsFileExistent(directory), false); -//} -// -//TEST(LicenseLibraryTest, GPU_INFO_TEST) { -// -// int device_count = 0; -// server::ServerError err = server::LicenseLibrary::GetDeviceCount(device_count); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -// std::cout << "Device Count: " << device_count << std::endl; -// -// std::vector uuid_array; -// err = server::LicenseLibrary::GetUUID(device_count, uuid_array); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -// for (long i = 0; i < device_count; ++i) { -// std::cout << "Device Id: " << i << ", UUID: " << uuid_array[i] << std::endl; -// } -// -// std::vector uuid_md5_array; -// err = server::LicenseLibrary::GetUUIDMD5(device_count, uuid_array, uuid_md5_array); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -// for (long i = 0; i < device_count; ++i) { -// std::cout << "Device Id: " << i << ", UUID: " << uuid_array[i] << ", UUID_MD5: " << uuid_md5_array[i] -// << std::endl; -// } -// -// std::vector uuid_sha256_array; -// err = server::LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, uuid_sha256_array); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -// for (long i = 0; i < device_count; ++i) { -// std::cout << "Device Id: " << i << ", UUID: " << uuid_array[i] << ", UUID_SHA256: " -// << uuid_sha256_array[i] << std::endl; -// } -// -// time_t systemtime; -// err = server::LicenseLibrary::GetSystemTime(systemtime); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -// std::cout << "System Time: " << systemtime << std::endl; -// -//} -// -//TEST(LicenseLibraryTest, LICENSE_FILE_TEST) { -// -// // 0. File check -// std::string test("/tmp/a.test"); -// bool is = server::LicenseLibrary::IsFileExistent(test); -// ASSERT_EQ(is, false); -// -// // 1. Get Device Count -// int device_count = 0; -// server::ServerError err = server::LicenseLibrary::GetDeviceCount(device_count); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -// -// // 2. Get All GPU UUID -// std::vector uuid_array; -// err = server::LicenseLibrary::GetUUID(device_count, uuid_array); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -// -// // 3. Get UUID SHA256 -// std::vector uuid_sha256_array; -// err = server::LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, uuid_sha256_array); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -// -// // 4. Generate GPU ID map with GPU UUID -// std::map uuid_encrption_map; -// for (int i = 0; i < device_count; ++i) { -// uuid_encrption_map[i] = uuid_sha256_array[i]; -// } -// -// // 5.GPU_info File -// std::string GPU_info_file_path("/tmp/milvus.info"); -// -// -// // 6. Generate GPU_info File -// err = server::LicenseLibrary::GPUinfoFileSerialization(GPU_info_file_path, -// device_count, -// uuid_encrption_map); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -// -// // 7. Define output var -// int output_info_device_count = 0; -// std::map output_info_uuid_encrption_map; -// -// // 8. Read GPU_info File -// err = server::LicenseLibrary::GPUinfoFileDeserialization(GPU_info_file_path, -// output_info_device_count, -// output_info_uuid_encrption_map); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -// -// ASSERT_EQ(device_count, output_info_device_count); -// for (int i = 0; i < device_count; ++i) { -// ASSERT_EQ(uuid_encrption_map[i], output_info_uuid_encrption_map[i]); -// } -// -// // 9. Set license file name -// std::string license_file_path("/tmp/milvus.license"); -// -// // 10. Get System Time/starting_time ans End Time -// time_t sysyem_time; -// err = server::LicenseLibrary::GetSystemTime(sysyem_time); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -// -// // 11.GetDateTime -// time_t starting_time; -// time_t end_time; -// const char *string_starting_time = "2019-05-10"; -// const char *string_end_time = "2022-05-10"; -// err = server::LicenseLibrary::GetDateTime(string_starting_time, starting_time); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -// err = server::LicenseLibrary::GetDateTime(string_end_time, end_time); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -// -// // 12. Generate License File -// err = server::LicenseLibrary::LicenseFileSerialization(license_file_path, -// device_count, -// uuid_encrption_map, -// starting_time, -// end_time); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -// -// // 13. Define output var -// int output_device_count = 0; -// std::map output_uuid_encrption_map; -// time_t output_starting_time; -// time_t output_end_time; -// -// // 14. Read License File -// err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path, -// output_device_count, -// output_uuid_encrption_map, -// output_starting_time, -// output_end_time); -// ASSERT_EQ(err, server::SERVER_SUCCESS); -// -// ASSERT_EQ(device_count, output_device_count); -// ASSERT_EQ(starting_time, output_starting_time); -// ASSERT_EQ(end_time, output_end_time); -// -// for (int i = 0; i < device_count; ++i) { -// ASSERT_EQ(uuid_encrption_map[i], output_uuid_encrption_map[i]); -// } -// -//// // 15. Get License File Attribute -//// time_t update_time; -//// off_t file_size; -//// err = server::LicenseLibrary::GetFileUpdateTimeAndSize(license_file_path, update_time, file_size); -//// ASSERT_EQ(err, server::SERVER_SUCCESS); -//// -//// // 16. Get License File MD5 -//// std::string file_md5; -//// err = server::LicenseLibrary::GetFileMD5(license_file_path, file_md5); -//// ASSERT_EQ(err, server::SERVER_SUCCESS); -// -// -// -//// // 17. Generate Secret File -//// std::string secret_file_path("/tmp/milvus.secret"); -//// err = server::LicenseLibrary::SecretFileSerialization(secret_file_path, -//// update_time, -//// file_size, -//// starting_time, -//// end_time, -//// file_md5); -//// ASSERT_EQ(err, server::SERVER_SUCCESS); -// -//// // 18. Define output var -//// time_t output_update_time; -//// off_t output_file_size; -//// time_t output_starting_time; -//// time_t output_end_time; -//// std::string output_file_md5; -// -//// // 19. Read License File -//// err = server::LicenseLibrary::SecretFileDeserialization(secret_file_path, -//// output_update_time, -//// output_file_size, -//// output_starting_time, -//// output_end_time, -//// output_file_md5); -//// ASSERT_EQ(err, server::SERVER_SUCCESS); -//// -//// ASSERT_EQ(update_time, output_update_time); -//// ASSERT_EQ(file_size, output_file_size); -//// ASSERT_EQ(starting_time, output_starting_time); -//// ASSERT_EQ(end_time, output_end_time); -//// ASSERT_EQ(file_md5, output_file_md5); -// -// -//}