test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-01 11:29:48 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

Upgrade to knowhere v1.1.0 (#16186)

Browse Source 
Signed-off-by: yudong.cai <yudong.cai@zilliz.com>
This commit is contained in:
Cai Yudong 2022-03-25 13:49:25 +08:00 committed by GitHub
parent a00968fdfe
commit f4ebd3a9ce
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 587 additions and 580 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
internal/core/bench/bench_indexbuilder.cpp
View File

@ -27,15 +27,15 @@ constexpr int64_t NB = 1000000;
namespace indexcgo = milvus::proto::indexcgo;
auto index_type_collections = [] {
static std::map<int, milvus::knowhere::IndexType> collections{
{0, milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT},
static std::map<int, knowhere::IndexType> collections{
{0, knowhere::IndexEnum::INDEX_FAISS_IVFFLAT},
};
return collections;
}();
auto metric_type_collections = [] {
static std::map<int, milvus::knowhere::MetricType> collections{
{0, milvus::knowhere::Metric::L2},
static std::map<int, knowhere::MetricType> collections{
{0, knowhere::Metric::L2},
};
return collections;
}();
@ -60,7 +60,7 @@ IndexBuilder_build(benchmark::State& state) {
auto is_binary = state.range(2);
auto dataset = GenDataset(NB, metric_type, is_binary);
auto xb_data = dataset.get_col<float>(0);
auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data());
auto xb_dataset = knowhere::GenDataset(NB, DIM, xb_data.data());
for (auto _ : state) {
auto index =
@ -89,7 +89,7 @@ IndexBuilder_build_and_codec(benchmark::State& state) {
auto is_binary = state.range(2);
auto dataset = GenDataset(NB, metric_type, is_binary);
auto xb_data = dataset.get_col<float>(0);
auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data());
auto xb_dataset = knowhere::GenDataset(NB, DIM, xb_data.data());
for (auto _ : state) {
auto index =

2
internal/core/src/common/LoadInfo.h
View File

@ -24,7 +24,7 @@
struct LoadIndexInfo {
int64_t field_id;
std::map<std::string, std::string> index_params;
milvus::knowhere::VecIndexPtr index;
knowhere::VecIndexPtr index;
};
// NOTE: field_id can be system field

14
internal/core/src/common/vector_index_c.cpp
View File

@ -20,7 +20,7 @@
CStatus
NewBinarySet(CBinarySet* c_binary_set) {
try {
auto binary_set = std::make_unique<milvus::knowhere::BinarySet>();
auto binary_set = std::make_unique<knowhere::BinarySet>();
*c_binary_set = binary_set.release();
auto status = CStatus();
status.error_code = Success;
@ -36,7 +36,7 @@ NewBinarySet(CBinarySet* c_binary_set) {
void
DeleteBinarySet(CBinarySet c_binary_set) {
auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set;
auto binary_set = (knowhere::BinarySet*)c_binary_set;
delete binary_set;
}
@ -44,7 +44,7 @@ CStatus
AppendIndexBinary(CBinarySet c_binary_set, void* index_binary, int64_t index_size, const char* c_index_key) {
auto status = CStatus();
try {
auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set;
auto binary_set = (knowhere::BinarySet*)c_binary_set;
std::string index_key(c_index_key);
uint8_t* index = (uint8_t*)index_binary;
uint8_t* dup = new uint8_t[index_size]();
@ -63,13 +63,13 @@ AppendIndexBinary(CBinarySet c_binary_set, void* index_binary, int64_t index_siz
int
GetBinarySetSize(CBinarySet c_binary_set) {
auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set;
auto binary_set = (knowhere::BinarySet*)c_binary_set;
return binary_set->binary_map_.size();
}
void
GetBinarySetKeys(CBinarySet c_binary_set, void* datas) {
auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set;
auto binary_set = (knowhere::BinarySet*)c_binary_set;
auto& map_ = binary_set->binary_map_;
const char** datas_ = (const char**)datas;
std::size_t i = 0;
@ -80,7 +80,7 @@ GetBinarySetKeys(CBinarySet c_binary_set, void* datas) {
int
GetBinarySetValueSize(CBinarySet c_binary_set, const char* key) {
auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set;
auto binary_set = (knowhere::BinarySet*)c_binary_set;
int64_t ret_ = 0;
try {
std::string key_(key);
@ -94,7 +94,7 @@ GetBinarySetValueSize(CBinarySet c_binary_set, const char* key) {
CStatus
CopyBinarySetValue(void* data, const char* key, CBinarySet c_binary_set) {
auto status = CStatus();
auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set;
auto binary_set = (knowhere::BinarySet*)c_binary_set;
try {
auto binary = binary_set->GetByName(key);
status.error_code = Success;

27
internal/core/src/config/ConfigKnowhere.cpp
View File

@ -22,19 +22,17 @@
#include "log/Log.h"
#include "knowhere/archive/KnowhereConfig.h"
namespace milvus {
namespace config {
namespace milvus::config {
std::once_flag init_knowhere_once_;
void
KnowhereInitImpl() {
auto init = []() {
namespace eg = milvus::engine;
eg::KnowhereConfig::SetBlasThreshold(16384);
eg::KnowhereConfig::SetEarlyStopThreshold(0);
eg::KnowhereConfig::SetLogHandler();
eg::KnowhereConfig::SetStatisticsLevel(0);
knowhere::KnowhereConfig::SetBlasThreshold(16384);
knowhere::KnowhereConfig::SetEarlyStopThreshold(0);
knowhere::KnowhereConfig::SetLogHandler();
knowhere::KnowhereConfig::SetStatisticsLevel(0);
el::Configurations el_conf;
el_conf.setGlobally(el::ConfigurationType::Enabled, std::to_string(false));
};
@ -44,25 +42,24 @@ KnowhereInitImpl() {
std::string
KnowhereSetSimdType(const char* value) {
milvus::engine::KnowhereConfig::SimdType simd_type;
knowhere::KnowhereConfig::SimdType simd_type;
if (strcmp(value, "auto") == 0) {
simd_type = milvus::engine::KnowhereConfig::SimdType::AUTO;
simd_type = knowhere::KnowhereConfig::SimdType::AUTO;
} else if (strcmp(value, "avx512") == 0) {
simd_type = milvus::engine::KnowhereConfig::SimdType::AVX512;
simd_type = knowhere::KnowhereConfig::SimdType::AVX512;
} else if (strcmp(value, "avx2") == 0) {
simd_type = milvus::engine::KnowhereConfig::SimdType::AVX2;
simd_type = knowhere::KnowhereConfig::SimdType::AVX2;
} else if (strcmp(value, "avx") == 0 || strcmp(value, "sse4_2") == 0) {
simd_type = milvus::engine::KnowhereConfig::SimdType::SSE4_2;
simd_type = knowhere::KnowhereConfig::SimdType::SSE4_2;
} else {
PanicInfo("invalid SIMD type: " + std::string(value));
}
try {
return milvus::engine::KnowhereConfig::SetSimdType(simd_type);
return knowhere::KnowhereConfig::SetSimdType(simd_type);
} catch (std::exception& e) {
LOG_SERVER_ERROR_ << e.what();
PanicInfo(e.what());
}
}
} // namespace config
} // namespace milvus
} // namespace milvus::config

2
internal/core/src/index/Index.h
View File

@ -17,7 +17,7 @@
#include <knowhere/index/structured_index_simple/StructuredIndex.h>
namespace milvus::scalar {
using Index = milvus::knowhere::Index;
using Index = knowhere::Index;
using IndexPtr = std::unique_ptr<Index>;
using BinarySet = knowhere::BinarySet;
using Config = knowhere::Config;

4
internal/core/src/indexbuilder/ScalarIndexCreator-inl.h
View File

@ -52,7 +52,7 @@ ScalarIndexCreator<T>::Load(const knowhere::BinarySet& binary_set) {
// not sure that the pointer of a golang bool array acts like other types.
template <>
inline void
ScalarIndexCreator<bool>::Build(const milvus::knowhere::DatasetPtr& dataset) {
ScalarIndexCreator<bool>::Build(const knowhere::DatasetPtr& dataset) {
auto size = dataset->Get<int64_t>(knowhere::meta::ROWS);
auto data = dataset->Get<const void*>(knowhere::meta::TENSOR);
proto::schema::BoolArray arr;
@ -71,7 +71,7 @@ inline ScalarIndexCreator<std::string>::ScalarIndexCreator(const char* type_para
template <>
inline void
ScalarIndexCreator<std::string>::Build(const milvus::knowhere::DatasetPtr& dataset) {
ScalarIndexCreator<std::string>::Build(const knowhere::DatasetPtr& dataset) {
auto size = dataset->Get<int64_t>(knowhere::meta::ROWS);
auto data = dataset->Get<const void*>(knowhere::meta::TENSOR);
proto::schema::StringArray arr;

2
internal/core/src/indexbuilder/ScalarIndexCreator.h
View File

@ -42,7 +42,7 @@ class ScalarIndexCreator : public IndexCreatorBase {
std::unique_ptr<knowhere::scalar::StructuredIndex<T>> index_ = nullptr;
proto::indexcgo::TypeParams type_params_;
proto::indexcgo::IndexParams index_params_;
milvus::json config_;
knowhere::Config config_;
};
} // namespace milvus::indexbuilder

74
internal/core/src/indexbuilder/VecIndexCreator.cpp
View File

@ -65,50 +65,50 @@ VecIndexCreator::parse_impl(const std::string& serialized_params_str, knowhere::
};
/***************************** meta *******************************/
check_parameter<int>(conf, milvus::knowhere::meta::DIM, stoi_closure, std::nullopt);
check_parameter<int>(conf, milvus::knowhere::meta::TOPK, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::meta::DIM, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::meta::TOPK, stoi_closure, std::nullopt);
/***************************** IVF Params *******************************/
check_parameter<int>(conf, milvus::knowhere::IndexParams::nprobe, stoi_closure, std::nullopt);
check_parameter<int>(conf, milvus::knowhere::IndexParams::nlist, stoi_closure, std::nullopt);
check_parameter<int>(conf, milvus::knowhere::IndexParams::m, stoi_closure, std::nullopt);
check_parameter<int>(conf, milvus::knowhere::IndexParams::nbits, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::nprobe, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::nlist, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::m, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::nbits, stoi_closure, std::nullopt);
/************************** NSG Parameter **************************/
check_parameter<int>(conf, milvus::knowhere::IndexParams::knng, stoi_closure, std::nullopt);
check_parameter<int>(conf, milvus::knowhere::IndexParams::search_length, stoi_closure, std::nullopt);
check_parameter<int>(conf, milvus::knowhere::IndexParams::out_degree, stoi_closure, std::nullopt);
check_parameter<int>(conf, milvus::knowhere::IndexParams::candidate, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::knng, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::search_length, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::out_degree, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::candidate, stoi_closure, std::nullopt);
/************************** HNSW Params *****************************/
check_parameter<int>(conf, milvus::knowhere::IndexParams::efConstruction, stoi_closure, std::nullopt);
check_parameter<int>(conf, milvus::knowhere::IndexParams::M, stoi_closure, std::nullopt);
check_parameter<int>(conf, milvus::knowhere::IndexParams::ef, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::efConstruction, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::M, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::ef, stoi_closure, std::nullopt);
/************************** Annoy Params *****************************/
check_parameter<int>(conf, milvus::knowhere::IndexParams::n_trees, stoi_closure, std::nullopt);
check_parameter<int>(conf, milvus::knowhere::IndexParams::search_k, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::n_trees, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::search_k, stoi_closure, std::nullopt);
/************************** PQ Params *****************************/
check_parameter<int>(conf, milvus::knowhere::IndexParams::PQM, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::PQM, stoi_closure, std::nullopt);
/************************** NGT Params *****************************/
check_parameter<int>(conf, milvus::knowhere::IndexParams::edge_size, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::edge_size, stoi_closure, std::nullopt);
/************************** NGT Search Params *****************************/
check_parameter<float>(conf, milvus::knowhere::IndexParams::epsilon, stof_closure, std::nullopt);
check_parameter<int>(conf, milvus::knowhere::IndexParams::max_search_edges, stoi_closure, std::nullopt);
check_parameter<float>(conf, knowhere::IndexParams::epsilon, stof_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::max_search_edges, stoi_closure, std::nullopt);
/************************** NGT_PANNG Params *****************************/
check_parameter<int>(conf, milvus::knowhere::IndexParams::forcedly_pruned_edge_size, stoi_closure, std::nullopt);
check_parameter<int>(conf, milvus::knowhere::IndexParams::selectively_pruned_edge_size, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::forcedly_pruned_edge_size, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::selectively_pruned_edge_size, stoi_closure, std::nullopt);
/************************** NGT_ONNG Params *****************************/
check_parameter<int>(conf, milvus::knowhere::IndexParams::outgoing_edge_size, stoi_closure, std::nullopt);
check_parameter<int>(conf, milvus::knowhere::IndexParams::incoming_edge_size, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::outgoing_edge_size, stoi_closure, std::nullopt);
check_parameter<int>(conf, knowhere::IndexParams::incoming_edge_size, stoi_closure, std::nullopt);
/************************** Serialize Params *******************************/
check_parameter<int>(conf, milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, stoi_closure, std::optional{4});
check_parameter<int>(conf, knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, stoi_closure, std::optional{4});
}
void
@ -149,7 +149,7 @@ VecIndexCreator::get_config_by_name(std::string name) {
int64_t
VecIndexCreator::dim() {
auto dimension = get_config_by_name<int64_t>(milvus::knowhere::meta::DIM);
auto dimension = get_config_by_name<int64_t>(knowhere::meta::DIM);
AssertInfo(dimension.has_value(), "[VecIndexCreator]Dimension doesn't have value");
return (dimension.value());
}
@ -190,7 +190,7 @@ VecIndexCreator::BuildWithoutIds(const knowhere::DatasetPtr& dataset) {
void
VecIndexCreator::BuildWithIds(const knowhere::DatasetPtr& dataset) {
AssertInfo(dataset->data().find(milvus::knowhere::meta::IDS) != dataset->data().end(),
AssertInfo(dataset->data().find(knowhere::meta::IDS) != dataset->data().end(),
"[VecIndexCreator]Can't find ids field in dataset");
auto index_type = get_index_type();
auto index_mode = get_index_mode();
@ -215,9 +215,9 @@ void
VecIndexCreator::StoreRawData(const knowhere::DatasetPtr& dataset) {
auto index_type = get_index_type();
if (is_in_nm_list(index_type)) {
auto tensor = dataset->Get<const void*>(milvus::knowhere::meta::TENSOR);
auto row_num = dataset->Get<int64_t>(milvus::knowhere::meta::ROWS);
auto dim = dataset->Get<int64_t>(milvus::knowhere::meta::DIM);
auto tensor = dataset->Get<const void*>(knowhere::meta::TENSOR);
auto row_num = dataset->Get<int64_t>(knowhere::meta::ROWS);
auto dim = dataset->Get<int64_t>(knowhere::meta::DIM);
int64_t data_size;
if (is_in_bin_list(index_type)) {
data_size = dim / 8 * row_num;
@ -229,7 +229,7 @@ VecIndexCreator::StoreRawData(const knowhere::DatasetPtr& dataset) {
}
}
milvus::knowhere::BinarySet
knowhere::BinarySet
VecIndexCreator::Serialize() {
auto ret = index_->Serialize(config_);
auto index_type = get_index_type();
@ -247,7 +247,7 @@ VecIndexCreator::Serialize() {
}
void
VecIndexCreator::Load(const milvus::knowhere::BinarySet& binary_set) {
VecIndexCreator::Load(const knowhere::BinarySet& binary_set) {
auto& map_ = binary_set.binary_map_;
for (auto it = map_.begin(); it != map_.end(); ++it) {
if (it->first == RAW_DATA) {
@ -311,7 +311,7 @@ VecIndexCreator::Query(const knowhere::DatasetPtr& dataset) {
std::unique_ptr<VecIndexCreator::QueryResult>
VecIndexCreator::QueryWithParam(const knowhere::DatasetPtr& dataset, const char* serialized_search_params) {
namespace indexcgo = milvus::proto::indexcgo;
milvus::knowhere::Config search_conf;
knowhere::Config search_conf;
parse_impl<indexcgo::MapParams>(std::string(serialized_search_params), search_conf);
return std::move(QueryImpl(dataset, search_conf));
@ -326,10 +326,10 @@ VecIndexCreator::QueryImpl(const knowhere::DatasetPtr& dataset, const knowhere::
}
auto res = index_->Query(dataset, conf, nullptr);
auto ids = res->Get<int64_t*>(milvus::knowhere::meta::IDS);
auto distances = res->Get<float*>(milvus::knowhere::meta::DISTANCE);
auto nq = dataset->Get<int64_t>(milvus::knowhere::meta::ROWS);
auto k = config_[milvus::knowhere::meta::TOPK].get<int64_t>();
auto ids = res->Get<int64_t*>(knowhere::meta::IDS);
auto distances = res->Get<float*>(knowhere::meta::DISTANCE);
auto nq = dataset->Get<int64_t>(knowhere::meta::ROWS);
auto k = config_[knowhere::meta::TOPK].get<int64_t>();
auto query_res = std::make_unique<VecIndexCreator::QueryResult>();
query_res->nq = nq;
@ -347,7 +347,7 @@ VecIndexCreator::LoadRawData() {
auto index_type = get_index_type();
if (is_in_nm_list(index_type)) {
auto bs = index_->Serialize(config_);
auto bptr = std::make_shared<milvus::knowhere::Binary>();
auto bptr = std::make_shared<knowhere::Binary>();
auto deleter = [&](uint8_t*) {}; // avoid repeated deconstruction
bptr->data = std::shared_ptr<uint8_t[]>(static_cast<uint8_t*>(raw_data_.data()), deleter);
bptr->size = raw_data_.size();

6
internal/core/src/indexbuilder/VecIndexCreator.h
View File

@ -44,7 +44,7 @@ class VecIndexCreator : public IndexCreatorBase {
public:
// used for tests
struct QueryResult {
std::vector<milvus::knowhere::IDType> ids;
std::vector<knowhere::IDType> ids;
std::vector<float> distances;
int64_t nq;
int64_t topk;
@ -107,8 +107,8 @@ class VecIndexCreator : public IndexCreatorBase {
knowhere::VecIndexPtr index_ = nullptr;
std::string type_params_;
std::string index_params_;
milvus::json type_config_;
milvus::json index_config_;
knowhere::Config type_config_;
knowhere::Config index_config_;
knowhere::Config config_;
std::vector<uint8_t> raw_data_;
std::once_flag raw_data_loaded_;

18
internal/core/src/indexbuilder/index_c.cpp
View File

@ -73,7 +73,7 @@ BuildFloatVecIndex(CIndex index, int64_t float_value_num, const float* vectors)
auto cIndex = dynamic_cast<milvus::indexbuilder::VecIndexCreator*>(real_index);
auto dim = cIndex->dim();
auto row_nums = float_value_num / dim;
auto ds = milvus::knowhere::GenDataset(row_nums, dim, vectors);
auto ds = knowhere::GenDataset(row_nums, dim, vectors);
cIndex->Build(ds);
status.error_code = Success;
status.error_msg = "";
@ -93,7 +93,7 @@ BuildBinaryVecIndex(CIndex index, int64_t data_size, const uint8_t* vectors) {
auto cIndex = dynamic_cast<milvus::indexbuilder::VecIndexCreator*>(real_index);
auto dim = cIndex->dim();
auto row_nums = (data_size * 8) / dim;
auto ds = milvus::knowhere::GenDataset(row_nums, dim, vectors);
auto ds = knowhere::GenDataset(row_nums, dim, vectors);
cIndex->Build(ds);
status.error_code = Success;
status.error_msg = "";
@ -117,7 +117,7 @@ BuildScalarIndex(CIndex c_index, int64_t size, const void* field_data) {
auto real_index = reinterpret_cast<milvus::indexbuilder::IndexCreatorBase*>(c_index);
const int64_t dim = 8; // not important here
auto dataset = milvus::knowhere::GenDataset(size, dim, field_data);
auto dataset = knowhere::GenDataset(size, dim, field_data);
real_index->Build(dataset);
status.error_code = Success;
@ -135,7 +135,7 @@ SerializeIndexToBinarySet(CIndex index, CBinarySet* c_binary_set) {
try {
AssertInfo(index, "failed to serialize index to binary set, passed index was null");
auto real_index = reinterpret_cast<milvus::indexbuilder::IndexCreatorBase*>(index);
auto binary = std::make_unique<milvus::knowhere::BinarySet>(real_index->Serialize());
auto binary = std::make_unique<knowhere::BinarySet>(real_index->Serialize());
*c_binary_set = binary.release();
status.error_code = Success;
status.error_msg = "";
@ -152,7 +152,7 @@ LoadIndexFromBinarySet(CIndex index, CBinarySet c_binary_set) {
try {
AssertInfo(index, "failed to load index from binary set, passed index was null");
auto real_index = reinterpret_cast<milvus::indexbuilder::IndexCreatorBase*>(index);
auto binary_set = reinterpret_cast<milvus::knowhere::BinarySet*>(c_binary_set);
auto binary_set = reinterpret_cast<knowhere::BinarySet*>(c_binary_set);
real_index->Load(*binary_set);
status.error_code = Success;
status.error_msg = "";
@ -170,7 +170,7 @@ QueryOnFloatVecIndex(CIndex index, int64_t float_value_num, const float* vectors
auto cIndex = (milvus::indexbuilder::VecIndexCreator*)index;
auto dim = cIndex->dim();
auto row_nums = float_value_num / dim;
auto query_ds = milvus::knowhere::GenDataset(row_nums, dim, vectors);
auto query_ds = knowhere::GenDataset(row_nums, dim, vectors);
auto query_res = cIndex->Query(query_ds);
*res = query_res.release();
@ -194,7 +194,7 @@ QueryOnFloatVecIndexWithParam(CIndex index,
auto cIndex = (milvus::indexbuilder::VecIndexCreator*)index;
auto dim = cIndex->dim();
auto row_nums = float_value_num / dim;
auto query_ds = milvus::knowhere::GenDataset(row_nums, dim, vectors);
auto query_ds = knowhere::GenDataset(row_nums, dim, vectors);
auto query_res = cIndex->QueryWithParam(query_ds, serialized_search_params);
*res = query_res.release();
@ -214,7 +214,7 @@ QueryOnBinaryVecIndex(CIndex index, int64_t data_size, const uint8_t* vectors, C
auto cIndex = (milvus::indexbuilder::VecIndexCreator*)index;
auto dim = cIndex->dim();
auto row_nums = (data_size * 8) / dim;
auto query_ds = milvus::knowhere::GenDataset(row_nums, dim, vectors);
auto query_ds = knowhere::GenDataset(row_nums, dim, vectors);
auto query_res = cIndex->Query(query_ds);
*res = query_res.release();
@ -238,7 +238,7 @@ QueryOnBinaryVecIndexWithParam(CIndex index,
auto cIndex = (milvus::indexbuilder::VecIndexCreator*)index;
auto dim = cIndex->dim();
auto row_nums = (data_size * 8) / dim;
auto query_ds = milvus::knowhere::GenDataset(row_nums, dim, vectors);
auto query_ds = knowhere::GenDataset(row_nums, dim, vectors);
auto query_res = cIndex->QueryWithParam(query_ds, serialized_search_params);
*res = query_res.release();

26
internal/core/src/indexbuilder/utils.cpp
View File

@ -24,9 +24,9 @@ namespace milvus::indexbuilder {
std::vector<std::string>
NM_List() {
static std::vector<std::string> ret{
milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT,
milvus::knowhere::IndexEnum::INDEX_NSG,
milvus::knowhere::IndexEnum::INDEX_RHNSWFlat,
knowhere::IndexEnum::INDEX_FAISS_IVFFLAT,
knowhere::IndexEnum::INDEX_NSG,
knowhere::IndexEnum::INDEX_RHNSWFlat,
};
return ret;
}
@ -34,8 +34,8 @@ NM_List() {
std::vector<std::string>
BIN_List() {
static std::vector<std::string> ret{
milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP,
milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT,
knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP,
knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT,
};
return ret;
}
@ -43,8 +43,8 @@ BIN_List() {
std::vector<std::string>
Need_ID_List() {
static std::vector<std::string> ret{
// milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT,
// milvus::knowhere::IndexEnum::INDEX_NSG,
// knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT,
// knowhere::IndexEnum::INDEX_NSG,
};
return ret;
@ -53,7 +53,7 @@ Need_ID_List() {
std::vector<std::string>
Need_BuildAll_list() {
static std::vector<std::string> ret{
milvus::knowhere::IndexEnum::INDEX_NSG,
knowhere::IndexEnum::INDEX_NSG,
};
return ret;
}
@ -74,27 +74,27 @@ is_in_list(const T& t, std::function<std::vector<T>()> list_func) {
}
bool
is_in_bin_list(const milvus::knowhere::IndexType& index_type) {
is_in_bin_list(const knowhere::IndexType& index_type) {
return is_in_list<std::string>(index_type, BIN_List);
}
bool
is_in_nm_list(const milvus::knowhere::IndexType& index_type) {
is_in_nm_list(const knowhere::IndexType& index_type) {
return is_in_list<std::string>(index_type, NM_List);
}
bool
is_in_need_build_all_list(const milvus::knowhere::IndexType& index_type) {
is_in_need_build_all_list(const knowhere::IndexType& index_type) {
return is_in_list<std::string>(index_type, Need_BuildAll_list);
}
bool
is_in_need_id_list(const milvus::knowhere::IndexType& index_type) {
is_in_need_id_list(const knowhere::IndexType& index_type) {
return is_in_list<std::string>(index_type, Need_ID_List);
}
bool
is_unsupported(const milvus::knowhere::IndexType& index_type, const milvus::knowhere::MetricType& metric_type) {
is_unsupported(const knowhere::IndexType& index_type, const knowhere::MetricType& metric_type) {
return is_in_list<std::tuple<std::string, std::string>>(std::make_tuple(index_type, metric_type),
unsupported_index_combinations);
}

10
internal/core/src/indexbuilder/utils.h
View File

@ -42,18 +42,18 @@ bool
is_in_list(const T& t, std::function<std::vector<T>()> list_func);
bool
is_in_bin_list(const milvus::knowhere::IndexType& index_type);
is_in_bin_list(const knowhere::IndexType& index_type);
bool
is_in_nm_list(const milvus::knowhere::IndexType& index_type);
is_in_nm_list(const knowhere::IndexType& index_type);
bool
is_in_need_build_all_list(const milvus::knowhere::IndexType& index_type);
is_in_need_build_all_list(const knowhere::IndexType& index_type);
bool
is_in_need_id_list(const milvus::knowhere::IndexType& index_type);
is_in_need_id_list(const knowhere::IndexType& index_type);
bool
is_unsupported(const milvus::knowhere::IndexType& index_type, const milvus::knowhere::MetricType& metric_type);
is_unsupported(const knowhere::IndexType& index_type, const knowhere::MetricType& metric_type);
} // namespace milvus::indexbuilder

2
internal/core/src/query/SearchBruteForce.cpp
View File

@ -124,7 +124,7 @@ FloatSearchBruteForce(const dataset::SearchDataset& dataset,
#elif defined(__linux__) || defined(__MINGW64__)
if (metric_type == MetricType::METRIC_L2) {
faiss::float_maxheap_array_t buf{(size_t)num_queries, (size_t)topk, sub_qr.get_ids(), sub_qr.get_distances()};
faiss::knn_L2sqr(query_data, chunk_data, dim, num_queries, size_per_chunk, &buf, bitset);
faiss::knn_L2sqr(query_data, chunk_data, dim, num_queries, size_per_chunk, &buf, nullptr, bitset);
} else {
faiss::float_minheap_array_t buf{(size_t)num_queries, (size_t)topk, sub_qr.get_ids(), sub_qr.get_distances()};
faiss::knn_inner_product(query_data, chunk_data, dim, num_queries, size_per_chunk, &buf, bitset);

4
internal/core/src/query/SearchOnIndex.cpp
View File

@ -30,8 +30,8 @@ SearchOnIndex(const dataset::SearchDataset& search_dataset,
auto& indexing_nonconst = const_cast<knowhere::VecIndex&>(indexing);
auto ans = indexing_nonconst.Query(dataset, search_conf, bitset);
auto dis = ans->Get<float*>(milvus::knowhere::meta::DISTANCE);
auto uids = ans->Get<int64_t*>(milvus::knowhere::meta::IDS);
auto dis = ans->Get<float*>(knowhere::meta::DISTANCE);
auto uids = ans->Get<int64_t*>(knowhere::meta::IDS);
SubSearchResult sub_qr(num_queries, topK, metric_type, round_decimal);
std::copy_n(dis, num_queries * topK, sub_qr.get_distances());

6
internal/core/src/query/SearchOnSealed.cpp
View File

@ -83,10 +83,10 @@ SearchOnSealed(const Schema& schema,
auto ds = knowhere::GenDataset(num_queries, dim, query_data);
auto conf = search_info.search_params_;
conf[milvus::knowhere::meta::TOPK] = search_info.topk_;
conf[milvus::knowhere::Metric::TYPE] = MetricTypeToName(field_indexing->metric_type_);
conf[knowhere::meta::TOPK] = search_info.topk_;
conf[knowhere::Metric::TYPE] = MetricTypeToName(field_indexing->metric_type_);
auto index_type = field_indexing->indexing_->index_type();
auto adapter = milvus::knowhere::AdapterMgr::GetInstance().GetAdapter(index_type);
auto adapter = knowhere::AdapterMgr::GetInstance().GetAdapter(index_type);
AssertInfo(adapter->CheckSearch(conf, index_type, field_indexing->indexing_->index_mode()),
"[SearchOnSealed]Search params check failed");
return field_indexing->indexing_->Query(ds, conf, bitset);

11
internal/core/src/segcore/load_index_c.cpp
View File

@ -80,20 +80,19 @@ CStatus
AppendIndex(CLoadIndexInfo c_load_index_info, CBinarySet c_binary_set) {
try {
auto load_index_info = (LoadIndexInfo*)c_load_index_info;
auto binary_set = (milvus::knowhere::BinarySet*)c_binary_set;
auto binary_set = (knowhere::BinarySet*)c_binary_set;
auto& index_params = load_index_info->index_params;
bool find_index_type = index_params.count("index_type") > 0 ? true : false;
bool find_index_mode = index_params.count("index_mode") > 0 ? true : false;
AssertInfo(find_index_type == true, "Can't find index type in index_params");
milvus::knowhere::IndexMode mode;
knowhere::IndexMode mode;
if (find_index_mode) {
mode = index_params["index_mode"] == "CPU" ? milvus::knowhere::IndexMode::MODE_CPU
: milvus::knowhere::IndexMode::MODE_GPU;
mode = index_params["index_mode"] == "CPU" ? knowhere::IndexMode::MODE_CPU : knowhere::IndexMode::MODE_GPU;
} else {
mode = milvus::knowhere::IndexMode::MODE_CPU;
mode = knowhere::IndexMode::MODE_CPU;
}
load_index_info->index =
milvus::knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_params["index_type"], mode);
knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_params["index_type"], mode);
load_index_info->index->Load(*binary_set);
auto status = CStatus();
status.error_code = Success;

6
internal/core/thirdparty/knowhere/CMakeLists.txt vendored
View File

@ -11,8 +11,8 @@
# or implied. See the License for the specific language governing permissions and limitations under the License.
#-------------------------------------------------------------------------------
set( KNOWHERE_VERSION v1.0.1 )
set( KNOWHERE_SOURCE_MD5 "0ac8a0802f94c2370f13a40527c57ced" )
set( KNOWHERE_VERSION v1.1.0 )
set( KNOWHERE_SOURCE_MD5 "9c326d3ba94184f4f646903f1ad36d49" )
if ( DEFINED ENV{MILVUS_KNOWHERE_URL} )
set( KNOWHERE_SOURCE_URL "$ENV{MILVUS_KNOWHERE_URL}" )
@ -34,7 +34,7 @@ macro(build_knowhere)
set(KNOWHERE_CMAKE_ARGS
${EP_COMMON_CMAKE_ARGS}
-DCMAKE_BUILD_TYPE={BUILD_TYPE}
-DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_PREFIX=${KNOWHERE_INSTALL_PREFIX}
)

332
internal/core/unittest/test_c_api.cpp
View File

@ -34,7 +34,7 @@ namespace chrono = std::chrono;
using namespace milvus;
using namespace milvus::segcore;
using namespace milvus::knowhere;
using namespace knowhere;
namespace {
const int DIM = 16;
@ -187,10 +187,10 @@ generate_collection_schema(std::string metric_type, int dim, bool is_binary) {
VecIndexPtr
generate_index(
void* raw_data, milvus::knowhere::Config conf, int64_t dim, int64_t topK, int64_t N, std::string index_type) {
void* raw_data, knowhere::Config conf, int64_t dim, int64_t topK, int64_t N, std::string index_type) {
auto indexing = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type, knowhere::IndexMode::MODE_CPU);
auto database = milvus::knowhere::GenDataset(N, dim, raw_data);
auto database = knowhere::GenDataset(N, dim, raw_data);
indexing->Train(database, conf);
indexing->AddWithoutIds(database, conf);
EXPECT_EQ(indexing->Count(), N);
@ -1005,17 +1005,17 @@ TEST(CApiTest, LoadIndexInfo) {
auto N = 1024 * 10;
auto [raw_data, timestamps, uids] = generate_data(N);
auto indexing = std::make_shared<milvus::knowhere::IVFPQ>();
auto conf = milvus::knowhere::Config{{milvus::knowhere::meta::DIM, DIM},
{milvus::knowhere::meta::TOPK, TOPK},
{milvus::knowhere::IndexParams::nlist, 100},
{milvus::knowhere::IndexParams::nprobe, 4},
{milvus::knowhere::IndexParams::m, 4},
{milvus::knowhere::IndexParams::nbits, 8},
{milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::L2},
{milvus::knowhere::meta::DEVICEID, 0}};
auto indexing = std::make_shared<knowhere::IVFPQ>();
auto conf = knowhere::Config{{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::nprobe, 4},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto database = milvus::knowhere::GenDataset(N, DIM, raw_data.data());
auto database = knowhere::GenDataset(N, DIM, raw_data.data());
indexing->Train(database, conf);
indexing->AddWithoutIds(database, conf);
EXPECT_EQ(indexing->Count(), N);
@ -1048,17 +1048,17 @@ TEST(CApiTest, LoadIndex_Search) {
auto N = 1024 * 1024;
auto num_query = 100;
auto [raw_data, timestamps, uids] = generate_data(N);
auto indexing = std::make_shared<milvus::knowhere::IVFPQ>();
auto conf = milvus::knowhere::Config{{milvus::knowhere::meta::DIM, DIM},
{milvus::knowhere::meta::TOPK, TOPK},
{milvus::knowhere::IndexParams::nlist, 100},
{milvus::knowhere::IndexParams::nprobe, 4},
{milvus::knowhere::IndexParams::m, 4},
{milvus::knowhere::IndexParams::nbits, 8},
{milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::L2},
{milvus::knowhere::meta::DEVICEID, 0}};
auto indexing = std::make_shared<knowhere::IVFPQ>();
auto conf = knowhere::Config{{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::nprobe, 4},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto database = milvus::knowhere::GenDataset(N, DIM, raw_data.data());
auto database = knowhere::GenDataset(N, DIM, raw_data.data());
indexing->Train(database, conf);
indexing->AddWithoutIds(database, conf);
@ -1073,18 +1073,18 @@ TEST(CApiTest, LoadIndex_Search) {
auto& index_params = load_index_info.index_params;
index_params["index_type"] = "IVF_PQ";
index_params["index_mode"] = "CPU";
auto mode = milvus::knowhere::IndexMode::MODE_CPU;
auto mode = knowhere::IndexMode::MODE_CPU;
load_index_info.index =
milvus::knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_params["index_type"], mode);
knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_params["index_type"], mode);
load_index_info.index->Load(binary_set);
// search
auto query_dataset = milvus::knowhere::GenDataset(num_query, DIM, raw_data.data() + DIM * 4200);
auto query_dataset = knowhere::GenDataset(num_query, DIM, raw_data.data() + DIM * 4200);
auto result = indexing->Query(query_dataset, conf, nullptr);
auto ids = result->Get<int64_t*>(milvus::knowhere::meta::IDS);
auto dis = result->Get<float*>(milvus::knowhere::meta::DISTANCE);
auto ids = result->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result->Get<float*>(knowhere::meta::DISTANCE);
// for (int i = 0; i < std::min(num_query * K, 100); ++i) {
// std::cout << ids[i] << "->" << dis[i] << std::endl;
//}
@ -1150,21 +1150,21 @@ TEST(CApiTest, Indexing_Without_Predicate) {
assert(res_before_load_index.error_code == Success);
// load index to segment
auto conf = milvus::knowhere::Config{{milvus::knowhere::meta::DIM, DIM},
{milvus::knowhere::meta::TOPK, TOPK},
{milvus::knowhere::IndexParams::nlist, 100},
{milvus::knowhere::IndexParams::nprobe, 10},
{milvus::knowhere::IndexParams::m, 4},
{milvus::knowhere::IndexParams::nbits, 8},
{milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::L2},
{milvus::knowhere::meta::DEVICEID, 0}};
auto conf = knowhere::Config{{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::nprobe, 10},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ);
// gen query dataset
auto query_dataset = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr);
auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr);
auto result_on_index = indexing->Query(query_dataset, conf, nullptr);
auto ids = result_on_index->Get<int64_t*>(milvus::knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(milvus::knowhere::meta::DISTANCE);
auto ids = result_on_index->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(knowhere::meta::DISTANCE);
std::vector<int64_t> vec_ids(ids, ids + TOPK * num_queries);
std::vector<float> vec_dis;
for (int j = 0; j < TOPK * num_queries; ++j) {
@ -1269,21 +1269,21 @@ TEST(CApiTest, Indexing_Expr_Without_Predicate) {
assert(res_before_load_index.error_code == Success);
// load index to segment
auto conf = milvus::knowhere::Config{{milvus::knowhere::meta::DIM, DIM},
{milvus::knowhere::meta::TOPK, TOPK},
{milvus::knowhere::IndexParams::nlist, 100},
{milvus::knowhere::IndexParams::nprobe, 10},
{milvus::knowhere::IndexParams::m, 4},
{milvus::knowhere::IndexParams::nbits, 8},
{milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::L2},
{milvus::knowhere::meta::DEVICEID, 0}};
auto conf = knowhere::Config{{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::nprobe, 10},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ);
// gen query dataset
auto query_dataset = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr);
auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr);
auto result_on_index = indexing->Query(query_dataset, conf, nullptr);
auto ids = result_on_index->Get<int64_t*>(milvus::knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(milvus::knowhere::meta::DISTANCE);
auto ids = result_on_index->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(knowhere::meta::DISTANCE);
std::vector<int64_t> vec_ids(ids, ids + TOPK * num_queries);
std::vector<float> vec_dis;
for (int j = 0; j < TOPK * num_queries; ++j) {
@ -1405,22 +1405,22 @@ TEST(CApiTest, Indexing_With_float_Predicate_Range) {
assert(res_before_load_index.error_code == Success);
// load index to segment
auto conf = milvus::knowhere::Config{{milvus::knowhere::meta::DIM, DIM},
{milvus::knowhere::meta::TOPK, TOPK},
{milvus::knowhere::IndexParams::nlist, 100},
{milvus::knowhere::IndexParams::nprobe, 10},
{milvus::knowhere::IndexParams::m, 4},
{milvus::knowhere::IndexParams::nbits, 8},
{milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::L2},
{milvus::knowhere::meta::DEVICEID, 0}};
auto conf = knowhere::Config{{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::nprobe, 10},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ);
// gen query dataset
auto query_dataset = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr);
auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr);
auto result_on_index = indexing->Query(query_dataset, conf, nullptr);
auto ids = result_on_index->Get<int64_t*>(milvus::knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(milvus::knowhere::meta::DISTANCE);
auto ids = result_on_index->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(knowhere::meta::DISTANCE);
std::vector<int64_t> vec_ids(ids, ids + TOPK * num_queries);
std::vector<float> vec_dis;
for (int j = 0; j < TOPK * num_queries; ++j) {
@ -1556,22 +1556,22 @@ TEST(CApiTest, Indexing_Expr_With_float_Predicate_Range) {
assert(res_before_load_index.error_code == Success);
// load index to segment
auto conf = milvus::knowhere::Config{{milvus::knowhere::meta::DIM, DIM},
{milvus::knowhere::meta::TOPK, TOPK},
{milvus::knowhere::IndexParams::nlist, 100},
{milvus::knowhere::IndexParams::nprobe, 10},
{milvus::knowhere::IndexParams::m, 4},
{milvus::knowhere::IndexParams::nbits, 8},
{milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::L2},
{milvus::knowhere::meta::DEVICEID, 0}};
auto conf = knowhere::Config{{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::nprobe, 10},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ);
// gen query dataset
auto query_dataset = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr);
auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr);
auto result_on_index = indexing->Query(query_dataset, conf, nullptr);
auto ids = result_on_index->Get<int64_t*>(milvus::knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(milvus::knowhere::meta::DISTANCE);
auto ids = result_on_index->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(knowhere::meta::DISTANCE);
std::vector<int64_t> vec_ids(ids, ids + TOPK * num_queries);
std::vector<float> vec_dis;
for (int j = 0; j < TOPK * num_queries; ++j) {
@ -1691,22 +1691,22 @@ TEST(CApiTest, Indexing_With_float_Predicate_Term) {
assert(res_before_load_index.error_code == Success);
// load index to segment
auto conf = milvus::knowhere::Config{{milvus::knowhere::meta::DIM, DIM},
{milvus::knowhere::meta::TOPK, TOPK},
{milvus::knowhere::IndexParams::nlist, 100},
{milvus::knowhere::IndexParams::nprobe, 10},
{milvus::knowhere::IndexParams::m, 4},
{milvus::knowhere::IndexParams::nbits, 8},
{milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::L2},
{milvus::knowhere::meta::DEVICEID, 0}};
auto conf = knowhere::Config{{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::nprobe, 10},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ);
// gen query dataset
auto query_dataset = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr);
auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr);
auto result_on_index = indexing->Query(query_dataset, conf, nullptr);
auto ids = result_on_index->Get<int64_t*>(milvus::knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(milvus::knowhere::meta::DISTANCE);
auto ids = result_on_index->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(knowhere::meta::DISTANCE);
std::vector<int64_t> vec_ids(ids, ids + TOPK * num_queries);
std::vector<float> vec_dis;
for (int j = 0; j < TOPK * num_queries; ++j) {
@ -1835,22 +1835,22 @@ TEST(CApiTest, Indexing_Expr_With_float_Predicate_Term) {
assert(res_before_load_index.error_code == Success);
// load index to segment
auto conf = milvus::knowhere::Config{{milvus::knowhere::meta::DIM, DIM},
{milvus::knowhere::meta::TOPK, TOPK},
{milvus::knowhere::IndexParams::nlist, 100},
{milvus::knowhere::IndexParams::nprobe, 10},
{milvus::knowhere::IndexParams::m, 4},
{milvus::knowhere::IndexParams::nbits, 8},
{milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::L2},
{milvus::knowhere::meta::DEVICEID, 0}};
auto conf = knowhere::Config{{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::nprobe, 10},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ);
// gen query dataset
auto query_dataset = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr);
auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr);
auto result_on_index = indexing->Query(query_dataset, conf, nullptr);
auto ids = result_on_index->Get<int64_t*>(milvus::knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(milvus::knowhere::meta::DISTANCE);
auto ids = result_on_index->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(knowhere::meta::DISTANCE);
std::vector<int64_t> vec_ids(ids, ids + TOPK * num_queries);
std::vector<float> vec_dis;
for (int j = 0; j < TOPK * num_queries; ++j) {
@ -1971,23 +1971,23 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Range) {
assert(res_before_load_index.error_code == Success);
// load index to segment
auto conf = milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
{milvus::knowhere::meta::TOPK, TOPK},
{milvus::knowhere::IndexParams::nprobe, 10},
{milvus::knowhere::IndexParams::nlist, 100},
{milvus::knowhere::IndexParams::m, 4},
{milvus::knowhere::IndexParams::nbits, 8},
{milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::JACCARD},
auto conf = knowhere::Config{
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nprobe, 10},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, knowhere::Metric::JACCARD},
};
auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_BIN_IVFFLAT);
// gen query dataset
auto query_dataset = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr);
auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr);
auto result_on_index = indexing->Query(query_dataset, conf, nullptr);
auto ids = result_on_index->Get<int64_t*>(milvus::knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(milvus::knowhere::meta::DISTANCE);
auto ids = result_on_index->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(knowhere::meta::DISTANCE);
std::vector<int64_t> vec_ids(ids, ids + TOPK * num_queries);
std::vector<float> vec_dis;
for (int j = 0; j < TOPK * num_queries; ++j) {
@ -2121,23 +2121,23 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Range) {
ASSERT_TRUE(res_before_load_index.error_code == Success) << res_before_load_index.error_msg;
// load index to segment
auto conf = milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
{milvus::knowhere::meta::TOPK, TOPK},
{milvus::knowhere::IndexParams::nprobe, 10},
{milvus::knowhere::IndexParams::nlist, 100},
{milvus::knowhere::IndexParams::m, 4},
{milvus::knowhere::IndexParams::nbits, 8},
{milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::JACCARD},
auto conf = knowhere::Config{
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nprobe, 10},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, knowhere::Metric::JACCARD},
};
auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_BIN_IVFFLAT);
// gen query dataset
auto query_dataset = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr);
auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr);
auto result_on_index = indexing->Query(query_dataset, conf, nullptr);
auto ids = result_on_index->Get<int64_t*>(milvus::knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(milvus::knowhere::meta::DISTANCE);
auto ids = result_on_index->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(knowhere::meta::DISTANCE);
std::vector<int64_t> vec_ids(ids, ids + TOPK * num_queries);
std::vector<float> vec_dis;
for (int j = 0; j < TOPK * num_queries; ++j) {
@ -2257,23 +2257,23 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Term) {
assert(res_before_load_index.error_code == Success);
// load index to segment
auto conf = milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
{milvus::knowhere::meta::TOPK, TOPK},
{milvus::knowhere::IndexParams::nprobe, 10},
{milvus::knowhere::IndexParams::nlist, 100},
{milvus::knowhere::IndexParams::m, 4},
{milvus::knowhere::IndexParams::nbits, 8},
{milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::JACCARD},
auto conf = knowhere::Config{
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nprobe, 10},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, knowhere::Metric::JACCARD},
};
auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_BIN_IVFFLAT);
// gen query dataset
auto query_dataset = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr);
auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr);
auto result_on_index = indexing->Query(query_dataset, conf, nullptr);
auto ids = result_on_index->Get<int64_t*>(milvus::knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(milvus::knowhere::meta::DISTANCE);
auto ids = result_on_index->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(knowhere::meta::DISTANCE);
std::vector<int64_t> vec_ids(ids, ids + TOPK * num_queries);
std::vector<float> vec_dis;
for (int j = 0; j < TOPK * num_queries; ++j) {
@ -2406,23 +2406,23 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Term) {
assert(res_before_load_index.error_code == Success);
// load index to segment
auto conf = milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
{milvus::knowhere::meta::TOPK, TOPK},
{milvus::knowhere::IndexParams::nprobe, 10},
{milvus::knowhere::IndexParams::nlist, 100},
{milvus::knowhere::IndexParams::m, 4},
{milvus::knowhere::IndexParams::nbits, 8},
{milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::JACCARD},
auto conf = knowhere::Config{
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nprobe, 10},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, knowhere::Metric::JACCARD},
};
auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_BIN_IVFFLAT);
// gen query dataset
auto query_dataset = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr);
auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr);
auto result_on_index = indexing->Query(query_dataset, conf, nullptr);
auto ids = result_on_index->Get<int64_t*>(milvus::knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(milvus::knowhere::meta::DISTANCE);
auto ids = result_on_index->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(knowhere::meta::DISTANCE);
std::vector<int64_t> vec_ids(ids, ids + TOPK * num_queries);
std::vector<float> vec_dis;
for (int j = 0; j < TOPK * num_queries; ++j) {
@ -2585,22 +2585,22 @@ TEST(CApiTest, SealedSegment_search_float_Predicate_Range) {
Timestamp time = 10000000;
// load index to segment
auto conf = milvus::knowhere::Config{{milvus::knowhere::meta::DIM, DIM},
{milvus::knowhere::meta::TOPK, TOPK},
{milvus::knowhere::IndexParams::nlist, 100},
{milvus::knowhere::IndexParams::nprobe, 10},
{milvus::knowhere::IndexParams::m, 4},
{milvus::knowhere::IndexParams::nbits, 8},
{milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::L2},
{milvus::knowhere::meta::DEVICEID, 0}};
auto conf = knowhere::Config{{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::nprobe, 10},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ);
// gen query dataset
auto query_dataset = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr);
auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr);
auto result_on_index = indexing->Query(query_dataset, conf, nullptr);
auto ids = result_on_index->Get<int64_t*>(milvus::knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(milvus::knowhere::meta::DISTANCE);
auto ids = result_on_index->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(knowhere::meta::DISTANCE);
std::vector<int64_t> vec_ids(ids, ids + TOPK * num_queries);
std::vector<float> vec_dis;
for (int j = 0; j < TOPK * num_queries; ++j) {
@ -2625,11 +2625,11 @@ TEST(CApiTest, SealedSegment_search_float_Predicate_Range) {
AppendIndex(c_load_index_info, (CBinarySet)&binary_set);
auto load_index_info = (LoadIndexInfo*)c_load_index_info;
auto query_dataset2 = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr);
auto query_dataset2 = knowhere::GenDataset(num_queries, DIM, query_ptr);
auto fuck2 = load_index_info->index;
auto result_on_index2 = fuck2->Query(query_dataset2, conf, nullptr);
auto ids2 = result_on_index2->Get<int64_t*>(milvus::knowhere::meta::IDS);
auto dis2 = result_on_index2->Get<float*>(milvus::knowhere::meta::DISTANCE);
auto ids2 = result_on_index2->Get<int64_t*>(knowhere::meta::IDS);
auto dis2 = result_on_index2->Get<float*>(knowhere::meta::DISTANCE);
int i = 1 + 1;
++i;
@ -2844,22 +2844,22 @@ TEST(CApiTest, SealedSegment_search_float_With_Expr_Predicate_Range) {
Timestamp time = 10000000;
// load index to segment
auto conf = milvus::knowhere::Config{{milvus::knowhere::meta::DIM, DIM},
{milvus::knowhere::meta::TOPK, TOPK},
{milvus::knowhere::IndexParams::nlist, 100},
{milvus::knowhere::IndexParams::nprobe, 10},
{milvus::knowhere::IndexParams::m, 4},
{milvus::knowhere::IndexParams::nbits, 8},
{milvus::knowhere::Metric::TYPE, milvus::knowhere::Metric::L2},
{milvus::knowhere::meta::DEVICEID, 0}};
auto conf = knowhere::Config{{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::nprobe, 10},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto indexing = generate_index(vec_col.data(), conf, DIM, TOPK, N, IndexEnum::INDEX_FAISS_IVFPQ);
// gen query dataset
auto query_dataset = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr);
auto query_dataset = knowhere::GenDataset(num_queries, DIM, query_ptr);
auto result_on_index = indexing->Query(query_dataset, conf, nullptr);
auto ids = result_on_index->Get<int64_t*>(milvus::knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(milvus::knowhere::meta::DISTANCE);
auto ids = result_on_index->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result_on_index->Get<float*>(knowhere::meta::DISTANCE);
std::vector<int64_t> vec_ids(ids, ids + TOPK * num_queries);
std::vector<float> vec_dis;
for (int j = 0; j < TOPK * num_queries; ++j) {
@ -2884,11 +2884,11 @@ TEST(CApiTest, SealedSegment_search_float_With_Expr_Predicate_Range) {
AppendIndex(c_load_index_info, (CBinarySet)&binary_set);
auto load_index_info = (LoadIndexInfo*)c_load_index_info;
auto query_dataset2 = milvus::knowhere::GenDataset(num_queries, DIM, query_ptr);
auto query_dataset2 = knowhere::GenDataset(num_queries, DIM, query_ptr);
auto fuck2 = load_index_info->index;
auto result_on_index2 = fuck2->Query(query_dataset2, conf, nullptr);
auto ids2 = result_on_index2->Get<int64_t*>(milvus::knowhere::meta::IDS);
auto dis2 = result_on_index2->Get<float*>(milvus::knowhere::meta::DISTANCE);
auto ids2 = result_on_index2->Get<int64_t*>(knowhere::meta::IDS);
auto dis2 = result_on_index2->Get<float*>(knowhere::meta::DISTANCE);
int i = 1 + 1;
++i;

4
internal/core/unittest/test_conf_adapter_mgr.cpp
View File

@ -16,8 +16,8 @@
TEST(AdapterMgr, MultiThread) {
auto run_case = [&]() {
auto& ins = milvus::knowhere::AdapterMgr::GetInstance();
auto adapter = ins.GetAdapter(milvus::knowhere::IndexEnum::INDEX_HNSW);
auto& ins = knowhere::AdapterMgr::GetInstance();
auto adapter = ins.GetAdapter(knowhere::IndexEnum::INDEX_HNSW);
ASSERT_TRUE(adapter != nullptr);
ASSERT_ANY_THROW(ins.GetAdapter("not supported now!"));
};

22
internal/core/unittest/test_index_c_api.cpp
View File

@ -31,8 +31,8 @@
constexpr int NB = 10;
TEST(FloatVecIndex, All) {
auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_IVFPQ;
auto metric_type = milvus::knowhere::Metric::L2;
auto index_type = knowhere::IndexEnum::INDEX_FAISS_IVFPQ;
auto metric_type = knowhere::Metric::L2;
indexcgo::TypeParams type_params;
indexcgo::IndexParams index_params;
std::tie(type_params, index_params) = generate_params(index_type, metric_type);
@ -82,8 +82,8 @@ TEST(FloatVecIndex, All) {
}
TEST(BinaryVecIndex, All) {
auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT;
auto metric_type = milvus::knowhere::Metric::JACCARD;
auto index_type = knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT;
auto metric_type = knowhere::Metric::JACCARD;
indexcgo::TypeParams type_params;
indexcgo::IndexParams index_params;
std::tie(type_params, index_params) = generate_params(index_type, metric_type);
@ -134,7 +134,7 @@ TEST(BinaryVecIndex, All) {
TEST(CBoolIndexTest, All) {
schemapb::BoolArray half;
milvus::knowhere::DatasetPtr half_ds;
knowhere::DatasetPtr half_ds;
for (size_t i = 0; i < NB; i++) {
*(half.mutable_data()->Add()) = (i % 2) == 0;
@ -159,8 +159,8 @@ TEST(CBoolIndexTest, All) {
ASSERT_EQ(Success, status.error_code);
}
{
status = BuildScalarIndex(index, half_ds->Get<int64_t>(milvus::knowhere::meta::ROWS),
half_ds->Get<const void*>(milvus::knowhere::meta::TENSOR));
status = BuildScalarIndex(index, half_ds->Get<int64_t>(knowhere::meta::ROWS),
half_ds->Get<const void*>(knowhere::meta::TENSOR));
ASSERT_EQ(Success, status.error_code);
}
{
@ -185,7 +185,7 @@ TEST(CBoolIndexTest, All) {
}
}
delete[](char*) half_ds->Get<const void*>(milvus::knowhere::meta::TENSOR);
delete[](char*) half_ds->Get<const void*>(knowhere::meta::TENSOR);
}
// TODO: more scalar type.
@ -260,8 +260,8 @@ TEST(CStringIndexTest, All) {
ASSERT_EQ(Success, status.error_code);
}
{
status = BuildScalarIndex(index, str_ds->Get<int64_t>(milvus::knowhere::meta::ROWS),
str_ds->Get<const void*>(milvus::knowhere::meta::TENSOR));
status = BuildScalarIndex(index, str_ds->Get<int64_t>(knowhere::meta::ROWS),
str_ds->Get<const void*>(knowhere::meta::TENSOR));
ASSERT_EQ(Success, status.error_code);
}
{
@ -286,5 +286,5 @@ TEST(CStringIndexTest, All) {
}
}
delete[](char*) str_ds->Get<const void*>(milvus::knowhere::meta::TENSOR);
delete[](char*) str_ds->Get<const void*>(knowhere::meta::TENSOR);
}

213
internal/core/unittest/test_index_wrapper.cpp
View File

@ -28,13 +28,13 @@
constexpr int64_t NB = 1000;
namespace indexcgo = milvus::proto::indexcgo;
using Param = std::pair<milvus::knowhere::IndexType, milvus::knowhere::MetricType>;
using Param = std::pair<knowhere::IndexType, knowhere::MetricType>;
class IndexWrapperTest : public ::testing::TestWithParam<Param> {
protected:
void
SetUp() override {
milvus::engine::KnowhereConfig::SetStatisticsLevel(3);
knowhere::KnowhereConfig::SetStatisticsLevel(3);
auto param = GetParam();
index_type = param.first;
@ -42,24 +42,28 @@ class IndexWrapperTest : public ::testing::TestWithParam<Param> {
std::tie(type_params, index_params) = generate_params(index_type, metric_type);
std::map<std::string, bool> is_binary_map = {
{milvus::knowhere::IndexEnum::INDEX_FAISS_IDMAP, false},
{milvus::knowhere::IndexEnum::INDEX_FAISS_IVFPQ, false},
{milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, false},
{milvus::knowhere::IndexEnum::INDEX_FAISS_IVFSQ8, false},
{milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, true},
{milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP, true},
{knowhere::IndexEnum::INDEX_FAISS_IDMAP, false},
{knowhere::IndexEnum::INDEX_FAISS_IVFPQ, false},
{knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, false},
{knowhere::IndexEnum::INDEX_FAISS_IVFSQ8, false},
{knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, true},
{knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP, true},
#ifdef MILVUS_SUPPORT_SPTAG
{milvus::knowhere::IndexEnum::INDEX_SPTAG_KDT_RNT, false},
{milvus::knowhere::IndexEnum::INDEX_SPTAG_BKT_RNT, false},
{knowhere::IndexEnum::INDEX_SPTAG_KDT_RNT, false},
{knowhere::IndexEnum::INDEX_SPTAG_BKT_RNT, false},
#endif
{knowhere::IndexEnum::INDEX_HNSW, false},
{knowhere::IndexEnum::INDEX_ANNOY, false},
{knowhere::IndexEnum::INDEX_RHNSWFlat, false},
{knowhere::IndexEnum::INDEX_RHNSWPQ, false},
{knowhere::IndexEnum::INDEX_RHNSWSQ, false},
#ifdef MILVUS_SUPPORT_NGT
{knowhere::IndexEnum::INDEX_NGTPANNG, false},
{knowhere::IndexEnum::INDEX_NGTONNG, false},
#endif
#ifdef MILVUS_SUPPORT_NSG
{knowhere::IndexEnum::INDEX_NSG, false},
#endif
{milvus::knowhere::IndexEnum::INDEX_HNSW, false},
{milvus::knowhere::IndexEnum::INDEX_ANNOY, false},
{milvus::knowhere::IndexEnum::INDEX_RHNSWFlat, false},
{milvus::knowhere::IndexEnum::INDEX_RHNSWPQ, false},
{milvus::knowhere::IndexEnum::INDEX_RHNSWSQ, false},
{milvus::knowhere::IndexEnum::INDEX_NGTPANNG, false},
{milvus::knowhere::IndexEnum::INDEX_NGTONNG, false},
{milvus::knowhere::IndexEnum::INDEX_NSG, false},
};
is_binary = is_binary_map[index_type];
@ -73,14 +77,14 @@ class IndexWrapperTest : public ::testing::TestWithParam<Param> {
auto dataset = GenDataset(NB, metric_type, is_binary);
if (!is_binary) {
xb_data = dataset.get_col<float>(0);
xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data());
xb_dataset = knowhere::GenDataset(NB, DIM, xb_data.data());
xq_data = dataset.get_col<float>(0);
xq_dataset = milvus::knowhere::GenDataset(NQ, DIM, xq_data.data());
xq_dataset = knowhere::GenDataset(NQ, DIM, xq_data.data());
} else {
xb_bin_data = dataset.get_col<uint8_t>(0);
xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_bin_data.data());
xb_dataset = knowhere::GenDataset(NB, DIM, xb_bin_data.data());
xq_bin_data = dataset.get_col<uint8_t>(0);
xq_dataset = milvus::knowhere::GenDataset(NQ, DIM, xq_bin_data.data());
xq_dataset = knowhere::GenDataset(NQ, DIM, xq_bin_data.data());
}
}
@ -94,59 +98,59 @@ class IndexWrapperTest : public ::testing::TestWithParam<Param> {
indexcgo::IndexParams index_params;
std::string type_params_str, index_params_str;
bool is_binary;
milvus::knowhere::DatasetPtr xb_dataset;
knowhere::DatasetPtr xb_dataset;
std::vector<float> xb_data;
std::vector<uint8_t> xb_bin_data;
std::vector<milvus::knowhere::IDType> ids;
milvus::knowhere::DatasetPtr xq_dataset;
std::vector<knowhere::IDType> ids;
knowhere::DatasetPtr xq_dataset;
std::vector<float> xq_data;
std::vector<uint8_t> xq_bin_data;
};
TEST(PQ, Build) {
auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_IVFPQ;
auto metric_type = milvus::knowhere::Metric::L2;
auto index_type = knowhere::IndexEnum::INDEX_FAISS_IVFPQ;
auto metric_type = knowhere::Metric::L2;
auto conf = generate_conf(index_type, metric_type);
auto index = milvus::knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type);
auto index = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type);
auto dataset = GenDataset(NB, metric_type, false);
auto xb_data = dataset.get_col<float>(0);
auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data());
auto xb_dataset = knowhere::GenDataset(NB, DIM, xb_data.data());
ASSERT_NO_THROW(index->Train(xb_dataset, conf));
ASSERT_NO_THROW(index->AddWithoutIds(xb_dataset, conf));
}
TEST(IVFFLATNM, Build) {
auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT;
auto metric_type = milvus::knowhere::Metric::L2;
auto index_type = knowhere::IndexEnum::INDEX_FAISS_IVFFLAT;
auto metric_type = knowhere::Metric::L2;
auto conf = generate_conf(index_type, metric_type);
auto index = milvus::knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type);
auto index = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type);
auto dataset = GenDataset(NB, metric_type, false);
auto xb_data = dataset.get_col<float>(0);
auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data());
auto xb_dataset = knowhere::GenDataset(NB, DIM, xb_data.data());
ASSERT_NO_THROW(index->Train(xb_dataset, conf));
ASSERT_NO_THROW(index->AddWithoutIds(xb_dataset, conf));
}
TEST(IVFFLATNM, Query) {
milvus::engine::KnowhereConfig::SetStatisticsLevel(3);
knowhere::KnowhereConfig::SetStatisticsLevel(3);
auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT;
auto metric_type = milvus::knowhere::Metric::L2;
auto index_type = knowhere::IndexEnum::INDEX_FAISS_IVFFLAT;
auto metric_type = knowhere::Metric::L2;
auto conf = generate_conf(index_type, metric_type);
auto index = milvus::knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type);
auto index = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type);
auto dataset = GenDataset(NB, metric_type, false);
auto xb_data = dataset.get_col<float>(0);
auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data());
auto xb_dataset = knowhere::GenDataset(NB, DIM, xb_data.data());
ASSERT_NO_THROW(index->Train(xb_dataset, conf));
ASSERT_NO_THROW(index->AddWithoutIds(xb_dataset, conf));
auto bs = index->Serialize(conf);
auto bptr = std::make_shared<milvus::knowhere::Binary>();
auto bptr = std::make_shared<knowhere::Binary>();
bptr->data = std::shared_ptr<uint8_t[]>((uint8_t*)xb_data.data(), [&](uint8_t*) {});
bptr->size = DIM * NB * sizeof(float);
bs.Append(RAW_DATA, bptr);
index->Load(bs);
auto xq_data = dataset.get_col<float>(0);
auto xq_dataset = milvus::knowhere::GenDataset(NQ, DIM, xq_data.data());
auto xq_dataset = knowhere::GenDataset(NQ, DIM, xq_data.data());
auto result = index->Query(xq_dataset, conf, nullptr);
index->UpdateIndexSize();
@ -157,36 +161,38 @@ TEST(IVFFLATNM, Query) {
index->ClearStatistics();
}
#ifdef MILVUS_SUPPORT_NSG
TEST(NSG, Query) {
auto index_type = milvus::knowhere::IndexEnum::INDEX_NSG;
auto metric_type = milvus::knowhere::Metric::L2;
auto index_type = knowhere::IndexEnum::INDEX_NSG;
auto metric_type = knowhere::Metric::L2;
auto conf = generate_conf(index_type, metric_type);
auto index = milvus::knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type);
auto index = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type);
auto dataset = GenDataset(NB, metric_type, false);
auto xb_data = dataset.get_col<float>(0);
auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data());
auto xb_dataset = knowhere::GenDataset(NB, DIM, xb_data.data());
index->BuildAll(xb_dataset, conf);
auto bs = index->Serialize(conf);
auto bptr = std::make_shared<milvus::knowhere::Binary>();
auto bptr = std::make_shared<knowhere::Binary>();
bptr->data = std::shared_ptr<uint8_t[]>((uint8_t*)xb_data.data(), [&](uint8_t*) {});
bptr->size = DIM * NB * sizeof(float);
bs.Append(RAW_DATA, bptr);
index->Load(bs);
auto xq_data = dataset.get_col<float>(0);
auto xq_dataset = milvus::knowhere::GenDataset(NQ, DIM, xq_data.data());
auto xq_dataset = knowhere::GenDataset(NQ, DIM, xq_data.data());
auto result = index->Query(xq_dataset, conf, nullptr);
}
#endif
TEST(BINFLAT, Build) {
auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT;
auto metric_type = milvus::knowhere::Metric::JACCARD;
auto index_type = knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT;
auto metric_type = knowhere::Metric::JACCARD;
auto conf = generate_conf(index_type, metric_type);
auto index = milvus::knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type);
auto index = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type);
auto dataset = GenDataset(NB, metric_type, true);
auto xb_data = dataset.get_col<uint8_t>(0);
std::vector<milvus::knowhere::IDType> ids(NB, 0);
std::vector<knowhere::IDType> ids(NB, 0);
std::iota(ids.begin(), ids.end(), 0);
auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data());
auto xb_dataset = knowhere::GenDataset(NB, DIM, xb_data.data());
ASSERT_NO_THROW(index->BuildAll(xb_dataset, conf));
}
@ -203,30 +209,30 @@ print_query_result(const std::unique_ptr<milvus::indexbuilder::VecIndexCreator::
// test for: https://github.com/milvus-io/milvus/issues/6569
TEST(BinIVFFlat, Build_and_Query) {
milvus::engine::KnowhereConfig::SetStatisticsLevel(2);
knowhere::KnowhereConfig::SetStatisticsLevel(2);
auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT;
auto metric_type = milvus::knowhere::Metric::TANIMOTO;
auto index_type = knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT;
auto metric_type = knowhere::Metric::TANIMOTO;
auto conf = generate_conf(index_type, metric_type);
auto topk = 10;
conf[milvus::knowhere::meta::TOPK] = topk;
conf[milvus::knowhere::IndexParams::nlist] = 1;
auto index = milvus::knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type);
conf[knowhere::meta::TOPK] = topk;
conf[knowhere::IndexParams::nlist] = 1;
auto index = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type);
auto nb = 2;
auto dim = 128;
auto nq = 10;
auto dataset = GenDataset(std::max(nq, nb), metric_type, true);
auto xb_data = dataset.get_col<uint8_t>(0);
std::vector<milvus::knowhere::IDType> ids(nb, 0);
std::vector<knowhere::IDType> ids(nb, 0);
std::iota(ids.begin(), ids.end(), 0);
auto xb_dataset = milvus::knowhere::GenDataset(nb, dim, xb_data.data());
auto xb_dataset = knowhere::GenDataset(nb, dim, xb_data.data());
index->BuildAll(xb_dataset, conf);
auto xq_data = dataset.get_col<float>(0);
auto xq_dataset = milvus::knowhere::GenDataset(nq, dim, xq_data.data());
auto xq_dataset = knowhere::GenDataset(nq, dim, xq_data.data());
auto result = index->Query(xq_dataset, conf, nullptr);
auto hit_ids = result->Get<int64_t*>(milvus::knowhere::meta::IDS);
auto distances = result->Get<float*>(milvus::knowhere::meta::DISTANCE);
auto hit_ids = result->Get<int64_t*>(knowhere::meta::IDS);
auto distances = result->Get<float*>(knowhere::meta::DISTANCE);
auto query_res = std::make_unique<milvus::indexbuilder::VecIndexCreator::QueryResult>();
query_res->nq = nq;
@ -247,21 +253,21 @@ TEST(BinIVFFlat, Build_and_Query) {
}
TEST(BINIDMAP, Build) {
auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP;
auto metric_type = milvus::knowhere::Metric::JACCARD;
auto index_type = knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP;
auto metric_type = knowhere::Metric::JACCARD;
auto conf = generate_conf(index_type, metric_type);
auto index = milvus::knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type);
auto index = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(index_type);
auto dataset = GenDataset(NB, metric_type, true);
auto xb_data = dataset.get_col<uint8_t>(0);
std::vector<milvus::knowhere::IDType> ids(NB, 0);
std::vector<knowhere::IDType> ids(NB, 0);
std::iota(ids.begin(), ids.end(), 0);
auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data());
auto xb_dataset = knowhere::GenDataset(NB, DIM, xb_data.data());
ASSERT_NO_THROW(index->BuildAll(xb_dataset, conf));
}
TEST(PQWrapper, Build) {
auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_IVFPQ;
auto metric_type = milvus::knowhere::Metric::L2;
auto index_type = knowhere::IndexEnum::INDEX_FAISS_IVFPQ;
auto metric_type = knowhere::Metric::L2;
indexcgo::TypeParams type_params;
indexcgo::IndexParams index_params;
std::tie(type_params, index_params) = generate_params(index_type, metric_type);
@ -273,15 +279,15 @@ TEST(PQWrapper, Build) {
assert(ok);
auto dataset = GenDataset(NB, metric_type, false);
auto xb_data = dataset.get_col<float>(0);
auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data());
auto xb_dataset = knowhere::GenDataset(NB, DIM, xb_data.data());
auto index =
std::make_unique<milvus::indexbuilder::VecIndexCreator>(type_params_str.c_str(), index_params_str.c_str());
ASSERT_NO_THROW(index->BuildWithoutIds(xb_dataset));
}
TEST(IVFFLATNMWrapper, Build) {
auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT;
auto metric_type = milvus::knowhere::Metric::L2;
auto index_type = knowhere::IndexEnum::INDEX_FAISS_IVFFLAT;
auto metric_type = knowhere::Metric::L2;
indexcgo::TypeParams type_params;
indexcgo::IndexParams index_params;
std::tie(type_params, index_params) = generate_params(index_type, metric_type);
@ -293,7 +299,7 @@ TEST(IVFFLATNMWrapper, Build) {
assert(ok);
auto dataset = GenDataset(NB, metric_type, false);
auto xb_data = dataset.get_col<float>(0);
auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data());
auto xb_dataset = knowhere::GenDataset(NB, DIM, xb_data.data());
auto index =
std::make_unique<milvus::indexbuilder::VecIndexCreator>(type_params_str.c_str(), index_params_str.c_str());
ASSERT_NO_THROW(index->BuildWithoutIds(xb_dataset));
@ -301,8 +307,8 @@ TEST(IVFFLATNMWrapper, Build) {
TEST(IVFFLATNMWrapper, Codec) {
int64_t flat_nb = 100000;
auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT;
auto metric_type = milvus::knowhere::Metric::L2;
auto index_type = knowhere::IndexEnum::INDEX_FAISS_IVFFLAT;
auto metric_type = knowhere::Metric::L2;
indexcgo::TypeParams type_params;
indexcgo::IndexParams index_params;
std::tie(type_params, index_params) = generate_params(index_type, metric_type);
@ -314,7 +320,7 @@ TEST(IVFFLATNMWrapper, Codec) {
assert(ok);
auto dataset = GenDataset(flat_nb, metric_type, false);
auto xb_data = dataset.get_col<float>(0);
auto xb_dataset = milvus::knowhere::GenDataset(flat_nb, DIM, xb_data.data());
auto xb_dataset = knowhere::GenDataset(flat_nb, DIM, xb_data.data());
auto index_wrapper =
std::make_unique<milvus::indexbuilder::VecIndexCreator>(type_params_str.c_str(), index_params_str.c_str());
ASSERT_NO_THROW(index_wrapper->BuildWithoutIds(xb_dataset));
@ -335,8 +341,8 @@ TEST(IVFFLATNMWrapper, Codec) {
}
TEST(BinFlatWrapper, Build) {
auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT;
auto metric_type = milvus::knowhere::Metric::JACCARD;
auto index_type = knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT;
auto metric_type = knowhere::Metric::JACCARD;
indexcgo::TypeParams type_params;
indexcgo::IndexParams index_params;
std::tie(type_params, index_params) = generate_params(index_type, metric_type);
@ -348,9 +354,9 @@ TEST(BinFlatWrapper, Build) {
assert(ok);
auto dataset = GenDataset(NB, metric_type, true);
auto xb_data = dataset.get_col<uint8_t>(0);
std::vector<milvus::knowhere::IDType> ids(NB, 0);
std::vector<knowhere::IDType> ids(NB, 0);
std::iota(ids.begin(), ids.end(), 0);
auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data());
auto xb_dataset = knowhere::GenDataset(NB, DIM, xb_data.data());
auto index =
std::make_unique<milvus::indexbuilder::VecIndexCreator>(type_params_str.c_str(), index_params_str.c_str());
ASSERT_NO_THROW(index->BuildWithoutIds(xb_dataset));
@ -358,8 +364,8 @@ TEST(BinFlatWrapper, Build) {
}
TEST(BinIdMapWrapper, Build) {
auto index_type = milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP;
auto metric_type = milvus::knowhere::Metric::JACCARD;
auto index_type = knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP;
auto metric_type = knowhere::Metric::JACCARD;
indexcgo::TypeParams type_params;
indexcgo::IndexParams index_params;
std::tie(type_params, index_params) = generate_params(index_type, metric_type);
@ -371,9 +377,9 @@ TEST(BinIdMapWrapper, Build) {
assert(ok);
auto dataset = GenDataset(NB, metric_type, true);
auto xb_data = dataset.get_col<uint8_t>(0);
std::vector<milvus::knowhere::IDType> ids(NB, 0);
std::vector<knowhere::IDType> ids(NB, 0);
std::iota(ids.begin(), ids.end(), 0);
auto xb_dataset = milvus::knowhere::GenDataset(NB, DIM, xb_data.data());
auto xb_dataset = knowhere::GenDataset(NB, DIM, xb_data.data());
auto index =
std::make_unique<milvus::indexbuilder::VecIndexCreator>(type_params_str.c_str(), index_params_str.c_str());
ASSERT_NO_THROW(index->BuildWithoutIds(xb_dataset));
@ -384,25 +390,30 @@ INSTANTIATE_TEST_CASE_P(
IndexTypeParameters,
IndexWrapperTest,
::testing::Values(
std::pair(milvus::knowhere::IndexEnum::INDEX_FAISS_IDMAP, milvus::knowhere::Metric::L2),
std::pair(milvus::knowhere::IndexEnum::INDEX_FAISS_IVFPQ, milvus::knowhere::Metric::L2),
std::pair(milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, milvus::knowhere::Metric::L2),
std::pair(milvus::knowhere::IndexEnum::INDEX_FAISS_IVFSQ8, milvus::knowhere::Metric::L2),
std::pair(milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, milvus::knowhere::Metric::JACCARD),
std::pair(milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, milvus::knowhere::Metric::TANIMOTO),
std::pair(milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP, milvus::knowhere::Metric::JACCARD),
std::pair(knowhere::IndexEnum::INDEX_FAISS_IDMAP, knowhere::Metric::L2),
std::pair(knowhere::IndexEnum::INDEX_FAISS_IVFPQ, knowhere::Metric::L2),
std::pair(knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, knowhere::Metric::L2),
std::pair(knowhere::IndexEnum::INDEX_FAISS_IVFSQ8, knowhere::Metric::L2),
std::pair(knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, knowhere::Metric::JACCARD),
std::pair(knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, knowhere::Metric::TANIMOTO),
std::pair(knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP, knowhere::Metric::JACCARD),
#ifdef MILVUS_SUPPORT_SPTAG
std::pair(milvus::knowhere::IndexEnum::INDEX_SPTAG_KDT_RNT, milvus::knowhere::Metric::L2),
std::pair(milvus::knowhere::IndexEnum::INDEX_SPTAG_BKT_RNT, milvus::knowhere::Metric::L2),
std::pair(knowhere::IndexEnum::INDEX_SPTAG_KDT_RNT, knowhere::Metric::L2),
std::pair(knowhere::IndexEnum::INDEX_SPTAG_BKT_RNT, knowhere::Metric::L2),
#endif
std::pair(milvus::knowhere::IndexEnum::INDEX_HNSW, milvus::knowhere::Metric::L2),
std::pair(milvus::knowhere::IndexEnum::INDEX_ANNOY, milvus::knowhere::Metric::L2),
std::pair(milvus::knowhere::IndexEnum::INDEX_RHNSWFlat, milvus::knowhere::Metric::L2),
std::pair(milvus::knowhere::IndexEnum::INDEX_RHNSWPQ, milvus::knowhere::Metric::L2),
std::pair(milvus::knowhere::IndexEnum::INDEX_RHNSWSQ, milvus::knowhere::Metric::L2),
std::pair(milvus::knowhere::IndexEnum::INDEX_NGTPANNG, milvus::knowhere::Metric::L2),
std::pair(milvus::knowhere::IndexEnum::INDEX_NGTONNG, milvus::knowhere::Metric::L2),
std::pair(milvus::knowhere::IndexEnum::INDEX_NSG, milvus::knowhere::Metric::L2)));
std::pair(knowhere::IndexEnum::INDEX_HNSW, knowhere::Metric::L2),
std::pair(knowhere::IndexEnum::INDEX_ANNOY, knowhere::Metric::L2),
std::pair(knowhere::IndexEnum::INDEX_RHNSWFlat, knowhere::Metric::L2),
std::pair(knowhere::IndexEnum::INDEX_RHNSWPQ, knowhere::Metric::L2),
std::pair(knowhere::IndexEnum::INDEX_RHNSWSQ, knowhere::Metric::L2)
#ifdef MILVUS_SUPPORT_NGT
std::pair(knowhere::IndexEnum::INDEX_NGTPANNG, knowhere::Metric::L2),
std::pair(knowhere::IndexEnum::INDEX_NGTONNG, knowhere::Metric::L2),
#endif
#ifdef MILVUS_SUPPORT_NSG
std::pair(knowhere::IndexEnum::INDEX_NSG, knowhere::Metric::L2)
#endif
));
TEST_P(IndexWrapperTest, Constructor) {
auto index =

20
internal/core/unittest/test_indexing.cpp
View File

@ -111,14 +111,14 @@ TEST(Indexing, Naive) {
auto index = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(knowhere::IndexEnum::INDEX_FAISS_IVFPQ,
knowhere::IndexMode::MODE_CPU);
auto conf = milvus::knowhere::Config{
auto conf = knowhere::Config{
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::nprobe, 4},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, milvus::knowhere::Metric::L2},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0},
};
@ -193,7 +193,7 @@ TEST(Indexing, IVFFlat) {
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nlist, NLIST},
{knowhere::IndexParams::nprobe, NPROBE},
{knowhere::Metric::TYPE, milvus::knowhere::Metric::L2},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto database = knowhere::GenDataset(N, DIM, raw_data.data());
@ -210,8 +210,8 @@ TEST(Indexing, IVFFlat) {
auto result = indexing->Query(dataset, conf, nullptr);
std::cout << "query ivf " << timer.get_step_seconds() << " seconds" << std::endl;
auto ids = result->Get<int64_t*>(milvus::knowhere::meta::IDS);
auto dis = result->Get<float*>(milvus::knowhere::meta::DISTANCE);
auto ids = result->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result->Get<float*>(knowhere::meta::DISTANCE);
for (int i = 0; i < std::min(NQ * TOPK, 100); ++i) {
std::cout << ids[i] << "->" << dis[i] << std::endl;
}
@ -233,7 +233,7 @@ TEST(Indexing, IVFFlatNM) {
{knowhere::meta::TOPK, TOPK},
{knowhere::IndexParams::nlist, NLIST},
{knowhere::IndexParams::nprobe, NPROBE},
{knowhere::Metric::TYPE, milvus::knowhere::Metric::L2},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto database = knowhere::GenDataset(N, DIM, raw_data.data());
@ -244,9 +244,9 @@ TEST(Indexing, IVFFlatNM) {
std::cout << "insert ivf_nm " << timer.get_step_seconds() << " seconds" << std::endl;
indexing->SetIndexSize(NQ * DIM * sizeof(float));
milvus::knowhere::BinarySet bs = indexing->Serialize(conf);
knowhere::BinarySet bs = indexing->Serialize(conf);
milvus::knowhere::BinaryPtr bptr = std::make_shared<milvus::knowhere::Binary>();
knowhere::BinaryPtr bptr = std::make_shared<knowhere::Binary>();
bptr->data = std::shared_ptr<uint8_t[]>((uint8_t*)raw_data.data(), [&](uint8_t*) {});
bptr->size = DIM * N * sizeof(float);
bs.Append(RAW_DATA, bptr);
@ -259,8 +259,8 @@ TEST(Indexing, IVFFlatNM) {
auto result = indexing->Query(dataset, conf, nullptr);
std::cout << "query ivf_nm " << timer.get_step_seconds() << " seconds" << std::endl;
auto ids = result->Get<int64_t*>(milvus::knowhere::meta::IDS);
auto dis = result->Get<float*>(milvus::knowhere::meta::DISTANCE);
auto ids = result->Get<int64_t*>(knowhere::meta::IDS);
auto dis = result->Get<float*>(knowhere::meta::DISTANCE);
for (int i = 0; i < std::min(NQ * TOPK, 100); ++i) {
std::cout << ids[i] << "->" << dis[i] << std::endl;
}

20
internal/core/unittest/test_scalar_index_creator.cpp
View File

@ -38,7 +38,7 @@ namespace indexcgo = milvus::proto::indexcgo;
namespace schemapb = milvus::proto::schema;
using milvus::indexbuilder::MapParams;
using milvus::indexbuilder::ScalarIndexCreator;
using milvus::knowhere::scalar::OperatorType;
using knowhere::scalar::OperatorType;
using ScalarTestParams = std::pair<MapParams, MapParams>;
namespace {
@ -225,7 +225,7 @@ TYPED_TEST_P(TypedScalarIndexCreatorTest, Codec) {
std::make_unique<ScalarIndexCreator<T>>(serialized_type_params.c_str(), serialized_index_params.c_str());
auto arr = GenArr<T>(nb);
const int64_t dim = 8; // not important here
auto dataset = milvus::knowhere::GenDataset(arr.size(), dim, arr.data());
auto dataset = knowhere::GenDataset(arr.size(), dim, arr.data());
creator->Build(dataset);
auto binary_set = creator->Serialize();
@ -262,9 +262,9 @@ class BoolIndexTest : public ::testing::Test {
void
TearDown() override {
delete[](char*)(all_true_ds->Get<const void*>(milvus::knowhere::meta::TENSOR));
delete[](char*) all_false_ds->Get<const void*>(milvus::knowhere::meta::TENSOR);
delete[](char*) half_ds->Get<const void*>(milvus::knowhere::meta::TENSOR);
delete[](char*)(all_true_ds->Get<const void*>(knowhere::meta::TENSOR));
delete[](char*) all_false_ds->Get<const void*>(knowhere::meta::TENSOR);
delete[](char*) half_ds->Get<const void*>(knowhere::meta::TENSOR);
}
private:
@ -277,9 +277,9 @@ class BoolIndexTest : public ::testing::Test {
schemapb::BoolArray all_true;
schemapb::BoolArray all_false;
schemapb::BoolArray half;
milvus::knowhere::DatasetPtr all_true_ds;
milvus::knowhere::DatasetPtr all_false_ds;
milvus::knowhere::DatasetPtr half_ds;
knowhere::DatasetPtr all_true_ds;
knowhere::DatasetPtr all_false_ds;
knowhere::DatasetPtr half_ds;
size_t n;
std::vector<ScalarTestParams> params;
};
@ -489,7 +489,7 @@ class StringIndexTest : public ::testing::Test {
void
TearDown() override {
delete[](char*)(str_ds->Get<const void*>(milvus::knowhere::meta::TENSOR));
delete[](char*)(str_ds->Get<const void*>(knowhere::meta::TENSOR));
}
private:
@ -501,7 +501,7 @@ class StringIndexTest : public ::testing::Test {
protected:
std::vector<std::string> strs;
schemapb::StringArray str_arr;
milvus::knowhere::DatasetPtr str_ds;
knowhere::DatasetPtr str_ds;
std::vector<ScalarTestParams> params;
};

10
internal/core/unittest/test_sealed.cpp
View File

@ -80,7 +80,7 @@ TEST(Sealed, without_predicate) {
{knowhere::meta::TOPK, topK},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::nprobe, 10},
{knowhere::Metric::TYPE, milvus::knowhere::Metric::L2},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto database = knowhere::GenDataset(N, dim, vec_col.data() + 1000 * dim);
@ -94,8 +94,8 @@ TEST(Sealed, without_predicate) {
auto result = indexing->Query(query_dataset, conf, nullptr);
auto ids = result->Get<int64_t*>(milvus::knowhere::meta::IDS); // for comparison
auto dis = result->Get<float*>(milvus::knowhere::meta::DISTANCE); // for comparison
auto ids = result->Get<int64_t*>(knowhere::meta::IDS); // for comparison
auto dis = result->Get<float*>(knowhere::meta::DISTANCE); // for comparison
std::vector<int64_t> vec_ids(ids, ids + topK * num_queries);
std::vector<float> vec_dis(dis, dis + topK * num_queries);
@ -180,7 +180,7 @@ TEST(Sealed, with_predicate) {
{knowhere::meta::TOPK, topK},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::nprobe, 10},
{knowhere::Metric::TYPE, milvus::knowhere::Metric::L2},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto database = knowhere::GenDataset(N, dim, vec_col.data());
@ -273,7 +273,7 @@ TEST(Sealed, LoadFieldData) {
LoadIndexInfo vec_info;
vec_info.field_id = fakevec_id.get();
vec_info.index = indexing;
vec_info.index_params["metric_type"] = milvus::knowhere::Metric::L2;
vec_info.index_params["metric_type"] = knowhere::Metric::L2;
segment->LoadIndex(vec_info);
ASSERT_EQ(segment->num_chunk(), 1);

2
internal/core/unittest/test_utils/DataGen.h
View File

@ -339,7 +339,7 @@ GenIndexing(int64_t N, int64_t dim, const float* vec) {
// {knowhere::IndexParams::nprobe, 10},
auto conf = knowhere::Config{{knowhere::meta::DIM, dim},
{knowhere::IndexParams::nlist, 1024},
{knowhere::Metric::TYPE, milvus::knowhere::Metric::L2},
{knowhere::Metric::TYPE, knowhere::Metric::L2},
{knowhere::meta::DEVICEID, 0}};
auto database = knowhere::GenDataset(N, dim, vec);
auto indexing = std::make_shared<knowhere::IVF>();

306
internal/core/unittest/test_utils/indexbuilder_test_utils.h
View File

@ -42,176 +42,176 @@ namespace indexcgo = milvus::proto::indexcgo;
namespace schemapb = milvus::proto::schema;
using milvus::indexbuilder::MapParams;
using milvus::indexbuilder::ScalarIndexCreator;
using milvus::knowhere::scalar::OperatorType;
using knowhere::scalar::OperatorType;
using ScalarTestParams = std::pair<MapParams, MapParams>;
namespace {
auto
generate_conf(const milvus::knowhere::IndexType& index_type, const milvus::knowhere::MetricType& metric_type) {
if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_IDMAP) {
return milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
{milvus::knowhere::meta::TOPK, K},
{milvus::knowhere::Metric::TYPE, metric_type},
{milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
generate_conf(const knowhere::IndexType& index_type, const knowhere::MetricType& metric_type) {
if (index_type == knowhere::IndexEnum::INDEX_FAISS_IDMAP) {
return knowhere::Config{
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, K},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
};
} else if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_IVFPQ) {
return milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
{milvus::knowhere::meta::TOPK, K},
{milvus::knowhere::IndexParams::nlist, 16},
{milvus::knowhere::IndexParams::nprobe, 4},
{milvus::knowhere::IndexParams::m, 4},
{milvus::knowhere::IndexParams::nbits, 8},
{milvus::knowhere::Metric::TYPE, metric_type},
{milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
} else if (index_type == knowhere::IndexEnum::INDEX_FAISS_IVFPQ) {
return knowhere::Config{
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, K},
{knowhere::IndexParams::nlist, 16},
{knowhere::IndexParams::nprobe, 4},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
};
} else if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_IVFFLAT) {
return milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
{milvus::knowhere::meta::TOPK, K},
{milvus::knowhere::IndexParams::nlist, 16},
{milvus::knowhere::IndexParams::nprobe, 4},
{milvus::knowhere::Metric::TYPE, metric_type},
{milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
} else if (index_type == knowhere::IndexEnum::INDEX_FAISS_IVFFLAT) {
return knowhere::Config{
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, K},
{knowhere::IndexParams::nlist, 16},
{knowhere::IndexParams::nprobe, 4},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
#ifdef MILVUS_GPU_VERSION
{milvus::knowhere::meta::DEVICEID, DEVICEID},
{knowhere::meta::DEVICEID, DEVICEID},
#endif
};
} else if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_IVFSQ8) {
return milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
{milvus::knowhere::meta::TOPK, K},
{milvus::knowhere::IndexParams::nlist, 16},
{milvus::knowhere::IndexParams::nprobe, 4},
{milvus::knowhere::IndexParams::nbits, 8},
{milvus::knowhere::Metric::TYPE, metric_type},
{milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
} else if (index_type == knowhere::IndexEnum::INDEX_FAISS_IVFSQ8) {
return knowhere::Config{
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, K},
{knowhere::IndexParams::nlist, 16},
{knowhere::IndexParams::nprobe, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
#ifdef MILVUS_GPU_VERSION
{milvus::knowhere::meta::DEVICEID, DEVICEID},
{knowhere::meta::DEVICEID, DEVICEID},
#endif
};
} else if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT) {
return milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
{milvus::knowhere::meta::TOPK, K},
{milvus::knowhere::IndexParams::nlist, 16},
{milvus::knowhere::IndexParams::nprobe, 4},
{milvus::knowhere::IndexParams::m, 4},
{milvus::knowhere::IndexParams::nbits, 8},
{milvus::knowhere::Metric::TYPE, metric_type},
{milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
} else if (index_type == knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT) {
return knowhere::Config{
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, K},
{knowhere::IndexParams::nlist, 16},
{knowhere::IndexParams::nprobe, 4},
{knowhere::IndexParams::m, 4},
{knowhere::IndexParams::nbits, 8},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
};
} else if (index_type == milvus::knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP) {
return milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
{milvus::knowhere::meta::TOPK, K},
{milvus::knowhere::Metric::TYPE, metric_type},
} else if (index_type == knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP) {
return knowhere::Config{
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, K},
{knowhere::Metric::TYPE, metric_type},
};
} else if (index_type == milvus::knowhere::IndexEnum::INDEX_NSG) {
return milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
{milvus::knowhere::IndexParams::nlist, 163},
{milvus::knowhere::meta::TOPK, K},
{milvus::knowhere::IndexParams::nprobe, 8},
{milvus::knowhere::IndexParams::knng, 20},
{milvus::knowhere::IndexParams::search_length, 40},
{milvus::knowhere::IndexParams::out_degree, 30},
{milvus::knowhere::IndexParams::candidate, 100},
{milvus::knowhere::Metric::TYPE, metric_type},
} else if (index_type == knowhere::IndexEnum::INDEX_NSG) {
return knowhere::Config{
{knowhere::meta::DIM, DIM},
{knowhere::IndexParams::nlist, 163},
{knowhere::meta::TOPK, K},
{knowhere::IndexParams::nprobe, 8},
{knowhere::IndexParams::knng, 20},
{knowhere::IndexParams::search_length, 40},
{knowhere::IndexParams::out_degree, 30},
{knowhere::IndexParams::candidate, 100},
{knowhere::Metric::TYPE, metric_type},
};
#ifdef MILVUS_SUPPORT_SPTAG
} else if (index_type == milvus::knowhere::IndexEnum::INDEX_SPTAG_KDT_RNT) {
return milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
// {milvus::knowhere::meta::TOPK, 10},
{milvus::knowhere::Metric::TYPE, metric_type},
{milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
} else if (index_type == knowhere::IndexEnum::INDEX_SPTAG_KDT_RNT) {
return knowhere::Config{
{knowhere::meta::DIM, DIM},
// {knowhere::meta::TOPK, 10},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
};
} else if (index_type == milvus::knowhere::IndexEnum::INDEX_SPTAG_BKT_RNT) {
return milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
// {milvus::knowhere::meta::TOPK, 10},
{milvus::knowhere::Metric::TYPE, metric_type},
{milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
} else if (index_type == knowhere::IndexEnum::INDEX_SPTAG_BKT_RNT) {
return knowhere::Config{
{knowhere::meta::DIM, DIM},
// {knowhere::meta::TOPK, 10},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
};
#endif
} else if (index_type == milvus::knowhere::IndexEnum::INDEX_HNSW) {
return milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM}, {milvus::knowhere::meta::TOPK, K},
{milvus::knowhere::IndexParams::M, 16}, {milvus::knowhere::IndexParams::efConstruction, 200},
{milvus::knowhere::IndexParams::ef, 200}, {milvus::knowhere::Metric::TYPE, metric_type},
} else if (index_type == knowhere::IndexEnum::INDEX_HNSW) {
return knowhere::Config{
{knowhere::meta::DIM, DIM}, {knowhere::meta::TOPK, K},
{knowhere::IndexParams::M, 16}, {knowhere::IndexParams::efConstruction, 200},
{knowhere::IndexParams::ef, 200}, {knowhere::Metric::TYPE, metric_type},
};
} else if (index_type == milvus::knowhere::IndexEnum::INDEX_ANNOY) {
return milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
{milvus::knowhere::meta::TOPK, K},
{milvus::knowhere::IndexParams::n_trees, 4},
{milvus::knowhere::IndexParams::search_k, 100},
{milvus::knowhere::Metric::TYPE, metric_type},
{milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
} else if (index_type == knowhere::IndexEnum::INDEX_ANNOY) {
return knowhere::Config{
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, K},
{knowhere::IndexParams::n_trees, 4},
{knowhere::IndexParams::search_k, 100},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
};
} else if (index_type == milvus::knowhere::IndexEnum::INDEX_RHNSWFlat) {
return milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
{milvus::knowhere::meta::TOPK, K},
{milvus::knowhere::IndexParams::M, 16},
{milvus::knowhere::IndexParams::efConstruction, 200},
{milvus::knowhere::IndexParams::ef, 200},
{milvus::knowhere::Metric::TYPE, metric_type},
{milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
} else if (index_type == knowhere::IndexEnum::INDEX_RHNSWFlat) {
return knowhere::Config{
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, K},
{knowhere::IndexParams::M, 16},
{knowhere::IndexParams::efConstruction, 200},
{knowhere::IndexParams::ef, 200},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
};
} else if (index_type == milvus::knowhere::IndexEnum::INDEX_RHNSWPQ) {
return milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
{milvus::knowhere::meta::TOPK, K},
{milvus::knowhere::IndexParams::M, 16},
{milvus::knowhere::IndexParams::efConstruction, 200},
{milvus::knowhere::IndexParams::ef, 200},
{milvus::knowhere::Metric::TYPE, metric_type},
{milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
{milvus::knowhere::IndexParams::PQM, 8},
} else if (index_type == knowhere::IndexEnum::INDEX_RHNSWPQ) {
return knowhere::Config{
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, K},
{knowhere::IndexParams::M, 16},
{knowhere::IndexParams::efConstruction, 200},
{knowhere::IndexParams::ef, 200},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
{knowhere::IndexParams::PQM, 8},
};
} else if (index_type == milvus::knowhere::IndexEnum::INDEX_RHNSWSQ) {
return milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
{milvus::knowhere::meta::TOPK, K},
{milvus::knowhere::IndexParams::M, 16},
{milvus::knowhere::IndexParams::efConstruction, 200},
{milvus::knowhere::IndexParams::ef, 200},
{milvus::knowhere::Metric::TYPE, metric_type},
{milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
} else if (index_type == knowhere::IndexEnum::INDEX_RHNSWSQ) {
return knowhere::Config{
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, K},
{knowhere::IndexParams::M, 16},
{knowhere::IndexParams::efConstruction, 200},
{knowhere::IndexParams::ef, 200},
{knowhere::Metric::TYPE, metric_type},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
};
} else if (index_type == milvus::knowhere::IndexEnum::INDEX_NGTPANNG) {
return milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
{milvus::knowhere::meta::TOPK, K},
{milvus::knowhere::Metric::TYPE, metric_type},
{milvus::knowhere::IndexParams::edge_size, 10},
{milvus::knowhere::IndexParams::epsilon, 0.1},
{milvus::knowhere::IndexParams::max_search_edges, 50},
{milvus::knowhere::IndexParams::forcedly_pruned_edge_size, 60},
{milvus::knowhere::IndexParams::selectively_pruned_edge_size, 30},
{milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
} else if (index_type == knowhere::IndexEnum::INDEX_NGTPANNG) {
return knowhere::Config{
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, K},
{knowhere::Metric::TYPE, metric_type},
{knowhere::IndexParams::edge_size, 10},
{knowhere::IndexParams::epsilon, 0.1},
{knowhere::IndexParams::max_search_edges, 50},
{knowhere::IndexParams::forcedly_pruned_edge_size, 60},
{knowhere::IndexParams::selectively_pruned_edge_size, 30},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
};
} else if (index_type == milvus::knowhere::IndexEnum::INDEX_NGTONNG) {
return milvus::knowhere::Config{
{milvus::knowhere::meta::DIM, DIM},
{milvus::knowhere::meta::TOPK, K},
{milvus::knowhere::Metric::TYPE, metric_type},
{milvus::knowhere::IndexParams::edge_size, 20},
{milvus::knowhere::IndexParams::epsilon, 0.1},
{milvus::knowhere::IndexParams::max_search_edges, 50},
{milvus::knowhere::IndexParams::outgoing_edge_size, 5},
{milvus::knowhere::IndexParams::incoming_edge_size, 40},
{milvus::knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
} else if (index_type == knowhere::IndexEnum::INDEX_NGTONNG) {
return knowhere::Config{
{knowhere::meta::DIM, DIM},
{knowhere::meta::TOPK, K},
{knowhere::Metric::TYPE, metric_type},
{knowhere::IndexParams::edge_size, 20},
{knowhere::IndexParams::epsilon, 0.1},
{knowhere::IndexParams::max_search_edges, 50},
{knowhere::IndexParams::outgoing_edge_size, 5},
{knowhere::IndexParams::incoming_edge_size, 40},
{knowhere::INDEX_FILE_SLICE_SIZE_IN_MEGABYTE, 4},
};
}
return milvus::knowhere::Config();
return knowhere::Config();
}
auto
generate_params(const milvus::knowhere::IndexType& index_type, const milvus::knowhere::MetricType& metric_type) {
generate_params(const knowhere::IndexType& index_type, const knowhere::MetricType& metric_type) {
namespace indexcgo = milvus::proto::indexcgo;
indexcgo::TypeParams type_params;
@ -233,9 +233,9 @@ generate_params(const milvus::knowhere::IndexType& index_type, const milvus::kno
}
auto
GenDataset(int64_t N, const milvus::knowhere::MetricType& metric_type, bool is_binary, int64_t dim = DIM) {
GenDataset(int64_t N, const knowhere::MetricType& metric_type, bool is_binary, int64_t dim = DIM) {
auto schema = std::make_shared<milvus::Schema>();
auto faiss_metric_type = milvus::knowhere::GetMetricType(metric_type);
auto faiss_metric_type = knowhere::GetMetricType(metric_type);
if (!is_binary) {
schema->AddDebugField("fakevec", milvus::engine::DataType::VECTOR_FLOAT, dim, faiss_metric_type);
return milvus::segcore::DataGen(schema, N);
@ -304,14 +304,14 @@ float
CountDistance(const void* point_a,
const void* point_b,
int dim,
const milvus::knowhere::MetricType& metric,
const knowhere::MetricType& metric,
bool is_binary = false) {
if (point_a == nullptr || point_b == nullptr) {
return std::numeric_limits<float>::max();
}
if (metric == milvus::knowhere::Metric::L2) {
if (metric == knowhere::Metric::L2) {
return L2(static_cast<const float*>(point_a), static_cast<const float*>(point_b), dim);
} else if (metric == milvus::knowhere::Metric::JACCARD) {
} else if (metric == knowhere::Metric::JACCARD) {
return Jaccard(static_cast<const uint8_t*>(point_a), static_cast<const uint8_t*>(point_b), dim);
} else {
return std::numeric_limits<float>::max();
@ -320,13 +320,13 @@ CountDistance(const void* point_a,
void
CheckDistances(const QueryResultPtr& result,
const milvus::knowhere::DatasetPtr& base_dataset,
const milvus::knowhere::DatasetPtr& query_dataset,
const milvus::knowhere::MetricType& metric,
const knowhere::DatasetPtr& base_dataset,
const knowhere::DatasetPtr& query_dataset,
const knowhere::MetricType& metric,
const float threshold = 1.0e-5) {
auto base_vecs = base_dataset->Get<float*>(milvus::knowhere::meta::TENSOR);
auto query_vecs = query_dataset->Get<float*>(milvus::knowhere::meta::TENSOR);
auto dim = base_dataset->Get<int64_t>(milvus::knowhere::meta::DIM);
auto base_vecs = base_dataset->Get<float*>(knowhere::meta::TENSOR);
auto query_vecs = query_dataset->Get<float*>(knowhere::meta::TENSOR);
auto dim = base_dataset->Get<int64_t>(knowhere::meta::DIM);
auto nq = result->nq;
auto k = result->topk;
for (auto i = 0; i < nq; i++) {
@ -445,7 +445,7 @@ template <typename T>
inline void
build_index(const std::unique_ptr<ScalarIndexCreator<T>>& creator, const std::vector<T>& arr) {
const int64_t dim = 8; // not important here
auto dataset = milvus::knowhere::GenDataset(arr.size(), dim, arr.data());
auto dataset = knowhere::GenDataset(arr.size(), dim, arr.data());
creator->Build(dataset);
}
@ -454,6 +454,6 @@ auto
GenDsFromPB(const google::protobuf::Message& msg) {
auto data = new char[msg.ByteSize()];
msg.SerializeToArray(data, msg.ByteSize());
return milvus::knowhere::GenDataset(msg.ByteSize(), 8, data);
return knowhere::GenDataset(msg.ByteSize(), 8, data);
}
} // namespace

12
internal/util/indexcgowrapper/index_test.go
View File

@ -72,8 +72,8 @@ func generateFloatVectorTestCases() []vecTestCase {
{IndexFaissIVFSQ8, IP, false, schemapb.DataType_FloatVector},
//{IndexFaissIVFSQ8H, L2, false, schemapb.DataType_FloatVector}, // TODO: enable gpu
//{IndexFaissIVFSQ8H, IP, false, schemapb.DataType_FloatVector},
{IndexNsg, L2, false, schemapb.DataType_FloatVector},
{IndexNsg, IP, false, schemapb.DataType_FloatVector},
//{IndexNsg, L2, false, schemapb.DataType_FloatVector},
//{IndexNsg, IP, false, schemapb.DataType_FloatVector},
{IndexHNSW, L2, false, schemapb.DataType_FloatVector},
{IndexHNSW, IP, false, schemapb.DataType_FloatVector},
{IndexRHNSWFlat, L2, false, schemapb.DataType_FloatVector},
@ -84,10 +84,10 @@ func generateFloatVectorTestCases() []vecTestCase {
{IndexRHNSWSQ, IP, false, schemapb.DataType_FloatVector},
{IndexANNOY, L2, false, schemapb.DataType_FloatVector},
{IndexANNOY, IP, false, schemapb.DataType_FloatVector},
{IndexNGTPANNG, L2, false, schemapb.DataType_FloatVector},
{IndexNGTPANNG, IP, false, schemapb.DataType_FloatVector},
{IndexNGTONNG, L2, false, schemapb.DataType_FloatVector},
{IndexNGTONNG, IP, false, schemapb.DataType_FloatVector},
//{IndexNGTPANNG, L2, false, schemapb.DataType_FloatVector},
//{IndexNGTPANNG, IP, false, schemapb.DataType_FloatVector},
//{IndexNGTONNG, L2, false, schemapb.DataType_FloatVector},
//{IndexNGTONNG, IP, false, schemapb.DataType_FloatVector},
}
}