test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-11-30 10:59:32 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

enhance: Update knowhere commit (#34223)

Browse Source 
Signed-off-by: Cai Yudong <yudong.cai@zilliz.com>
This commit is contained in:
Cai Yudong 2024-06-27 18:20:06 +08:00 committed by GitHub
parent f2107ad263
commit ad90360162
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 32 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
internal/core/src/common/QueryResult.h
View File

@ -93,7 +93,7 @@ struct VectorIterator {
return !heap_.empty(); return !heap_.empty();
} }
bool bool
AddIterator(std::shared_ptr<knowhere::IndexNode::iterator> iter) { AddIterator(knowhere::IndexNode::IteratorPtr iter) {
if (!sealed && iter != nullptr) { if (!sealed && iter != nullptr) {
iterators_.emplace_back(iter); iterators_.emplace_back(iter);
return true; return true;
@ -130,7 +130,7 @@ struct VectorIterator {
} }
private: private:
std::vector<std::shared_ptr<knowhere::IndexNode::iterator>> iterators_; std::vector<knowhere::IndexNode::IteratorPtr> iterators_;
std::priority_queue<std::shared_ptr<OffsetDisPair>, std::priority_queue<std::shared_ptr<OffsetDisPair>,
std::vector<std::shared_ptr<OffsetDisPair>>, std::vector<std::shared_ptr<OffsetDisPair>>,
OffsetDisPairComparator> OffsetDisPairComparator>
@ -161,8 +161,7 @@ struct SearchResult {
int64_t nq, int64_t nq,
int chunk_count, int chunk_count,
int64_t rows_per_chunk, int64_t rows_per_chunk,
const std::vector<std::shared_ptr<knowhere::IndexNode::iterator>>& const std::vector<knowhere::IndexNode::IteratorPtr>& kw_iterators) {
kw_iterators) {
AssertInfo(kw_iterators.size() == nq * chunk_count, AssertInfo(kw_iterators.size() == nq * chunk_count,
"kw_iterators count:{} is not equal to nq*chunk_count:{}, " "kw_iterators count:{} is not equal to nq*chunk_count:{}, "
"wrong state", "wrong state",

10
internal/core/src/index/VectorDiskIndex.cpp
View File

@ -375,7 +375,7 @@ VectorDiskAnnIndex<T>::Query(const DatasetPtr dataset,
search_config[RANGE_FILTER], search_config[RANGE_FILTER],
GetMetricType()); GetMetricType());
} }
auto res = index_.RangeSearch(*dataset, search_config, bitset); auto res = index_.RangeSearch(dataset, search_config, bitset);
if (!res.has_value()) { if (!res.has_value()) {
PanicInfo(ErrorCode::UnexpectedError, PanicInfo(ErrorCode::UnexpectedError,
@ -386,7 +386,7 @@ VectorDiskAnnIndex<T>::Query(const DatasetPtr dataset,
return ReGenRangeSearchResult( return ReGenRangeSearchResult(
res.value(), topk, num_queries, GetMetricType()); res.value(), topk, num_queries, GetMetricType());
} else { } else {
auto res = index_.Search(*dataset, search_config, bitset); auto res = index_.Search(dataset, search_config, bitset);
if (!res.has_value()) { if (!res.has_value()) {
PanicInfo(ErrorCode::UnexpectedError, PanicInfo(ErrorCode::UnexpectedError,
fmt::format("failed to search: {}: {}", fmt::format("failed to search: {}: {}",
@ -419,11 +419,11 @@ VectorDiskAnnIndex<T>::Query(const DatasetPtr dataset,
} }
template <typename T> template <typename T>
knowhere::expected<std::vector<std::shared_ptr<knowhere::IndexNode::iterator>>> knowhere::expected<std::vector<knowhere::IndexNode::IteratorPtr>>
VectorDiskAnnIndex<T>::VectorIterators(const DatasetPtr dataset, VectorDiskAnnIndex<T>::VectorIterators(const DatasetPtr dataset,
const knowhere::Json& conf, const knowhere::Json& conf,
const BitsetView& bitset) const { const BitsetView& bitset) const {
return this->index_.AnnIterator(*dataset, conf, bitset); return this->index_.AnnIterator(dataset, conf, bitset);
} }
template <typename T> template <typename T>
@ -440,7 +440,7 @@ VectorDiskAnnIndex<T>::GetVector(const DatasetPtr dataset) const {
PanicInfo(ErrorCode::UnexpectedError, PanicInfo(ErrorCode::UnexpectedError,
"failed to get vector, index is sparse"); "failed to get vector, index is sparse");
} }
auto res = index_.GetVectorByIds(*dataset); auto res = index_.GetVectorByIds(dataset);
if (!res.has_value()) { if (!res.has_value()) {
PanicInfo(ErrorCode::UnexpectedError, PanicInfo(ErrorCode::UnexpectedError,
fmt::format("failed to get vector: {}: {}", fmt::format("failed to get vector: {}: {}",

3
internal/core/src/index/VectorDiskIndex.h
View File

@ -106,8 +106,7 @@ class VectorDiskAnnIndex : public VectorIndex {
void CleanLocalData() override; void CleanLocalData() override;
knowhere::expected< knowhere::expected<std::vector<knowhere::IndexNode::IteratorPtr>>
std::vector<std::shared_ptr<knowhere::IndexNode::iterator>>>
VectorIterators(const DatasetPtr dataset, VectorIterators(const DatasetPtr dataset,
const knowhere::Json& json, const knowhere::Json& json,
const BitsetView& bitset) const override; const BitsetView& bitset) const override;

3
internal/core/src/index/VectorIndex.h
View File

@ -60,8 +60,7 @@ class VectorIndex : public IndexBase {
const BitsetView& bitset, const BitsetView& bitset,
SearchResult& search_result) const = 0; SearchResult& search_result) const = 0;
virtual knowhere::expected< virtual knowhere::expected<std::vector<knowhere::IndexNode::IteratorPtr>>
std::vector<std::shared_ptr<knowhere::IndexNode::iterator>>>
VectorIterators(const DatasetPtr dataset, VectorIterators(const DatasetPtr dataset,
const knowhere::Json& json, const knowhere::Json& json,
const BitsetView& bitset) const { const BitsetView& bitset) const {

16
internal/core/src/index/VectorMemIndex.cpp
View File

@ -147,11 +147,11 @@ VectorMemIndex<T>::UploadV2(const Config& config) {
} }
template <typename T> template <typename T>
knowhere::expected<std::vector<std::shared_ptr<knowhere::IndexNode::iterator>>> knowhere::expected<std::vector<knowhere::IndexNode::IteratorPtr>>
VectorMemIndex<T>::VectorIterators(const milvus::DatasetPtr dataset, VectorMemIndex<T>::VectorIterators(const milvus::DatasetPtr dataset,
const knowhere::Json& conf, const knowhere::Json& conf,
const milvus::BitsetView& bitset) const { const milvus::BitsetView& bitset) const {
return this->index_.AnnIterator(*dataset, conf, bitset); return this->index_.AnnIterator(dataset, conf, bitset);
} }
template <typename T> template <typename T>
@ -434,7 +434,7 @@ VectorMemIndex<T>::BuildWithDataset(const DatasetPtr& dataset,
SetDim(dataset->GetDim()); SetDim(dataset->GetDim());
knowhere::TimeRecorder rc("BuildWithoutIds", 1); knowhere::TimeRecorder rc("BuildWithoutIds", 1);
auto stat = index_.Build(*dataset, index_config); auto stat = index_.Build(dataset, index_config);
if (stat != knowhere::Status::success) if (stat != knowhere::Status::success)
PanicInfo(ErrorCode::IndexBuildError, PanicInfo(ErrorCode::IndexBuildError,
"failed to build index, " + KnowhereStatusString(stat)); "failed to build index, " + KnowhereStatusString(stat));
@ -572,7 +572,7 @@ VectorMemIndex<T>::AddWithDataset(const DatasetPtr& dataset,
index_config.update(config); index_config.update(config);
knowhere::TimeRecorder rc("AddWithDataset", 1); knowhere::TimeRecorder rc("AddWithDataset", 1);
auto stat = index_.Add(*dataset, index_config); auto stat = index_.Add(dataset, index_config);
if (stat != knowhere::Status::success) if (stat != knowhere::Status::success)
PanicInfo(ErrorCode::IndexBuildError, PanicInfo(ErrorCode::IndexBuildError,
"failed to append index, " + KnowhereStatusString(stat)); "failed to append index, " + KnowhereStatusString(stat));
@ -601,7 +601,7 @@ VectorMemIndex<T>::Query(const DatasetPtr dataset,
GetMetricType()); GetMetricType());
} }
milvus::tracer::AddEvent("start_knowhere_index_range_search"); milvus::tracer::AddEvent("start_knowhere_index_range_search");
auto res = index_.RangeSearch(*dataset, search_conf, bitset); auto res = index_.RangeSearch(dataset, search_conf, bitset);
milvus::tracer::AddEvent("finish_knowhere_index_range_search"); milvus::tracer::AddEvent("finish_knowhere_index_range_search");
if (!res.has_value()) { if (!res.has_value()) {
PanicInfo(ErrorCode::UnexpectedError, PanicInfo(ErrorCode::UnexpectedError,
@ -615,7 +615,7 @@ VectorMemIndex<T>::Query(const DatasetPtr dataset,
return result; return result;
} else { } else {
milvus::tracer::AddEvent("start_knowhere_index_search"); milvus::tracer::AddEvent("start_knowhere_index_search");
auto res = index_.Search(*dataset, search_conf, bitset); auto res = index_.Search(dataset, search_conf, bitset);
milvus::tracer::AddEvent("finish_knowhere_index_search"); milvus::tracer::AddEvent("finish_knowhere_index_search");
if (!res.has_value()) { if (!res.has_value()) {
PanicInfo(ErrorCode::UnexpectedError, PanicInfo(ErrorCode::UnexpectedError,
@ -662,7 +662,7 @@ VectorMemIndex<T>::GetVector(const DatasetPtr dataset) const {
"failed to get vector, index is sparse"); "failed to get vector, index is sparse");
} }
auto res = index_.GetVectorByIds(*dataset); auto res = index_.GetVectorByIds(dataset);
if (!res.has_value()) { if (!res.has_value()) {
PanicInfo(ErrorCode::UnexpectedError, PanicInfo(ErrorCode::UnexpectedError,
"failed to get vector, " + KnowhereStatusString(res.error())); "failed to get vector, " + KnowhereStatusString(res.error()));
@ -680,7 +680,7 @@ VectorMemIndex<T>::GetVector(const DatasetPtr dataset) const {
template <typename T> template <typename T>
std::unique_ptr<const knowhere::sparse::SparseRow<float>[]> std::unique_ptr<const knowhere::sparse::SparseRow<float>[]>
VectorMemIndex<T>::GetSparseVector(const DatasetPtr dataset) const { VectorMemIndex<T>::GetSparseVector(const DatasetPtr dataset) const {
auto res = index_.GetVectorByIds(*dataset); auto res = index_.GetVectorByIds(dataset);
if (!res.has_value()) { if (!res.has_value()) {
PanicInfo(ErrorCode::UnexpectedError, PanicInfo(ErrorCode::UnexpectedError,
"failed to get vector, " + KnowhereStatusString(res.error())); "failed to get vector, " + KnowhereStatusString(res.error()));

3
internal/core/src/index/VectorMemIndex.h
View File

@ -94,8 +94,7 @@ class VectorMemIndex : public VectorIndex {
BinarySet BinarySet
UploadV2(const Config& config = {}) override; UploadV2(const Config& config = {}) override;
knowhere::expected< knowhere::expected<std::vector<knowhere::IndexNode::IteratorPtr>>
std::vector<std::shared_ptr<knowhere::IndexNode::iterator>>>
VectorIterators(const DatasetPtr dataset, VectorIterators(const DatasetPtr dataset,
const knowhere::Json& json, const knowhere::Json& json,
const BitsetView& bitset) const override; const BitsetView& bitset) const override;

3
internal/core/src/query/GroupByOperator.h
View File

@ -133,8 +133,7 @@ PrepareVectorIteratorsFromIndex(const SearchInfo& search_info,
if (search_info.group_by_field_id_.has_value()) { if (search_info.group_by_field_id_.has_value()) {
try { try {
auto search_conf = search_info.search_params_; auto search_conf = search_info.search_params_;
knowhere::expected< knowhere::expected<std::vector<knowhere::IndexNode::IteratorPtr>>
std::vector<std::shared_ptr<knowhere::IndexNode::iterator>>>
iterators_val = iterators_val =
index.VectorIterators(dataset, search_conf, bitset); index.VectorIterators(dataset, search_conf, bitset);
if (iterators_val.has_value()) { if (iterators_val.has_value()) {

3
internal/core/src/query/SearchBruteForce.cpp
View File

@ -205,8 +205,7 @@ BruteForceSearchIterators(const dataset::SearchDataset& dataset,
} }
auto search_cfg = PrepareBFSearchParams(search_info); auto search_cfg = PrepareBFSearchParams(search_info);
knowhere::expected< knowhere::expected<std::vector<knowhere::IndexNode::IteratorPtr>>
std::vector<std::shared_ptr<knowhere::IndexNode::iterator>>>
iterators_val; iterators_val;
switch (data_type) { switch (data_type) {
case DataType::VECTOR_FLOAT: case DataType::VECTOR_FLOAT:

9
internal/core/src/query/SubSearchResult.h
View File

@ -27,8 +27,7 @@ class SubSearchResult {
int64_t topk, int64_t topk,
const MetricType& metric_type, const MetricType& metric_type,
int64_t round_decimal, int64_t round_decimal,
const std::vector<std::shared_ptr<knowhere::IndexNode::iterator>>& const std::vector<knowhere::IndexNode::IteratorPtr>& iters)
iters)
: num_queries_(num_queries), : num_queries_(num_queries),
topk_(topk), topk_(topk),
round_decimal_(round_decimal), round_decimal_(round_decimal),
@ -47,7 +46,7 @@ class SubSearchResult {
topk, topk,
metric_type, metric_type,
round_decimal, round_decimal,
std::vector<std::shared_ptr<knowhere::IndexNode::iterator>>{}) { std::vector<knowhere::IndexNode::IteratorPtr>{}) {
} }
SubSearchResult(SubSearchResult&& other) noexcept SubSearchResult(SubSearchResult&& other) noexcept
@ -114,7 +113,7 @@ class SubSearchResult {
void void
merge(const SubSearchResult& other); merge(const SubSearchResult& other);
const std::vector<std::shared_ptr<knowhere::IndexNode::iterator>>& const std::vector<knowhere::IndexNode::IteratorPtr>&
chunk_iterators() { chunk_iterators() {
return this->chunk_iterators_; return this->chunk_iterators_;
} }
@ -131,7 +130,7 @@ class SubSearchResult {
knowhere::MetricType metric_type_; knowhere::MetricType metric_type_;
std::vector<int64_t> seg_offsets_; std::vector<int64_t> seg_offsets_;
std::vector<float> distances_; std::vector<float> distances_;
std::vector<std::shared_ptr<knowhere::IndexNode::iterator>> std::vector<knowhere::IndexNode::IteratorPtr>
chunk_iterators_; chunk_iterators_;
}; };

2
internal/core/thirdparty/knowhere/CMakeLists.txt vendored
View File

@ -13,7 +13,7 @@
# Update KNOWHERE_VERSION for the first occurrence # Update KNOWHERE_VERSION for the first occurrence
set_property(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES "") set_property(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES "")
set( KNOWHERE_VERSION d69e0f33 ) set( KNOWHERE_VERSION 6bd3ba4c )
set( GIT_REPOSITORY "https://github.com/zilliztech/knowhere.git") set( GIT_REPOSITORY "https://github.com/zilliztech/knowhere.git")
message(STATUS "Knowhere repo: ${GIT_REPOSITORY}") message(STATUS "Knowhere repo: ${GIT_REPOSITORY}")
message(STATUS "Knowhere version: ${KNOWHERE_VERSION}") message(STATUS "Knowhere version: ${KNOWHERE_VERSION}")

10
internal/core/unittest/test_c_api.cpp
View File

@ -1958,8 +1958,8 @@ TEST(CApiTest, LoadIndexInfo) {
{knowhere::indexparam::NPROBE, 4}}; {knowhere::indexparam::NPROBE, 4}};
auto database = knowhere::GenDataSet(N, DIM, raw_data.data()); auto database = knowhere::GenDataSet(N, DIM, raw_data.data());
indexing.Train(*database, conf); indexing.Train(database, conf);
indexing.Add(*database, conf); indexing.Add(database, conf);
EXPECT_EQ(indexing.Count(), N); EXPECT_EQ(indexing.Count(), N);
EXPECT_EQ(indexing.Dim(), DIM); EXPECT_EQ(indexing.Dim(), DIM);
knowhere::BinarySet binary_set; knowhere::BinarySet binary_set;
@ -2009,8 +2009,8 @@ TEST(CApiTest, LoadIndexSearch) {
{knowhere::indexparam::NPROBE, 4}}; {knowhere::indexparam::NPROBE, 4}};
auto database = knowhere::GenDataSet(N, DIM, raw_data.data()); auto database = knowhere::GenDataSet(N, DIM, raw_data.data());
indexing.Train(*database, conf); indexing.Train(database, conf);
indexing.Add(*database, conf); indexing.Add(database, conf);
EXPECT_EQ(indexing.Count(), N); EXPECT_EQ(indexing.Count(), N);
EXPECT_EQ(indexing.Dim(), DIM); EXPECT_EQ(indexing.Dim(), DIM);
@ -2033,7 +2033,7 @@ TEST(CApiTest, LoadIndexSearch) {
auto query_dataset = auto query_dataset =
knowhere::GenDataSet(num_query, DIM, raw_data.data() + BIAS * DIM); knowhere::GenDataSet(num_query, DIM, raw_data.data() + BIAS * DIM);
auto result = indexing.Search(*query_dataset, conf, nullptr); auto result = indexing.Search(query_dataset, conf, nullptr);
} }
TEST(CApiTest, Indexing_Without_Predicate) { TEST(CApiTest, Indexing_Without_Predicate) {

3
internal/core/unittest/test_indexing.cpp
View File

@ -449,8 +449,7 @@ TEST(Indexing, Iterator) {
searchInfo.search_params_ = search_conf; searchInfo.search_params_ = search_conf;
auto vec_index = dynamic_cast<index::VectorIndex*>(index.get()); auto vec_index = dynamic_cast<index::VectorIndex*>(index.get());
knowhere::expected< knowhere::expected<std::vector<knowhere::IndexNode::IteratorPtr>>
std::vector<std::shared_ptr<knowhere::IndexNode::iterator>>>
kw_iterators = vec_index->VectorIterators( kw_iterators = vec_index->VectorIterators(
query_ds, searchInfo.search_params_, view); query_ds, searchInfo.search_params_, view);
ASSERT_TRUE(kw_iterators.has_value()); ASSERT_TRUE(kw_iterators.has_value());