test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-11-29 18:38:44 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

Add float16 vector (#25852)

Browse Source 
Signed-off-by: Writer-X <1256866856@qq.com>
This commit is contained in:
Xu Tong 2023-09-08 10:03:16 +08:00 committed by GitHub
parent eb3ea3a1df
commit 9166011c4a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
87 changed files with 2154 additions and 401 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
internal/core/src/common/FieldMeta.h
View File

@ -49,6 +49,9 @@ datatype_sizeof(DataType data_type, int dim = 1) {
AssertInfo(dim % 8 == 0, "dim=" + std::to_string(dim));
return dim / 8;
}
case DataType::VECTOR_FLOAT16: {
return sizeof(float16) * dim;
}
default: {
throw std::invalid_argument("unsupported data type");
}
@ -84,6 +87,9 @@ datatype_name(DataType data_type) {
case DataType::VECTOR_BINARY: {
return "vector_binary";
}
case DataType::VECTOR_FLOAT16: {
return "vector_float16";
}
default: {
auto err_msg =
"Unsupported DataType(" + std::to_string((int)data_type) + ")";
@ -95,7 +101,8 @@ datatype_name(DataType data_type) {
inline bool
datatype_is_vector(DataType datatype) {
return datatype == DataType::VECTOR_BINARY ||
datatype == DataType::VECTOR_FLOAT;
datatype == DataType::VECTOR_FLOAT ||
datatype == DataType::VECTOR_FLOAT16;
}
inline bool

31
internal/core/src/common/Types.h
View File

@ -51,6 +51,33 @@ using offset_t = int32_t;
using date_t = int32_t;
using distance_t = float;
union float16 {
unsigned short bits;
struct {
unsigned short mantissa : 10;
unsigned short exponent : 5;
unsigned short sign : 1;
} parts;
float16() {
}
float16(float f) {
unsigned int i = *(unsigned int*)&f;
unsigned int sign = (i >> 31) & 0x0001;
unsigned int exponent = ((i >> 23) & 0xff) - 127 + 15;
unsigned int mantissa = (i >> 13) & 0x3ff;
parts.sign = sign;
parts.exponent = exponent;
parts.mantissa = mantissa;
}
operator float() const {
unsigned int sign = parts.sign << 31;
unsigned int exponent = (parts.exponent - 15 + 127) << 23;
unsigned int mantissa = parts.mantissa << 13;
unsigned int bits = sign | exponent | mantissa;
return *(float*)&bits;
}
};
enum class DataType {
NONE = 0,
BOOL = 1,
@ -69,6 +96,7 @@ enum class DataType {
VECTOR_BINARY = 100,
VECTOR_FLOAT = 101,
VECTOR_FLOAT16 = 102,
};
using Timestamp = uint64_t; // TODO: use TiKV-like timestamp
@ -202,6 +230,9 @@ struct fmt::formatter<milvus::DataType> : formatter<string_view> {
case milvus::DataType::VECTOR_FLOAT:
name = "VECTOR_FLOAT";
break;
case milvus::DataType::VECTOR_FLOAT16:
name = "VECTOR_FLOAT16";
break;
}
return formatter<string_view>::format(name, ctx);
}

14
internal/core/src/common/VectorTrait.h
View File

@ -35,12 +35,20 @@ class BinaryVector : public VectorTrait {
static constexpr auto metric_type = DataType::VECTOR_BINARY;
};
class Float16Vector : public VectorTrait {
public:
using embedded_type = float16;
static constexpr auto metric_type = DataType::VECTOR_FLOAT16;
};
template <typename VectorType>
inline constexpr int64_t
element_sizeof(int64_t dim) {
static_assert(std::is_base_of_v<VectorType, VectorTrait>);
if constexpr (std::is_same_v<VectorType, FloatVector>) {
return dim * sizeof(float);
} else if constexpr (std::is_same_v<VectorType, Float16Vector>) {
return dim * sizeof(float16);
} else {
return dim / 8;
}
@ -64,8 +72,10 @@ struct EmbeddedTypeImpl<T, std::enable_if_t<IsScalar<T>>> {
template <typename T>
struct EmbeddedTypeImpl<T, std::enable_if_t<IsVector<T>>> {
using type =
std::conditional_t<std::is_same_v<T, FloatVector>, float, uint8_t>;
using type = std::conditional_t<
std::is_same_v<T, FloatVector>,
float,
std::conditional_t<std::is_same_v<T, Float16Vector>, float16, uint8_t>>;
};
template <typename T>

1
internal/core/src/common/type_c.h
View File

@ -56,6 +56,7 @@ enum CDataType {
BinaryVector = 100,
FloatVector = 101,
Float16Vector = 102,
};
typedef enum CDataType CDataType;

105
internal/core/src/pb/plan.pb.cc
View File

@ -309,7 +309,7 @@ PROTOBUF_CONSTEXPR VectorANNS::VectorANNS(
, /*decltype(_impl_.predicates_)*/nullptr
, /*decltype(_impl_.query_info_)*/nullptr
, /*decltype(_impl_.field_id_)*/int64_t{0}
, /*decltype(_impl_.is_binary_)*/false
, /*decltype(_impl_.vector_type_)*/0
, /*decltype(_impl_._cached_size_)*/{}} {}
struct VectorANNSDefaultTypeInternal {
PROTOBUF_CONSTEXPR VectorANNSDefaultTypeInternal()
@ -355,7 +355,7 @@ PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORIT
} // namespace proto
} // namespace milvus
static ::_pb::Metadata file_level_metadata_plan_2eproto[22];
static const ::_pb::EnumDescriptor* file_level_enum_descriptors_plan_2eproto[5];
static const ::_pb::EnumDescriptor* file_level_enum_descriptors_plan_2eproto[6];
static constexpr ::_pb::ServiceDescriptor const** file_level_service_descriptors_plan_2eproto = nullptr;
const uint32_t TableStruct_plan_2eproto::offsets[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {
@ -548,7 +548,7 @@ const uint32_t TableStruct_plan_2eproto::offsets[] PROTOBUF_SECTION_VARIABLE(pro
~0u, // no _oneof_case_
~0u, // no _weak_field_map_
~0u, // no _inlined_string_donated_
PROTOBUF_FIELD_OFFSET(::milvus::proto::plan::VectorANNS, _impl_.is_binary_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::plan::VectorANNS, _impl_.vector_type_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::plan::VectorANNS, _impl_.field_id_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::plan::VectorANNS, _impl_.predicates_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::plan::VectorANNS, _impl_.query_info_),
@ -711,33 +711,36 @@ const char descriptor_table_protodef_plan_2eproto[] PROTOBUF_SECTION_VARIABLE(pr
"_true_expr\030\014 \001(\0132!.milvus.proto.plan.Alw"
"aysTrueExprH\000\022A\n\022json_contains_expr\030\r \001("
"\0132#.milvus.proto.plan.JSONContainsExprH\000"
"B\006\n\004expr\"\251\001\n\nVectorANNS\022\021\n\tis_binary\030\001 \001"
"(\010\022\020\n\010field_id\030\002 \001(\003\022+\n\npredicates\030\003 \001(\013"
"2\027.milvus.proto.plan.Expr\0220\n\nquery_info\030"
"\004 \001(\0132\034.milvus.proto.plan.QueryInfo\022\027\n\017p"
"laceholder_tag\030\005 \001(\t\"]\n\rQueryPlanNode\022+\n"
"\npredicates\030\001 \001(\0132\027.milvus.proto.plan.Ex"
"pr\022\020\n\010is_count\030\002 \001(\010\022\r\n\005limit\030\003 \001(\003\"\304\001\n\010"
"PlanNode\0224\n\013vector_anns\030\001 \001(\0132\035.milvus.p"
"roto.plan.VectorANNSH\000\022-\n\npredicates\030\002 \001"
"(\0132\027.milvus.proto.plan.ExprH\000\0221\n\005query\030\004"
" \001(\0132 .milvus.proto.plan.QueryPlanNodeH\000"
"\022\030\n\020output_field_ids\030\003 \003(\003B\006\n\004node*\272\001\n\006O"
"pType\022\013\n\007Invalid\020\000\022\017\n\013GreaterThan\020\001\022\020\n\014G"
"reaterEqual\020\002\022\014\n\010LessThan\020\003\022\r\n\tLessEqual"
"\020\004\022\t\n\005Equal\020\005\022\014\n\010NotEqual\020\006\022\017\n\013PrefixMat"
"ch\020\007\022\020\n\014PostfixMatch\020\010\022\t\n\005Match\020\t\022\t\n\005Ran"
"ge\020\n\022\006\n\002In\020\013\022\t\n\005NotIn\020\014*G\n\013ArithOpType\022\013"
"\n\007Unknown\020\000\022\007\n\003Add\020\001\022\007\n\003Sub\020\002\022\007\n\003Mul\020\003\022\007"
"\n\003Div\020\004\022\007\n\003Mod\020\005B3Z1github.com/milvus-io"
"/milvus/internal/proto/planpbb\006proto3"
"B\006\n\004expr\"\312\001\n\nVectorANNS\0222\n\013vector_type\030\001"
" \001(\0162\035.milvus.proto.plan.VectorType\022\020\n\010f"
"ield_id\030\002 \001(\003\022+\n\npredicates\030\003 \001(\0132\027.milv"
"us.proto.plan.Expr\0220\n\nquery_info\030\004 \001(\0132\034"
".milvus.proto.plan.QueryInfo\022\027\n\017placehol"
"der_tag\030\005 \001(\t\"]\n\rQueryPlanNode\022+\n\npredic"
"ates\030\001 \001(\0132\027.milvus.proto.plan.Expr\022\020\n\010i"
"s_count\030\002 \001(\010\022\r\n\005limit\030\003 \001(\003\"\304\001\n\010PlanNod"
"e\0224\n\013vector_anns\030\001 \001(\0132\035.milvus.proto.pl"
"an.VectorANNSH\000\022-\n\npredicates\030\002 \001(\0132\027.mi"
"lvus.proto.plan.ExprH\000\0221\n\005query\030\004 \001(\0132 ."
"milvus.proto.plan.QueryPlanNodeH\000\022\030\n\020out"
"put_field_ids\030\003 \003(\003B\006\n\004node*\272\001\n\006OpType\022\013"
"\n\007Invalid\020\000\022\017\n\013GreaterThan\020\001\022\020\n\014GreaterE"
"qual\020\002\022\014\n\010LessThan\020\003\022\r\n\tLessEqual\020\004\022\t\n\005E"
"qual\020\005\022\014\n\010NotEqual\020\006\022\017\n\013PrefixMatch\020\007\022\020\n"
"\014PostfixMatch\020\010\022\t\n\005Match\020\t\022\t\n\005Range\020\n\022\006\n"
"\002In\020\013\022\t\n\005NotIn\020\014*G\n\013ArithOpType\022\013\n\007Unkno"
"wn\020\000\022\007\n\003Add\020\001\022\007\n\003Sub\020\002\022\007\n\003Mul\020\003\022\007\n\003Div\020\004"
"\022\007\n\003Mod\020\005*B\n\nVectorType\022\020\n\014BinaryVector\020"
"\000\022\017\n\013FloatVector\020\001\022\021\n\rFloat16Vector\020\002B3Z"
"1github.com/milvus-io/milvus/internal/pr"
"oto/planpbb\006proto3"
;
static const ::_pbi::DescriptorTable* const descriptor_table_plan_2eproto_deps[1] = {
&::descriptor_table_schema_2eproto,
};
static ::_pbi::once_flag descriptor_table_plan_2eproto_once;
const ::_pbi::DescriptorTable descriptor_table_plan_2eproto = {
false, false, 4237, descriptor_table_protodef_plan_2eproto,
false, false, 4338, descriptor_table_protodef_plan_2eproto,
"plan.proto",
&descriptor_table_plan_2eproto_once, descriptor_table_plan_2eproto_deps, 1, 22,
schemas, file_default_instances, TableStruct_plan_2eproto::offsets,
@ -865,6 +868,21 @@ bool ArithOpType_IsValid(int value) {
}
}
const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* VectorType_descriptor() {
::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&descriptor_table_plan_2eproto);
return file_level_enum_descriptors_plan_2eproto[5];
}
bool VectorType_IsValid(int value) {
switch (value) {
case 0:
case 1:
case 2:
return true;
default:
return false;
}
}
// ===================================================================
@ -6475,7 +6493,7 @@ VectorANNS::VectorANNS(const VectorANNS& from)
, decltype(_impl_.predicates_){nullptr}
, decltype(_impl_.query_info_){nullptr}
, decltype(_impl_.field_id_){}
, decltype(_impl_.is_binary_){}
, decltype(_impl_.vector_type_){}
, /*decltype(_impl_._cached_size_)*/{}};
_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_);
@ -6494,8 +6512,8 @@ VectorANNS::VectorANNS(const VectorANNS& from)
_this->_impl_.query_info_ = new ::milvus::proto::plan::QueryInfo(*from._impl_.query_info_);
}
::memcpy(&_impl_.field_id_, &from._impl_.field_id_,
static_cast<size_t>(reinterpret_cast<char*>(&_impl_.is_binary_) -
reinterpret_cast<char*>(&_impl_.field_id_)) + sizeof(_impl_.is_binary_));
static_cast<size_t>(reinterpret_cast<char*>(&_impl_.vector_type_) -
reinterpret_cast<char*>(&_impl_.field_id_)) + sizeof(_impl_.vector_type_));
// @@protoc_insertion_point(copy_constructor:milvus.proto.plan.VectorANNS)
}
@ -6508,7 +6526,7 @@ inline void VectorANNS::SharedCtor(
, decltype(_impl_.predicates_){nullptr}
, decltype(_impl_.query_info_){nullptr}
, decltype(_impl_.field_id_){int64_t{0}}
, decltype(_impl_.is_binary_){false}
, decltype(_impl_.vector_type_){0}
, /*decltype(_impl_._cached_size_)*/{}
};
_impl_.placeholder_tag_.InitDefault();
@ -6553,8 +6571,8 @@ void VectorANNS::Clear() {
}
_impl_.query_info_ = nullptr;
::memset(&_impl_.field_id_, 0, static_cast<size_t>(
reinterpret_cast<char*>(&_impl_.is_binary_) -
reinterpret_cast<char*>(&_impl_.field_id_)) + sizeof(_impl_.is_binary_));
reinterpret_cast<char*>(&_impl_.vector_type_) -
reinterpret_cast<char*>(&_impl_.field_id_)) + sizeof(_impl_.vector_type_));
_internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>();
}
@ -6564,11 +6582,12 @@ const char* VectorANNS::_InternalParse(const char* ptr, ::_pbi::ParseContext* ct
uint32_t tag;
ptr = ::_pbi::ReadTag(ptr, &tag);
switch (tag >> 3) {
// bool is_binary = 1;
// .milvus.proto.plan.VectorType vector_type = 1;
case 1:
if (PROTOBUF_PREDICT_TRUE(static_cast<uint8_t>(tag) == 8)) {
_impl_.is_binary_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
uint64_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
_internal_set_vector_type(static_cast<::milvus::proto::plan::VectorType>(val));
} else
goto handle_unusual;
continue;
@ -6635,10 +6654,11 @@ uint8_t* VectorANNS::_InternalSerialize(
uint32_t cached_has_bits = 0;
(void) cached_has_bits;
// bool is_binary = 1;
if (this->_internal_is_binary() != 0) {
// .milvus.proto.plan.VectorType vector_type = 1;
if (this->_internal_vector_type() != 0) {
target = stream->EnsureSpace(target);
target = ::_pbi::WireFormatLite::WriteBoolToArray(1, this->_internal_is_binary(), target);
target = ::_pbi::WireFormatLite::WriteEnumToArray(
1, this->_internal_vector_type(), target);
}
// int64 field_id = 2;
@ -6713,9 +6733,10 @@ size_t VectorANNS::ByteSizeLong() const {
total_size += ::_pbi::WireFormatLite::Int64SizePlusOne(this->_internal_field_id());
}
// bool is_binary = 1;
if (this->_internal_is_binary() != 0) {
total_size += 1 + 1;
// .milvus.proto.plan.VectorType vector_type = 1;
if (this->_internal_vector_type() != 0) {
total_size += 1 +
::_pbi::WireFormatLite::EnumSize(this->_internal_vector_type());
}
return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_);
@ -6750,8 +6771,8 @@ void VectorANNS::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PRO
if (from._internal_field_id() != 0) {
_this->_internal_set_field_id(from._internal_field_id());
}
if (from._internal_is_binary() != 0) {
_this->_internal_set_is_binary(from._internal_is_binary());
if (from._internal_vector_type() != 0) {
_this->_internal_set_vector_type(from._internal_vector_type());
}
_this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_);
}
@ -6777,8 +6798,8 @@ void VectorANNS::InternalSwap(VectorANNS* other) {
&other->_impl_.placeholder_tag_, rhs_arena
);
::PROTOBUF_NAMESPACE_ID::internal::memswap<
PROTOBUF_FIELD_OFFSET(VectorANNS, _impl_.is_binary_)
+ sizeof(VectorANNS::_impl_.is_binary_)
PROTOBUF_FIELD_OFFSET(VectorANNS, _impl_.vector_type_)
+ sizeof(VectorANNS::_impl_.vector_type_)
- PROTOBUF_FIELD_OFFSET(VectorANNS, _impl_.predicates_)>(
reinterpret_cast<char*>(&_impl_.predicates_),
reinterpret_cast<char*>(&other->_impl_.predicates_));

73
internal/core/src/pb/plan.pb.h
View File

@ -290,6 +290,32 @@ inline bool ArithOpType_Parse(
return ::PROTOBUF_NAMESPACE_ID::internal::ParseNamedEnum<ArithOpType>(
ArithOpType_descriptor(), name, value);
}
enum VectorType : int {
BinaryVector = 0,
FloatVector = 1,
Float16Vector = 2,
VectorType_INT_MIN_SENTINEL_DO_NOT_USE_ = std::numeric_limits<int32_t>::min(),
VectorType_INT_MAX_SENTINEL_DO_NOT_USE_ = std::numeric_limits<int32_t>::max()
};
bool VectorType_IsValid(int value);
constexpr VectorType VectorType_MIN = BinaryVector;
constexpr VectorType VectorType_MAX = Float16Vector;
constexpr int VectorType_ARRAYSIZE = VectorType_MAX + 1;
const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* VectorType_descriptor();
template<typename T>
inline const std::string& VectorType_Name(T enum_t_value) {
static_assert(::std::is_same<T, VectorType>::value ||
::std::is_integral<T>::value,
"Incorrect type passed to function VectorType_Name.");
return ::PROTOBUF_NAMESPACE_ID::internal::NameOfEnum(
VectorType_descriptor(), enum_t_value);
}
inline bool VectorType_Parse(
::PROTOBUF_NAMESPACE_ID::ConstStringParam name, VectorType* value) {
return ::PROTOBUF_NAMESPACE_ID::internal::ParseNamedEnum<VectorType>(
VectorType_descriptor(), name, value);
}
// ===================================================================
class GenericValue final :
@ -4297,7 +4323,7 @@ class VectorANNS final :
kPredicatesFieldNumber = 3,
kQueryInfoFieldNumber = 4,
kFieldIdFieldNumber = 2,
kIsBinaryFieldNumber = 1,
kVectorTypeFieldNumber = 1,
};
// string placeholder_tag = 5;
void clear_placeholder_tag();
@ -4358,13 +4384,13 @@ class VectorANNS final :
void _internal_set_field_id(int64_t value);
public:
// bool is_binary = 1;
void clear_is_binary();
bool is_binary() const;
void set_is_binary(bool value);
// .milvus.proto.plan.VectorType vector_type = 1;
void clear_vector_type();
::milvus::proto::plan::VectorType vector_type() const;
void set_vector_type(::milvus::proto::plan::VectorType value);
private:
bool _internal_is_binary() const;
void _internal_set_is_binary(bool value);
::milvus::proto::plan::VectorType _internal_vector_type() const;
void _internal_set_vector_type(::milvus::proto::plan::VectorType value);
public:
// @@protoc_insertion_point(class_scope:milvus.proto.plan.VectorANNS)
@ -4379,7 +4405,7 @@ class VectorANNS final :
::milvus::proto::plan::Expr* predicates_;
::milvus::proto::plan::QueryInfo* query_info_;
int64_t field_id_;
bool is_binary_;
int vector_type_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
};
union { Impl_ _impl_; };
@ -8834,24 +8860,24 @@ inline Expr::ExprCase Expr::expr_case() const {
// VectorANNS
// bool is_binary = 1;
inline void VectorANNS::clear_is_binary() {
_impl_.is_binary_ = false;
// .milvus.proto.plan.VectorType vector_type = 1;
inline void VectorANNS::clear_vector_type() {
_impl_.vector_type_ = 0;
}
inline bool VectorANNS::_internal_is_binary() const {
return _impl_.is_binary_;
inline ::milvus::proto::plan::VectorType VectorANNS::_internal_vector_type() const {
return static_cast< ::milvus::proto::plan::VectorType >(_impl_.vector_type_);
}
inline bool VectorANNS::is_binary() const {
// @@protoc_insertion_point(field_get:milvus.proto.plan.VectorANNS.is_binary)
return _internal_is_binary();
inline ::milvus::proto::plan::VectorType VectorANNS::vector_type() const {
// @@protoc_insertion_point(field_get:milvus.proto.plan.VectorANNS.vector_type)
return _internal_vector_type();
}
inline void VectorANNS::_internal_set_is_binary(bool value) {
inline void VectorANNS::_internal_set_vector_type(::milvus::proto::plan::VectorType value) {
_impl_.is_binary_ = value;
_impl_.vector_type_ = value;
}
inline void VectorANNS::set_is_binary(bool value) {
_internal_set_is_binary(value);
// @@protoc_insertion_point(field_set:milvus.proto.plan.VectorANNS.is_binary)
inline void VectorANNS::set_vector_type(::milvus::proto::plan::VectorType value) {
_internal_set_vector_type(value);
// @@protoc_insertion_point(field_set:milvus.proto.plan.VectorANNS.vector_type)
}
// int64 field_id = 2;
@ -9599,6 +9625,11 @@ template <>
inline const EnumDescriptor* GetEnumDescriptor< ::milvus::proto::plan::ArithOpType>() {
return ::milvus::proto::plan::ArithOpType_descriptor();
}
template <> struct is_proto_enum< ::milvus::proto::plan::VectorType> : ::std::true_type {};
template <>
inline const EnumDescriptor* GetEnumDescriptor< ::milvus::proto::plan::VectorType>() {
return ::milvus::proto::plan::VectorType_descriptor();
}
PROTOBUF_NAMESPACE_CLOSE

6
internal/core/src/query/PlanNode.h
View File

@ -52,6 +52,12 @@ struct BinaryVectorANNS : VectorPlanNode {
accept(PlanNodeVisitor&) override;
};
struct Float16VectorANNS : VectorPlanNode {
public:
void
accept(PlanNodeVisitor&) override;
};
struct RetrievePlanNode : PlanNode {
public:
void

6
internal/core/src/query/PlanProto.cpp
View File

@ -185,8 +185,12 @@ ProtoParser::PlanNodeFromProto(const planpb::PlanNode& plan_node_proto) {
search_info.search_params_ = json::parse(query_info_proto.search_params());
auto plan_node = [&]() -> std::unique_ptr<VectorPlanNode> {
if (anns_proto.is_binary()) {
if (anns_proto.vector_type() ==
milvus::proto::plan::VectorType::BinaryVector) {
return std::make_unique<BinaryVectorANNS>();
} else if (anns_proto.vector_type() ==
milvus::proto::plan::VectorType::Float16Vector) {
return std::make_unique<Float16VectorANNS>();
} else {
return std::make_unique<FloatVectorANNS>();
}

38
internal/core/src/query/SearchBruteForce.cpp
View File

@ -30,7 +30,8 @@ CheckBruteForceSearchParam(const FieldMeta& field,
AssertInfo(datatype_is_vector(data_type),
"[BruteForceSearch] Data type isn't vector type");
bool is_float_data_type = (data_type == DataType::VECTOR_FLOAT);
bool is_float_data_type = (data_type == DataType::VECTOR_FLOAT ||
data_type == DataType::VECTOR_FLOAT16);
bool is_float_metric_type = IsFloatMetricType(metric_type);
AssertInfo(is_float_data_type == is_float_metric_type,
"[BruteForceSearch] Data type and metric type miss-match");
@ -41,7 +42,8 @@ BruteForceSearch(const dataset::SearchDataset& dataset,
const void* chunk_data_raw,
int64_t chunk_rows,
const knowhere::Json& conf,
const BitsetView& bitset) {
const BitsetView& bitset,
DataType data_type) {
SubSearchResult sub_result(dataset.num_queries,
dataset.topk,
dataset.metric_type,
@ -54,6 +56,38 @@ BruteForceSearch(const dataset::SearchDataset& dataset,
auto base_dataset =
knowhere::GenDataSet(chunk_rows, dim, chunk_data_raw);
auto query_dataset = knowhere::GenDataSet(nq, dim, dataset.query_data);
if (data_type == DataType::VECTOR_FLOAT16) {
// Todo: Temporarily use cast to float32 to achieve, need to optimize
// first, First, transfer the cast to knowhere part
// second, knowhere partially supports float16 and removes the forced conversion to float32
auto xb = base_dataset->GetTensor();
std::vector<float> float_xb(base_dataset->GetRows() *
base_dataset->GetDim());
auto xq = query_dataset->GetTensor();
std::vector<float> float_xq(query_dataset->GetRows() *
query_dataset->GetDim());
auto fp16_xb = (const float16*)xb;
for (int i = 0;
i < base_dataset->GetRows() * base_dataset->GetDim();
i++) {
float_xb[i] = (float)fp16_xb[i];
}
auto fp16_xq = (const float16*)xq;
for (int i = 0;
i < query_dataset->GetRows() * query_dataset->GetDim();
i++) {
float_xq[i] = (float)fp16_xq[i];
}
void* void_ptr_xb = static_cast<void*>(float_xb.data());
void* void_ptr_xq = static_cast<void*>(float_xq.data());
base_dataset = knowhere::GenDataSet(chunk_rows, dim, void_ptr_xb);
query_dataset = knowhere::GenDataSet(nq, dim, void_ptr_xq);
}
auto config = knowhere::Json{
{knowhere::meta::METRIC_TYPE, dataset.metric_type},
{knowhere::meta::DIM, dim},

3
internal/core/src/query/SearchBruteForce.h
View File

@ -28,6 +28,7 @@ BruteForceSearch(const dataset::SearchDataset& dataset,
const void* chunk_data_raw,
int64_t chunk_rows,
const knowhere::Json& conf,
const BitsetView& bitset);
const BitsetView& bitset,
DataType data_type = DataType::VECTOR_FLOAT);
} // namespace milvus::query

4
internal/core/src/query/SearchOnGrowing.cpp
View File

@ -123,7 +123,9 @@ SearchOnGrowing(const segcore::SegmentGrowingImpl& segment,
chunk_data,
size_per_chunk,
info.search_params_,
sub_view);
sub_view,
data_type);
// convert chunk uid to segment uid
for (auto& x : sub_qr.mutable_seg_offsets()) {
if (x != -1) {

9
internal/core/src/query/SearchOnSealed.cpp
View File

@ -84,9 +84,14 @@ SearchOnSealed(const Schema& schema,
field.get_dim(),
query_data};
auto data_type = field.get_data_type();
CheckBruteForceSearchParam(field, search_info);
auto sub_qr = BruteForceSearch(
dataset, vec_data, row_count, search_info.search_params_, bitset);
auto sub_qr = BruteForceSearch(dataset,
vec_data,
row_count,
search_info.search_params_,
bitset,
data_type);
result.distances_ = std::move(sub_qr.mutable_distances());
result.seg_offsets_ = std::move(sub_qr.mutable_seg_offsets());

3
internal/core/src/query/generated/ExecPlanNodeVisitor.h
View File

@ -27,6 +27,9 @@ class ExecPlanNodeVisitor : public PlanNodeVisitor {
void
visit(BinaryVectorANNS& node) override;
void
visit(Float16VectorANNS& node) override;
void
visit(RetrievePlanNode& node) override;

3
internal/core/src/query/generated/ExtractInfoPlanNodeVisitor.h
View File

@ -24,6 +24,9 @@ class ExtractInfoPlanNodeVisitor : public PlanNodeVisitor {
void
visit(BinaryVectorANNS& node) override;
void
visit(Float16VectorANNS& node) override;
void
visit(RetrievePlanNode& node) override;

5
internal/core/src/query/generated/PlanNode.cpp
View File

@ -25,6 +25,11 @@ BinaryVectorANNS::accept(PlanNodeVisitor& visitor) {
visitor.visit(*this);
}
void
Float16VectorANNS::accept(PlanNodeVisitor& visitor) {
visitor.visit(*this);
}
void
RetrievePlanNode::accept(PlanNodeVisitor& visitor) {
visitor.visit(*this);

3
internal/core/src/query/generated/PlanNodeVisitor.h
View File

@ -25,6 +25,9 @@ class PlanNodeVisitor {
virtual void
visit(BinaryVectorANNS&) = 0;
virtual void
visit(Float16VectorANNS&) = 0;
virtual void
visit(RetrievePlanNode&) = 0;
};

3
internal/core/src/query/generated/ShowPlanNodeVisitor.h
View File

@ -28,6 +28,9 @@ class ShowPlanNodeVisitor : public PlanNodeVisitor {
void
visit(BinaryVectorANNS& node) override;
void
visit(Float16VectorANNS& node) override;
void
visit(RetrievePlanNode& node) override;

3
internal/core/src/query/generated/VerifyPlanNodeVisitor.h
View File

@ -27,6 +27,9 @@ class VerifyPlanNodeVisitor : public PlanNodeVisitor {
void
visit(BinaryVectorANNS& node) override;
void
visit(Float16VectorANNS& node) override;
void
visit(RetrievePlanNode& node) override;

5
internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp
View File

@ -212,4 +212,9 @@ ExecPlanNodeVisitor::visit(BinaryVectorANNS& node) {
VectorVisitorImpl<BinaryVector>(node);
}
void
ExecPlanNodeVisitor::visit(Float16VectorANNS& node) {
VectorVisitorImpl<Float16Vector>(node);
}
} // namespace milvus::query

9
internal/core/src/query/visitors/ExtractInfoPlanNodeVisitor.cpp
View File

@ -47,6 +47,15 @@ ExtractInfoPlanNodeVisitor::visit(BinaryVectorANNS& node) {
}
}
void
ExtractInfoPlanNodeVisitor::visit(Float16VectorANNS& node) {
plan_info_.add_involved_field(node.search_info_.field_id_);
if (node.predicate_.has_value()) {
ExtractInfoExprVisitor expr_visitor(plan_info_);
node.predicate_.value()->accept(expr_visitor);
}
}
void
ExtractInfoPlanNodeVisitor::visit(RetrievePlanNode& node) {
// Assert(node.predicate_.has_value());

24
internal/core/src/query/visitors/ShowPlanNodeVisitor.cpp
View File

@ -96,6 +96,30 @@ ShowPlanNodeVisitor::visit(BinaryVectorANNS& node) {
ret_ = json_body;
}
void
ShowPlanNodeVisitor::visit(Float16VectorANNS& node) {
assert(!ret_);
auto& info = node.search_info_;
Json json_body{
{"node_type", "Float16VectorANNS"}, //
{"metric_type", info.metric_type_}, //
{"field_id_", info.field_id_.get()}, //
{"topk", info.topk_}, //
{"search_params", info.search_params_}, //
{"placeholder_tag", node.placeholder_tag_}, //
};
if (node.predicate_.has_value()) {
ShowExprVisitor expr_show;
AssertInfo(node.predicate_.value(),
"[ShowPlanNodeVisitor]Can't get value from node predict");
json_body["predicate"] =
expr_show.call_child(node.predicate_->operator*());
} else {
json_body["predicate"] = "None";
}
ret_ = json_body;
}
void
ShowPlanNodeVisitor::visit(RetrievePlanNode& node) {
}

4
internal/core/src/query/visitors/VerifyPlanNodeVisitor.cpp
View File

@ -67,6 +67,10 @@ void
VerifyPlanNodeVisitor::visit(BinaryVectorANNS&) {
}
void
VerifyPlanNodeVisitor::visit(Float16VectorANNS&) {
}
void
VerifyPlanNodeVisitor::visit(RetrievePlanNode&) {
}

3
internal/core/src/segcore/ConcurrentVector.cpp
View File

@ -30,6 +30,9 @@ VectorBase::set_data_raw(ssize_t element_offset,
} else if (field_meta.get_data_type() == DataType::VECTOR_BINARY) {
return set_data_raw(
element_offset, VEC_FIELD_DATA(data, binary), element_count);
} else if (field_meta.get_data_type() == DataType::VECTOR_FLOAT16) {
return set_data_raw(
element_offset, VEC_FIELD_DATA(data, float16), element_count);
} else {
PanicInfo("unsupported");
}

24
internal/core/src/segcore/ConcurrentVector.h
View File

@ -148,12 +148,14 @@ class ConcurrentVectorImpl : public VectorBase {
ConcurrentVectorImpl&
operator=(const ConcurrentVectorImpl&) = delete;
using TraitType =
std::conditional_t<is_scalar,
Type,
std::conditional_t<std::is_same_v<Type, float>,
FloatVector,
BinaryVector>>;
using TraitType = std::conditional_t<
is_scalar,
Type,
std::conditional_t<std::is_same_v<Type, float>,
FloatVector,
std::conditional_t<std::is_same_v<Type, float16>,
Float16Vector,
BinaryVector>>>;
public:
explicit ConcurrentVectorImpl(ssize_t dim, int64_t size_per_chunk)
@ -389,4 +391,14 @@ class ConcurrentVector<BinaryVector>
int64_t binary_dim_;
};
template <>
class ConcurrentVector<Float16Vector>
: public ConcurrentVectorImpl<float16, false> {
public:
ConcurrentVector(int64_t dim, int64_t size_per_chunk)
: ConcurrentVectorImpl<float16, false>::ConcurrentVectorImpl(
dim, size_per_chunk) {
}
};
} // namespace milvus::segcore

5
internal/core/src/segcore/FieldIndexing.cpp
View File

@ -237,6 +237,11 @@ CreateIndex(const FieldMeta& field_meta,
field_index_meta,
segment_max_row_count,
segcore_config);
} else if (field_meta.get_data_type() == DataType::VECTOR_FLOAT16) {
return std::make_unique<VectorFieldIndexing>(field_meta,
field_index_meta,
segment_max_row_count,
segcore_config);
} else {
// TODO
PanicInfo("unsupported");

5
internal/core/src/segcore/InsertRecord.h
View File

@ -280,6 +280,11 @@ struct InsertRecord {
this->append_field_data<BinaryVector>(
field_id, field_meta.get_dim(), size_per_chunk);
continue;
} else if (field_meta.get_data_type() ==
DataType::VECTOR_FLOAT16) {
this->append_field_data<Float16Vector>(
field_id, field_meta.get_dim(), size_per_chunk);
continue;
} else {
PanicInfo("unsupported");
}

7
internal/core/src/segcore/SegmentGrowingImpl.cpp
View File

@ -346,6 +346,13 @@ SegmentGrowingImpl::bulk_subscript(FieldId field_id,
seg_offsets,
count,
output.data());
} else if (field_meta.get_data_type() == DataType::VECTOR_FLOAT16) {
bulk_subscript_impl<Float16Vector>(field_id,
field_meta.get_sizeof(),
vec_ptr,
seg_offsets,
count,
output.data());
} else {
PanicInfo("logical error");
}

1
internal/core/src/segcore/SegmentSealedImpl.cpp
View File

@ -892,6 +892,7 @@ SegmentSealedImpl::bulk_subscript(FieldId field_id,
}
case DataType::VECTOR_FLOAT:
case DataType::VECTOR_FLOAT16:
case DataType::VECTOR_BINARY: {
aligned_vector<char> output(field_meta.get_sizeof() * count);
bulk_subscript_impl(field_meta.get_sizeof(),

13
internal/core/src/segcore/Utils.cpp
View File

@ -243,6 +243,12 @@ CreateVectorDataArray(int64_t count, const FieldMeta& field_meta) {
obj->resize(num_bytes);
break;
}
case DataType::VECTOR_FLOAT16: {
auto length = count * dim;
auto obj = vector_array->mutable_float16_vector();
obj->resize(length * sizeof(float16));
break;
}
default: {
PanicInfo("unsupported datatype");
}
@ -358,6 +364,13 @@ CreateVectorDataArrayFrom(const void* data_raw,
obj->assign(data, num_bytes);
break;
}
case DataType::VECTOR_FLOAT16: {
auto length = count * dim;
auto data = reinterpret_cast<const char*>(data_raw);
auto obj = vector_array->mutable_float16_vector();
obj->assign(data, length * sizeof(float16));
break;
}
default: {
PanicInfo("unsupported datatype");
}

2
internal/core/src/storage/FieldData.cpp
View File

@ -131,6 +131,7 @@ FieldDataImpl<Type, is_scalar>::FillFieldData(
return FillFieldData(values.data(), element_count);
}
case DataType::VECTOR_FLOAT:
case DataType::VECTOR_FLOAT16:
case DataType::VECTOR_BINARY: {
auto array_info =
GetDataInfoFromArray<arrow::FixedSizeBinaryArray,
@ -161,5 +162,6 @@ template class FieldDataImpl<Json, true>;
// vector data
template class FieldDataImpl<int8_t, false>;
template class FieldDataImpl<float, false>;
template class FieldDataImpl<float16, false>;
} // namespace milvus::storage

11
internal/core/src/storage/FieldData.h
View File

@ -85,6 +85,17 @@ class FieldData<BinaryVector> : public FieldDataImpl<uint8_t, false> {
int64_t binary_dim_;
};
template <>
class FieldData<Float16Vector> : public FieldDataImpl<float16, false> {
public:
explicit FieldData(int64_t dim,
DataType data_type,
int64_t buffered_num_rows = 0)
: FieldDataImpl<float16, false>::FieldDataImpl(
dim, data_type, buffered_num_rows) {
}
};
using FieldDataPtr = std::shared_ptr<FieldDataBase>;
using FieldDataChannel = Channel<storage::FieldDataPtr>;
using FieldDataChannelPtr = std::shared_ptr<FieldDataChannel>;

17
internal/core/src/storage/Util.cpp
View File

@ -121,6 +121,7 @@ AddPayloadToArrowBuilder(std::shared_ptr<arrow::ArrayBuilder> builder,
builder, double_data, length);
break;
}
case DataType::VECTOR_FLOAT16:
case DataType::VECTOR_BINARY:
case DataType::VECTOR_FLOAT: {
add_vector_payload(builder, const_cast<uint8_t*>(raw_data), length);
@ -215,6 +216,11 @@ CreateArrowBuilder(DataType data_type, int dim) {
return std::make_shared<arrow::FixedSizeBinaryBuilder>(
arrow::fixed_size_binary(dim / 8));
}
case DataType::VECTOR_FLOAT16: {
AssertInfo(dim > 0, "invalid dim value");
return std::make_shared<arrow::FixedSizeBinaryBuilder>(
arrow::fixed_size_binary(dim * sizeof(float16)));
}
default: {
PanicInfo("unsupported vector data type");
}
@ -272,6 +278,11 @@ CreateArrowSchema(DataType data_type, int dim) {
return arrow::schema(
{arrow::field("val", arrow::fixed_size_binary(dim / 8))});
}
case DataType::VECTOR_FLOAT16: {
AssertInfo(dim > 0, "invalid dim value");
return arrow::schema({arrow::field(
"val", arrow::fixed_size_binary(dim * sizeof(float16)))});
}
default: {
PanicInfo("unsupported vector data type");
}
@ -288,6 +299,9 @@ GetDimensionFromFileMetaData(const parquet::ColumnDescriptor* schema,
case DataType::VECTOR_BINARY: {
return schema->type_length() * 8;
}
case DataType::VECTOR_FLOAT16: {
return schema->type_length() / sizeof(float16);
}
default:
PanicInfo("unsupported data type");
}
@ -560,6 +574,9 @@ CreateFieldData(const DataType& type, int64_t dim, int64_t total_num_rows) {
case DataType::VECTOR_BINARY:
return std::make_shared<FieldData<BinaryVector>>(
dim, type, total_num_rows);
case DataType::VECTOR_FLOAT16:
return std::make_shared<FieldData<Float16Vector>>(
dim, type, total_num_rows);
default:
throw NotSupportedDataTypeException(
"CreateFieldData not support data type " + datatype_name(type));

1
internal/core/unittest/CMakeLists.txt
View File

@ -25,6 +25,7 @@ set(MILVUS_TEST_FILES
test_concurrent_vector.cpp
test_c_api.cpp
test_expr.cpp
test_float16.cpp
test_growing.cpp
test_growing_index.cpp
test_indexing.cpp

10
internal/core/unittest/test_c_api.cpp
View File

@ -311,7 +311,7 @@ TEST(CApiTest, CPlan) {
milvus::proto::plan::PlanNode plan_node;
auto vector_anns = plan_node.mutable_vector_anns();
vector_anns->set_is_binary(true);
vector_anns->set_vector_type(milvus::proto::plan::VectorType::BinaryVector);
vector_anns->set_placeholder_tag("$0");
vector_anns->set_field_id(100);
auto query_info = vector_anns->mutable_query_info();
@ -950,7 +950,7 @@ TEST(CApiTest, SearchTest) {
milvus::proto::plan::PlanNode plan_node;
auto vector_anns = plan_node.mutable_vector_anns();
vector_anns->set_is_binary(false);
vector_anns->set_vector_type(milvus::proto::plan::VectorType::FloatVector);
vector_anns->set_placeholder_tag("$0");
vector_anns->set_field_id(100);
auto query_info = vector_anns->mutable_query_info();
@ -1275,7 +1275,7 @@ TEST(CApiTest, ReudceNullResult) {
milvus::proto::plan::PlanNode plan_node;
auto vector_anns = plan_node.mutable_vector_anns();
vector_anns->set_is_binary(false);
vector_anns->set_vector_type(milvus::proto::plan::VectorType::FloatVector);
vector_anns->set_placeholder_tag("$0");
vector_anns->set_field_id(100);
auto query_info = vector_anns->mutable_query_info();
@ -1359,7 +1359,7 @@ TEST(CApiTest, ReduceRemoveDuplicates) {
milvus::proto::plan::PlanNode plan_node;
auto vector_anns = plan_node.mutable_vector_anns();
vector_anns->set_is_binary(false);
vector_anns->set_vector_type(milvus::proto::plan::VectorType::FloatVector);
vector_anns->set_placeholder_tag("$0");
vector_anns->set_field_id(100);
auto query_info = vector_anns->mutable_query_info();
@ -1703,7 +1703,7 @@ TEST(CApiTest, Indexing_Without_Predicate) {
milvus::proto::plan::PlanNode plan_node;
auto vector_anns = plan_node.mutable_vector_anns();
vector_anns->set_is_binary(false);
vector_anns->set_vector_type(milvus::proto::plan::VectorType::FloatVector);
vector_anns->set_placeholder_tag("$0");
vector_anns->set_field_id(100);
auto query_info = vector_anns->mutable_query_info();

30
internal/core/unittest/test_data_codec.cpp
View File

@ -302,6 +302,36 @@ TEST(storage, InsertDataBinaryVector) {
ASSERT_EQ(data, new_data);
}
TEST(storage, InsertDataFloat16Vector) {
std::vector<float16> data = {1, 2, 3, 4, 5, 6, 7, 8};
int DIM = 2;
auto field_data =
milvus::storage::CreateFieldData(storage::DataType::VECTOR_FLOAT16, DIM);
field_data->FillFieldData(data.data(), data.size() / DIM);
storage::InsertData insert_data(field_data);
storage::FieldDataMeta field_data_meta{100, 101, 102, 103};
insert_data.SetFieldDataMeta(field_data_meta);
insert_data.SetTimestamps(0, 100);
auto serialized_bytes = insert_data.Serialize(storage::StorageType::Remote);
std::shared_ptr<uint8_t[]> serialized_data_ptr(serialized_bytes.data(),
[&](uint8_t*) {});
auto new_insert_data = storage::DeserializeFileData(
serialized_data_ptr, serialized_bytes.size());
ASSERT_EQ(new_insert_data->GetCodecType(), storage::InsertDataType);
ASSERT_EQ(new_insert_data->GetTimeRage(),
std::make_pair(Timestamp(0), Timestamp(100)));
auto new_payload = new_insert_data->GetFieldData();
ASSERT_EQ(new_payload->get_data_type(), storage::DataType::VECTOR_FLOAT16);
ASSERT_EQ(new_payload->get_num_rows(), data.size() / DIM);
std::vector<float16> new_data(data.size());
memcpy(new_data.data(),
new_payload->Data(),
new_payload->get_num_rows() * sizeof(float16) * DIM);
ASSERT_EQ(data, new_data);
}
TEST(storage, IndexData) {
std::vector<uint8_t> data = {1, 2, 3, 4, 5, 6, 7, 8};
auto field_data = milvus::storage::CreateFieldData(storage::DataType::INT8);

420
internal/core/unittest/test_float16.cpp Normal file
View File

@ -0,0 +1,420 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#include <gtest/gtest.h>
#include "common/LoadInfo.h"
#include "common/Types.h"
#include "index/IndexFactory.h"
#include "knowhere/comp/index_param.h"
#include "query/ExprImpl.h"
#include "segcore/Reduce.h"
#include "segcore/reduce_c.h"
#include "test_utils/PbHelper.h"
#include "test_utils/indexbuilder_test_utils.h"
#include "pb/schema.pb.h"
#include "pb/plan.pb.h"
#include "query/Expr.h"
#include "query/Plan.h"
#include "query/Utils.h"
#include "query/PlanImpl.h"
#include "query/PlanNode.h"
#include "query/PlanProto.h"
#include "query/SearchBruteForce.h"
#include "query/generated/ExecPlanNodeVisitor.h"
#include "query/generated/PlanNodeVisitor.h"
#include "query/generated/ExecExprVisitor.h"
#include "query/generated/ExprVisitor.h"
#include "query/generated/ShowPlanNodeVisitor.h"
#include "segcore/Collection.h"
#include "segcore/SegmentSealed.h"
#include "segcore/SegmentGrowing.h"
#include "segcore/SegmentGrowingImpl.h"
#include "test_utils/AssertUtils.h"
#include "test_utils/DataGen.h"
using namespace milvus::segcore;
using namespace milvus;
using namespace milvus::index;
using namespace knowhere;
using milvus::index::VectorIndex;
using milvus::segcore::LoadIndexInfo;
const int64_t ROW_COUNT = 100 * 1000;
TEST(Float16, Insert) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
int64_t N = ROW_COUNT;
constexpr int64_t size_per_chunk = 32 * 1024;
auto schema = std::make_shared<Schema>();
auto float16_vec_fid = schema->AddDebugField(
"float16vec", DataType::VECTOR_FLOAT16, 32, knowhere::metric::L2);
auto i64_fid = schema->AddDebugField("counter", DataType::INT64);
schema->set_primary_field_id(i64_fid);
auto dataset = DataGen(schema, N);
// auto seg_conf = SegcoreConfig::default_config();
auto segment = CreateGrowingSegment(schema, empty_index_meta);
segment->PreInsert(N);
segment->Insert(0,
N,
dataset.row_ids_.data(),
dataset.timestamps_.data(),
dataset.raw_);
auto float16_ptr = dataset.get_col<float16>(float16_vec_fid);
SegmentInternalInterface& interface = *segment;
auto num_chunk = interface.num_chunk();
ASSERT_EQ(num_chunk, upper_div(N, size_per_chunk));
auto row_count = interface.get_row_count();
ASSERT_EQ(N, row_count);
for (auto chunk_id = 0; chunk_id < num_chunk; ++chunk_id) {
auto float16_span =
interface.chunk_data<milvus::Float16Vector>(float16_vec_fid, chunk_id);
auto begin = chunk_id * size_per_chunk;
auto end = std::min((chunk_id + 1) * size_per_chunk, N);
auto size_of_chunk = end - begin;
for (int i = 0; i < size_of_chunk; ++i) {
// std::cout << float16_span.data()[i] << " " << float16_ptr[i + begin * 32] << std::endl;
ASSERT_EQ(float16_span.data()[i], float16_ptr[i + begin * 32]);
}
}
}
TEST(Float16, ShowExecutor) {
using namespace milvus::query;
using namespace milvus::segcore;
using namespace milvus;
auto metric_type = knowhere::metric::L2;
auto node = std::make_unique<Float16VectorANNS>();
auto schema = std::make_shared<Schema>();
auto field_id = schema->AddDebugField(
"fakevec", DataType::VECTOR_FLOAT16, 16, metric_type);
int64_t num_queries = 100L;
auto raw_data = DataGen(schema, num_queries);
auto& info = node->search_info_;
info.metric_type_ = metric_type;
info.topk_ = 20;
info.field_id_ = field_id;
node->predicate_ = std::nullopt;
ShowPlanNodeVisitor show_visitor;
PlanNodePtr base(node.release());
auto res = show_visitor.call_child(*base);
auto dup = res;
std::cout << dup.dump(4);
}
TEST(Float16, ExecWithoutPredicateFlat) {
using namespace milvus::query;
using namespace milvus::segcore;
using namespace milvus;
auto schema = std::make_shared<Schema>();
auto vec_fid = schema->AddDebugField(
"fakevec", DataType::VECTOR_FLOAT16, 32, knowhere::metric::L2);
schema->AddDebugField("age", DataType::FLOAT);
auto i64_fid = schema->AddDebugField("counter", DataType::INT64);
schema->set_primary_field_id(i64_fid);
const char* raw_plan = R"(vector_anns: <
field_id: 100
query_info: <
topk: 5
round_decimal: 3
metric_type: "L2"
search_params: "{\"nprobe\": 10}"
>
placeholder_tag: "$0"
>)";
auto plan_str = translate_text_plan_to_binary_plan(raw_plan);
auto plan =
CreateSearchPlanByExpr(*schema, plan_str.data(), plan_str.size());
int64_t N = ROW_COUNT;
auto dataset = DataGen(schema, N);
auto segment = CreateGrowingSegment(schema, empty_index_meta);
segment->PreInsert(N);
segment->Insert(0,
N,
dataset.row_ids_.data(),
dataset.timestamps_.data(),
dataset.raw_);
auto vec_ptr = dataset.get_col<float16>(vec_fid);
auto num_queries = 5;
auto ph_group_raw = CreateFloat16PlaceholderGroup(
num_queries, 32, 1024);
auto ph_group =
ParsePlaceholderGroup(plan.get(), ph_group_raw.SerializeAsString());
auto sr = segment->Search(plan.get(), ph_group.get());
int topk = 5;
query::Json json = SearchResultToJson(*sr);
std::cout << json.dump(2);
}
TEST(Float16, GetVector) {
auto metricType = knowhere::metric::L2;
auto schema = std::make_shared<Schema>();
auto pk = schema->AddDebugField("pk", DataType::INT64);
auto random = schema->AddDebugField("random", DataType::DOUBLE);
auto vec = schema->AddDebugField(
"embeddings", DataType::VECTOR_FLOAT16, 128, metricType);
schema->set_primary_field_id(pk);
std::map<std::string, std::string> index_params = {
{"index_type", "IVF_FLAT"},
{"metric_type", metricType},
{"nlist", "128"}};
std::map<std::string, std::string> type_params = {{"dim", "128"}};
FieldIndexMeta fieldIndexMeta(
vec, std::move(index_params), std::move(type_params));
auto& config = SegcoreConfig::default_config();
config.set_chunk_rows(1024);
config.set_enable_growing_segment_index(true);
std::map<FieldId, FieldIndexMeta> filedMap = {{vec, fieldIndexMeta}};
IndexMetaPtr metaPtr =
std::make_shared<CollectionIndexMeta>(100000, std::move(filedMap));
auto segment_growing = CreateGrowingSegment(schema, metaPtr);
auto segment = dynamic_cast<SegmentGrowingImpl*>(segment_growing.get());
int64_t per_batch = 5000;
int64_t n_batch = 20;
int64_t dim = 128;
for (int64_t i = 0; i < n_batch; i++) {
auto dataset = DataGen(schema, per_batch);
auto fakevec = dataset.get_col<float16>(vec);
auto offset = segment->PreInsert(per_batch);
segment->Insert(offset,
per_batch,
dataset.row_ids_.data(),
dataset.timestamps_.data(),
dataset.raw_);
auto num_inserted = (i + 1) * per_batch;
auto ids_ds = GenRandomIds(num_inserted);
auto result =
segment->bulk_subscript(vec, ids_ds->GetIds(), num_inserted);
auto vector = result.get()->mutable_vectors()->float16_vector();
EXPECT_TRUE(vector.size() == num_inserted * dim * sizeof(float16));
// EXPECT_TRUE(vector.size() == num_inserted * dim);
// for (size_t i = 0; i < num_inserted; ++i) {
// auto id = ids_ds->GetIds()[i];
// for (size_t j = 0; j < 128; ++j) {
// EXPECT_TRUE(vector[i * dim + j] ==
// fakevec[(id % per_batch) * dim + j]);
// }
// }
}
}
std::string
generate_collection_schema(std::string metric_type, int dim, bool is_fp16) {
namespace schema = milvus::proto::schema;
schema::CollectionSchema collection_schema;
collection_schema.set_name("collection_test");
auto vec_field_schema = collection_schema.add_fields();
vec_field_schema->set_name("fakevec");
vec_field_schema->set_fieldid(100);
if (is_fp16) {
vec_field_schema->set_data_type(schema::DataType::Float16Vector);
} else {
vec_field_schema->set_data_type(schema::DataType::FloatVector);
}
auto metric_type_param = vec_field_schema->add_index_params();
metric_type_param->set_key("metric_type");
metric_type_param->set_value(metric_type);
auto dim_param = vec_field_schema->add_type_params();
dim_param->set_key("dim");
dim_param->set_value(std::to_string(dim));
auto other_field_schema = collection_schema.add_fields();
other_field_schema->set_name("counter");
other_field_schema->set_fieldid(101);
other_field_schema->set_data_type(schema::DataType::Int64);
other_field_schema->set_is_primary_key(true);
auto other_field_schema2 = collection_schema.add_fields();
other_field_schema2->set_name("doubleField");
other_field_schema2->set_fieldid(102);
other_field_schema2->set_data_type(schema::DataType::Double);
std::string schema_string;
auto marshal = google::protobuf::TextFormat::PrintToString(
collection_schema, &schema_string);
assert(marshal);
return schema_string;
}
CCollection
NewCollection(const char* schema_proto_blob) {
auto proto = std::string(schema_proto_blob);
auto collection = std::make_unique<milvus::segcore::Collection>(proto);
return (void*)collection.release();
}
TEST(Float16, CApiCPlan) {
std::string schema_string =
generate_collection_schema(knowhere::metric::L2, 16, true);
auto collection = NewCollection(schema_string.c_str());
// const char* dsl_string = R"(
// {
// "bool": {
// "vector": {
// "fakevec": {
// "metric_type": "L2",
// "params": {
// "nprobe": 10
// },
// "query": "$0",
// "topk": 10,
// "round_decimal": 3
// }
// }
// }
// })";
milvus::proto::plan::PlanNode plan_node;
auto vector_anns = plan_node.mutable_vector_anns();
vector_anns->set_vector_type(milvus::proto::plan::VectorType::Float16Vector);
vector_anns->set_placeholder_tag("$0");
vector_anns->set_field_id(100);
auto query_info = vector_anns->mutable_query_info();
query_info->set_topk(10);
query_info->set_round_decimal(3);
query_info->set_metric_type("L2");
query_info->set_search_params(R"({"nprobe": 10})");
auto plan_str = plan_node.SerializeAsString();
void* plan = nullptr;
auto status = CreateSearchPlanByExpr(
collection, plan_str.data(), plan_str.size(), &plan);
ASSERT_EQ(status.error_code, Success);
int64_t field_id = -1;
status = GetFieldID(plan, &field_id);
ASSERT_EQ(status.error_code, Success);
auto col = static_cast<Collection*>(collection);
for (auto& [target_field_id, field_meta] :
col->get_schema()->get_fields()) {
if (field_meta.is_vector()) {
ASSERT_EQ(field_id, target_field_id.get());
}
}
ASSERT_NE(field_id, -1);
DeleteSearchPlan(plan);
DeleteCollection(collection);
}
TEST(Float16, RetrieveEmpty) {
auto schema = std::make_shared<Schema>();
auto fid_64 = schema->AddDebugField("i64", DataType::INT64);
auto DIM = 16;
auto fid_vec = schema->AddDebugField(
"vector_64", DataType::VECTOR_FLOAT16, DIM, knowhere::metric::L2);
schema->set_primary_field_id(fid_64);
int64_t N = 100;
int64_t req_size = 10;
auto choose = [=](int i) { return i * 3 % N; };
auto segment = CreateSealedSegment(schema);
auto plan = std::make_unique<query::RetrievePlan>(*schema);
std::vector<int64_t> values;
for (int i = 0; i < req_size; ++i) {
values.emplace_back(choose(i));
}
auto term_expr = std::make_unique<query::TermExprImpl<int64_t>>(
milvus::query::ColumnInfo(
fid_64, DataType::INT64, std::vector<std::string>()),
values,
proto::plan::GenericValue::kInt64Val);
plan->plan_node_ = std::make_unique<query::RetrievePlanNode>();
plan->plan_node_->predicate_ = std::move(term_expr);
std::vector<FieldId> target_offsets{fid_64, fid_vec};
plan->field_ids_ = target_offsets;
auto retrieve_results =
segment->Retrieve(plan.get(), 100, DEFAULT_MAX_OUTPUT_SIZE);
Assert(retrieve_results->fields_data_size() == target_offsets.size());
auto field0 = retrieve_results->fields_data(0);
auto field1 = retrieve_results->fields_data(1);
Assert(field0.has_scalars());
auto field0_data = field0.scalars().long_data();
Assert(field0_data.data_size() == 0);
Assert(field1.vectors().float16_vector().size() == 0);
}
TEST(Float16, ExecWithPredicate) {
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
schema->AddDebugField(
"fakevec", DataType::VECTOR_FLOAT16, 16, knowhere::metric::L2);
schema->AddDebugField("age", DataType::FLOAT);
auto i64_fid = schema->AddDebugField("counter", DataType::INT64);
schema->set_primary_field_id(i64_fid);
const char* raw_plan = R"(vector_anns: <
field_id: 100
predicates: <
binary_range_expr: <
column_info: <
field_id: 101
data_type: Float
>
lower_inclusive: true,
upper_inclusive: false,
lower_value: <
float_val: -1
>
upper_value: <
float_val: 1
>
>
>
query_info: <
topk: 5
round_decimal: 3
metric_type: "L2"
search_params: "{\"nprobe\": 10}"
>
placeholder_tag: "$0"
>)";
int64_t N = ROW_COUNT;
auto dataset = DataGen(schema, N);
auto segment = CreateGrowingSegment(schema, empty_index_meta);
segment->PreInsert(N);
segment->Insert(0,
N,
dataset.row_ids_.data(),
dataset.timestamps_.data(),
dataset.raw_);
auto plan_str = translate_text_plan_to_binary_plan(raw_plan);
auto plan =
CreateSearchPlanByExpr(*schema, plan_str.data(), plan_str.size());
auto num_queries = 5;
auto ph_group_raw = CreateFloat16PlaceholderGroup(num_queries, 16, 1024);
auto ph_group =
ParsePlaceholderGroup(plan.get(), ph_group_raw.SerializeAsString());
auto sr = segment->Search(plan.get(), ph_group.get());
int topk = 5;
query::Json json = SearchResultToJson(*sr);
std::cout << json.dump(2);
}

4
internal/core/unittest/test_growing_index.cpp
View File

@ -46,7 +46,7 @@ TEST(GrowingIndex, Correctness) {
milvus::proto::plan::PlanNode plan_node;
auto vector_anns = plan_node.mutable_vector_anns();
vector_anns->set_is_binary(false);
vector_anns->set_vector_type(milvus::proto::plan::VectorType::FloatVector);
vector_anns->set_placeholder_tag("$0");
vector_anns->set_field_id(102);
auto query_info = vector_anns->mutable_query_info();
@ -58,7 +58,7 @@ TEST(GrowingIndex, Correctness) {
milvus::proto::plan::PlanNode range_query_plan_node;
auto vector_range_querys = range_query_plan_node.mutable_vector_anns();
vector_range_querys->set_is_binary(false);
vector_range_querys->set_vector_type(milvus::proto::plan::VectorType::FloatVector);
vector_range_querys->set_placeholder_tag("$0");
vector_range_querys->set_field_id(102);
auto range_query_info = vector_range_querys->mutable_query_info();

65
internal/core/unittest/test_string_expr.cpp
View File

@ -73,12 +73,12 @@ GenQueryInfo(int64_t topk,
auto
GenAnns(proto::plan::Expr* predicate,
bool is_binary,
proto::plan::VectorType vectorType,
int64_t field_id,
std::string placeholder_tag = "$0") {
auto query_info = GenQueryInfo(10, "L2", "{\"nprobe\": 10}", -1);
auto anns = new proto::plan::VectorANNS();
anns->set_is_binary(is_binary);
anns->set_vector_type(vectorType);
anns->set_field_id(field_id);
anns->set_allocated_predicates(predicate);
anns->set_allocated_query_info(query_info);
@ -177,8 +177,17 @@ GenTermPlan(const FieldMeta& fvec_meta,
auto expr = GenExpr().release();
expr->set_allocated_term_expr(term_expr);
proto::plan::VectorType vector_type;
if (fvec_meta.get_data_type() == DataType::VECTOR_FLOAT) {
vector_type = proto::plan::VectorType::FloatVector;
} else if (fvec_meta.get_data_type() == DataType::VECTOR_BINARY) {
vector_type = proto::plan::VectorType::BinaryVector;
} else if (fvec_meta.get_data_type() == DataType::VECTOR_FLOAT16) {
vector_type = proto::plan::VectorType::Float16Vector;
}
auto anns = GenAnns(expr,
fvec_meta.get_data_type() == DataType::VECTOR_BINARY,
vector_type,
fvec_meta.get_id().get(),
"$0");
@ -215,8 +224,16 @@ GenAlwaysTrueExpr(const FieldMeta& fvec_meta, const FieldMeta& str_meta) {
auto
GenAlwaysFalsePlan(const FieldMeta& fvec_meta, const FieldMeta& str_meta) {
auto always_false_expr = GenAlwaysFalseExpr(fvec_meta, str_meta);
proto::plan::VectorType vector_type;
if (fvec_meta.get_data_type() == DataType::VECTOR_FLOAT) {
vector_type = proto::plan::VectorType::FloatVector;
} else if (fvec_meta.get_data_type() == DataType::VECTOR_BINARY) {
vector_type = proto::plan::VectorType::BinaryVector;
} else if (fvec_meta.get_data_type() == DataType::VECTOR_FLOAT16) {
vector_type = proto::plan::VectorType::Float16Vector;
}
auto anns = GenAnns(always_false_expr,
fvec_meta.get_data_type() == DataType::VECTOR_BINARY,
vector_type,
fvec_meta.get_id().get(),
"$0");
@ -228,8 +245,16 @@ GenAlwaysFalsePlan(const FieldMeta& fvec_meta, const FieldMeta& str_meta) {
auto
GenAlwaysTruePlan(const FieldMeta& fvec_meta, const FieldMeta& str_meta) {
auto always_true_expr = GenAlwaysTrueExpr(fvec_meta, str_meta);
proto::plan::VectorType vector_type;
if (fvec_meta.get_data_type() == DataType::VECTOR_FLOAT) {
vector_type = proto::plan::VectorType::FloatVector;
} else if (fvec_meta.get_data_type() == DataType::VECTOR_BINARY) {
vector_type = proto::plan::VectorType::BinaryVector;
} else if (fvec_meta.get_data_type() == DataType::VECTOR_FLOAT16) {
vector_type = proto::plan::VectorType::Float16Vector;
}
auto anns = GenAnns(always_true_expr,
fvec_meta.get_data_type() == DataType::VECTOR_BINARY,
vector_type,
fvec_meta.get_id().get(),
"$0");
@ -353,9 +378,17 @@ TEST(StringExpr, Compare) {
auto expr = GenExpr().release();
expr->set_allocated_compare_expr(compare_expr);
proto::plan::VectorType vector_type;
if (fvec_meta.get_data_type() == DataType::VECTOR_FLOAT) {
vector_type = proto::plan::VectorType::FloatVector;
} else if (fvec_meta.get_data_type() == DataType::VECTOR_BINARY) {
vector_type = proto::plan::VectorType::BinaryVector;
} else if (fvec_meta.get_data_type() == DataType::VECTOR_FLOAT16) {
vector_type = proto::plan::VectorType::Float16Vector;
}
auto anns =
GenAnns(expr,
fvec_meta.get_data_type() == DataType::VECTOR_BINARY,
vector_type,
fvec_meta.get_id().get(),
"$0");
@ -456,9 +489,17 @@ TEST(StringExpr, UnaryRange) {
auto expr = GenExpr().release();
expr->set_allocated_unary_range_expr(unary_range_expr);
proto::plan::VectorType vector_type;
if (fvec_meta.get_data_type() == DataType::VECTOR_FLOAT) {
vector_type = proto::plan::VectorType::FloatVector;
} else if (fvec_meta.get_data_type() == DataType::VECTOR_BINARY) {
vector_type = proto::plan::VectorType::BinaryVector;
} else if (fvec_meta.get_data_type() == DataType::VECTOR_FLOAT16) {
vector_type = proto::plan::VectorType::Float16Vector;
}
auto anns =
GenAnns(expr,
fvec_meta.get_data_type() == DataType::VECTOR_BINARY,
vector_type,
fvec_meta.get_id().get(),
"$0");
@ -551,9 +592,17 @@ TEST(StringExpr, BinaryRange) {
auto expr = GenExpr().release();
expr->set_allocated_binary_range_expr(binary_range_expr);
proto::plan::VectorType vector_type;
if (fvec_meta.get_data_type() == DataType::VECTOR_FLOAT) {
vector_type = proto::plan::VectorType::FloatVector;
} else if (fvec_meta.get_data_type() == DataType::VECTOR_BINARY) {
vector_type = proto::plan::VectorType::BinaryVector;
} else if (fvec_meta.get_data_type() == DataType::VECTOR_FLOAT16) {
vector_type = proto::plan::VectorType::Float16Vector;
}
auto anns =
GenAnns(expr,
fvec_meta.get_data_type() == DataType::VECTOR_BINARY,
vector_type,
fvec_meta.get_id().get(),
"$0");

40
internal/core/unittest/test_utils/DataGen.h
View File

@ -105,6 +105,16 @@ struct GeneratedData {
auto src_data = reinterpret_cast<const T*>(
target_field_data.vectors().binary_vector().data());
std::copy_n(src_data, len, ret.data());
} else if (field_meta.get_data_type() ==
DataType::VECTOR_FLOAT16) {
// int len = raw_->num_rows() * field_meta.get_dim() * sizeof(float16);
int len = raw_->num_rows() * field_meta.get_dim();
ret.resize(len);
auto src_data =
reinterpret_cast<const T*>(target_field_data.vectors()
.float16_vector()
.data());
std::copy_n(src_data, len, ret.data());
} else {
PanicInfo("unsupported");
}
@ -258,6 +268,15 @@ DataGen(SchemaPtr schema,
}
insert_cols(data, N, field_meta);
break;
}
case DataType::VECTOR_FLOAT16: {
auto dim = field_meta.get_dim();
vector<float16> final(dim * N);
for (auto& x : final) {
x = float16(distr(er) + offset);
}
insert_cols(final, N, field_meta);
break;
}
case DataType::BOOL: {
FixedVector<bool> data(N);
@ -549,6 +568,27 @@ CreateBinaryPlaceholderGroupFromBlob(int64_t num_queries,
return raw_group;
}
inline auto
CreateFloat16PlaceholderGroup(int64_t num_queries,
int64_t dim,
int64_t seed = 42) {
namespace ser = milvus::proto::common;
ser::PlaceholderGroup raw_group;
auto value = raw_group.add_placeholders();
value->set_tag("$0");
value->set_type(ser::PlaceholderType::Float16Vector);
std::normal_distribution<double> dis(0, 1);
std::default_random_engine e(seed);
for (int i = 0; i < num_queries; ++i) {
std::vector<float16> vec;
for (int d = 0; d < dim; ++d) {
vec.push_back(float16(dis(e)));
}
value->add_values(vec.data(), vec.size() * sizeof(float16));
}
return raw_group;
}
inline auto
SearchResultToVector(const SearchResult& sr) {
int64_t num_queries = sr.total_nq_;

3
internal/datacoord/util.go
View File

@ -104,7 +104,8 @@ func FilterInIndexedSegments(handler Handler, mt *meta, segments ...*SegmentInfo
}
for _, field := range coll.Schema.GetFields() {
if field.GetDataType() == schemapb.DataType_BinaryVector ||
field.GetDataType() == schemapb.DataType_FloatVector {
field.GetDataType() == schemapb.DataType_FloatVector ||
field.GetDataType() == schemapb.DataType_Float16Vector {
vecFieldID[collection] = field.GetFieldID()
break
}

16
internal/datanode/compactor.go
View File

@ -881,6 +881,22 @@ func interface2FieldData(schemaDataType schemapb.DataType, content []interface{}
data.Dim = len(data.Data) / int(numRows)
rst = data
case schemapb.DataType_Float16Vector:
var data = &storage.Float16VectorFieldData{
Data: []byte{},
}
for _, c := range content {
r, ok := c.([]byte)
if !ok {
return nil, errTransferType
}
data.Data = append(data.Data, r...)
}
data.Dim = len(data.Data) / 2 / int(numRows)
rst = data
case schemapb.DataType_BinaryVector:
var data = &storage.BinaryVectorFieldData{
Data: []byte{},

2
internal/datanode/compactor_test.go
View File

@ -108,6 +108,7 @@ func TestCompactionTaskInnerMethods(t *testing.T) {
{true, schemapb.DataType_JSON, []interface{}{[]byte("{\"key\":\"value\"}"), []byte("{\"hello\":\"world\"}")}, "valid json"},
{true, schemapb.DataType_FloatVector, []interface{}{[]float32{1.0, 2.0}}, "valid floatvector"},
{true, schemapb.DataType_BinaryVector, []interface{}{[]byte{255}}, "valid binaryvector"},
{true, schemapb.DataType_Float16Vector, []interface{}{[]byte{255, 255, 255, 255}}, "valid float16vector"},
{false, schemapb.DataType_Bool, []interface{}{1, 2}, "invalid bool"},
{false, schemapb.DataType_Int8, []interface{}{nil, nil}, "invalid int8"},
{false, schemapb.DataType_Int16, []interface{}{nil, nil}, "invalid int16"},
@ -119,6 +120,7 @@ func TestCompactionTaskInnerMethods(t *testing.T) {
{false, schemapb.DataType_JSON, []interface{}{nil, nil}, "invalid json"},
{false, schemapb.DataType_FloatVector, []interface{}{nil, nil}, "invalid floatvector"},
{false, schemapb.DataType_BinaryVector, []interface{}{nil, nil}, "invalid binaryvector"},
{false, schemapb.DataType_Float16Vector, []interface{}{nil, nil}, "invalid float16vector"},
{false, schemapb.DataType_None, nil, "invalid data type"},
}

4
internal/indexnode/util.go
View File

@ -31,6 +31,8 @@ func estimateFieldDataSize(dim int64, numRows int64, dataType schemapb.DataType)
if dataType == schemapb.DataType_BinaryVector {
return uint64(dim) / 8 * uint64(numRows), nil
}
if dataType == schemapb.DataType_Float16Vector {
return uint64(dim) * uint64(numRows) * 2, nil
}
return 0, nil
}

2
internal/parser/planparserv2/check_identical.go
View File

@ -37,7 +37,7 @@ func CheckQueryInfoIdentical(info1, info2 *planpb.QueryInfo) bool {
}
func CheckVectorANNSIdentical(node1, node2 *planpb.VectorANNS) bool {
if node1.GetIsBinary() != node2.GetIsBinary() {
if node1.GetVectorType() != node2.GetVectorType() {
return false
}
if node1.GetFieldId() != node2.GetFieldId() {

32
internal/parser/planparserv2/check_identical_test.go
View File

@ -94,35 +94,35 @@ func TestCheckVectorANNSIdentical(t *testing.T) {
}{
{
args: args{
node1: &planpb.VectorANNS{IsBinary: true},
node2: &planpb.VectorANNS{IsBinary: false},
node1: &planpb.VectorANNS{VectorType: planpb.VectorType_BinaryVector},
node2: &planpb.VectorANNS{VectorType: planpb.VectorType_FloatVector},
},
want: false,
},
{
args: args{
node1: &planpb.VectorANNS{IsBinary: false, FieldId: 100},
node2: &planpb.VectorANNS{IsBinary: false, FieldId: 101},
node1: &planpb.VectorANNS{VectorType: planpb.VectorType_FloatVector, FieldId: 100},
node2: &planpb.VectorANNS{VectorType: planpb.VectorType_FloatVector, FieldId: 101},
},
want: false,
},
{
args: args{
node1: &planpb.VectorANNS{IsBinary: false, FieldId: 100, PlaceholderTag: "$0"},
node2: &planpb.VectorANNS{IsBinary: false, FieldId: 100, PlaceholderTag: "$1"},
node1: &planpb.VectorANNS{VectorType: planpb.VectorType_FloatVector, FieldId: 100, PlaceholderTag: "$0"},
node2: &planpb.VectorANNS{VectorType: planpb.VectorType_FloatVector, FieldId: 100, PlaceholderTag: "$1"},
},
want: false,
},
{
args: args{
node1: &planpb.VectorANNS{IsBinary: false, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 100}},
node2: &planpb.VectorANNS{IsBinary: false, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 10}},
node1: &planpb.VectorANNS{VectorType: planpb.VectorType_FloatVector, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 100}},
node2: &planpb.VectorANNS{VectorType: planpb.VectorType_FloatVector, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 10}},
},
want: false,
},
{
args: args{
node1: &planpb.VectorANNS{IsBinary: false, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 1, MetricType: "L2", SearchParams: `{"nprobe": 10}`, RoundDecimal: 6},
node1: &planpb.VectorANNS{VectorType: planpb.VectorType_FloatVector, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 1, MetricType: "L2", SearchParams: `{"nprobe": 10}`, RoundDecimal: 6},
Predicates: &planpb.Expr{
Expr: &planpb.Expr_ColumnExpr{
ColumnExpr: &planpb.ColumnExpr{
@ -130,7 +130,7 @@ func TestCheckVectorANNSIdentical(t *testing.T) {
},
},
}},
node2: &planpb.VectorANNS{IsBinary: false, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 1, MetricType: "L2", SearchParams: `{"nprobe": 10}`, RoundDecimal: 6},
node2: &planpb.VectorANNS{VectorType: planpb.VectorType_FloatVector, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 1, MetricType: "L2", SearchParams: `{"nprobe": 10}`, RoundDecimal: 6},
Predicates: &planpb.Expr{
Expr: &planpb.Expr_ValueExpr{
ValueExpr: &planpb.ValueExpr{Value: NewInt(100)},
@ -141,13 +141,13 @@ func TestCheckVectorANNSIdentical(t *testing.T) {
},
{
args: args{
node1: &planpb.VectorANNS{IsBinary: false, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 1, MetricType: "L2", SearchParams: `{"nprobe": 10}`, RoundDecimal: 6},
node1: &planpb.VectorANNS{VectorType: planpb.VectorType_FloatVector, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 1, MetricType: "L2", SearchParams: `{"nprobe": 10}`, RoundDecimal: 6},
Predicates: &planpb.Expr{
Expr: &planpb.Expr_ValueExpr{
ValueExpr: &planpb.ValueExpr{Value: NewInt(100)},
},
}},
node2: &planpb.VectorANNS{IsBinary: false, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 1, MetricType: "L2", SearchParams: `{"nprobe": 10}`, RoundDecimal: 6},
node2: &planpb.VectorANNS{VectorType: planpb.VectorType_FloatVector, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 1, MetricType: "L2", SearchParams: `{"nprobe": 10}`, RoundDecimal: 6},
Predicates: &planpb.Expr{
Expr: &planpb.Expr_ValueExpr{
ValueExpr: &planpb.ValueExpr{Value: NewInt(100)},
@ -194,7 +194,7 @@ func TestCheckPlanNodeIdentical(t *testing.T) {
node1: &planpb.PlanNode{
Node: &planpb.PlanNode_VectorAnns{
VectorAnns: &planpb.VectorANNS{
IsBinary: true,
VectorType: planpb.VectorType_BinaryVector,
},
},
OutputFieldIds: []int64{100},
@ -202,7 +202,7 @@ func TestCheckPlanNodeIdentical(t *testing.T) {
node2: &planpb.PlanNode{
Node: &planpb.PlanNode_VectorAnns{
VectorAnns: &planpb.VectorANNS{
IsBinary: false,
VectorType: planpb.VectorType_FloatVector,
},
},
OutputFieldIds: []int64{100},
@ -214,7 +214,7 @@ func TestCheckPlanNodeIdentical(t *testing.T) {
args: args{
node1: &planpb.PlanNode{
Node: &planpb.PlanNode_VectorAnns{
VectorAnns: &planpb.VectorANNS{IsBinary: false, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 1, MetricType: "L2", SearchParams: `{"nprobe": 10}`, RoundDecimal: 6},
VectorAnns: &planpb.VectorANNS{VectorType: planpb.VectorType_FloatVector, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 1, MetricType: "L2", SearchParams: `{"nprobe": 10}`, RoundDecimal: 6},
Predicates: &planpb.Expr{
Expr: &planpb.Expr_ValueExpr{
ValueExpr: &planpb.ValueExpr{Value: NewInt(100)},
@ -225,7 +225,7 @@ func TestCheckPlanNodeIdentical(t *testing.T) {
},
node2: &planpb.PlanNode{
Node: &planpb.PlanNode_VectorAnns{
VectorAnns: &planpb.VectorANNS{IsBinary: false, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 1, MetricType: "L2", SearchParams: `{"nprobe": 10}`, RoundDecimal: 6},
VectorAnns: &planpb.VectorANNS{VectorType: planpb.VectorType_FloatVector, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 1, MetricType: "L2", SearchParams: `{"nprobe": 10}`, RoundDecimal: 6},
Predicates: &planpb.Expr{
Expr: &planpb.Expr_ValueExpr{
ValueExpr: &planpb.ValueExpr{Value: NewInt(100)},

12
internal/parser/planparserv2/plan_parser_v2.go
View File

@ -134,14 +134,22 @@ func CreateSearchPlan(schemaPb *schemapb.CollectionSchema, exprStr string, vecto
fieldID := vectorField.FieldID
dataType := vectorField.DataType
var vectorType planpb.VectorType
if !typeutil.IsVectorType(dataType) {
return nil, fmt.Errorf("field (%s) to search is not of vector data type", vectorFieldName)
} else {
if dataType == schemapb.DataType_FloatVector {
vectorType = planpb.VectorType_FloatVector
} else if dataType == schemapb.DataType_BinaryVector {
vectorType = planpb.VectorType_BinaryVector
} else {
vectorType = planpb.VectorType_Float16Vector
}
}
planNode := &planpb.PlanNode{
Node: &planpb.PlanNode_VectorAnns{
VectorAnns: &planpb.VectorANNS{
IsBinary: dataType == schemapb.DataType_BinaryVector,
VectorType: vectorType,
Predicates: expr,
QueryInfo: queryInfo,
PlaceholderTag: "$0",

8
internal/proto/plan.proto
View File

@ -29,6 +29,12 @@ enum ArithOpType {
Mod = 5;
};
enum VectorType {
BinaryVector = 0;
FloatVector = 1;
Float16Vector = 2;
};
message GenericValue {
oneof val {
bool bool_val = 1;
@ -176,7 +182,7 @@ message Expr {
}
message VectorANNS {
bool is_binary = 1;
VectorType vector_type = 1;
int64 field_id = 2;
Expr predicates = 3;
QueryInfo query_info = 4;

266
internal/proto/planpb/plan.pb.go
View File

@ -116,6 +116,34 @@ func (ArithOpType) EnumDescriptor() ([]byte, []int) {
return fileDescriptor_2d655ab2f7683c23, []int{1}
}
type VectorType int32
const (
VectorType_BinaryVector VectorType = 0
VectorType_FloatVector VectorType = 1
VectorType_Float16Vector VectorType = 2
)
var VectorType_name = map[int32]string{
0: "BinaryVector",
1: "FloatVector",
2: "Float16Vector",
}
var VectorType_value = map[string]int32{
"BinaryVector": 0,
"FloatVector": 1,
"Float16Vector": 2,
}
func (x VectorType) String() string {
return proto.EnumName(VectorType_name, int32(x))
}
func (VectorType) EnumDescriptor() ([]byte, []int) {
return fileDescriptor_2d655ab2f7683c23, []int{2}
}
// 0: invalid
// 1: json_contains
// 2: json_contains_all
@ -206,7 +234,6 @@ func (BinaryExpr_BinaryOp) EnumDescriptor() ([]byte, []int) {
type GenericValue struct {
// Types that are valid to be assigned to Val:
//
// *GenericValue_BoolVal
// *GenericValue_Int64Val
// *GenericValue_FloatVal
@ -1251,7 +1278,6 @@ var xxx_messageInfo_AlwaysTrueExpr proto.InternalMessageInfo
type Expr struct {
// Types that are valid to be assigned to Expr:
//
// *Expr_TermExpr
// *Expr_UnaryExpr
// *Expr_BinaryExpr
@ -1496,7 +1522,7 @@ func (*Expr) XXX_OneofWrappers() []interface{} {
}
type VectorANNS struct {
IsBinary bool `protobuf:"varint,1,opt,name=is_binary,json=isBinary,proto3" json:"is_binary,omitempty"`
VectorType VectorType `protobuf:"varint,1,opt,name=vector_type,json=vectorType,proto3,enum=milvus.proto.plan.VectorType" json:"vector_type,omitempty"`
FieldId int64 `protobuf:"varint,2,opt,name=field_id,json=fieldId,proto3" json:"field_id,omitempty"`
Predicates *Expr `protobuf:"bytes,3,opt,name=predicates,proto3" json:"predicates,omitempty"`
QueryInfo *QueryInfo `protobuf:"bytes,4,opt,name=query_info,json=queryInfo,proto3" json:"query_info,omitempty"`
@ -1531,11 +1557,11 @@ func (m *VectorANNS) XXX_DiscardUnknown() {
var xxx_messageInfo_VectorANNS proto.InternalMessageInfo
func (m *VectorANNS) GetIsBinary() bool {
func (m *VectorANNS) GetVectorType() VectorType {
if m != nil {
return m.IsBinary
return m.VectorType
}
return false
return VectorType_BinaryVector
}
func (m *VectorANNS) GetFieldId() int64 {
@ -1623,7 +1649,6 @@ func (m *QueryPlanNode) GetLimit() int64 {
type PlanNode struct {
// Types that are valid to be assigned to Node:
//
// *PlanNode_VectorAnns
// *PlanNode_Predicates
// *PlanNode_Query
@ -1728,6 +1753,7 @@ func (*PlanNode) XXX_OneofWrappers() []interface{} {
func init() {
proto.RegisterEnum("milvus.proto.plan.OpType", OpType_name, OpType_value)
proto.RegisterEnum("milvus.proto.plan.ArithOpType", ArithOpType_name, ArithOpType_value)
proto.RegisterEnum("milvus.proto.plan.VectorType", VectorType_name, VectorType_value)
proto.RegisterEnum("milvus.proto.plan.JSONContainsExpr_JSONOp", JSONContainsExpr_JSONOp_name, JSONContainsExpr_JSONOp_value)
proto.RegisterEnum("milvus.proto.plan.UnaryExpr_UnaryOp", UnaryExpr_UnaryOp_name, UnaryExpr_UnaryOp_value)
proto.RegisterEnum("milvus.proto.plan.BinaryExpr_BinaryOp", BinaryExpr_BinaryOp_name, BinaryExpr_BinaryOp_value)
@ -1758,116 +1784,118 @@ func init() {
func init() { proto.RegisterFile("plan.proto", fileDescriptor_2d655ab2f7683c23) }
var fileDescriptor_2d655ab2f7683c23 = []byte{
// 1762 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x58, 0x4f, 0x93, 0xdb, 0x48,
0x15, 0xb7, 0x2c, 0x7b, 0x2c, 0x3d, 0x79, 0x3c, 0x8a, 0x8a, 0x2a, 0x26, 0x09, 0x9b, 0x19, 0xb4,
0x5b, 0xec, 0x10, 0xc8, 0xa4, 0x36, 0xbb, 0x9b, 0xb0, 0xd9, 0x5a, 0x18, 0xcf, 0x9f, 0x8d, 0xcd,
0x6e, 0x66, 0x06, 0xcd, 0x6c, 0x0e, 0x70, 0x50, 0xb5, 0xa5, 0x9e, 0x71, 0x13, 0xb9, 0xa5, 0x48,
0x2d, 0x27, 0xfe, 0x0a, 0xdc, 0xf8, 0x00, 0x9c, 0x38, 0x70, 0xe7, 0xc8, 0x05, 0xce, 0x14, 0x07,
0x8e, 0x1c, 0xa9, 0xe2, 0x13, 0x50, 0x7c, 0x01, 0xaa, 0x5f, 0x4b, 0x96, 0x3d, 0xd8, 0x19, 0x0f,
0xa4, 0x8a, 0x5b, 0xeb, 0xf5, 0x7b, 0xbf, 0x7e, 0xff, 0xfa, 0xbd, 0xd7, 0x02, 0x48, 0x22, 0xc2,
0x77, 0x93, 0x34, 0x16, 0xb1, 0x73, 0x6b, 0xc4, 0xa2, 0x71, 0x9e, 0xa9, 0xaf, 0x5d, 0xb9, 0x71,
0xa7, 0x9d, 0x05, 0x43, 0x3a, 0x22, 0x8a, 0xe4, 0xfe, 0x59, 0x83, 0xf6, 0x33, 0xca, 0x69, 0xca,
0x82, 0x17, 0x24, 0xca, 0xa9, 0x73, 0x17, 0x8c, 0x41, 0x1c, 0x47, 0xfe, 0x98, 0x44, 0x9b, 0xda,
0xb6, 0xb6, 0x63, 0xf4, 0x6a, 0x5e, 0x4b, 0x52, 0x5e, 0x90, 0xc8, 0x79, 0x0f, 0x4c, 0xc6, 0xc5,
0xe3, 0x4f, 0x70, 0xb7, 0xbe, 0xad, 0xed, 0xe8, 0xbd, 0x9a, 0x67, 0x20, 0xa9, 0xd8, 0xbe, 0x88,
0x62, 0x22, 0x70, 0x5b, 0xdf, 0xd6, 0x76, 0x34, 0xb9, 0x8d, 0x24, 0xb9, 0xbd, 0x05, 0x90, 0x89,
0x94, 0xf1, 0x4b, 0xdc, 0x6f, 0x6c, 0x6b, 0x3b, 0x66, 0xaf, 0xe6, 0x99, 0x8a, 0x26, 0x19, 0x9e,
0x80, 0x49, 0xd2, 0x94, 0x4c, 0x70, 0xbf, 0xb9, 0xad, 0xed, 0x58, 0x8f, 0x36, 0x77, 0xff, 0xc3,
0x82, 0xdd, 0xae, 0xe4, 0x91, 0xc8, 0xc8, 0xfc, 0x82, 0x44, 0xfb, 0x4d, 0xd0, 0xc7, 0x24, 0x72,
0x7f, 0x01, 0x4d, 0xdc, 0x73, 0x3e, 0x85, 0x26, 0xee, 0x6d, 0x6a, 0xdb, 0xfa, 0x8e, 0xf5, 0x68,
0x6b, 0x01, 0xc8, 0xac, 0xd1, 0x9e, 0xe2, 0x76, 0xee, 0x82, 0x99, 0x91, 0x11, 0xf5, 0xc5, 0x24,
0xa1, 0x68, 0x9e, 0xe1, 0x19, 0x92, 0x70, 0x3e, 0x49, 0xa8, 0xfb, 0x2b, 0x0d, 0xcc, 0x9f, 0xe5,
0x34, 0x9d, 0xf4, 0xf9, 0x45, 0xec, 0x38, 0xd0, 0x10, 0x71, 0xf2, 0x12, 0x5d, 0xa4, 0x7b, 0xb8,
0x76, 0xb6, 0xc0, 0x1a, 0x51, 0x91, 0xb2, 0x40, 0x01, 0x48, 0x07, 0x98, 0x1e, 0x28, 0x92, 0x84,
0x70, 0xde, 0x87, 0xf5, 0x8c, 0x92, 0x34, 0x18, 0xfa, 0x09, 0x49, 0xc9, 0x28, 0x53, 0x3e, 0xf0,
0xda, 0x8a, 0x78, 0x8a, 0x34, 0xc9, 0x94, 0xc6, 0x39, 0x0f, 0xfd, 0x90, 0x06, 0x6c, 0x54, 0x38,
0x42, 0xf7, 0xda, 0x48, 0x3c, 0x54, 0x34, 0xf7, 0x9f, 0x1a, 0xc0, 0x41, 0x1c, 0xe5, 0x23, 0x8e,
0xda, 0xdc, 0x06, 0xe3, 0x82, 0xd1, 0x28, 0xf4, 0x59, 0x58, 0x68, 0xd4, 0xc2, 0xef, 0x7e, 0xe8,
0x3c, 0x05, 0x33, 0x24, 0x82, 0x54, 0x36, 0x75, 0x1e, 0xbd, 0x37, 0xef, 0x8e, 0x22, 0x1f, 0x0e,
0x89, 0x20, 0x52, 0x4b, 0xcf, 0x08, 0x8b, 0x95, 0xf3, 0x01, 0x74, 0x58, 0xe6, 0x27, 0x29, 0x1b,
0x91, 0x74, 0xe2, 0xbf, 0xa4, 0x13, 0xb4, 0xc9, 0xf0, 0xda, 0x2c, 0x3b, 0x55, 0xc4, 0xaf, 0x28,
0x7a, 0x8d, 0x65, 0x3e, 0xc9, 0x45, 0xdc, 0x3f, 0x44, 0x8b, 0x0c, 0xcf, 0x60, 0x59, 0x17, 0xbf,
0xa5, 0x4f, 0x38, 0xcd, 0x04, 0x0d, 0xfd, 0x84, 0x88, 0xe1, 0x66, 0x73, 0x5b, 0x97, 0x3e, 0x51,
0xa4, 0x53, 0x22, 0x86, 0xce, 0x0e, 0xd8, 0xf2, 0x0c, 0x92, 0x0a, 0x26, 0x58, 0xcc, 0xf1, 0x94,
0x35, 0x04, 0xe9, 0xb0, 0xec, 0xb4, 0x24, 0x7f, 0x45, 0x27, 0xee, 0x4f, 0x4a, 0x93, 0x8f, 0xde,
0x24, 0xa9, 0xf3, 0x11, 0x34, 0x18, 0xbf, 0x88, 0xd1, 0x5c, 0xeb, 0xaa, 0x49, 0x18, 0xe1, 0xca,
0x3f, 0x1e, 0xb2, 0x4a, 0x80, 0xa3, 0x37, 0x2c, 0x13, 0xd9, 0x7f, 0x0b, 0xb0, 0x0f, 0x26, 0xe6,
0x0b, 0xca, 0x7f, 0x0a, 0xcd, 0xb1, 0xfc, 0x28, 0x00, 0xae, 0xcf, 0x31, 0xe4, 0x76, 0x7f, 0xaf,
0x41, 0xe7, 0x1b, 0x4e, 0xd2, 0x89, 0x47, 0xf8, 0xa5, 0x42, 0xfa, 0x31, 0x58, 0x01, 0x1e, 0xe5,
0xaf, 0xae, 0x10, 0x04, 0x55, 0xf4, 0xbf, 0x0f, 0xf5, 0x38, 0x29, 0x62, 0x7b, 0x7b, 0x81, 0xd8,
0x49, 0x82, 0x71, 0xad, 0xc7, 0x49, 0xa5, 0xb4, 0x7e, 0x23, 0xa5, 0x7f, 0x57, 0x87, 0x8d, 0x7d,
0xf6, 0x6e, 0xb5, 0xfe, 0x10, 0x36, 0xa2, 0xf8, 0x35, 0x4d, 0x7d, 0xc6, 0x83, 0x28, 0xcf, 0xd8,
0xb8, 0xbc, 0x72, 0x1d, 0x24, 0xf7, 0x4b, 0xaa, 0x64, 0xcc, 0x93, 0x64, 0x8e, 0x51, 0xa5, 0x61,
0x07, 0xc9, 0x15, 0xe3, 0x1e, 0x58, 0x0a, 0x51, 0x99, 0xd8, 0x58, 0xcd, 0x44, 0x40, 0x19, 0x55,
0xfc, 0xf6, 0xc0, 0x52, 0x47, 0x29, 0x84, 0xe6, 0x8a, 0x08, 0x28, 0x83, 0x6b, 0xf7, 0x2f, 0x1a,
0x58, 0x07, 0xf1, 0x28, 0x21, 0xa9, 0xf2, 0xd2, 0x33, 0xb0, 0x23, 0x7a, 0x21, 0xfc, 0x1b, 0xbb,
0xaa, 0x23, 0xc5, 0x66, 0xae, 0x78, 0x1f, 0x6e, 0xa5, 0xec, 0x72, 0x38, 0x8f, 0x54, 0x5f, 0x05,
0x69, 0x03, 0xe5, 0x0e, 0xae, 0xe6, 0x8b, 0xbe, 0x42, 0xbe, 0xb8, 0xbf, 0xd5, 0xc0, 0x38, 0xa7,
0xe9, 0xe8, 0x9d, 0x44, 0xfc, 0x09, 0xac, 0xa1, 0x5f, 0xb3, 0xcd, 0xfa, 0x6a, 0x65, 0xb9, 0x60,
0x77, 0xee, 0x81, 0xc5, 0x32, 0x9f, 0x71, 0x1f, 0x8b, 0x5a, 0x11, 0x7d, 0x93, 0x65, 0x7d, 0xfe,
0xa5, 0x24, 0xb8, 0x7f, 0xaa, 0x83, 0xfd, 0xd3, 0xb3, 0x93, 0xe3, 0x83, 0x98, 0x0b, 0xc2, 0x78,
0xf6, 0x4e, 0xb4, 0xfd, 0x1c, 0x0c, 0x1a, 0xd1, 0x11, 0xe5, 0x62, 0x65, 0x7d, 0xa7, 0x02, 0xce,
0xd3, 0x19, 0x17, 0xdf, 0x5f, 0x20, 0x76, 0x55, 0x5b, 0x24, 0x9c, 0x24, 0x78, 0x47, 0x7f, 0x08,
0x4e, 0x89, 0xe3, 0x57, 0xed, 0x48, 0x15, 0x56, 0xbb, 0xdc, 0x39, 0x2b, 0xdb, 0xd2, 0x11, 0xac,
0x29, 0x59, 0xc7, 0x82, 0x56, 0x9f, 0x8f, 0x49, 0xc4, 0x42, 0xbb, 0xe6, 0xb4, 0xc1, 0x28, 0xf1,
0x6d, 0xcd, 0xd9, 0x90, 0x49, 0xa9, 0xbe, 0xba, 0x51, 0x64, 0xd7, 0xe7, 0x08, 0x7c, 0x62, 0xeb,
0xee, 0xaf, 0x35, 0x30, 0xb1, 0x2c, 0xa1, 0xef, 0x3e, 0x41, 0xf5, 0x35, 0x54, 0xff, 0x83, 0x05,
0xea, 0x4f, 0x39, 0xd5, 0xaa, 0x50, 0xfc, 0x01, 0x34, 0x83, 0x21, 0x8b, 0xc2, 0x22, 0x2d, 0xbf,
0xbd, 0x40, 0x50, 0xca, 0x78, 0x8a, 0xcb, 0xdd, 0x82, 0x56, 0x21, 0x3d, 0xaf, 0x7a, 0x0b, 0xf4,
0xe3, 0x58, 0xd8, 0x9a, 0xfb, 0x37, 0x0d, 0x40, 0x55, 0x1d, 0x54, 0xea, 0xf1, 0x8c, 0x52, 0xdf,
0x5b, 0x80, 0x5d, 0xb1, 0x16, 0xcb, 0x42, 0xad, 0x1f, 0x40, 0x43, 0xde, 0xa5, 0xeb, 0xb4, 0x42,
0x26, 0x69, 0x03, 0x5e, 0x97, 0xa2, 0x40, 0x2e, 0xb7, 0x01, 0xb9, 0xdc, 0xc7, 0x60, 0x94, 0x67,
0xcd, 0x1b, 0xd1, 0x01, 0xf8, 0x3a, 0xbe, 0x64, 0x01, 0x89, 0xba, 0x3c, 0xb4, 0x35, 0x67, 0x1d,
0xcc, 0xe2, 0xfb, 0x24, 0xb5, 0xeb, 0xee, 0x5f, 0x35, 0x58, 0x57, 0x82, 0xdd, 0x94, 0x89, 0xe1,
0x49, 0xf2, 0x3f, 0xa7, 0xeb, 0x67, 0x60, 0x10, 0x09, 0xe5, 0x4f, 0x5b, 0xc1, 0xbd, 0x85, 0xa3,
0x13, 0x9e, 0x86, 0xf7, 0xbb, 0x45, 0x8a, 0xa3, 0x0f, 0x61, 0x5d, 0x95, 0x96, 0x38, 0xa1, 0x29,
0xe1, 0xe1, 0xaa, 0xcd, 0xa1, 0x8d, 0x52, 0x27, 0x4a, 0xc8, 0xfd, 0x8d, 0x56, 0xf6, 0x08, 0x3c,
0x04, 0x43, 0x56, 0xba, 0x5e, 0xbb, 0x91, 0xeb, 0xeb, 0xab, 0xb8, 0xde, 0xd9, 0x9d, 0xb9, 0x62,
0xd7, 0x99, 0x2a, 0x4b, 0xd9, 0x1f, 0xeb, 0x70, 0x67, 0xce, 0xe5, 0x47, 0x63, 0x12, 0xbd, 0xbb,
0x76, 0xf6, 0xff, 0xf6, 0x7f, 0x51, 0xd5, 0x1b, 0x37, 0x9a, 0x02, 0x9a, 0x37, 0x9a, 0x02, 0x6c,
0xe8, 0x74, 0xa3, 0xd7, 0x64, 0x92, 0x9d, 0xa7, 0x6a, 0x06, 0x72, 0xff, 0xde, 0x82, 0x06, 0x7a,
0xef, 0x29, 0x98, 0x82, 0xa6, 0x23, 0x9f, 0xbe, 0x49, 0xd2, 0xc2, 0x77, 0x77, 0x17, 0xa0, 0x96,
0xad, 0x44, 0x0e, 0xef, 0xa2, 0x6c, 0x2b, 0x5f, 0x00, 0xe4, 0x32, 0x2c, 0x4a, 0x58, 0x05, 0xff,
0x3b, 0x6f, 0x2b, 0x3a, 0xf2, 0xd1, 0x90, 0x4f, 0xcb, 0xc2, 0x1e, 0x58, 0x03, 0x56, 0xc9, 0xeb,
0x4b, 0x03, 0x57, 0xd5, 0x87, 0x5e, 0xcd, 0x83, 0x41, 0x55, 0x58, 0x0e, 0xa0, 0x1d, 0xa8, 0x96,
0xad, 0x20, 0xd4, 0xe0, 0x70, 0x6f, 0x61, 0xec, 0xa7, 0x9d, 0xbd, 0x57, 0xf3, 0xac, 0x60, 0xa6,
0xd1, 0x3f, 0x07, 0x5b, 0x59, 0x91, 0xca, 0x94, 0x52, 0x40, 0xca, 0xbd, 0xdf, 0x5d, 0x66, 0xcb,
0x34, 0xf9, 0x7a, 0x35, 0xaf, 0x93, 0xcf, 0x4f, 0x57, 0xa7, 0x70, 0xab, 0xb0, 0x6a, 0x06, 0x6f,
0x0d, 0xf1, 0xdc, 0xa5, 0xb6, 0xcd, 0x02, 0x6e, 0x0c, 0xae, 0xcc, 0x6b, 0x02, 0xb6, 0x0a, 0xc4,
0x32, 0x4f, 0x7d, 0x3a, 0x26, 0xd1, 0x2c, 0x7e, 0x0b, 0xf1, 0x1f, 0x2c, 0xc5, 0x5f, 0x74, 0x71,
0x7a, 0x35, 0xef, 0xce, 0x60, 0xf9, 0xb5, 0xaa, 0xec, 0x50, 0xa7, 0xe2, 0x39, 0xc6, 0x35, 0x76,
0x4c, 0x0b, 0x48, 0x65, 0x47, 0x55, 0x53, 0xbe, 0x00, 0xc0, 0x74, 0x54, 0x50, 0xe6, 0xd2, 0x74,
0x99, 0x4e, 0xea, 0x32, 0x5d, 0xc6, 0xd3, 0xb1, 0x7d, 0x6f, 0x7a, 0xcf, 0x51, 0x1e, 0xae, 0xb9,
0xe7, 0x65, 0xba, 0x04, 0xd5, 0xcb, 0x63, 0x0f, 0x2c, 0x8a, 0xcf, 0x08, 0x85, 0x60, 0x2d, 0x45,
0xa8, 0x1e, 0x1b, 0x12, 0x81, 0x56, 0x4f, 0x8f, 0xe7, 0x60, 0x13, 0xbc, 0x48, 0xbe, 0x48, 0x4b,
0x43, 0xda, 0x4b, 0x73, 0x65, 0xfe, 0xce, 0xc9, 0x5c, 0x21, 0x73, 0x14, 0xe7, 0x0c, 0x9c, 0x5f,
0x66, 0x31, 0xf7, 0x83, 0xa2, 0xa3, 0x2b, 0xc0, 0x75, 0x04, 0x7c, 0x7f, 0x85, 0xe1, 0xa3, 0x57,
0xf3, 0x6c, 0x09, 0x30, 0x4b, 0xdb, 0x5f, 0x83, 0x86, 0x84, 0x71, 0xff, 0xa1, 0x01, 0xbc, 0xa0,
0x81, 0x88, 0xd3, 0xee, 0xf1, 0xf1, 0x59, 0xf1, 0xd8, 0x53, 0x31, 0x51, 0xff, 0x07, 0xe4, 0x63,
0x4f, 0x85, 0x6d, 0xee, 0x19, 0x5a, 0x9f, 0x7f, 0x86, 0x3e, 0x01, 0x48, 0x52, 0x1a, 0xb2, 0x80,
0x08, 0x9a, 0x5d, 0xd7, 0x5c, 0x67, 0x58, 0x9d, 0xcf, 0x01, 0x5e, 0xc9, 0x57, 0xb7, 0x2a, 0xcb,
0x8d, 0xa5, 0xe1, 0x9e, 0x3e, 0xcd, 0x3d, 0xf3, 0xd5, 0xf4, 0x95, 0xfe, 0x21, 0x6c, 0x24, 0x11,
0x09, 0xe8, 0x30, 0x8e, 0x42, 0x9a, 0xfa, 0x82, 0x5c, 0xe2, 0x9d, 0x34, 0xbd, 0xce, 0x0c, 0xf9,
0x9c, 0x5c, 0xba, 0x13, 0x58, 0x47, 0x80, 0xd3, 0x88, 0xf0, 0xe3, 0x38, 0xa4, 0x57, 0xf4, 0xd5,
0x56, 0xd7, 0xf7, 0x36, 0x18, 0x2c, 0xf3, 0x83, 0x38, 0xe7, 0xa2, 0x78, 0xcf, 0xb4, 0x58, 0x76,
0x20, 0x3f, 0x9d, 0x6f, 0x41, 0x33, 0x62, 0x23, 0xa6, 0x66, 0x0b, 0xdd, 0x53, 0x1f, 0xee, 0xbf,
0x34, 0x30, 0xa6, 0xc7, 0xee, 0x81, 0x35, 0x46, 0x67, 0xfb, 0x84, 0xf3, 0xec, 0x2d, 0x5d, 0xa8,
0x0a, 0x89, 0xcc, 0x2d, 0x25, 0xd3, 0xe5, 0x3c, 0x73, 0x3e, 0x9b, 0x53, 0xfc, 0xed, 0xad, 0x54,
0x8a, 0xce, 0xa8, 0xfe, 0x23, 0x68, 0xa2, 0xeb, 0x0a, 0x2f, 0x6f, 0x2f, 0xf3, 0x72, 0xa9, 0x6d,
0xaf, 0xe6, 0x29, 0x01, 0xf9, 0x88, 0x8f, 0x73, 0x91, 0xe4, 0xc2, 0x2f, 0xe3, 0x2f, 0x63, 0xac,
0xef, 0xe8, 0x5e, 0x47, 0xd1, 0xbf, 0x54, 0x69, 0x90, 0xc9, 0xb4, 0xe2, 0x71, 0x48, 0xef, 0xff,
0x41, 0x83, 0x35, 0xd5, 0x91, 0xe6, 0xe7, 0xa6, 0x0d, 0xb0, 0x9e, 0xa5, 0x94, 0x08, 0x9a, 0x9e,
0x0f, 0x09, 0xb7, 0x35, 0xc7, 0x86, 0x76, 0x41, 0x38, 0x7a, 0x95, 0x13, 0x39, 0xbb, 0xb6, 0xc1,
0xf8, 0x9a, 0x66, 0x19, 0xee, 0xeb, 0x38, 0x58, 0xd1, 0x2c, 0x53, 0x9b, 0x0d, 0xc7, 0x84, 0xa6,
0x5a, 0x36, 0x25, 0xdf, 0x71, 0x2c, 0xd4, 0xd7, 0x9a, 0x04, 0x3e, 0x4d, 0xe9, 0x05, 0x7b, 0xf3,
0x9c, 0x88, 0x60, 0x68, 0xb7, 0x24, 0xf0, 0x69, 0x9c, 0x89, 0x29, 0xc5, 0x90, 0xb2, 0x6a, 0x69,
0xca, 0x25, 0xd6, 0x30, 0x1b, 0x9c, 0x35, 0xa8, 0xf7, 0xb9, 0x6d, 0x49, 0xd2, 0x71, 0x2c, 0xfa,
0xdc, 0x6e, 0xdf, 0x7f, 0x06, 0xd6, 0x4c, 0x23, 0x97, 0x06, 0x7c, 0xc3, 0x5f, 0xf2, 0xf8, 0x35,
0x57, 0xd3, 0x6b, 0x37, 0x94, 0x13, 0x5f, 0x0b, 0xf4, 0xb3, 0x7c, 0x60, 0xd7, 0xe5, 0xe2, 0x79,
0x1e, 0xd9, 0xba, 0x5c, 0x1c, 0xb2, 0xb1, 0xdd, 0x40, 0x4a, 0x1c, 0xda, 0xcd, 0xfd, 0x8f, 0x7f,
0xfe, 0xd1, 0x25, 0x13, 0xc3, 0x7c, 0xb0, 0x1b, 0xc4, 0xa3, 0x87, 0xca, 0xdd, 0x0f, 0x58, 0x5c,
0xac, 0x1e, 0x32, 0x2e, 0x68, 0xca, 0x49, 0xf4, 0x10, 0x23, 0xf0, 0x50, 0x46, 0x20, 0x19, 0x0c,
0xd6, 0xf0, 0xeb, 0xe3, 0x7f, 0x07, 0x00, 0x00, 0xff, 0xff, 0x48, 0x4f, 0x55, 0x7b, 0xe8, 0x13,
0x00, 0x00,
// 1796 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x58, 0x4f, 0x73, 0xdc, 0x48,
0x15, 0x1f, 0x8d, 0xe6, 0x8f, 0xf4, 0x66, 0x3c, 0x56, 0x54, 0x54, 0xe1, 0x24, 0x6c, 0x6c, 0xb4,
0x5b, 0xac, 0x09, 0xc4, 0xa9, 0x64, 0x77, 0x13, 0x36, 0x5b, 0x0b, 0x1e, 0xff, 0x49, 0x66, 0xd8,
0x8d, 0x6d, 0x64, 0x6f, 0x0e, 0x70, 0x50, 0xf5, 0x48, 0x6d, 0x4f, 0x13, 0x4d, 0x4b, 0x91, 0x5a,
0x93, 0xcc, 0x17, 0xe0, 0xc0, 0x8d, 0x0f, 0xc0, 0x89, 0x03, 0x77, 0x8e, 0x5c, 0xe0, 0x4c, 0x71,
0xe0, 0xc8, 0x91, 0xaf, 0x40, 0xf1, 0x05, 0xa8, 0x7e, 0x2d, 0x8d, 0x66, 0xcc, 0x4c, 0x3c, 0x06,
0x57, 0x71, 0x6b, 0xbd, 0x7e, 0xef, 0xd7, 0xef, 0x5f, 0xbf, 0xf7, 0x5a, 0x00, 0x71, 0x48, 0xf8,
0x4e, 0x9c, 0x44, 0x22, 0xb2, 0x6f, 0x8d, 0x58, 0x38, 0xce, 0x52, 0xf5, 0xb5, 0x23, 0x37, 0xee,
0xb4, 0x53, 0x7f, 0x48, 0x47, 0x44, 0x91, 0x9c, 0xbf, 0x68, 0xd0, 0x7e, 0x41, 0x39, 0x4d, 0x98,
0xff, 0x8a, 0x84, 0x19, 0xb5, 0xef, 0x82, 0x31, 0x88, 0xa2, 0xd0, 0x1b, 0x93, 0x70, 0x43, 0xdb,
0xd2, 0xb6, 0x8d, 0x5e, 0xc5, 0x6d, 0x4a, 0xca, 0x2b, 0x12, 0xda, 0x1f, 0x80, 0xc9, 0xb8, 0x78,
0xf2, 0x29, 0xee, 0x56, 0xb7, 0xb4, 0x6d, 0xbd, 0x57, 0x71, 0x0d, 0x24, 0xe5, 0xdb, 0xe7, 0x61,
0x44, 0x04, 0x6e, 0xeb, 0x5b, 0xda, 0xb6, 0x26, 0xb7, 0x91, 0x24, 0xb7, 0x37, 0x01, 0x52, 0x91,
0x30, 0x7e, 0x81, 0xfb, 0xb5, 0x2d, 0x6d, 0xdb, 0xec, 0x55, 0x5c, 0x53, 0xd1, 0x24, 0xc3, 0x53,
0x30, 0x49, 0x92, 0x90, 0x09, 0xee, 0xd7, 0xb7, 0xb4, 0xed, 0xd6, 0xe3, 0x8d, 0x9d, 0xff, 0xb0,
0x60, 0xa7, 0x2b, 0x79, 0x24, 0x32, 0x32, 0xbf, 0x22, 0xe1, 0x5e, 0x1d, 0xf4, 0x31, 0x09, 0x9d,
0x5f, 0x40, 0x1d, 0xf7, 0xec, 0xcf, 0xa0, 0x8e, 0x7b, 0x1b, 0xda, 0x96, 0xbe, 0xdd, 0x7a, 0xbc,
0xb9, 0x00, 0x64, 0xd6, 0x68, 0x57, 0x71, 0xdb, 0x77, 0xc1, 0x4c, 0xc9, 0x88, 0x7a, 0x62, 0x12,
0x53, 0x34, 0xcf, 0x70, 0x0d, 0x49, 0x38, 0x9b, 0xc4, 0xd4, 0xf9, 0xb5, 0x06, 0xe6, 0xcf, 0x32,
0x9a, 0x4c, 0xfa, 0xfc, 0x3c, 0xb2, 0x6d, 0xa8, 0x89, 0x28, 0x7e, 0x8d, 0x2e, 0xd2, 0x5d, 0x5c,
0xdb, 0x9b, 0xd0, 0x1a, 0x51, 0x91, 0x30, 0x5f, 0x01, 0x48, 0x07, 0x98, 0x2e, 0x28, 0x92, 0x84,
0xb0, 0x3f, 0x84, 0xb5, 0x94, 0x92, 0xc4, 0x1f, 0x7a, 0x31, 0x49, 0xc8, 0x28, 0x55, 0x3e, 0x70,
0xdb, 0x8a, 0x78, 0x82, 0x34, 0xc9, 0x94, 0x44, 0x19, 0x0f, 0xbc, 0x80, 0xfa, 0x6c, 0x94, 0x3b,
0x42, 0x77, 0xdb, 0x48, 0x3c, 0x50, 0x34, 0xe7, 0x9f, 0x1a, 0xc0, 0x7e, 0x14, 0x66, 0x23, 0x8e,
0xda, 0xdc, 0x06, 0xe3, 0x9c, 0xd1, 0x30, 0xf0, 0x58, 0x90, 0x6b, 0xd4, 0xc4, 0xef, 0x7e, 0x60,
0x3f, 0x03, 0x33, 0x20, 0x82, 0x94, 0x36, 0x75, 0x1e, 0x7f, 0x30, 0xef, 0x8e, 0x3c, 0x1f, 0x0e,
0x88, 0x20, 0x52, 0x4b, 0xd7, 0x08, 0xf2, 0x95, 0xfd, 0x11, 0x74, 0x58, 0xea, 0xc5, 0x09, 0x1b,
0x91, 0x64, 0xe2, 0xbd, 0xa6, 0x13, 0xb4, 0xc9, 0x70, 0xdb, 0x2c, 0x3d, 0x51, 0xc4, 0xaf, 0x28,
0x7a, 0x8d, 0xa5, 0x1e, 0xc9, 0x44, 0xd4, 0x3f, 0x40, 0x8b, 0x0c, 0xd7, 0x60, 0x69, 0x17, 0xbf,
0xa5, 0x4f, 0x38, 0x4d, 0x05, 0x0d, 0xbc, 0x98, 0x88, 0xe1, 0x46, 0x7d, 0x4b, 0x97, 0x3e, 0x51,
0xa4, 0x13, 0x22, 0x86, 0xf6, 0x36, 0x58, 0xf2, 0x0c, 0x92, 0x08, 0x26, 0x58, 0xc4, 0xf1, 0x94,
0x06, 0x82, 0x74, 0x58, 0x7a, 0x52, 0x90, 0xbf, 0xa2, 0x13, 0xe7, 0x27, 0x85, 0xc9, 0x87, 0xef,
0xe2, 0xc4, 0x7e, 0x04, 0x35, 0xc6, 0xcf, 0x23, 0x34, 0xb7, 0x75, 0xd9, 0x24, 0x8c, 0x70, 0xe9,
0x1f, 0x17, 0x59, 0x25, 0xc0, 0xe1, 0x3b, 0x96, 0x8a, 0xf4, 0xbf, 0x05, 0xd8, 0x03, 0x13, 0xf3,
0x05, 0xe5, 0x3f, 0x83, 0xfa, 0x58, 0x7e, 0xe4, 0x00, 0x57, 0xe7, 0x18, 0x72, 0x3b, 0x7f, 0xd0,
0xa0, 0xf3, 0x0d, 0x27, 0xc9, 0xc4, 0x25, 0xfc, 0x42, 0x21, 0xfd, 0x18, 0x5a, 0x3e, 0x1e, 0xe5,
0xad, 0xae, 0x10, 0xf8, 0x65, 0xf4, 0xbf, 0x0f, 0xd5, 0x28, 0xce, 0x63, 0x7b, 0x7b, 0x81, 0xd8,
0x71, 0x8c, 0x71, 0xad, 0x46, 0x71, 0xa9, 0xb4, 0x7e, 0x2d, 0xa5, 0x7f, 0x5f, 0x85, 0xf5, 0x3d,
0x76, 0xb3, 0x5a, 0x7f, 0x0c, 0xeb, 0x61, 0xf4, 0x96, 0x26, 0x1e, 0xe3, 0x7e, 0x98, 0xa5, 0x6c,
0x5c, 0x5c, 0xb9, 0x0e, 0x92, 0xfb, 0x05, 0x55, 0x32, 0x66, 0x71, 0x3c, 0xc7, 0xa8, 0xd2, 0xb0,
0x83, 0xe4, 0x92, 0x71, 0x17, 0x5a, 0x0a, 0x51, 0x99, 0x58, 0x5b, 0xcd, 0x44, 0x40, 0x19, 0x55,
0xfc, 0x76, 0xa1, 0xa5, 0x8e, 0x52, 0x08, 0xf5, 0x15, 0x11, 0x50, 0x06, 0xd7, 0xce, 0x5f, 0x35,
0x68, 0xed, 0x47, 0xa3, 0x98, 0x24, 0xca, 0x4b, 0x2f, 0xc0, 0x0a, 0xe9, 0xb9, 0xf0, 0xae, 0xed,
0xaa, 0x8e, 0x14, 0x9b, 0xb9, 0xe2, 0x7d, 0xb8, 0x95, 0xb0, 0x8b, 0xe1, 0x3c, 0x52, 0x75, 0x15,
0xa4, 0x75, 0x94, 0xdb, 0xbf, 0x9c, 0x2f, 0xfa, 0x0a, 0xf9, 0xe2, 0xfc, 0x4e, 0x03, 0xe3, 0x8c,
0x26, 0xa3, 0x1b, 0x89, 0xf8, 0x53, 0x68, 0xa0, 0x5f, 0xd3, 0x8d, 0xea, 0x6a, 0x65, 0x39, 0x67,
0xb7, 0xef, 0x41, 0x8b, 0xa5, 0x1e, 0xe3, 0x1e, 0x16, 0xb5, 0x3c, 0xfa, 0x26, 0x4b, 0xfb, 0xfc,
0xb9, 0x24, 0x38, 0x7f, 0xae, 0x82, 0xf5, 0xd3, 0xd3, 0xe3, 0xa3, 0xfd, 0x88, 0x0b, 0xc2, 0x78,
0x7a, 0x23, 0xda, 0x7e, 0x01, 0x06, 0x0d, 0xe9, 0x88, 0x72, 0xb1, 0xb2, 0xbe, 0x53, 0x01, 0xfb,
0xd9, 0x8c, 0x8b, 0xef, 0x2f, 0x10, 0xbb, 0xac, 0x2d, 0x12, 0x8e, 0x63, 0xbc, 0xa3, 0x3f, 0x04,
0xbb, 0xc0, 0xf1, 0xca, 0x76, 0xa4, 0x0a, 0xab, 0x55, 0xec, 0x9c, 0x16, 0x6d, 0xe9, 0x10, 0x1a,
0x4a, 0xd6, 0x6e, 0x41, 0xb3, 0xcf, 0xc7, 0x24, 0x64, 0x81, 0x55, 0xb1, 0xdb, 0x60, 0x14, 0xf8,
0x96, 0x66, 0xaf, 0xcb, 0xa4, 0x54, 0x5f, 0xdd, 0x30, 0xb4, 0xaa, 0x73, 0x04, 0x3e, 0xb1, 0x74,
0xe7, 0x37, 0x1a, 0x98, 0x58, 0x96, 0xd0, 0x77, 0x9f, 0xa2, 0xfa, 0x1a, 0xaa, 0xff, 0xd1, 0x02,
0xf5, 0xa7, 0x9c, 0x6a, 0x95, 0x2b, 0xfe, 0x00, 0xea, 0xfe, 0x90, 0x85, 0x41, 0x9e, 0x96, 0xdf,
0x5e, 0x20, 0x28, 0x65, 0x5c, 0xc5, 0xe5, 0x6c, 0x42, 0x33, 0x97, 0x9e, 0x57, 0xbd, 0x09, 0xfa,
0x51, 0x24, 0x2c, 0xcd, 0xf9, 0xbb, 0x06, 0xa0, 0xaa, 0x0e, 0x2a, 0xf5, 0x64, 0x46, 0xa9, 0xef,
0x2d, 0xc0, 0x2e, 0x59, 0xf3, 0x65, 0xae, 0xd6, 0x0f, 0xa0, 0x26, 0xef, 0xd2, 0x55, 0x5a, 0x21,
0x93, 0xb4, 0x01, 0xaf, 0x4b, 0x5e, 0x20, 0x97, 0xdb, 0x80, 0x5c, 0xce, 0x13, 0x30, 0x8a, 0xb3,
0xe6, 0x8d, 0xe8, 0x00, 0x7c, 0x1d, 0x5d, 0x30, 0x9f, 0x84, 0x5d, 0x1e, 0x58, 0x9a, 0xbd, 0x06,
0x66, 0xfe, 0x7d, 0x9c, 0x58, 0x55, 0xe7, 0x6f, 0x1a, 0xac, 0x29, 0xc1, 0x6e, 0xc2, 0xc4, 0xf0,
0x38, 0xfe, 0x9f, 0xd3, 0xf5, 0x73, 0x30, 0x88, 0x84, 0xf2, 0xa6, 0xad, 0xe0, 0xde, 0xc2, 0xd1,
0x09, 0x4f, 0xc3, 0xfb, 0xdd, 0x24, 0xf9, 0xd1, 0x07, 0xb0, 0xa6, 0x4a, 0x4b, 0x14, 0xd3, 0x84,
0xf0, 0x60, 0xd5, 0xe6, 0xd0, 0x46, 0xa9, 0x63, 0x25, 0xe4, 0xfc, 0x56, 0x2b, 0x7a, 0x04, 0x1e,
0x82, 0x21, 0x2b, 0x5c, 0xaf, 0x5d, 0xcb, 0xf5, 0xd5, 0x55, 0x5c, 0x6f, 0xef, 0xcc, 0x5c, 0xb1,
0xab, 0x4c, 0x95, 0xa5, 0xec, 0x4f, 0x55, 0xb8, 0x33, 0xe7, 0xf2, 0xc3, 0x31, 0x09, 0x6f, 0xae,
0x9d, 0xfd, 0xbf, 0xfd, 0x9f, 0x57, 0xf5, 0xda, 0xb5, 0xa6, 0x80, 0xfa, 0xb5, 0xa6, 0x00, 0x0b,
0x3a, 0xdd, 0xf0, 0x2d, 0x99, 0xa4, 0x67, 0x89, 0x9a, 0x81, 0x9c, 0x7f, 0x34, 0xa1, 0x86, 0xde,
0x7b, 0x06, 0xa6, 0xa0, 0xc9, 0xc8, 0xa3, 0xef, 0xe2, 0x24, 0xf7, 0xdd, 0xdd, 0x05, 0xa8, 0x45,
0x2b, 0x91, 0xc3, 0xbb, 0x28, 0xda, 0xca, 0x97, 0x00, 0x99, 0x0c, 0x8b, 0x12, 0x56, 0xc1, 0xff,
0xce, 0xfb, 0x8a, 0x8e, 0x7c, 0x34, 0x64, 0xd3, 0xb2, 0xb0, 0x0b, 0xad, 0x01, 0x2b, 0xe5, 0xf5,
0xa5, 0x81, 0x2b, 0xeb, 0x43, 0xaf, 0xe2, 0xc2, 0xa0, 0x2c, 0x2c, 0xfb, 0xd0, 0xf6, 0x55, 0xcb,
0x56, 0x10, 0x6a, 0x70, 0xb8, 0xb7, 0x30, 0xf6, 0xd3, 0xce, 0xde, 0xab, 0xb8, 0x2d, 0x7f, 0xa6,
0xd1, 0xbf, 0x04, 0x4b, 0x59, 0x91, 0xc8, 0x94, 0x52, 0x40, 0xca, 0xbd, 0xdf, 0x5d, 0x66, 0xcb,
0x34, 0xf9, 0x7a, 0x15, 0xb7, 0x93, 0xcd, 0x4f, 0x57, 0x27, 0x70, 0x2b, 0xb7, 0x6a, 0x06, 0xaf,
0x81, 0x78, 0xce, 0x52, 0xdb, 0x66, 0x01, 0xd7, 0x07, 0x97, 0xe6, 0x35, 0x01, 0x9b, 0x39, 0x62,
0x91, 0xa7, 0x1e, 0x1d, 0x93, 0x70, 0x16, 0xbf, 0x89, 0xf8, 0x0f, 0x96, 0xe2, 0x2f, 0xba, 0x38,
0xbd, 0x8a, 0x7b, 0x67, 0xb0, 0xfc, 0x5a, 0x95, 0x76, 0xa8, 0x53, 0xf1, 0x1c, 0xe3, 0x0a, 0x3b,
0xa6, 0x05, 0xa4, 0xb4, 0xa3, 0xac, 0x29, 0x5f, 0x02, 0x60, 0x3a, 0x2a, 0x28, 0x73, 0x69, 0xba,
0x4c, 0x27, 0x75, 0x99, 0x2e, 0xe3, 0xe9, 0xd8, 0xbe, 0x3b, 0xbd, 0xe7, 0x28, 0x0f, 0x57, 0xdc,
0xf3, 0x22, 0x5d, 0xfc, 0xf2, 0xe5, 0xb1, 0x0b, 0x2d, 0x8a, 0xcf, 0x08, 0x85, 0xd0, 0x5a, 0x8a,
0x50, 0x3e, 0x36, 0x24, 0x02, 0x2d, 0x9f, 0x1e, 0x2f, 0xc1, 0x22, 0x78, 0x91, 0x3c, 0x91, 0x14,
0x86, 0xb4, 0x97, 0xe6, 0xca, 0xfc, 0x9d, 0x93, 0xb9, 0x42, 0xe6, 0x28, 0xf6, 0x29, 0xd8, 0xbf,
0x4c, 0x23, 0xee, 0xf9, 0x79, 0x47, 0x57, 0x80, 0x6b, 0x08, 0xf8, 0xe1, 0x0a, 0xc3, 0x47, 0xaf,
0xe2, 0x5a, 0x12, 0x60, 0x96, 0xb6, 0xd7, 0x80, 0x9a, 0x84, 0x71, 0x7e, 0x55, 0x05, 0x78, 0x45,
0x7d, 0x11, 0x25, 0xdd, 0xa3, 0xa3, 0x53, 0x59, 0x26, 0xc7, 0xf8, 0xa5, 0xa6, 0x12, 0x6d, 0xd1,
0x83, 0x52, 0xb9, 0x1f, 0xb9, 0xb0, 0xe4, 0xc0, 0x78, 0xba, 0x9e, 0x7b, 0xa9, 0x56, 0xe7, 0x5f,
0xaa, 0x4f, 0x01, 0xe2, 0x84, 0x06, 0xcc, 0x27, 0x82, 0xa6, 0x57, 0xf5, 0xdf, 0x19, 0x56, 0xfb,
0x0b, 0x80, 0x37, 0xf2, 0x61, 0xae, 0x2a, 0x77, 0x6d, 0x69, 0x46, 0x4c, 0x5f, 0xef, 0xae, 0xf9,
0x66, 0xfa, 0x90, 0xff, 0x18, 0xd6, 0xe3, 0x90, 0xf8, 0x74, 0x18, 0x85, 0x01, 0x4d, 0x3c, 0x41,
0x2e, 0xf0, 0xda, 0x9a, 0x6e, 0x67, 0x86, 0x7c, 0x46, 0x2e, 0x9c, 0x09, 0xac, 0x21, 0xc0, 0x49,
0x48, 0xf8, 0x51, 0x14, 0xd0, 0x4b, 0xfa, 0x6a, 0xab, 0xeb, 0x7b, 0x1b, 0x0c, 0x96, 0x7a, 0x7e,
0x94, 0x71, 0x91, 0x3f, 0x79, 0x9a, 0x2c, 0xdd, 0x97, 0x9f, 0xf6, 0xb7, 0xa0, 0x1e, 0xb2, 0x11,
0x53, 0xe3, 0x87, 0xee, 0xaa, 0x0f, 0xe7, 0x5f, 0x1a, 0x18, 0xd3, 0x63, 0x77, 0xa7, 0x11, 0x20,
0x9c, 0xa7, 0xef, 0x69, 0x54, 0x65, 0xd4, 0x64, 0xfa, 0x29, 0x99, 0x2e, 0xe7, 0xa9, 0xfd, 0xf9,
0x9c, 0xe2, 0xef, 0xef, 0xb6, 0x52, 0x74, 0x46, 0xf5, 0x1f, 0x41, 0x1d, 0x5d, 0x97, 0x7b, 0x79,
0x6b, 0x99, 0x97, 0x0b, 0x6d, 0x7b, 0x15, 0x57, 0x09, 0xc8, 0x77, 0x7e, 0x94, 0x89, 0x38, 0x13,
0x5e, 0x11, 0x7f, 0x19, 0x63, 0x7d, 0x5b, 0x77, 0x3b, 0x8a, 0xfe, 0x5c, 0xa5, 0x41, 0x2a, 0x33,
0x8f, 0x47, 0x01, 0xbd, 0xff, 0x47, 0x0d, 0x1a, 0xaa, 0x69, 0xcd, 0x8f, 0x56, 0xeb, 0xd0, 0x7a,
0x91, 0x50, 0x22, 0x68, 0x72, 0x36, 0x24, 0xdc, 0xd2, 0x6c, 0x0b, 0xda, 0x39, 0xe1, 0xf0, 0x4d,
0x46, 0xe4, 0x78, 0xdb, 0x06, 0xe3, 0x6b, 0x9a, 0xa6, 0xb8, 0xaf, 0xe3, 0xec, 0x45, 0xd3, 0x54,
0x6d, 0xd6, 0x6c, 0x13, 0xea, 0x6a, 0x59, 0x97, 0x7c, 0x47, 0x91, 0x50, 0x5f, 0x0d, 0x09, 0x7c,
0x92, 0xd0, 0x73, 0xf6, 0xee, 0x25, 0x11, 0xfe, 0xd0, 0x6a, 0x4a, 0xe0, 0x93, 0x28, 0x15, 0x53,
0x8a, 0x21, 0x65, 0xd5, 0xd2, 0x94, 0x4b, 0x2c, 0x73, 0x16, 0xd8, 0x0d, 0xa8, 0xf6, 0xb9, 0xd5,
0x92, 0xa4, 0xa3, 0x48, 0xf4, 0xb9, 0xd5, 0xbe, 0xff, 0x02, 0x5a, 0x33, 0xbd, 0x5e, 0x1a, 0xf0,
0x0d, 0x7f, 0xcd, 0xa3, 0xb7, 0x5c, 0x0d, 0xb8, 0xdd, 0x40, 0x0e, 0x85, 0x4d, 0xd0, 0x4f, 0xb3,
0x81, 0x55, 0x95, 0x8b, 0x97, 0x59, 0x68, 0xe9, 0x72, 0x71, 0xc0, 0xc6, 0x56, 0x0d, 0x29, 0x51,
0x60, 0xd5, 0xef, 0xef, 0x15, 0xd7, 0x0f, 0x71, 0x2c, 0x68, 0xab, 0x12, 0xa9, 0x68, 0xca, 0x1b,
0xcf, 0xf1, 0x07, 0x9b, 0x22, 0x68, 0xf6, 0x2d, 0x58, 0x43, 0xc2, 0xa3, 0x27, 0x39, 0xa9, 0xba,
0xf7, 0xc9, 0xcf, 0x1f, 0x5d, 0x30, 0x31, 0xcc, 0x06, 0x3b, 0x7e, 0x34, 0x7a, 0xa8, 0x42, 0xf6,
0x80, 0x45, 0xf9, 0xea, 0x21, 0xe3, 0x82, 0x26, 0x9c, 0x84, 0x0f, 0x31, 0x8a, 0x0f, 0x65, 0x14,
0xe3, 0xc1, 0xa0, 0x81, 0x5f, 0x9f, 0xfc, 0x3b, 0x00, 0x00, 0xff, 0xff, 0x3f, 0x1a, 0x78, 0xf1,
0x4f, 0x14, 0x00, 0x00,
}

5
internal/proto/proxypb/proxy.pb.go
View File

@ -29,9 +29,8 @@ const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
type InvalidateCollMetaCacheRequest struct {
// MsgType:
//
// DropCollection -> {meta cache, dml channels}
// Other -> {meta cache}
// DropCollection -> {meta cache, dml channels}
// Other -> {meta cache}
Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"`
DbName string `protobuf:"bytes,2,opt,name=db_name,json=dbName,proto3" json:"db_name,omitempty"`
CollectionName string `protobuf:"bytes,3,opt,name=collection_name,json=collectionName,proto3" json:"collection_name,omitempty"`

8
internal/proto/querypb/query_coord.pb.go
View File

@ -251,7 +251,7 @@ func (SyncType) EnumDescriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{6}
}
// --------------------QueryCoord grpc request and response proto------------------
//--------------------QueryCoord grpc request and response proto------------------
type ShowCollectionsRequest struct {
Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"`
// Not useful for now
@ -1336,7 +1336,7 @@ func (m *SyncNewCreatedPartitionRequest) GetPartitionID() int64 {
return 0
}
// -----------------query node grpc request and response proto----------------
//-----------------query node grpc request and response proto----------------
type LoadMetaInfo struct {
LoadType LoadType `protobuf:"varint,1,opt,name=load_type,json=loadType,proto3,enum=milvus.proto.query.LoadType" json:"load_type,omitempty"`
CollectionID int64 `protobuf:"varint,2,opt,name=collectionID,proto3" json:"collectionID,omitempty"`
@ -2488,7 +2488,7 @@ func (m *GetLoadInfoResponse) GetPartitions() []int64 {
return nil
}
// ----------------request auto triggered by QueryCoord-----------------
//----------------request auto triggered by QueryCoord-----------------
type HandoffSegmentsRequest struct {
Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"`
SegmentInfos []*SegmentInfo `protobuf:"bytes,2,rep,name=segmentInfos,proto3" json:"segmentInfos,omitempty"`
@ -3185,7 +3185,7 @@ func (m *UnsubscribeChannelInfo) GetCollectionChannels() []*UnsubscribeChannels
return nil
}
// ---- synchronize messages proto between QueryCoord and QueryNode -----
//---- synchronize messages proto between QueryCoord and QueryNode -----
type SegmentChangeInfo struct {
OnlineNodeID int64 `protobuf:"varint,1,opt,name=online_nodeID,json=onlineNodeID,proto3" json:"online_nodeID,omitempty"`
OnlineSegments []*SegmentInfo `protobuf:"bytes,2,rep,name=online_segments,json=onlineSegments,proto3" json:"online_segments,omitempty"`

40
internal/proto/rootcoordpb/root_coord.pb.go
View File

@ -793,28 +793,28 @@ type RootCoordClient interface {
GetComponentStates(ctx context.Context, in *milvuspb.GetComponentStatesRequest, opts ...grpc.CallOption) (*milvuspb.ComponentStates, error)
GetTimeTickChannel(ctx context.Context, in *internalpb.GetTimeTickChannelRequest, opts ...grpc.CallOption) (*milvuspb.StringResponse, error)
GetStatisticsChannel(ctx context.Context, in *internalpb.GetStatisticsChannelRequest, opts ...grpc.CallOption) (*milvuspb.StringResponse, error)
// *
//*
// @brief This method is used to create collection
//
// @param CreateCollectionRequest, use to provide collection information to be created.
//
// @return Status
CreateCollection(ctx context.Context, in *milvuspb.CreateCollectionRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
// *
//*
// @brief This method is used to delete collection.
//
// @param DropCollectionRequest, collection name is going to be deleted.
//
// @return Status
DropCollection(ctx context.Context, in *milvuspb.DropCollectionRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
// *
//*
// @brief This method is used to test collection existence.
//
// @param HasCollectionRequest, collection name is going to be tested.
//
// @return BoolResponse
HasCollection(ctx context.Context, in *milvuspb.HasCollectionRequest, opts ...grpc.CallOption) (*milvuspb.BoolResponse, error)
// *
//*
// @brief This method is used to get collection schema.
//
// @param DescribeCollectionRequest, target collection name.
@ -825,28 +825,28 @@ type RootCoordClient interface {
CreateAlias(ctx context.Context, in *milvuspb.CreateAliasRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
DropAlias(ctx context.Context, in *milvuspb.DropAliasRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
AlterAlias(ctx context.Context, in *milvuspb.AlterAliasRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
// *
//*
// @brief This method is used to list all collections.
//
// @return StringListResponse, collection name list
ShowCollections(ctx context.Context, in *milvuspb.ShowCollectionsRequest, opts ...grpc.CallOption) (*milvuspb.ShowCollectionsResponse, error)
AlterCollection(ctx context.Context, in *milvuspb.AlterCollectionRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
// *
//*
// @brief This method is used to create partition
//
// @return Status
CreatePartition(ctx context.Context, in *milvuspb.CreatePartitionRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
// *
//*
// @brief This method is used to drop partition
//
// @return Status
DropPartition(ctx context.Context, in *milvuspb.DropPartitionRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
// *
//*
// @brief This method is used to test partition existence.
//
// @return BoolResponse
HasPartition(ctx context.Context, in *milvuspb.HasPartitionRequest, opts ...grpc.CallOption) (*milvuspb.BoolResponse, error)
// *
//*
// @brief This method is used to show partition information
//
// @param ShowPartitionRequest, target collection name.
@ -854,7 +854,7 @@ type RootCoordClient interface {
// @return StringListResponse
ShowPartitions(ctx context.Context, in *milvuspb.ShowPartitionsRequest, opts ...grpc.CallOption) (*milvuspb.ShowPartitionsResponse, error)
ShowPartitionsInternal(ctx context.Context, in *milvuspb.ShowPartitionsRequest, opts ...grpc.CallOption) (*milvuspb.ShowPartitionsResponse, error)
// rpc DescribeSegment(milvus.DescribeSegmentRequest) returns (milvus.DescribeSegmentResponse) {}
// rpc DescribeSegment(milvus.DescribeSegmentRequest) returns (milvus.DescribeSegmentResponse) {}
ShowSegments(ctx context.Context, in *milvuspb.ShowSegmentsRequest, opts ...grpc.CallOption) (*milvuspb.ShowSegmentsResponse, error)
AllocTimestamp(ctx context.Context, in *AllocTimestampRequest, opts ...grpc.CallOption) (*AllocTimestampResponse, error)
AllocID(ctx context.Context, in *AllocIDRequest, opts ...grpc.CallOption) (*AllocIDResponse, error)
@ -1327,28 +1327,28 @@ type RootCoordServer interface {
GetComponentStates(context.Context, *milvuspb.GetComponentStatesRequest) (*milvuspb.ComponentStates, error)
GetTimeTickChannel(context.Context, *internalpb.GetTimeTickChannelRequest) (*milvuspb.StringResponse, error)
GetStatisticsChannel(context.Context, *internalpb.GetStatisticsChannelRequest) (*milvuspb.StringResponse, error)
// *
//*
// @brief This method is used to create collection
//
// @param CreateCollectionRequest, use to provide collection information to be created.
//
// @return Status
CreateCollection(context.Context, *milvuspb.CreateCollectionRequest) (*commonpb.Status, error)
// *
//*
// @brief This method is used to delete collection.
//
// @param DropCollectionRequest, collection name is going to be deleted.
//
// @return Status
DropCollection(context.Context, *milvuspb.DropCollectionRequest) (*commonpb.Status, error)
// *
//*
// @brief This method is used to test collection existence.
//
// @param HasCollectionRequest, collection name is going to be tested.
//
// @return BoolResponse
HasCollection(context.Context, *milvuspb.HasCollectionRequest) (*milvuspb.BoolResponse, error)
// *
//*
// @brief This method is used to get collection schema.
//
// @param DescribeCollectionRequest, target collection name.
@ -1359,28 +1359,28 @@ type RootCoordServer interface {
CreateAlias(context.Context, *milvuspb.CreateAliasRequest) (*commonpb.Status, error)
DropAlias(context.Context, *milvuspb.DropAliasRequest) (*commonpb.Status, error)
AlterAlias(context.Context, *milvuspb.AlterAliasRequest) (*commonpb.Status, error)
// *
//*
// @brief This method is used to list all collections.
//
// @return StringListResponse, collection name list
ShowCollections(context.Context, *milvuspb.ShowCollectionsRequest) (*milvuspb.ShowCollectionsResponse, error)
AlterCollection(context.Context, *milvuspb.AlterCollectionRequest) (*commonpb.Status, error)
// *
//*
// @brief This method is used to create partition
//
// @return Status
CreatePartition(context.Context, *milvuspb.CreatePartitionRequest) (*commonpb.Status, error)
// *
//*
// @brief This method is used to drop partition
//
// @return Status
DropPartition(context.Context, *milvuspb.DropPartitionRequest) (*commonpb.Status, error)
// *
//*
// @brief This method is used to test partition existence.
//
// @return BoolResponse
HasPartition(context.Context, *milvuspb.HasPartitionRequest) (*milvuspb.BoolResponse, error)
// *
//*
// @brief This method is used to show partition information
//
// @param ShowPartitionRequest, target collection name.
@ -1388,7 +1388,7 @@ type RootCoordServer interface {
// @return StringListResponse
ShowPartitions(context.Context, *milvuspb.ShowPartitionsRequest) (*milvuspb.ShowPartitionsResponse, error)
ShowPartitionsInternal(context.Context, *milvuspb.ShowPartitionsRequest) (*milvuspb.ShowPartitionsResponse, error)
// rpc DescribeSegment(milvus.DescribeSegmentRequest) returns (milvus.DescribeSegmentResponse) {}
// rpc DescribeSegment(milvus.DescribeSegmentRequest) returns (milvus.DescribeSegmentResponse) {}
ShowSegments(context.Context, *milvuspb.ShowSegmentsRequest) (*milvuspb.ShowSegmentsResponse, error)
AllocTimestamp(context.Context, *AllocTimestampRequest) (*AllocTimestampResponse, error)
AllocID(context.Context, *AllocIDRequest) (*AllocIDResponse, error)

11
internal/proxy/plan_parser.go
View File

@ -880,14 +880,23 @@ func createQueryPlan(schemaPb *schemapb.CollectionSchema, exprStr string, vector
fieldID := vectorField.FieldID
dataType := vectorField.DataType
var vectorType planpb.VectorType
if !typeutil.IsVectorType(dataType) {
return nil, fmt.Errorf("field (%s) to search is not of vector data type", vectorFieldName)
} else {
if dataType == schemapb.DataType_FloatVector {
vectorType = planpb.VectorType_FloatVector
} else if dataType == schemapb.DataType_BinaryVector {
vectorType = planpb.VectorType_BinaryVector
} else {
vectorType = planpb.VectorType_Float16Vector
}
}
planNode := &planpb.PlanNode{
Node: &planpb.PlanNode_VectorAnns{
VectorAnns: &planpb.VectorANNS{
IsBinary: dataType == schemapb.DataType_BinaryVector,
VectorType: vectorType,
Predicates: expr,
QueryInfo: queryInfo,
PlaceholderTag: "$0",

6
internal/proxy/task.go
View File

@ -265,7 +265,7 @@ func (cct *createCollectionTask) PreExecute(ctx context.Context) error {
return err
}
// validate vector field type parameters
if field.DataType == schemapb.DataType_FloatVector || field.DataType == schemapb.DataType_BinaryVector {
if isVectorType(field.DataType) {
err = validateDimension(field)
if err != nil {
return err
@ -1426,7 +1426,7 @@ func (lct *loadCollectionTask) Execute(ctx context.Context) (err error) {
for _, index := range indexResponse.IndexInfos {
fieldIndexIDs[index.FieldID] = index.IndexID
for _, field := range collSchema.Fields {
if index.FieldID == field.FieldID && (field.DataType == schemapb.DataType_FloatVector || field.DataType == schemapb.DataType_BinaryVector) {
if index.FieldID == field.FieldID && (field.DataType == schemapb.DataType_FloatVector || field.DataType == schemapb.DataType_BinaryVector || field.DataType == schemapb.DataType_Float16Vector) {
hasVecIndex = true
}
}
@ -1654,7 +1654,7 @@ func (lpt *loadPartitionsTask) Execute(ctx context.Context) error {
for _, index := range indexResponse.IndexInfos {
fieldIndexIDs[index.FieldID] = index.IndexID
for _, field := range collSchema.Fields {
if index.FieldID == field.FieldID && (field.DataType == schemapb.DataType_FloatVector || field.DataType == schemapb.DataType_BinaryVector) {
if index.FieldID == field.FieldID && (field.DataType == schemapb.DataType_FloatVector || field.DataType == schemapb.DataType_BinaryVector || field.DataType == schemapb.DataType_Float16Vector) {
hasVecIndex = true
}
}

2
internal/proxy/task_index.go
View File

@ -291,6 +291,7 @@ func fillDimension(field *schemapb.FieldSchema, indexParams map[string]string) e
vecDataTypes := []schemapb.DataType{
schemapb.DataType_FloatVector,
schemapb.DataType_BinaryVector,
schemapb.DataType_Float16Vector,
}
if !funcutil.SliceContain(vecDataTypes, field.GetDataType()) {
return nil
@ -319,6 +320,7 @@ func checkTrain(field *schemapb.FieldSchema, indexParams map[string]string) erro
vecDataTypes := []schemapb.DataType{
schemapb.DataType_FloatVector,
schemapb.DataType_BinaryVector,
schemapb.DataType_Float16Vector,
}
if !funcutil.SliceContain(vecDataTypes, field.GetDataType()) {
return indexparamcheck.CheckIndexValid(field.GetDataType(), indexType, indexParams)

2
internal/proxy/task_query.go
View File

@ -72,7 +72,7 @@ func translateToOutputFieldIDs(outputFields []string, schema *schemapb.Collectio
outputFieldIDs := make([]UniqueID, 0, len(outputFields)+1)
if len(outputFields) == 0 {
for _, field := range schema.Fields {
if field.FieldID >= common.StartOfUserFieldID && field.DataType != schemapb.DataType_FloatVector && field.DataType != schemapb.DataType_BinaryVector {
if field.FieldID >= common.StartOfUserFieldID && field.DataType != schemapb.DataType_FloatVector && field.DataType != schemapb.DataType_BinaryVector && field.DataType != schemapb.DataType_Float16Vector {
outputFieldIDs = append(outputFieldIDs, field.FieldID)
}
}

59
internal/proxy/task_test.go
View File

@ -65,6 +65,7 @@ const (
testVarCharField = "varChar"
testFloatVecField = "fvec"
testBinaryVecField = "bvec"
testFloat16VecField = "f16vec"
testVecDim = 128
testMaxVarCharLength = 100
)
@ -173,7 +174,7 @@ func constructCollectionSchemaByDataType(collectionName string, fieldName2DataTy
Name: fieldName,
DataType: dataType,
}
if dataType == schemapb.DataType_FloatVector || dataType == schemapb.DataType_BinaryVector {
if dataType == schemapb.DataType_FloatVector || dataType == schemapb.DataType_BinaryVector || dataType == schemapb.DataType_Float16Vector {
fieldSchema.TypeParams = []*commonpb.KeyValuePair{
{
Key: common.DimKey,
@ -205,7 +206,7 @@ func constructCollectionSchemaByDataType(collectionName string, fieldName2DataTy
func constructCollectionSchemaWithAllType(
boolField, int32Field, int64Field, floatField, doubleField string,
floatVecField, binaryVecField string,
floatVecField, binaryVecField, float16VecField string,
dim int,
collectionName string,
) *schemapb.CollectionSchema {
@ -290,6 +291,21 @@ func constructCollectionSchemaWithAllType(
IndexParams: nil,
AutoID: false,
}
f16Vec := &schemapb.FieldSchema{
FieldID: 0,
Name: float16VecField,
IsPrimaryKey: false,
Description: "",
DataType: schemapb.DataType_Float16Vector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: common.DimKey,
Value: strconv.Itoa(dim),
},
},
IndexParams: nil,
AutoID: false,
}
if enableMultipleVectorFields {
return &schemapb.CollectionSchema{
@ -304,6 +320,7 @@ func constructCollectionSchemaWithAllType(
d,
fVec,
bVec,
f16Vec,
},
}
}
@ -409,10 +426,11 @@ func constructSearchRequest(
func TestTranslateOutputFields(t *testing.T) {
const (
idFieldName = "id"
tsFieldName = "timestamp"
floatVectorFieldName = "float_vector"
binaryVectorFieldName = "binary_vector"
idFieldName = "id"
tsFieldName = "timestamp"
floatVectorFieldName = "float_vector"
binaryVectorFieldName = "binary_vector"
float16VectorFieldName = "float16_vector"
)
var outputFields []string
var userOutputFields []string
@ -427,6 +445,7 @@ func TestTranslateOutputFields(t *testing.T) {
{Name: tsFieldName, FieldID: 1, DataType: schemapb.DataType_Int64},
{Name: floatVectorFieldName, FieldID: 100, DataType: schemapb.DataType_FloatVector},
{Name: binaryVectorFieldName, FieldID: 101, DataType: schemapb.DataType_BinaryVector},
{Name: float16VectorFieldName, FieldID: 102, DataType: schemapb.DataType_Float16Vector},
},
}
@ -452,23 +471,23 @@ func TestTranslateOutputFields(t *testing.T) {
outputFields, userOutputFields, err = translateOutputFields([]string{"*"}, schema, false)
assert.Equal(t, nil, err)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName}, outputFields)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName}, userOutputFields)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName}, outputFields)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName}, userOutputFields)
outputFields, userOutputFields, err = translateOutputFields([]string{" * "}, schema, false)
assert.Equal(t, nil, err)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName}, outputFields)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName}, userOutputFields)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName}, outputFields)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName}, userOutputFields)
outputFields, userOutputFields, err = translateOutputFields([]string{"*", tsFieldName}, schema, false)
assert.Equal(t, nil, err)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName}, outputFields)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName}, userOutputFields)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName}, outputFields)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName}, userOutputFields)
outputFields, userOutputFields, err = translateOutputFields([]string{"*", floatVectorFieldName}, schema, false)
assert.Equal(t, nil, err)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName}, outputFields)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName}, userOutputFields)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName}, outputFields)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName}, userOutputFields)
//=========================================================================
outputFields, userOutputFields, err = translateOutputFields([]string{}, schema, true)
@ -493,18 +512,18 @@ func TestTranslateOutputFields(t *testing.T) {
outputFields, userOutputFields, err = translateOutputFields([]string{"*"}, schema, true)
assert.Equal(t, nil, err)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName}, outputFields)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName}, userOutputFields)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName}, outputFields)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName}, userOutputFields)
outputFields, userOutputFields, err = translateOutputFields([]string{"*", tsFieldName}, schema, true)
assert.Equal(t, nil, err)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName}, outputFields)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName}, userOutputFields)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName}, outputFields)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName}, userOutputFields)
outputFields, userOutputFields, err = translateOutputFields([]string{"*", floatVectorFieldName}, schema, true)
assert.Equal(t, nil, err)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName}, outputFields)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName}, userOutputFields)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName}, outputFields)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName}, userOutputFields)
outputFields, userOutputFields, err = translateOutputFields([]string{"A"}, schema, true)
assert.Error(t, err)

14
internal/proxy/util.go
View File

@ -84,6 +84,12 @@ func isNumber(c uint8) bool {
return true
}
func isVectorType(dataType schemapb.DataType) bool {
return dataType == schemapb.DataType_FloatVector ||
dataType == schemapb.DataType_BinaryVector ||
dataType == schemapb.DataType_Float16Vector
}
func validateMaxQueryResultWindow(offset int64, limit int64) error {
if offset < 0 {
return fmt.Errorf("%s [%d] is invalid, should be gte than 0", OffsetKey, offset)
@ -319,7 +325,7 @@ func validateMaxLengthPerRow(collectionName string, field *schemapb.FieldSchema)
}
func validateVectorFieldMetricType(field *schemapb.FieldSchema) error {
if (field.DataType != schemapb.DataType_FloatVector) && (field.DataType != schemapb.DataType_BinaryVector) {
if !isVectorType(field.DataType) {
return nil
}
for _, params := range field.IndexParams {
@ -432,7 +438,7 @@ func isVector(dataType schemapb.DataType) (bool, error) {
schemapb.DataType_Float, schemapb.DataType_Double:
return false, nil
case schemapb.DataType_FloatVector, schemapb.DataType_BinaryVector:
case schemapb.DataType_FloatVector, schemapb.DataType_BinaryVector, schemapb.DataType_Float16Vector:
return true, nil
}
@ -443,7 +449,7 @@ func validateMetricType(dataType schemapb.DataType, metricTypeStrRaw string) err
metricTypeStr := strings.ToUpper(metricTypeStrRaw)
switch metricTypeStr {
case metric.L2, metric.IP, metric.COSINE:
if dataType == schemapb.DataType_FloatVector {
if dataType == schemapb.DataType_FloatVector || dataType == schemapb.DataType_Float16Vector {
return nil
}
case metric.JACCARD, metric.HAMMING, metric.SUBSTRUCTURE, metric.SUPERSTRUCTURE:
@ -548,7 +554,7 @@ func validateMultipleVectorFields(schema *schemapb.CollectionSchema) error {
for i := range schema.Fields {
name := schema.Fields[i].Name
dType := schema.Fields[i].DataType
isVec := dType == schemapb.DataType_BinaryVector || dType == schemapb.DataType_FloatVector
isVec := dType == schemapb.DataType_BinaryVector || dType == schemapb.DataType_FloatVector || dType == schemapb.DataType_Float16Vector
if isVec && vecExist && !enableMultipleVectorFields {
return fmt.Errorf(
"multiple vector fields is not supported, fields name: %s, %s",

29
internal/proxy/validate_util.go
View File

@ -63,6 +63,10 @@ func (v *validateUtil) Validate(data []*schemapb.FieldData, schema *schemapb.Col
if err := v.checkFloatVectorFieldData(field, fieldSchema); err != nil {
return err
}
case schemapb.DataType_Float16Vector:
if err := v.checkFloat16VectorFieldData(field, fieldSchema); err != nil {
return err
}
case schemapb.DataType_BinaryVector:
if err := v.checkBinaryVectorFieldData(field, fieldSchema); err != nil {
return err
@ -143,6 +147,26 @@ func (v *validateUtil) checkAligned(data []*schemapb.FieldData, schema *typeutil
return errNumRowsMismatch(field.GetFieldName(), n, numRows)
}
case schemapb.DataType_Float16Vector:
f, err := schema.GetFieldFromName(field.GetFieldName())
if err != nil {
return err
}
dim, err := typeutil.GetDim(f)
if err != nil {
return err
}
n, err := funcutil.GetNumRowsOfFloat16VectorField(field.GetVectors().GetFloat16Vector(), dim)
if err != nil {
return err
}
if n != numRows {
return errNumRowsMismatch(field.GetFieldName(), n, numRows)
}
default:
// error won't happen here.
n, err := funcutil.GetNumRowOfFieldData(field)
@ -249,6 +273,11 @@ func (v *validateUtil) checkFloatVectorFieldData(field *schemapb.FieldData, fiel
return nil
}
func (v *validateUtil) checkFloat16VectorFieldData(field *schemapb.FieldData, fieldSchema *schemapb.FieldSchema) error {
// TODO
return nil
}
func (v *validateUtil) checkBinaryVectorFieldData(field *schemapb.FieldData, fieldSchema *schemapb.FieldSchema) error {
// TODO
return nil

125
internal/proxy/validate_util_test.go
View File

@ -523,6 +523,131 @@ func Test_validateUtil_checkAligned(t *testing.T) {
assert.Error(t, err)
})
//////////////////////////////////////////////////////////////////////
t.Run("float16 vector column not found", func(t *testing.T) {
data := []*schemapb.FieldData{
{
FieldName: "test",
Type: schemapb.DataType_Float16Vector,
},
}
schema := &schemapb.CollectionSchema{}
h, err := typeutil.CreateSchemaHelper(schema)
assert.NoError(t, err)
v := newValidateUtil()
err = v.checkAligned(data, h, 100)
assert.Error(t, err)
})
t.Run("float16 vector column dimension not found", func(t *testing.T) {
data := []*schemapb.FieldData{
{
FieldName: "test",
Type: schemapb.DataType_Float16Vector,
},
}
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{
Name: "test",
DataType: schemapb.DataType_Float16Vector,
},
},
}
h, err := typeutil.CreateSchemaHelper(schema)
assert.NoError(t, err)
v := newValidateUtil()
err = v.checkAligned(data, h, 100)
assert.Error(t, err)
})
t.Run("invalid num rows", func(t *testing.T) {
data := []*schemapb.FieldData{
{
FieldName: "test",
Type: schemapb.DataType_Float16Vector,
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Data: &schemapb.VectorField_Float16Vector{
Float16Vector: []byte("not128"),
},
},
},
},
}
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{
Name: "test",
DataType: schemapb.DataType_Float16Vector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: common.DimKey,
Value: "128",
},
},
},
},
}
h, err := typeutil.CreateSchemaHelper(schema)
assert.NoError(t, err)
v := newValidateUtil()
err = v.checkAligned(data, h, 100)
assert.Error(t, err)
})
t.Run("num rows mismatch", func(t *testing.T) {
data := []*schemapb.FieldData{
{
FieldName: "test",
Type: schemapb.DataType_Float16Vector,
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Data: &schemapb.VectorField_Float16Vector{
Float16Vector: []byte{'1', '2'},
},
},
},
},
}
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{
Name: "test",
DataType: schemapb.DataType_Float16Vector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: common.DimKey,
Value: "8",
},
},
},
},
}
h, err := typeutil.CreateSchemaHelper(schema)
assert.NoError(t, err)
v := newValidateUtil()
err = v.checkAligned(data, h, 100)
assert.Error(t, err)
})
//////////////////////////////////////////////////////////////////
t.Run("mismatch", func(t *testing.T) {

53
internal/querynodev2/segments/mock_data.go
View File

@ -114,6 +114,14 @@ var simpleBinVecField = vecFieldParam{
fieldName: "binVectorField",
}
var simpleFloat16VecField = vecFieldParam{
id: 112,
dim: defaultDim,
metricType: defaultMetricType,
vecType: schemapb.DataType_Float16Vector,
fieldName: "float16VectorField",
}
var simpleBoolField = constFieldParam{
id: 102,
dataType: schemapb.DataType_Bool,
@ -434,6 +442,16 @@ func generateBinaryVectors(numRows, dim int) []byte {
return ret
}
func generateFloat16Vectors(numRows, dim int) []byte {
total := numRows * dim * 2
ret := make([]byte, total)
_, err := rand.Read(ret)
if err != nil {
panic(err)
}
return ret
}
func GenTestScalarFieldData(dType schemapb.DataType, fieldName string, fieldID int64, numRows int) *schemapb.FieldData {
ret := &schemapb.FieldData{
Type: dType,
@ -589,6 +607,16 @@ func GenTestVectorFiledData(dType schemapb.DataType, fieldName string, fieldID i
},
},
}
case schemapb.DataType_Float16Vector:
ret.FieldId = fieldID
ret.Field = &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: int64(dim),
Data: &schemapb.VectorField_Float16Vector{
Float16Vector: generateFloat16Vectors(numRows, dim),
},
},
}
default:
panic("data type not supported")
}
@ -778,6 +806,12 @@ func genInsertData(msgLength int, schema *schemapb.CollectionSchema) (*storage.I
Data: generateFloatVectors(msgLength, dim),
Dim: dim,
}
case schemapb.DataType_Float16Vector:
dim := simpleFloat16VecField.dim
insertData.Data[f.FieldID] = &storage.Float16VectorFieldData{
Data: generateFloat16Vectors(msgLength, dim),
Dim: dim,
}
case schemapb.DataType_BinaryVector:
dim := simpleBinVecField.dim
insertData.Data[f.FieldID] = &storage.BinaryVectorFieldData{
@ -1042,7 +1076,7 @@ func genBruteForceDSL(schema *schemapb.CollectionSchema, topK int64, roundDecima
roundDecimalStr := strconv.FormatInt(roundDecimal, 10)
var fieldID int64
for _, f := range schema.Fields {
if f.DataType == schemapb.DataType_FloatVector {
if f.DataType == schemapb.DataType_FloatVector || f.DataType == schemapb.DataType_Float16Vector {
vecFieldName = f.Name
fieldID = f.FieldID
for _, p := range f.IndexParams {
@ -1195,6 +1229,9 @@ func genInsertMsg(collection *Collection, partitionID, segment int64, numRows in
case schemapb.DataType_BinaryVector:
dim := simpleBinVecField.dim // if no dim specified, use simpleFloatVecField's dim
fieldsData = append(fieldsData, GenTestVectorFiledData(f.DataType, f.Name, f.FieldID, numRows, dim))
case schemapb.DataType_Float16Vector:
dim := simpleFloat16VecField.dim // if no dim specified, use simpleFloatVecField's dim
fieldsData = append(fieldsData, GenTestVectorFiledData(f.DataType, f.Name, f.FieldID, numRows, dim))
default:
err := errors.New("data type not supported")
return nil, err
@ -1429,6 +1466,20 @@ func genFieldData(fieldName string, fieldID int64, fieldType schemapb.DataType,
},
FieldId: fieldID,
}
case schemapb.DataType_Float16Vector:
fieldData = &schemapb.FieldData{
Type: schemapb.DataType_Float16Vector,
FieldName: fieldName,
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: dim,
Data: &schemapb.VectorField_Float16Vector{
Float16Vector: fieldValue.([]byte),
},
},
},
FieldId: fieldID,
}
case schemapb.DataType_JSON:
fieldData = &schemapb.FieldData{
Type: schemapb.DataType_JSON,

73
internal/storage/data_codec.go
View File

@ -148,20 +148,25 @@ type FloatVectorFieldData struct {
Data []float32
Dim int
}
type Float16VectorFieldData struct {
Data []byte
Dim int
}
// RowNum implements FieldData.RowNum
func (data *BoolFieldData) RowNum() int { return len(data.Data) }
func (data *Int8FieldData) RowNum() int { return len(data.Data) }
func (data *Int16FieldData) RowNum() int { return len(data.Data) }
func (data *Int32FieldData) RowNum() int { return len(data.Data) }
func (data *Int64FieldData) RowNum() int { return len(data.Data) }
func (data *FloatFieldData) RowNum() int { return len(data.Data) }
func (data *DoubleFieldData) RowNum() int { return len(data.Data) }
func (data *StringFieldData) RowNum() int { return len(data.Data) }
func (data *BinaryVectorFieldData) RowNum() int { return len(data.Data) * 8 / data.Dim }
func (data *FloatVectorFieldData) RowNum() int { return len(data.Data) / data.Dim }
func (data *ArrayFieldData) RowNum() int { return len(data.Data) }
func (data *JSONFieldData) RowNum() int { return len(data.Data) }
func (data *BoolFieldData) RowNum() int { return len(data.Data) }
func (data *Int8FieldData) RowNum() int { return len(data.Data) }
func (data *Int16FieldData) RowNum() int { return len(data.Data) }
func (data *Int32FieldData) RowNum() int { return len(data.Data) }
func (data *Int64FieldData) RowNum() int { return len(data.Data) }
func (data *FloatFieldData) RowNum() int { return len(data.Data) }
func (data *DoubleFieldData) RowNum() int { return len(data.Data) }
func (data *StringFieldData) RowNum() int { return len(data.Data) }
func (data *BinaryVectorFieldData) RowNum() int { return len(data.Data) * 8 / data.Dim }
func (data *FloatVectorFieldData) RowNum() int { return len(data.Data) / data.Dim }
func (data *Float16VectorFieldData) RowNum() int { return len(data.Data) / 2 / data.Dim }
func (data *ArrayFieldData) RowNum() int { return len(data.Data) }
func (data *JSONFieldData) RowNum() int { return len(data.Data) }
// GetRow implements FieldData.GetRow
func (data *BoolFieldData) GetRow(i int) any { return data.Data[i] }
@ -180,6 +185,9 @@ func (data *BinaryVectorFieldData) GetRow(i int) any {
func (data *FloatVectorFieldData) GetRow(i int) any {
return data.Data[i*data.Dim : (i+1)*data.Dim]
}
func (data *Float16VectorFieldData) GetRow(i int) any {
return data.Data[i*data.Dim*2 : (i+1)*data.Dim*2]
}
// why not binary.Size(data) directly? binary.Size(data) return -1
// binary.Size returns how many bytes Write would generate to encode the value v, which
@ -266,6 +274,10 @@ func (data *FloatVectorFieldData) GetMemorySize() int {
return binary.Size(data.Data) + 4
}
func (data *Float16VectorFieldData) GetMemorySize() int {
return binary.Size(data.Data) + 4
}
// system field id:
// 0: unique row id
// 1: timestamp
@ -438,6 +450,8 @@ func (insertCodec *InsertCodec) Serialize(partitionID UniqueID, segmentID Unique
eventWriter, err = writer.NextInsertEventWriter(singleData.(*FloatVectorFieldData).Dim)
case schemapb.DataType_BinaryVector:
eventWriter, err = writer.NextInsertEventWriter(singleData.(*BinaryVectorFieldData).Dim)
case schemapb.DataType_Float16Vector:
eventWriter, err = writer.NextInsertEventWriter(singleData.(*Float16VectorFieldData).Dim)
default:
return nil, fmt.Errorf("undefined data type %d", field.DataType)
}
@ -553,6 +567,14 @@ func (insertCodec *InsertCodec) Serialize(partitionID UniqueID, segmentID Unique
return nil, err
}
writer.AddExtra(originalSizeKey, fmt.Sprintf("%v", singleData.(*FloatVectorFieldData).GetMemorySize()))
case schemapb.DataType_Float16Vector:
err = eventWriter.AddFloat16VectorToPayload(singleData.(*Float16VectorFieldData).Data, singleData.(*Float16VectorFieldData).Dim)
if err != nil {
eventWriter.Close()
writer.Close()
return nil, err
}
writer.AddExtra(originalSizeKey, fmt.Sprintf("%v", singleData.(*Float16VectorFieldData).GetMemorySize()))
default:
return nil, fmt.Errorf("undefined data type %d", field.DataType)
}
@ -857,6 +879,33 @@ func (insertCodec *InsertCodec) DeserializeInto(fieldBinlogs []*Blob, rowNum int
binaryVectorFieldData.Dim = dim
insertData.Data[fieldID] = binaryVectorFieldData
case schemapb.DataType_Float16Vector:
var singleData []byte
singleData, dim, err = eventReader.GetFloat16VectorFromPayload()
if err != nil {
eventReader.Close()
binlogReader.Close()
return InvalidUniqueID, InvalidUniqueID, InvalidUniqueID, err
}
if insertData.Data[fieldID] == nil {
insertData.Data[fieldID] = &Float16VectorFieldData{
Data: make([]byte, 0, rowNum*dim),
}
}
float16VectorFieldData := insertData.Data[fieldID].(*Float16VectorFieldData)
float16VectorFieldData.Data = append(float16VectorFieldData.Data, singleData...)
length, err := eventReader.GetPayloadLengthFromReader()
if err != nil {
eventReader.Close()
binlogReader.Close()
return InvalidUniqueID, InvalidUniqueID, InvalidUniqueID, err
}
totalLength += length
float16VectorFieldData.Dim = dim
insertData.Data[fieldID] = float16VectorFieldData
case schemapb.DataType_FloatVector:
var singleData []float32
singleData, dim, err = eventReader.GetFloatVectorFromPayload()

80
internal/storage/data_codec_test.go
View File

@ -30,23 +30,24 @@ import (
)
const (
CollectionID = 1
PartitionID = 1
SegmentID = 1
RowIDField = 0
TimestampField = 1
BoolField = 100
Int8Field = 101
Int16Field = 102
Int32Field = 103
Int64Field = 104
FloatField = 105
DoubleField = 106
StringField = 107
BinaryVectorField = 108
FloatVectorField = 109
ArrayField = 110
JSONField = 111
CollectionID = 1
PartitionID = 1
SegmentID = 1
RowIDField = 0
TimestampField = 1
BoolField = 100
Int8Field = 101
Int16Field = 102
Int32Field = 103
Int64Field = 104
FloatField = 105
DoubleField = 106
StringField = 107
BinaryVectorField = 108
FloatVectorField = 109
ArrayField = 110
JSONField = 111
Float16VectorField = 112
)
func TestInsertCodec(t *testing.T) {
@ -157,6 +158,13 @@ func TestInsertCodec(t *testing.T) {
Description: "float_vector",
DataType: schemapb.DataType_FloatVector,
},
{
FieldID: Float16VectorField,
Name: "field_float16_vector",
IsPrimaryKey: false,
Description: "float16_vector",
DataType: schemapb.DataType_Float16Vector,
},
},
},
}
@ -222,6 +230,10 @@ func TestInsertCodec(t *testing.T) {
[]byte(`{"key":"world"}`),
},
},
Float16VectorField: &Float16VectorFieldData{
Data: []byte{0, 255, 0, 255, 0, 255, 0, 255},
Dim: 4,
},
},
}
@ -265,6 +277,10 @@ func TestInsertCodec(t *testing.T) {
Data: []float32{0, 1, 2, 3, 0, 1, 2, 3},
Dim: 4,
},
Float16VectorField: &Float16VectorFieldData{
Data: []byte{0, 255, 0, 255, 0, 255, 0, 255},
Dim: 4,
},
ArrayField: &ArrayFieldData{
ElementType: schemapb.DataType_Int32,
Data: []*schemapb.ScalarField{
@ -291,20 +307,21 @@ func TestInsertCodec(t *testing.T) {
insertDataEmpty := &InsertData{
Data: map[int64]FieldData{
RowIDField: &Int64FieldData{[]int64{}},
TimestampField: &Int64FieldData{[]int64{}},
BoolField: &BoolFieldData{[]bool{}},
Int8Field: &Int8FieldData{[]int8{}},
Int16Field: &Int16FieldData{[]int16{}},
Int32Field: &Int32FieldData{[]int32{}},
Int64Field: &Int64FieldData{[]int64{}},
FloatField: &FloatFieldData{[]float32{}},
DoubleField: &DoubleFieldData{[]float64{}},
StringField: &StringFieldData{[]string{}},
BinaryVectorField: &BinaryVectorFieldData{[]byte{}, 8},
FloatVectorField: &FloatVectorFieldData{[]float32{}, 4},
ArrayField: &ArrayFieldData{schemapb.DataType_Int32, []*schemapb.ScalarField{}},
JSONField: &JSONFieldData{[][]byte{}},
RowIDField: &Int64FieldData{[]int64{}},
TimestampField: &Int64FieldData{[]int64{}},
BoolField: &BoolFieldData{[]bool{}},
Int8Field: &Int8FieldData{[]int8{}},
Int16Field: &Int16FieldData{[]int16{}},
Int32Field: &Int32FieldData{[]int32{}},
Int64Field: &Int64FieldData{[]int64{}},
FloatField: &FloatFieldData{[]float32{}},
DoubleField: &DoubleFieldData{[]float64{}},
StringField: &StringFieldData{[]string{}},
BinaryVectorField: &BinaryVectorFieldData{[]byte{}, 8},
FloatVectorField: &FloatVectorFieldData{[]float32{}, 4},
Float16VectorField: &Float16VectorFieldData{[]byte{}, 4},
ArrayField: &ArrayFieldData{schemapb.DataType_Int32, []*schemapb.ScalarField{}},
JSONField: &JSONFieldData{[][]byte{}},
},
}
b, err := insertCodec.Serialize(PartitionID, SegmentID, insertDataEmpty)
@ -345,6 +362,7 @@ func TestInsertCodec(t *testing.T) {
assert.Equal(t, []string{"1", "2", "3", "4"}, resultData.Data[StringField].(*StringFieldData).Data)
assert.Equal(t, []byte{0, 255, 0, 255}, resultData.Data[BinaryVectorField].(*BinaryVectorFieldData).Data)
assert.Equal(t, []float32{0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 4, 5, 6, 7}, resultData.Data[FloatVectorField].(*FloatVectorFieldData).Data)
assert.Equal(t, []byte{0, 255, 0, 255, 0, 255, 0, 255, 0, 255, 0, 255, 0, 255, 0, 255}, resultData.Data[Float16VectorField].(*Float16VectorFieldData).Data)
int32ArrayList := [][]int32{{1, 2, 3}, {4, 5, 6}, {3, 2, 1}, {6, 5, 4}}
resultArrayList := [][]int32{}

7
internal/storage/data_sorter.go
View File

@ -94,6 +94,13 @@ func (ds *DataSorter) Swap(i, j int) {
for idx := 0; idx < dim; idx++ {
data[i*dim+idx], data[j*dim+idx] = data[j*dim+idx], data[i*dim+idx]
}
case schemapb.DataType_Float16Vector:
data := singleData.(*Float16VectorFieldData).Data
dim := singleData.(*Float16VectorFieldData).Dim
steps := dim * 2
for idx := 0; idx < steps; idx++ {
data[i*steps+idx], data[j*steps+idx] = data[j*steps+idx], data[i*steps+idx]
}
case schemapb.DataType_Array:
data := singleData.(*ArrayFieldData).Data
data[i], data[j] = data[j], data[i]

12
internal/storage/data_sorter_test.go
View File

@ -120,6 +120,13 @@ func TestDataSorter(t *testing.T) {
Description: "description_11",
DataType: schemapb.DataType_FloatVector,
},
{
FieldID: 110,
Name: "field_float16_vector",
IsPrimaryKey: false,
Description: "description_12",
DataType: schemapb.DataType_Float16Vector,
},
},
},
}
@ -165,6 +172,10 @@ func TestDataSorter(t *testing.T) {
Data: []float32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23},
Dim: 8,
},
110: &Float16VectorFieldData{
Data: []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23},
Dim: 4,
},
},
}
@ -226,6 +237,7 @@ func TestDataSorter(t *testing.T) {
assert.Equal(t, []string{"5", "3", "4"}, dataSorter.InsertData.Data[107].(*StringFieldData).Data)
assert.Equal(t, []byte{128, 0, 255}, dataSorter.InsertData.Data[108].(*BinaryVectorFieldData).Data)
assert.Equal(t, []float32{16, 17, 18, 19, 20, 21, 22, 23, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, dataSorter.InsertData.Data[109].(*FloatVectorFieldData).Data)
assert.Equal(t, []byte{16, 17, 18, 19, 20, 21, 22, 23, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, dataSorter.InsertData.Data[110].(*Float16VectorFieldData).Data)
}
func TestDataSorter_Len(t *testing.T) {

2
internal/storage/payload.go
View File

@ -36,6 +36,7 @@ type PayloadWriterInterface interface {
AddOneJSONToPayload(msg []byte) error
AddBinaryVectorToPayload(binVec []byte, dim int) error
AddFloatVectorToPayload(binVec []float32, dim int) error
AddFloat16VectorToPayload(binVec []byte, dim int) error
FinishPayloadWriter() error
GetPayloadBufferFromWriter() ([]byte, error)
GetPayloadLengthFromWriter() (int, error)
@ -58,6 +59,7 @@ type PayloadReaderInterface interface {
GetArrayFromPayload() ([]*schemapb.ScalarField, error)
GetJSONFromPayload() ([][]byte, error)
GetBinaryVectorFromPayload() ([]byte, int, error)
GetFloat16VectorFromPayload() ([]byte, int, error)
GetFloatVectorFromPayload() ([]float32, int, error)
GetPayloadLengthFromReader() (int, error)
ReleasePayloadReader() error

25
internal/storage/payload_reader.go
View File

@ -67,6 +67,8 @@ func (r *PayloadReader) GetDataFromPayload() (interface{}, int, error) {
return r.GetBinaryVectorFromPayload()
case schemapb.DataType_FloatVector:
return r.GetFloatVectorFromPayload()
case schemapb.DataType_Float16Vector:
return r.GetFloat16VectorFromPayload()
case schemapb.DataType_String, schemapb.DataType_VarChar:
val, err := r.GetStringFromPayload()
return val, 0, err
@ -315,6 +317,29 @@ func (r *PayloadReader) GetBinaryVectorFromPayload() ([]byte, int, error) {
return ret, dim * 8, nil
}
// GetFloat16VectorFromPayload returns vector, dimension, error
func (r *PayloadReader) GetFloat16VectorFromPayload() ([]byte, int, error) {
if r.colType != schemapb.DataType_Float16Vector {
return nil, -1, fmt.Errorf("failed to get float vector from datatype %v", r.colType.String())
}
dim := r.reader.RowGroup(0).Column(0).Descriptor().TypeLength() / 2
values := make([]parquet.FixedLenByteArray, r.numRows)
valuesRead, err := ReadDataFromAllRowGroups[parquet.FixedLenByteArray, *file.FixedLenByteArrayColumnChunkReader](r.reader, values, 0, r.numRows)
if err != nil {
return nil, -1, err
}
if valuesRead != r.numRows {
return nil, -1, fmt.Errorf("expect %d rows, but got valuesRead = %d", r.numRows, valuesRead)
}
ret := make([]byte, int64(dim*2)*r.numRows)
for i := 0; i < int(r.numRows); i++ {
copy(ret[i*dim*2:(i+1)*dim*2], values[i])
}
return ret, dim, nil
}
// GetFloatVectorFromPayload returns vector, dimension, error
func (r *PayloadReader) GetFloatVectorFromPayload() ([]float32, int, error) {
if r.colType != schemapb.DataType_FloatVector {

33
internal/storage/payload_test.go
View File

@ -536,6 +536,39 @@ func TestPayload_ReaderAndWriter(t *testing.T) {
defer r.ReleasePayloadReader()
})
t.Run("TestFloat16Vector", func(t *testing.T) {
w, err := NewPayloadWriter(schemapb.DataType_Float16Vector, 1)
require.Nil(t, err)
require.NotNil(t, w)
err = w.AddFloat16VectorToPayload([]byte{1, 2}, 1)
assert.NoError(t, err)
err = w.AddDataToPayload([]byte{3, 4}, 1)
assert.NoError(t, err)
err = w.FinishPayloadWriter()
assert.NoError(t, err)
length, err := w.GetPayloadLengthFromWriter()
assert.NoError(t, err)
assert.Equal(t, 2, length)
defer w.ReleasePayloadWriter()
buffer, err := w.GetPayloadBufferFromWriter()
assert.NoError(t, err)
r, err := NewPayloadReader(schemapb.DataType_Float16Vector, buffer)
require.Nil(t, err)
length, err = r.GetPayloadLengthFromReader()
assert.NoError(t, err)
assert.Equal(t, length, 2)
float16Vecs, dim, err := r.GetFloat16VectorFromPayload()
assert.NoError(t, err)
assert.Equal(t, 1, dim)
assert.Equal(t, 4, len(float16Vecs))
assert.ElementsMatch(t, []byte{1, 2, 3, 4}, float16Vecs)
})
// t.Run("TestAddDataToPayload", func(t *testing.T) {
// w, err := NewPayloadWriter(schemapb.DataType_Bool)
// w.colType = 999

35
internal/storage/payload_writer.go
View File

@ -151,6 +151,12 @@ func (w *NativePayloadWriter) AddDataToPayload(data interface{}, dim ...int) err
return errors.New("incorrect data type")
}
return w.AddFloatVectorToPayload(val, dim[0])
case schemapb.DataType_Float16Vector:
val, ok := data.([]byte)
if !ok {
return errors.New("incorrect data type")
}
return w.AddFloat16VectorToPayload(val, dim[0])
default:
return errors.New("incorrect datatype")
}
@ -412,6 +418,31 @@ func (w *NativePayloadWriter) AddFloatVectorToPayload(data []float32, dim int) e
return nil
}
func (w *NativePayloadWriter) AddFloat16VectorToPayload(data []byte, dim int) error {
if w.finished {
return errors.New("can't append data to finished writer")
}
if len(data) == 0 {
return errors.New("can't add empty msgs into payload")
}
builder, ok := w.builder.(*array.FixedSizeBinaryBuilder)
if !ok {
return errors.New("failed to cast ArrayBuilder")
}
byteLength := dim * 2
length := len(data) / byteLength
builder.Reserve(length)
for i := 0; i < length; i++ {
builder.Append(data[i*byteLength : (i+1)*byteLength])
}
return nil
}
func (w *NativePayloadWriter) FinishPayloadWriter() error {
if w.finished {
return errors.New("can't reuse a finished writer")
@ -503,6 +534,10 @@ func milvusDataTypeToArrowType(dataType schemapb.DataType, dim int) arrow.DataTy
return &arrow.FixedSizeBinaryType{
ByteWidth: dim / 8,
}
case schemapb.DataType_Float16Vector:
return &arrow.FixedSizeBinaryType{
ByteWidth: dim * 2,
}
default:
panic("unsupported data type")
}

54
internal/storage/utils.go
View File

@ -224,6 +224,16 @@ func readBinaryVectors(blobReaders []io.Reader, dim int) []byte {
return ret
}
func readFloat16Vectors(blobReaders []io.Reader, dim int) []byte {
ret := make([]byte, 0)
for _, r := range blobReaders {
var v = make([]byte, dim*2)
ReadBinary(r, &v, schemapb.DataType_Float16Vector)
ret = append(ret, v...)
}
return ret
}
func readBoolArray(blobReaders []io.Reader) []bool {
ret := make([]bool, 0)
for _, r := range blobReaders {
@ -321,6 +331,19 @@ func RowBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *schemap
Dim: dim,
}
case schemapb.DataType_Float16Vector:
dim, err := GetDimFromParams(field.TypeParams)
if err != nil {
log.Error("failed to get dim", zap.Error(err))
return nil, err
}
vecs := readFloat16Vectors(blobReaders, dim)
idata.Data[field.FieldID] = &Float16VectorFieldData{
Data: vecs,
Dim: dim,
}
case schemapb.DataType_BinaryVector:
var dim int
dim, err := GetDimFromParams(field.TypeParams)
@ -435,6 +458,23 @@ func ColumnBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *sche
idata.Data[field.FieldID] = fieldData
case schemapb.DataType_Float16Vector:
dim, err := GetDimFromParams(field.TypeParams)
if err != nil {
log.Error("failed to get dim", zap.Error(err))
return nil, err
}
srcData := srcFields[field.FieldID].GetVectors().GetFloat16Vector()
fieldData := &Float16VectorFieldData{
Data: make([]byte, 0, len(srcData)),
Dim: dim,
}
fieldData.Data = append(fieldData.Data, srcData...)
idata.Data[field.FieldID] = fieldData
case schemapb.DataType_Bool:
srcData := srcFields[field.FieldID].GetScalars().GetBoolData().GetData()
@ -698,6 +738,18 @@ func mergeFloatVectorField(data *InsertData, fid FieldID, field *FloatVectorFiel
fieldData.Data = append(fieldData.Data, field.Data...)
}
func mergeFloat16VectorField(data *InsertData, fid FieldID, field *Float16VectorFieldData) {
if _, ok := data.Data[fid]; !ok {
fieldData := &Float16VectorFieldData{
Data: nil,
Dim: field.Dim,
}
data.Data[fid] = fieldData
}
fieldData := data.Data[fid].(*Float16VectorFieldData)
fieldData.Data = append(fieldData.Data, field.Data...)
}
// MergeFieldData merge field into data.
func MergeFieldData(data *InsertData, fid FieldID, field FieldData) {
if field == nil {
@ -728,6 +780,8 @@ func MergeFieldData(data *InsertData, fid FieldID, field FieldData) {
mergeBinaryVectorField(data, fid, field)
case *FloatVectorFieldData:
mergeFloatVectorField(data, fid, field)
case *Float16VectorFieldData:
mergeFloat16VectorField(data, fid, field)
}
}

87
internal/storage/utils_test.go
View File

@ -169,6 +169,10 @@ func TestTransferColumnBasedInsertDataToRowBased(t *testing.T) {
Dim: 1,
Data: []float32{0, 0, 0},
}
f11 := &Float16VectorFieldData{
Dim: 1,
Data: []byte{1, 1, 2, 2, 3, 3},
}
data.Data[101] = f1
data.Data[102] = f2
@ -180,6 +184,7 @@ func TestTransferColumnBasedInsertDataToRowBased(t *testing.T) {
// data.Data[108] = f8
data.Data[109] = f9
data.Data[110] = f10
data.Data[111] = f11
utss, rowIds, rows, err := TransferColumnBasedInsertDataToRowBased(data)
assert.NoError(t, err)
@ -202,6 +207,7 @@ func TestTransferColumnBasedInsertDataToRowBased(t *testing.T) {
// b + 1, // "1"
1, // 1
0, 0, 0, 0, // 0
1, 1,
},
rows[0].Value)
assert.ElementsMatch(t,
@ -216,6 +222,7 @@ func TestTransferColumnBasedInsertDataToRowBased(t *testing.T) {
// b + 2, // "2"
2, // 2
0, 0, 0, 0, // 0
2, 2,
},
rows[1].Value)
assert.ElementsMatch(t,
@ -230,6 +237,7 @@ func TestTransferColumnBasedInsertDataToRowBased(t *testing.T) {
// b + 3, // "3"
3, // 3
0, 0, 0, 0, // 0
3, 3,
},
rows[2].Value)
}
@ -313,7 +321,7 @@ func TestReadBinary(t *testing.T) {
}
}
func genAllFieldsSchema(fVecDim, bVecDim int) (schema *schemapb.CollectionSchema, pkFieldID UniqueID, fieldIDs []UniqueID) {
func genAllFieldsSchema(fVecDim, bVecDim, f16VecDim int) (schema *schemapb.CollectionSchema, pkFieldID UniqueID, fieldIDs []UniqueID) {
schema = &schemapb.CollectionSchema{
Name: "all_fields_schema",
Description: "all_fields_schema",
@ -359,6 +367,15 @@ func genAllFieldsSchema(fVecDim, bVecDim int) (schema *schemapb.CollectionSchema
},
},
},
{
DataType: schemapb.DataType_Float16Vector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: common.DimKey,
Value: strconv.Itoa(f16VecDim),
},
},
},
{
DataType: schemapb.DataType_Array,
},
@ -412,6 +429,16 @@ func generateBinaryVectors(numRows, dim int) []byte {
return ret
}
func generateFloat16Vectors(numRows, dim int) []byte {
total := (numRows * dim) * 2
ret := make([]byte, total)
_, err := rand.Read(ret)
if err != nil {
panic(err)
}
return ret
}
func generateBoolArray(numRows int) []bool {
ret := make([]bool, 0, numRows)
for i := 0; i < numRows; i++ {
@ -474,8 +501,8 @@ func generateInt32ArrayList(numRows int) []*schemapb.ScalarField {
return ret
}
func genRowWithAllFields(fVecDim, bVecDim int) (blob *commonpb.Blob, pk int64, row []interface{}) {
schema, _, _ := genAllFieldsSchema(fVecDim, bVecDim)
func genRowWithAllFields(fVecDim, bVecDim, f16VecDim int) (blob *commonpb.Blob, pk int64, row []interface{}) {
schema, _, _ := genAllFieldsSchema(fVecDim, bVecDim, f16VecDim)
ret := &commonpb.Blob{
Value: nil,
}
@ -493,6 +520,11 @@ func genRowWithAllFields(fVecDim, bVecDim int) (blob *commonpb.Blob, pk int64, r
_ = binary.Write(&buffer, common.Endian, bVec)
ret.Value = append(ret.Value, buffer.Bytes()...)
row = append(row, bVec)
case schemapb.DataType_Float16Vector:
f16Vec := generateFloat16Vectors(1, f16VecDim)
_ = binary.Write(&buffer, common.Endian, f16Vec)
ret.Value = append(ret.Value, buffer.Bytes()...)
row = append(row, f16Vec)
case schemapb.DataType_Bool:
data := rand.Int()%2 == 0
_ = binary.Write(&buffer, common.Endian, data)
@ -550,7 +582,7 @@ func genRowWithAllFields(fVecDim, bVecDim int) (blob *commonpb.Blob, pk int64, r
return ret, pk, row
}
func genRowBasedInsertMsg(numRows, fVecDim, bVecDim int) (msg *msgstream.InsertMsg, pks []int64, columns [][]interface{}) {
func genRowBasedInsertMsg(numRows, fVecDim, bVecDim, f16VecDim int) (msg *msgstream.InsertMsg, pks []int64, columns [][]interface{}) {
msg = &msgstream.InsertMsg{
BaseMsg: msgstream.BaseMsg{
Ctx: nil,
@ -573,7 +605,7 @@ func genRowBasedInsertMsg(numRows, fVecDim, bVecDim int) (msg *msgstream.InsertM
pks = make([]int64, 0)
raws := make([][]interface{}, 0)
for i := 0; i < numRows; i++ {
row, pk, raw := genRowWithAllFields(fVecDim, bVecDim)
row, pk, raw := genRowWithAllFields(fVecDim, bVecDim, f16VecDim)
msg.InsertRequest.RowData = append(msg.InsertRequest.RowData, row)
pks = append(pks, pk)
raws = append(raws, raw)
@ -588,7 +620,7 @@ func genRowBasedInsertMsg(numRows, fVecDim, bVecDim int) (msg *msgstream.InsertM
return msg, pks, columns
}
func genColumnBasedInsertMsg(schema *schemapb.CollectionSchema, numRows, fVecDim, bVecDim int) (msg *msgstream.InsertMsg, pks []int64, columns [][]interface{}) {
func genColumnBasedInsertMsg(schema *schemapb.CollectionSchema, numRows, fVecDim, bVecDim, f16VecDim int) (msg *msgstream.InsertMsg, pks []int64, columns [][]interface{}) {
msg = &msgstream.InsertMsg{
BaseMsg: msgstream.BaseMsg{
Ctx: nil,
@ -795,6 +827,25 @@ func genColumnBasedInsertMsg(schema *schemapb.CollectionSchema, numRows, fVecDim
for nrows := 0; nrows < numRows; nrows++ {
columns[idx] = append(columns[idx], data[nrows*bVecDim/8:(nrows+1)*bVecDim/8])
}
case schemapb.DataType_Float16Vector:
data := generateFloat16Vectors(numRows, f16VecDim)
f := &schemapb.FieldData{
Type: schemapb.DataType_Float16Vector,
FieldName: field.Name,
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: int64(f16VecDim),
Data: &schemapb.VectorField_Float16Vector{
Float16Vector: data,
},
},
},
FieldId: field.FieldID,
}
msg.FieldsData = append(msg.FieldsData, f)
for nrows := 0; nrows < numRows; nrows++ {
columns[idx] = append(columns[idx], data[nrows*f16VecDim*2:(nrows+1)*f16VecDim*2])
}
case schemapb.DataType_Array:
data := generateInt32ArrayList(numRows)
@ -845,10 +896,10 @@ func genColumnBasedInsertMsg(schema *schemapb.CollectionSchema, numRows, fVecDim
}
func TestRowBasedInsertMsgToInsertData(t *testing.T) {
numRows, fVecDim, bVecDim := 10, 8, 8
schema, _, fieldIDs := genAllFieldsSchema(fVecDim, bVecDim)
numRows, fVecDim, bVecDim, f16VecDim := 10, 8, 8, 8
schema, _, fieldIDs := genAllFieldsSchema(fVecDim, bVecDim, f16VecDim)
fieldIDs = fieldIDs[:len(fieldIDs)-2]
msg, _, columns := genRowBasedInsertMsg(numRows, fVecDim, bVecDim)
msg, _, columns := genRowBasedInsertMsg(numRows, fVecDim, bVecDim, f16VecDim)
idata, err := RowBasedInsertMsgToInsertData(msg, schema)
assert.NoError(t, err)
@ -864,9 +915,9 @@ func TestRowBasedInsertMsgToInsertData(t *testing.T) {
}
func TestColumnBasedInsertMsgToInsertData(t *testing.T) {
numRows, fVecDim, bVecDim := 2, 2, 8
schema, _, fieldIDs := genAllFieldsSchema(fVecDim, bVecDim)
msg, _, columns := genColumnBasedInsertMsg(schema, numRows, fVecDim, bVecDim)
numRows, fVecDim, bVecDim, f16VecDim := 2, 2, 8, 2
schema, _, fieldIDs := genAllFieldsSchema(fVecDim, bVecDim, f16VecDim)
msg, _, columns := genColumnBasedInsertMsg(schema, numRows, fVecDim, bVecDim, f16VecDim)
idata, err := ColumnBasedInsertMsgToInsertData(msg, schema)
assert.NoError(t, err)
@ -882,10 +933,10 @@ func TestColumnBasedInsertMsgToInsertData(t *testing.T) {
}
func TestInsertMsgToInsertData(t *testing.T) {
numRows, fVecDim, bVecDim := 10, 8, 8
schema, _, fieldIDs := genAllFieldsSchema(fVecDim, bVecDim)
numRows, fVecDim, bVecDim, f16VecDim := 10, 8, 8, 8
schema, _, fieldIDs := genAllFieldsSchema(fVecDim, bVecDim, f16VecDim)
fieldIDs = fieldIDs[:len(fieldIDs)-2]
msg, _, columns := genRowBasedInsertMsg(numRows, fVecDim, bVecDim)
msg, _, columns := genRowBasedInsertMsg(numRows, fVecDim, bVecDim, f16VecDim)
idata, err := InsertMsgToInsertData(msg, schema)
assert.NoError(t, err)
@ -901,9 +952,9 @@ func TestInsertMsgToInsertData(t *testing.T) {
}
func TestInsertMsgToInsertData2(t *testing.T) {
numRows, fVecDim, bVecDim := 2, 2, 8
schema, _, fieldIDs := genAllFieldsSchema(fVecDim, bVecDim)
msg, _, columns := genColumnBasedInsertMsg(schema, numRows, fVecDim, bVecDim)
numRows, fVecDim, bVecDim, f16VecDim := 2, 2, 8, 2
schema, _, fieldIDs := genAllFieldsSchema(fVecDim, bVecDim, f16VecDim)
msg, _, columns := genColumnBasedInsertMsg(schema, numRows, fVecDim, bVecDim, f16VecDim)
idata, err := InsertMsgToInsertData(msg, schema)
assert.NoError(t, err)

43
internal/util/importutil/binlog_file.go
View File

@ -524,6 +524,49 @@ func (p *BinlogFile) ReadBinaryVector() ([]byte, int, error) {
return result, dim, nil
}
func (p *BinlogFile) ReadFloat16Vector() ([]byte, int, error) {
if p.reader == nil {
log.Warn("Binlog file: binlog reader not yet initialized")
return nil, 0, errors.New("binlog reader not yet initialized")
}
dim := 0
result := make([]byte, 0)
for {
event, err := p.reader.NextEventReader()
if err != nil {
log.Warn("Binlog file: failed to iterate events reader", zap.Error(err))
return nil, 0, fmt.Errorf("failed to iterate events reader, error: %w", err)
}
// end of the file
if event == nil {
break
}
if event.TypeCode != storage.InsertEventType {
log.Warn("Binlog file: binlog file is not insert log")
return nil, 0, errors.New("binlog file is not insert log")
}
if p.DataType() != schemapb.DataType_Float16Vector {
log.Warn("Binlog file: binlog data type is not float16 vector")
return nil, 0, errors.New("binlog data type is not float16 vector")
}
data, dimenson, err := event.PayloadReaderInterface.GetFloat16VectorFromPayload()
if err != nil {
log.Warn("Binlog file: failed to read float16 vector data", zap.Error(err))
return nil, 0, fmt.Errorf("failed to read float16 vector data, error: %w", err)
}
dim = dimenson
result = append(result, data...)
}
return result, dim, nil
}
// ReadFloatVector method reads all the blocks of a binlog by a data type.
// A binlog is designed to support multiple blocks, but so far each binlog always contains only one block.
// return vectors data and the dimension

88
internal/util/importutil/binlog_file_test.go
View File

@ -43,6 +43,11 @@ func createBinlogBuf(t *testing.T, dataType schemapb.DataType, data interface{})
if len(vectors) > 0 {
dim = len(vectors[0])
}
} else if dataType == schemapb.DataType_Float16Vector {
vectors := data.([][]byte)
if len(vectors) > 0 {
dim = len(vectors[0]) / 2
}
}
evt, err := w.NextInsertEventWriter(dim)
@ -144,6 +149,16 @@ func createBinlogBuf(t *testing.T, dataType schemapb.DataType, data interface{})
// the "original_size" is come from storage.originalSizeKey
sizeTotal := len(vectors) * dim * 4
w.AddExtra("original_size", fmt.Sprintf("%v", sizeTotal))
case schemapb.DataType_Float16Vector:
vectors := data.([][]byte)
for i := 0; i < len(vectors); i++ {
err = evt.AddFloat16VectorToPayload(vectors[i], dim)
assert.NoError(t, err)
}
// without the two lines, the case will crash at here.
// the "original_size" is come from storage.originalSizeKey
sizeTotal := len(vectors) * dim * 2
w.AddExtra("original_size", fmt.Sprintf("%v", sizeTotal))
default:
assert.True(t, false)
return nil
@ -256,6 +271,11 @@ func Test_BinlogFileOpen(t *testing.T) {
assert.Nil(t, dataFloatVector)
assert.Equal(t, 0, dim)
assert.Error(t, err)
dataFloat16Vector, dim, err := binlogFile.ReadFloat16Vector()
assert.Nil(t, dataFloat16Vector)
assert.Equal(t, 0, dim)
assert.Error(t, err)
}
func Test_BinlogFileBool(t *testing.T) {
@ -894,3 +914,71 @@ func Test_BinlogFileFloatVector(t *testing.T) {
binlogFile.Close()
}
func Test_BinlogFileFloat16Vector(t *testing.T) {
vectors := make([][]byte, 0)
vectors = append(vectors, []byte{1, 3, 5, 7})
vectors = append(vectors, []byte{2, 4, 6, 8})
dim := len(vectors[0]) / 2
vecCount := len(vectors)
chunkManager := &MockChunkManager{
readBuf: map[string][]byte{
"dummy": createBinlogBuf(t, schemapb.DataType_Float16Vector, vectors),
},
}
binlogFile, err := NewBinlogFile(chunkManager)
assert.NoError(t, err)
assert.NotNil(t, binlogFile)
// correct reading
err = binlogFile.Open("dummy")
assert.NoError(t, err)
assert.Equal(t, schemapb.DataType_Float16Vector, binlogFile.DataType())
data, d, err := binlogFile.ReadFloat16Vector()
assert.NoError(t, err)
assert.Equal(t, dim, d)
assert.NotNil(t, data)
assert.Equal(t, vecCount*dim*2, len(data))
for i := 0; i < vecCount; i++ {
for j := 0; j < dim*2; j++ {
assert.Equal(t, vectors[i][j], data[i*dim*2+j])
}
}
binlogFile.Close()
// wrong data type reading
binlogFile, err = NewBinlogFile(chunkManager)
assert.NoError(t, err)
err = binlogFile.Open("dummy")
assert.NoError(t, err)
dt, d, err := binlogFile.ReadFloatVector()
assert.Zero(t, len(dt))
assert.Zero(t, d)
assert.Error(t, err)
binlogFile.Close()
// wrong log type
chunkManager.readBuf["dummy"] = createDeltalogBuf(t, []int64{1}, false)
err = binlogFile.Open("dummy")
assert.NoError(t, err)
data, d, err = binlogFile.ReadFloat16Vector()
assert.Zero(t, len(data))
assert.Zero(t, d)
assert.Error(t, err)
// failed to iterate events reader
binlogFile.reader.Close()
data, d, err = binlogFile.ReadFloat16Vector()
assert.Zero(t, len(data))
assert.Zero(t, d)
assert.Error(t, err)
binlogFile.Close()
}

2
internal/util/indexcgowrapper/index.go
View File

@ -115,6 +115,8 @@ func (index *CgoIndex) Build(dataset *Dataset) error {
return fmt.Errorf("build index on supported data type: %s", dataset.DType.String())
case schemapb.DataType_FloatVector:
return index.buildFloatVecIndex(dataset)
case schemapb.DataType_Float16Vector:
return fmt.Errorf("build index on supported data type: %s", dataset.DType.String())
case schemapb.DataType_BinaryVector:
return index.buildBinaryVecIndex(dataset)
case schemapb.DataType_Bool:

1
internal/util/typeutil/result_helper_test.go
View File

@ -67,6 +67,7 @@ func TestGenEmptyFieldData(t *testing.T) {
vectorTypes := []schemapb.DataType{
schemapb.DataType_BinaryVector,
schemapb.DataType_FloatVector,
schemapb.DataType_Float16Vector,
}
field := &schemapb.FieldSchema{Name: "field_name", FieldID: 100}

2
pkg/go.mod
View File

@ -13,7 +13,7 @@ require (
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/klauspost/compress v1.16.5
github.com/lingdor/stackerror v0.0.0-20191119040541-976d8885ed76
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.0-dev.1.0.20230716112827-c3fe148f5e1d
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.0
github.com/nats-io/nats-server/v2 v2.9.17
github.com/nats-io/nats.go v1.24.0
github.com/panjf2000/ants/v2 v2.7.2

4
pkg/go.sum
View File

@ -477,8 +477,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfr
github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.0-dev.1.0.20230716112827-c3fe148f5e1d h1:XsQQ/MigebXEE2VXPKKmA3K7OHC+mkEUiErWvaWMikI=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.0-dev.1.0.20230716112827-c3fe148f5e1d/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.0 h1:t5CKm7+FXuD2rDLv/H8tpN9iY8F2dZvHF87xWBx8muU=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.0/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
github.com/milvus-io/pulsar-client-go v0.6.10 h1:eqpJjU+/QX0iIhEo3nhOqMNXL+TyInAs1IAHZCrCM/A=
github.com/milvus-io/pulsar-client-go v0.6.10/go.mod h1:lQqCkgwDF8YFYjKA+zOheTk1tev2B+bKj5j7+nm8M1w=
github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=

20
pkg/util/funcutil/func.go
View File

@ -108,7 +108,7 @@ func CheckCtxValid(ctx context.Context) bool {
func GetVecFieldIDs(schema *schemapb.CollectionSchema) []int64 {
var vecFieldIDs []int64
for _, field := range schema.Fields {
if field.DataType == schemapb.DataType_BinaryVector || field.DataType == schemapb.DataType_FloatVector {
if field.DataType == schemapb.DataType_BinaryVector || field.DataType == schemapb.DataType_FloatVector || field.DataType == schemapb.DataType_Float16Vector {
vecFieldIDs = append(vecFieldIDs, field.FieldID)
}
}
@ -222,7 +222,17 @@ func GetNumRowsOfBinaryVectorField(bDatas []byte, dim int64) (uint64, error) {
return uint64((8 * int64(l)) / dim), nil
}
// GetNumRowOfFieldData return num rows of the field data
func GetNumRowsOfFloat16VectorField(f16Datas []byte, dim int64) (uint64, error) {
if dim <= 0 {
return 0, fmt.Errorf("dim(%d) should be greater than 0", dim)
}
l := len(f16Datas)
if int64(l)%dim != 0 {
return 0, fmt.Errorf("the length(%d) of float data should divide the dim(%d)", l, dim)
}
return uint64((int64(l)) / dim / 2), nil
}
func GetNumRowOfFieldData(fieldData *schemapb.FieldData) (uint64, error) {
var fieldNumRows uint64
var err error
@ -264,6 +274,12 @@ func GetNumRowOfFieldData(fieldData *schemapb.FieldData) (uint64, error) {
if err != nil {
return 0, err
}
case *schemapb.VectorField_Float16Vector:
dim := vectorField.GetDim()
fieldNumRows, err = GetNumRowsOfFloat16VectorField(vectorField.GetFloat16Vector(), dim)
if err != nil {
return 0, err
}
default:
return 0, fmt.Errorf("%s is not supported now", vectorFieldType)
}

28
pkg/util/funcutil/func_test.go
View File

@ -219,6 +219,34 @@ func TestGetNumRowsOfFloatVectorField(t *testing.T) {
}
}
func TestGetNumRowsOfFloat16VectorField(t *testing.T) {
cases := []struct {
bDatas []byte
dim int64
want uint64
errIsNil bool
}{
{[]byte{}, -1, 0, false}, // dim <= 0
{[]byte{}, 0, 0, false}, // dim <= 0
{[]byte{1.0}, 128, 0, false}, // length % dim != 0
{[]byte{}, 128, 0, true},
{[]byte{1.0, 2.0}, 1, 1, true},
{[]byte{1.0, 2.0, 3.0, 4.0}, 2, 1, true},
}
for _, test := range cases {
got, err := GetNumRowsOfFloat16VectorField(test.bDatas, test.dim)
if test.errIsNil {
assert.Equal(t, nil, err)
if got != test.want {
t.Errorf("GetNumRowsOfFloat16VectorField(%v, %v) = %v, %v", test.bDatas, test.dim, test.want, nil)
}
} else {
assert.NotEqual(t, nil, err)
}
}
}
func TestGetNumRowsOfBinaryVectorField(t *testing.T) {
cases := []struct {
bDatas []byte

7
pkg/util/indexparamcheck/float_vector_base_checker.go
View File

@ -29,10 +29,11 @@ func (c floatVectorBaseChecker) CheckTrain(params map[string]string) error {
}
func (c floatVectorBaseChecker) CheckValidDataType(dType schemapb.DataType) error {
if dType != schemapb.DataType_FloatVector {
return fmt.Errorf("float vector is only supported")
if dType == schemapb.DataType_FloatVector || dType == schemapb.DataType_Float16Vector {
return nil
} else {
return fmt.Errorf("float or float16 vector are only supported")
}
return nil
}
func (c floatVectorBaseChecker) SetDefaultMetricTypeIfNotExist(params map[string]string) {

2
pkg/util/indexparamcheck/hnsw_checker.go
View File

@ -31,7 +31,7 @@ func (c hnswChecker) CheckTrain(params map[string]string) error {
}
func (c hnswChecker) CheckValidDataType(dType schemapb.DataType) error {
if dType != schemapb.DataType_FloatVector && dType != schemapb.DataType_BinaryVector {
if dType != schemapb.DataType_FloatVector && dType != schemapb.DataType_BinaryVector && dType != schemapb.DataType_Float16Vector {
return fmt.Errorf("only support float vector or binary vector")
}
return nil

22
pkg/util/typeutil/gen_empty_field_data.go
View File

@ -150,6 +150,26 @@ func genEmptyFloatVectorFieldData(field *schemapb.FieldSchema) (*schemapb.FieldD
}, nil
}
func genEmptyFloat16VectorFieldData(field *schemapb.FieldSchema) (*schemapb.FieldData, error) {
dim, err := GetDim(field)
if err != nil {
return nil, err
}
return &schemapb.FieldData{
Type: field.GetDataType(),
FieldName: field.GetName(),
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: dim,
Data: &schemapb.VectorField_Float16Vector{
Float16Vector: nil,
},
},
},
FieldId: field.GetFieldID(),
}, nil
}
func GenEmptyFieldData(field *schemapb.FieldSchema) (*schemapb.FieldData, error) {
dataType := field.GetDataType()
switch dataType {
@ -173,6 +193,8 @@ func GenEmptyFieldData(field *schemapb.FieldSchema) (*schemapb.FieldData, error)
return genEmptyBinaryVectorFieldData(field)
case schemapb.DataType_FloatVector:
return genEmptyFloatVectorFieldData(field)
case schemapb.DataType_Float16Vector:
return genEmptyFloat16VectorFieldData(field)
default:
return nil, fmt.Errorf("unsupported data type: %s", dataType.String())
}

31
pkg/util/typeutil/schema.go
View File

@ -105,6 +105,17 @@ func EstimateSizePerRecord(schema *schemapb.CollectionSchema) (int, error) {
break
}
}
case schemapb.DataType_Float16Vector:
for _, kv := range fs.TypeParams {
if kv.Key == common.DimKey {
v, err := strconv.Atoi(kv.Value)
if err != nil {
return -1, err
}
res += v * 2
break
}
}
}
}
return res, nil
@ -305,7 +316,7 @@ func (helper *SchemaHelper) GetVectorDimFromID(fieldID int64) (int, error) {
// IsVectorType returns true if input is a vector type, otherwise false
func IsVectorType(dataType schemapb.DataType) bool {
switch dataType {
case schemapb.DataType_FloatVector, schemapb.DataType_BinaryVector:
case schemapb.DataType_FloatVector, schemapb.DataType_BinaryVector, schemapb.DataType_Float16Vector:
return true
default:
return false
@ -510,6 +521,17 @@ func AppendFieldData(dst []*schemapb.FieldData, src []*schemapb.FieldData, idx i
} else {
dstVector.GetFloatVector().Data = append(dstVector.GetFloatVector().Data, srcVector.FloatVector.Data[idx*dim:(idx+1)*dim]...)
}
case *schemapb.VectorField_Float16Vector:
if dstVector.GetFloat16Vector() == nil {
srcToCopy := srcVector.Float16Vector[idx*(dim*2) : (idx+1)*(dim*2)]
dstVector.Data = &schemapb.VectorField_Float16Vector{
Float16Vector: make([]byte, len(srcToCopy)),
}
copy(dstVector.Data.(*schemapb.VectorField_Float16Vector).Float16Vector, srcToCopy)
} else {
dstFloat16Vector := dstVector.Data.(*schemapb.VectorField_Float16Vector)
dstFloat16Vector.Float16Vector = append(dstFloat16Vector.Float16Vector, srcVector.Float16Vector[idx*(dim*2):(idx+1)*(dim*2)]...)
}
default:
log.Error("Not supported field type", zap.String("field type", fieldData.Type.String()))
}
@ -558,6 +580,9 @@ func DeleteFieldData(dst []*schemapb.FieldData) {
dstBinaryVector.BinaryVector = dstBinaryVector.BinaryVector[:len(dstBinaryVector.BinaryVector)-int(dim/8)]
case *schemapb.VectorField_FloatVector:
dstVector.GetFloatVector().Data = dstVector.GetFloatVector().Data[:len(dstVector.GetFloatVector().Data)-int(dim)]
case *schemapb.VectorField_Float16Vector:
dstFloat16Vector := dstVector.Data.(*schemapb.VectorField_Float16Vector)
dstFloat16Vector.Float16Vector = dstFloat16Vector.Float16Vector[:len(dstFloat16Vector.Float16Vector)-int(dim*2)]
default:
log.Error("wrong field type added", zap.String("field type", fieldData.Type.String()))
}
@ -873,6 +898,10 @@ func GetData(field *schemapb.FieldData, idx int) interface{} {
dim := int(field.GetVectors().GetDim())
dataBytes := dim / 8
return field.GetVectors().GetBinaryVector()[idx*dataBytes : (idx+1)*dataBytes]
case schemapb.DataType_Float16Vector:
dim := int(field.GetVectors().GetDim())
dataBytes := dim * 2
return field.GetVectors().GetFloat16Vector()[idx*dataBytes : (idx+1)*dataBytes]
}
return nil
}

88
pkg/util/typeutil/schema_test.go
View File

@ -588,6 +588,20 @@ func genFieldData(fieldName string, fieldID int64, fieldType schemapb.DataType,
},
FieldId: fieldID,
}
case schemapb.DataType_Float16Vector:
fieldData = &schemapb.FieldData{
Type: schemapb.DataType_Float16Vector,
FieldName: fieldName,
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: dim,
Data: &schemapb.VectorField_Float16Vector{
Float16Vector: fieldValue.([]byte),
},
},
},
FieldId: fieldID,
}
case schemapb.DataType_Array:
data := fieldValue.([][]int32)
fieldData = &schemapb.FieldData{
@ -640,21 +654,23 @@ func genFieldData(fieldName string, fieldID int64, fieldType schemapb.DataType,
func TestAppendFieldData(t *testing.T) {
const (
Dim = 8
BoolFieldName = "BoolField"
Int32FieldName = "Int32Field"
Int64FieldName = "Int64Field"
FloatFieldName = "FloatField"
DoubleFieldName = "DoubleField"
BinaryVectorFieldName = "BinaryVectorField"
FloatVectorFieldName = "FloatVectorField"
BoolFieldID = common.StartOfUserFieldID + 1
Int32FieldID = common.StartOfUserFieldID + 2
Int64FieldID = common.StartOfUserFieldID + 3
FloatFieldID = common.StartOfUserFieldID + 4
DoubleFieldID = common.StartOfUserFieldID + 5
BinaryVectorFieldID = common.StartOfUserFieldID + 6
FloatVectorFieldID = common.StartOfUserFieldID + 7
Dim = 8
BoolFieldName = "BoolField"
Int32FieldName = "Int32Field"
Int64FieldName = "Int64Field"
FloatFieldName = "FloatField"
DoubleFieldName = "DoubleField"
BinaryVectorFieldName = "BinaryVectorField"
FloatVectorFieldName = "FloatVectorField"
Float16VectorFieldName = "Float16VectorField"
BoolFieldID = common.StartOfUserFieldID + 1
Int32FieldID = common.StartOfUserFieldID + 2
Int64FieldID = common.StartOfUserFieldID + 3
FloatFieldID = common.StartOfUserFieldID + 4
DoubleFieldID = common.StartOfUserFieldID + 5
BinaryVectorFieldID = common.StartOfUserFieldID + 6
FloatVectorFieldID = common.StartOfUserFieldID + 7
Float16VectorFieldID = common.StartOfUserFieldID + 8
)
BoolArray := []bool{true, false}
Int32Array := []int32{1, 2}
@ -663,8 +679,10 @@ func TestAppendFieldData(t *testing.T) {
DoubleArray := []float64{11.0, 22.0}
BinaryVector := []byte{0x12, 0x34}
FloatVector := []float32{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 11.0, 22.0, 33.0, 44.0, 55.0, 66.0, 77.0, 88.0}
Float16Vector := []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff}
result := make([]*schemapb.FieldData, 7)
result := make([]*schemapb.FieldData, 8)
var fieldDataArray1 []*schemapb.FieldData
fieldDataArray1 = append(fieldDataArray1, genFieldData(BoolFieldName, BoolFieldID, schemapb.DataType_Bool, BoolArray[0:1], 1))
fieldDataArray1 = append(fieldDataArray1, genFieldData(Int32FieldName, Int32FieldID, schemapb.DataType_Int32, Int32Array[0:1], 1))
@ -673,6 +691,7 @@ func TestAppendFieldData(t *testing.T) {
fieldDataArray1 = append(fieldDataArray1, genFieldData(DoubleFieldName, DoubleFieldID, schemapb.DataType_Double, DoubleArray[0:1], 1))
fieldDataArray1 = append(fieldDataArray1, genFieldData(BinaryVectorFieldName, BinaryVectorFieldID, schemapb.DataType_BinaryVector, BinaryVector[0:Dim/8], Dim))
fieldDataArray1 = append(fieldDataArray1, genFieldData(FloatVectorFieldName, FloatVectorFieldID, schemapb.DataType_FloatVector, FloatVector[0:Dim], Dim))
fieldDataArray1 = append(fieldDataArray1, genFieldData(Float16VectorFieldName, Float16VectorFieldID, schemapb.DataType_Float16Vector, Float16Vector[0:Dim*2], Dim))
var fieldDataArray2 []*schemapb.FieldData
fieldDataArray2 = append(fieldDataArray2, genFieldData(BoolFieldName, BoolFieldID, schemapb.DataType_Bool, BoolArray[1:2], 1))
@ -682,6 +701,7 @@ func TestAppendFieldData(t *testing.T) {
fieldDataArray2 = append(fieldDataArray2, genFieldData(DoubleFieldName, DoubleFieldID, schemapb.DataType_Double, DoubleArray[1:2], 1))
fieldDataArray2 = append(fieldDataArray2, genFieldData(BinaryVectorFieldName, BinaryVectorFieldID, schemapb.DataType_BinaryVector, BinaryVector[Dim/8:2*Dim/8], Dim))
fieldDataArray2 = append(fieldDataArray2, genFieldData(FloatVectorFieldName, FloatVectorFieldID, schemapb.DataType_FloatVector, FloatVector[Dim:2*Dim], Dim))
fieldDataArray2 = append(fieldDataArray2, genFieldData(Float16VectorFieldName, Float16VectorFieldID, schemapb.DataType_Float16Vector, Float16Vector[2*Dim:4*Dim], Dim))
AppendFieldData(result, fieldDataArray1, 0)
AppendFieldData(result, fieldDataArray2, 0)
@ -693,19 +713,21 @@ func TestAppendFieldData(t *testing.T) {
assert.Equal(t, DoubleArray, result[4].GetScalars().GetDoubleData().Data)
assert.Equal(t, BinaryVector, result[5].GetVectors().Data.(*schemapb.VectorField_BinaryVector).BinaryVector)
assert.Equal(t, FloatVector, result[6].GetVectors().GetFloatVector().Data)
assert.Equal(t, Float16Vector, result[7].GetVectors().Data.(*schemapb.VectorField_Float16Vector).Float16Vector)
}
func TestDeleteFieldData(t *testing.T) {
const (
Dim = 8
BoolFieldName = "BoolField"
Int32FieldName = "Int32Field"
Int64FieldName = "Int64Field"
FloatFieldName = "FloatField"
DoubleFieldName = "DoubleField"
JSONFieldName = "JSONField"
BinaryVectorFieldName = "BinaryVectorField"
FloatVectorFieldName = "FloatVectorField"
Dim = 8
BoolFieldName = "BoolField"
Int32FieldName = "Int32Field"
Int64FieldName = "Int64Field"
FloatFieldName = "FloatField"
DoubleFieldName = "DoubleField"
JSONFieldName = "JSONField"
BinaryVectorFieldName = "BinaryVectorField"
FloatVectorFieldName = "FloatVectorField"
Float16VectorFieldName = "Float16VectorField"
)
const (
@ -717,6 +739,7 @@ func TestDeleteFieldData(t *testing.T) {
JSONFieldID
BinaryVectorFieldID
FloatVectorFieldID
Float16VectorFieldID
)
BoolArray := []bool{true, false}
Int32Array := []int32{1, 2}
@ -726,9 +749,11 @@ func TestDeleteFieldData(t *testing.T) {
JSONArray := [][]byte{[]byte("{\"hello\":0}"), []byte("{\"key\":1}")}
BinaryVector := []byte{0x12, 0x34}
FloatVector := []float32{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 11.0, 22.0, 33.0, 44.0, 55.0, 66.0, 77.0, 88.0}
Float16Vector := []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff}
result1 := make([]*schemapb.FieldData, 8)
result2 := make([]*schemapb.FieldData, 8)
result1 := make([]*schemapb.FieldData, 9)
result2 := make([]*schemapb.FieldData, 9)
var fieldDataArray1 []*schemapb.FieldData
fieldDataArray1 = append(fieldDataArray1, genFieldData(BoolFieldName, BoolFieldID, schemapb.DataType_Bool, BoolArray[0:1], 1))
fieldDataArray1 = append(fieldDataArray1, genFieldData(Int32FieldName, Int32FieldID, schemapb.DataType_Int32, Int32Array[0:1], 1))
@ -738,6 +763,7 @@ func TestDeleteFieldData(t *testing.T) {
fieldDataArray1 = append(fieldDataArray1, genFieldData(JSONFieldName, JSONFieldID, schemapb.DataType_JSON, JSONArray[0:1], 1))
fieldDataArray1 = append(fieldDataArray1, genFieldData(BinaryVectorFieldName, BinaryVectorFieldID, schemapb.DataType_BinaryVector, BinaryVector[0:Dim/8], Dim))
fieldDataArray1 = append(fieldDataArray1, genFieldData(FloatVectorFieldName, FloatVectorFieldID, schemapb.DataType_FloatVector, FloatVector[0:Dim], Dim))
fieldDataArray1 = append(fieldDataArray1, genFieldData(Float16VectorFieldName, Float16VectorFieldID, schemapb.DataType_Float16Vector, Float16Vector[0:2*Dim], Dim))
var fieldDataArray2 []*schemapb.FieldData
fieldDataArray2 = append(fieldDataArray2, genFieldData(BoolFieldName, BoolFieldID, schemapb.DataType_Bool, BoolArray[1:2], 1))
@ -748,6 +774,7 @@ func TestDeleteFieldData(t *testing.T) {
fieldDataArray2 = append(fieldDataArray2, genFieldData(JSONFieldName, JSONFieldID, schemapb.DataType_JSON, JSONArray[1:2], 1))
fieldDataArray2 = append(fieldDataArray2, genFieldData(BinaryVectorFieldName, BinaryVectorFieldID, schemapb.DataType_BinaryVector, BinaryVector[Dim/8:2*Dim/8], Dim))
fieldDataArray2 = append(fieldDataArray2, genFieldData(FloatVectorFieldName, FloatVectorFieldID, schemapb.DataType_FloatVector, FloatVector[Dim:2*Dim], Dim))
fieldDataArray2 = append(fieldDataArray2, genFieldData(Float16VectorFieldName, Float16VectorFieldID, schemapb.DataType_Float16Vector, Float16Vector[2*Dim:4*Dim], Dim))
AppendFieldData(result1, fieldDataArray1, 0)
AppendFieldData(result1, fieldDataArray2, 0)
@ -760,6 +787,7 @@ func TestDeleteFieldData(t *testing.T) {
assert.Equal(t, JSONArray[0:1], result1[JSONFieldID-common.StartOfUserFieldID].GetScalars().GetJsonData().Data)
assert.Equal(t, BinaryVector[0:Dim/8], result1[BinaryVectorFieldID-common.StartOfUserFieldID].GetVectors().Data.(*schemapb.VectorField_BinaryVector).BinaryVector)
assert.Equal(t, FloatVector[0:Dim], result1[FloatVectorFieldID-common.StartOfUserFieldID].GetVectors().GetFloatVector().Data)
assert.Equal(t, Float16Vector[0:2*Dim], result1[Float16VectorFieldID-common.StartOfUserFieldID].GetVectors().Data.(*schemapb.VectorField_Float16Vector).Float16Vector)
AppendFieldData(result2, fieldDataArray2, 0)
AppendFieldData(result2, fieldDataArray1, 0)
@ -772,6 +800,7 @@ func TestDeleteFieldData(t *testing.T) {
assert.Equal(t, JSONArray[1:2], result2[JSONFieldID-common.StartOfUserFieldID].GetScalars().GetJsonData().Data)
assert.Equal(t, BinaryVector[Dim/8:2*Dim/8], result2[BinaryVectorFieldID-common.StartOfUserFieldID].GetVectors().Data.(*schemapb.VectorField_BinaryVector).BinaryVector)
assert.Equal(t, FloatVector[Dim:2*Dim], result2[FloatVectorFieldID-common.StartOfUserFieldID].GetVectors().GetFloatVector().Data)
assert.Equal(t, Float16Vector[2*Dim:4*Dim], result2[Float16VectorFieldID-common.StartOfUserFieldID].GetVectors().Data.(*schemapb.VectorField_Float16Vector).Float16Vector)
}
func TestGetPrimaryFieldSchema(t *testing.T) {
@ -1091,6 +1120,8 @@ func TestGetDataAndGetDataSize(t *testing.T) {
VarCharArray := []string{"a", "b"}
BinaryVector := []byte{0x12, 0x34}
FloatVector := []float32{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 11.0, 22.0, 33.0, 44.0, 55.0, 66.0, 77.0, 88.0}
Float16Vector := []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77,
0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77}
boolData := genFieldData(fieldName, fieldID, schemapb.DataType_Bool, BoolArray, 1)
int8Data := genFieldData(fieldName, fieldID, schemapb.DataType_Int8, Int8Array, 1)
@ -1102,6 +1133,7 @@ func TestGetDataAndGetDataSize(t *testing.T) {
varCharData := genFieldData(fieldName, fieldID, schemapb.DataType_VarChar, VarCharArray, 1)
binVecData := genFieldData(fieldName, fieldID, schemapb.DataType_BinaryVector, BinaryVector, Dim)
floatVecData := genFieldData(fieldName, fieldID, schemapb.DataType_FloatVector, FloatVector, Dim)
float16VecData := genFieldData(fieldName, fieldID, schemapb.DataType_Float16Vector, Float16Vector, Dim)
invalidData := &schemapb.FieldData{
Type: schemapb.DataType_None,
}
@ -1125,6 +1157,7 @@ func TestGetDataAndGetDataSize(t *testing.T) {
varCharDataRes := GetData(varCharData, 0)
binVecDataRes := GetData(binVecData, 0)
floatVecDataRes := GetData(floatVecData, 0)
float16VecDataRes := GetData(float16VecData, 0)
invalidDataRes := GetData(invalidData, 0)
assert.Equal(t, BoolArray[0], boolDataRes)
@ -1137,6 +1170,7 @@ func TestGetDataAndGetDataSize(t *testing.T) {
assert.Equal(t, VarCharArray[0], varCharDataRes)
assert.ElementsMatch(t, BinaryVector[:Dim/8], binVecDataRes)
assert.ElementsMatch(t, FloatVector[:Dim], floatVecDataRes)
assert.ElementsMatch(t, Float16Vector[:2*Dim], float16VecDataRes)
assert.Nil(t, invalidDataRes)
})
}

32
tests/integration/getvector/get_vector_test.go
View File

@ -125,6 +125,8 @@ func (s *TestGetVectorSuite) run() {
var vecFieldData *schemapb.FieldData
if s.vecType == schemapb.DataType_FloatVector {
vecFieldData = integration.NewFloatVectorFieldData(vecFieldName, NB, dim)
} else if s.vecType == schemapb.DataType_Float16Vector {
vecFieldData = integration.NewFloat16VectorFieldData(vecFieldName, NB, dim)
} else {
vecFieldData = integration.NewBinaryVectorFieldData(vecFieldName, NB, dim)
}
@ -235,6 +237,25 @@ func (s *TestGetVectorSuite) run() {
s.Require().ElementsMatch(expect, actual)
}
}
} else if s.vecType == schemapb.DataType_Float16Vector {
// s.Require().Len(result.GetFieldsData()[vecFieldIndex].GetVectors().GetFloat16Vector(), nq*topk*dim*2)
// rawData := vecFieldData.GetVectors().GetFloat16Vector()
// resData := result.GetFieldsData()[vecFieldIndex].GetVectors().GetFloat16Vector()
// if s.pkType == schemapb.DataType_Int64 {
// for i, id := range result.GetIds().GetIntId().GetData() {
// expect := rawData[int(id)*dim : (int(id)+1)*dim]
// actual := resData[i*dim : (i+1)*dim]
// s.Require().ElementsMatch(expect, actual)
// }
// } else {
// for i, idStr := range result.GetIds().GetStrId().GetData() {
// id, err := strconv.Atoi(idStr)
// s.Require().NoError(err)
// expect := rawData[id*dim : (id+1)*dim]
// actual := resData[i*dim : (i+1)*dim]
// s.Require().ElementsMatch(expect, actual)
// }
// }
} else {
s.Require().Len(result.GetFieldsData()[vecFieldIndex].GetVectors().GetBinaryVector(), nq*topk*dim/8)
rawData := vecFieldData.GetVectors().GetBinaryVector()
@ -281,6 +302,17 @@ func (s *TestGetVectorSuite) TestGetVector_FLAT() {
s.run()
}
func (s *TestGetVectorSuite) TestGetVector_Float16Vector() {
s.nq = 10
s.topK = 10
s.indexType = integration.IndexFaissIDMap
s.metricType = metric.L2
s.pkType = schemapb.DataType_Int64
s.vecType = schemapb.DataType_Float16Vector
s.searchFailed = false
s.run()
}
func (s *TestGetVectorSuite) TestGetVector_IVF_FLAT() {
s.nq = 10
s.topK = 10

25
tests/integration/util_insert.go
View File

@ -116,6 +116,21 @@ func NewFloatVectorFieldData(fieldName string, numRows, dim int) *schemapb.Field
}
}
func NewFloat16VectorFieldData(fieldName string, numRows, dim int) *schemapb.FieldData {
return &schemapb.FieldData{
Type: schemapb.DataType_Float16Vector,
FieldName: fieldName,
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: int64(dim),
Data: &schemapb.VectorField_Float16Vector{
Float16Vector: GenerateFloat16Vectors(numRows, dim),
},
},
},
}
}
func NewBinaryVectorFieldData(fieldName string, numRows, dim int) *schemapb.FieldData {
return &schemapb.FieldData{
Type: schemapb.DataType_BinaryVector,
@ -166,6 +181,16 @@ func GenerateBinaryVectors(numRows, dim int) []byte {
return ret
}
func GenerateFloat16Vectors(numRows, dim int) []byte {
total := numRows * dim * 2
ret := make([]byte, total)
_, err := rand.Read(ret)
if err != nil {
panic(err)
}
return ret
}
func GenerateHashKeys(numRows int) []uint32 {
ret := make([]uint32, 0, numRows)
for i := 0; i < numRows; i++ {

11
tests/integration/util_query.go
View File

@ -181,6 +181,17 @@ func constructPlaceholderGroup(nq, dim int, vectorType schemapb.DataType) *commo
}
values = append(values, ret)
}
case schemapb.DataType_Float16Vector:
placeholderType = commonpb.PlaceholderType_Float16Vector
for i := 0; i < nq; i++ {
total := dim * 2
ret := make([]byte, total)
_, err := rand.Read(ret)
if err != nil {
panic(err)
}
values = append(values, ret)
}
default:
panic("invalid vector data type")
}

23
tests/integration/util_schema.go
View File

@ -25,17 +25,18 @@ import (
)
const (
BoolField = "boolField"
Int8Field = "int8Field"
Int16Field = "int16Field"
Int32Field = "int32Field"
Int64Field = "int64Field"
FloatField = "floatField"
DoubleField = "doubleField"
VarCharField = "varCharField"
JSONField = "jsonField"
FloatVecField = "floatVecField"
BinVecField = "binVecField"
BoolField = "boolField"
Int8Field = "int8Field"
Int16Field = "int16Field"
Int32Field = "int32Field"
Int64Field = "int64Field"
FloatField = "floatField"
DoubleField = "doubleField"
VarCharField = "varCharField"
JSONField = "jsonField"
FloatVecField = "floatVecField"
BinVecField = "binVecField"
Float16VecField = "float16VecField"
)
func ConstructSchema(collection string, dim int, autoID bool, fields ...*schemapb.FieldSchema) *schemapb.CollectionSchema {