test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-01 03:18:29 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

Support TermExpr, NotExpr, LogicalExpr (#5096)

Browse Source 
1. Support Term, like `A in [1, 2, 3]`
2. Support Not, like `! A < 3`
3. Support logical combination, like `A < 3 && B > 5 or C == 0`

Type: Feature

Signed-off-by: fluorinedog <fluorinedog@gmail.com>
This commit is contained in:
FluorineDog 2021-04-30 15:19:52 +08:00 committed by GitHub
parent b46ae44087
commit 396b3f33e9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 1242 additions and 247 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
internal/core/src/common/FieldMeta.h
View File

@ -83,6 +83,30 @@ datatype_is_vector(DataType datatype) {
return datatype == DataType::VECTOR_BINARY || datatype == DataType::VECTOR_FLOAT;
}
inline bool
datatype_is_interger(DataType datatype) {
switch (datatype) {
case DataType::INT8:
case DataType::INT16:
case DataType::INT32:
case DataType::INT64:
return true;
default:
return false;
}
}
inline bool
datatype_is_floating(DataType datatype) {
switch (datatype) {
case DataType::DOUBLE:
case DataType::FLOAT:
return true;
default:
return false;
}
}
class FieldMeta {
public:
FieldMeta(const FieldMeta&) = delete;

289
internal/core/src/pb/plan.pb.cc
View File

@ -15,9 +15,8 @@
#include <google/protobuf/wire_format.h>
// @@protoc_insertion_point(includes)
#include <google/protobuf/port_def.inc>
extern PROTOBUF_INTERNAL_EXPORT_plan_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_BinaryExpr_plan_2eproto;
extern PROTOBUF_INTERNAL_EXPORT_plan_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<2> scc_info_BinaryExpr_plan_2eproto;
extern PROTOBUF_INTERNAL_EXPORT_plan_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_ColumnInfo_plan_2eproto;
extern PROTOBUF_INTERNAL_EXPORT_plan_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<3> scc_info_Expr_plan_2eproto;
extern PROTOBUF_INTERNAL_EXPORT_plan_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_GenericValue_plan_2eproto;
extern PROTOBUF_INTERNAL_EXPORT_plan_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_QueryInfo_plan_2eproto;
extern PROTOBUF_INTERNAL_EXPORT_plan_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<2> scc_info_RangeExpr_plan_2eproto;
@ -80,16 +79,30 @@ class PlanNodeDefaultTypeInternal {
static void InitDefaultsscc_info_BinaryExpr_plan_2eproto() {
GOOGLE_PROTOBUF_VERIFY_VERSION;
{
void* ptr = &::milvus::proto::plan::_UnaryExpr_default_instance_;
new (ptr) ::milvus::proto::plan::UnaryExpr();
::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr);
}
{
void* ptr = &::milvus::proto::plan::_BinaryExpr_default_instance_;
new (ptr) ::milvus::proto::plan::BinaryExpr();
::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr);
}
{
void* ptr = &::milvus::proto::plan::_Expr_default_instance_;
new (ptr) ::milvus::proto::plan::Expr();
::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr);
}
::milvus::proto::plan::UnaryExpr::InitAsDefaultInstance();
::milvus::proto::plan::BinaryExpr::InitAsDefaultInstance();
::milvus::proto::plan::Expr::InitAsDefaultInstance();
}
::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_BinaryExpr_plan_2eproto =
{{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 0, InitDefaultsscc_info_BinaryExpr_plan_2eproto}, {}};
::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<2> scc_info_BinaryExpr_plan_2eproto =
{{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 2, InitDefaultsscc_info_BinaryExpr_plan_2eproto}, {
&scc_info_RangeExpr_plan_2eproto.base,
&scc_info_TermExpr_plan_2eproto.base,}};
static void InitDefaultsscc_info_ColumnInfo_plan_2eproto() {
GOOGLE_PROTOBUF_VERIFY_VERSION;
@ -105,29 +118,6 @@ static void InitDefaultsscc_info_ColumnInfo_plan_2eproto() {
::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_ColumnInfo_plan_2eproto =
{{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 0, InitDefaultsscc_info_ColumnInfo_plan_2eproto}, {}};
static void InitDefaultsscc_info_Expr_plan_2eproto() {
GOOGLE_PROTOBUF_VERIFY_VERSION;
{
void* ptr = &::milvus::proto::plan::_UnaryExpr_default_instance_;
new (ptr) ::milvus::proto::plan::UnaryExpr();
::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr);
}
{
void* ptr = &::milvus::proto::plan::_Expr_default_instance_;
new (ptr) ::milvus::proto::plan::Expr();
::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr);
}
::milvus::proto::plan::UnaryExpr::InitAsDefaultInstance();
::milvus::proto::plan::Expr::InitAsDefaultInstance();
}
::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<3> scc_info_Expr_plan_2eproto =
{{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 3, InitDefaultsscc_info_Expr_plan_2eproto}, {
&scc_info_RangeExpr_plan_2eproto.base,
&scc_info_TermExpr_plan_2eproto.base,
&scc_info_BinaryExpr_plan_2eproto.base,}};
static void InitDefaultsscc_info_GenericValue_plan_2eproto() {
GOOGLE_PROTOBUF_VERIFY_VERSION;
@ -216,7 +206,7 @@ static void InitDefaultsscc_info_VectorANNS_plan_2eproto() {
::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<2> scc_info_VectorANNS_plan_2eproto =
{{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 2, InitDefaultsscc_info_VectorANNS_plan_2eproto}, {
&scc_info_Expr_plan_2eproto.base,
&scc_info_BinaryExpr_plan_2eproto.base,
&scc_info_QueryInfo_plan_2eproto.base,}};
static ::PROTOBUF_NAMESPACE_ID::Metadata file_level_metadata_plan_2eproto[10];
@ -275,6 +265,9 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_plan_2eproto::offsets[] PROTOB
~0u, // no _extensions_
~0u, // no _oneof_case_
~0u, // no _weak_field_map_
PROTOBUF_FIELD_OFFSET(::milvus::proto::plan::BinaryExpr, op_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::plan::BinaryExpr, left_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::plan::BinaryExpr, right_),
~0u, // no _has_bits_
PROTOBUF_FIELD_OFFSET(::milvus::proto::plan::Expr, _internal_metadata_),
~0u, // no _extensions_
@ -311,9 +304,9 @@ static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOB
{ 32, -1, sizeof(::milvus::proto::plan::TermExpr)},
{ 39, -1, sizeof(::milvus::proto::plan::UnaryExpr)},
{ 46, -1, sizeof(::milvus::proto::plan::BinaryExpr)},
{ 51, -1, sizeof(::milvus::proto::plan::Expr)},
{ 61, -1, sizeof(::milvus::proto::plan::VectorANNS)},
{ 71, -1, sizeof(::milvus::proto::plan::PlanNode)},
{ 54, -1, sizeof(::milvus::proto::plan::Expr)},
{ 64, -1, sizeof(::milvus::proto::plan::VectorANNS)},
{ 74, -1, sizeof(::milvus::proto::plan::PlanNode)},
};
static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = {
@ -350,30 +343,32 @@ const char descriptor_table_protodef_plan_2eproto[] PROTOBUF_SECTION_VARIABLE(pr
"aryExpr\0220\n\002op\030\001 \001(\0162$.milvus.proto.plan."
"UnaryExpr.UnaryOp\022&\n\005child\030\002 \001(\0132\027.milvu"
"s.proto.plan.Expr\"\037\n\007UnaryOp\022\013\n\007Invalid\020"
"\000\022\007\n\003Not\020\001\"D\n\nBinaryExpr\"6\n\010BinaryOp\022\013\n\007"
"Invalid\020\000\022\016\n\nLogicalAnd\020\001\022\r\n\tLogicalOr\020\002"
"\"\336\001\n\004Expr\0222\n\nrange_expr\030\001 \001(\0132\034.milvus.p"
"roto.plan.RangeExprH\000\0220\n\tterm_expr\030\002 \001(\013"
"2\033.milvus.proto.plan.TermExprH\000\0222\n\nunary"
"_expr\030\003 \001(\0132\034.milvus.proto.plan.UnaryExp"
"rH\000\0224\n\013binary_expr\030\004 \001(\0132\035.milvus.proto."
"plan.BinaryExprH\000B\006\n\004expr\"\251\001\n\nVectorANNS"
"\022\021\n\tis_binary\030\001 \001(\010\022\020\n\010field_id\030\002 \001(\003\022+\n"
"\npredicates\030\003 \001(\0132\027.milvus.proto.plan.Ex"
"pr\0220\n\nquery_info\030\004 \001(\0132\034.milvus.proto.pl"
"an.QueryInfo\022\027\n\017placeholder_tag\030\005 \001(\t\"H\n"
"\010PlanNode\0224\n\013vector_anns\030\001 \001(\0132\035.milvus."
"proto.plan.VectorANNSH\000B\006\n\004nodeB3Z1githu"
"b.com/milvus-io/milvus/internal/proto/pl"
"anpbb\006proto3"
"\000\022\007\n\003Not\020\001\"\307\001\n\nBinaryExpr\0222\n\002op\030\001 \001(\0162&."
"milvus.proto.plan.BinaryExpr.BinaryOp\022%\n"
"\004left\030\002 \001(\0132\027.milvus.proto.plan.Expr\022&\n\005"
"right\030\003 \001(\0132\027.milvus.proto.plan.Expr\"6\n\010"
"BinaryOp\022\013\n\007Invalid\020\000\022\016\n\nLogicalAnd\020\001\022\r\n"
"\tLogicalOr\020\002\"\336\001\n\004Expr\0222\n\nrange_expr\030\001 \001("
"\0132\034.milvus.proto.plan.RangeExprH\000\0220\n\tter"
"m_expr\030\002 \001(\0132\033.milvus.proto.plan.TermExp"
"rH\000\0222\n\nunary_expr\030\003 \001(\0132\034.milvus.proto.p"
"lan.UnaryExprH\000\0224\n\013binary_expr\030\004 \001(\0132\035.m"
"ilvus.proto.plan.BinaryExprH\000B\006\n\004expr\"\251\001"
"\n\nVectorANNS\022\021\n\tis_binary\030\001 \001(\010\022\020\n\010field"
"_id\030\002 \001(\003\022+\n\npredicates\030\003 \001(\0132\027.milvus.p"
"roto.plan.Expr\0220\n\nquery_info\030\004 \001(\0132\034.mil"
"vus.proto.plan.QueryInfo\022\027\n\017placeholder_"
"tag\030\005 \001(\t\"H\n\010PlanNode\0224\n\013vector_anns\030\001 \001"
"(\0132\035.milvus.proto.plan.VectorANNSH\000B\006\n\004n"
"odeB3Z1github.com/milvus-io/milvus/inter"
"nal/proto/planpbb\006proto3"
;
static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_plan_2eproto_deps[1] = {
&::descriptor_table_schema_2eproto,
};
static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_plan_2eproto_sccs[9] = {
static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_plan_2eproto_sccs[8] = {
&scc_info_BinaryExpr_plan_2eproto.base,
&scc_info_ColumnInfo_plan_2eproto.base,
&scc_info_Expr_plan_2eproto.base,
&scc_info_GenericValue_plan_2eproto.base,
&scc_info_PlanNode_plan_2eproto.base,
&scc_info_QueryInfo_plan_2eproto.base,
@ -384,8 +379,8 @@ static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_pla
static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_plan_2eproto_once;
static bool descriptor_table_plan_2eproto_initialized = false;
const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_plan_2eproto = {
&descriptor_table_plan_2eproto_initialized, descriptor_table_protodef_plan_2eproto, "plan.proto", 1412,
&descriptor_table_plan_2eproto_once, descriptor_table_plan_2eproto_sccs, descriptor_table_plan_2eproto_deps, 9, 1,
&descriptor_table_plan_2eproto_initialized, descriptor_table_protodef_plan_2eproto, "plan.proto", 1544,
&descriptor_table_plan_2eproto_once, descriptor_table_plan_2eproto_sccs, descriptor_table_plan_2eproto_deps, 8, 1,
schemas, file_default_instances, TableStruct_plan_2eproto::offsets,
file_level_metadata_plan_2eproto, 10, file_level_enum_descriptors_plan_2eproto, file_level_service_descriptors_plan_2eproto,
};
@ -2295,7 +2290,7 @@ UnaryExpr::UnaryExpr(const UnaryExpr& from)
}
void UnaryExpr::SharedCtor() {
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_Expr_plan_2eproto.base);
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_BinaryExpr_plan_2eproto.base);
::memset(&child_, 0, static_cast<size_t>(
reinterpret_cast<char*>(&op_) -
reinterpret_cast<char*>(&child_)) + sizeof(op_));
@ -2314,7 +2309,7 @@ void UnaryExpr::SetCachedSize(int size) const {
_cached_size_.Set(size);
}
const UnaryExpr& UnaryExpr::default_instance() {
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_Expr_plan_2eproto.base);
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_BinaryExpr_plan_2eproto.base);
return *internal_default_instance();
}
@ -2578,11 +2573,25 @@ void UnaryExpr::InternalSwap(UnaryExpr* other) {
// ===================================================================
void BinaryExpr::InitAsDefaultInstance() {
::milvus::proto::plan::_BinaryExpr_default_instance_._instance.get_mutable()->left_ = const_cast< ::milvus::proto::plan::Expr*>(
::milvus::proto::plan::Expr::internal_default_instance());
::milvus::proto::plan::_BinaryExpr_default_instance_._instance.get_mutable()->right_ = const_cast< ::milvus::proto::plan::Expr*>(
::milvus::proto::plan::Expr::internal_default_instance());
}
class BinaryExpr::_Internal {
public:
static const ::milvus::proto::plan::Expr& left(const BinaryExpr* msg);
static const ::milvus::proto::plan::Expr& right(const BinaryExpr* msg);
};
const ::milvus::proto::plan::Expr&
BinaryExpr::_Internal::left(const BinaryExpr* msg) {
return *msg->left_;
}
const ::milvus::proto::plan::Expr&
BinaryExpr::_Internal::right(const BinaryExpr* msg) {
return *msg->right_;
}
BinaryExpr::BinaryExpr()
: ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr) {
SharedCtor();
@ -2592,10 +2601,25 @@ BinaryExpr::BinaryExpr(const BinaryExpr& from)
: ::PROTOBUF_NAMESPACE_ID::Message(),
_internal_metadata_(nullptr) {
_internal_metadata_.MergeFrom(from._internal_metadata_);
if (from.has_left()) {
left_ = new ::milvus::proto::plan::Expr(*from.left_);
} else {
left_ = nullptr;
}
if (from.has_right()) {
right_ = new ::milvus::proto::plan::Expr(*from.right_);
} else {
right_ = nullptr;
}
op_ = from.op_;
// @@protoc_insertion_point(copy_constructor:milvus.proto.plan.BinaryExpr)
}
void BinaryExpr::SharedCtor() {
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_BinaryExpr_plan_2eproto.base);
::memset(&left_, 0, static_cast<size_t>(
reinterpret_cast<char*>(&op_) -
reinterpret_cast<char*>(&left_)) + sizeof(op_));
}
BinaryExpr::~BinaryExpr() {
@ -2604,6 +2628,8 @@ BinaryExpr::~BinaryExpr() {
}
void BinaryExpr::SharedDtor() {
if (this != internal_default_instance()) delete left_;
if (this != internal_default_instance()) delete right_;
}
void BinaryExpr::SetCachedSize(int size) const {
@ -2621,6 +2647,15 @@ void BinaryExpr::Clear() {
// Prevent compiler warnings about cached_has_bits being unused
(void) cached_has_bits;
if (GetArenaNoVirtual() == nullptr && left_ != nullptr) {
delete left_;
}
left_ = nullptr;
if (GetArenaNoVirtual() == nullptr && right_ != nullptr) {
delete right_;
}
right_ = nullptr;
op_ = 0;
_internal_metadata_.Clear();
}
@ -2632,7 +2667,30 @@ const char* BinaryExpr::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID:
ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag);
CHK_(ptr);
switch (tag >> 3) {
// .milvus.proto.plan.BinaryExpr.BinaryOp op = 1;
case 1:
if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 8)) {
::PROTOBUF_NAMESPACE_ID::uint64 val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr);
CHK_(ptr);
set_op(static_cast<::milvus::proto::plan::BinaryExpr_BinaryOp>(val));
} else goto handle_unusual;
continue;
// .milvus.proto.plan.Expr left = 2;
case 2:
if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) {
ptr = ctx->ParseMessage(mutable_left(), ptr);
CHK_(ptr);
} else goto handle_unusual;
continue;
// .milvus.proto.plan.Expr right = 3;
case 3:
if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 26)) {
ptr = ctx->ParseMessage(mutable_right(), ptr);
CHK_(ptr);
} else goto handle_unusual;
continue;
default: {
handle_unusual:
if ((tag & 7) == 4 || tag == 0) {
ctx->SetLastTag(tag);
goto success;
@ -2660,12 +2718,53 @@ bool BinaryExpr::MergePartialFromCodedStream(
::std::pair<::PROTOBUF_NAMESPACE_ID::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u);
tag = p.first;
if (!p.second) goto handle_unusual;
handle_unusual:
if (tag == 0) {
goto success;
switch (::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::GetTagFieldNumber(tag)) {
// .milvus.proto.plan.BinaryExpr.BinaryOp op = 1;
case 1: {
if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (8 & 0xFF)) {
int value = 0;
DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive<
int, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_ENUM>(
input, &value)));
set_op(static_cast< ::milvus::proto::plan::BinaryExpr_BinaryOp >(value));
} else {
goto handle_unusual;
}
break;
}
// .milvus.proto.plan.Expr left = 2;
case 2: {
if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (18 & 0xFF)) {
DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadMessage(
input, mutable_left()));
} else {
goto handle_unusual;
}
break;
}
// .milvus.proto.plan.Expr right = 3;
case 3: {
if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (26 & 0xFF)) {
DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadMessage(
input, mutable_right()));
} else {
goto handle_unusual;
}
break;
}
default: {
handle_unusual:
if (tag == 0) {
goto success;
}
DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SkipField(
input, tag, _internal_metadata_.mutable_unknown_fields()));
break;
}
}
DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SkipField(
input, tag, _internal_metadata_.mutable_unknown_fields()));
}
success:
// @@protoc_insertion_point(parse_success:milvus.proto.plan.BinaryExpr)
@ -2683,6 +2782,24 @@ void BinaryExpr::SerializeWithCachedSizes(
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
(void) cached_has_bits;
// .milvus.proto.plan.BinaryExpr.BinaryOp op = 1;
if (this->op() != 0) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteEnum(
1, this->op(), output);
}
// .milvus.proto.plan.Expr left = 2;
if (this->has_left()) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray(
2, _Internal::left(this), output);
}
// .milvus.proto.plan.Expr right = 3;
if (this->has_right()) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray(
3, _Internal::right(this), output);
}
if (_internal_metadata_.have_unknown_fields()) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields(
_internal_metadata_.unknown_fields(), output);
@ -2696,6 +2813,26 @@ void BinaryExpr::SerializeWithCachedSizes(
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
(void) cached_has_bits;
// .milvus.proto.plan.BinaryExpr.BinaryOp op = 1;
if (this->op() != 0) {
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteEnumToArray(
1, this->op(), target);
}
// .milvus.proto.plan.Expr left = 2;
if (this->has_left()) {
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
InternalWriteMessageToArray(
2, _Internal::left(this), target);
}
// .milvus.proto.plan.Expr right = 3;
if (this->has_right()) {
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
InternalWriteMessageToArray(
3, _Internal::right(this), target);
}
if (_internal_metadata_.have_unknown_fields()) {
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray(
_internal_metadata_.unknown_fields(), target);
@ -2717,6 +2854,26 @@ size_t BinaryExpr::ByteSizeLong() const {
// Prevent compiler warnings about cached_has_bits being unused
(void) cached_has_bits;
// .milvus.proto.plan.Expr left = 2;
if (this->has_left()) {
total_size += 1 +
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(
*left_);
}
// .milvus.proto.plan.Expr right = 3;
if (this->has_right()) {
total_size += 1 +
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(
*right_);
}
// .milvus.proto.plan.BinaryExpr.BinaryOp op = 1;
if (this->op() != 0) {
total_size += 1 +
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::EnumSize(this->op());
}
int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size);
SetCachedSize(cached_size);
return total_size;
@ -2744,6 +2901,15 @@ void BinaryExpr::MergeFrom(const BinaryExpr& from) {
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
(void) cached_has_bits;
if (from.has_left()) {
mutable_left()->::milvus::proto::plan::Expr::MergeFrom(from.left());
}
if (from.has_right()) {
mutable_right()->::milvus::proto::plan::Expr::MergeFrom(from.right());
}
if (from.op() != 0) {
set_op(from.op());
}
}
void BinaryExpr::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) {
@ -2767,6 +2933,9 @@ bool BinaryExpr::IsInitialized() const {
void BinaryExpr::InternalSwap(BinaryExpr* other) {
using std::swap;
_internal_metadata_.Swap(&other->_internal_metadata_);
swap(left_, other->left_);
swap(right_, other->right_);
swap(op_, other->op_);
}
::PROTOBUF_NAMESPACE_ID::Metadata BinaryExpr::GetMetadata() const {
@ -2901,7 +3070,7 @@ Expr::Expr(const Expr& from)
}
void Expr::SharedCtor() {
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_Expr_plan_2eproto.base);
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_BinaryExpr_plan_2eproto.base);
clear_has_expr();
}
@ -2920,7 +3089,7 @@ void Expr::SetCachedSize(int size) const {
_cached_size_.Set(size);
}
const Expr& Expr::default_instance() {
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_Expr_plan_2eproto.base);
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_BinaryExpr_plan_2eproto.base);
return *internal_default_instance();
}

145
internal/core/src/pb/plan.pb.h
View File

@ -1321,11 +1321,40 @@ class BinaryExpr :
// accessors -------------------------------------------------------
enum : int {
kLeftFieldNumber = 2,
kRightFieldNumber = 3,
kOpFieldNumber = 1,
};
// .milvus.proto.plan.Expr left = 2;
bool has_left() const;
void clear_left();
const ::milvus::proto::plan::Expr& left() const;
::milvus::proto::plan::Expr* release_left();
::milvus::proto::plan::Expr* mutable_left();
void set_allocated_left(::milvus::proto::plan::Expr* left);
// .milvus.proto.plan.Expr right = 3;
bool has_right() const;
void clear_right();
const ::milvus::proto::plan::Expr& right() const;
::milvus::proto::plan::Expr* release_right();
::milvus::proto::plan::Expr* mutable_right();
void set_allocated_right(::milvus::proto::plan::Expr* right);
// .milvus.proto.plan.BinaryExpr.BinaryOp op = 1;
void clear_op();
::milvus::proto::plan::BinaryExpr_BinaryOp op() const;
void set_op(::milvus::proto::plan::BinaryExpr_BinaryOp value);
// @@protoc_insertion_point(class_scope:milvus.proto.plan.BinaryExpr)
private:
class _Internal;
::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_;
::milvus::proto::plan::Expr* left_;
::milvus::proto::plan::Expr* right_;
int op_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
friend struct ::TableStruct_plan_2eproto;
};
@ -2368,6 +2397,122 @@ inline void UnaryExpr::set_allocated_child(::milvus::proto::plan::Expr* child) {
// BinaryExpr
// .milvus.proto.plan.BinaryExpr.BinaryOp op = 1;
inline void BinaryExpr::clear_op() {
op_ = 0;
}
inline ::milvus::proto::plan::BinaryExpr_BinaryOp BinaryExpr::op() const {
// @@protoc_insertion_point(field_get:milvus.proto.plan.BinaryExpr.op)
return static_cast< ::milvus::proto::plan::BinaryExpr_BinaryOp >(op_);
}
inline void BinaryExpr::set_op(::milvus::proto::plan::BinaryExpr_BinaryOp value) {
op_ = value;
// @@protoc_insertion_point(field_set:milvus.proto.plan.BinaryExpr.op)
}
// .milvus.proto.plan.Expr left = 2;
inline bool BinaryExpr::has_left() const {
return this != internal_default_instance() && left_ != nullptr;
}
inline void BinaryExpr::clear_left() {
if (GetArenaNoVirtual() == nullptr && left_ != nullptr) {
delete left_;
}
left_ = nullptr;
}
inline const ::milvus::proto::plan::Expr& BinaryExpr::left() const {
const ::milvus::proto::plan::Expr* p = left_;
// @@protoc_insertion_point(field_get:milvus.proto.plan.BinaryExpr.left)
return p != nullptr ? *p : *reinterpret_cast<const ::milvus::proto::plan::Expr*>(
&::milvus::proto::plan::_Expr_default_instance_);
}
inline ::milvus::proto::plan::Expr* BinaryExpr::release_left() {
// @@protoc_insertion_point(field_release:milvus.proto.plan.BinaryExpr.left)
::milvus::proto::plan::Expr* temp = left_;
left_ = nullptr;
return temp;
}
inline ::milvus::proto::plan::Expr* BinaryExpr::mutable_left() {
if (left_ == nullptr) {
auto* p = CreateMaybeMessage<::milvus::proto::plan::Expr>(GetArenaNoVirtual());
left_ = p;
}
// @@protoc_insertion_point(field_mutable:milvus.proto.plan.BinaryExpr.left)
return left_;
}
inline void BinaryExpr::set_allocated_left(::milvus::proto::plan::Expr* left) {
::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaNoVirtual();
if (message_arena == nullptr) {
delete left_;
}
if (left) {
::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = nullptr;
if (message_arena != submessage_arena) {
left = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage(
message_arena, left, submessage_arena);
}
} else {
}
left_ = left;
// @@protoc_insertion_point(field_set_allocated:milvus.proto.plan.BinaryExpr.left)
}
// .milvus.proto.plan.Expr right = 3;
inline bool BinaryExpr::has_right() const {
return this != internal_default_instance() && right_ != nullptr;
}
inline void BinaryExpr::clear_right() {
if (GetArenaNoVirtual() == nullptr && right_ != nullptr) {
delete right_;
}
right_ = nullptr;
}
inline const ::milvus::proto::plan::Expr& BinaryExpr::right() const {
const ::milvus::proto::plan::Expr* p = right_;
// @@protoc_insertion_point(field_get:milvus.proto.plan.BinaryExpr.right)
return p != nullptr ? *p : *reinterpret_cast<const ::milvus::proto::plan::Expr*>(
&::milvus::proto::plan::_Expr_default_instance_);
}
inline ::milvus::proto::plan::Expr* BinaryExpr::release_right() {
// @@protoc_insertion_point(field_release:milvus.proto.plan.BinaryExpr.right)
::milvus::proto::plan::Expr* temp = right_;
right_ = nullptr;
return temp;
}
inline ::milvus::proto::plan::Expr* BinaryExpr::mutable_right() {
if (right_ == nullptr) {
auto* p = CreateMaybeMessage<::milvus::proto::plan::Expr>(GetArenaNoVirtual());
right_ = p;
}
// @@protoc_insertion_point(field_mutable:milvus.proto.plan.BinaryExpr.right)
return right_;
}
inline void BinaryExpr::set_allocated_right(::milvus::proto::plan::Expr* right) {
::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaNoVirtual();
if (message_arena == nullptr) {
delete right_;
}
if (right) {
::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena = nullptr;
if (message_arena != submessage_arena) {
right = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage(
message_arena, right, submessage_arena);
}
} else {
}
right_ = right;
// @@protoc_insertion_point(field_set_allocated:milvus.proto.plan.BinaryExpr.right)
}
// -------------------------------------------------------------------
// Expr

1
internal/core/src/query/Plan.cpp
View File

@ -484,7 +484,6 @@ Plan::check_identical(Plan& other) {
Assert(&schema_ == &other.schema_);
auto json = ShowPlanNodeVisitor().call_child(*this->plan_node_);
auto other_json = ShowPlanNodeVisitor().call_child(*other.plan_node_);
std::cout << json.dump(2) << std::endl << other_json.dump(2) << std::endl;
Assert(json.dump(2) == other_json.dump(2));
Assert(this->extra_info_opt_.has_value() == other.extra_info_opt_.has_value());
if (this->extra_info_opt_.has_value()) {

220
internal/core/src/query/PlanProto.cpp
View File

@ -14,44 +14,65 @@
#include "ExprImpl.h"
#include "pb/plan.pb.h"
#include <google/protobuf/text_format.h>
#include <query/generated/ExtractInfoPlanNodeVisitor.h>
#include "query/generated/ExtractInfoExprVisitor.h"
#include "common/Types.h"
namespace milvus::query {
namespace planpb = milvus::proto::plan;
ExprPtr
ProtoParser::ExprFromProto(const planpb::Expr& expr_proto) {
// TODO: make naive works
Assert(expr_proto.has_range_expr());
auto& range_expr = expr_proto.range_expr();
auto& columen_info = range_expr.column_info();
auto field_id = FieldId(columen_info.field_id());
auto field_offset = schema.get_offset(field_id);
auto data_type = schema[field_offset].get_data_type();
involved_fields.set(field_offset.get(), true);
// auto& field_meta = schema[field_offset];
ExprPtr result = [&]() {
switch ((DataType)columen_info.data_type()) {
case DataType::INT64: {
auto result = std::make_unique<RangeExprImpl<int64_t>>();
result->field_offset_ = field_offset;
result->data_type_ = data_type;
Assert(range_expr.ops_size() == range_expr.values_size());
auto sz = range_expr.ops_size();
// TODO simplify this
for (int i = 0; i < sz; ++i) {
result->conditions_.emplace_back((RangeExpr::OpType)range_expr.ops(i),
range_expr.values(i).int64_val());
}
return result;
}
default: {
PanicInfo("unsupported yet");
}
template <typename T>
std::unique_ptr<TermExprImpl<T>>
ExtractTermExprImpl(FieldOffset field_offset, DataType data_type, const planpb::TermExpr& expr_proto) {
static_assert(std::is_fundamental_v<T>);
auto result = std::make_unique<TermExprImpl<T>>();
result->field_offset_ = field_offset;
result->data_type_ = data_type;
auto size = expr_proto.values_size();
for (int i = 0; i < size; ++i) {
auto& value_proto = expr_proto.values(i);
if constexpr (std::is_same_v<T, bool>) {
Assert(value_proto.val_case() == planpb::GenericValue::kBoolVal);
result->terms_.emplace_back(static_cast<T>(value_proto.bool_val()));
} else if constexpr (std::is_integral_v<T>) {
Assert(value_proto.val_case() == planpb::GenericValue::kInt64Val);
result->terms_.emplace_back(static_cast<T>(value_proto.int64_val()));
} else if constexpr (std::is_floating_point_v<T>) {
Assert(value_proto.val_case() == planpb::GenericValue::kFloatVal);
result->terms_.emplace_back(static_cast<T>(value_proto.float_val()));
} else {
static_assert(always_false<T>);
}
}();
}
return result;
}
template <typename T>
std::unique_ptr<RangeExprImpl<T>>
ExtractRangeExprImpl(FieldOffset field_offset, DataType data_type, const planpb::RangeExpr& expr_proto) {
static_assert(std::is_fundamental_v<T>);
auto result = std::make_unique<RangeExprImpl<T>>();
result->field_offset_ = field_offset;
result->data_type_ = data_type;
Assert(expr_proto.ops_size() == expr_proto.values_size());
auto sz = expr_proto.ops_size();
for (int i = 0; i < sz; ++i) {
auto op = static_cast<RangeExpr::OpType>(expr_proto.ops(i));
auto& value_proto = expr_proto.values(i);
if constexpr (std::is_same_v<T, bool>) {
Assert(value_proto.val_case() == planpb::GenericValue::kBoolVal);
result->conditions_.emplace_back(op, static_cast<T>(value_proto.bool_val()));
} else if constexpr (std::is_integral_v<T>) {
Assert(value_proto.val_case() == planpb::GenericValue::kInt64Val);
result->conditions_.emplace_back(op, static_cast<T>(value_proto.int64_val()));
} else if constexpr (std::is_floating_point_v<T>) {
Assert(value_proto.val_case() == planpb::GenericValue::kFloatVal);
result->conditions_.emplace_back(op, static_cast<T>(value_proto.float_val()));
} else {
static_assert(always_false<T>);
}
}
return result;
}
@ -65,7 +86,7 @@ ProtoParser::PlanNodeFromProto(const planpb::PlanNode& plan_node_proto) {
if (!anns_proto.has_predicates()) {
return std::nullopt;
} else {
return ExprFromProto(anns_proto.predicates());
return ParseExpr(anns_proto.predicates());
}
}();
@ -75,7 +96,6 @@ ProtoParser::PlanNodeFromProto(const planpb::PlanNode& plan_node_proto) {
auto field_id = FieldId(anns_proto.field_id());
auto field_offset = schema.get_offset(field_id);
query_info.field_offset_ = field_offset;
this->involved_fields.set(field_offset.get(), true);
query_info.metric_type_ = GetMetricType(query_info_proto.metric_type());
query_info.topK_ = query_info_proto.topk();
@ -99,12 +119,138 @@ ProtoParser::CreatePlan(const proto::plan::PlanNode& plan_node_proto) {
auto plan = std::make_unique<Plan>(schema);
auto plan_node = PlanNodeFromProto(plan_node_proto);
ExtractedPlanInfo plan_info(schema.size());
ExtractInfoPlanNodeVisitor extractor(plan_info);
plan_node->accept(extractor);
plan->tag2field_["$0"] = plan_node->query_info_.field_offset_;
plan->plan_node_ = std::move(plan_node);
ExtractedPlanInfo extract_info(schema.size());
extract_info.involved_fields_ = std::move(involved_fields);
plan->extra_info_opt_ = std::move(extract_info);
plan->extra_info_opt_ = std::move(plan_info);
return plan;
}
ExprPtr
ProtoParser::ParseRangeExpr(const proto::plan::RangeExpr& expr_pb) {
auto& columen_info = expr_pb.column_info();
auto field_id = FieldId(columen_info.field_id());
auto field_offset = schema.get_offset(field_id);
auto data_type = schema[field_offset].get_data_type();
Assert(data_type == (DataType)columen_info.data_type());
// auto& field_meta = schema[field_offset];
auto result = [&]() -> ExprPtr {
switch (data_type) {
case DataType::BOOL: {
return ExtractRangeExprImpl<bool>(field_offset, data_type, expr_pb);
}
case DataType::INT8: {
return ExtractRangeExprImpl<int8_t>(field_offset, data_type, expr_pb);
}
case DataType::INT16: {
return ExtractRangeExprImpl<int16_t>(field_offset, data_type, expr_pb);
}
case DataType::INT32: {
return ExtractRangeExprImpl<int32_t>(field_offset, data_type, expr_pb);
}
case DataType::INT64: {
return ExtractRangeExprImpl<int64_t>(field_offset, data_type, expr_pb);
}
case DataType::FLOAT: {
return ExtractRangeExprImpl<float>(field_offset, data_type, expr_pb);
}
case DataType::DOUBLE: {
return ExtractRangeExprImpl<double>(field_offset, data_type, expr_pb);
}
default: {
PanicInfo("unsupported data type");
}
}
}();
return result;
}
ExprPtr
ProtoParser::ParseTermExpr(const proto::plan::TermExpr& expr_pb) {
auto& columen_info = expr_pb.column_info();
auto field_id = FieldId(columen_info.field_id());
auto field_offset = schema.get_offset(field_id);
auto data_type = schema[field_offset].get_data_type();
Assert(data_type == (DataType)columen_info.data_type());
// auto& field_meta = schema[field_offset];
auto result = [&]() -> ExprPtr {
switch (data_type) {
case DataType::BOOL: {
return ExtractTermExprImpl<bool>(field_offset, data_type, expr_pb);
}
case DataType::INT8: {
return ExtractTermExprImpl<int8_t>(field_offset, data_type, expr_pb);
}
case DataType::INT16: {
return ExtractTermExprImpl<int16_t>(field_offset, data_type, expr_pb);
}
case DataType::INT32: {
return ExtractTermExprImpl<int32_t>(field_offset, data_type, expr_pb);
}
case DataType::INT64: {
return ExtractTermExprImpl<int64_t>(field_offset, data_type, expr_pb);
}
case DataType::FLOAT: {
return ExtractTermExprImpl<float>(field_offset, data_type, expr_pb);
}
case DataType::DOUBLE: {
return ExtractTermExprImpl<double>(field_offset, data_type, expr_pb);
}
default: {
PanicInfo("unsupported data type");
}
}
}();
return result;
}
ExprPtr
ProtoParser::ParseUnaryExpr(const proto::plan::UnaryExpr& expr_pb) {
auto op = static_cast<BoolUnaryExpr::OpType>(expr_pb.op());
Assert(op == BoolUnaryExpr::OpType::LogicalNot);
auto expr = this->ParseExpr(expr_pb.child());
auto result = std::make_unique<BoolUnaryExpr>();
result->child_ = std::move(expr);
result->op_type_ = op;
return result;
}
ExprPtr
ProtoParser::ParseBinaryExpr(const proto::plan::BinaryExpr& expr_pb) {
auto op = static_cast<BoolBinaryExpr::OpType>(expr_pb.op());
auto left_expr = this->ParseExpr(expr_pb.left());
auto right_expr = this->ParseExpr(expr_pb.right());
auto result = std::make_unique<BoolBinaryExpr>();
result->op_type_ = op;
result->left_ = std::move(left_expr);
result->right_ = std::move(right_expr);
return result;
}
ExprPtr
ProtoParser::ParseExpr(const proto::plan::Expr& expr_pb) {
using ppe = proto::plan::Expr;
switch (expr_pb.expr_case()) {
case ppe::kBinaryExpr: {
return ParseBinaryExpr(expr_pb.binary_expr());
}
case ppe::kUnaryExpr: {
return ParseUnaryExpr(expr_pb.unary_expr());
}
case ppe::kTermExpr: {
return ParseTermExpr(expr_pb.term_expr());
}
case ppe::kRangeExpr: {
return ParseRangeExpr(expr_pb.range_expr());
}
default:
PanicInfo("unsupported expr proto node");
}
}
} // namespace milvus::query

21
internal/core/src/query/PlanProto.h
View File

@ -20,11 +20,26 @@ namespace milvus::query {
class ProtoParser {
public:
explicit ProtoParser(const Schema& schema) : schema(schema), involved_fields(schema.size()) {
explicit ProtoParser(const Schema& schema) : schema(schema) {
}
// ExprPtr
// ExprFromProto(const proto::plan::Expr& expr_proto);
ExprPtr
ExprFromProto(const proto::plan::Expr& expr_proto);
ParseRangeExpr(const proto::plan::RangeExpr& expr_pb);
ExprPtr
ParseTermExpr(const proto::plan::TermExpr& expr_pb);
ExprPtr
ParseUnaryExpr(const proto::plan::UnaryExpr& expr_pb);
ExprPtr
ParseBinaryExpr(const proto::plan::BinaryExpr& expr_pb);
ExprPtr
ParseExpr(const proto::plan::Expr& expr_pb);
std::unique_ptr<VectorPlanNode>
PlanNodeFromProto(const proto::plan::PlanNode& plan_node_proto);
@ -34,7 +49,7 @@ class ProtoParser {
private:
const Schema& schema;
boost::dynamic_bitset<> involved_fields;
// boost::dynamic_bitset<> involved_fields;
};
} // namespace milvus::query

1
internal/core/src/segcore/SegcoreConfig.cpp
View File

@ -96,6 +96,7 @@ SegcoreConfig::parse_from(const std::string& config_path) {
result.table_[metric_type] = conf;
}
}
return result;
} catch (const SegcoreError& e) {
// re-throw
throw e;

402
internal/core/unittest/test_plan_proto.cpp
View File

@ -17,27 +17,77 @@
#include <vector>
#include <queue>
#include <random>
#include <boost/format.hpp>
using namespace milvus;
using namespace milvus::query;
namespace planpb = proto::plan;
using std::string;
TEST(PlanProto, Naive) {
auto schema = std::make_unique<Schema>();
schema->AddField(FieldName("vectorfield"), FieldId(101), DataType::VECTOR_FLOAT, 16, MetricType::METRIC_L2);
schema->AddField(FieldName("int64field"), FieldId(100), DataType::INT64);
std::string proto_text = R"(
namespace spb = proto::schema;
static SchemaPtr
getStandardSchema() {
auto schema = std::make_shared<Schema>();
schema->AddField(FieldName("FloatVectorField"), FieldId(100 + spb::DataType::FloatVector), DataType::VECTOR_FLOAT,
16, MetricType::METRIC_L2);
schema->AddField(FieldName("BinaryVectorField"), FieldId(100 + spb::DataType::BinaryVector),
DataType::VECTOR_BINARY, 16, MetricType::METRIC_Jaccard);
schema->AddField(FieldName("Int64Field"), FieldId(100 + spb::DataType::Int64), DataType::INT64);
schema->AddField(FieldName("Int32Field"), FieldId(100 + spb::DataType::Int32), DataType::INT32);
schema->AddField(FieldName("Int16Field"), FieldId(100 + spb::DataType::Int16), DataType::INT16);
schema->AddField(FieldName("Int8Field"), FieldId(100 + spb::DataType::Int8), DataType::INT8);
schema->AddField(FieldName("DoubleField"), FieldId(100 + spb::DataType::Double), DataType::DOUBLE);
schema->AddField(FieldName("FloatField"), FieldId(100 + spb::DataType::Float), DataType::FLOAT);
return schema;
}
class PlanProtoTest : public ::testing::TestWithParam<std::tuple<spb::DataType>> {
public:
PlanProtoTest() {
schema = getStandardSchema();
}
protected:
SchemaPtr schema;
};
INSTANTIATE_TEST_CASE_P(InstName,
PlanProtoTest,
::testing::Values( //
std::make_tuple(spb::DataType::Double), //
std::make_tuple(spb::DataType::Float), //
std::make_tuple(spb::DataType::Int64), //
std::make_tuple(spb::DataType::Int32), //
std::make_tuple(spb::DataType::Int16), //
std::make_tuple(spb::DataType::Int8) //
));
TEST_P(PlanProtoTest, Range) {
// xxx.query(predicates = "int64field > 3", topk = 10, ...)
// xxx.query(predicates = "int64field > 3", topk = 10, ...)
auto data_type = std::get<0>(GetParam());
auto data_type_str = spb::DataType_Name(data_type);
auto field_id = 100 + (int)data_type;
auto field_name = data_type_str + "Field";
string value_tag = "bool_val";
if (datatype_is_floating((DataType)data_type)) {
value_tag = "float_val";
} else if (datatype_is_interger((DataType)data_type)) {
value_tag = "int64_val";
}
auto fmt1 = boost::format(R"(
vector_anns: <
field_id: 101
field_id: 201
predicates: <
range_expr: <
column_info: <
field_id: 100
data_type: Int64
field_id: %1%
data_type: %2%
>
ops: GreaterThan
values: <
int64_val: 3
%3%: 3
>
>
>
@ -48,31 +98,34 @@ vector_anns: <
>
placeholder_tag: "$0"
>
)";
)") % field_id % data_type_str %
value_tag;
auto proto_text = fmt1.str();
planpb::PlanNode node_proto;
google::protobuf::TextFormat::ParseFromString(proto_text, &node_proto);
std::cout << node_proto.DebugString();
// std::cout << node_proto.DebugString();
auto plan = ProtoParser(*schema).CreatePlan(node_proto);
ShowPlanNodeVisitor visitor;
auto json = visitor.call_child(*plan->plan_node_);
std::cout << json.dump(2);
// std::cout << json.dump(2);
auto extra_info = plan->extra_info_opt_.value();
std::string dsl_text = R"(
std::string dsl_text = boost::str(boost::format(R"(
{
"bool": {
"must": [
{
"range": {
"int64field": {
"%1%": {
"GT": 3
}
}
},
{
"vector": {
"vectorfield": {
"FloatVectorField": {
"metric_type": "L2",
"params": {
"nprobe": 10
@ -85,7 +138,324 @@ vector_anns: <
]
}
}
)";
)") % field_name);
auto ref_plan = CreatePlan(*schema, dsl_text);
plan->check_identical(*ref_plan);
}
TEST_P(PlanProtoTest, TermExpr) {
// xxx.query(predicates = "int64field > 3", topk = 10, ...)
// xxx.query(predicates = "int64field > 3", topk = 10, ...)
auto data_type = std::get<0>(GetParam());
auto data_type_str = spb::DataType_Name(data_type);
auto field_id = 100 + (int)data_type;
auto field_name = data_type_str + "Field";
string value_tag = "bool_val";
if (datatype_is_floating((DataType)data_type)) {
value_tag = "float_val";
} else if (datatype_is_interger((DataType)data_type)) {
value_tag = "int64_val";
}
auto fmt1 = boost::format(R"(
vector_anns: <
field_id: 201
predicates: <
term_expr: <
column_info: <
field_id: %1%
data_type: %2%
>
values: <
%3%: 1
>
values: <
%3%: 2
>
values: <
%3%: 3
>
>
>
query_info: <
topk: 10
metric_type: "L2"
search_params: "{\"nprobe\": 10}"
>
placeholder_tag: "$0"
>
)") % field_id % data_type_str %
value_tag;
auto proto_text = fmt1.str();
planpb::PlanNode node_proto;
google::protobuf::TextFormat::ParseFromString(proto_text, &node_proto);
// std::cout << node_proto.DebugString();
auto plan = ProtoParser(*schema).CreatePlan(node_proto);
ShowPlanNodeVisitor visitor;
auto json = visitor.call_child(*plan->plan_node_);
// std::cout << json.dump(2);
auto extra_info = plan->extra_info_opt_.value();
std::string dsl_text = boost::str(boost::format(R"(
{
"bool": {
"must": [
{
"term": {
"%1%": {
"values": [1,2,3]
}
}
},
{
"vector": {
"FloatVectorField": {
"metric_type": "L2",
"params": {
"nprobe": 10
},
"query": "$0",
"topk": 10
}
}
}
]
}
}
)") % field_name);
auto ref_plan = CreatePlan(*schema, dsl_text);
plan->check_identical(*ref_plan);
}
TEST(PlanProtoXTest, NotExpr) {
auto schema = getStandardSchema();
// xxx.query(predicates = "int64field > 3", topk = 10, ...)
// xxx.query(predicates = "int64field > 3", topk = 10, ...)
auto data_type = spb::DataType::Int64;
auto data_type_str = spb::DataType_Name(data_type);
auto field_id = 100 + (int)data_type;
auto field_name = data_type_str + "Field";
string value_tag = "bool_val";
if (datatype_is_floating((DataType)data_type)) {
value_tag = "float_val";
} else if (datatype_is_interger((DataType)data_type)) {
value_tag = "int64_val";
}
auto fmt1 = boost::format(R"(
vector_anns: <
field_id: 201
predicates: <
unary_expr: <
op: Not
child: <
range_expr: <
column_info: <
field_id: %1%
data_type: %2%
>
ops: GreaterThan
values: <
%3%: 3
>
>
>
>
>
query_info: <
topk: 10
metric_type: "L2"
search_params: "{\"nprobe\": 10}"
>
placeholder_tag: "$0"
>
)") % field_id % data_type_str %
value_tag;
auto proto_text = fmt1.str();
planpb::PlanNode node_proto;
google::protobuf::TextFormat::ParseFromString(proto_text, &node_proto);
// std::cout << node_proto.DebugString();
auto plan = ProtoParser(*schema).CreatePlan(node_proto);
ShowPlanNodeVisitor visitor;
auto json = visitor.call_child(*plan->plan_node_);
// std::cout << json.dump(2);
auto extra_info = plan->extra_info_opt_.value();
std::string dsl_text = boost::str(boost::format(R"(
{
"bool": {
"must": [
{
"should_not": [{
"range": {
"%1%": {
"GT": 3
}
}
}]
},
{
"vector": {
"FloatVectorField": {
"metric_type": "L2",
"params": {
"nprobe": 10
},
"query": "$0",
"topk": 10
}
}
}
]
}
}
)") % field_name);
auto ref_plan = CreatePlan(*schema, dsl_text);
auto ref_json = ShowPlanNodeVisitor().call_child(*ref_plan->plan_node_);
EXPECT_EQ(json.dump(2), ref_json.dump(2));
plan->check_identical(*ref_plan);
}
TEST(PlanProtoXTest, AndOrExpr) {
auto schema = getStandardSchema();
// xxx.query(predicates = "int64field > 3", topk = 10, ...)
// xxx.query(predicates = "int64field > 3", topk = 10, ...)
auto data_type = spb::DataType::Int64;
auto data_type_str = spb::DataType_Name(data_type);
auto field_id = 100 + (int)data_type;
auto field_name = data_type_str + "Field";
string value_tag = "bool_val";
if (datatype_is_floating((DataType)data_type)) {
value_tag = "float_val";
} else if (datatype_is_interger((DataType)data_type)) {
value_tag = "int64_val";
}
auto fmt1 = boost::format(R"(
vector_anns: <
field_id: 201
predicates: <
binary_expr: <
op: LogicalAnd
left: <
range_expr: <
column_info: <
field_id: 105
data_type: Int64
>
ops: LessThan
values: <
int64_val: 3
>
>
>
right: <
binary_expr: <
op: LogicalOr
left: <
range_expr: <
column_info: <
field_id: 105
data_type: Int64
>
ops: GreaterThan
values: <
int64_val: 2
>
>
>
right: <
range_expr: <
column_info: <
field_id: 105
data_type: Int64
>
ops: Equal
values: <
int64_val: 1
>
>
>
>
>
>
>
query_info: <
topk: 10
metric_type: "L2"
search_params: "{\"nprobe\": 10}"
>
placeholder_tag: "$0"
>
)");
auto proto_text = fmt1.str();
planpb::PlanNode node_proto;
google::protobuf::TextFormat::ParseFromString(proto_text, &node_proto);
// std::cout << node_proto.DebugString();
auto plan = ProtoParser(*schema).CreatePlan(node_proto);
ShowPlanNodeVisitor visitor;
auto json = visitor.call_child(*plan->plan_node_);
// std::cout << json.dump(2);
auto extra_info = plan->extra_info_opt_.value();
std::string dsl_text = boost::str(boost::format(R"(
{
"bool": {
"must": [
{
"range": {
"%1%": {
"LT": 3
}
}
},
{
"should": [
{
"range": {
"%1%": {
"GT": 2
}
}
},
{
"range": {
"%1%": {
"EQ": 1
}
}
}
]
},
{
"vector": {
"FloatVectorField": {
"metric_type": "L2",
"params": {
"nprobe": 10
},
"query": "$0",
"topk": 10
}
}
}
]
}
}
)") % field_name);
auto ref_plan = CreatePlan(*schema, dsl_text);
auto ref_json = ShowPlanNodeVisitor().call_child(*ref_plan->plan_node_);
EXPECT_EQ(json.dump(2), ref_json.dump(2));
plan->check_identical(*ref_plan);
}

3
internal/proto/plan.proto
View File

@ -59,6 +59,9 @@ message BinaryExpr {
LogicalAnd = 1;
LogicalOr = 2;
}
BinaryOp op = 1;
Expr left = 2;
Expr right = 3;
}
message Expr {

134
internal/proto/planpb/plan.pb.go
View File

@ -461,9 +461,12 @@ func (m *UnaryExpr) GetChild() *Expr {
}
type BinaryExpr struct {
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
Op BinaryExpr_BinaryOp `protobuf:"varint,1,opt,name=op,proto3,enum=milvus.proto.plan.BinaryExpr_BinaryOp" json:"op,omitempty"`
Left *Expr `protobuf:"bytes,2,opt,name=left,proto3" json:"left,omitempty"`
Right *Expr `protobuf:"bytes,3,opt,name=right,proto3" json:"right,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *BinaryExpr) Reset() { *m = BinaryExpr{} }
@ -491,6 +494,27 @@ func (m *BinaryExpr) XXX_DiscardUnknown() {
var xxx_messageInfo_BinaryExpr proto.InternalMessageInfo
func (m *BinaryExpr) GetOp() BinaryExpr_BinaryOp {
if m != nil {
return m.Op
}
return BinaryExpr_Invalid
}
func (m *BinaryExpr) GetLeft() *Expr {
if m != nil {
return m.Left
}
return nil
}
func (m *BinaryExpr) GetRight() *Expr {
if m != nil {
return m.Right
}
return nil
}
type Expr struct {
// Types that are valid to be assigned to Expr:
// *Expr_RangeExpr
@ -756,55 +780,57 @@ func init() {
func init() { proto.RegisterFile("plan.proto", fileDescriptor_2d655ab2f7683c23) }
var fileDescriptor_2d655ab2f7683c23 = []byte{
// 792 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x54, 0x4f, 0x6f, 0xdb, 0x36,
0x14, 0xb7, 0x24, 0xff, 0x91, 0x9e, 0x5d, 0xd7, 0xe3, 0x65, 0xde, 0xb2, 0x22, 0x86, 0x3a, 0x60,
0xbe, 0xd4, 0xc1, 0xdc, 0xae, 0x05, 0x3a, 0x74, 0x58, 0xb2, 0x16, 0x4d, 0x80, 0xc2, 0xe9, 0x34,
0x2f, 0x87, 0x5d, 0x04, 0x5a, 0x62, 0x6c, 0x62, 0x34, 0xc9, 0x50, 0x94, 0xd1, 0x9c, 0x77, 0xdb,
0x6d, 0x9f, 0x63, 0x1f, 0x6b, 0x5f, 0x64, 0x20, 0xa9, 0xc8, 0x71, 0xe0, 0x04, 0x18, 0xb0, 0x1b,
0xdf, 0x9f, 0xdf, 0xfb, 0x91, 0xbf, 0xf7, 0xf8, 0x00, 0x24, 0xc3, 0x7c, 0x22, 0x95, 0xd0, 0x02,
0x7d, 0xb6, 0xa6, 0x6c, 0x53, 0x16, 0xce, 0x9a, 0x98, 0xc0, 0x97, 0xbd, 0x22, 0x5b, 0x91, 0x35,
0x76, 0xae, 0x58, 0x42, 0xef, 0x3d, 0xe1, 0x44, 0xd1, 0xec, 0x02, 0xb3, 0x92, 0xa0, 0x03, 0x08,
0x17, 0x42, 0xb0, 0x74, 0x83, 0xd9, 0xd0, 0x1b, 0x79, 0xe3, 0xf0, 0xb4, 0x91, 0x74, 0x8c, 0xe7,
0x02, 0x33, 0xf4, 0x04, 0x22, 0xca, 0xf5, 0xcb, 0x17, 0x36, 0xea, 0x8f, 0xbc, 0x71, 0x70, 0xda,
0x48, 0x42, 0xeb, 0xaa, 0xc2, 0x97, 0x4c, 0x60, 0x6d, 0xc3, 0xc1, 0xc8, 0x1b, 0x7b, 0x26, 0x6c,
0x5d, 0x17, 0x98, 0x9d, 0xb4, 0x20, 0xd8, 0x60, 0x16, 0x13, 0x88, 0x7e, 0x2e, 0x89, 0xba, 0x3e,
0xe3, 0x97, 0x02, 0x21, 0x68, 0x6a, 0x21, 0x7f, 0xb7, 0x54, 0x41, 0x62, 0xcf, 0xe8, 0x10, 0xba,
0x6b, 0xa2, 0x15, 0xcd, 0x52, 0x7d, 0x2d, 0x89, 0x2d, 0x14, 0x25, 0xe0, 0x5c, 0xf3, 0x6b, 0x49,
0xd0, 0x53, 0x78, 0x54, 0x10, 0xac, 0xb2, 0x55, 0x2a, 0xb1, 0xc2, 0xeb, 0x62, 0xd8, 0xb4, 0x29,
0x3d, 0xe7, 0xfc, 0x68, 0x7d, 0x71, 0x06, 0xf0, 0x93, 0x60, 0xe5, 0x9a, 0x5b, 0x9e, 0x2f, 0x20,
0xbc, 0xa4, 0x84, 0xe5, 0x29, 0xcd, 0x2b, 0xae, 0x8e, 0xb5, 0xcf, 0x72, 0xf4, 0x1a, 0xa2, 0x1c,
0x6b, 0xec, 0xc8, 0xcc, 0xa3, 0xfa, 0xd3, 0x27, 0x93, 0x1d, 0xd9, 0x2a, 0xc1, 0xde, 0x62, 0x8d,
0x0d, 0x7f, 0x12, 0xe6, 0xd5, 0x29, 0xfe, 0xdb, 0x87, 0x28, 0xc1, 0x7c, 0x49, 0xde, 0x7d, 0x92,
0x0a, 0xfd, 0x00, 0xdd, 0xcc, 0x52, 0xa6, 0x94, 0x5f, 0x0a, 0xcb, 0xd3, 0xbd, 0x5b, 0xcb, 0xf6,
0x66, 0x7b, 0xb1, 0x04, 0xb2, 0xed, 0x25, 0xbf, 0x83, 0x40, 0xc8, 0x62, 0xe8, 0x8f, 0x82, 0x71,
0x7f, 0xfa, 0x74, 0x0f, 0xae, 0xa6, 0x9a, 0x9c, 0x4b, 0x7b, 0x13, 0x93, 0x8f, 0x5e, 0x41, 0x7b,
0x63, 0x7a, 0x57, 0x0c, 0x83, 0x51, 0x30, 0xee, 0x4e, 0x0f, 0xf7, 0x20, 0x6f, 0xf7, 0x38, 0xa9,
0xd2, 0x63, 0x0e, 0x6d, 0x57, 0x07, 0x75, 0xa1, 0x73, 0xc6, 0x37, 0x98, 0xd1, 0x7c, 0xd0, 0x40,
0x8f, 0xa1, 0xfb, 0x5e, 0x11, 0xac, 0x89, 0x9a, 0xaf, 0x30, 0x1f, 0x78, 0x68, 0x00, 0xbd, 0xca,
0xf1, 0xee, 0xaa, 0xc4, 0x6c, 0xe0, 0xa3, 0x1e, 0x84, 0x1f, 0x48, 0x51, 0xd8, 0x78, 0x80, 0x1e,
0x41, 0x64, 0x2c, 0x17, 0x6c, 0xa2, 0x08, 0x5a, 0xee, 0xd8, 0x32, 0x79, 0x33, 0xa1, 0x9d, 0xd5,
0x8e, 0xff, 0xf0, 0x20, 0x9c, 0x13, 0xb5, 0xfe, 0x5f, 0xc4, 0xda, 0xbe, 0xda, 0xff, 0x6f, 0xaf,
0xfe, 0xcb, 0x83, 0xe8, 0x57, 0x8e, 0xd5, 0xb5, 0xbd, 0xc6, 0x0b, 0xf0, 0x85, 0xb4, 0xec, 0xfd,
0xe9, 0xd7, 0x7b, 0x4a, 0xd4, 0x99, 0xee, 0x74, 0x2e, 0x13, 0x5f, 0x48, 0xf4, 0x0c, 0x5a, 0xd9,
0x8a, 0xb2, 0xdc, 0xce, 0x4b, 0x77, 0xfa, 0xf9, 0x1e, 0xa0, 0xc1, 0x24, 0x2e, 0x2b, 0x3e, 0x84,
0x4e, 0x85, 0xde, 0x55, 0xba, 0x03, 0xc1, 0x4c, 0xe8, 0x81, 0x17, 0xbf, 0x05, 0x38, 0xa1, 0x37,
0x4c, 0xf1, 0x4b, 0x08, 0x9d, 0x75, 0x37, 0xbf, 0x0f, 0xf0, 0x41, 0x2c, 0x69, 0x86, 0xd9, 0x31,
0xcf, 0x07, 0x9e, 0x15, 0xde, 0xd9, 0xe7, 0x6a, 0xe0, 0xc7, 0x7f, 0xfa, 0xd0, 0xb4, 0x8f, 0x7a,
0x03, 0xa0, 0xcc, 0xa8, 0xa4, 0xe4, 0x93, 0x54, 0x95, 0xb4, 0x5f, 0x3d, 0x34, 0x4f, 0xa7, 0x8d,
0x24, 0x52, 0xf5, 0x1c, 0xbf, 0x86, 0x48, 0x13, 0xb5, 0x76, 0x68, 0xf7, 0xc2, 0x83, 0x3d, 0xe8,
0x9b, 0x56, 0x9a, 0x4f, 0xae, 0x6f, 0xda, 0xfa, 0x06, 0xa0, 0x34, 0x57, 0x77, 0xe0, 0xe0, 0x5e,
0xea, 0x5a, 0x57, 0x43, 0x5d, 0xd6, 0xed, 0xf8, 0x11, 0xba, 0x0b, 0xba, 0xc5, 0x37, 0xef, 0x9d,
0x8a, 0xad, 0x5c, 0xa7, 0x8d, 0x04, 0x16, 0xb5, 0x75, 0xd2, 0x86, 0xa6, 0x81, 0xc6, 0xff, 0x78,
0x00, 0x17, 0x24, 0xd3, 0x42, 0x1d, 0xcf, 0x66, 0xbf, 0xa0, 0x03, 0x88, 0x68, 0x91, 0xba, 0x3c,
0xb7, 0xd8, 0x92, 0x90, 0x16, 0xae, 0xca, 0xce, 0x76, 0xf0, 0x77, 0xb7, 0xc3, 0x2b, 0x00, 0xa9,
0x48, 0x4e, 0x33, 0xac, 0xed, 0x07, 0x7b, 0xb0, 0xdd, 0xb7, 0x52, 0xd1, 0xf7, 0x00, 0x57, 0x66,
0xcd, 0xb9, 0xf1, 0x6e, 0xde, 0x2b, 0x44, 0xbd, 0x0b, 0x93, 0xe8, 0xaa, 0x5e, 0x8b, 0xdf, 0xc0,
0x63, 0xc9, 0x70, 0x46, 0x56, 0x82, 0xe5, 0x44, 0xa5, 0x1a, 0x2f, 0x87, 0x2d, 0xbb, 0xe3, 0xfa,
0xb7, 0xdc, 0x73, 0xbc, 0x8c, 0xe7, 0x10, 0x7e, 0x64, 0x98, 0xcf, 0x44, 0x4e, 0x8c, 0x76, 0x1b,
0xfb, 0xe0, 0x14, 0x73, 0x5e, 0x3c, 0xf0, 0xa3, 0xb6, 0xb2, 0x18, 0xed, 0x1c, 0xe6, 0x98, 0xf3,
0xc2, 0x68, 0xc7, 0x45, 0x4e, 0x4e, 0x9e, 0xff, 0xf6, 0xed, 0x92, 0xea, 0x55, 0xb9, 0x98, 0x64,
0x62, 0x7d, 0xe4, 0x0a, 0x3c, 0xa3, 0xa2, 0x3a, 0x1d, 0x51, 0xae, 0x89, 0xe2, 0x98, 0x1d, 0xd9,
0x9a, 0x47, 0xa6, 0xa6, 0x5c, 0x2c, 0xda, 0xd6, 0x7a, 0xfe, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff,
0x33, 0x17, 0x7b, 0x52, 0x7f, 0x06, 0x00, 0x00,
// 829 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x54, 0x4f, 0x6f, 0x23, 0x35,
0x14, 0xcf, 0xcc, 0xe4, 0xcf, 0xcc, 0x4b, 0x36, 0x1b, 0x7c, 0x21, 0x50, 0x56, 0x8d, 0x66, 0x11,
0x44, 0x42, 0x9b, 0x8a, 0xec, 0xd2, 0x95, 0x16, 0x2d, 0xa2, 0x85, 0xd5, 0xb6, 0xd2, 0x2a, 0x5d,
0x86, 0xd0, 0x03, 0x97, 0x91, 0x33, 0xe3, 0x24, 0x16, 0x8e, 0xed, 0x7a, 0x9c, 0xa8, 0x3d, 0x73,
0xe3, 0xc6, 0xe7, 0xe0, 0x0b, 0x71, 0xe7, 0x8b, 0x20, 0xdb, 0xd3, 0x49, 0x83, 0xd2, 0x22, 0x24,
0x6e, 0x7e, 0x7f, 0x7e, 0xef, 0xf7, 0xfe, 0xf9, 0x01, 0x48, 0x86, 0xf9, 0x48, 0x2a, 0xa1, 0x05,
0xfa, 0x60, 0x45, 0xd9, 0x66, 0x5d, 0x38, 0x69, 0x64, 0x0c, 0x1f, 0x77, 0x8a, 0x6c, 0x49, 0x56,
0xd8, 0xa9, 0x62, 0x09, 0x9d, 0xb7, 0x84, 0x13, 0x45, 0xb3, 0x4b, 0xcc, 0xd6, 0x04, 0x1d, 0x40,
0x38, 0x13, 0x82, 0xa5, 0x1b, 0xcc, 0xfa, 0xde, 0xc0, 0x1b, 0x86, 0x67, 0xb5, 0xa4, 0x65, 0x34,
0x97, 0x98, 0xa1, 0x27, 0x10, 0x51, 0xae, 0x8f, 0x5f, 0x58, 0xab, 0x3f, 0xf0, 0x86, 0xc1, 0x59,
0x2d, 0x09, 0xad, 0xaa, 0x34, 0xcf, 0x99, 0xc0, 0xda, 0x9a, 0x83, 0x81, 0x37, 0xf4, 0x8c, 0xd9,
0xaa, 0x2e, 0x31, 0x3b, 0x6d, 0x40, 0xb0, 0xc1, 0x2c, 0x26, 0x10, 0xfd, 0xb0, 0x26, 0xea, 0xe6,
0x9c, 0xcf, 0x05, 0x42, 0x50, 0xd7, 0x42, 0xfe, 0x62, 0xa9, 0x82, 0xc4, 0xbe, 0xd1, 0x21, 0xb4,
0x57, 0x44, 0x2b, 0x9a, 0xa5, 0xfa, 0x46, 0x12, 0x1b, 0x28, 0x4a, 0xc0, 0xa9, 0xa6, 0x37, 0x92,
0xa0, 0xa7, 0xf0, 0xa8, 0x20, 0x58, 0x65, 0xcb, 0x54, 0x62, 0x85, 0x57, 0x45, 0xbf, 0x6e, 0x5d,
0x3a, 0x4e, 0xf9, 0xde, 0xea, 0xe2, 0x0c, 0xe0, 0x3b, 0xc1, 0xd6, 0x2b, 0x6e, 0x79, 0x3e, 0x82,
0x70, 0x4e, 0x09, 0xcb, 0x53, 0x9a, 0x97, 0x5c, 0x2d, 0x2b, 0x9f, 0xe7, 0xe8, 0x15, 0x44, 0x39,
0xd6, 0xd8, 0x91, 0x99, 0xa2, 0xba, 0xe3, 0x27, 0xa3, 0x9d, 0xb6, 0x95, 0x0d, 0xfb, 0x1e, 0x6b,
0x6c, 0xf8, 0x93, 0x30, 0x2f, 0x5f, 0xf1, 0x1f, 0x3e, 0x44, 0x09, 0xe6, 0x0b, 0xf2, 0xe6, 0x5a,
0x2a, 0xf4, 0x0d, 0xb4, 0x33, 0x4b, 0x99, 0x52, 0x3e, 0x17, 0x96, 0xa7, 0xfd, 0xcf, 0x58, 0x76,
0x36, 0xdb, 0xc4, 0x12, 0xc8, 0xb6, 0x49, 0x7e, 0x05, 0x81, 0x90, 0x45, 0xdf, 0x1f, 0x04, 0xc3,
0xee, 0xf8, 0xe9, 0x1e, 0x5c, 0x45, 0x35, 0xba, 0x90, 0x36, 0x13, 0xe3, 0x8f, 0x5e, 0x42, 0x73,
0x63, 0x66, 0x57, 0xf4, 0x83, 0x41, 0x30, 0x6c, 0x8f, 0x0f, 0xf7, 0x20, 0xef, 0xce, 0x38, 0x29,
0xdd, 0x63, 0x0e, 0x4d, 0x17, 0x07, 0xb5, 0xa1, 0x75, 0xce, 0x37, 0x98, 0xd1, 0xbc, 0x57, 0x43,
0x8f, 0xa1, 0xfd, 0x56, 0x11, 0xac, 0x89, 0x9a, 0x2e, 0x31, 0xef, 0x79, 0xa8, 0x07, 0x9d, 0x52,
0xf1, 0xe6, 0x6a, 0x8d, 0x59, 0xcf, 0x47, 0x1d, 0x08, 0xdf, 0x91, 0xa2, 0xb0, 0xf6, 0x00, 0x3d,
0x82, 0xc8, 0x48, 0xce, 0x58, 0x47, 0x11, 0x34, 0xdc, 0xb3, 0x61, 0xfc, 0x26, 0x42, 0x3b, 0xa9,
0x19, 0xff, 0xea, 0x41, 0x38, 0x25, 0x6a, 0xf5, 0xbf, 0x34, 0x6b, 0x5b, 0xb5, 0xff, 0xdf, 0xaa,
0xfe, 0xdd, 0x83, 0xe8, 0x27, 0x8e, 0xd5, 0x8d, 0x4d, 0xe3, 0x05, 0xf8, 0x42, 0x5a, 0xf6, 0xee,
0xf8, 0xd3, 0x3d, 0x21, 0x2a, 0x4f, 0xf7, 0xba, 0x90, 0x89, 0x2f, 0x24, 0x7a, 0x06, 0x8d, 0x6c,
0x49, 0x59, 0x6e, 0xf7, 0xa5, 0x3d, 0xfe, 0x70, 0x0f, 0xd0, 0x60, 0x12, 0xe7, 0x15, 0x1f, 0x42,
0xab, 0x44, 0xef, 0x76, 0xba, 0x05, 0xc1, 0x44, 0xe8, 0x9e, 0x17, 0xff, 0xe9, 0x01, 0x9c, 0xd2,
0x2a, 0xa9, 0xe3, 0x3b, 0x49, 0x7d, 0xb6, 0x27, 0xf6, 0xd6, 0xb5, 0x7c, 0x96, 0x69, 0x7d, 0x01,
0x75, 0x46, 0xe6, 0xfa, 0xdf, 0xb2, 0xb2, 0x4e, 0xa6, 0x06, 0x45, 0x17, 0x4b, 0x6d, 0x3f, 0xd8,
0x43, 0x35, 0x58, 0xaf, 0xf8, 0x18, 0xc2, 0x5b, 0xae, 0xdd, 0x22, 0xba, 0x00, 0xef, 0xc4, 0x82,
0x66, 0x98, 0x9d, 0xf0, 0xbc, 0xe7, 0xd9, 0x6d, 0x70, 0xf2, 0x85, 0xea, 0xf9, 0xf1, 0x6f, 0x3e,
0xd4, 0x6d, 0x51, 0xaf, 0x01, 0x94, 0xd9, 0xdf, 0x94, 0x5c, 0x4b, 0x55, 0xce, 0xfb, 0x93, 0x87,
0x96, 0xfc, 0xac, 0x96, 0x44, 0xaa, 0xfa, 0x5c, 0xaf, 0x20, 0xd2, 0x44, 0xad, 0x1c, 0xda, 0x15,
0x78, 0xb0, 0x07, 0x7d, 0xbb, 0x5f, 0xe6, 0xf2, 0xe8, 0xdb, 0x5d, 0x7b, 0x0d, 0xb0, 0x36, 0xa9,
0x3b, 0x70, 0x70, 0x2f, 0x75, 0x35, 0x6c, 0x43, 0xbd, 0xae, 0xc6, 0xf1, 0x2d, 0xb4, 0x67, 0x74,
0x8b, 0xaf, 0xdf, 0xbb, 0xaa, 0xdb, 0xb9, 0x9c, 0xd5, 0x12, 0x98, 0x55, 0xd2, 0x69, 0x13, 0xea,
0x06, 0x1a, 0xff, 0xe5, 0x01, 0x5c, 0x92, 0x4c, 0x0b, 0x75, 0x32, 0x99, 0xfc, 0x88, 0x0e, 0x20,
0xa2, 0x45, 0xea, 0xfc, 0xdc, 0xb5, 0x4d, 0x42, 0x5a, 0xb8, 0x28, 0x3b, 0x27, 0xcb, 0xdf, 0x3d,
0x59, 0x2f, 0x01, 0xa4, 0x22, 0x39, 0xcd, 0xb0, 0xb6, 0xbf, 0xfe, 0xc1, 0xf9, 0xdd, 0x71, 0x45,
0x5f, 0x03, 0x5c, 0x99, 0xdb, 0xeb, 0xfe, 0x5c, 0xfd, 0xde, 0x46, 0x54, 0x07, 0x3a, 0x89, 0xae,
0xaa, 0x5b, 0xfd, 0x39, 0x3c, 0x96, 0x0c, 0x67, 0x64, 0x29, 0x58, 0x4e, 0x54, 0xaa, 0xf1, 0xa2,
0xdf, 0xb0, 0x87, 0xb7, 0x7b, 0x47, 0x3d, 0xc5, 0x8b, 0x78, 0x0a, 0xe1, 0x7b, 0x86, 0xf9, 0x44,
0xe4, 0xc4, 0xf4, 0x6e, 0x63, 0x0b, 0x4e, 0x31, 0xe7, 0xc5, 0x03, 0xdf, 0x7c, 0xdb, 0x16, 0xd3,
0x3b, 0x87, 0x39, 0xe1, 0xbc, 0x30, 0xbd, 0xe3, 0x22, 0x27, 0xa7, 0xcf, 0x7f, 0xfe, 0x72, 0x41,
0xf5, 0x72, 0x3d, 0x1b, 0x65, 0x62, 0x75, 0xe4, 0x02, 0x3c, 0xa3, 0xa2, 0x7c, 0x1d, 0x51, 0xae,
0x89, 0xe2, 0x98, 0x1d, 0xd9, 0x98, 0x47, 0x26, 0xa6, 0x9c, 0xcd, 0x9a, 0x56, 0x7a, 0xfe, 0x77,
0x00, 0x00, 0x00, 0xff, 0xff, 0x17, 0x9f, 0xfa, 0x86, 0x14, 0x07, 0x00, 0x00,
}

183
internal/proxynode/plan_parser.go
View File

@ -1,3 +1,14 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
package proxynode
import (
@ -11,46 +22,6 @@ import (
"github.com/milvus-io/milvus/internal/util/typeutil"
)
//func parseQueryExprNaive(schema *typeutil.SchemaHelper, exprStr string) (*planpb.Expr, error) {
// // TODO: handle more cases
// // TODO: currently just A > 3
//
// tmps := strings.Split(exprStr, ">")
// if len(tmps) != 2 {
// return nil, errors.New("unsupported yet")
// }
// for i, str := range tmps {
// tmps[i] = strings.TrimSpace(str)
// }
// fieldName := tmps[0]
// tmpValue, err := strconv.ParseInt(tmps[1], 10, 64)
// if err != nil {
// return nil, err
// }
// fieldValue := &planpb.GenericValue{
// Val: &planpb.GenericValue_Int64Val{Int64Val: tmpValue},
// }
//
// field, err := schema.GetFieldFromName(fieldName)
// if err != nil {
// return nil, err
// }
//
// expr := &planpb.Expr{
// Expr: &planpb.Expr_RangeExpr{
// RangeExpr: &planpb.RangeExpr{
// ColumnInfo: &planpb.ColumnInfo{
// FieldId: field.FieldID,
// DataType: field.DataType,
// },
// Ops: []planpb.RangeExpr_OpType{planpb.RangeExpr_GreaterThan},
// Values: []*planpb.GenericValue{fieldValue},
// },
// },
// }
// return expr, nil
//}
func parseQueryExpr(schema *typeutil.SchemaHelper, exprStrNullable *string) (*planpb.Expr, error) {
if exprStrNullable == nil {
return nil, nil
@ -70,7 +41,7 @@ func parseQueryExprAdvanced(schema *typeutil.SchemaHelper, exprStr string) (*pla
}
context := ParserContext{schema}
return context.parseExpr(&ast.Node)
return context.handleExpr(&ast.Node)
}
func (context *ParserContext) createColumnInfo(field *schemapb.FieldSchema) *planpb.ColumnInfo {
@ -80,9 +51,10 @@ func (context *ParserContext) createColumnInfo(field *schemapb.FieldSchema) *pla
}
}
func createSingleOps(opStr string, reverse bool) planpb.RangeExpr_OpType {
func getCompareOpType(opStr string, reverse bool) planpb.RangeExpr_OpType {
type OpType = planpb.RangeExpr_OpType
var op planpb.RangeExpr_OpType
if !reverse {
switch opStr {
case "<":
@ -121,27 +93,49 @@ func createSingleOps(opStr string, reverse bool) planpb.RangeExpr_OpType {
return op
}
func (context *ParserContext) handleCmpExpr(node *ant_ast.BinaryNode) (*planpb.Expr, error) {
idNode, leftOk := node.Left.(*ant_ast.IdentifierNode)
if !leftOk {
return nil, fmt.Errorf("compare require left to be identifier")
func getLogicalOpType(opStr string) planpb.BinaryExpr_BinaryOp {
switch opStr {
case "&&", "and":
return planpb.BinaryExpr_LogicalAnd
case "||", "or":
return planpb.BinaryExpr_LogicalOr
default:
return planpb.BinaryExpr_Invalid
}
}
func (context *ParserContext) handleCmpExpr(node *ant_ast.BinaryNode) (*planpb.Expr, error) {
var idNode *ant_ast.IdentifierNode
var isReversed bool
var valueNode *ant_ast.Node
if idNodeLeft, leftOk := node.Left.(*ant_ast.IdentifierNode); leftOk {
idNode = idNodeLeft
isReversed = false
valueNode = &node.Right
} else if idNodeRight, rightOk := node.Right.(*ant_ast.IdentifierNode); rightOk {
idNode = idNodeRight
isReversed = true
valueNode = &node.Left
} else {
return nil, fmt.Errorf("compare expr has no identifier")
}
field, err := context.handleIdentifier(idNode)
if err != nil {
return nil, err
}
val, err := context.handleLeafValue(&node.Right, field.DataType)
val, err := context.handleLeafValue(valueNode, field.DataType)
if err != nil {
return nil, err
}
op := createSingleOps(node.Operator, false)
op := getCompareOpType(node.Operator, isReversed)
if op == planpb.RangeExpr_Invalid {
return nil, fmt.Errorf("invalid binary operator %s", node.Operator)
}
final := &planpb.Expr{
expr := &planpb.Expr{
Expr: &planpb.Expr_RangeExpr{
RangeExpr: &planpb.RangeExpr{
ColumnInfo: context.createColumnInfo(field),
@ -150,15 +144,76 @@ func (context *ParserContext) handleCmpExpr(node *ant_ast.BinaryNode) (*planpb.E
},
},
}
return final, nil
return expr, nil
}
func (context *ParserContext) handleLogicalExpr(node *ant_ast.BinaryNode) (*planpb.Expr, error) {
return nil, fmt.Errorf("unimplemented")
op := getLogicalOpType(node.Operator)
if op == planpb.BinaryExpr_Invalid {
return nil, fmt.Errorf("invalid logical op(%s)", node.Operator)
}
leftExpr, err := context.handleExpr(&node.Left)
if err != nil {
return nil, err
}
rightExpr, err := context.handleExpr(&node.Right)
if err != nil {
return nil, err
}
expr := &planpb.Expr{
Expr: &planpb.Expr_BinaryExpr{
BinaryExpr: &planpb.BinaryExpr{
Op: op,
Left: leftExpr,
Right: rightExpr,
},
},
}
return expr, nil
}
func (context *ParserContext) handleArrayExpr(node *ant_ast.Node, dataType schemapb.DataType) ([]*planpb.GenericValue, error) {
arrayNode, ok2 := (*node).(*ant_ast.ArrayNode)
if !ok2 {
return nil, fmt.Errorf("right operand of the InExpr must be array")
}
var arr []*planpb.GenericValue
for _, element := range arrayNode.Nodes {
val, err := context.handleLeafValue(&element, dataType)
if err != nil {
return nil, err
}
arr = append(arr, val)
}
return arr, nil
}
func (context *ParserContext) handleInExpr(node *ant_ast.BinaryNode) (*planpb.Expr, error) {
return nil, fmt.Errorf("unimplemented")
idNode, ok := node.Left.(*ant_ast.IdentifierNode)
if !ok {
return nil, fmt.Errorf("left operand of the InExpr must be identifier")
}
field, err := context.handleIdentifier(idNode)
if err != nil {
return nil, err
}
arrayData, err := context.handleArrayExpr(&node.Right, field.DataType)
if err != nil {
return nil, err
}
expr := &planpb.Expr{
Expr: &planpb.Expr_TermExpr{
TermExpr: &planpb.TermExpr{
ColumnInfo: context.createColumnInfo(field),
Values: arrayData,
},
},
}
return expr, nil
}
func (context *ParserContext) handleBinaryExpr(node *ant_ast.BinaryNode) (*planpb.Expr, error) {
@ -166,16 +221,24 @@ func (context *ParserContext) handleBinaryExpr(node *ant_ast.BinaryNode) (*planp
switch node.Operator {
case "<", "<=", ">", ">=", "==", "!=":
return context.handleCmpExpr(node)
case "and", "or":
case "and", "or", "&&", "||":
return context.handleLogicalExpr(node)
case "in", "not in":
return context.handleInExpr(node)
}
return nil, fmt.Errorf("unimplemented")
return nil, fmt.Errorf("unsupported binary operator %s", node.Operator)
}
func (context *ParserContext) handleNotExpr(node *ant_ast.UnaryNode) (*planpb.Expr, error) {
return nil, fmt.Errorf("unimplemented")
func (context *ParserContext) createNotExpr(childExpr *planpb.Expr) (*planpb.Expr, error) {
expr := &planpb.Expr{
Expr: &planpb.Expr_UnaryExpr{
UnaryExpr: &planpb.UnaryExpr{
Op: planpb.UnaryExpr_Not,
Child: childExpr,
},
},
}
return expr, nil
}
func (context *ParserContext) handleLeafValue(nodeRaw *ant_ast.Node, dataType schemapb.DataType) (gv *planpb.GenericValue, err error) {
@ -232,13 +295,17 @@ func (context *ParserContext) handleIdentifier(node *ant_ast.IdentifierNode) (*s
func (context *ParserContext) handleUnaryExpr(node *ant_ast.UnaryNode) (*planpb.Expr, error) {
switch node.Operator {
case "!", "not":
return context.handleNotExpr(node)
subExpr, err := context.handleExpr(&node.Node)
if err != nil {
return nil, err
}
return context.createNotExpr(subExpr)
default:
return nil, fmt.Errorf("invalid unary operator(%s)", node.Operator)
}
}
func (context *ParserContext) parseExpr(nodeRaw *ant_ast.Node) (*planpb.Expr, error) {
func (context *ParserContext) handleExpr(nodeRaw *ant_ast.Node) (*planpb.Expr, error) {
switch node := (*nodeRaw).(type) {
case *ant_ast.IdentifierNode,
*ant_ast.FloatNode,

66
internal/proxynode/plan_parser_test.go
View File

@ -1,9 +1,20 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
package proxynode
import (
"fmt"
"testing"
ant_ast "github.com/antonmedv/expr/ast"
ant_parser "github.com/antonmedv/expr/parser"
"github.com/golang/protobuf/proto"
@ -15,20 +26,31 @@ import (
)
func newTestSchema() *schemapb.CollectionSchema {
fields := []*schemapb.FieldSchema{
{FieldID: 0, Name: "FieldID", IsPrimaryKey: false, Description: "field no.1", DataType: schemapb.DataType_Int64},
}
for name, value := range schemapb.DataType_value {
dataType := schemapb.DataType(value)
if !typeutil.IsIntergerType(dataType) && !typeutil.IsFloatingType(dataType) && !typeutil.IsVectorType(dataType) {
continue
}
newField := &schemapb.FieldSchema{
FieldID: int64(100 + value), Name: name + "Field", IsPrimaryKey: false, Description: "", DataType: dataType,
}
fields = append(fields, newField)
}
return &schemapb.CollectionSchema{
Name: "test",
Description: "schema for test used",
AutoID: true,
Fields: []*schemapb.FieldSchema{
{FieldID: 0, Name: "FieldID", IsPrimaryKey: false, Description: "field no.1", DataType: schemapb.DataType_Int64},
{FieldID: 101, Name: "vectorField", IsPrimaryKey: false, Description: "field no.2", DataType: schemapb.DataType_FloatVector},
{FieldID: 100, Name: "int64Field", IsPrimaryKey: false, Description: "field no.1", DataType: schemapb.DataType_Int64},
},
Fields: fields,
}
}
func TestParseQueryExpr_Naive(t *testing.T) {
exprStr := "int64Field > 3"
exprStr := "Int64Field > 3"
schemaPb := newTestSchema()
schema, err := typeutil.CreateSchemaHelper(schemaPb)
assert.Nil(t, err)
@ -39,7 +61,16 @@ func TestParseQueryExpr_Naive(t *testing.T) {
}
func TestParsePlanNode_Naive(t *testing.T) {
exprStr := "int64Field > 3"
// exprStr := "not (int64Field > 3)"
exprStrs := []string{
"not (Int64Field > 3)",
"not (3 > Int64Field)",
"Int64Field in [1, 2, 3]",
"Int64Field < 3 and (Int64Field > 2 || Int64Field == 1)",
"DoubleField in [1.0, 2, 3]",
"DoubleField in [1.0, 2, 3] && Int64Field < 3 or Int64Field > 2",
}
schema := newTestSchema()
queryInfo := &planpb.QueryInfo{
Topk: 10,
@ -49,20 +80,19 @@ func TestParsePlanNode_Naive(t *testing.T) {
// Note: use pointer to string to represent nullable string
// TODO: change it to better solution
planProto, err := CreateQueryPlan(schema, &exprStr, "vectorField", queryInfo)
for offset, exprStr := range exprStrs {
fmt.Printf("case %d: %s\n", offset, exprStr)
planProto, err := CreateQueryPlan(schema, &exprStr, "FloatVectorField", queryInfo)
assert.Nil(t, err)
dbgStr := proto.MarshalTextString(planProto)
println(dbgStr)
}
assert.Nil(t, err)
dbgStr := proto.MarshalTextString(planProto)
println(dbgStr)
}
func TestExternalParser(t *testing.T) {
ast, err := ant_parser.Parse("!(1 < a < 2 or b in [1, 2, 3]) or (c < 3 and b > 5) or ")
var node ant_ast.Node = nil
if node == nil {
// TODO
}
ast, err := ant_parser.Parse("!(1 < a < 2 or b in [1, 2, 3]) or (c < 3 and b > 5)")
// NOTE: probe ast here via IDE
assert.Nil(t, err)
println(ast.Node.Location().Column)