From d64aa37db3e8417575cc2ed433e74eb7e42a3fd3 Mon Sep 17 00:00:00 2001 From: XuanYang-cn Date: Fri, 11 Dec 2020 12:01:20 +0800 Subject: [PATCH] Add KFlush msg type Signed-off-by: XuanYang-cn --- .gitignore | 1 + go.mod | 3 + go.sum | 26 ++ internal/indexbuilder/client/client.go | 98 ++++++ internal/master/master_test.go | 17 +- internal/msgstream/msg.go | 33 ++ internal/proto/internal_msg.proto | 7 + internal/proto/internalpb/internal_msg.pb.go | 290 +++++++++++------- internal/querynode/search_service.go | 5 +- internal/storage/data_codec.go | 5 +- internal/writenode/data_sync_service.go | 12 +- internal/writenode/data_sync_service_test.go | 20 +- .../writenode/flow_graph_filter_dm_node.go | 2 + .../flow_graph_insert_buffer_node.go | 69 ++++- internal/writenode/flow_graph_message.go | 1 + internal/writenode/write_node.go | 12 +- 16 files changed, 429 insertions(+), 172 deletions(-) create mode 100644 internal/indexbuilder/client/client.go diff --git a/.gitignore b/.gitignore index f62f7bb3a8..57c9149088 100644 --- a/.gitignore +++ b/.gitignore @@ -59,3 +59,4 @@ cwrapper_build **/.clangd/* **/compile_commands.json **/.lint +typescript diff --git a/go.mod b/go.mod index 5016aaf715..47afde2bff 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 github.com/frankban/quicktest v1.10.2 // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect + github.com/git-hooks/git-hooks v1.3.1 // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.3.2 github.com/google/btree v1.0.0 @@ -28,12 +29,14 @@ require ( github.com/prometheus/client_golang v1.5.1 // indirect github.com/prometheus/common v0.10.0 // indirect github.com/prometheus/procfs v0.1.3 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sirupsen/logrus v1.6.0 // indirect github.com/spaolacci/murmur3 v1.1.0 github.com/spf13/cast v1.3.0 github.com/spf13/viper v1.7.1 github.com/stretchr/testify v1.6.1 github.com/tikv/client-go v0.0.0-20200824032810-95774393107b + github.com/urfave/cli v1.22.5 // indirect github.com/yahoo/athenz v1.9.16 // indirect go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 go.uber.org/zap v1.15.0 diff --git a/go.sum b/go.sum index 16af9ce827..eb4ef6b6a4 100644 --- a/go.sum +++ b/go.sum @@ -40,8 +40,12 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= +github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q= github.com/boynton/repl v0.0.0-20170116235056-348863958e3e/go.mod h1:Crc/GCZ3NXDVCio7Yr0o+SSrytpcFhLmVCIzi0s49t4= +github.com/cattail/go-exclude v0.0.0-20141118090525-7e63167c2dab h1:1WOH7EEbhb6OZWcIU5RpQx5rmHm1xEUda8Qiw4UzNlU= +github.com/cattail/go-exclude v0.0.0-20141118090525-7e63167c2dab/go.mod h1:5MSsYMW59C/HfIUsthTRDxRoMQctcmAVb1JnNSQXERA= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -63,6 +67,9 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= @@ -89,6 +96,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/git-hooks/git-hooks v1.3.1 h1:ypdvNTXMiITXQxuqIl6t1f8R3V1FrUXPi1CwQ4guClo= +github.com/git-hooks/git-hooks v1.3.1/go.mod h1:RBqjgxUpRSLI4AxVZGV1FsODclMjpsJIaDEyJSoK3ws= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -125,6 +134,10 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -336,8 +349,14 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= @@ -370,6 +389,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v0.0.0-20141015234014-d6577e08ec30/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -388,8 +408,14 @@ github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43/go.mod h1:iT03XoTw github.com/unrolled/render v1.0.0 h1:XYtvhA3UkpB7PqkvhUFYmpKD55OudoIeygcfus4vcd4= github.com/unrolled/render v1.0.0/go.mod h1:tu82oB5W2ykJRVioYsB+IQKcft7ryBr7w12qMBUPyXg= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU= +github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/gozstd v1.7.0 h1:Ljh5c9zboqLhwTI33al32R72iCZfn0mCbVGcFWbGwRQ= github.com/valyala/gozstd v1.7.0/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ= +github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= +github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yahoo/athenz v1.8.55/go.mod h1:G7LLFUH7Z/r4QAB7FfudfuA7Am/eCzO1GlzBhDL6Kv0= diff --git a/internal/indexbuilder/client/client.go b/internal/indexbuilder/client/client.go new file mode 100644 index 0000000000..24d935b3bc --- /dev/null +++ b/internal/indexbuilder/client/client.go @@ -0,0 +1,98 @@ +package indexbuilder + +import ( + "context" + "time" + + "google.golang.org/grpc" + + "github.com/zilliztech/milvus-distributed/internal/proto/commonpb" + "github.com/zilliztech/milvus-distributed/internal/proto/indexbuilderpb" + "github.com/zilliztech/milvus-distributed/internal/util/typeutil" +) + +type UniqueID = typeutil.UniqueID + +type Client struct { + client indexbuilderpb.IndexBuildServiceClient +} + +type IndexStatus int32 + +const ( + NOTEXIST IndexStatus = 0 + UNISSUED IndexStatus = 1 + INPROGRESS IndexStatus = 2 + FINISHED IndexStatus = 3 +) + +type IndexDescription struct { + ID UniqueID + Status IndexStatus + EnqueueTime time.Time + ScheduleTime time.Time + BuildCompleteTime time.Time +} + +func NewBuildIndexClient(conn *grpc.ClientConn) *Client { + return &Client{ + client: indexbuilderpb.NewIndexBuildServiceClient(conn), + } +} + +func (c *Client) BuildIndexWithoutID(columnDataPaths []string, typeParams map[string]string, indexParams map[string]string) (UniqueID, error) { + //first new a build service client + + var typeParamsKV []*commonpb.KeyValuePair + for typeParam := range typeParams { + typeParamsKV = append(typeParamsKV, &commonpb.KeyValuePair{ + Key: typeParam, + Value: typeParams[typeParam], + }) + } + + var indexParamsKV []*commonpb.KeyValuePair + for indexParam := range indexParams { + indexParamsKV = append(indexParamsKV, &commonpb.KeyValuePair{ + Key: indexParam, + Value: indexParams[indexParam], + }) + } + + ctx := context.TODO() + requset := &indexbuilderpb.BuildIndexRequest{ + DataPaths: columnDataPaths, + TypeParams: typeParamsKV, + IndexParams: indexParamsKV, + } + response, err := c.client.BuildIndex(ctx, requset) + if err != nil { + return 0, err + } + + indexID := response.IndexID + return indexID, err +} + +func (c *Client) DescribeIndex(indexID UniqueID) (IndexDescription, error) { + //ctx := context.TODO() + //request := &indexbuilderpb.DescribleIndexRequest{ + // IndexID: indexID, + //} + //response, err := c.client.DescribeIndex(ctx, request) + //if err != nil { + // return IndexDescription{}, err + //} + // + //indexDescription := &IndexDescription{ + // ID: indexID, + // Status: response.IndexStatus, + // EnqueueTime: time.Unix(), + //} + return IndexDescription{}, nil +} + +func (c *Client) GetIndexFilePaths(IndexID UniqueID) ([]string, error) { + + return nil, nil +} diff --git a/internal/master/master_test.go b/internal/master/master_test.go index 5af957c527..632e45deaf 100644 --- a/internal/master/master_test.go +++ b/internal/master/master_test.go @@ -14,7 +14,6 @@ import ( "github.com/stretchr/testify/assert" ms "github.com/zilliztech/milvus-distributed/internal/msgstream" "github.com/zilliztech/milvus-distributed/internal/proto/commonpb" - "github.com/zilliztech/milvus-distributed/internal/proto/internalpb" internalPb "github.com/zilliztech/milvus-distributed/internal/proto/internalpb" "github.com/zilliztech/milvus-distributed/internal/proto/masterpb" "github.com/zilliztech/milvus-distributed/internal/proto/schemapb" @@ -165,8 +164,8 @@ func TestMaster(t *testing.T) { schemaBytes, err := proto.Marshal(&sch) assert.Nil(t, err) - createCollectionReq := internalpb.CreateCollectionRequest{ - MsgType: internalpb.MsgType_kCreateCollection, + createCollectionReq := internalPb.CreateCollectionRequest{ + MsgType: internalPb.MsgType_kCreateCollection, ReqID: 1, Timestamp: 11, ProxyID: 1, @@ -196,8 +195,8 @@ func TestMaster(t *testing.T) { ////////////////////////////CreatePartition//////////////////////// partitionName := "partitionName" + strconv.FormatUint(rand.Uint64(), 10) - createPartitionReq := internalpb.CreatePartitionRequest{ - MsgType: internalpb.MsgType_kCreatePartition, + createPartitionReq := internalPb.CreatePartitionRequest{ + MsgType: internalPb.MsgType_kCreatePartition, ReqID: 1, Timestamp: 11, ProxyID: 1, @@ -230,8 +229,8 @@ func TestMaster(t *testing.T) { assert.Equal(t, createPartitionReq.PartitionName.Tag, createPartitionMsg.CreatePartitionRequest.PartitionName.Tag) ////////////////////////////DropPartition//////////////////////// - dropPartitionReq := internalpb.DropPartitionRequest{ - MsgType: internalpb.MsgType_kDropPartition, + dropPartitionReq := internalPb.DropPartitionRequest{ + MsgType: internalPb.MsgType_kDropPartition, ReqID: 1, Timestamp: 11, ProxyID: 1, @@ -263,8 +262,8 @@ func TestMaster(t *testing.T) { assert.Equal(t, dropPartitionReq.PartitionName.CollectionName, dropPartitionMsg.DropPartitionRequest.PartitionName.CollectionName) ////////////////////////////DropCollection//////////////////////// - dropCollectionReq := internalpb.DropCollectionRequest{ - MsgType: internalpb.MsgType_kDropCollection, + dropCollectionReq := internalPb.DropCollectionRequest{ + MsgType: internalPb.MsgType_kDropCollection, ReqID: 1, Timestamp: 11, ProxyID: 1, diff --git a/internal/msgstream/msg.go b/internal/msgstream/msg.go index e26f60dcf2..518bcfa7af 100644 --- a/internal/msgstream/msg.go +++ b/internal/msgstream/msg.go @@ -78,6 +78,39 @@ func (it *InsertMsg) Unmarshal(input []byte) (TsMsg, error) { return insertMsg, nil } +/////////////////////////////////////////Flush////////////////////////////////////////// +type FlushMsg struct { + BaseMsg + internalPb.FlushMsg +} + +func (fl *FlushMsg) Type() MsgType { + return fl.GetMsgType() +} + +func (fl *FlushMsg) Marshal(input TsMsg) ([]byte, error) { + flushMsgTask := input.(*FlushMsg) + flushMsg := &flushMsgTask.FlushMsg + mb, err := proto.Marshal(flushMsg) + if err != nil { + return nil, err + } + return mb, nil +} + +func (fl *FlushMsg) Unmarshal(input []byte) (TsMsg, error) { + flushMsg := internalPb.FlushMsg{} + err := proto.Unmarshal(input, &flushMsg) + if err != nil { + return nil, err + } + flushMsgTask := &FlushMsg{FlushMsg: flushMsg} + flushMsgTask.BeginTimestamp = flushMsgTask.Timestamp + flushMsgTask.EndTimestamp = flushMsgTask.Timestamp + + return flushMsgTask, nil +} + /////////////////////////////////////////Delete////////////////////////////////////////// type DeleteMsg struct { BaseMsg diff --git a/internal/proto/internal_msg.proto b/internal/proto/internal_msg.proto index 53737b495e..209d4767fd 100644 --- a/internal/proto/internal_msg.proto +++ b/internal/proto/internal_msg.proto @@ -26,6 +26,7 @@ enum MsgType { /* Manipulation Requests */ kInsert = 400; kDelete = 401; + kFlush = 402; /* Query */ kSearch = 500; @@ -262,6 +263,12 @@ message TimeTickMsg { uint64 timestamp = 3; } +message FlushMsg { + MsgType msg_type = 1; + int64 segmentID = 2; + uint64 timestamp = 3; +} + message Key2Seg { int64 rowID = 1; diff --git a/internal/proto/internalpb/internal_msg.pb.go b/internal/proto/internalpb/internal_msg.pb.go index 75e9953f11..24560b7da3 100644 --- a/internal/proto/internalpb/internal_msg.pb.go +++ b/internal/proto/internalpb/internal_msg.pb.go @@ -42,6 +42,7 @@ const ( // Manipulation Requests MsgType_kInsert MsgType = 400 MsgType_kDelete MsgType = 401 + MsgType_kFlush MsgType = 402 // Query MsgType_kSearch MsgType = 500 MsgType_kSearchResult MsgType = 501 @@ -66,6 +67,7 @@ var MsgType_name = map[int32]string{ 204: "kShowPartitions", 400: "kInsert", 401: "kDelete", + 402: "kFlush", 500: "kSearch", 501: "kSearchResult", 1200: "kTimeTick", @@ -88,6 +90,7 @@ var MsgType_value = map[string]int32{ "kShowPartitions": 204, "kInsert": 400, "kDelete": 401, + "kFlush": 402, "kSearch": 500, "kSearchResult": 501, "kTimeTick": 1200, @@ -1864,6 +1867,61 @@ func (m *TimeTickMsg) GetTimestamp() uint64 { return 0 } +type FlushMsg struct { + MsgType MsgType `protobuf:"varint,1,opt,name=msg_type,json=msgType,proto3,enum=milvus.proto.internal.MsgType" json:"msg_type,omitempty"` + SegmentID int64 `protobuf:"varint,2,opt,name=segmentID,proto3" json:"segmentID,omitempty"` + Timestamp uint64 `protobuf:"varint,3,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *FlushMsg) Reset() { *m = FlushMsg{} } +func (m *FlushMsg) String() string { return proto.CompactTextString(m) } +func (*FlushMsg) ProtoMessage() {} +func (*FlushMsg) Descriptor() ([]byte, []int) { + return fileDescriptor_7eb37f6b80b23116, []int{24} +} + +func (m *FlushMsg) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_FlushMsg.Unmarshal(m, b) +} +func (m *FlushMsg) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_FlushMsg.Marshal(b, m, deterministic) +} +func (m *FlushMsg) XXX_Merge(src proto.Message) { + xxx_messageInfo_FlushMsg.Merge(m, src) +} +func (m *FlushMsg) XXX_Size() int { + return xxx_messageInfo_FlushMsg.Size(m) +} +func (m *FlushMsg) XXX_DiscardUnknown() { + xxx_messageInfo_FlushMsg.DiscardUnknown(m) +} + +var xxx_messageInfo_FlushMsg proto.InternalMessageInfo + +func (m *FlushMsg) GetMsgType() MsgType { + if m != nil { + return m.MsgType + } + return MsgType_kNone +} + +func (m *FlushMsg) GetSegmentID() int64 { + if m != nil { + return m.SegmentID + } + return 0 +} + +func (m *FlushMsg) GetTimestamp() uint64 { + if m != nil { + return m.Timestamp + } + return 0 +} + type Key2Seg struct { RowID int64 `protobuf:"varint,1,opt,name=rowID,proto3" json:"rowID,omitempty"` PrimaryKey int64 `protobuf:"varint,2,opt,name=primary_key,json=primaryKey,proto3" json:"primary_key,omitempty"` @@ -1879,7 +1937,7 @@ func (m *Key2Seg) Reset() { *m = Key2Seg{} } func (m *Key2Seg) String() string { return proto.CompactTextString(m) } func (*Key2Seg) ProtoMessage() {} func (*Key2Seg) Descriptor() ([]byte, []int) { - return fileDescriptor_7eb37f6b80b23116, []int{24} + return fileDescriptor_7eb37f6b80b23116, []int{25} } func (m *Key2Seg) XXX_Unmarshal(b []byte) error { @@ -1947,7 +2005,7 @@ func (m *Key2SegMsg) Reset() { *m = Key2SegMsg{} } func (m *Key2SegMsg) String() string { return proto.CompactTextString(m) } func (*Key2SegMsg) ProtoMessage() {} func (*Key2SegMsg) Descriptor() ([]byte, []int) { - return fileDescriptor_7eb37f6b80b23116, []int{25} + return fileDescriptor_7eb37f6b80b23116, []int{26} } func (m *Key2SegMsg) XXX_Unmarshal(b []byte) error { @@ -1996,7 +2054,7 @@ func (m *LoadIndex) Reset() { *m = LoadIndex{} } func (m *LoadIndex) String() string { return proto.CompactTextString(m) } func (*LoadIndex) ProtoMessage() {} func (*LoadIndex) Descriptor() ([]byte, []int) { - return fileDescriptor_7eb37f6b80b23116, []int{26} + return fileDescriptor_7eb37f6b80b23116, []int{27} } func (m *LoadIndex) XXX_Unmarshal(b []byte) error { @@ -2057,7 +2115,7 @@ func (m *IndexStats) Reset() { *m = IndexStats{} } func (m *IndexStats) String() string { return proto.CompactTextString(m) } func (*IndexStats) ProtoMessage() {} func (*IndexStats) Descriptor() ([]byte, []int) { - return fileDescriptor_7eb37f6b80b23116, []int{27} + return fileDescriptor_7eb37f6b80b23116, []int{28} } func (m *IndexStats) XXX_Unmarshal(b []byte) error { @@ -2105,7 +2163,7 @@ func (m *FieldStats) Reset() { *m = FieldStats{} } func (m *FieldStats) String() string { return proto.CompactTextString(m) } func (*FieldStats) ProtoMessage() {} func (*FieldStats) Descriptor() ([]byte, []int) { - return fileDescriptor_7eb37f6b80b23116, []int{28} + return fileDescriptor_7eb37f6b80b23116, []int{29} } func (m *FieldStats) XXX_Unmarshal(b []byte) error { @@ -2161,7 +2219,7 @@ func (m *SegmentStats) Reset() { *m = SegmentStats{} } func (m *SegmentStats) String() string { return proto.CompactTextString(m) } func (*SegmentStats) ProtoMessage() {} func (*SegmentStats) Descriptor() ([]byte, []int) { - return fileDescriptor_7eb37f6b80b23116, []int{29} + return fileDescriptor_7eb37f6b80b23116, []int{30} } func (m *SegmentStats) XXX_Unmarshal(b []byte) error { @@ -2224,7 +2282,7 @@ func (m *QueryNodeStats) Reset() { *m = QueryNodeStats{} } func (m *QueryNodeStats) String() string { return proto.CompactTextString(m) } func (*QueryNodeStats) ProtoMessage() {} func (*QueryNodeStats) Descriptor() ([]byte, []int) { - return fileDescriptor_7eb37f6b80b23116, []int{30} + return fileDescriptor_7eb37f6b80b23116, []int{31} } func (m *QueryNodeStats) XXX_Unmarshal(b []byte) error { @@ -2300,6 +2358,7 @@ func init() { proto.RegisterType((*SysConfigRequest)(nil), "milvus.proto.internal.SysConfigRequest") proto.RegisterType((*SearchResult)(nil), "milvus.proto.internal.SearchResult") proto.RegisterType((*TimeTickMsg)(nil), "milvus.proto.internal.TimeTickMsg") + proto.RegisterType((*FlushMsg)(nil), "milvus.proto.internal.FlushMsg") proto.RegisterType((*Key2Seg)(nil), "milvus.proto.internal.Key2Seg") proto.RegisterType((*Key2SegMsg)(nil), "milvus.proto.internal.Key2SegMsg") proto.RegisterType((*LoadIndex)(nil), "milvus.proto.internal.LoadIndex") @@ -2312,112 +2371,113 @@ func init() { func init() { proto.RegisterFile("internal_msg.proto", fileDescriptor_7eb37f6b80b23116) } var fileDescriptor_7eb37f6b80b23116 = []byte{ - // 1701 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x58, 0xcd, 0x6f, 0x23, 0x49, - 0x15, 0xdf, 0xee, 0xf6, 0x47, 0xfc, 0xec, 0x38, 0x3d, 0x95, 0x64, 0xc6, 0xb3, 0x8b, 0x76, 0x32, - 0x3d, 0x88, 0x0d, 0x8b, 0x48, 0x20, 0xc3, 0x81, 0xbd, 0x41, 0x62, 0xc1, 0x9a, 0x21, 0xa3, 0xd0, - 0x8e, 0x16, 0x09, 0xad, 0xd4, 0xea, 0xd8, 0x2f, 0x76, 0xa9, 0x3f, 0x53, 0xd5, 0x9e, 0x8c, 0xe7, - 0xc0, 0x69, 0xce, 0x08, 0x84, 0x38, 0x70, 0x40, 0xf0, 0x17, 0xac, 0x58, 0xf8, 0x27, 0xf8, 0x3a, - 0xf1, 0x5f, 0xc0, 0x81, 0x95, 0x60, 0x39, 0x20, 0x2e, 0xa8, 0xaa, 0xfa, 0xc3, 0x9d, 0xd8, 0xce, - 0x88, 0x64, 0x96, 0x41, 0xbb, 0xb7, 0xaa, 0xd7, 0xd5, 0x55, 0xef, 0xf7, 0xfb, 0x55, 0xbd, 0xaa, - 0xf7, 0x80, 0xd0, 0x30, 0x41, 0x16, 0xba, 0xbe, 0x13, 0xf0, 0xd1, 0x4e, 0xcc, 0xa2, 0x24, 0x22, - 0x9b, 0x01, 0xf5, 0x9f, 0x4c, 0xb8, 0xea, 0xed, 0x64, 0x03, 0x5e, 0x6f, 0x0d, 0xa2, 0x20, 0x88, - 0x42, 0x65, 0x7e, 0xfd, 0x16, 0x47, 0xf6, 0x84, 0x0e, 0xb0, 0xf8, 0xcf, 0x0a, 0xa1, 0xd1, 0xeb, - 0xda, 0x78, 0x36, 0x41, 0x9e, 0x90, 0xdb, 0x50, 0x8b, 0x11, 0x59, 0xaf, 0xdb, 0xd1, 0xb6, 0xb4, - 0x6d, 0xc3, 0x4e, 0x7b, 0xe4, 0x21, 0x54, 0x58, 0xe4, 0x63, 0x47, 0xdf, 0xd2, 0xb6, 0xdb, 0x7b, - 0xf7, 0x76, 0xe6, 0xae, 0xb5, 0x73, 0x84, 0xc8, 0xec, 0xc8, 0x47, 0x5b, 0x0e, 0x26, 0x1b, 0x50, - 0x1d, 0x44, 0x93, 0x30, 0xe9, 0x18, 0x5b, 0xda, 0xf6, 0xaa, 0xad, 0x3a, 0xd6, 0x08, 0x40, 0xac, - 0xc7, 0xe3, 0x28, 0xe4, 0x48, 0x1e, 0x42, 0x8d, 0x27, 0x6e, 0x32, 0xe1, 0x72, 0xc1, 0xe6, 0xde, - 0x1b, 0xe5, 0xa9, 0x53, 0xe7, 0xfb, 0x72, 0x88, 0x9d, 0x0e, 0x25, 0x6d, 0xd0, 0x7b, 0x5d, 0xe9, - 0x8b, 0x61, 0xeb, 0xbd, 0xee, 0x82, 0x85, 0x22, 0x80, 0x63, 0x1e, 0x7d, 0x82, 0xc8, 0x9e, 0x40, - 0x53, 0x2e, 0x78, 0x1d, 0x68, 0x9f, 0x83, 0x46, 0x42, 0x03, 0xe4, 0x89, 0x1b, 0xc4, 0xd2, 0xa7, - 0x8a, 0x5d, 0x18, 0x16, 0xac, 0xfb, 0x5c, 0x83, 0x56, 0x1f, 0x47, 0x85, 0x8a, 0xf9, 0x30, 0x6d, - 0x66, 0x98, 0x98, 0x7a, 0x30, 0x76, 0xc3, 0x10, 0xfd, 0x94, 0xbc, 0xaa, 0x5d, 0x18, 0xc8, 0x1b, - 0xd0, 0x18, 0x44, 0xbe, 0xef, 0x84, 0x6e, 0x80, 0x72, 0xfa, 0x86, 0xbd, 0x22, 0x0c, 0x8f, 0xdd, - 0x00, 0xc9, 0x03, 0x58, 0x8d, 0x5d, 0x96, 0xd0, 0x84, 0x46, 0xa1, 0x93, 0xb8, 0xa3, 0x4e, 0x45, - 0x0e, 0x68, 0xe5, 0xc6, 0x63, 0x77, 0x64, 0x7d, 0xa0, 0x01, 0xf9, 0x26, 0xe7, 0x74, 0x14, 0x96, - 0x9c, 0xb9, 0x51, 0xe2, 0x1f, 0xc1, 0x5a, 0x8c, 0xcc, 0x49, 0xdd, 0x76, 0x18, 0x9e, 0x75, 0x8c, - 0x2d, 0x63, 0xbb, 0xb9, 0xf7, 0x60, 0xc1, 0xff, 0xb3, 0xae, 0xd8, 0xab, 0x31, 0xb2, 0x03, 0xf5, - 0xab, 0x8d, 0x67, 0xd6, 0x2f, 0x34, 0x58, 0x93, 0xdf, 0x95, 0xd7, 0x01, 0x86, 0x92, 0x3a, 0x2e, - 0x4c, 0xa9, 0xb3, 0xaa, 0x73, 0x05, 0x75, 0x73, 0x55, 0x29, 0x13, 0x5a, 0xb9, 0x8a, 0xd0, 0xea, - 0x1c, 0x42, 0x3f, 0xd2, 0x60, 0xbd, 0x44, 0xe8, 0xcb, 0xdb, 0x58, 0x6f, 0xc1, 0x1a, 0x3e, 0x8d, - 0x29, 0x43, 0x67, 0x38, 0x61, 0xae, 0x70, 0x40, 0x82, 0xa9, 0xd8, 0x6d, 0x65, 0xee, 0xa6, 0x56, - 0xf2, 0x3e, 0xdc, 0x9e, 0x15, 0xc0, 0xcd, 0x99, 0xeb, 0x54, 0xa4, 0x0e, 0x5f, 0x58, 0xa6, 0x43, - 0xc1, 0xb3, 0xbd, 0x51, 0x48, 0x51, 0x58, 0xad, 0x7f, 0x69, 0x70, 0xe7, 0x80, 0xa1, 0x9b, 0xe0, - 0x41, 0xe4, 0xfb, 0x38, 0x10, 0x4b, 0x66, 0xfb, 0xe8, 0x1d, 0x58, 0x09, 0xf8, 0xc8, 0x49, 0xa6, - 0x31, 0x4a, 0xdc, 0xed, 0xbd, 0x37, 0x17, 0xac, 0x75, 0xc8, 0x47, 0xc7, 0xd3, 0x18, 0xed, 0x7a, - 0xa0, 0x1a, 0xc4, 0x82, 0xd6, 0x20, 0x9f, 0x2f, 0x8f, 0x1c, 0x25, 0x9b, 0x10, 0x91, 0xe1, 0x59, - 0xaf, 0x2b, 0x71, 0x1b, 0xb6, 0xea, 0x94, 0x59, 0xab, 0x5c, 0x64, 0xad, 0x03, 0xf5, 0x98, 0x45, - 0x4f, 0xa7, 0xbd, 0xae, 0xd4, 0xcf, 0xb0, 0xb3, 0x2e, 0xf9, 0x2a, 0xd4, 0xf8, 0x60, 0x8c, 0x81, - 0xdb, 0xa9, 0x49, 0x89, 0xee, 0xce, 0x95, 0x68, 0xdf, 0x8f, 0x4e, 0xec, 0x74, 0xa0, 0xf5, 0x53, - 0x1d, 0x36, 0xbb, 0x2c, 0x8a, 0xff, 0xcf, 0x91, 0x1f, 0xc2, 0x5a, 0x31, 0xbb, 0xda, 0xfc, 0x8a, - 0x82, 0xcf, 0x97, 0x7d, 0x4e, 0x2f, 0xa2, 0x9d, 0x02, 0xae, 0x38, 0x18, 0x76, 0x7b, 0x50, 0xea, - 0x5b, 0x7f, 0xd3, 0x60, 0xe3, 0x5d, 0x97, 0xdf, 0x28, 0x29, 0x39, 0x60, 0x7d, 0x21, 0x60, 0x63, - 0x09, 0xe0, 0xca, 0x95, 0x80, 0xab, 0xd7, 0x00, 0xfc, 0x0f, 0x0d, 0xee, 0x76, 0x91, 0x0f, 0x18, - 0x3d, 0xc1, 0x4f, 0x0f, 0xea, 0x5f, 0x69, 0xb0, 0xd9, 0x1f, 0x47, 0xe7, 0xaf, 0x2e, 0x62, 0xeb, - 0xb7, 0x3a, 0xdc, 0x56, 0xb1, 0xe9, 0x28, 0x0b, 0xd2, 0x9f, 0xd0, 0x01, 0xdd, 0x82, 0x66, 0x7e, - 0x2f, 0xe4, 0xc7, 0x74, 0xd6, 0x54, 0x20, 0xad, 0x2c, 0x44, 0x5a, 0x5d, 0x82, 0xb4, 0x56, 0xd6, - 0xf6, 0x3b, 0xd0, 0x2e, 0x2e, 0x27, 0x29, 0x6d, 0x5d, 0x4a, 0xfb, 0x60, 0xbe, 0xb4, 0x39, 0x1d, - 0x52, 0xd9, 0xe2, 0x5e, 0x93, 0xc2, 0x7e, 0xa8, 0xc3, 0x86, 0x88, 0x6a, 0x9f, 0x71, 0xf6, 0xe2, - 0x9c, 0xfd, 0x55, 0x83, 0xf5, 0x77, 0x5d, 0x7e, 0x93, 0x94, 0xdd, 0xec, 0xe1, 0xbf, 0x0c, 0xb6, - 0xfa, 0x5f, 0x83, 0xfd, 0x48, 0x83, 0x4e, 0x16, 0xef, 0x3e, 0x1d, 0x88, 0xc5, 0x95, 0x26, 0x62, - 0xdd, 0xab, 0x8b, 0xf6, 0x86, 0x83, 0xfb, 0xdf, 0x75, 0x58, 0xed, 0x85, 0x1c, 0x59, 0xf2, 0xd2, - 0x90, 0xbe, 0x75, 0xd9, 0x63, 0x95, 0xc3, 0x5c, 0xf0, 0xe5, 0x85, 0x32, 0x19, 0xc1, 0x1b, 0xc7, - 0x91, 0x78, 0x91, 0xe6, 0xef, 0x9b, 0xc2, 0x50, 0x4e, 0x06, 0x54, 0x18, 0x98, 0x49, 0x06, 0x66, - 0x58, 0xad, 0x97, 0x59, 0x7d, 0x13, 0x20, 0x27, 0x9f, 0x77, 0x56, 0xb6, 0x8c, 0xed, 0x8a, 0x3d, - 0x63, 0x11, 0x89, 0x12, 0x8b, 0xce, 0x7b, 0x5d, 0xde, 0x69, 0x6c, 0x19, 0x22, 0x51, 0x52, 0x3d, - 0xf2, 0x35, 0x58, 0x61, 0xd1, 0xb9, 0x33, 0x74, 0x13, 0xb7, 0x03, 0xf2, 0x91, 0xbd, 0xe4, 0x35, - 0x59, 0x67, 0xd1, 0x79, 0xd7, 0x4d, 0x5c, 0xeb, 0xb9, 0x0e, 0xab, 0x5d, 0xf4, 0x31, 0xc1, 0xff, - 0x3d, 0xe9, 0x25, 0xc6, 0x2a, 0x4b, 0x18, 0xab, 0x2e, 0x63, 0xac, 0x76, 0x89, 0xb1, 0xfb, 0xd0, - 0x8a, 0x19, 0x0d, 0x5c, 0x36, 0x75, 0x3c, 0x9c, 0xf2, 0x4e, 0x5d, 0xf2, 0xd6, 0x4c, 0x6d, 0x8f, - 0x70, 0xca, 0xad, 0x8f, 0x35, 0x58, 0xed, 0xa3, 0xcb, 0x06, 0xe3, 0x97, 0x46, 0xc3, 0x8c, 0xff, - 0x46, 0xd9, 0xff, 0xe5, 0x6f, 0xe8, 0x2f, 0x82, 0xc9, 0x90, 0x4f, 0xfc, 0xc4, 0x29, 0xc8, 0x51, - 0x04, 0xac, 0x29, 0xfb, 0x41, 0x4e, 0xd1, 0x2e, 0x54, 0xcf, 0x26, 0xc8, 0xa6, 0x57, 0x67, 0x13, - 0x6a, 0x9c, 0xf5, 0x67, 0x0d, 0xcc, 0xfe, 0x94, 0x1f, 0x44, 0xe1, 0x29, 0x1d, 0xbd, 0x72, 0xc8, - 0x09, 0x54, 0xa4, 0x5e, 0xd5, 0x2d, 0x63, 0xbb, 0x61, 0xcb, 0xb6, 0xd0, 0xd2, 0xc3, 0xa9, 0x13, - 0x33, 0x3c, 0xa5, 0x4f, 0x51, 0xa9, 0xdd, 0xb0, 0x9b, 0x1e, 0x4e, 0x8f, 0x52, 0x93, 0xf5, 0x81, - 0x0e, 0xad, 0x4c, 0x4b, 0xc1, 0xcf, 0x75, 0x00, 0x15, 0x39, 0xb4, 0xfe, 0xe2, 0x39, 0xf4, 0xfc, - 0x4c, 0x69, 0x71, 0x1c, 0xbd, 0x0f, 0x2d, 0x29, 0x87, 0x13, 0x46, 0x43, 0xcc, 0xd5, 0x6d, 0x4a, - 0xdb, 0x63, 0x69, 0x2a, 0x13, 0x55, 0x7b, 0x91, 0x2d, 0x52, 0x9f, 0xbf, 0x45, 0x08, 0x54, 0xc6, - 0x34, 0x51, 0x71, 0xa5, 0x65, 0xcb, 0xb6, 0xf5, 0x43, 0x68, 0x1e, 0xd3, 0x00, 0x8f, 0xe9, 0xc0, - 0x3b, 0xe4, 0xa3, 0xeb, 0xd0, 0x55, 0x14, 0x71, 0xf4, 0x52, 0x11, 0x67, 0xe9, 0x0d, 0x63, 0xfd, - 0x5c, 0x83, 0xfa, 0x23, 0x9c, 0xee, 0xf5, 0x71, 0x24, 0xb9, 0x13, 0xf1, 0x2c, 0x2b, 0xac, 0xc8, - 0x0e, 0xb9, 0x07, 0xcd, 0x99, 0x13, 0x9c, 0x4e, 0x0e, 0xc5, 0x01, 0xbe, 0xe2, 0x0a, 0xbb, 0x0b, - 0x2b, 0x94, 0x3b, 0x4f, 0x5c, 0x9f, 0x0e, 0x25, 0xf7, 0x2b, 0x76, 0x9d, 0xf2, 0xf7, 0x44, 0x57, - 0xc4, 0x8e, 0x3c, 0x64, 0xab, 0x9d, 0x66, 0xd8, 0x33, 0x16, 0xeb, 0x7d, 0x80, 0xd4, 0x35, 0x41, - 0x4d, 0xae, 0xac, 0x36, 0xab, 0xec, 0xd7, 0xa1, 0xee, 0xe1, 0x74, 0x8f, 0xe3, 0xa8, 0xa3, 0xcb, - 0xc0, 0xbb, 0x88, 0xaf, 0x74, 0x26, 0x3b, 0x1b, 0x6e, 0xfd, 0x52, 0x83, 0xc6, 0x77, 0x23, 0x77, - 0xd8, 0x0b, 0x87, 0xf8, 0xf4, 0x3a, 0xc4, 0x97, 0xae, 0x22, 0xfd, 0xe2, 0x55, 0xd4, 0x81, 0xfa, - 0x29, 0x45, 0x7f, 0x58, 0x1c, 0xc0, 0xb4, 0x2b, 0x88, 0xa5, 0x62, 0x6d, 0x27, 0x76, 0x93, 0x31, - 0x97, 0xc5, 0x99, 0x86, 0x0d, 0xd2, 0x74, 0x24, 0x2c, 0xd6, 0x73, 0x0d, 0x40, 0x7a, 0x27, 0xf6, - 0x38, 0x27, 0x5d, 0x68, 0x65, 0xe3, 0x99, 0x1b, 0xf0, 0x8e, 0x26, 0xf1, 0xde, 0x9f, 0x7b, 0x2a, - 0x1e, 0xe1, 0xf4, 0x3d, 0xd7, 0x9f, 0xe0, 0x91, 0x4b, 0x99, 0xdd, 0x4c, 0xe7, 0x14, 0x7f, 0x91, - 0xaf, 0xc0, 0x46, 0x38, 0x09, 0x1c, 0x86, 0xbe, 0x9b, 0xe0, 0xd0, 0x49, 0x1d, 0xe5, 0xa9, 0xe3, - 0x24, 0x9c, 0x04, 0xb6, 0xfa, 0xd4, 0x4f, 0xbf, 0x58, 0x3f, 0xd2, 0x00, 0xbe, 0x25, 0x7c, 0x56, - 0x6e, 0x5c, 0x7c, 0xda, 0x6b, 0x73, 0x9e, 0xf6, 0x33, 0xa0, 0xf5, 0x32, 0xe8, 0xfd, 0x0c, 0xb4, - 0x38, 0xaf, 0x3c, 0xad, 0x0c, 0xde, 0x5f, 0x40, 0x75, 0x01, 0x3e, 0xe5, 0x45, 0xb6, 0xad, 0x9f, - 0xa9, 0x62, 0xaa, 0xf0, 0x4e, 0xb9, 0x54, 0x52, 0x40, 0xbb, 0xa8, 0xc0, 0x3d, 0x68, 0x06, 0x18, - 0x44, 0x6c, 0xea, 0x70, 0xfa, 0x0c, 0xb3, 0x0d, 0xac, 0x4c, 0x7d, 0xfa, 0x0c, 0xc5, 0x16, 0x95, - 0x94, 0x44, 0xe7, 0x3c, 0xd3, 0x48, 0xd0, 0x10, 0x9d, 0x73, 0xf2, 0x25, 0xb8, 0xc5, 0x70, 0x80, - 0x61, 0xe2, 0x4f, 0x9d, 0x20, 0x1a, 0xd2, 0x53, 0x8a, 0xd9, 0x36, 0x36, 0xb3, 0x0f, 0x87, 0xa9, - 0xdd, 0xfa, 0x8b, 0x06, 0xed, 0xef, 0x65, 0x81, 0x43, 0x79, 0xf6, 0x12, 0xce, 0xf3, 0x37, 0x24, - 0xd8, 0x12, 0x7f, 0x4b, 0x2a, 0xab, 0x39, 0x49, 0xf6, 0x0a, 0xc7, 0x91, 0x72, 0x6a, 0x1f, 0x9a, - 0x52, 0x8e, 0x74, 0x8e, 0xca, 0x52, 0x0d, 0x0a, 0xe5, 0x6d, 0x38, 0xcd, 0xdb, 0x6f, 0xff, 0x5b, - 0x87, 0x7a, 0xea, 0x32, 0x69, 0x40, 0xd5, 0x7b, 0x1c, 0x85, 0x68, 0xbe, 0x46, 0x36, 0xe1, 0x96, - 0x77, 0xb1, 0x3a, 0x68, 0x0e, 0xc9, 0x3a, 0xac, 0x79, 0xe5, 0xc2, 0x99, 0x89, 0x84, 0x40, 0xdb, - 0x2b, 0xd5, 0x8d, 0xcc, 0x53, 0x72, 0x07, 0xd6, 0xbd, 0xcb, 0xa5, 0x15, 0x53, 0x9c, 0x7e, 0xd3, - 0x2b, 0x57, 0x1f, 0xb8, 0x39, 0x96, 0x53, 0x7c, 0x1b, 0x93, 0xfc, 0x1a, 0xe5, 0x26, 0x25, 0x9b, - 0x60, 0x7a, 0x17, 0x8a, 0x00, 0xe6, 0xef, 0x34, 0xb2, 0x0e, 0x6d, 0xaf, 0x94, 0xe5, 0x9a, 0xbf, - 0xd7, 0x08, 0x81, 0x55, 0x6f, 0x36, 0x8d, 0x33, 0xff, 0xa0, 0x91, 0x3b, 0x40, 0xbc, 0x4b, 0xd9, - 0x8e, 0xf9, 0x47, 0x8d, 0x6c, 0xc0, 0x9a, 0x57, 0x4a, 0x0a, 0xb8, 0xf9, 0x27, 0x8d, 0xb4, 0xa0, - 0xee, 0xa9, 0x97, 0xb3, 0xf9, 0x63, 0x43, 0xf6, 0xd4, 0x93, 0xce, 0xfc, 0x89, 0xea, 0xa9, 0xdb, - 0xd0, 0xfc, 0xd8, 0x90, 0x8b, 0xcd, 0xde, 0x8d, 0xe6, 0x3f, 0x0d, 0xd2, 0x86, 0x86, 0x97, 0xc5, - 0x7f, 0xf3, 0xd7, 0x0d, 0xb9, 0x46, 0x79, 0x0b, 0x99, 0x1f, 0x36, 0xc8, 0x1a, 0x80, 0x97, 0x87, - 0x2a, 0xf3, 0x37, 0x8d, 0xb7, 0xdf, 0x81, 0x95, 0xac, 0xea, 0x4e, 0x00, 0x6a, 0x87, 0x2e, 0x4f, - 0x90, 0x99, 0xaf, 0x89, 0xb6, 0x8d, 0xee, 0x10, 0x99, 0xa9, 0x89, 0xf6, 0xf7, 0x19, 0x15, 0x76, - 0x5d, 0x28, 0x74, 0x24, 0x2e, 0x3c, 0xd3, 0xd8, 0xef, 0xfe, 0x60, 0x7f, 0x44, 0x93, 0xf1, 0xe4, - 0x44, 0x84, 0x8a, 0xdd, 0x67, 0xd4, 0xf7, 0xe9, 0xb3, 0x04, 0x07, 0xe3, 0x5d, 0x25, 0xff, 0x97, - 0x87, 0x94, 0x27, 0x8c, 0x9e, 0x4c, 0x12, 0x1c, 0xee, 0x66, 0x9b, 0x60, 0x57, 0xee, 0x89, 0xbc, - 0x1b, 0x9f, 0x9c, 0xd4, 0xa4, 0xe5, 0xe1, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x9a, 0xfa, 0x64, - 0x88, 0xe6, 0x1a, 0x00, 0x00, + // 1725 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x59, 0xcd, 0x6f, 0x23, 0x49, + 0x15, 0xdf, 0x72, 0xfb, 0xf3, 0xd9, 0x71, 0x7a, 0x2a, 0xc9, 0x8c, 0x67, 0x17, 0xed, 0x64, 0x7a, + 0x10, 0x1b, 0x16, 0x91, 0x40, 0x86, 0x03, 0x7b, 0x83, 0xc4, 0x5a, 0xd6, 0x0c, 0x19, 0x85, 0x76, + 0xb4, 0x48, 0x68, 0xa5, 0x56, 0xc7, 0x7e, 0xb1, 0x5b, 0xfd, 0x99, 0xaa, 0xf6, 0x64, 0x3c, 0x07, + 0x24, 0xa4, 0x39, 0x70, 0x42, 0x7c, 0x88, 0x03, 0x07, 0x04, 0x7f, 0xc1, 0x8a, 0x85, 0x7f, 0x82, + 0xaf, 0x13, 0xff, 0x05, 0x1c, 0x58, 0x09, 0x96, 0x03, 0x37, 0x54, 0x55, 0xfd, 0xe1, 0x4e, 0x6c, + 0x67, 0x44, 0x92, 0x65, 0xd0, 0x70, 0xab, 0x7a, 0x5d, 0xae, 0x7a, 0xbf, 0xdf, 0xef, 0xd5, 0xab, + 0x7a, 0x65, 0xa0, 0x4e, 0x10, 0x23, 0x0b, 0x6c, 0xcf, 0xf2, 0xf9, 0x68, 0x3b, 0x62, 0x61, 0x1c, + 0xd2, 0x0d, 0xdf, 0xf1, 0x9e, 0x4c, 0xb8, 0xea, 0x6d, 0xa7, 0x03, 0x5e, 0x6f, 0x0d, 0x42, 0xdf, + 0x0f, 0x03, 0x65, 0x7e, 0xfd, 0x16, 0x47, 0xf6, 0xc4, 0x19, 0x60, 0xfe, 0x3b, 0x23, 0x80, 0x46, + 0xaf, 0x6b, 0xe2, 0xe9, 0x04, 0x79, 0x4c, 0x6f, 0x43, 0x35, 0x42, 0x64, 0xbd, 0x6e, 0x87, 0x6c, + 0x92, 0x2d, 0xcd, 0x4c, 0x7a, 0xf4, 0x21, 0x94, 0x59, 0xe8, 0x61, 0xa7, 0xb4, 0x49, 0xb6, 0xda, + 0xbb, 0xf7, 0xb6, 0xe7, 0xae, 0xb5, 0x7d, 0x88, 0xc8, 0xcc, 0xd0, 0x43, 0x53, 0x0e, 0xa6, 0xeb, + 0x50, 0x19, 0x84, 0x93, 0x20, 0xee, 0x68, 0x9b, 0x64, 0x6b, 0xc5, 0x54, 0x1d, 0x63, 0x04, 0x20, + 0xd6, 0xe3, 0x51, 0x18, 0x70, 0xa4, 0x0f, 0xa1, 0xca, 0x63, 0x3b, 0x9e, 0x70, 0xb9, 0x60, 0x73, + 0xf7, 0x8d, 0xe2, 0xd4, 0x89, 0xf3, 0x7d, 0x39, 0xc4, 0x4c, 0x86, 0xd2, 0x36, 0x94, 0x7a, 0x5d, + 0xe9, 0x8b, 0x66, 0x96, 0x7a, 0xdd, 0x05, 0x0b, 0x85, 0x00, 0x47, 0x3c, 0xfc, 0x14, 0x91, 0x3d, + 0x81, 0xa6, 0x5c, 0xf0, 0x2a, 0xd0, 0x3e, 0x03, 0x8d, 0xd8, 0xf1, 0x91, 0xc7, 0xb6, 0x1f, 0x49, + 0x9f, 0xca, 0x66, 0x6e, 0x58, 0xb0, 0xee, 0x73, 0x02, 0xad, 0x3e, 0x8e, 0x72, 0x15, 0xb3, 0x61, + 0x64, 0x66, 0x98, 0x98, 0x7a, 0x30, 0xb6, 0x83, 0x00, 0xbd, 0x84, 0xbc, 0x8a, 0x99, 0x1b, 0xe8, + 0x1b, 0xd0, 0x18, 0x84, 0x9e, 0x67, 0x05, 0xb6, 0x8f, 0x72, 0xfa, 0x86, 0x59, 0x17, 0x86, 0xc7, + 0xb6, 0x8f, 0xf4, 0x01, 0xac, 0x44, 0x36, 0x8b, 0x9d, 0xd8, 0x09, 0x03, 0x2b, 0xb6, 0x47, 0x9d, + 0xb2, 0x1c, 0xd0, 0xca, 0x8c, 0x47, 0xf6, 0xc8, 0xf8, 0x90, 0x00, 0xfd, 0x3a, 0xe7, 0xce, 0x28, + 0x28, 0x38, 0x73, 0xad, 0xc4, 0x3f, 0x82, 0xd5, 0x08, 0x99, 0x95, 0xb8, 0x6d, 0x31, 0x3c, 0xed, + 0x68, 0x9b, 0xda, 0x56, 0x73, 0xf7, 0xc1, 0x82, 0xdf, 0xcf, 0xba, 0x62, 0xae, 0x44, 0xc8, 0xf6, + 0xd5, 0x4f, 0x4d, 0x3c, 0x35, 0x7e, 0x41, 0x60, 0x55, 0x7e, 0x57, 0x5e, 0xfb, 0x18, 0x48, 0xea, + 0xb8, 0x30, 0x25, 0xce, 0xaa, 0xce, 0x25, 0xd4, 0xcd, 0x55, 0xa5, 0x48, 0x68, 0xf9, 0x32, 0x42, + 0x2b, 0x73, 0x08, 0xfd, 0x98, 0xc0, 0x5a, 0x81, 0xd0, 0x9b, 0x0b, 0xac, 0xb7, 0x60, 0x15, 0x9f, + 0x46, 0x0e, 0x43, 0x6b, 0x38, 0x61, 0xb6, 0x70, 0x40, 0x82, 0x29, 0x9b, 0x6d, 0x65, 0xee, 0x26, + 0x56, 0xfa, 0x01, 0xdc, 0x9e, 0x15, 0xc0, 0xce, 0x98, 0xeb, 0x94, 0xa5, 0x0e, 0x9f, 0x5b, 0xa6, + 0x43, 0xce, 0xb3, 0xb9, 0x9e, 0x4b, 0x91, 0x5b, 0x8d, 0x7f, 0x11, 0xb8, 0xb3, 0xcf, 0xd0, 0x8e, + 0x71, 0x3f, 0xf4, 0x3c, 0x1c, 0x88, 0x25, 0xd3, 0x38, 0x7a, 0x07, 0xea, 0x3e, 0x1f, 0x59, 0xf1, + 0x34, 0x42, 0x89, 0xbb, 0xbd, 0xfb, 0xe6, 0x82, 0xb5, 0x0e, 0xf8, 0xe8, 0x68, 0x1a, 0xa1, 0x59, + 0xf3, 0x55, 0x83, 0x1a, 0xd0, 0x1a, 0x64, 0xf3, 0x65, 0x99, 0xa3, 0x60, 0x13, 0x22, 0x32, 0x3c, + 0xed, 0x75, 0x25, 0x6e, 0xcd, 0x54, 0x9d, 0x22, 0x6b, 0xe5, 0xf3, 0xac, 0x75, 0xa0, 0x16, 0xb1, + 0xf0, 0xe9, 0xb4, 0xd7, 0x95, 0xfa, 0x69, 0x66, 0xda, 0xa5, 0x5f, 0x86, 0x2a, 0x1f, 0x8c, 0xd1, + 0xb7, 0x3b, 0x55, 0x29, 0xd1, 0xdd, 0xb9, 0x12, 0xed, 0x79, 0xe1, 0xb1, 0x99, 0x0c, 0x34, 0x7e, + 0x5a, 0x82, 0x8d, 0x2e, 0x0b, 0xa3, 0xff, 0x71, 0xe4, 0x07, 0xb0, 0x9a, 0xcf, 0xae, 0x82, 0x5f, + 0x51, 0xf0, 0xd9, 0xa2, 0xcf, 0xc9, 0x41, 0xb4, 0x9d, 0xc3, 0x15, 0x1b, 0xc3, 0x6c, 0x0f, 0x0a, + 0x7d, 0xe3, 0x6f, 0x04, 0xd6, 0xdf, 0xb3, 0xf9, 0xb5, 0x92, 0x92, 0x01, 0x2e, 0x2d, 0x04, 0xac, + 0x2d, 0x01, 0x5c, 0xbe, 0x14, 0x70, 0xe5, 0x0a, 0x80, 0xff, 0x41, 0xe0, 0x6e, 0x17, 0xf9, 0x80, + 0x39, 0xc7, 0xf8, 0xea, 0xa0, 0xfe, 0x15, 0x81, 0x8d, 0xfe, 0x38, 0x3c, 0x7b, 0x79, 0x11, 0x1b, + 0xbf, 0x2d, 0xc1, 0x6d, 0x95, 0x9b, 0x0e, 0xd3, 0x24, 0xfd, 0x29, 0x6d, 0xd0, 0x4d, 0x68, 0x66, + 0xe7, 0x42, 0xb6, 0x4d, 0x67, 0x4d, 0x39, 0xd2, 0xf2, 0x42, 0xa4, 0x95, 0x25, 0x48, 0xab, 0x45, + 0x6d, 0xbf, 0x09, 0xed, 0xfc, 0x70, 0x92, 0xd2, 0xd6, 0xa4, 0xb4, 0x0f, 0xe6, 0x4b, 0x9b, 0xd1, + 0x21, 0x95, 0xcd, 0xcf, 0x35, 0x29, 0xec, 0x47, 0x25, 0x58, 0x17, 0x59, 0xed, 0xff, 0x9c, 0xbd, + 0x38, 0x67, 0x7f, 0x25, 0xb0, 0xf6, 0x9e, 0xcd, 0xaf, 0x93, 0xb2, 0xeb, 0xdd, 0xfc, 0x17, 0xc1, + 0x56, 0xfe, 0x63, 0xb0, 0x1f, 0x13, 0xe8, 0xa4, 0xf9, 0xee, 0xd5, 0x40, 0x2c, 0x8e, 0x34, 0x91, + 0xeb, 0x5e, 0x5e, 0xb4, 0xd7, 0x9c, 0xdc, 0xff, 0x5e, 0x82, 0x95, 0x5e, 0xc0, 0x91, 0xc5, 0x37, + 0x86, 0xf4, 0xad, 0x8b, 0x1e, 0xab, 0x1a, 0xe6, 0x9c, 0x2f, 0x2f, 0x54, 0xc9, 0x08, 0xde, 0x38, + 0x8e, 0xc4, 0x8d, 0x34, 0xbb, 0xdf, 0xe4, 0x86, 0x62, 0x31, 0xa0, 0xd2, 0xc0, 0x4c, 0x31, 0x30, + 0xc3, 0x6a, 0xad, 0xc8, 0xea, 0x9b, 0x00, 0x19, 0xf9, 0xbc, 0x53, 0xdf, 0xd4, 0xb6, 0xca, 0xe6, + 0x8c, 0x45, 0x14, 0x4a, 0x2c, 0x3c, 0xeb, 0x75, 0x79, 0xa7, 0xb1, 0xa9, 0x89, 0x42, 0x49, 0xf5, + 0xe8, 0x57, 0xa0, 0xce, 0xc2, 0x33, 0x6b, 0x68, 0xc7, 0x76, 0x07, 0xe4, 0x25, 0x7b, 0xc9, 0x6d, + 0xb2, 0xc6, 0xc2, 0xb3, 0xae, 0x1d, 0xdb, 0xc6, 0xf3, 0x12, 0xac, 0x74, 0xd1, 0xc3, 0x18, 0xff, + 0xfb, 0xa4, 0x17, 0x18, 0x2b, 0x2f, 0x61, 0xac, 0xb2, 0x8c, 0xb1, 0xea, 0x05, 0xc6, 0xee, 0x43, + 0x2b, 0x62, 0x8e, 0x6f, 0xb3, 0xa9, 0xe5, 0xe2, 0x94, 0x77, 0x6a, 0x92, 0xb7, 0x66, 0x62, 0x7b, + 0x84, 0x53, 0x6e, 0x7c, 0x42, 0x60, 0xa5, 0x8f, 0x36, 0x1b, 0x8c, 0x6f, 0x8c, 0x86, 0x19, 0xff, + 0xb5, 0xa2, 0xff, 0xcb, 0xef, 0xd0, 0x9f, 0x07, 0x9d, 0x21, 0x9f, 0x78, 0xb1, 0x95, 0x93, 0xa3, + 0x08, 0x58, 0x55, 0xf6, 0xfd, 0x8c, 0xa2, 0x1d, 0xa8, 0x9c, 0x4e, 0x90, 0x4d, 0x2f, 0xaf, 0x26, + 0xd4, 0x38, 0xe3, 0xcf, 0x04, 0xf4, 0xfe, 0x94, 0xef, 0x87, 0xc1, 0x89, 0x33, 0x7a, 0xe9, 0x90, + 0x53, 0x28, 0x4b, 0xbd, 0x2a, 0x9b, 0xda, 0x56, 0xc3, 0x94, 0x6d, 0xa1, 0xa5, 0x8b, 0x53, 0x2b, + 0x62, 0x78, 0xe2, 0x3c, 0x45, 0xa5, 0x76, 0xc3, 0x6c, 0xba, 0x38, 0x3d, 0x4c, 0x4c, 0xc6, 0x87, + 0x25, 0x68, 0xa5, 0x5a, 0x0a, 0x7e, 0xae, 0x02, 0x28, 0xaf, 0xa1, 0x4b, 0x2f, 0x5e, 0x43, 0xcf, + 0xaf, 0x94, 0x16, 0xe7, 0xd1, 0xfb, 0xd0, 0x92, 0x72, 0x58, 0x41, 0x38, 0xc4, 0x4c, 0xdd, 0xa6, + 0xb4, 0x3d, 0x96, 0xa6, 0x22, 0x51, 0xd5, 0x17, 0x09, 0x91, 0xda, 0xfc, 0x10, 0xa1, 0x50, 0x1e, + 0x3b, 0xb1, 0xca, 0x2b, 0x2d, 0x53, 0xb6, 0x8d, 0xef, 0x41, 0xf3, 0xc8, 0xf1, 0xf1, 0xc8, 0x19, + 0xb8, 0x07, 0x7c, 0x74, 0x15, 0xba, 0xf2, 0x47, 0x9c, 0x52, 0xe1, 0x11, 0x67, 0xe9, 0x09, 0x63, + 0x7c, 0x9f, 0x40, 0xfd, 0x5d, 0x6f, 0xc2, 0xc7, 0x57, 0x5c, 0xbd, 0x90, 0x8f, 0x4b, 0x73, 0xf2, + 0xf1, 0x12, 0x1f, 0x7e, 0x4e, 0xa0, 0xf6, 0x08, 0xa7, 0xbb, 0x7d, 0x1c, 0x49, 0xfd, 0x44, 0x4e, + 0x4d, 0x1f, 0x77, 0x64, 0x87, 0xde, 0x83, 0xe6, 0x4c, 0x16, 0x49, 0xe6, 0x87, 0x3c, 0x89, 0x5c, + 0x72, 0x8c, 0xde, 0x85, 0xba, 0xc3, 0xad, 0x27, 0xb6, 0xe7, 0x0c, 0xa5, 0xfe, 0x75, 0xb3, 0xe6, + 0xf0, 0xf7, 0x45, 0x57, 0xe4, 0xaf, 0xcc, 0x4d, 0x15, 0xed, 0x9a, 0x39, 0x63, 0x31, 0x3e, 0x00, + 0x48, 0x5c, 0x13, 0x04, 0x65, 0xd1, 0x45, 0x66, 0xa3, 0xeb, 0xab, 0x50, 0x73, 0x71, 0xba, 0xcb, + 0x71, 0xd4, 0x29, 0xc9, 0xe4, 0xbf, 0x88, 0xb5, 0x64, 0x26, 0x33, 0x1d, 0x6e, 0xfc, 0x92, 0x40, + 0xe3, 0x5b, 0xa1, 0x3d, 0xec, 0x05, 0x43, 0x7c, 0x7a, 0x73, 0xf4, 0x77, 0xa0, 0x76, 0xe2, 0xa0, + 0x37, 0xcc, 0x93, 0x40, 0xd2, 0x15, 0xc4, 0x3a, 0x62, 0x6d, 0x2b, 0xb2, 0xe3, 0x31, 0x97, 0x0f, + 0x44, 0x0d, 0x13, 0xa4, 0xe9, 0x50, 0x58, 0x8c, 0xe7, 0x04, 0x40, 0x7a, 0x27, 0xf6, 0x19, 0xa7, + 0x5d, 0x68, 0xa5, 0xe3, 0x99, 0xed, 0xf3, 0x0e, 0x91, 0x78, 0xef, 0xcf, 0xdd, 0x99, 0x8f, 0x70, + 0xfa, 0xbe, 0xed, 0x4d, 0xf0, 0xd0, 0x76, 0x98, 0xd9, 0x4c, 0xe6, 0x14, 0xbf, 0xa2, 0x5f, 0x82, + 0xf5, 0x60, 0xe2, 0x5b, 0x0c, 0x3d, 0x3b, 0xc6, 0xa1, 0x95, 0x38, 0xca, 0x13, 0xc7, 0x69, 0x30, + 0xf1, 0x4d, 0xf5, 0xa9, 0x9f, 0x7c, 0x31, 0x7e, 0x48, 0x00, 0xde, 0x15, 0x3e, 0x2b, 0x37, 0xce, + 0x97, 0x17, 0x64, 0x4e, 0x79, 0x31, 0x03, 0xba, 0x54, 0x04, 0xbd, 0x97, 0x82, 0x16, 0x39, 0x83, + 0x27, 0xaf, 0x93, 0xf7, 0x17, 0x50, 0x9d, 0x83, 0x4f, 0x78, 0x91, 0x6d, 0xe3, 0x67, 0xea, 0x41, + 0x57, 0x78, 0xa7, 0x5c, 0x2a, 0x28, 0x40, 0xce, 0x2b, 0x70, 0x0f, 0x9a, 0x3e, 0xfa, 0x21, 0x9b, + 0x5a, 0xdc, 0x79, 0x86, 0x69, 0x00, 0x2b, 0x53, 0xdf, 0x79, 0x86, 0x22, 0x44, 0x25, 0x25, 0xe1, + 0x19, 0x4f, 0x35, 0x12, 0x34, 0x84, 0x67, 0x9c, 0x7e, 0x01, 0x6e, 0x31, 0x1c, 0x60, 0x10, 0x7b, + 0x53, 0xcb, 0x0f, 0x87, 0xce, 0x89, 0x83, 0x69, 0x18, 0xeb, 0xe9, 0x87, 0x83, 0xc4, 0x6e, 0xfc, + 0x85, 0x40, 0xfb, 0xdb, 0x69, 0xf2, 0x52, 0x9e, 0xdd, 0x40, 0x4e, 0xf9, 0x9a, 0x04, 0x5b, 0xe0, + 0x6f, 0xc9, 0xeb, 0x6e, 0x46, 0x92, 0x59, 0xe7, 0x38, 0x52, 0x4e, 0xed, 0x41, 0x53, 0xca, 0x91, + 0xcc, 0x51, 0x5e, 0xaa, 0x41, 0xae, 0xbc, 0x09, 0x27, 0x59, 0xfb, 0xed, 0x1f, 0x68, 0x50, 0x4b, + 0x5c, 0xa6, 0x0d, 0xa8, 0xb8, 0x8f, 0xc3, 0x00, 0xf5, 0xd7, 0xe8, 0x06, 0xdc, 0x72, 0xcf, 0xbf, + 0x50, 0xea, 0x43, 0xba, 0x06, 0xab, 0x6e, 0xf1, 0xf1, 0x4e, 0x47, 0x4a, 0xa1, 0xed, 0x16, 0xde, + 0xae, 0xf4, 0x13, 0x7a, 0x07, 0xd6, 0xdc, 0x8b, 0xcf, 0x3b, 0xba, 0xd8, 0xfd, 0xba, 0x5b, 0x7c, + 0x01, 0xe1, 0xfa, 0x58, 0x4e, 0xf1, 0x0d, 0x8c, 0xb3, 0xa3, 0x9c, 0xeb, 0x0e, 0xdd, 0x00, 0xdd, + 0x3d, 0xf7, 0x10, 0xa1, 0xff, 0x8e, 0xd0, 0x35, 0x68, 0xbb, 0x85, 0x4a, 0x5b, 0xff, 0x3d, 0xa1, + 0x14, 0x56, 0xdc, 0xd9, 0x52, 0x52, 0xff, 0x03, 0xa1, 0x77, 0x80, 0xba, 0x17, 0x2a, 0x2e, 0xfd, + 0x8f, 0x84, 0xae, 0xc3, 0xaa, 0x5b, 0x28, 0x4c, 0xb8, 0xfe, 0x27, 0x42, 0x5b, 0x50, 0x73, 0xd5, + 0xed, 0x5d, 0xff, 0x91, 0x26, 0x7b, 0xea, 0x5a, 0xa9, 0xff, 0x58, 0xa3, 0x4d, 0xa8, 0xba, 0x32, + 0xc1, 0xeb, 0x3f, 0x51, 0x9f, 0xd4, 0xf1, 0xac, 0x7f, 0xa2, 0xc9, 0x95, 0x67, 0x0f, 0x6b, 0xfd, + 0x9f, 0x1a, 0x6d, 0x43, 0xc3, 0x4d, 0x0f, 0x24, 0xfd, 0xd7, 0x0d, 0xb9, 0x60, 0x31, 0x9e, 0xf4, + 0x8f, 0x1a, 0x74, 0x15, 0xc0, 0xcd, 0xf2, 0x96, 0xfe, 0x9b, 0xc6, 0xdb, 0xef, 0x40, 0x3d, 0xfd, + 0x1b, 0x80, 0x02, 0x54, 0x0f, 0x6c, 0x1e, 0x23, 0xd3, 0x5f, 0x13, 0x6d, 0x13, 0xed, 0x21, 0x32, + 0x9d, 0x88, 0xf6, 0x77, 0x98, 0x23, 0xec, 0x25, 0x21, 0xd7, 0xa1, 0x38, 0x81, 0x75, 0x6d, 0xaf, + 0xfb, 0xdd, 0xbd, 0x91, 0x13, 0x8f, 0x27, 0xc7, 0x22, 0x6f, 0xec, 0x3c, 0x73, 0x3c, 0xcf, 0x79, + 0x16, 0xe3, 0x60, 0xbc, 0xa3, 0x62, 0xe1, 0x8b, 0x43, 0x87, 0xc7, 0xcc, 0x39, 0x9e, 0xc4, 0x38, + 0xdc, 0x49, 0x23, 0x62, 0x47, 0x06, 0x48, 0xd6, 0x8d, 0x8e, 0x8f, 0xab, 0xd2, 0xf2, 0xf0, 0xdf, + 0x01, 0x00, 0x00, 0xff, 0xff, 0x0f, 0x64, 0x4d, 0xeb, 0x77, 0x1b, 0x00, 0x00, } diff --git a/internal/querynode/search_service.go b/internal/querynode/search_service.go index df6006fa1c..6642eed1ff 100644 --- a/internal/querynode/search_service.go +++ b/internal/querynode/search_service.go @@ -359,10 +359,7 @@ func (ss *searchService) publishSearchResult(msg msgstream.TsMsg) error { msgPack := msgstream.MsgPack{} msgPack.Msgs = append(msgPack.Msgs, msg) err := ss.searchResultMsgStream.Produce(&msgPack) - if err != nil { - return err - } - return nil + return err } func (ss *searchService) publishFailedSearchResult(msg msgstream.TsMsg, errMsg string) error { diff --git a/internal/storage/data_codec.go b/internal/storage/data_codec.go index 55fc3b1252..942f48b5a1 100644 --- a/internal/storage/data_codec.go +++ b/internal/storage/data_codec.go @@ -15,6 +15,7 @@ const ( type ( UniqueID = typeutil.UniqueID + FieldID = typeutil.UniqueID Timestamp = typeutil.Timestamp ) @@ -86,7 +87,7 @@ type FloatVectorFieldData struct { // example row_schema: {float_field, int_field, float_vector_field, string_field} // Data {<0, row_id>, <1, timestamp>, <100, float_field>, <101, int_field>, <102, float_vector_field>, <103, string_field>} type InsertData struct { - Data map[int64]FieldData // field id to field data + Data map[FieldID]FieldData // field id to field data } // Blob key example: @@ -179,7 +180,7 @@ func (insertCodec *InsertCodec) Deserialize(blobs []*Blob) (partitionID UniqueID var resultData InsertData var pID UniqueID var sID UniqueID - resultData.Data = make(map[int64]FieldData) + resultData.Data = make(map[FieldID]FieldData) for _, blob := range blobs { binlogReader, err := NewBinlogReader(blob.value) if err != nil { diff --git a/internal/writenode/data_sync_service.go b/internal/writenode/data_sync_service.go index 4c5e8303b7..a58c3afd33 100644 --- a/internal/writenode/data_sync_service.go +++ b/internal/writenode/data_sync_service.go @@ -38,12 +38,12 @@ func (dsService *dataSyncService) initNodes() { var dmStreamNode Node = newDmInputNode(dsService.ctx) var filterDmNode Node = newFilteredDmNode() - var writeNode Node = newWriteNode() + var insertBufferNode Node = newInsertBufferNode() var serviceTimeNode Node = newServiceTimeNode() dsService.fg.AddNode(&dmStreamNode) dsService.fg.AddNode(&filterDmNode) - dsService.fg.AddNode(&writeNode) + dsService.fg.AddNode(&insertBufferNode) dsService.fg.AddNode(&serviceTimeNode) var err = dsService.fg.SetEdges(dmStreamNode.Name(), @@ -56,22 +56,22 @@ func (dsService *dataSyncService) initNodes() { err = dsService.fg.SetEdges(filterDmNode.Name(), []string{dmStreamNode.Name()}, - []string{writeNode.Name()}, + []string{insertBufferNode.Name()}, ) if err != nil { log.Fatal("set edges failed in node:", filterDmNode.Name()) } - err = dsService.fg.SetEdges(writeNode.Name(), + err = dsService.fg.SetEdges(insertBufferNode.Name(), []string{filterDmNode.Name()}, []string{serviceTimeNode.Name()}, ) if err != nil { - log.Fatal("set edges failed in node:", writeNode.Name()) + log.Fatal("set edges failed in node:", insertBufferNode.Name()) } err = dsService.fg.SetEdges(serviceTimeNode.Name(), - []string{writeNode.Name()}, + []string{insertBufferNode.Name()}, []string{}, ) if err != nil { diff --git a/internal/writenode/data_sync_service_test.go b/internal/writenode/data_sync_service_test.go index 095699f1a9..646fb64ad3 100644 --- a/internal/writenode/data_sync_service_test.go +++ b/internal/writenode/data_sync_service_test.go @@ -4,7 +4,6 @@ import ( "context" "encoding/binary" "math" - "math/rand" "testing" "time" @@ -66,8 +65,6 @@ func TestDataSyncService_Start(t *testing.T) { const MSGLENGTH = 10 insertMessages := make([]msgstream.TsMsg, 0) for i := 0; i < MSGLENGTH; i++ { - randt := rand.Intn(MSGLENGTH) - // randt := i var msg msgstream.TsMsg = &msgstream.InsertMsg{ BaseMsg: msgstream.BaseMsg{ HashValues: []uint32{ @@ -76,14 +73,15 @@ func TestDataSyncService_Start(t *testing.T) { }, InsertRequest: internalPb.InsertRequest{ MsgType: internalPb.MsgType_kInsert, - ReqID: int64(0), + ReqID: UniqueID(0), CollectionName: "collection0", PartitionTag: "default", - SegmentID: int64(0), - ChannelID: int64(0), - ProxyID: int64(0), - Timestamps: []uint64{uint64(randt + 1000), uint64(randt + 1000)}, - RowIDs: []int64{int64(i), int64(i)}, + SegmentID: UniqueID(0), + ChannelID: UniqueID(0), + ProxyID: UniqueID(0), + Timestamps: []Timestamp{Timestamp(i + 1000), Timestamp(i + 1000)}, + RowIDs: []UniqueID{UniqueID(i), UniqueID(i)}, + RowData: []*commonpb.Blob{ {Value: rawData}, {Value: rawData}, @@ -104,8 +102,8 @@ func TestDataSyncService_Start(t *testing.T) { timeTickMsg := &msgstream.TimeTickMsg{ BaseMsg: msgstream.BaseMsg{ - BeginTimestamp: 0, - EndTimestamp: 0, + BeginTimestamp: Timestamp(0), + EndTimestamp: Timestamp(0), HashValues: []uint32{0}, }, TimeTickMsg: internalPb.TimeTickMsg{ diff --git a/internal/writenode/flow_graph_filter_dm_node.go b/internal/writenode/flow_graph_filter_dm_node.go index 23533f07e2..b01ecfcb56 100644 --- a/internal/writenode/flow_graph_filter_dm_node.go +++ b/internal/writenode/flow_graph_filter_dm_node.go @@ -39,6 +39,8 @@ func (fdmNode *filterDmNode) Operate(in []*Msg) []*Msg { switch msg.Type() { case internalPb.MsgType_kInsert: iMsg.insertMessages = append(iMsg.insertMessages, msg.(*msgstream.InsertMsg)) + case internalPb.MsgType_kFlush: + iMsg.insertMessages = append(iMsg.insertMessages, msg.(*msgstream.InsertMsg)) // case internalPb.MsgType_kDelete: // dmMsg.deleteMessages = append(dmMsg.deleteMessages, (*msg).(*msgstream.DeleteTask)) default: diff --git a/internal/writenode/flow_graph_insert_buffer_node.go b/internal/writenode/flow_graph_insert_buffer_node.go index aa7731802e..d027f6a94b 100644 --- a/internal/writenode/flow_graph_insert_buffer_node.go +++ b/internal/writenode/flow_graph_insert_buffer_node.go @@ -2,23 +2,39 @@ package writenode import ( "log" + + "github.com/zilliztech/milvus-distributed/internal/storage" ) type ( - writeNode struct { + insertBufferNode struct { BaseNode + binLogs map[SegmentID][]*storage.Blob // Binary logs of a segment. + buffer *insertBuffer + } + + insertBufferData struct { + logIdx int // TODO What's it for? + partitionID UniqueID + segmentID UniqueID + data *storage.InsertData + } + + insertBuffer struct { + buffer []*insertBufferData + maxSize int // TODO set from write_node.yaml } ) -func (wNode *writeNode) Name() string { - return "wNode" +func (ibNode *insertBufferNode) Name() string { + return "ibNode" } -func (wNode *writeNode) Operate(in []*Msg) []*Msg { - log.Println("=========== WriteNode Operating") +func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg { + log.Println("=========== insert buffer Node Operating") if len(in) != 1 { - log.Println("Invalid operate message input in writetNode, input length = ", len(in)) + log.Println("Invalid operate message input in insertBuffertNode, input length = ", len(in)) // TODO: add error handling } @@ -28,15 +44,25 @@ func (wNode *writeNode) Operate(in []*Msg) []*Msg { // TODO: add error handling } - log.Println("=========== insertMsg length:", len(iMsg.insertMessages)) - for _, task := range iMsg.insertMessages { - if len(task.RowIDs) != len(task.Timestamps) || len(task.RowIDs) != len(task.RowData) { - log.Println("Error, misaligned messages detected") - continue - } - log.Println("Timestamp: ", task.Timestamps[0]) - log.Printf("t(%d) : %v ", task.Timestamps[0], task.RowData[0]) - } + // iMsg is insertMsg + // 1. iMsg -> insertBufferData -> insertBuffer + // 2. Send hardTimeTick msg + // 3. if insertBuffer full + // 3.1 insertBuffer -> binLogs + // 3.2 binLogs -> minIO/S3 + // iMsg is Flush() msg from master + // 1. insertBuffer(not empty) -> binLogs -> minIO/S3 + // Return + + // log.Println("=========== insertMsg length:", len(iMsg.insertMessages)) + // for _, task := range iMsg.insertMessages { + // if len(task.RowIDs) != len(task.Timestamps) || len(task.RowIDs) != len(task.RowData) { + // log.Println("Error, misaligned messages detected") + // continue + // } + // log.Println("Timestamp: ", task.Timestamps[0]) + // log.Printf("t(%d) : %v ", task.Timestamps[0], task.RowData[0]) + // } var res Msg = &serviceTimeMsg{ timeRange: iMsg.timeRange, @@ -47,7 +73,7 @@ func (wNode *writeNode) Operate(in []*Msg) []*Msg { } -func newWriteNode() *writeNode { +func newInsertBufferNode() *insertBufferNode { maxQueueLength := Params.flowGraphMaxQueueLength() maxParallelism := Params.flowGraphMaxParallelism() @@ -55,7 +81,16 @@ func newWriteNode() *writeNode { baseNode.SetMaxQueueLength(maxQueueLength) baseNode.SetMaxParallelism(maxParallelism) - return &writeNode{ + // TODO read from yaml + maxSize := 10 + iBuffer := &insertBuffer{ + buffer: make([]*insertBufferData, maxSize), + maxSize: maxSize, + } + + return &insertBufferNode{ BaseNode: baseNode, + binLogs: make(map[SegmentID][]*storage.Blob), + buffer: iBuffer, } } diff --git a/internal/writenode/flow_graph_message.go b/internal/writenode/flow_graph_message.go index e6941150b8..5b387f36eb 100644 --- a/internal/writenode/flow_graph_message.go +++ b/internal/writenode/flow_graph_message.go @@ -24,6 +24,7 @@ type ( insertMsg struct { insertMessages []*msgstream.InsertMsg + flushMessages []*msgstream.FlushMsg timeRange TimeRange } diff --git a/internal/writenode/write_node.go b/internal/writenode/write_node.go index 1542467c8f..33adde07f3 100644 --- a/internal/writenode/write_node.go +++ b/internal/writenode/write_node.go @@ -5,20 +5,16 @@ import ( ) type WriteNode struct { - ctx context.Context - - WriteNodeID uint64 - + ctx context.Context + WriteNodeID uint64 dataSyncService *dataSyncService } func NewWriteNode(ctx context.Context, writeNodeID uint64) *WriteNode { return &WriteNode{ - ctx: ctx, - - WriteNodeID: writeNodeID, - + ctx: ctx, + WriteNodeID: writeNodeID, dataSyncService: nil, } }