From c2382adf04e42aa8c27808afdda567a9fa68fd7e Mon Sep 17 00:00:00 2001 From: dragondriver Date: Sat, 14 Nov 2020 11:24:49 +0800 Subject: [PATCH] Initialize the task scheduler of Proxy Signed-off-by: dragondriver --- cmd/master/main.go | 23 +- cmd/proxy/proxy.go | 11 +- cmd/reader/reader.go | 17 +- go.mod | 1 + go.sum | 151 ++++++++++++ internal/allocator/id_allocator.go | 13 +- internal/allocator/timestamp_allocator.go | 14 +- internal/core/src/pb/CMakeLists.txt | 1 - internal/core/src/query/CMakeLists.txt | 3 +- internal/core/src/query/Expr.h | 18 +- internal/core/src/query/ExprImpl.h | 16 -- internal/core/src/query/Plan.cpp | 16 +- internal/core/src/query/Plan.h | 4 +- internal/core/src/query/PlanNode.h | 1 + .../src/query/generated/ExecPlanNodeVisitor.h | 4 - .../src/query/generated/ShowExprVisitor.h | 34 --- .../src/query/generated/ShowPlanNodeVisitor.h | 6 - .../query/visitors/ExecPlanNodeVisitor.cpp | 6 +- .../src/query/visitors/ShowExprVisitor.cpp | 173 ------------- .../query/visitors/ShowPlanNodeVisitor.cpp | 9 +- internal/core/src/segcore/SegmentDefs.h | 30 --- .../core/src/segcore/SegmentSmallIndex.cpp | 11 +- internal/core/src/segcore/SegmentSmallIndex.h | 1 - internal/core/unittest/CMakeLists.txt | 1 - internal/core/unittest/test_expr.cpp | 70 ------ internal/core/unittest/test_query.cpp | 88 +------ internal/kv/etcd_kv_test.go | 47 +++- internal/kv/mockkv/mock_etcd.go | 10 +- internal/master/collection_task_test.go | 13 +- internal/master/controller/collection.go | 18 +- internal/master/controller/segment.go | 17 +- internal/master/controller/segment_test.go | 22 +- internal/master/grpc_service_test.go | 13 +- internal/master/id/id.go | 6 +- internal/master/id/id_test.go | 7 +- internal/master/informer/pulsar.go | 17 +- internal/master/master.go | 9 +- internal/master/meta_table_test.go | 35 ++- internal/master/mock/grpc_service.go | 4 +- internal/master/mock/master.go | 25 +- internal/master/partition_task_test.go | 15 +- internal/master/tso/global_allocator.go | 7 +- internal/master/tso/global_allocator_test.go | 7 +- internal/master/tso/tso.go | 4 +- internal/proxy/grpc_service.go | 232 +++++++++++++----- internal/proxy/proxy.go | 89 +++++-- internal/proxy/proxy_test.go | 89 +++++++ internal/proxy/task.go | 40 +-- internal/proxy/task_scheduler.go | 221 +++++++++-------- internal/proxy/timetick.go | 27 +- internal/proxy/timetick_test.go | 102 +++----- internal/proxy/util.go | 32 +++ internal/reader/meta_service.go | 65 ++++- internal/storage/internal/S3/s3_engine.go | 15 +- .../storage/internal/minio/minio_store.go | 11 + .../internal/minio/minio_storeEngine.go | 3 - internal/storage/internal/tikv/tikv_store.go | 20 +- internal/util/flowgraph/flow_graph_test.go | 4 +- internal/{kv => util/kvutil}/kv.go | 2 +- internal/{kv => util/kvutil}/mem_kv.go | 2 +- internal/util/paramtableutil/kv.go | 20 ++ internal/util/paramtableutil/paramtable.go | 66 +++++ .../util/paramtableutil/paramtable_test.go | 114 +++++++++ internal/util/tsoutil/tso.go | 22 +- tools/core_gen/all_generate.py | 17 +- tools/core_gen/templates/visitor_derived.h | 2 - 66 files changed, 1276 insertions(+), 917 deletions(-) delete mode 100644 internal/core/src/query/ExprImpl.h delete mode 100644 internal/core/src/query/visitors/ShowExprVisitor.cpp delete mode 100644 internal/core/unittest/test_expr.cpp create mode 100644 internal/proxy/proxy_test.go create mode 100644 internal/proxy/util.go rename internal/{kv => util/kvutil}/kv.go (96%) rename internal/{kv => util/kvutil}/mem_kv.go (99%) create mode 100644 internal/util/paramtableutil/kv.go create mode 100644 internal/util/paramtableutil/paramtable.go create mode 100644 internal/util/paramtableutil/paramtable_test.go diff --git a/cmd/master/main.go b/cmd/master/main.go index ff3ca927c7..6fa759b5c8 100644 --- a/cmd/master/main.go +++ b/cmd/master/main.go @@ -9,8 +9,8 @@ import ( "strconv" "syscall" - "github.com/zilliztech/milvus-distributed/internal/conf" "github.com/zilliztech/milvus-distributed/internal/master" + gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil" "go.uber.org/zap" ) @@ -21,15 +21,18 @@ func main() { flag.Parse() // flag.Usage() log.Println("yaml file: ", yamlFile) - conf.LoadConfig(yamlFile) + err := gparams.GParams.LoadYaml(yamlFile) + if err != nil { + panic(err) + } // Creates server. ctx, cancel := context.WithCancel(context.Background()) - etcdAddr := conf.Config.Etcd.Address - etcdAddr += ":" - etcdAddr += strconv.FormatInt(int64(conf.Config.Etcd.Port), 10) - - svr, err := master.CreateServer(ctx, conf.Config.Etcd.Rootpath, conf.Config.Etcd.Rootpath, conf.Config.Etcd.Rootpath, []string{etcdAddr}) + etcdAddress, _ := gparams.GParams.Load("etcd.address") + etcdPort, _ := gparams.GParams.Load("etcd.port") + etcdAddr := etcdAddress + ":" + etcdPort + etcdRootPath, _ := gparams.GParams.Load("etcd.rootpath") + svr, err := master.CreateServer(ctx, etcdRootPath, etcdRootPath, etcdRootPath, []string{etcdAddr}) if err != nil { log.Print("create server failed", zap.Error(err)) } @@ -47,7 +50,11 @@ func main() { cancel() }() - grpcPort := int64(conf.Config.Master.Port) + masterPort, _ := gparams.GParams.Load("master.port") + grpcPort, err := strconv.ParseInt(masterPort, 10, 64) + if err != nil { + panic(err) + } if err := svr.Run(grpcPort); err != nil { log.Fatal("run server failed", zap.Error(err)) diff --git a/cmd/proxy/proxy.go b/cmd/proxy/proxy.go index ddd02ceb2a..203cf1760a 100644 --- a/cmd/proxy/proxy.go +++ b/cmd/proxy/proxy.go @@ -8,10 +8,9 @@ import ( "os/signal" "syscall" - "go.uber.org/zap" - - "github.com/zilliztech/milvus-distributed/internal/conf" "github.com/zilliztech/milvus-distributed/internal/proxy" + gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil" + "go.uber.org/zap" ) func main() { @@ -20,7 +19,11 @@ func main() { flag.Parse() flag.Usage() log.Println("yaml file: ", yamlFile) - conf.LoadConfig(yamlFile) + + err := gparams.GParams.LoadYaml(yamlFile) + if err != nil { + panic(err) + } // Creates server. ctx, cancel := context.WithCancel(context.Background()) diff --git a/cmd/reader/reader.go b/cmd/reader/reader.go index 9cb0de3057..9f74abd673 100644 --- a/cmd/reader/reader.go +++ b/cmd/reader/reader.go @@ -6,11 +6,10 @@ import ( "fmt" "os" "os/signal" - "strconv" "syscall" - "github.com/zilliztech/milvus-distributed/internal/conf" "github.com/zilliztech/milvus-distributed/internal/reader" + gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil" ) func main() { @@ -22,7 +21,11 @@ func main() { flag.Parse() // flag.Usage() fmt.Println("yaml file: ", yamlFile) - conf.LoadConfig(yamlFile) + + err := gparams.GParams.LoadYaml(yamlFile) + if err != nil { + panic(err) + } sc := make(chan os.Signal, 1) signal.Notify(sc, @@ -36,11 +39,9 @@ func main() { sig = <-sc cancel() }() - - pulsarAddr := "pulsar://" - pulsarAddr += conf.Config.Pulsar.Address - pulsarAddr += ":" - pulsarAddr += strconv.FormatInt(int64(conf.Config.Pulsar.Port), 10) + pulsarAddr, _ := gparams.GParams.Load("pulsar.address") + pulsarPort, _ := gparams.GParams.Load("pulsar.port") + pulsarAddr += ":" + pulsarPort reader.StartQueryNode(ctx, pulsarAddr) switch sig { diff --git a/go.mod b/go.mod index 1eb6a53930..b5f9d7c194 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( github.com/sirupsen/logrus v1.6.0 // indirect github.com/smartystreets/goconvey v1.6.4 // indirect github.com/spaolacci/murmur3 v1.1.0 + github.com/spf13/viper v1.7.1 github.com/stretchr/testify v1.6.1 github.com/tikv/client-go v0.0.0-20200824032810-95774393107b github.com/yahoo/athenz v1.9.16 // indirect diff --git a/go.sum b/go.sum index 1c3093f8b0..8f530f558f 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,21 @@ +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= code.cloudfoundry.org/bytefmt v0.0.0-20200131002437-cf55d5288a48 h1:/EMHruHCFXR9xClkGV/t0rmHrdhX4+trQUcBqjwc9xE= code.cloudfoundry.org/bytefmt v0.0.0-20200131002437-cf55d5288a48/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -11,6 +25,9 @@ github.com/apache/pulsar-client-go v0.1.1/go.mod h1:mlxC65KL1BLhGO2bnT9zWMttVzR2 github.com/ardielle/ardielle-go v1.5.2 h1:TilHTpHIQJ27R1Tl/iITBzMwiUGSlVfiVhwDNGM3Zj4= github.com/ardielle/ardielle-go v1.5.2/go.mod h1:I4hy1n795cUhaVt/ojz83SNVCYIGsAFAONtv2Dr7HUI= github.com/ardielle/ardielle-tools v1.5.4/go.mod h1:oZN+JRMnqGiIhrzkRN9l26Cej9dEx4jeNG6A+AdkShk= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.30.8 h1:4BHbh8K3qKmcnAgToZ2LShldRF9inoqIBccpCLNCy3I= github.com/aws/aws-sdk-go v1.30.8/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6/go.mod h1:6YNgTHLutezwnBvyneBbwvB8C82y3dcoOj5EQJIdGXA= @@ -20,9 +37,12 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q= github.com/boynton/repl v0.0.0-20170116235056-348863958e3e/go.mod h1:Crc/GCZ3NXDVCio7Yr0o+SSrytpcFhLmVCIzi0s49t4= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= @@ -31,9 +51,13 @@ github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190212144455-93d5ec2c7f76 h1:FE783w8WFh+Rvg+7bZ5g8p7gP4SeVS4AoNwkvazlsBg= github.com/coreos/go-systemd v0.0.0-20190212144455-93d5ec2c7f76/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= @@ -45,6 +69,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dimfeld/httptreemux v5.0.1+incompatible h1:Qj3gVcDNoOthBAqftuD596rm4wg/adLLz5xh5CmpiCA= github.com/dimfeld/httptreemux v5.0.1+incompatible/go.mod h1:rbUlSV+CCpv/SuqUTP/8Bk2O3LyUV436/yaRGkhP6Z0= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -61,6 +86,7 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -78,6 +104,8 @@ github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -89,15 +117,21 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= @@ -107,14 +141,38 @@ github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.8.1/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jawher/mow.cli v1.0.4/go.mod h1:5hQj2V8g+qYmLUVWqu4Wuja1pI57M83EChYLVZ0sMKk= @@ -129,6 +187,7 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= @@ -152,19 +211,31 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= github.com/minio/minio-go/v7 v7.0.5 h1:I2NIJ2ojwJqD/YByemC1M59e1b4FW9kS7NlOar7HPV4= github.com/minio/minio-go/v7 v7.0.5/go.mod h1:TA0CQCjJZHM5SJj9IjqR0NmpmQJ6bCbXifAJ3mUU6Hw= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -178,6 +249,7 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWb github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -192,6 +264,9 @@ github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+ github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/ozonru/etcd v3.3.20-grpc1.27-origmodule+incompatible h1:CAG0PUvo1fen+ZEfxKJjFIc8GuuN5RuaBuCAuaP2Hno= github.com/ozonru/etcd v3.3.20-grpc1.27-origmodule+incompatible/go.mod h1:iIubILNIN6Jq9h8uiSLrN9L1tuj3iSSFwz3R61skm/A= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI= github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -221,8 +296,10 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA= @@ -233,7 +310,9 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCb github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= @@ -241,17 +320,21 @@ github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lN github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/protocolbuffers/protobuf v3.13.0+incompatible h1:omZA3Tuq+U2kJ2uMuqMR9c1VO5qLEgZ19m9878fXNtg= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= @@ -268,10 +351,18 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= +github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= @@ -282,6 +373,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tikv/client-go v0.0.0-20200824032810-95774393107b h1:VOG2GkM7RpRrT0St7HIIwCWrc3mVdf+DjcT8r2ucusI= github.com/tikv/client-go v0.0.0-20200824032810-95774393107b/go.mod h1:K0NcdVNrXDq92YPLytsrAwRMyuXi7GZCO6dXNH7OzQc= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -304,7 +397,10 @@ go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 h1:lWF4f9Nypl1ZqSb4gLeh/DGvBYVaUYHuiB93teOmwgc= go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -321,8 +417,10 @@ go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -330,23 +428,40 @@ golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -357,19 +472,29 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA= golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -383,23 +508,35 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f h1:Fqb3ao1hUmOR3GkUOg/Y+BadLwykBIzs5q8Ez2SbHyc= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190808195139-e713427fea3f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191107010934-f79515f33823/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -410,10 +547,20 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150 h1:VPpdpQkGvFicX9yo4G5oxZPi9ALBnEOZblPSa/Wa2m4= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= @@ -427,6 +574,7 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= @@ -447,9 +595,12 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/internal/allocator/id_allocator.go b/internal/allocator/id_allocator.go index c29d0661e1..ee214a6d99 100644 --- a/internal/allocator/id_allocator.go +++ b/internal/allocator/id_allocator.go @@ -4,8 +4,11 @@ import ( "context" "fmt" "log" + "strconv" "time" + "github.com/zilliztech/milvus-distributed/internal/conf" + "github.com/zilliztech/milvus-distributed/internal/proto/internalpb" "github.com/zilliztech/milvus-distributed/internal/util/typeutil" ) @@ -20,11 +23,17 @@ type IDAllocator struct { } func NewIDAllocator(ctx context.Context) (*IDAllocator, error) { + masterAddr := conf.Config.Etcd.Address + masterAddr += ":" + masterAddr += strconv.FormatInt(int64(conf.Config.Master.Port), 10) + ctx1, cancel := context.WithCancel(ctx) a := &IDAllocator{ Allocator: Allocator{reqs: make(chan request, maxMergeRequests), - ctx: ctx1, - cancel: cancel, + ctx: ctx1, + cancel: cancel, + masterAddress: masterAddr, + countPerRPC: maxMergeRequests, }, } a.tChan = &emptyTicker{} diff --git a/internal/allocator/timestamp_allocator.go b/internal/allocator/timestamp_allocator.go index f330ff1e5d..75efbb76ac 100644 --- a/internal/allocator/timestamp_allocator.go +++ b/internal/allocator/timestamp_allocator.go @@ -4,8 +4,11 @@ import ( "context" "fmt" "log" + "strconv" "time" + "github.com/zilliztech/milvus-distributed/internal/conf" + "github.com/zilliztech/milvus-distributed/internal/util/typeutil" "github.com/zilliztech/milvus-distributed/internal/proto/internalpb" @@ -25,11 +28,18 @@ type TimestampAllocator struct { } func NewTimestampAllocator(ctx context.Context) (*TimestampAllocator, error) { + + masterAddr := conf.Config.Etcd.Address + masterAddr += ":" + masterAddr += strconv.FormatInt(int64(conf.Config.Master.Port), 10) + ctx1, cancel := context.WithCancel(ctx) a := &TimestampAllocator{ Allocator: Allocator{reqs: make(chan request, maxMergeRequests), - ctx: ctx1, - cancel: cancel, + ctx: ctx1, + cancel: cancel, + masterAddress: masterAddr, + countPerRPC: maxMergeRequests, }, } a.tChan = &ticker{ diff --git a/internal/core/src/pb/CMakeLists.txt b/internal/core/src/pb/CMakeLists.txt index 0ae4e9abcc..dfb48bfda5 100644 --- a/internal/core/src/pb/CMakeLists.txt +++ b/internal/core/src/pb/CMakeLists.txt @@ -21,7 +21,6 @@ ENDFOREACH(proto_file) add_library(milvus_proto STATIC ${MILVUS_PROTO_SRCS} ) -message(${MILVUS_PROTO_SRCS}) target_link_libraries(milvus_proto libprotobuf diff --git a/internal/core/src/query/CMakeLists.txt b/internal/core/src/query/CMakeLists.txt index 7ffb1fa50e..d330469c2c 100644 --- a/internal/core/src/query/CMakeLists.txt +++ b/internal/core/src/query/CMakeLists.txt @@ -5,9 +5,8 @@ set(MILVUS_QUERY_SRCS generated/Expr.cpp visitors/ShowPlanNodeVisitor.cpp visitors/ExecPlanNodeVisitor.cpp - visitors/ShowExprVisitor.cpp Parser.cpp Plan.cpp ) add_library(milvus_query ${MILVUS_QUERY_SRCS}) -target_link_libraries(milvus_query milvus_proto) +target_link_libraries(milvus_query libprotobuf) diff --git a/internal/core/src/query/Expr.h b/internal/core/src/query/Expr.h index a5f121880d..1ae8949657 100644 --- a/internal/core/src/query/Expr.h +++ b/internal/core/src/query/Expr.h @@ -4,8 +4,6 @@ #include #include #include -#include "segcore/SegmentDefs.h" - namespace milvus::query { class ExprVisitor; @@ -60,13 +58,7 @@ using FieldId = std::string; struct TermExpr : Expr { FieldId field_id_; - segcore::DataType data_type_; - // std::vector terms_; - - protected: - // prevent accidential instantiation - TermExpr() = default; - + std::vector terms_; // public: void accept(ExprVisitor&) override; @@ -74,14 +66,12 @@ struct TermExpr : Expr { struct RangeExpr : Expr { FieldId field_id_; - segcore::DataType data_type_; - // std::vector> conditions_; - protected: - // prevent accidential instantiation - RangeExpr() = default; + enum class OpType { LessThan, LessEqual, GreaterThan, GreaterEqual, Equal, NotEqual }; + std::vector> conditions_; public: void accept(ExprVisitor&) override; }; + } // namespace milvus::query diff --git a/internal/core/src/query/ExprImpl.h b/internal/core/src/query/ExprImpl.h deleted file mode 100644 index 6a148e4acb..0000000000 --- a/internal/core/src/query/ExprImpl.h +++ /dev/null @@ -1,16 +0,0 @@ -#pragma once -#include "Expr.h" - -namespace milvus::query { -template -struct TermExprImpl : TermExpr { - std::vector terms_; -}; - -template -struct RangeExprImpl : RangeExpr { - enum class OpType { LessThan, LessEqual, GreaterThan, GreaterEqual, Equal, NotEqual }; - std::vector> conditions_; -}; - -} // namespace milvus::query \ No newline at end of file diff --git a/internal/core/src/query/Plan.cpp b/internal/core/src/query/Plan.cpp index b2073759e7..e0a82da555 100644 --- a/internal/core/src/query/Plan.cpp +++ b/internal/core/src/query/Plan.cpp @@ -23,7 +23,7 @@ CreateVec(const std::string& field_name, const json& vec_info) { static std::unique_ptr CreatePlanImplNaive(const std::string& dsl_str) { - auto plan = std::make_unique(); + auto plan = std::unique_ptr(); auto dsl = nlohmann::json::parse(dsl_str); nlohmann::json vec_pack; @@ -36,19 +36,17 @@ CreatePlanImplNaive(const std::string& dsl_str) { auto key = iter.key(); auto& body = iter.value(); plan->plan_node_ = CreateVec(key, body); - return plan; } } - PanicInfo("Unsupported DSL: vector node not detected"); } else if (bool_dsl.contains("vector")) { auto iter = bool_dsl["vector"].begin(); auto key = iter.key(); auto& body = iter.value(); plan->plan_node_ = CreateVec(key, body); - return plan; } else { PanicInfo("Unsupported DSL: vector node not detected"); } + return plan; } void @@ -57,7 +55,6 @@ CheckNull(const Json& json) { } class PlanParser { - public: void ParseBoolBody(const Json& dsl) { CheckNull(dsl); @@ -77,8 +74,6 @@ class PlanParser { } PanicInfo("unimplemented"); } - - private: }; std::unique_ptr @@ -88,12 +83,11 @@ CreatePlan(const std::string& dsl_str) { } std::unique_ptr -ParsePlaceholderGroup(const std::string& blob) { +ParsePlaceholderGroup(const char* placeholder_group_blob) { namespace ser = milvus::proto::service; - auto result = std::make_unique(); + auto result = std::unique_ptr(); ser::PlaceholderGroup ph_group; - auto ok = ph_group.ParseFromString(blob); - Assert(ok); + GOOGLE_PROTOBUF_PARSER_ASSERT(ph_group.ParseFromString(placeholder_group_blob)); for (auto& info : ph_group.placeholders()) { Placeholder element; element.tag_ = info.tag(); diff --git a/internal/core/src/query/Plan.h b/internal/core/src/query/Plan.h index 78fec8bbac..c0e943fd0e 100644 --- a/internal/core/src/query/Plan.h +++ b/internal/core/src/query/Plan.h @@ -13,7 +13,7 @@ std::unique_ptr CreatePlan(const std::string& dsl); std::unique_ptr -ParsePlaceholderGroup(const std::string& placeholder_group_blob); +ParsePlaceholderGroup(const char* placeholder_group_blob); int64_t GetNumOfQueries(const PlaceholderGroup*); @@ -24,5 +24,3 @@ int64_t GetTopK(const Plan*); } // namespace milvus::query - -#include "PlanImpl.h" \ No newline at end of file diff --git a/internal/core/src/query/PlanNode.h b/internal/core/src/query/PlanNode.h index 47ce41b641..d2de5eb20e 100644 --- a/internal/core/src/query/PlanNode.h +++ b/internal/core/src/query/PlanNode.h @@ -28,6 +28,7 @@ struct PlanNode { using PlanNodePtr = std::unique_ptr; struct QueryInfo { + int64_t num_queries_; int64_t topK_; FieldId field_id_; std::string metric_type_; // TODO: use enum diff --git a/internal/core/src/query/generated/ExecPlanNodeVisitor.h b/internal/core/src/query/generated/ExecPlanNodeVisitor.h index fed4813a86..2c6f0e8db7 100644 --- a/internal/core/src/query/generated/ExecPlanNodeVisitor.h +++ b/internal/core/src/query/generated/ExecPlanNodeVisitor.h @@ -1,11 +1,7 @@ #pragma once // Generated File // DO NOT EDIT -#include "utils/Json.h" -#include "query/PlanImpl.h" -#include "segcore/SegmentBase.h" #include "PlanNodeVisitor.h" - namespace milvus::query { class ExecPlanNodeVisitor : PlanNodeVisitor { public: diff --git a/internal/core/src/query/generated/ShowExprVisitor.h b/internal/core/src/query/generated/ShowExprVisitor.h index 53053480ae..dfda5f09a9 100644 --- a/internal/core/src/query/generated/ShowExprVisitor.h +++ b/internal/core/src/query/generated/ShowExprVisitor.h @@ -1,11 +1,7 @@ #pragma once // Generated File // DO NOT EDIT -#include "query/Plan.h" -#include "utils/EasyAssert.h" -#include "utils/Json.h" #include "ExprVisitor.h" - namespace milvus::query { class ShowExprVisitor : ExprVisitor { public: @@ -22,35 +18,5 @@ class ShowExprVisitor : ExprVisitor { visit(RangeExpr& expr) override; public: - using RetType = Json; - - public: - RetType - call_child(Expr& expr) { - assert(!ret_.has_value()); - expr.accept(*this); - assert(ret_.has_value()); - auto ret = std::move(ret_); - ret_ = std::nullopt; - return std::move(ret.value()); - } - - Json - combine(Json&& extra, UnaryExpr& expr) { - auto result = std::move(extra); - result["child"] = call_child(*expr.child_); - return result; - } - - Json - combine(Json&& extra, BinaryExpr& expr) { - auto result = std::move(extra); - result["left_child"] = call_child(*expr.left_); - result["right_child"] = call_child(*expr.right_); - return result; - } - - private: - std::optional ret_; }; } // namespace milvus::query diff --git a/internal/core/src/query/generated/ShowPlanNodeVisitor.h b/internal/core/src/query/generated/ShowPlanNodeVisitor.h index 1835cb5547..603f3649b4 100644 --- a/internal/core/src/query/generated/ShowPlanNodeVisitor.h +++ b/internal/core/src/query/generated/ShowPlanNodeVisitor.h @@ -1,12 +1,7 @@ #pragma once // Generated File // DO NOT EDIT -#include "utils/EasyAssert.h" -#include "utils/Json.h" -#include - #include "PlanNodeVisitor.h" - namespace milvus::query { class ShowPlanNodeVisitor : PlanNodeVisitor { public: @@ -26,7 +21,6 @@ class ShowPlanNodeVisitor : PlanNodeVisitor { node.accept(*this); assert(ret_.has_value()); auto ret = std::move(ret_); - ret_ = std::nullopt; return std::move(ret.value()); } diff --git a/internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp b/internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp index 888e0e20b2..8d71bbd4cb 100644 --- a/internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp +++ b/internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp @@ -48,10 +48,8 @@ ExecPlanNodeVisitor::visit(FloatVectorANNS& node) { auto segment = dynamic_cast(&segment_); AssertInfo(segment, "support SegmentSmallIndex Only"); RetType ret; - auto& ph = placeholder_group_.at(0); - auto src_data = ph.get_blob(); - auto num_queries = ph.num_of_queries_; - segment->QueryBruteForceImpl(node.query_info_, src_data, num_queries, timestamp_, ret); + auto src_data = placeholder_group_.at(0).get_blob(); + segment->QueryBruteForceImpl(node.query_info_, src_data, timestamp_, ret); ret_ = ret; } diff --git a/internal/core/src/query/visitors/ShowExprVisitor.cpp b/internal/core/src/query/visitors/ShowExprVisitor.cpp deleted file mode 100644 index 07765842ab..0000000000 --- a/internal/core/src/query/visitors/ShowExprVisitor.cpp +++ /dev/null @@ -1,173 +0,0 @@ -#include "query/Plan.h" -#include "utils/EasyAssert.h" -#include "utils/Json.h" -#include "query/generated/ShowExprVisitor.h" -#include "query/ExprImpl.h" - -namespace milvus::query { -using Json = nlohmann::json; - -#if 1 -// THIS CONTAINS EXTRA BODY FOR VISITOR -// WILL BE USED BY GENERATOR -namespace impl { -class ShowExprNodeVisitor : ExprVisitor { - public: - using RetType = Json; - - public: - RetType - call_child(Expr& expr) { - assert(!ret_.has_value()); - expr.accept(*this); - assert(ret_.has_value()); - auto ret = std::move(ret_); - ret_ = std::nullopt; - return std::move(ret.value()); - } - - Json - combine(Json&& extra, UnaryExpr& expr) { - auto result = std::move(extra); - result["child"] = call_child(*expr.child_); - return result; - } - - Json - combine(Json&& extra, BinaryExpr& expr) { - auto result = std::move(extra); - result["left_child"] = call_child(*expr.left_); - result["right_child"] = call_child(*expr.right_); - return result; - } - - private: - std::optional ret_; -}; -} // namespace impl -#endif - -void -ShowExprVisitor::visit(BoolUnaryExpr& expr) { - Assert(!ret_.has_value()); - using OpType = BoolUnaryExpr::OpType; - - // TODO: use magic_enum if available - Assert(expr.op_type_ == OpType::LogicalNot); - auto op_name = "LogicalNot"; - - Json extra{ - {"expr_type", "BoolUnary"}, - {"op", op_name}, - }; - ret_ = this->combine(std::move(extra), expr); -} - -void -ShowExprVisitor::visit(BoolBinaryExpr& expr) { - Assert(!ret_.has_value()); - using OpType = BoolBinaryExpr::OpType; - - // TODO: use magic_enum if available - auto op_name = [](OpType op) { - switch (op) { - case OpType::LogicalAnd: - return "LogicalAnd"; - case OpType::LogicalOr: - return "LogicalOr"; - case OpType::LogicalXor: - return "LogicalXor"; - default: - PanicInfo("unsupported op"); - } - }(expr.op_type_); - - Json extra{ - {"expr_type", "BoolBinary"}, - {"op", op_name}, - }; - ret_ = this->combine(std::move(extra), expr); -} - -template -static Json -TermExtract(const TermExpr& expr_raw) { - auto expr = dynamic_cast*>(&expr_raw); - Assert(expr); - return Json{expr->terms_}; -} - -void -ShowExprVisitor::visit(TermExpr& expr) { - Assert(!ret_.has_value()); - Assert(segcore::field_is_vector(expr.data_type_) == false); - using segcore::DataType; - auto terms = [&] { - switch (expr.data_type_) { - case DataType::INT8: - return TermExtract(expr); - case DataType::INT16: - return TermExtract(expr); - case DataType::INT32: - return TermExtract(expr); - case DataType::INT64: - return TermExtract(expr); - case DataType::DOUBLE: - return TermExtract(expr); - case DataType::FLOAT: - return TermExtract(expr); - case DataType::BOOL: - return TermExtract(expr); - default: - PanicInfo("unsupported type"); - } - }(); - - Json res{{"expr_type", "Term"}, - {"field_id", expr.field_id_}, - {"data_type", segcore::datatype_name(expr.data_type_)}, - {"terms", std::move(terms)}}; - - ret_ = res; -} - -template -static Json -CondtionExtract(const RangeExpr& expr_raw) { - auto expr = dynamic_cast*>(&expr_raw); - Assert(expr); - return Json{expr->terms_}; -} - -void -ShowExprVisitor::visit(RangeExpr& expr) { - Assert(!ret_.has_value()); - Assert(segcore::field_is_vector(expr.data_type_) == false); - using segcore::DataType; - auto conditions = [&] { - switch (expr.data_type_) { - case DataType::BOOL: - return CondtionExtract(expr); - case DataType::INT8: - return CondtionExtract(expr); - case DataType::INT16: - return CondtionExtract(expr); - case DataType::INT32: - return CondtionExtract(expr); - case DataType::INT64: - return CondtionExtract(expr); - case DataType::DOUBLE: - return CondtionExtract(expr); - case DataType::FLOAT: - return CondtionExtract(expr); - default: - PanicInfo("unsupported type"); - } - }(); - - Json res{{"expr_type", "Range"}, - {"field_id", expr.field_id_}, - {"data_type", segcore::datatype_name(expr.data_type_)}, - {"conditions", std::move(conditions)}}; -} -} // namespace milvus::query diff --git a/internal/core/src/query/visitors/ShowPlanNodeVisitor.cpp b/internal/core/src/query/visitors/ShowPlanNodeVisitor.cpp index 4f31677cfe..04b24b78b5 100644 --- a/internal/core/src/query/visitors/ShowPlanNodeVisitor.cpp +++ b/internal/core/src/query/visitors/ShowPlanNodeVisitor.cpp @@ -19,7 +19,6 @@ class ShowPlanNodeVisitorImpl : PlanNodeVisitor { node.accept(*this); assert(ret_.has_value()); auto ret = std::move(ret_); - ret_ = std::nullopt; return std::move(ret.value()); } @@ -41,9 +40,11 @@ ShowPlanNodeVisitor::visit(FloatVectorANNS& node) { assert(!ret_); auto& info = node.query_info_; Json json_body{ - {"node_type", "FloatVectorANNS"}, // - {"metric_type", info.metric_type_}, // + {"node_type", "FloatVectorANNS"}, // + {"metric_type", info.metric_type_}, // + // {"dim", info.dim_}, // {"field_id_", info.field_id_}, // + {"num_queries", info.num_queries_}, // {"topK", info.topK_}, // {"search_params", info.search_params_}, // {"placeholder_tag", node.placeholder_tag_}, // @@ -51,7 +52,7 @@ ShowPlanNodeVisitor::visit(FloatVectorANNS& node) { if (node.predicate_.has_value()) { PanicInfo("unimplemented"); } else { - json_body["predicate"] = "None"; + json_body["predicate"] = "nullopt"; } ret_ = json_body; } diff --git a/internal/core/src/segcore/SegmentDefs.h b/internal/core/src/segcore/SegmentDefs.h index 36e1afc46e..df560f9afd 100644 --- a/internal/core/src/segcore/SegmentDefs.h +++ b/internal/core/src/segcore/SegmentDefs.h @@ -50,36 +50,6 @@ field_sizeof(DataType data_type, int dim = 1) { } } -// TODO: use magic_enum when available -inline std::string -datatype_name(DataType data_type) { - switch (data_type) { - case DataType::BOOL: - return "bool"; - case DataType::DOUBLE: - return "double"; - case DataType::FLOAT: - return "float"; - case DataType::INT8: - return "int8_t"; - case DataType::INT16: - return "int16_t"; - case DataType::INT32: - return "int32_t"; - case DataType::INT64: - return "int64_t"; - case DataType::VECTOR_FLOAT: - return "vector_float"; - case DataType::VECTOR_BINARY: { - return "vector_binary"; - } - default: { - auto err_msg = "Unsupported DataType(" + std::to_string((int)data_type) + ")"; - PanicInfo(err_msg); - } - } -} - inline bool field_is_vector(DataType datatype) { return datatype == DataType::VECTOR_BINARY || datatype == DataType::VECTOR_FLOAT; diff --git a/internal/core/src/segcore/SegmentSmallIndex.cpp b/internal/core/src/segcore/SegmentSmallIndex.cpp index 4396b4fde8..319d2014fe 100644 --- a/internal/core/src/segcore/SegmentSmallIndex.cpp +++ b/internal/core/src/segcore/SegmentSmallIndex.cpp @@ -223,7 +223,6 @@ get_barrier(const RecordType& record, Timestamp timestamp) { Status SegmentSmallIndex::QueryBruteForceImpl(const query::QueryInfo& info, const float* query_data, - int64_t num_queries, Timestamp timestamp, QueryResult& results) { // step 1: binary search to find the barrier of the snapshot @@ -248,6 +247,7 @@ SegmentSmallIndex::QueryBruteForceImpl(const query::QueryInfo& info, Assert(field.get_data_type() == DataType::VECTOR_FLOAT); auto dim = field.get_dim(); auto topK = info.topK_; + auto num_queries = info.num_queries_; auto total_count = topK * num_queries; // TODO: optimize @@ -321,6 +321,7 @@ SegmentSmallIndex::QueryDeprecated(query::QueryDeprecatedPtr query_info, Timesta int64_t inferred_dim = query_info->query_raw_data.size() / query_info->num_queries; // TODO query::QueryInfo info{ + query_info->num_queries, query_info->topK, query_info->field_name, "L2", @@ -328,8 +329,7 @@ SegmentSmallIndex::QueryDeprecated(query::QueryDeprecatedPtr query_info, Timesta {"nprobe", 10}, }, }; - auto num_queries = query_info->num_queries; - return QueryBruteForceImpl(info, query_info->query_raw_data.data(), num_queries, timestamp, result); + return QueryBruteForceImpl(info, query_info->query_raw_data.data(), timestamp, result); } Status @@ -453,15 +453,14 @@ SegmentSmallIndex::GetMemoryUsageInBytes() { } Status -SegmentSmallIndex::Search(const query::Plan* plan, +SegmentSmallIndex::Search(const query::Plan* Plan, const query::PlaceholderGroup** placeholder_groups, const Timestamp* timestamps, int num_groups, QueryResult& results) { Assert(num_groups == 1); query::ExecPlanNodeVisitor visitor(*this, timestamps[0], *placeholder_groups[0]); - results = visitor.get_moved_result(*plan->plan_node_); - return Status::OK(); + PanicInfo("unimplemented"); } } // namespace milvus::segcore diff --git a/internal/core/src/segcore/SegmentSmallIndex.h b/internal/core/src/segcore/SegmentSmallIndex.h index 0a4282c08c..c643bb3fe4 100644 --- a/internal/core/src/segcore/SegmentSmallIndex.h +++ b/internal/core/src/segcore/SegmentSmallIndex.h @@ -143,7 +143,6 @@ class SegmentSmallIndex : public SegmentBase { Status QueryBruteForceImpl(const query::QueryInfo& info, const float* query_data, - int64_t num_queries, Timestamp timestamp, QueryResult& results); diff --git a/internal/core/unittest/CMakeLists.txt b/internal/core/unittest/CMakeLists.txt index 8ae28c95c5..1c10a7e351 100644 --- a/internal/core/unittest/CMakeLists.txt +++ b/internal/core/unittest/CMakeLists.txt @@ -9,7 +9,6 @@ set(MILVUS_TEST_FILES test_c_api.cpp test_indexing.cpp test_query.cpp - test_expr.cpp ) add_executable(all_tests ${MILVUS_TEST_FILES} diff --git a/internal/core/unittest/test_expr.cpp b/internal/core/unittest/test_expr.cpp deleted file mode 100644 index d43d6d4589..0000000000 --- a/internal/core/unittest/test_expr.cpp +++ /dev/null @@ -1,70 +0,0 @@ -#include -#include "query/Parser.h" -#include "query/Expr.h" -#include "query/PlanNode.h" -#include "query/generated/ExprVisitor.h" -#include "query/generated/PlanNodeVisitor.h" -#include "test_utils/DataGen.h" -#include "query/generated/ShowPlanNodeVisitor.h" - -TEST(Expr, Naive) { - SUCCEED(); - using namespace milvus::wtf; - std::string dsl_string = R"( -{ - "bool": { - "must": [ - { - "term": { - "A": [ - 1, - 2, - 5 - ] - } - }, - { - "range": { - "B": { - "GT": 1, - "LT": 100 - } - } - }, - { - "vector": { - "Vec": { - "metric_type": "L2", - "params": { - "nprobe": 10 - }, - "query": "$0", - "topk": 10 - } - } - } - ] - } -})"; -} - -TEST(Expr, ShowExecutor) { - using namespace milvus::query; - using namespace milvus::segcore; - auto node = std::make_unique(); - auto schema = std::make_shared(); - int64_t num_queries = 100L; - schema->AddField("fakevec", DataType::VECTOR_FLOAT, 16); - auto raw_data = DataGen(schema, num_queries); - auto& info = node->query_info_; - info.metric_type_ = "L2"; - info.topK_ = 20; - info.field_id_ = "fakevec"; - node->predicate_ = std::nullopt; - ShowPlanNodeVisitor show_visitor; - PlanNodePtr base(node.release()); - auto res = show_visitor.call_child(*base); - auto dup = res; - dup["data"] = "...collased..."; - std::cout << dup.dump(4); -} \ No newline at end of file diff --git a/internal/core/unittest/test_query.cpp b/internal/core/unittest/test_query.cpp index d77d55f657..29003ec23e 100644 --- a/internal/core/unittest/test_query.cpp +++ b/internal/core/unittest/test_query.cpp @@ -6,8 +6,6 @@ #include "query/generated/PlanNodeVisitor.h" #include "test_utils/DataGen.h" #include "query/generated/ShowPlanNodeVisitor.h" -#include "query/generated/ExecPlanNodeVisitor.h" -#include "query/PlanImpl.h" TEST(Query, Naive) { SUCCEED(); @@ -60,6 +58,7 @@ TEST(Query, ShowExecutor) { auto raw_data = DataGen(schema, num_queries); auto& info = node->query_info_; info.metric_type_ = "L2"; + info.num_queries_ = 10; info.topK_ = 20; info.field_id_ = "fakevec"; node->predicate_ = std::nullopt; @@ -67,87 +66,6 @@ TEST(Query, ShowExecutor) { PlanNodePtr base(node.release()); auto res = show_visitor.call_child(*base); auto dup = res; + dup["data"] = "...collased..."; std::cout << dup.dump(4); -} - -TEST(Query, DSL) { - using namespace milvus::query; - using namespace milvus::segcore; - ShowPlanNodeVisitor shower; - - std::string dsl_string = R"( -{ - "bool": { - "must": [ - { - "vector": { - "Vec": { - "metric_type": "L2", - "params": { - "nprobe": 10 - }, - "query": "$0", - "topk": 10 - } - } - } - ] - } -})"; - auto plan = CreatePlan(dsl_string); - auto res = shower.call_child(*plan->plan_node_); - std::cout << res.dump(4) << std::endl; - - std::string dsl_string2 = R"( -{ - "bool": { - "vector": { - "Vec": { - "metric_type": "L2", - "params": { - "nprobe": 10 - }, - "query": "$0", - "topk": 10 - } - } - } -})"; - auto plan2 = CreatePlan(dsl_string2); - auto res2 = shower.call_child(*plan2->plan_node_); - std::cout << res2.dump(4) << std::endl; - ASSERT_EQ(res, res2); -} - -TEST(Query, ParsePlaceholderGroup) { - using namespace milvus::query; - using namespace milvus::segcore; - namespace ser = milvus::proto::service; - int num_queries = 10; - int dim = 16; - std::default_random_engine e; - std::normal_distribution dis(0, 1); - ser::PlaceholderGroup raw_group; - auto value = raw_group.add_placeholders(); - value->set_tag("$0"); - value->set_type(ser::PlaceholderType::VECTOR_FLOAT); - for(int i = 0; i < num_queries; ++i) { - std::vector vec; - for(int d = 0; d < dim; ++d) { - vec.push_back(dis(e)); - } - // std::string line((char*)vec.data(), (char*)vec.data() + vec.size() * sizeof(float)); - value->add_values(vec.data(), vec.size() * sizeof(float)); - } - auto blob = raw_group.SerializeAsString(); - //ser::PlaceholderGroup new_group; - //new_group.ParseFromString() - auto fuck = ParsePlaceholderGroup(blob); - int x = 1+1; -} - - -TEST(Query, Exec) { - using namespace milvus::query; - using namespace milvus::segcore; -} +} \ No newline at end of file diff --git a/internal/kv/etcd_kv_test.go b/internal/kv/etcd_kv_test.go index f3f84b7102..6afc0882ab 100644 --- a/internal/kv/etcd_kv_test.go +++ b/internal/kv/etcd_kv_test.go @@ -3,17 +3,23 @@ package kv import ( "context" "path" - "strconv" "testing" "github.com/stretchr/testify/assert" - "github.com/zilliztech/milvus-distributed/internal/conf" + gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil" "go.etcd.io/etcd/clientv3" ) func TestEtcdKV_Load(t *testing.T) { - conf.LoadConfig("config.yaml") - etcdPort := strconv.Itoa(int(conf.Config.Etcd.Port)) + err := gparams.GParams.LoadYaml("config.yaml") + if err != nil { + panic(err) + } + etcdPort, err := gparams.GParams.Load("etcd.port") + if err != nil { + panic(err) + } + cli, err := clientv3.New(clientv3.Config{Endpoints: []string{"127.0.0.1:" + etcdPort}}) assert.Nil(t, err) rootpath := "/etcd/test/root" @@ -68,8 +74,15 @@ func TestEtcdKV_Load(t *testing.T) { } func TestEtcdKV_MultiSave(t *testing.T) { - conf.LoadConfig("config.yaml") - etcdPort := strconv.Itoa(int(conf.Config.Etcd.Port)) + err := gparams.GParams.LoadYaml("config.yaml") + if err != nil { + panic(err) + } + etcdPort, err := gparams.GParams.Load("etcd.port") + if err != nil { + panic(err) + } + cli, err := clientv3.New(clientv3.Config{Endpoints: []string{"127.0.0.1:" + etcdPort}}) assert.Nil(t, err) rootpath := "/etcd/test/root" @@ -97,8 +110,15 @@ func TestEtcdKV_MultiSave(t *testing.T) { } func TestEtcdKV_Remove(t *testing.T) { - conf.LoadConfig("config.yaml") - etcdPort := strconv.Itoa(int(conf.Config.Etcd.Port)) + err := gparams.GParams.LoadYaml("config.yaml") + if err != nil { + panic(err) + } + etcdPort, err := gparams.GParams.Load("etcd.port") + if err != nil { + panic(err) + } + cli, err := clientv3.New(clientv3.Config{Endpoints: []string{"127.0.0.1:" + etcdPort}}) assert.Nil(t, err) rootpath := "/etcd/test/root" @@ -166,8 +186,15 @@ func TestEtcdKV_Remove(t *testing.T) { } func TestEtcdKV_MultiSaveAndRemove(t *testing.T) { - conf.LoadConfig("config.yaml") - etcdPort := strconv.Itoa(int(conf.Config.Etcd.Port)) + err := gparams.GParams.LoadYaml("config.yaml") + if err != nil { + panic(err) + } + etcdPort, err := gparams.GParams.Load("etcd.port") + if err != nil { + panic(err) + } + cli, err := clientv3.New(clientv3.Config{Endpoints: []string{"127.0.0.1:" + etcdPort}}) assert.Nil(t, err) rootpath := "/etcd/test/root" diff --git a/internal/kv/mockkv/mock_etcd.go b/internal/kv/mockkv/mock_etcd.go index f5af8f9bca..04a3b95a23 100644 --- a/internal/kv/mockkv/mock_etcd.go +++ b/internal/kv/mockkv/mock_etcd.go @@ -1,14 +1,14 @@ package mockkv import ( - "github.com/zilliztech/milvus-distributed/internal/kv" + "github.com/zilliztech/milvus-distributed/internal/util/kvutil" ) // use MemoryKV to mock EtcdKV -func NewEtcdKV() *kv.MemoryKV { - return kv.NewMemoryKV() +func NewEtcdKV() *kvutil.MemoryKV { + return kvutil.NewMemoryKV() } -func NewMemoryKV() *kv.MemoryKV { - return kv.NewMemoryKV() +func NewMemoryKV() *kvutil.MemoryKV { + return kvutil.NewMemoryKV() } diff --git a/internal/master/collection_task_test.go b/internal/master/collection_task_test.go index d81a785517..256ad1352e 100644 --- a/internal/master/collection_task_test.go +++ b/internal/master/collection_task_test.go @@ -3,27 +3,32 @@ package master import ( "context" "log" - "strconv" "testing" "github.com/golang/protobuf/proto" "github.com/stretchr/testify/assert" - "github.com/zilliztech/milvus-distributed/internal/conf" "github.com/zilliztech/milvus-distributed/internal/proto/commonpb" "github.com/zilliztech/milvus-distributed/internal/proto/internalpb" "github.com/zilliztech/milvus-distributed/internal/proto/masterpb" "github.com/zilliztech/milvus-distributed/internal/proto/schemapb" "github.com/zilliztech/milvus-distributed/internal/proto/servicepb" + gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil" "go.etcd.io/etcd/clientv3" "google.golang.org/grpc" ) func TestMaster_CreateCollectionTask(t *testing.T) { - conf.LoadConfig("config.yaml") + err := gparams.GParams.LoadYaml("config.yaml") + if err != nil { + panic(err) + } ctx, cancel := context.WithCancel(context.TODO()) defer cancel() - etcdPort := strconv.Itoa(int(conf.Config.Etcd.Port)) + etcdPort, err := gparams.GParams.Load("etcd.port") + if err != nil { + panic(err) + } etcdAddr := "127.0.0.1:" + etcdPort etcdCli, err := clientv3.New(clientv3.Config{Endpoints: []string{etcdAddr}}) diff --git a/internal/master/controller/collection.go b/internal/master/controller/collection.go index 6c95ced485..b132ae3e00 100644 --- a/internal/master/controller/collection.go +++ b/internal/master/controller/collection.go @@ -5,15 +5,13 @@ import ( "strconv" "time" - "github.com/zilliztech/milvus-distributed/internal/util/typeutil" - - "github.com/zilliztech/milvus-distributed/internal/master/id" - - "github.com/zilliztech/milvus-distributed/internal/conf" "github.com/zilliztech/milvus-distributed/internal/kv" "github.com/zilliztech/milvus-distributed/internal/master/collection" + "github.com/zilliztech/milvus-distributed/internal/master/id" "github.com/zilliztech/milvus-distributed/internal/master/segment" "github.com/zilliztech/milvus-distributed/internal/proto/schemapb" + gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil" + "github.com/zilliztech/milvus-distributed/internal/util/typeutil" ) type UniqueID = typeutil.UniqueID @@ -68,7 +66,15 @@ func WriteCollection2Datastore(collectionMeta *schemapb.CollectionSchema, kvbase time.Now(), fieldMetas, []UniqueID{sID}, []string{"default"}) cm := collection.GrpcMarshal(&c) - s := segment.NewSegment(sID, cID, collectionMeta.Name, "default", 0, conf.Config.Pulsar.TopicNum, time.Now(), time.Unix(1<<46-1, 0)) + pulsarTopicNum, err := gparams.GParams.Load("pulsar.topicnum") + if err != nil { + panic(err) + } + topicNum, err := strconv.Atoi(pulsarTopicNum) + if err != nil { + panic(err) + } + s := segment.NewSegment(sID, cID, collectionMeta.Name, "default", 0, topicNum, time.Now(), time.Unix(1<<46-1, 0)) collectionData, err := collection.Collection2JSON(*cm) if err != nil { log.Fatal(err) diff --git a/internal/master/controller/segment.go b/internal/master/controller/segment.go index 8e9b2da2b3..cd137c4a94 100644 --- a/internal/master/controller/segment.go +++ b/internal/master/controller/segment.go @@ -5,24 +5,31 @@ import ( "strconv" "time" - "github.com/zilliztech/milvus-distributed/internal/master/id" - - "github.com/zilliztech/milvus-distributed/internal/conf" "github.com/zilliztech/milvus-distributed/internal/kv" "github.com/zilliztech/milvus-distributed/internal/master/collection" + "github.com/zilliztech/milvus-distributed/internal/master/id" "github.com/zilliztech/milvus-distributed/internal/master/segment" "github.com/zilliztech/milvus-distributed/internal/proto/internalpb" + gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil" ) func ComputeCloseTime(ss internalpb.SegmentStats, kvbase *kv.EtcdKV) error { - if int(ss.MemorySize) > int(conf.Config.Master.SegmentThreshole*0.8) { + masterSegmentThreshole, err := gparams.GParams.Load("master.segmentthreshole") + if err != nil { + panic(err) + } + segmentThreshole, err := strconv.ParseFloat(masterSegmentThreshole, 32) + if err != nil { + panic(err) + } + if int(ss.MemorySize) > int(segmentThreshole*0.8) { currentTime := time.Now() //memRate := int(ss.MemoryRate) memRate := 1 // to do if memRate == 0 { memRate = 1 } - sec := int(conf.Config.Master.SegmentThreshole*0.2) / memRate + sec := int(segmentThreshole*0.2) / memRate data, err := kvbase.Load("segment/" + strconv.Itoa(int(ss.SegmentID))) if err != nil { return err diff --git a/internal/master/controller/segment_test.go b/internal/master/controller/segment_test.go index 049342167a..79df4c3d1a 100644 --- a/internal/master/controller/segment_test.go +++ b/internal/master/controller/segment_test.go @@ -1,26 +1,34 @@ package controller import ( - "strconv" "testing" "time" - "github.com/zilliztech/milvus-distributed/internal/conf" "github.com/zilliztech/milvus-distributed/internal/kv" "github.com/zilliztech/milvus-distributed/internal/proto/internalpb" + gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil" "go.etcd.io/etcd/clientv3" ) func newKvBase() *kv.EtcdKV { - //etcdAddr := conf.Config.Etcd.Address - //etcdAddr += ":" - //etcdAddr += strconv.FormatInt(int64(conf.Config.Etcd.Port), 10) - etcdAddr := "127.0.0.1:" + strconv.Itoa(int(conf.Config.Etcd.Port)) + err := gparams.GParams.LoadYaml("config.yaml") + if err != nil { + panic(err) + } + etcdPort, err := gparams.GParams.Load("etcd.port") + if err != nil { + panic(err) + } + etcdAddr := "127.0.0.1:" + etcdPort cli, _ := clientv3.New(clientv3.Config{ Endpoints: []string{etcdAddr}, DialTimeout: 5 * time.Second, }) - kvbase := kv.NewEtcdKV(cli, conf.Config.Etcd.Rootpath) + etcdRootPath, err := gparams.GParams.Load("etcd.rootpath") + if err != nil { + panic(err) + } + kvbase := kv.NewEtcdKV(cli, etcdRootPath) return kvbase } diff --git a/internal/master/grpc_service_test.go b/internal/master/grpc_service_test.go index d657171087..3ef81843d4 100644 --- a/internal/master/grpc_service_test.go +++ b/internal/master/grpc_service_test.go @@ -2,26 +2,31 @@ package master import ( "context" - "strconv" "testing" "github.com/golang/protobuf/proto" "github.com/stretchr/testify/assert" - "github.com/zilliztech/milvus-distributed/internal/conf" "github.com/zilliztech/milvus-distributed/internal/proto/commonpb" "github.com/zilliztech/milvus-distributed/internal/proto/internalpb" "github.com/zilliztech/milvus-distributed/internal/proto/masterpb" "github.com/zilliztech/milvus-distributed/internal/proto/schemapb" + gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil" "go.etcd.io/etcd/clientv3" "google.golang.org/grpc" ) func TestMaster_CreateCollection(t *testing.T) { - conf.LoadConfig("config.yaml") + err := gparams.GParams.LoadYaml("config.yaml") + if err != nil { + panic(err) + } ctx, cancel := context.WithCancel(context.TODO()) defer cancel() - etcdPort := strconv.Itoa(int(conf.Config.Etcd.Port)) + etcdPort, err := gparams.GParams.Load("etcd.port") + if err != nil { + panic(err) + } etcdAddr := "127.0.0.1:" + etcdPort etcdCli, err := clientv3.New(clientv3.Config{Endpoints: []string{etcdAddr}}) diff --git a/internal/master/id/id.go b/internal/master/id/id.go index 4d76f5b0c2..d8dbcd5abd 100644 --- a/internal/master/id/id.go +++ b/internal/master/id/id.go @@ -1,8 +1,8 @@ package id import ( - "github.com/zilliztech/milvus-distributed/internal/kv" "github.com/zilliztech/milvus-distributed/internal/master/tso" + "github.com/zilliztech/milvus-distributed/internal/util/kvutil" "github.com/zilliztech/milvus-distributed/internal/util/tsoutil" "github.com/zilliztech/milvus-distributed/internal/util/typeutil" ) @@ -20,12 +20,12 @@ func Init() { InitGlobalIDAllocator("idTimestamp", tsoutil.NewTSOKVBase("gid")) } -func InitGlobalIDAllocator(key string, base kv.Base) { +func InitGlobalIDAllocator(key string, base kvutil.Base) { allocator = NewGlobalIDAllocator(key, base) allocator.Initialize() } -func NewGlobalIDAllocator(key string, base kv.Base) *GlobalIDAllocator { +func NewGlobalIDAllocator(key string, base kvutil.Base) *GlobalIDAllocator { return &GlobalIDAllocator{ allocator: tso.NewGlobalTSOAllocator(key, base), } diff --git a/internal/master/id/id_test.go b/internal/master/id/id_test.go index b6ae47b753..8c31987191 100644 --- a/internal/master/id/id_test.go +++ b/internal/master/id/id_test.go @@ -6,14 +6,17 @@ import ( "github.com/stretchr/testify/assert" - "github.com/zilliztech/milvus-distributed/internal/conf" + gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil" "github.com/zilliztech/milvus-distributed/internal/util/tsoutil" ) var GIdAllocator *GlobalIDAllocator func TestMain(m *testing.M) { - conf.LoadConfig("config.yaml") + err := gparams.GParams.LoadYaml("config.yaml") + if err != nil { + panic(err) + } GIdAllocator = NewGlobalIDAllocator("idTimestamp", tsoutil.NewTSOKVBase("gid")) exitCode := m.Run() os.Exit(exitCode) diff --git a/internal/master/informer/pulsar.go b/internal/master/informer/pulsar.go index 04aba9f34c..7aea6fc54d 100644 --- a/internal/master/informer/pulsar.go +++ b/internal/master/informer/pulsar.go @@ -2,19 +2,22 @@ package informer import ( "log" - "strconv" "time" - "github.com/zilliztech/milvus-distributed/internal/conf" - "github.com/apache/pulsar-client-go/pulsar" + gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil" ) func NewPulsarClient() *PulsarClient { - pulsarAddr := "pulsar://" - pulsarAddr += conf.Config.Pulsar.Address - pulsarAddr += ":" - pulsarAddr += strconv.FormatInt(int64(conf.Config.Pulsar.Port), 10) + pulsarAddr, err := gparams.GParams.Load("pulsar.address") + if err != nil { + panic(err) + } + pulsarPort, err := gparams.GParams.Load("pulsar.port") + if err != nil { + panic(err) + } + pulsarAddr = "pulsar://" + pulsarAddr + ":" + pulsarPort client, err := pulsar.NewClient(pulsar.ClientOptions{ URL: pulsarAddr, OperationTimeout: 30 * time.Second, diff --git a/internal/master/master.go b/internal/master/master.go index a8e4e6ac14..17e414b17e 100644 --- a/internal/master/master.go +++ b/internal/master/master.go @@ -16,7 +16,6 @@ import ( "go.etcd.io/etcd/clientv3" "google.golang.org/grpc" - "github.com/zilliztech/milvus-distributed/internal/conf" "github.com/zilliztech/milvus-distributed/internal/errors" "github.com/zilliztech/milvus-distributed/internal/kv" "github.com/zilliztech/milvus-distributed/internal/master/controller" @@ -25,6 +24,7 @@ import ( "github.com/zilliztech/milvus-distributed/internal/master/tso" "github.com/zilliztech/milvus-distributed/internal/proto/internalpb" "github.com/zilliztech/milvus-distributed/internal/proto/masterpb" + gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil" ) // Server is the pd server. @@ -236,8 +236,13 @@ func (s *Master) pulsarLoop() { ctx, cancel := context.WithCancel(s.serverLoopCtx) + pulsarTopic, err := gparams.GParams.Load("master.pulsartopic") + if err != nil { + panic(err) + } + consumer, err := s.pc.Client.Subscribe(pulsar.ConsumerOptions{ - Topic: conf.Config.Master.PulsarTopic, + Topic: pulsarTopic, SubscriptionName: "my-sub", Type: pulsar.Shared, }) diff --git a/internal/master/meta_table_test.go b/internal/master/meta_table_test.go index f959a354b1..33af155f80 100644 --- a/internal/master/meta_table_test.go +++ b/internal/master/meta_table_test.go @@ -2,20 +2,25 @@ package master import ( "context" - "strconv" "testing" "github.com/stretchr/testify/assert" - "github.com/zilliztech/milvus-distributed/internal/conf" "github.com/zilliztech/milvus-distributed/internal/kv" pb "github.com/zilliztech/milvus-distributed/internal/proto/etcdpb" "github.com/zilliztech/milvus-distributed/internal/proto/schemapb" + gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil" "go.etcd.io/etcd/clientv3" ) func TestMetaTable_Collection(t *testing.T) { - conf.LoadConfig("config.yaml") - etcdPort := strconv.Itoa(int(conf.Config.Etcd.Port)) + err := gparams.GParams.LoadYaml("config.yaml") + if err != nil { + panic(err) + } + etcdPort, err := gparams.GParams.Load("etcd.port") + if err != nil { + panic(err) + } cli, err := clientv3.New(clientv3.Config{Endpoints: []string{"127.0.0.1:" + etcdPort}}) assert.Nil(t, err) etcdKV := kv.NewEtcdKV(cli, "/etcd/test/root") @@ -137,8 +142,15 @@ func TestMetaTable_Collection(t *testing.T) { } func TestMetaTable_DeletePartition(t *testing.T) { - conf.LoadConfig("config.yaml") - etcdPort := strconv.Itoa(int(conf.Config.Etcd.Port)) + err := gparams.GParams.LoadYaml("config.yaml") + if err != nil { + panic(err) + } + etcdPort, err := gparams.GParams.Load("etcd.port") + if err != nil { + panic(err) + } + cli, err := clientv3.New(clientv3.Config{Endpoints: []string{"127.0.0.1:" + etcdPort}}) assert.Nil(t, err) etcdKV := kv.NewEtcdKV(cli, "/etcd/test/root") @@ -220,8 +232,15 @@ func TestMetaTable_DeletePartition(t *testing.T) { } func TestMetaTable_Segment(t *testing.T) { - conf.LoadConfig("config.yaml") - etcdPort := strconv.Itoa(int(conf.Config.Etcd.Port)) + err := gparams.GParams.LoadYaml("config.yaml") + if err != nil { + panic(err) + } + etcdPort, err := gparams.GParams.Load("etcd.port") + if err != nil { + panic(err) + } + cli, err := clientv3.New(clientv3.Config{Endpoints: []string{"127.0.0.1:" + etcdPort}}) assert.Nil(t, err) etcdKV := kv.NewEtcdKV(cli, "/etcd/test/root") diff --git a/internal/master/mock/grpc_service.go b/internal/master/mock/grpc_service.go index 8296054a61..84e199d3da 100644 --- a/internal/master/mock/grpc_service.go +++ b/internal/master/mock/grpc_service.go @@ -3,6 +3,8 @@ package mockmaster import ( "context" + "github.com/zilliztech/milvus-distributed/internal/master/tso" + "github.com/zilliztech/milvus-distributed/internal/master/id" "github.com/zilliztech/milvus-distributed/internal/proto/commonpb" @@ -73,7 +75,7 @@ func (s *Master) ShowPartitions(ctx context.Context, in *internalpb.ShowPartitio func (s *Master) AllocTimestamp(ctx context.Context, request *internalpb.TsoRequest) (*internalpb.TsoResponse, error) { count := request.GetCount() - ts, err := s.tsoAllocator.GenerateTSO(count) + ts, err := tso.Alloc(count) if err != nil { return &internalpb.TsoResponse{ diff --git a/internal/master/mock/master.go b/internal/master/mock/master.go index a33861e644..3166aa6c1e 100644 --- a/internal/master/mock/master.go +++ b/internal/master/mock/master.go @@ -6,19 +6,18 @@ import ( "log" "math/rand" "net" - "strconv" "sync" "sync/atomic" "time" - "github.com/zilliztech/milvus-distributed/internal/conf" - "github.com/zilliztech/milvus-distributed/internal/kv" - "github.com/zilliztech/milvus-distributed/internal/kv/mockkv" - "github.com/zilliztech/milvus-distributed/internal/master/id" - "github.com/zilliztech/milvus-distributed/internal/proto/masterpb" "google.golang.org/grpc" + "github.com/zilliztech/milvus-distributed/internal/kv/mockkv" + "github.com/zilliztech/milvus-distributed/internal/master/id" "github.com/zilliztech/milvus-distributed/internal/master/tso" + "github.com/zilliztech/milvus-distributed/internal/proto/masterpb" + "github.com/zilliztech/milvus-distributed/internal/util/kvutil" + gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil" ) // Server is the pd server. @@ -36,7 +35,7 @@ type Master struct { // for tso. tsoAllocator tso.Allocator - kvBase kv.Base + kvBase kvutil.Base // error chans grpcErr chan error @@ -183,9 +182,15 @@ func (s *Master) checkReady(ctx context.Context, targetCh chan error) { func (s *Master) grpcLoop() { defer s.serverLoopWg.Done() - masterAddr := conf.Config.Etcd.Address - masterAddr += ":" - masterAddr += strconv.FormatInt(int64(conf.Config.Master.Port), 10) + masterAddr, err := gparams.GParams.Load("master.address") + if err != nil { + panic(err) + } + masterPort, err := gparams.GParams.Load("master.port") + if err != nil { + panic(err) + } + masterAddr = masterAddr + ":" + masterPort lis, err := net.Listen("tcp", masterAddr) if err != nil { log.Printf("failed to listen: %v", err) diff --git a/internal/master/partition_task_test.go b/internal/master/partition_task_test.go index 4bdb123323..e3cdb5aa12 100644 --- a/internal/master/partition_task_test.go +++ b/internal/master/partition_task_test.go @@ -6,25 +6,30 @@ import ( "strconv" "testing" - "github.com/zilliztech/milvus-distributed/internal/conf" - "github.com/zilliztech/milvus-distributed/internal/proto/servicepb" - "github.com/golang/protobuf/proto" "github.com/stretchr/testify/assert" "github.com/zilliztech/milvus-distributed/internal/proto/commonpb" "github.com/zilliztech/milvus-distributed/internal/proto/internalpb" "github.com/zilliztech/milvus-distributed/internal/proto/masterpb" "github.com/zilliztech/milvus-distributed/internal/proto/schemapb" + "github.com/zilliztech/milvus-distributed/internal/proto/servicepb" + gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil" "go.etcd.io/etcd/clientv3" "google.golang.org/grpc" ) func TestMaster_Partition(t *testing.T) { - conf.LoadConfig("config.yaml") + err := gparams.GParams.LoadYaml("config.yaml") + if err != nil { + panic(err) + } ctx, cancel := context.WithCancel(context.TODO()) defer cancel() - etcdPort := strconv.Itoa(int(conf.Config.Etcd.Port)) + etcdPort, err := gparams.GParams.Load("etcd.port") + if err != nil { + panic(err) + } etcdAddr := "127.0.0.1:" + etcdPort etcdCli, err := clientv3.New(clientv3.Config{Endpoints: []string{etcdAddr}}) diff --git a/internal/master/tso/global_allocator.go b/internal/master/tso/global_allocator.go index 5c94acb6fd..0307107ef6 100644 --- a/internal/master/tso/global_allocator.go +++ b/internal/master/tso/global_allocator.go @@ -5,9 +5,8 @@ import ( "sync/atomic" "time" - "github.com/zilliztech/milvus-distributed/internal/kv" - "github.com/zilliztech/milvus-distributed/internal/errors" + "github.com/zilliztech/milvus-distributed/internal/util/kvutil" "github.com/zilliztech/milvus-distributed/internal/util/tsoutil" "github.com/zilliztech/milvus-distributed/internal/util/typeutil" "go.uber.org/zap" @@ -42,13 +41,13 @@ func Init() { InitGlobalTsoAllocator("timestamp", tsoutil.NewTSOKVBase("tso")) } -func InitGlobalTsoAllocator(key string, base kv.Base) { +func InitGlobalTsoAllocator(key string, base kvutil.Base) { allocator = NewGlobalTSOAllocator(key, base) allocator.Initialize() } // NewGlobalTSOAllocator creates a new global TSO allocator. -func NewGlobalTSOAllocator(key string, kvBase kv.Base) *GlobalTSOAllocator { +func NewGlobalTSOAllocator(key string, kvBase kvutil.Base) *GlobalTSOAllocator { var saveInterval = 3 * time.Second return &GlobalTSOAllocator{ tso: ×tampOracle{ diff --git a/internal/master/tso/global_allocator_test.go b/internal/master/tso/global_allocator_test.go index 9db77177e6..0d4d033eaa 100644 --- a/internal/master/tso/global_allocator_test.go +++ b/internal/master/tso/global_allocator_test.go @@ -7,14 +7,17 @@ import ( "github.com/stretchr/testify/assert" - "github.com/zilliztech/milvus-distributed/internal/conf" + gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil" "github.com/zilliztech/milvus-distributed/internal/util/tsoutil" ) var GTsoAllocator Allocator func TestMain(m *testing.M) { - conf.LoadConfig("config.yaml") + err := gparams.GParams.LoadYaml("config.yaml") + if err != nil { + panic(err) + } GTsoAllocator = NewGlobalTSOAllocator("timestamp", tsoutil.NewTSOKVBase("tso")) exitCode := m.Run() diff --git a/internal/master/tso/tso.go b/internal/master/tso/tso.go index 02a99931bf..81f246deb1 100644 --- a/internal/master/tso/tso.go +++ b/internal/master/tso/tso.go @@ -22,7 +22,7 @@ import ( "go.uber.org/zap" "github.com/zilliztech/milvus-distributed/internal/errors" - "github.com/zilliztech/milvus-distributed/internal/kv" + "github.com/zilliztech/milvus-distributed/internal/util/kvutil" "github.com/zilliztech/milvus-distributed/internal/util/tsoutil" "github.com/zilliztech/milvus-distributed/internal/util/typeutil" ) @@ -47,7 +47,7 @@ type atomicObject struct { // timestampOracle is used to maintain the logic of tso. type timestampOracle struct { key string - kvBase kv.Base + kvBase kvutil.Base // TODO: remove saveInterval saveInterval time.Duration diff --git a/internal/proxy/grpc_service.go b/internal/proxy/grpc_service.go index 788aa6eada..88ed746fcd 100644 --- a/internal/proxy/grpc_service.go +++ b/internal/proxy/grpc_service.go @@ -3,10 +3,8 @@ package proxy import ( "context" "errors" - "log" "github.com/golang/protobuf/proto" - "github.com/zilliztech/milvus-distributed/internal/msgstream" "github.com/zilliztech/milvus-distributed/internal/proto/commonpb" "github.com/zilliztech/milvus-distributed/internal/proto/internalpb" @@ -28,27 +26,44 @@ func (p *Proxy) Insert(ctx context.Context, in *servicepb.RowBatch) (*servicepb. }, }, done: make(chan error), - resultChan: make(chan *servicepb.IntegerRangeResponse), manipulationMsgStream: p.manipulationMsgStream, } + it.ctx, it.cancel = context.WithCancel(ctx) // TODO: req_id, segment_id, channel_id, proxy_id, timestamps, row_ids defer it.cancel() - p.taskSch.DmQueue.Enqueue(it) - select { - case <-ctx.Done(): - log.Print("insert timeout!") + fn := func() error { + select { + case <-ctx.Done(): + return errors.New("insert timeout") + default: + return p.taskSch.DdQueue.Enqueue(it) + } + } + err := fn() + + if err != nil { return &servicepb.IntegerRangeResponse{ Status: &commonpb.Status{ ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR, - Reason: "insert timeout!", + Reason: err.Error(), }, - }, errors.New("insert timeout") - case result := <-it.resultChan: - return result, nil + }, nil } + + err = it.WaitToFinish() + if err != nil { + return &servicepb.IntegerRangeResponse{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR, + Reason: err.Error(), + }, + }, nil + } + + return it.result, nil } func (p *Proxy) CreateCollection(ctx context.Context, req *schemapb.CollectionSchema) (*commonpb.Status, error) { @@ -60,24 +75,37 @@ func (p *Proxy) CreateCollection(ctx context.Context, req *schemapb.CollectionSc }, masterClient: p.masterClient, done: make(chan error), - resultChan: make(chan *commonpb.Status), } schemaBytes, _ := proto.Marshal(req) cct.CreateCollectionRequest.Schema.Value = schemaBytes cct.ctx, cct.cancel = context.WithCancel(ctx) defer cct.cancel() - p.taskSch.DdQueue.Enqueue(cct) - select { - case <-ctx.Done(): - log.Print("create collection timeout!") + fn := func() error { + select { + case <-ctx.Done(): + return errors.New("create collection timeout") + default: + return p.taskSch.DdQueue.Enqueue(cct) + } + } + err := fn() + if err != nil { return &commonpb.Status{ ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR, - Reason: "create collection timeout!", - }, errors.New("create collection timeout") - case result := <-cct.resultChan: - return result, nil + Reason: err.Error(), + }, err } + + err = cct.WaitToFinish() + if err != nil { + return &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR, + Reason: err.Error(), + }, err + } + + return cct.result, nil } func (p *Proxy) Search(ctx context.Context, req *servicepb.Query) (*servicepb.QueryResult, error) { @@ -90,26 +118,41 @@ func (p *Proxy) Search(ctx context.Context, req *servicepb.Query) (*servicepb.Qu queryMsgStream: p.queryMsgStream, done: make(chan error), resultBuf: make(chan []*internalpb.SearchResult), - resultChan: make(chan *servicepb.QueryResult), } qt.ctx, qt.cancel = context.WithCancel(ctx) queryBytes, _ := proto.Marshal(req) qt.SearchRequest.Query.Value = queryBytes defer qt.cancel() - p.taskSch.DqQueue.Enqueue(qt) - select { - case <-ctx.Done(): - log.Print("query timeout!") + fn := func() error { + select { + case <-ctx.Done(): + return errors.New("create collection timeout") + default: + return p.taskSch.DdQueue.Enqueue(qt) + } + } + err := fn() + if err != nil { return &servicepb.QueryResult{ Status: &commonpb.Status{ ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR, - Reason: "query timeout!", + Reason: err.Error(), }, - }, errors.New("query timeout") - case result := <-qt.resultChan: - return result, nil + }, err } + + err = qt.WaitToFinish() + if err != nil { + return &servicepb.QueryResult{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR, + Reason: err.Error(), + }, + }, err + } + + return qt.result, nil } func (p *Proxy) DropCollection(ctx context.Context, req *servicepb.CollectionName) (*commonpb.Status, error) { @@ -121,22 +164,35 @@ func (p *Proxy) DropCollection(ctx context.Context, req *servicepb.CollectionNam }, masterClient: p.masterClient, done: make(chan error), - resultChan: make(chan *commonpb.Status), } dct.ctx, dct.cancel = context.WithCancel(ctx) defer dct.cancel() - p.taskSch.DdQueue.Enqueue(dct) - select { - case <-ctx.Done(): - log.Print("create collection timeout!") + fn := func() error { + select { + case <-ctx.Done(): + return errors.New("create collection timeout") + default: + return p.taskSch.DdQueue.Enqueue(dct) + } + } + err := fn() + if err != nil { return &commonpb.Status{ ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR, - Reason: "create collection timeout!", - }, errors.New("create collection timeout") - case result := <-dct.resultChan: - return result, nil + Reason: err.Error(), + }, err } + + err = dct.WaitToFinish() + if err != nil { + return &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR, + Reason: err.Error(), + }, err + } + + return dct.result, nil } func (p *Proxy) HasCollection(ctx context.Context, req *servicepb.CollectionName) (*servicepb.BoolResponse, error) { @@ -148,25 +204,39 @@ func (p *Proxy) HasCollection(ctx context.Context, req *servicepb.CollectionName }, masterClient: p.masterClient, done: make(chan error), - resultChan: make(chan *servicepb.BoolResponse), } hct.ctx, hct.cancel = context.WithCancel(ctx) defer hct.cancel() - p.taskSch.DqQueue.Enqueue(hct) - select { - case <-ctx.Done(): - log.Print("has collection timeout!") + fn := func() error { + select { + case <-ctx.Done(): + return errors.New("create collection timeout") + default: + return p.taskSch.DdQueue.Enqueue(hct) + } + } + err := fn() + if err != nil { return &servicepb.BoolResponse{ Status: &commonpb.Status{ ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR, - Reason: "has collection timeout!", + Reason: err.Error(), }, - Value: false, - }, errors.New("has collection timeout") - case result := <-hct.resultChan: - return result, nil + }, err } + + err = hct.WaitToFinish() + if err != nil { + return &servicepb.BoolResponse{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR, + Reason: err.Error(), + }, + }, err + } + + return hct.result, nil } func (p *Proxy) DescribeCollection(ctx context.Context, req *servicepb.CollectionName) (*servicepb.CollectionDescription, error) { @@ -178,24 +248,39 @@ func (p *Proxy) DescribeCollection(ctx context.Context, req *servicepb.Collectio }, masterClient: p.masterClient, done: make(chan error), - resultChan: make(chan *servicepb.CollectionDescription), } dct.ctx, dct.cancel = context.WithCancel(ctx) defer dct.cancel() - p.taskSch.DqQueue.Enqueue(dct) - select { - case <-ctx.Done(): - log.Print("has collection timeout!") + fn := func() error { + select { + case <-ctx.Done(): + return errors.New("create collection timeout") + default: + return p.taskSch.DdQueue.Enqueue(dct) + } + } + err := fn() + if err != nil { return &servicepb.CollectionDescription{ Status: &commonpb.Status{ ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR, - Reason: "describe collection timeout!", + Reason: err.Error(), }, - }, errors.New("describe collection timeout") - case result := <-dct.resultChan: - return result, nil + }, err } + + err = dct.WaitToFinish() + if err != nil { + return &servicepb.CollectionDescription{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR, + Reason: err.Error(), + }, + }, err + } + + return dct.result, nil } func (p *Proxy) ShowCollections(ctx context.Context, req *commonpb.Empty) (*servicepb.StringListResponse, error) { @@ -206,24 +291,39 @@ func (p *Proxy) ShowCollections(ctx context.Context, req *commonpb.Empty) (*serv }, masterClient: p.masterClient, done: make(chan error), - resultChan: make(chan *servicepb.StringListResponse), } sct.ctx, sct.cancel = context.WithCancel(ctx) defer sct.cancel() - p.taskSch.DqQueue.Enqueue(sct) - select { - case <-ctx.Done(): - log.Print("show collections timeout!") + fn := func() error { + select { + case <-ctx.Done(): + return errors.New("create collection timeout") + default: + return p.taskSch.DdQueue.Enqueue(sct) + } + } + err := fn() + if err != nil { return &servicepb.StringListResponse{ Status: &commonpb.Status{ ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR, - Reason: "show collections timeout!", + Reason: err.Error(), }, - }, errors.New("show collections timeout") - case result := <-sct.resultChan: - return result, nil + }, err } + + err = sct.WaitToFinish() + if err != nil { + return &servicepb.StringListResponse{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR, + Reason: err.Error(), + }, + }, err + } + + return sct.result, nil } func (p *Proxy) CreatePartition(ctx context.Context, in *servicepb.PartitionName) (*commonpb.Status, error) { diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index fa243bcc37..65d4cd56b6 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -5,9 +5,12 @@ import ( "log" "math/rand" "net" + "strconv" "sync" "time" + "github.com/zilliztech/milvus-distributed/internal/conf" + "google.golang.org/grpc" "github.com/zilliztech/milvus-distributed/internal/allocator" @@ -51,10 +54,30 @@ func CreateProxy(ctx context.Context) (*Proxy, error) { proxyLoopCtx: ctx1, proxyLoopCancel: cancel, } + + // TODO: use config instead + pulsarAddress := "pulsar://localhost:6650" bufSize := int64(1000) + manipulationChannels := []string{"manipulation"} + queryChannels := []string{"query"} + queryResultChannels := []string{"QueryResult"} + queryResultSubName := "QueryResultSubject" + unmarshal := msgstream.NewUnmarshalDispatcher() + p.manipulationMsgStream = msgstream.NewPulsarMsgStream(p.proxyLoopCtx, bufSize) + p.manipulationMsgStream.SetPulsarCient(pulsarAddress) + p.manipulationMsgStream.CreatePulsarProducers(manipulationChannels) + p.queryMsgStream = msgstream.NewPulsarMsgStream(p.proxyLoopCtx, bufSize) + p.queryMsgStream.SetPulsarCient(pulsarAddress) + p.queryMsgStream.CreatePulsarProducers(queryChannels) + p.queryResultMsgStream = msgstream.NewPulsarMsgStream(p.proxyLoopCtx, bufSize) + p.queryResultMsgStream.SetPulsarCient(pulsarAddress) + p.queryResultMsgStream.CreatePulsarConsumers(queryResultChannels, + queryResultSubName, + unmarshal, + bufSize) idAllocator, err := allocator.NewIDAllocator(p.proxyLoopCtx) @@ -111,7 +134,7 @@ func (p *Proxy) grpcLoop() { defer p.proxyLoopWg.Done() // TODO: use address in config instead - lis, err := net.Listen("tcp", "5053") + lis, err := net.Listen("tcp", ":5053") if err != nil { log.Fatalf("Proxy grpc server fatal error=%v", err) } @@ -124,16 +147,18 @@ func (p *Proxy) grpcLoop() { } func (p *Proxy) connectMaster() error { - log.Printf("Connected to master, master_addr=%s", "127.0.0.1:5053") + masterHost := conf.Config.Master.Address + masterPort := conf.Config.Master.Port + masterAddr := masterHost + ":" + strconv.FormatInt(int64(masterPort), 10) + log.Printf("Proxy connected to master, master_addr=%s", masterAddr) ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() - - conn, err := grpc.DialContext(ctx, "127.0.0.1:5053", grpc.WithInsecure(), grpc.WithBlock()) + conn, err := grpc.DialContext(ctx, masterAddr, grpc.WithInsecure(), grpc.WithBlock()) if err != nil { - log.Printf("Connect to master failed, error= %v", err) + log.Printf("Proxy connect to master failed, error= %v", err) return err } - log.Printf("Connected to master, master_addr=%s", "127.0.0.1:5053") + log.Printf("Proxy connected to master, master_addr=%s", masterAddr) p.masterConn = conn p.masterClient = masterpb.NewMasterClient(conn) return nil @@ -141,28 +166,39 @@ func (p *Proxy) connectMaster() error { func (p *Proxy) queryResultLoop() { defer p.proxyLoopWg.Done() + defer p.proxyLoopCancel() queryResultBuf := make(map[UniqueID][]*internalpb.SearchResult) for { - msgPack := p.queryResultMsgStream.Consume() - if msgPack == nil { - continue - } - tsMsg := msgPack.Msgs[0] - searchResultMsg, _ := (*tsMsg).(*msgstream.SearchResultMsg) - reqID := searchResultMsg.GetReqID() - _, ok := queryResultBuf[reqID] - if !ok { - queryResultBuf[reqID] = make([]*internalpb.SearchResult, 0) - } - queryResultBuf[reqID] = append(queryResultBuf[reqID], &searchResultMsg.SearchResult) - if len(queryResultBuf[reqID]) == 4 { - // TODO: use the number of query node instead - t := p.taskSch.getTaskByReqID(reqID) - qt := t.(*QueryTask) - qt.resultBuf <- queryResultBuf[reqID] - delete(queryResultBuf, reqID) + select { + case msgPack, ok := <-p.queryResultMsgStream.Chan(): + if !ok { + log.Print("buf chan closed") + return + } + log.Print("Consume message from query result stream...") + if msgPack == nil { + continue + } + tsMsg := msgPack.Msgs[0] + searchResultMsg, _ := (*tsMsg).(*msgstream.SearchResultMsg) + reqID := searchResultMsg.GetReqID() + _, ok = queryResultBuf[reqID] + if !ok { + queryResultBuf[reqID] = make([]*internalpb.SearchResult, 0) + } + queryResultBuf[reqID] = append(queryResultBuf[reqID], &searchResultMsg.SearchResult) + if len(queryResultBuf[reqID]) == 4 { + // TODO: use the number of query node instead + t := p.taskSch.getTaskByReqID(reqID) + qt := t.(*QueryTask) + qt.resultBuf <- queryResultBuf[reqID] + delete(queryResultBuf, reqID) + } + case <-p.proxyLoopCtx.Done(): + log.Print("proxy server is closed ...") + return } } } @@ -189,10 +225,15 @@ func (p *Proxy) stopProxyLoop() { p.grpcServer.GracefulStop() } p.tsoAllocator.Close() + p.idAllocator.Close() + p.taskSch.Close() + p.manipulationMsgStream.Close() + p.queryMsgStream.Close() + p.queryResultMsgStream.Close() p.proxyLoopWg.Wait() diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go new file mode 100644 index 0000000000..20b9273be1 --- /dev/null +++ b/internal/proxy/proxy_test.go @@ -0,0 +1,89 @@ +package proxy + +import ( + "context" + "fmt" + "log" + "os" + "testing" + + "github.com/zilliztech/milvus-distributed/internal/conf" + mockMaster "github.com/zilliztech/milvus-distributed/internal/master/mock" + "github.com/zilliztech/milvus-distributed/internal/proto/servicepb" + "go.uber.org/zap" + "google.golang.org/grpc" +) + +var ctx context.Context +var cancel func() +var proxyAddress = "127.0.0.1:5053" +var proxyConn *grpc.ClientConn +var proxyClient servicepb.MilvusServiceClient + +var proxyServer *Proxy + +var masterServer *mockMaster.Master + +func startMaster(ctx context.Context) { + //conf.LoadConfig("config.yaml") + fmt.Println("THIS is test before.") + svr, err := mockMaster.CreateServer(ctx) + masterServer = svr + if err != nil { + log.Print("create server failed", zap.Error(err)) + } + + if err := svr.Run(); err != nil { + log.Fatal("run server failed", zap.Error(err)) + } + + fmt.Println("Waiting for server!", svr.IsServing()) + +} + +func startProxy(ctx context.Context) { + + svr, err := CreateProxy(ctx) + proxyServer = svr + if err != nil { + log.Print("create proxy failed", zap.Error(err)) + } + + // TODO: change to wait until master is ready + if err := svr.Run(); err != nil { + log.Fatal("run proxy failed", zap.Error(err)) + } +} + +func setup() { + conf.LoadConfig("config.yaml") + ctx, cancel = context.WithCancel(context.Background()) + + startMaster(ctx) + startProxy(ctx) + + conn, err := grpc.DialContext(ctx, proxyAddress, grpc.WithInsecure(), grpc.WithBlock()) + if err != nil { + log.Fatalf("Connect to proxy failed, error= %v", err) + } + proxyConn = conn + proxyClient = servicepb.NewMilvusServiceClient(proxyConn) + +} + +func shutdown() { + cancel() + masterServer.Close() + proxyServer.Close() +} + +func TestProxy_CreateCollection(t *testing.T) { + fmt.Println(proxyClient) +} + +func TestMain(m *testing.M) { + setup() + code := m.Run() + shutdown() + os.Exit(code) +} diff --git a/internal/proxy/task.go b/internal/proxy/task.go index 4158d32c8b..20cc765b9e 100644 --- a/internal/proxy/task.go +++ b/internal/proxy/task.go @@ -31,7 +31,7 @@ type InsertTask struct { BaseInsertTask ts Timestamp done chan error - resultChan chan *servicepb.IntegerRangeResponse + result *servicepb.IntegerRangeResponse manipulationMsgStream *msgstream.PulsarMsgStream ctx context.Context cancel context.CancelFunc @@ -98,7 +98,7 @@ type CreateCollectionTask struct { internalpb.CreateCollectionRequest masterClient masterpb.MasterClient done chan error - resultChan chan *commonpb.Status + result *commonpb.Status ctx context.Context cancel context.CancelFunc } @@ -131,12 +131,12 @@ func (cct *CreateCollectionTask) Execute() error { resp, err := cct.masterClient.CreateCollection(cct.ctx, &cct.CreateCollectionRequest) if err != nil { log.Printf("create collection failed, error= %v", err) - cct.resultChan <- &commonpb.Status{ + cct.result = &commonpb.Status{ ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR, Reason: err.Error(), } } else { - cct.resultChan <- resp + cct.result = resp } return err } @@ -166,7 +166,7 @@ type DropCollectionTask struct { internalpb.DropCollectionRequest masterClient masterpb.MasterClient done chan error - resultChan chan *commonpb.Status + result *commonpb.Status ctx context.Context cancel context.CancelFunc } @@ -199,12 +199,12 @@ func (dct *DropCollectionTask) Execute() error { resp, err := dct.masterClient.DropCollection(dct.ctx, &dct.DropCollectionRequest) if err != nil { log.Printf("drop collection failed, error= %v", err) - dct.resultChan <- &commonpb.Status{ + dct.result = &commonpb.Status{ ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR, Reason: err.Error(), } } else { - dct.resultChan <- resp + dct.result = resp } return err } @@ -235,7 +235,7 @@ type QueryTask struct { queryMsgStream *msgstream.PulsarMsgStream done chan error resultBuf chan []*internalpb.SearchResult - resultChan chan *servicepb.QueryResult + result *servicepb.QueryResult ctx context.Context cancel context.CancelFunc } @@ -312,12 +312,12 @@ func (qt *QueryTask) Notify(err error) { case searchResults := <-qt.resultBuf: rlen := len(searchResults) // query num if rlen <= 0 { - qt.resultChan <- &servicepb.QueryResult{} + qt.result = &servicepb.QueryResult{} return } n := len(searchResults[0].Hits) // n if n <= 0 { - qt.resultChan <- &servicepb.QueryResult{} + qt.result = &servicepb.QueryResult{} return } k := len(searchResults[0].Hits[0].IDs) // k @@ -354,7 +354,7 @@ func (qt *QueryTask) Notify(err error) { } queryResult.Hits = append(queryResult.Hits, hits) } - qt.resultChan <- queryResult + qt.result = queryResult } } } @@ -363,7 +363,7 @@ type HasCollectionTask struct { internalpb.HasCollectionRequest masterClient masterpb.MasterClient done chan error - resultChan chan *servicepb.BoolResponse + result *servicepb.BoolResponse ctx context.Context cancel context.CancelFunc } @@ -396,7 +396,7 @@ func (hct *HasCollectionTask) Execute() error { resp, err := hct.masterClient.HasCollection(hct.ctx, &hct.HasCollectionRequest) if err != nil { log.Printf("has collection failed, error= %v", err) - hct.resultChan <- &servicepb.BoolResponse{ + hct.result = &servicepb.BoolResponse{ Status: &commonpb.Status{ ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR, Reason: "internal error", @@ -404,7 +404,7 @@ func (hct *HasCollectionTask) Execute() error { Value: false, } } else { - hct.resultChan <- resp + hct.result = resp } return err } @@ -434,7 +434,7 @@ type DescribeCollectionTask struct { internalpb.DescribeCollectionRequest masterClient masterpb.MasterClient done chan error - resultChan chan *servicepb.CollectionDescription + result *servicepb.CollectionDescription ctx context.Context cancel context.CancelFunc } @@ -467,14 +467,14 @@ func (dct *DescribeCollectionTask) Execute() error { resp, err := dct.masterClient.DescribeCollection(dct.ctx, &dct.DescribeCollectionRequest) if err != nil { log.Printf("describe collection failed, error= %v", err) - dct.resultChan <- &servicepb.CollectionDescription{ + dct.result = &servicepb.CollectionDescription{ Status: &commonpb.Status{ ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR, Reason: "internal error", }, } } else { - dct.resultChan <- resp + dct.result = resp } return err } @@ -504,7 +504,7 @@ type ShowCollectionsTask struct { internalpb.ShowCollectionRequest masterClient masterpb.MasterClient done chan error - resultChan chan *servicepb.StringListResponse + result *servicepb.StringListResponse ctx context.Context cancel context.CancelFunc } @@ -537,14 +537,14 @@ func (sct *ShowCollectionsTask) Execute() error { resp, err := sct.masterClient.ShowCollections(sct.ctx, &sct.ShowCollectionRequest) if err != nil { log.Printf("show collections failed, error= %v", err) - sct.resultChan <- &servicepb.StringListResponse{ + sct.result = &servicepb.StringListResponse{ Status: &commonpb.Status{ ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR, Reason: "internal error", }, } } else { - sct.resultChan <- resp + sct.result = resp } return err } diff --git a/internal/proxy/task_scheduler.go b/internal/proxy/task_scheduler.go index bfa2e579c6..0c6f570d9b 100644 --- a/internal/proxy/task_scheduler.go +++ b/internal/proxy/task_scheduler.go @@ -3,73 +3,113 @@ package proxy import ( "container/list" "context" + "errors" "log" "sync" "github.com/zilliztech/milvus-distributed/internal/allocator" ) +type TaskQueue interface { + utChan() <-chan int + utEmpty() bool + utFull() bool + addUnissuedTask(t task) error + FrontUnissuedTask() task + PopUnissuedTask() task + AddActiveTask(t task) + PopActiveTask(ts Timestamp) task + getTaskByReqID(reqID UniqueID) task + TaskDoneTest(ts Timestamp) bool + Enqueue(t task) error +} + type BaseTaskQueue struct { unissuedTasks *list.List activeTasks map[Timestamp]task utLock sync.Mutex atLock sync.Mutex + + // maxTaskNum should keep still + maxTaskNum int64 + + utBufChan chan int // to block scheduler } -func (queue *BaseTaskQueue) Empty() bool { - queue.utLock.Lock() - defer queue.utLock.Unlock() - queue.atLock.Lock() - defer queue.atLock.Unlock() - return queue.unissuedTasks.Len() <= 0 && len(queue.activeTasks) <= 0 +func (queue *BaseTaskQueue) utChan() <-chan int { + return queue.utBufChan } -func (queue *BaseTaskQueue) AddUnissuedTask(t task) { +func (queue *BaseTaskQueue) utEmpty() bool { + return queue.unissuedTasks.Len() == 0 +} + +func (queue *BaseTaskQueue) utFull() bool { + return int64(queue.unissuedTasks.Len()) >= queue.maxTaskNum +} + +func (queue *BaseTaskQueue) addUnissuedTask(t task) error { queue.utLock.Lock() defer queue.utLock.Unlock() + + if queue.utFull() { + return errors.New("task queue is full") + } queue.unissuedTasks.PushBack(t) + queue.utBufChan <- 1 + return nil } func (queue *BaseTaskQueue) FrontUnissuedTask() task { queue.utLock.Lock() defer queue.utLock.Unlock() + if queue.unissuedTasks.Len() <= 0 { - log.Fatal("sorry, but the unissued task list is empty!") + log.Panic("sorry, but the unissued task list is empty!") return nil } + return queue.unissuedTasks.Front().Value.(task) } func (queue *BaseTaskQueue) PopUnissuedTask() task { queue.utLock.Lock() defer queue.utLock.Unlock() + if queue.unissuedTasks.Len() <= 0 { log.Fatal("sorry, but the unissued task list is empty!") return nil } + ft := queue.unissuedTasks.Front() - return queue.unissuedTasks.Remove(ft).(task) + queue.unissuedTasks.Remove(ft) + + return ft.Value.(task) } func (queue *BaseTaskQueue) AddActiveTask(t task) { queue.atLock.Lock() defer queue.atLock.Lock() + ts := t.EndTs() _, ok := queue.activeTasks[ts] if ok { log.Fatalf("task with timestamp %v already in active task list!", ts) } + queue.activeTasks[ts] = t } func (queue *BaseTaskQueue) PopActiveTask(ts Timestamp) task { queue.atLock.Lock() defer queue.atLock.Lock() + t, ok := queue.activeTasks[ts] if ok { delete(queue.activeTasks, ts) return t } + log.Fatalf("sorry, but the timestamp %d was not found in the active task list!", ts) return nil } @@ -114,6 +154,11 @@ func (queue *BaseTaskQueue) TaskDoneTest(ts Timestamp) bool { return true } +func (queue *BaseTaskQueue) Enqueue(t task) error { + // TODO: set Ts, ReqId, ProxyId + return queue.addUnissuedTask(t) +} + type DdTaskQueue struct { BaseTaskQueue lock sync.Mutex @@ -128,23 +173,13 @@ type DqTaskQueue struct { } func (queue *DdTaskQueue) Enqueue(t task) error { + queue.lock.Lock() defer queue.lock.Unlock() - // TODO: set Ts, ReqID, ProxyID - queue.AddUnissuedTask(t) - return nil -} -func (queue *DmTaskQueue) Enqueue(t task) error { - // TODO: set Ts, ReqID, ProxyID - queue.AddUnissuedTask(t) - return nil -} + // TODO: set Ts, ReqId, ProxyId + return queue.addUnissuedTask(t) -func (queue *DqTaskQueue) Enqueue(t task) error { - // TODO: set Ts, ReqID, ProxyID - queue.AddUnissuedTask(t) - return nil } func NewDdTaskQueue() *DdTaskQueue { @@ -152,6 +187,8 @@ func NewDdTaskQueue() *DdTaskQueue { BaseTaskQueue: BaseTaskQueue{ unissuedTasks: list.New(), activeTasks: make(map[Timestamp]task), + maxTaskNum: 1024, + utBufChan: make(chan int, 1024), }, } } @@ -161,6 +198,8 @@ func NewDmTaskQueue() *DmTaskQueue { BaseTaskQueue: BaseTaskQueue{ unissuedTasks: list.New(), activeTasks: make(map[Timestamp]task), + maxTaskNum: 1024, + utBufChan: make(chan int, 1024), }, } } @@ -170,14 +209,16 @@ func NewDqTaskQueue() *DqTaskQueue { BaseTaskQueue: BaseTaskQueue{ unissuedTasks: list.New(), activeTasks: make(map[Timestamp]task), + maxTaskNum: 1024, + utBufChan: make(chan int, 1024), }, } } type TaskScheduler struct { - DdQueue *DdTaskQueue - DmQueue *DmTaskQueue - DqQueue *DqTaskQueue + DdQueue TaskQueue + DmQueue TaskQueue + DqQueue TaskQueue idAllocator *allocator.IDAllocator tsoAllocator *allocator.TimestampAllocator @@ -229,116 +270,82 @@ func (sched *TaskScheduler) getTaskByReqID(collMeta UniqueID) task { return nil } +func (sched *TaskScheduler) processTask(t task, q TaskQueue) { + + err := t.PreExecute() + + defer func() { + t.Notify(err) + }() + if err != nil { + return + } + + q.AddActiveTask(t) + defer q.PopActiveTask(t.EndTs()) + + err = t.Execute() + if err != nil { + log.Printf("execute definition task failed, error = %v", err) + return + } + err = t.PostExecute() +} + func (sched *TaskScheduler) definitionLoop() { defer sched.wg.Done() - defer sched.cancel() - for { - if sched.DdQueue.Empty() { - continue - } - - //sched.DdQueue.atLock.Lock() - t := sched.scheduleDdTask() - - err := t.PreExecute() - if err != nil { + select { + case <-sched.ctx.Done(): return + case <-sched.DdQueue.utChan(): + if !sched.DdQueue.utEmpty() { + t := sched.scheduleDdTask() + sched.processTask(t, sched.DdQueue) + } } - err = t.Execute() - if err != nil { - log.Printf("execute definition task failed, error = %v", err) - } - t.Notify(err) - - sched.DdQueue.AddActiveTask(t) - - t.WaitToFinish() - t.PostExecute() - - sched.DdQueue.PopActiveTask(t.EndTs()) } } func (sched *TaskScheduler) manipulationLoop() { defer sched.wg.Done() - defer sched.cancel() - for { - if sched.DmQueue.Empty() { - continue - } - - sched.DmQueue.atLock.Lock() - t := sched.scheduleDmTask() - - if err := t.PreExecute(); err != nil { + select { + case <-sched.ctx.Done(): return - } - - go func() { - err := t.Execute() - if err != nil { - log.Printf("execute manipulation task failed, error = %v", err) + case <-sched.DmQueue.utChan(): + if !sched.DmQueue.utEmpty() { + t := sched.scheduleDmTask() + go sched.processTask(t, sched.DmQueue) } - t.Notify(err) - }() - - sched.DmQueue.AddActiveTask(t) - sched.DmQueue.atLock.Unlock() - - go func() { - t.WaitToFinish() - t.PostExecute() - - // remove from active list - sched.DmQueue.PopActiveTask(t.EndTs()) - }() + } } } func (sched *TaskScheduler) queryLoop() { defer sched.wg.Done() - defer sched.cancel() for { - if sched.DqQueue.Empty() { - continue - } - - sched.DqQueue.atLock.Lock() - t := sched.scheduleDqTask() - - if err := t.PreExecute(); err != nil { + select { + case <-sched.ctx.Done(): return - } - - go func() { - err := t.Execute() - if err != nil { - log.Printf("execute query task failed, error = %v", err) + case <-sched.DqQueue.utChan(): + if !sched.DqQueue.utEmpty() { + t := sched.scheduleDqTask() + go sched.processTask(t, sched.DqQueue) } - t.Notify(err) - }() - - sched.DqQueue.AddActiveTask(t) - sched.DqQueue.atLock.Unlock() - - go func() { - t.WaitToFinish() - t.PostExecute() - - // remove from active list - sched.DqQueue.PopActiveTask(t.EndTs()) - }() + } } } func (sched *TaskScheduler) Start() error { - sched.wg.Add(3) - + sched.wg.Add(1) go sched.definitionLoop() + + sched.wg.Add(1) go sched.manipulationLoop() + + sched.wg.Add(1) go sched.queryLoop() return nil diff --git a/internal/proxy/timetick.go b/internal/proxy/timetick.go index 81b96b54e2..6ffb9e3c92 100644 --- a/internal/proxy/timetick.go +++ b/internal/proxy/timetick.go @@ -3,9 +3,12 @@ package proxy import ( "context" "log" + "strconv" "sync" "time" + "github.com/zilliztech/milvus-distributed/internal/conf" + "github.com/zilliztech/milvus-distributed/internal/allocator" "github.com/zilliztech/milvus-distributed/internal/msgstream" @@ -14,10 +17,12 @@ import ( "github.com/apache/pulsar-client-go/pulsar" ) +type tickCheckFunc = func(Timestamp) bool + type timeTick struct { lastTick Timestamp currentTick Timestamp - interval uint64 + interval int64 pulsarProducer pulsar.Producer @@ -30,21 +35,29 @@ type timeTick struct { cancel func() timer *time.Ticker - areRequestsDelivered func(ts Timestamp) bool + checkFunc tickCheckFunc } -func newTimeTick(ctx context.Context, tsoAllocator *allocator.TimestampAllocator) *timeTick { +func newTimeTick(ctx context.Context, + tsoAllocator *allocator.TimestampAllocator, + checkFunc tickCheckFunc) *timeTick { ctx1, cancel := context.WithCancel(ctx) t := &timeTick{ ctx: ctx1, cancel: cancel, tsoAllocator: tsoAllocator, + interval: 200, + peerID: 1, + checkFunc: checkFunc, } - bufSzie := int64(1000) - t.tickMsgStream = msgstream.NewPulsarMsgStream(t.ctx, bufSzie) + bufSize := int64(1000) + t.tickMsgStream = msgstream.NewPulsarMsgStream(t.ctx, bufSize) + pulsarAddress := "pulsar://" + pulsarAddress += conf.Config.Pulsar.Address + pulsarAddress += ":" + pulsarAddress += strconv.FormatInt(int64(conf.Config.Pulsar.Port), 10) - pulsarAddress := "pulsar://localhost:6650" producerChannels := []string{"timeTick"} t.tickMsgStream.SetPulsarCient(pulsarAddress) t.tickMsgStream.CreatePulsarProducers(producerChannels) @@ -61,7 +74,7 @@ func (tt *timeTick) tick() error { tt.currentTick = ts } - if !tt.areRequestsDelivered(tt.currentTick) { + if !tt.checkFunc(tt.currentTick) { return nil } msgPack := msgstream.MsgPack{} diff --git a/internal/proxy/timetick_test.go b/internal/proxy/timetick_test.go index ab092077a5..4dbcf1c693 100644 --- a/internal/proxy/timetick_test.go +++ b/internal/proxy/timetick_test.go @@ -2,84 +2,48 @@ package proxy import ( "context" + "fmt" + "math/rand" "testing" "time" - "github.com/apache/pulsar-client-go/pulsar" - "github.com/golang/protobuf/proto" "github.com/stretchr/testify/assert" - "github.com/zilliztech/milvus-distributed/internal/proto/internalpb" + "github.com/zilliztech/milvus-distributed/internal/allocator" ) -func TestTimeTick(t *testing.T) { - client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"}) - assert.Nil(t, err) +var trueCnt = 0 - producer, err := client.CreateProducer(pulsar.ProducerOptions{Topic: "timesync"}) - assert.Nil(t, err) - - consumer, err := client.Subscribe(pulsar.ConsumerOptions{ - Topic: "timesync", - SubscriptionName: "timesync_group", - Type: pulsar.KeyShared, - SubscriptionInitialPosition: pulsar.SubscriptionPositionEarliest, - }) - assert.Nil(t, err) - - ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second) - defer cancel() - - //var curTs Timestamp - //curTs = 0 - tt := timeTick{ - interval: 200, - pulsarProducer: producer, - peerID: 1, - ctx: ctx, - areRequestsDelivered: func(ts Timestamp) bool { return true }, +func checkFunc(timestamp Timestamp) bool { + ret := rand.Intn(2) == 1 + if ret { + trueCnt++ } + return ret +} + +func TestTimeTick_Start(t *testing.T) { + fmt.Println("HHH") +} + +func TestTimeTick_Start2(t *testing.T) { + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + + tsoAllocator, err := allocator.NewTimestampAllocator(ctx) + assert.Nil(t, err) + err = tsoAllocator.Start() + assert.Nil(t, err) + + tt := newTimeTick(ctx, tsoAllocator, checkFunc) + + defer func() { + cancel() + tsoAllocator.Close() + tt.Close() + }() + tt.Start() - ctx2, cancel2 := context.WithTimeout(context.Background(), time.Second*2) - defer cancel2() + <-ctx.Done() - isbreak := false - for { - if isbreak { - break - } - select { - case <-ctx2.Done(): - isbreak = true - break - case cm, ok := <-consumer.Chan(): - if !ok { - t.Fatalf("consumer closed") - } - consumer.AckID(cm.ID()) - break - } - } - - var lastTimestamp uint64 = 0 - for { - select { - case <-ctx.Done(): - return - case cm, ok := <-consumer.Chan(): - if ok == false { - return - } - msg := cm.Message - var tsm internalpb.TimeTickMsg - if err := proto.Unmarshal(msg.Payload(), &tsm); err != nil { - return - } - if tsm.Timestamp <= lastTimestamp { - t.Fatalf("current = %d, last = %d", tsm.Timestamp, lastTimestamp) - } - t.Log("current = ", tsm.Timestamp) - lastTimestamp = tsm.Timestamp - } - } } diff --git a/internal/proxy/util.go b/internal/proxy/util.go new file mode 100644 index 0000000000..7546aa4d27 --- /dev/null +++ b/internal/proxy/util.go @@ -0,0 +1,32 @@ +package proxy + +import ( + "log" + "time" +) + +// Reference: https://blog.cyeam.com/golang/2018/08/27/retry + +func Retry(attempts int, sleep time.Duration, fn func() error) error { + if err := fn(); err != nil { + if s, ok := err.(InterruptError); ok { + return s.error + } + + if attempts--; attempts > 0 { + log.Printf("retry func error: %s. attempts #%d after %s.", err.Error(), attempts, sleep) + time.Sleep(sleep) + return Retry(attempts, 2*sleep, fn) + } + return err + } + return nil +} + +type InterruptError struct { + error +} + +func NoRetryError(err error) InterruptError { + return InterruptError{err} +} diff --git a/internal/reader/meta_service.go b/internal/reader/meta_service.go index 2ab30a7edb..8f2d8863e0 100644 --- a/internal/reader/meta_service.go +++ b/internal/reader/meta_service.go @@ -12,9 +12,9 @@ import ( "github.com/golang/protobuf/proto" - "github.com/zilliztech/milvus-distributed/internal/conf" "github.com/zilliztech/milvus-distributed/internal/kv" "github.com/zilliztech/milvus-distributed/internal/proto/etcdpb" + gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil" "go.etcd.io/etcd/clientv3" "go.etcd.io/etcd/mvcc/mvccpb" ) @@ -31,10 +31,19 @@ type metaService struct { } func newMetaService(ctx context.Context, container *container) *metaService { - ETCDAddr := "http://" - ETCDAddr += conf.Config.Etcd.Address - ETCDPort := conf.Config.Etcd.Port - ETCDAddr = ETCDAddr + ":" + strconv.FormatInt(int64(ETCDPort), 10) + ETCDAddr, err := gparams.GParams.Load("etcd.address") + if err != nil { + panic(err) + } + ETCDPort, err := gparams.GParams.Load("etcd.port") + if err != nil { + panic(err) + } + ETCDAddr = "http://" + ETCDAddr + ":" + ETCDPort + ETCDRootPath, err := gparams.GParams.Load("etcd.rootpath") + if err != nil { + panic(err) + } cli, _ := clientv3.New(clientv3.Config{ Endpoints: []string{ETCDAddr}, @@ -43,7 +52,7 @@ func newMetaService(ctx context.Context, container *container) *metaService { return &metaService{ ctx: ctx, - kvBase: kv.NewEtcdKV(cli, conf.Config.Etcd.Rootpath), + kvBase: kv.NewEtcdKV(cli, ETCDRootPath), container: container, } } @@ -74,17 +83,29 @@ func (mService *metaService) start() { } func GetCollectionObjID(key string) string { - prefix := path.Join(conf.Config.Etcd.Rootpath, CollectionPrefix) + "/" + ETCDRootPath, err := gparams.GParams.Load("etcd.rootpath") + if err != nil { + panic(err) + } + prefix := path.Join(ETCDRootPath, CollectionPrefix) + "/" return strings.TrimPrefix(key, prefix) } func GetSegmentObjID(key string) string { - prefix := path.Join(conf.Config.Etcd.Rootpath, SegmentPrefix) + "/" + ETCDRootPath, err := gparams.GParams.Load("etcd.rootpath") + if err != nil { + panic(err) + } + prefix := path.Join(ETCDRootPath, SegmentPrefix) + "/" return strings.TrimPrefix(key, prefix) } func isCollectionObj(key string) bool { - prefix := path.Join(conf.Config.Etcd.Rootpath, CollectionPrefix) + "/" + ETCDRootPath, err := gparams.GParams.Load("etcd.rootpath") + if err != nil { + panic(err) + } + prefix := path.Join(ETCDRootPath, CollectionPrefix) + "/" prefix = strings.TrimSpace(prefix) index := strings.Index(key, prefix) @@ -92,7 +113,11 @@ func isCollectionObj(key string) bool { } func isSegmentObj(key string) bool { - prefix := path.Join(conf.Config.Etcd.Rootpath, SegmentPrefix) + "/" + ETCDRootPath, err := gparams.GParams.Load("etcd.rootpath") + if err != nil { + panic(err) + } + prefix := path.Join(ETCDRootPath, SegmentPrefix) + "/" prefix = strings.TrimSpace(prefix) index := strings.Index(key, prefix) @@ -105,8 +130,24 @@ func isSegmentChannelRangeInQueryNodeChannelRange(segment *etcdpb.SegmentMeta) b return false } - var queryNodeChannelStart = conf.Config.Reader.TopicStart - var queryNodeChannelEnd = conf.Config.Reader.TopicEnd + readerTopicStart, err := gparams.GParams.Load("reader.topicstart") + if err != nil { + panic(err) + } + TopicStart, err := strconv.Atoi(readerTopicStart) + if err != nil { + panic(err) + } + readerTopicEnd, err := gparams.GParams.Load("reader.topicend") + if err != nil { + panic(err) + } + TopicEnd, err := strconv.Atoi(readerTopicEnd) + if err != nil { + panic(err) + } + var queryNodeChannelStart = TopicStart + var queryNodeChannelEnd = TopicEnd if segment.ChannelStart >= int32(queryNodeChannelStart) && segment.ChannelEnd <= int32(queryNodeChannelEnd) { return true diff --git a/internal/storage/internal/S3/s3_engine.go b/internal/storage/internal/S3/s3_engine.go index aaf33d54c7..8d391a95fa 100644 --- a/internal/storage/internal/S3/s3_engine.go +++ b/internal/storage/internal/S3/s3_engine.go @@ -3,16 +3,17 @@ package s3driver import ( "bytes" "context" + "fmt" "io" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" - "github.com/zilliztech/milvus-distributed/internal/conf" . "github.com/zilliztech/milvus-distributed/internal/storage/type" + gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil" ) -var bucketName = conf.Config.Writer.Bucket +var bucketName string type S3Store struct { client *s3.S3 @@ -21,6 +22,16 @@ type S3Store struct { func NewS3Store(config aws.Config) (*S3Store, error) { sess := session.Must(session.NewSession(&config)) service := s3.New(sess) + err := gparams.GParams.LoadYaml("config.yaml") + if err != nil { + panic(err) + } + bucketName, err := gparams.GParams.Load("writer.bucket") + if err != nil { + panic(err) + } + fmt.Println(bucketName) + return &S3Store{ client: service, }, nil diff --git a/internal/storage/internal/minio/minio_store.go b/internal/storage/internal/minio/minio_store.go index be4265ff2f..ea8983be34 100644 --- a/internal/storage/internal/minio/minio_store.go +++ b/internal/storage/internal/minio/minio_store.go @@ -7,12 +7,15 @@ import ( "github.com/minio/minio-go/v7/pkg/credentials" "github.com/zilliztech/milvus-distributed/internal/storage/internal/minio/codec" storageType "github.com/zilliztech/milvus-distributed/internal/storage/type" + gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil" ) type MinioDriver struct { driver *minioStore } +var bucketName string + func NewMinioDriver(ctx context.Context) (*MinioDriver, error) { // to-do read conf var endPoint = "localhost:9000" @@ -20,6 +23,14 @@ func NewMinioDriver(ctx context.Context) (*MinioDriver, error) { var secretAccessKey = "testminio" var useSSL = false + err := gparams.GParams.LoadYaml("config.yaml") + if err != nil { + panic(err) + } + bucketName, err := gparams.GParams.Load("writer.bucket") + if err != nil { + panic(err) + } minioClient, err := minio.New(endPoint, &minio.Options{ Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), Secure: useSSL, diff --git a/internal/storage/internal/minio/minio_storeEngine.go b/internal/storage/internal/minio/minio_storeEngine.go index 117f73d352..64d74e8590 100644 --- a/internal/storage/internal/minio/minio_storeEngine.go +++ b/internal/storage/internal/minio/minio_storeEngine.go @@ -6,12 +6,9 @@ import ( "io" "github.com/minio/minio-go/v7" - "github.com/zilliztech/milvus-distributed/internal/conf" . "github.com/zilliztech/milvus-distributed/internal/storage/type" ) -var bucketName = conf.Config.Writer.Bucket - type minioStore struct { client *minio.Client } diff --git a/internal/storage/internal/tikv/tikv_store.go b/internal/storage/internal/tikv/tikv_store.go index 5d45e95ea2..87db3a5ce9 100644 --- a/internal/storage/internal/tikv/tikv_store.go +++ b/internal/storage/internal/tikv/tikv_store.go @@ -8,9 +8,9 @@ import ( "github.com/tikv/client-go/config" "github.com/tikv/client-go/rawkv" - "github.com/zilliztech/milvus-distributed/internal/conf" . "github.com/zilliztech/milvus-distributed/internal/storage/internal/tikv/codec" . "github.com/zilliztech/milvus-distributed/internal/storage/type" + gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil" ) func keyAddOne(key Key) Key { @@ -88,11 +88,21 @@ type TikvStore struct { } func NewTikvStore(ctx context.Context) (*TikvStore, error) { - conf.LoadConfig("config.yaml") - var pdAddress0 = conf.Config.Storage.Address + ":" + strconv.FormatInt(int64(conf.Config.Storage.Port), 10) - pdAddrs := []string{pdAddress0} + err := gparams.GParams.LoadYaml("config.yaml") + if err != nil { + panic(err) + } + pdAddress, err := gparams.GParams.Load("storage.address") + if err != nil { + panic(err) + } + pdPort, err := gparams.GParams.Load("storage.port") + if err != nil { + panic(err) + } + pdAddress = pdAddress + ":" + pdPort conf := config.Default() - client, err := rawkv.NewClient(ctx, pdAddrs, conf) + client, err := rawkv.NewClient(ctx, []string{pdAddress}, conf) if err != nil { return nil, err } diff --git a/internal/util/flowgraph/flow_graph_test.go b/internal/util/flowgraph/flow_graph_test.go index 9f170eb2bb..27f363ff86 100644 --- a/internal/util/flowgraph/flow_graph_test.go +++ b/internal/util/flowgraph/flow_graph_test.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "math" + "math/big" "math/rand" "sync" "testing" @@ -137,7 +138,8 @@ func sendMsgFromCmd(ctx context.Context, fg *TimeTickedFlowGraph) { return } // assert result - if res-(math.Pow(num, 2)+math.Sqrt(num)) > 0.000001 { + expect := math.Pow(num, 2) + math.Sqrt(num) + if big.NewFloat(res) != big.NewFloat(expect) { fmt.Println(res) fmt.Println(math.Pow(num, 2) + math.Sqrt(num)) panic("wrong answer") diff --git a/internal/kv/kv.go b/internal/util/kvutil/kv.go similarity index 96% rename from internal/kv/kv.go rename to internal/util/kvutil/kv.go index ac40309984..f35aa5b62e 100644 --- a/internal/kv/kv.go +++ b/internal/util/kvutil/kv.go @@ -1,4 +1,4 @@ -package kv +package kvutil type Base interface { Load(key string) (string, error) diff --git a/internal/kv/mem_kv.go b/internal/util/kvutil/mem_kv.go similarity index 99% rename from internal/kv/mem_kv.go rename to internal/util/kvutil/mem_kv.go index 0e03beb87f..60b35613be 100644 --- a/internal/kv/mem_kv.go +++ b/internal/util/kvutil/mem_kv.go @@ -1,4 +1,4 @@ -package kv +package kvutil import ( "sync" diff --git a/internal/util/paramtableutil/kv.go b/internal/util/paramtableutil/kv.go new file mode 100644 index 0000000000..8dfecf51b0 --- /dev/null +++ b/internal/util/paramtableutil/kv.go @@ -0,0 +1,20 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. + +package paramtableutil + +type Base interface { + Load(key string) (string, error) + LoadRange(key, endKey string, limit int) ([]string, []string, error) + LoadYaml(fileName string) error + Remove(key string) error + Save(key, value string) error +} diff --git a/internal/util/paramtableutil/paramtable.go b/internal/util/paramtableutil/paramtable.go new file mode 100644 index 0000000000..1152e0efe1 --- /dev/null +++ b/internal/util/paramtableutil/paramtable.go @@ -0,0 +1,66 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. + +package paramtableutil + +import ( + "path" + "runtime" + "strings" + + "github.com/spf13/viper" + "github.com/zilliztech/milvus-distributed/internal/util/kvutil" +) + +type GlobalParamsTable struct { + params *kvutil.MemoryKV +} + +func NewGlobalParamsTable() Base { + return &GlobalParamsTable{params: kvutil.NewMemoryKV()} +} + +var GParams = NewGlobalParamsTable() + +func (gparams *GlobalParamsTable) Load(key string) (string, error) { + return gparams.params.Load(strings.ToLower(key)) +} + +func (gparams *GlobalParamsTable) LoadRange(key, endKey string, limit int) ([]string, []string, error) { + return gparams.params.LoadRange(strings.ToLower(key), strings.ToLower(endKey), limit) +} + +func (gparams *GlobalParamsTable) LoadYaml(fileName string) error { + config := viper.New() + _, fpath, _, _ := runtime.Caller(0) + configPath := path.Dir(fpath) + "/../../../configs/" + config.SetConfigFile(configPath + fileName) + if err := config.ReadInConfig(); err != nil { + panic(err) + } + + for _, key := range config.AllKeys() { + err := gparams.params.Save(strings.ToLower(key), config.GetString(key)) + if err != nil { + panic(err) + } + } + + return nil +} + +func (gparams *GlobalParamsTable) Remove(key string) error { + return gparams.params.Remove(strings.ToLower(key)) +} + +func (gparams *GlobalParamsTable) Save(key, value string) error { + return gparams.params.Save(strings.ToLower(key), value) +} diff --git a/internal/util/paramtableutil/paramtable_test.go b/internal/util/paramtableutil/paramtable_test.go new file mode 100644 index 0000000000..ce5ac5d0ad --- /dev/null +++ b/internal/util/paramtableutil/paramtable_test.go @@ -0,0 +1,114 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. + +package paramtableutil + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +var param = NewGlobalParamsTable() + +func TestGlobalParamsTable_SaveAndLoad(t *testing.T) { + err1 := param.Save("int", "10") + assert.Nil(t, err1) + + err2 := param.Save("string", "testSaveAndLoad") + assert.Nil(t, err2) + + err3 := param.Save("float", "1.234") + assert.Nil(t, err3) + + r1, _ := param.Load("int") + assert.Equal(t, "10", r1) + + r2, _ := param.Load("string") + assert.Equal(t, "testSaveAndLoad", r2) + + r3, _ := param.Load("float") + assert.Equal(t, "1.234", r3) + + err4 := param.Remove("int") + assert.Nil(t, err4) + + err5 := param.Remove("string") + assert.Nil(t, err5) + + err6 := param.Remove("float") + assert.Nil(t, err6) +} + +func TestGlobalParamsTable_LoadRange(t *testing.T) { + _ = param.Save("abc", "10") + _ = param.Save("fghz", "20") + _ = param.Save("bcde", "1.1") + _ = param.Save("abcd", "testSaveAndLoad") + _ = param.Save("zhi", "12") + + keys, values, err := param.LoadRange("a", "g", 10) + assert.Nil(t, err) + assert.Equal(t, 4, len(keys)) + assert.Equal(t, "10", values[0]) + assert.Equal(t, "testSaveAndLoad", values[1]) + assert.Equal(t, "1.1", values[2]) + assert.Equal(t, "20", values[3]) + + _ = param.Remove("abc") + _ = param.Remove("fghz") + _ = param.Remove("bcde") + _ = param.Remove("abcd") + _ = param.Remove("zhi") +} + +func TestGlobalParamsTable_Remove(t *testing.T) { + err1 := param.Save("RemoveInt", "10") + assert.Nil(t, err1) + + err2 := param.Save("RemoveString", "testRemove") + assert.Nil(t, err2) + + err3 := param.Save("RemoveFloat", "1.234") + assert.Nil(t, err3) + + err4 := param.Remove("RemoveInt") + assert.Nil(t, err4) + + err5 := param.Remove("RemoveString") + assert.Nil(t, err5) + + err6 := param.Remove("RemoveFloat") + assert.Nil(t, err6) +} + +func TestGlobalParamsTable_LoadYaml(t *testing.T) { + err := param.LoadYaml("config.yaml") + assert.Nil(t, err) + + value1, err1 := param.Load("etcd.address") + value2, err2 := param.Load("pulsar.port") + value3, err3 := param.Load("reader.topicend") + value4, err4 := param.Load("proxy.pulsarTopics.readerTopicPrefix") + value5, err5 := param.Load("proxy.network.address") + + assert.Equal(t, value1, "localhost") + assert.Equal(t, value2, "6650") + assert.Equal(t, value3, "128") + assert.Equal(t, value4, "milvusReader") + assert.Equal(t, value5, "0.0.0.0") + + assert.Nil(t, err1) + assert.Nil(t, err2) + assert.Nil(t, err3) + assert.Nil(t, err4) + assert.Nil(t, err5) +} diff --git a/internal/util/tsoutil/tso.go b/internal/util/tsoutil/tso.go index c1e3b3491b..9e23dcdfe2 100644 --- a/internal/util/tsoutil/tso.go +++ b/internal/util/tsoutil/tso.go @@ -3,11 +3,10 @@ package tsoutil import ( "fmt" "path" - "strconv" "time" - "github.com/zilliztech/milvus-distributed/internal/conf" "github.com/zilliztech/milvus-distributed/internal/kv" + gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil" "go.etcd.io/etcd/clientv3" ) @@ -29,13 +28,24 @@ func ParseTS(ts uint64) (time.Time, uint64) { } func NewTSOKVBase(subPath string) *kv.EtcdKV { - etcdAddr := conf.Config.Etcd.Address - etcdAddr += ":" - etcdAddr += strconv.FormatInt(int64(conf.Config.Etcd.Port), 10) + etcdAddr, err := gparams.GParams.Load("etcd.address") + if err != nil { + panic(err) + } + etcdPort, err := gparams.GParams.Load("etcd.port") + if err != nil { + panic(err) + } + etcdAddr = etcdAddr + ":" + etcdPort fmt.Println("etcdAddr ::: ", etcdAddr) client, _ := clientv3.New(clientv3.Config{ Endpoints: []string{etcdAddr}, DialTimeout: 5 * time.Second, }) - return kv.NewEtcdKV(client, path.Join(conf.Config.Etcd.Rootpath, subPath)) + + etcdRootPath, err := gparams.GParams.Load("etcd.rootpath") + if err != nil { + panic(err) + } + return kv.NewEtcdKV(client, path.Join(etcdRootPath, subPath)) } diff --git a/tools/core_gen/all_generate.py b/tools/core_gen/all_generate.py index 3ce44022f0..cf40b73876 100755 --- a/tools/core_gen/all_generate.py +++ b/tools/core_gen/all_generate.py @@ -14,7 +14,7 @@ def gen_file(rootfile, template, output, **kwargs): def extract_extra_body(visitor_info, query_path): - pattern = re.compile(r"class(.*){\n((.|\n)*?)\n};", re.MULTILINE) + pattern = re.compile("class(.*){\n((.|\n)*?)\n};", re.MULTILINE) for node, visitors in visitor_info.items(): for visitor in visitors: @@ -22,24 +22,11 @@ def extract_extra_body(visitor_info, query_path): vis_file = query_path + "visitors/" + vis_name + ".cpp" body = ' public:' - inc_pattern_str = r'^(#include(.|\n)*)\n#include "query/generated/{}.h"'.format(vis_name) - inc_pattern = re.compile(inc_pattern_str, re.MULTILINE) - if os.path.exists(vis_file): - content = readfile(vis_file) - infos = pattern.findall(content) - assert len(infos) <= 1 + infos = pattern.findall(readfile(vis_file)) if len(infos) == 1: name, body, _ = infos[0] - - extra_inc_infos = inc_pattern.findall(content) - assert(len(extra_inc_infos) <= 1) - print(extra_inc_infos) - if len(extra_inc_infos) == 1: - extra_inc_body, _ = extra_inc_infos[0] - visitor["ctor_and_member"] = body - visitor["extra_inc"] = extra_inc_body if __name__ == "__main__": query_path = "../../internal/core/src/query/" diff --git a/tools/core_gen/templates/visitor_derived.h b/tools/core_gen/templates/visitor_derived.h index 49d31fa44c..ed0c92315a 100644 --- a/tools/core_gen/templates/visitor_derived.h +++ b/tools/core_gen/templates/visitor_derived.h @@ -9,9 +9,7 @@ #pragma once // Generated File // DO NOT EDIT -@@extra_inc@@ #include "@@base_visitor@@.h" - namespace @@namespace@@ { class @@visitor_name@@ : @@base_visitor@@ { public: