mirror of
https://gitee.com/milvus-io/milvus.git
synced 2024-11-29 18:38:44 +08:00
Initialize the task scheduler of Proxy
Signed-off-by: dragondriver <jiquan.long@zilliz.com>
This commit is contained in:
dragondriver
yefu.chen
parent
f47fc7fef1
commit
c2382adf04
@ -9,8 +9,8 @@ import (
|
||||
"strconv"
|
||||
"syscall"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
"github.com/zilliztech/milvus-distributed/internal/master"
|
||||
gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
@ -21,15 +21,18 @@ func main() {
|
||||
flag.Parse()
|
||||
// flag.Usage()
|
||||
log.Println("yaml file: ", yamlFile)
|
||||
conf.LoadConfig(yamlFile)
|
||||
|
||||
err := gparams.GParams.LoadYaml(yamlFile)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// Creates server.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
etcdAddr := conf.Config.Etcd.Address
|
||||
etcdAddr += ":"
|
||||
etcdAddr += strconv.FormatInt(int64(conf.Config.Etcd.Port), 10)
|
||||
|
||||
svr, err := master.CreateServer(ctx, conf.Config.Etcd.Rootpath, conf.Config.Etcd.Rootpath, conf.Config.Etcd.Rootpath, []string{etcdAddr})
|
||||
etcdAddress, _ := gparams.GParams.Load("etcd.address")
|
||||
etcdPort, _ := gparams.GParams.Load("etcd.port")
|
||||
etcdAddr := etcdAddress + ":" + etcdPort
|
||||
etcdRootPath, _ := gparams.GParams.Load("etcd.rootpath")
|
||||
svr, err := master.CreateServer(ctx, etcdRootPath, etcdRootPath, etcdRootPath, []string{etcdAddr})
|
||||
if err != nil {
|
||||
log.Print("create server failed", zap.Error(err))
|
||||
}
|
||||
@ -47,7 +50,11 @@ func main() {
|
||||
cancel()
|
||||
}()
|
||||
|
||||
grpcPort := int64(conf.Config.Master.Port)
|
||||
masterPort, _ := gparams.GParams.Load("master.port")
|
||||
grpcPort, err := strconv.ParseInt(masterPort, 10, 64)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
if err := svr.Run(grpcPort); err != nil {
|
||||
log.Fatal("run server failed", zap.Error(err))
|
||||
|
||||
@ -8,10 +8,9 @@ import (
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proxy"
|
||||
gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@ -20,7 +19,11 @@ func main() {
|
||||
flag.Parse()
|
||||
flag.Usage()
|
||||
log.Println("yaml file: ", yamlFile)
|
||||
conf.LoadConfig(yamlFile)
|
||||
|
||||
err := gparams.GParams.LoadYaml(yamlFile)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// Creates server.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
@ -6,11 +6,10 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"syscall"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
"github.com/zilliztech/milvus-distributed/internal/reader"
|
||||
gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@ -22,7 +21,11 @@ func main() {
|
||||
flag.Parse()
|
||||
// flag.Usage()
|
||||
fmt.Println("yaml file: ", yamlFile)
|
||||
conf.LoadConfig(yamlFile)
|
||||
|
||||
err := gparams.GParams.LoadYaml(yamlFile)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
sc := make(chan os.Signal, 1)
|
||||
signal.Notify(sc,
|
||||
@ -36,11 +39,9 @@ func main() {
|
||||
sig = <-sc
|
||||
cancel()
|
||||
}()
|
||||
|
||||
pulsarAddr := "pulsar://"
|
||||
pulsarAddr += conf.Config.Pulsar.Address
|
||||
pulsarAddr += ":"
|
||||
pulsarAddr += strconv.FormatInt(int64(conf.Config.Pulsar.Port), 10)
|
||||
pulsarAddr, _ := gparams.GParams.Load("pulsar.address")
|
||||
pulsarPort, _ := gparams.GParams.Load("pulsar.port")
|
||||
pulsarAddr += ":" + pulsarPort
|
||||
reader.StartQueryNode(ctx, pulsarAddr)
|
||||
|
||||
switch sig {
|
||||
|
||||
1
go.mod
1
go.mod
@ -31,6 +31,7 @@ require (
|
||||
github.com/sirupsen/logrus v1.6.0 // indirect
|
||||
github.com/smartystreets/goconvey v1.6.4 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0
|
||||
github.com/spf13/viper v1.7.1
|
||||
github.com/stretchr/testify v1.6.1
|
||||
github.com/tikv/client-go v0.0.0-20200824032810-95774393107b
|
||||
github.com/yahoo/athenz v1.9.16 // indirect
|
||||
|
||||
151
go.sum
151
go.sum
@ -1,7 +1,21 @@
|
||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
|
||||
cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
|
||||
cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
|
||||
cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
|
||||
cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
|
||||
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
|
||||
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
|
||||
cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
|
||||
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
|
||||
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
|
||||
code.cloudfoundry.org/bytefmt v0.0.0-20200131002437-cf55d5288a48 h1:/EMHruHCFXR9xClkGV/t0rmHrdhX4+trQUcBqjwc9xE=
|
||||
code.cloudfoundry.org/bytefmt v0.0.0-20200131002437-cf55d5288a48/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
@ -11,6 +25,9 @@ github.com/apache/pulsar-client-go v0.1.1/go.mod h1:mlxC65KL1BLhGO2bnT9zWMttVzR2
|
||||
github.com/ardielle/ardielle-go v1.5.2 h1:TilHTpHIQJ27R1Tl/iITBzMwiUGSlVfiVhwDNGM3Zj4=
|
||||
github.com/ardielle/ardielle-go v1.5.2/go.mod h1:I4hy1n795cUhaVt/ojz83SNVCYIGsAFAONtv2Dr7HUI=
|
||||
github.com/ardielle/ardielle-tools v1.5.4/go.mod h1:oZN+JRMnqGiIhrzkRN9l26Cej9dEx4jeNG6A+AdkShk=
|
||||
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
|
||||
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
|
||||
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
|
||||
github.com/aws/aws-sdk-go v1.30.8 h1:4BHbh8K3qKmcnAgToZ2LShldRF9inoqIBccpCLNCy3I=
|
||||
github.com/aws/aws-sdk-go v1.30.8/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
|
||||
github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6/go.mod h1:6YNgTHLutezwnBvyneBbwvB8C82y3dcoOj5EQJIdGXA=
|
||||
@ -20,9 +37,12 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
|
||||
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
|
||||
github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q=
|
||||
github.com/boynton/repl v0.0.0-20170116235056-348863958e3e/go.mod h1:Crc/GCZ3NXDVCio7Yr0o+SSrytpcFhLmVCIzi0s49t4=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y=
|
||||
@ -31,9 +51,13 @@ github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s=
|
||||
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
|
||||
github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY=
|
||||
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
|
||||
github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
|
||||
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
|
||||
github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
||||
github.com/coreos/go-systemd v0.0.0-20190212144455-93d5ec2c7f76 h1:FE783w8WFh+Rvg+7bZ5g8p7gP4SeVS4AoNwkvazlsBg=
|
||||
github.com/coreos/go-systemd v0.0.0-20190212144455-93d5ec2c7f76/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
||||
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8=
|
||||
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
||||
github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
|
||||
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg=
|
||||
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
|
||||
@ -45,6 +69,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
|
||||
github.com/dimfeld/httptreemux v5.0.1+incompatible h1:Qj3gVcDNoOthBAqftuD596rm4wg/adLLz5xh5CmpiCA=
|
||||
github.com/dimfeld/httptreemux v5.0.1+incompatible/go.mod h1:rbUlSV+CCpv/SuqUTP/8Bk2O3LyUV436/yaRGkhP6Z0=
|
||||
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
||||
@ -61,6 +86,7 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
|
||||
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
@ -78,6 +104,8 @@ github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4er
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
|
||||
github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
@ -89,15 +117,21 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z
|
||||
github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
|
||||
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
|
||||
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
|
||||
@ -107,14 +141,38 @@ github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z
|
||||
github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
|
||||
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
|
||||
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg=
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
|
||||
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
|
||||
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.8.1/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
|
||||
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
|
||||
github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
|
||||
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
|
||||
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
|
||||
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
|
||||
github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
|
||||
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
|
||||
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
|
||||
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
|
||||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
||||
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
|
||||
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
|
||||
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
|
||||
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/jawher/mow.cli v1.0.4/go.mod h1:5hQj2V8g+qYmLUVWqu4Wuja1pI57M83EChYLVZ0sMKk=
|
||||
@ -129,6 +187,7 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u
|
||||
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
|
||||
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||
@ -152,19 +211,31 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
|
||||
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
|
||||
github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
|
||||
github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
|
||||
github.com/minio/minio-go/v7 v7.0.5 h1:I2NIJ2ojwJqD/YByemC1M59e1b4FW9kS7NlOar7HPV4=
|
||||
github.com/minio/minio-go/v7 v7.0.5/go.mod h1:TA0CQCjJZHM5SJj9IjqR0NmpmQJ6bCbXifAJ3mUU6Hw=
|
||||
github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
|
||||
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
|
||||
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
|
||||
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
|
||||
github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
|
||||
github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
|
||||
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
@ -178,6 +249,7 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWb
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
|
||||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
||||
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
|
||||
github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
@ -192,6 +264,9 @@ github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+
|
||||
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
|
||||
github.com/ozonru/etcd v3.3.20-grpc1.27-origmodule+incompatible h1:CAG0PUvo1fen+ZEfxKJjFIc8GuuN5RuaBuCAuaP2Hno=
|
||||
github.com/ozonru/etcd v3.3.20-grpc1.27-origmodule+incompatible/go.mod h1:iIubILNIN6Jq9h8uiSLrN9L1tuj3iSSFwz3R61skm/A=
|
||||
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
|
||||
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
|
||||
github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI=
|
||||
github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
|
||||
@ -221,8 +296,10 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
|
||||
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
|
||||
github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM=
|
||||
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
|
||||
github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA=
|
||||
@ -233,7 +310,9 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCb
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
|
||||
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||
github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw=
|
||||
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||
github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
|
||||
@ -241,17 +320,21 @@ github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lN
|
||||
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||
github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs=
|
||||
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
|
||||
github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8=
|
||||
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
|
||||
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
|
||||
github.com/protocolbuffers/protobuf v3.13.0+incompatible h1:omZA3Tuq+U2kJ2uMuqMR9c1VO5qLEgZ19m9878fXNtg=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
|
||||
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
|
||||
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
|
||||
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
|
||||
@ -268,10 +351,18 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
|
||||
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
|
||||
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
|
||||
github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
|
||||
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
|
||||
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
|
||||
github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
|
||||
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
|
||||
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
|
||||
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk=
|
||||
github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
|
||||
@ -282,6 +373,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
|
||||
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
|
||||
github.com/tikv/client-go v0.0.0-20200824032810-95774393107b h1:VOG2GkM7RpRrT0St7HIIwCWrc3mVdf+DjcT8r2ucusI=
|
||||
github.com/tikv/client-go v0.0.0-20200824032810-95774393107b/go.mod h1:K0NcdVNrXDq92YPLytsrAwRMyuXi7GZCO6dXNH7OzQc=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
@ -304,7 +397,10 @@ go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
|
||||
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
|
||||
go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 h1:lWF4f9Nypl1ZqSb4gLeh/DGvBYVaUYHuiB93teOmwgc=
|
||||
go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
|
||||
go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk=
|
||||
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
|
||||
@ -321,8 +417,10 @@ go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
|
||||
go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM=
|
||||
go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
@ -330,23 +428,40 @@ golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPh
|
||||
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM=
|
||||
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
|
||||
golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k=
|
||||
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
|
||||
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
|
||||
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
|
||||
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
@ -357,19 +472,29 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R
|
||||
golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA=
|
||||
golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
@ -383,23 +508,35 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f h1:Fqb3ao1hUmOR3GkUOg/Y+BadLwykBIzs5q8Ez2SbHyc=
|
||||
golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190808195139-e713427fea3f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191107010934-f79515f33823/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
@ -410,10 +547,20 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
|
||||
google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150 h1:VPpdpQkGvFicX9yo4G5oxZPi9ALBnEOZblPSa/Wa2m4=
|
||||
google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
|
||||
google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg=
|
||||
@ -427,6 +574,7 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8
|
||||
gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
|
||||
gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
|
||||
@ -447,9 +595,12 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
|
||||
honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8=
|
||||
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
|
||||
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
||||
sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
|
||||
sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
|
||||
sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
|
||||
|
||||
@ -4,8 +4,11 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
|
||||
)
|
||||
@ -20,11 +23,17 @@ type IDAllocator struct {
|
||||
}
|
||||
|
||||
func NewIDAllocator(ctx context.Context) (*IDAllocator, error) {
|
||||
masterAddr := conf.Config.Etcd.Address
|
||||
masterAddr += ":"
|
||||
masterAddr += strconv.FormatInt(int64(conf.Config.Master.Port), 10)
|
||||
|
||||
ctx1, cancel := context.WithCancel(ctx)
|
||||
a := &IDAllocator{
|
||||
Allocator: Allocator{reqs: make(chan request, maxMergeRequests),
|
||||
ctx: ctx1,
|
||||
cancel: cancel,
|
||||
ctx: ctx1,
|
||||
cancel: cancel,
|
||||
masterAddress: masterAddr,
|
||||
countPerRPC: maxMergeRequests,
|
||||
},
|
||||
}
|
||||
a.tChan = &emptyTicker{}
|
||||
|
||||
@ -4,8 +4,11 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
|
||||
@ -25,11 +28,18 @@ type TimestampAllocator struct {
|
||||
}
|
||||
|
||||
func NewTimestampAllocator(ctx context.Context) (*TimestampAllocator, error) {
|
||||
|
||||
masterAddr := conf.Config.Etcd.Address
|
||||
masterAddr += ":"
|
||||
masterAddr += strconv.FormatInt(int64(conf.Config.Master.Port), 10)
|
||||
|
||||
ctx1, cancel := context.WithCancel(ctx)
|
||||
a := &TimestampAllocator{
|
||||
Allocator: Allocator{reqs: make(chan request, maxMergeRequests),
|
||||
ctx: ctx1,
|
||||
cancel: cancel,
|
||||
ctx: ctx1,
|
||||
cancel: cancel,
|
||||
masterAddress: masterAddr,
|
||||
countPerRPC: maxMergeRequests,
|
||||
},
|
||||
}
|
||||
a.tChan = &ticker{
|
||||
|
||||
@ -21,7 +21,6 @@ ENDFOREACH(proto_file)
|
||||
add_library(milvus_proto STATIC
|
||||
${MILVUS_PROTO_SRCS}
|
||||
)
|
||||
message(${MILVUS_PROTO_SRCS})
|
||||
|
||||
target_link_libraries(milvus_proto
|
||||
libprotobuf
|
||||
|
||||
@ -5,9 +5,8 @@ set(MILVUS_QUERY_SRCS
|
||||
generated/Expr.cpp
|
||||
visitors/ShowPlanNodeVisitor.cpp
|
||||
visitors/ExecPlanNodeVisitor.cpp
|
||||
visitors/ShowExprVisitor.cpp
|
||||
Parser.cpp
|
||||
Plan.cpp
|
||||
)
|
||||
add_library(milvus_query ${MILVUS_QUERY_SRCS})
|
||||
target_link_libraries(milvus_query milvus_proto)
|
||||
target_link_libraries(milvus_query libprotobuf)
|
||||
|
||||
@ -4,8 +4,6 @@
|
||||
#include <any>
|
||||
#include <string>
|
||||
#include <optional>
|
||||
#include "segcore/SegmentDefs.h"
|
||||
|
||||
namespace milvus::query {
|
||||
class ExprVisitor;
|
||||
|
||||
@ -60,13 +58,7 @@ using FieldId = std::string;
|
||||
|
||||
struct TermExpr : Expr {
|
||||
FieldId field_id_;
|
||||
segcore::DataType data_type_;
|
||||
// std::vector<std::any> terms_;
|
||||
|
||||
protected:
|
||||
// prevent accidential instantiation
|
||||
TermExpr() = default;
|
||||
|
||||
std::vector<std::any> terms_; //
|
||||
public:
|
||||
void
|
||||
accept(ExprVisitor&) override;
|
||||
@ -74,14 +66,12 @@ struct TermExpr : Expr {
|
||||
|
||||
struct RangeExpr : Expr {
|
||||
FieldId field_id_;
|
||||
segcore::DataType data_type_;
|
||||
// std::vector<std::tuple<OpType, std::any>> conditions_;
|
||||
protected:
|
||||
// prevent accidential instantiation
|
||||
RangeExpr() = default;
|
||||
enum class OpType { LessThan, LessEqual, GreaterThan, GreaterEqual, Equal, NotEqual };
|
||||
std::vector<std::tuple<OpType, std::any>> conditions_;
|
||||
|
||||
public:
|
||||
void
|
||||
accept(ExprVisitor&) override;
|
||||
};
|
||||
|
||||
} // namespace milvus::query
|
||||
|
||||
@ -1,16 +0,0 @@
|
||||
#pragma once
|
||||
#include "Expr.h"
|
||||
|
||||
namespace milvus::query {
|
||||
template <typename T>
|
||||
struct TermExprImpl : TermExpr {
|
||||
std::vector<T> terms_;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct RangeExprImpl : RangeExpr {
|
||||
enum class OpType { LessThan, LessEqual, GreaterThan, GreaterEqual, Equal, NotEqual };
|
||||
std::vector<std::tuple<OpType, T>> conditions_;
|
||||
};
|
||||
|
||||
} // namespace milvus::query
|
||||
@ -23,7 +23,7 @@ CreateVec(const std::string& field_name, const json& vec_info) {
|
||||
|
||||
static std::unique_ptr<Plan>
|
||||
CreatePlanImplNaive(const std::string& dsl_str) {
|
||||
auto plan = std::make_unique<Plan>();
|
||||
auto plan = std::unique_ptr<Plan>();
|
||||
auto dsl = nlohmann::json::parse(dsl_str);
|
||||
nlohmann::json vec_pack;
|
||||
|
||||
@ -36,19 +36,17 @@ CreatePlanImplNaive(const std::string& dsl_str) {
|
||||
auto key = iter.key();
|
||||
auto& body = iter.value();
|
||||
plan->plan_node_ = CreateVec(key, body);
|
||||
return plan;
|
||||
}
|
||||
}
|
||||
PanicInfo("Unsupported DSL: vector node not detected");
|
||||
} else if (bool_dsl.contains("vector")) {
|
||||
auto iter = bool_dsl["vector"].begin();
|
||||
auto key = iter.key();
|
||||
auto& body = iter.value();
|
||||
plan->plan_node_ = CreateVec(key, body);
|
||||
return plan;
|
||||
} else {
|
||||
PanicInfo("Unsupported DSL: vector node not detected");
|
||||
}
|
||||
return plan;
|
||||
}
|
||||
|
||||
void
|
||||
@ -57,7 +55,6 @@ CheckNull(const Json& json) {
|
||||
}
|
||||
|
||||
class PlanParser {
|
||||
public:
|
||||
void
|
||||
ParseBoolBody(const Json& dsl) {
|
||||
CheckNull(dsl);
|
||||
@ -77,8 +74,6 @@ class PlanParser {
|
||||
}
|
||||
PanicInfo("unimplemented");
|
||||
}
|
||||
|
||||
private:
|
||||
};
|
||||
|
||||
std::unique_ptr<Plan>
|
||||
@ -88,12 +83,11 @@ CreatePlan(const std::string& dsl_str) {
|
||||
}
|
||||
|
||||
std::unique_ptr<PlaceholderGroup>
|
||||
ParsePlaceholderGroup(const std::string& blob) {
|
||||
ParsePlaceholderGroup(const char* placeholder_group_blob) {
|
||||
namespace ser = milvus::proto::service;
|
||||
auto result = std::make_unique<PlaceholderGroup>();
|
||||
auto result = std::unique_ptr<PlaceholderGroup>();
|
||||
ser::PlaceholderGroup ph_group;
|
||||
auto ok = ph_group.ParseFromString(blob);
|
||||
Assert(ok);
|
||||
GOOGLE_PROTOBUF_PARSER_ASSERT(ph_group.ParseFromString(placeholder_group_blob));
|
||||
for (auto& info : ph_group.placeholders()) {
|
||||
Placeholder element;
|
||||
element.tag_ = info.tag();
|
||||
|
||||
@ -13,7 +13,7 @@ std::unique_ptr<Plan>
|
||||
CreatePlan(const std::string& dsl);
|
||||
|
||||
std::unique_ptr<PlaceholderGroup>
|
||||
ParsePlaceholderGroup(const std::string& placeholder_group_blob);
|
||||
ParsePlaceholderGroup(const char* placeholder_group_blob);
|
||||
|
||||
int64_t
|
||||
GetNumOfQueries(const PlaceholderGroup*);
|
||||
@ -24,5 +24,3 @@ int64_t
|
||||
GetTopK(const Plan*);
|
||||
|
||||
} // namespace milvus::query
|
||||
|
||||
#include "PlanImpl.h"
|
||||
@ -28,6 +28,7 @@ struct PlanNode {
|
||||
using PlanNodePtr = std::unique_ptr<PlanNode>;
|
||||
|
||||
struct QueryInfo {
|
||||
int64_t num_queries_;
|
||||
int64_t topK_;
|
||||
FieldId field_id_;
|
||||
std::string metric_type_; // TODO: use enum
|
||||
|
||||
@ -1,11 +1,7 @@
|
||||
#pragma once
|
||||
// Generated File
|
||||
// DO NOT EDIT
|
||||
#include "utils/Json.h"
|
||||
#include "query/PlanImpl.h"
|
||||
#include "segcore/SegmentBase.h"
|
||||
#include "PlanNodeVisitor.h"
|
||||
|
||||
namespace milvus::query {
|
||||
class ExecPlanNodeVisitor : PlanNodeVisitor {
|
||||
public:
|
||||
|
||||
@ -1,11 +1,7 @@
|
||||
#pragma once
|
||||
// Generated File
|
||||
// DO NOT EDIT
|
||||
#include "query/Plan.h"
|
||||
#include "utils/EasyAssert.h"
|
||||
#include "utils/Json.h"
|
||||
#include "ExprVisitor.h"
|
||||
|
||||
namespace milvus::query {
|
||||
class ShowExprVisitor : ExprVisitor {
|
||||
public:
|
||||
@ -22,35 +18,5 @@ class ShowExprVisitor : ExprVisitor {
|
||||
visit(RangeExpr& expr) override;
|
||||
|
||||
public:
|
||||
using RetType = Json;
|
||||
|
||||
public:
|
||||
RetType
|
||||
call_child(Expr& expr) {
|
||||
assert(!ret_.has_value());
|
||||
expr.accept(*this);
|
||||
assert(ret_.has_value());
|
||||
auto ret = std::move(ret_);
|
||||
ret_ = std::nullopt;
|
||||
return std::move(ret.value());
|
||||
}
|
||||
|
||||
Json
|
||||
combine(Json&& extra, UnaryExpr& expr) {
|
||||
auto result = std::move(extra);
|
||||
result["child"] = call_child(*expr.child_);
|
||||
return result;
|
||||
}
|
||||
|
||||
Json
|
||||
combine(Json&& extra, BinaryExpr& expr) {
|
||||
auto result = std::move(extra);
|
||||
result["left_child"] = call_child(*expr.left_);
|
||||
result["right_child"] = call_child(*expr.right_);
|
||||
return result;
|
||||
}
|
||||
|
||||
private:
|
||||
std::optional<RetType> ret_;
|
||||
};
|
||||
} // namespace milvus::query
|
||||
|
||||
@ -1,12 +1,7 @@
|
||||
#pragma once
|
||||
// Generated File
|
||||
// DO NOT EDIT
|
||||
#include "utils/EasyAssert.h"
|
||||
#include "utils/Json.h"
|
||||
#include <optional>
|
||||
|
||||
#include "PlanNodeVisitor.h"
|
||||
|
||||
namespace milvus::query {
|
||||
class ShowPlanNodeVisitor : PlanNodeVisitor {
|
||||
public:
|
||||
@ -26,7 +21,6 @@ class ShowPlanNodeVisitor : PlanNodeVisitor {
|
||||
node.accept(*this);
|
||||
assert(ret_.has_value());
|
||||
auto ret = std::move(ret_);
|
||||
ret_ = std::nullopt;
|
||||
return std::move(ret.value());
|
||||
}
|
||||
|
||||
|
||||
@ -48,10 +48,8 @@ ExecPlanNodeVisitor::visit(FloatVectorANNS& node) {
|
||||
auto segment = dynamic_cast<segcore::SegmentSmallIndex*>(&segment_);
|
||||
AssertInfo(segment, "support SegmentSmallIndex Only");
|
||||
RetType ret;
|
||||
auto& ph = placeholder_group_.at(0);
|
||||
auto src_data = ph.get_blob<float>();
|
||||
auto num_queries = ph.num_of_queries_;
|
||||
segment->QueryBruteForceImpl(node.query_info_, src_data, num_queries, timestamp_, ret);
|
||||
auto src_data = placeholder_group_.at(0).get_blob<float>();
|
||||
segment->QueryBruteForceImpl(node.query_info_, src_data, timestamp_, ret);
|
||||
ret_ = ret;
|
||||
}
|
||||
|
||||
|
||||
@ -1,173 +0,0 @@
|
||||
#include "query/Plan.h"
|
||||
#include "utils/EasyAssert.h"
|
||||
#include "utils/Json.h"
|
||||
#include "query/generated/ShowExprVisitor.h"
|
||||
#include "query/ExprImpl.h"
|
||||
|
||||
namespace milvus::query {
|
||||
using Json = nlohmann::json;
|
||||
|
||||
#if 1
|
||||
// THIS CONTAINS EXTRA BODY FOR VISITOR
|
||||
// WILL BE USED BY GENERATOR
|
||||
namespace impl {
|
||||
class ShowExprNodeVisitor : ExprVisitor {
|
||||
public:
|
||||
using RetType = Json;
|
||||
|
||||
public:
|
||||
RetType
|
||||
call_child(Expr& expr) {
|
||||
assert(!ret_.has_value());
|
||||
expr.accept(*this);
|
||||
assert(ret_.has_value());
|
||||
auto ret = std::move(ret_);
|
||||
ret_ = std::nullopt;
|
||||
return std::move(ret.value());
|
||||
}
|
||||
|
||||
Json
|
||||
combine(Json&& extra, UnaryExpr& expr) {
|
||||
auto result = std::move(extra);
|
||||
result["child"] = call_child(*expr.child_);
|
||||
return result;
|
||||
}
|
||||
|
||||
Json
|
||||
combine(Json&& extra, BinaryExpr& expr) {
|
||||
auto result = std::move(extra);
|
||||
result["left_child"] = call_child(*expr.left_);
|
||||
result["right_child"] = call_child(*expr.right_);
|
||||
return result;
|
||||
}
|
||||
|
||||
private:
|
||||
std::optional<RetType> ret_;
|
||||
};
|
||||
} // namespace impl
|
||||
#endif
|
||||
|
||||
void
|
||||
ShowExprVisitor::visit(BoolUnaryExpr& expr) {
|
||||
Assert(!ret_.has_value());
|
||||
using OpType = BoolUnaryExpr::OpType;
|
||||
|
||||
// TODO: use magic_enum if available
|
||||
Assert(expr.op_type_ == OpType::LogicalNot);
|
||||
auto op_name = "LogicalNot";
|
||||
|
||||
Json extra{
|
||||
{"expr_type", "BoolUnary"},
|
||||
{"op", op_name},
|
||||
};
|
||||
ret_ = this->combine(std::move(extra), expr);
|
||||
}
|
||||
|
||||
void
|
||||
ShowExprVisitor::visit(BoolBinaryExpr& expr) {
|
||||
Assert(!ret_.has_value());
|
||||
using OpType = BoolBinaryExpr::OpType;
|
||||
|
||||
// TODO: use magic_enum if available
|
||||
auto op_name = [](OpType op) {
|
||||
switch (op) {
|
||||
case OpType::LogicalAnd:
|
||||
return "LogicalAnd";
|
||||
case OpType::LogicalOr:
|
||||
return "LogicalOr";
|
||||
case OpType::LogicalXor:
|
||||
return "LogicalXor";
|
||||
default:
|
||||
PanicInfo("unsupported op");
|
||||
}
|
||||
}(expr.op_type_);
|
||||
|
||||
Json extra{
|
||||
{"expr_type", "BoolBinary"},
|
||||
{"op", op_name},
|
||||
};
|
||||
ret_ = this->combine(std::move(extra), expr);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static Json
|
||||
TermExtract(const TermExpr& expr_raw) {
|
||||
auto expr = dynamic_cast<const TermExprImpl<T>*>(&expr_raw);
|
||||
Assert(expr);
|
||||
return Json{expr->terms_};
|
||||
}
|
||||
|
||||
void
|
||||
ShowExprVisitor::visit(TermExpr& expr) {
|
||||
Assert(!ret_.has_value());
|
||||
Assert(segcore::field_is_vector(expr.data_type_) == false);
|
||||
using segcore::DataType;
|
||||
auto terms = [&] {
|
||||
switch (expr.data_type_) {
|
||||
case DataType::INT8:
|
||||
return TermExtract<int8_t>(expr);
|
||||
case DataType::INT16:
|
||||
return TermExtract<int16_t>(expr);
|
||||
case DataType::INT32:
|
||||
return TermExtract<int32_t>(expr);
|
||||
case DataType::INT64:
|
||||
return TermExtract<int64_t>(expr);
|
||||
case DataType::DOUBLE:
|
||||
return TermExtract<double>(expr);
|
||||
case DataType::FLOAT:
|
||||
return TermExtract<float>(expr);
|
||||
case DataType::BOOL:
|
||||
return TermExtract<bool>(expr);
|
||||
default:
|
||||
PanicInfo("unsupported type");
|
||||
}
|
||||
}();
|
||||
|
||||
Json res{{"expr_type", "Term"},
|
||||
{"field_id", expr.field_id_},
|
||||
{"data_type", segcore::datatype_name(expr.data_type_)},
|
||||
{"terms", std::move(terms)}};
|
||||
|
||||
ret_ = res;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static Json
|
||||
CondtionExtract(const RangeExpr& expr_raw) {
|
||||
auto expr = dynamic_cast<const TermExprImpl<T>*>(&expr_raw);
|
||||
Assert(expr);
|
||||
return Json{expr->terms_};
|
||||
}
|
||||
|
||||
void
|
||||
ShowExprVisitor::visit(RangeExpr& expr) {
|
||||
Assert(!ret_.has_value());
|
||||
Assert(segcore::field_is_vector(expr.data_type_) == false);
|
||||
using segcore::DataType;
|
||||
auto conditions = [&] {
|
||||
switch (expr.data_type_) {
|
||||
case DataType::BOOL:
|
||||
return CondtionExtract<bool>(expr);
|
||||
case DataType::INT8:
|
||||
return CondtionExtract<int8_t>(expr);
|
||||
case DataType::INT16:
|
||||
return CondtionExtract<int16_t>(expr);
|
||||
case DataType::INT32:
|
||||
return CondtionExtract<int32_t>(expr);
|
||||
case DataType::INT64:
|
||||
return CondtionExtract<int64_t>(expr);
|
||||
case DataType::DOUBLE:
|
||||
return CondtionExtract<double>(expr);
|
||||
case DataType::FLOAT:
|
||||
return CondtionExtract<float>(expr);
|
||||
default:
|
||||
PanicInfo("unsupported type");
|
||||
}
|
||||
}();
|
||||
|
||||
Json res{{"expr_type", "Range"},
|
||||
{"field_id", expr.field_id_},
|
||||
{"data_type", segcore::datatype_name(expr.data_type_)},
|
||||
{"conditions", std::move(conditions)}};
|
||||
}
|
||||
} // namespace milvus::query
|
||||
@ -19,7 +19,6 @@ class ShowPlanNodeVisitorImpl : PlanNodeVisitor {
|
||||
node.accept(*this);
|
||||
assert(ret_.has_value());
|
||||
auto ret = std::move(ret_);
|
||||
ret_ = std::nullopt;
|
||||
return std::move(ret.value());
|
||||
}
|
||||
|
||||
@ -41,9 +40,11 @@ ShowPlanNodeVisitor::visit(FloatVectorANNS& node) {
|
||||
assert(!ret_);
|
||||
auto& info = node.query_info_;
|
||||
Json json_body{
|
||||
{"node_type", "FloatVectorANNS"}, //
|
||||
{"metric_type", info.metric_type_}, //
|
||||
{"node_type", "FloatVectorANNS"}, //
|
||||
{"metric_type", info.metric_type_}, //
|
||||
// {"dim", info.dim_}, //
|
||||
{"field_id_", info.field_id_}, //
|
||||
{"num_queries", info.num_queries_}, //
|
||||
{"topK", info.topK_}, //
|
||||
{"search_params", info.search_params_}, //
|
||||
{"placeholder_tag", node.placeholder_tag_}, //
|
||||
@ -51,7 +52,7 @@ ShowPlanNodeVisitor::visit(FloatVectorANNS& node) {
|
||||
if (node.predicate_.has_value()) {
|
||||
PanicInfo("unimplemented");
|
||||
} else {
|
||||
json_body["predicate"] = "None";
|
||||
json_body["predicate"] = "nullopt";
|
||||
}
|
||||
ret_ = json_body;
|
||||
}
|
||||
|
||||
@ -50,36 +50,6 @@ field_sizeof(DataType data_type, int dim = 1) {
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: use magic_enum when available
|
||||
inline std::string
|
||||
datatype_name(DataType data_type) {
|
||||
switch (data_type) {
|
||||
case DataType::BOOL:
|
||||
return "bool";
|
||||
case DataType::DOUBLE:
|
||||
return "double";
|
||||
case DataType::FLOAT:
|
||||
return "float";
|
||||
case DataType::INT8:
|
||||
return "int8_t";
|
||||
case DataType::INT16:
|
||||
return "int16_t";
|
||||
case DataType::INT32:
|
||||
return "int32_t";
|
||||
case DataType::INT64:
|
||||
return "int64_t";
|
||||
case DataType::VECTOR_FLOAT:
|
||||
return "vector_float";
|
||||
case DataType::VECTOR_BINARY: {
|
||||
return "vector_binary";
|
||||
}
|
||||
default: {
|
||||
auto err_msg = "Unsupported DataType(" + std::to_string((int)data_type) + ")";
|
||||
PanicInfo(err_msg);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
inline bool
|
||||
field_is_vector(DataType datatype) {
|
||||
return datatype == DataType::VECTOR_BINARY || datatype == DataType::VECTOR_FLOAT;
|
||||
|
||||
@ -223,7 +223,6 @@ get_barrier(const RecordType& record, Timestamp timestamp) {
|
||||
Status
|
||||
SegmentSmallIndex::QueryBruteForceImpl(const query::QueryInfo& info,
|
||||
const float* query_data,
|
||||
int64_t num_queries,
|
||||
Timestamp timestamp,
|
||||
QueryResult& results) {
|
||||
// step 1: binary search to find the barrier of the snapshot
|
||||
@ -248,6 +247,7 @@ SegmentSmallIndex::QueryBruteForceImpl(const query::QueryInfo& info,
|
||||
Assert(field.get_data_type() == DataType::VECTOR_FLOAT);
|
||||
auto dim = field.get_dim();
|
||||
auto topK = info.topK_;
|
||||
auto num_queries = info.num_queries_;
|
||||
auto total_count = topK * num_queries;
|
||||
// TODO: optimize
|
||||
|
||||
@ -321,6 +321,7 @@ SegmentSmallIndex::QueryDeprecated(query::QueryDeprecatedPtr query_info, Timesta
|
||||
int64_t inferred_dim = query_info->query_raw_data.size() / query_info->num_queries;
|
||||
// TODO
|
||||
query::QueryInfo info{
|
||||
query_info->num_queries,
|
||||
query_info->topK,
|
||||
query_info->field_name,
|
||||
"L2",
|
||||
@ -328,8 +329,7 @@ SegmentSmallIndex::QueryDeprecated(query::QueryDeprecatedPtr query_info, Timesta
|
||||
{"nprobe", 10},
|
||||
},
|
||||
};
|
||||
auto num_queries = query_info->num_queries;
|
||||
return QueryBruteForceImpl(info, query_info->query_raw_data.data(), num_queries, timestamp, result);
|
||||
return QueryBruteForceImpl(info, query_info->query_raw_data.data(), timestamp, result);
|
||||
}
|
||||
|
||||
Status
|
||||
@ -453,15 +453,14 @@ SegmentSmallIndex::GetMemoryUsageInBytes() {
|
||||
}
|
||||
|
||||
Status
|
||||
SegmentSmallIndex::Search(const query::Plan* plan,
|
||||
SegmentSmallIndex::Search(const query::Plan* Plan,
|
||||
const query::PlaceholderGroup** placeholder_groups,
|
||||
const Timestamp* timestamps,
|
||||
int num_groups,
|
||||
QueryResult& results) {
|
||||
Assert(num_groups == 1);
|
||||
query::ExecPlanNodeVisitor visitor(*this, timestamps[0], *placeholder_groups[0]);
|
||||
results = visitor.get_moved_result(*plan->plan_node_);
|
||||
return Status::OK();
|
||||
PanicInfo("unimplemented");
|
||||
}
|
||||
|
||||
} // namespace milvus::segcore
|
||||
|
||||
@ -143,7 +143,6 @@ class SegmentSmallIndex : public SegmentBase {
|
||||
Status
|
||||
QueryBruteForceImpl(const query::QueryInfo& info,
|
||||
const float* query_data,
|
||||
int64_t num_queries,
|
||||
Timestamp timestamp,
|
||||
QueryResult& results);
|
||||
|
||||
|
||||
@ -9,7 +9,6 @@ set(MILVUS_TEST_FILES
|
||||
test_c_api.cpp
|
||||
test_indexing.cpp
|
||||
test_query.cpp
|
||||
test_expr.cpp
|
||||
)
|
||||
add_executable(all_tests
|
||||
${MILVUS_TEST_FILES}
|
||||
|
||||
@ -1,70 +0,0 @@
|
||||
#include <gtest/gtest.h>
|
||||
#include "query/Parser.h"
|
||||
#include "query/Expr.h"
|
||||
#include "query/PlanNode.h"
|
||||
#include "query/generated/ExprVisitor.h"
|
||||
#include "query/generated/PlanNodeVisitor.h"
|
||||
#include "test_utils/DataGen.h"
|
||||
#include "query/generated/ShowPlanNodeVisitor.h"
|
||||
|
||||
TEST(Expr, Naive) {
|
||||
SUCCEED();
|
||||
using namespace milvus::wtf;
|
||||
std::string dsl_string = R"(
|
||||
{
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"term": {
|
||||
"A": [
|
||||
1,
|
||||
2,
|
||||
5
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"range": {
|
||||
"B": {
|
||||
"GT": 1,
|
||||
"LT": 100
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"vector": {
|
||||
"Vec": {
|
||||
"metric_type": "L2",
|
||||
"params": {
|
||||
"nprobe": 10
|
||||
},
|
||||
"query": "$0",
|
||||
"topk": 10
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
})";
|
||||
}
|
||||
|
||||
TEST(Expr, ShowExecutor) {
|
||||
using namespace milvus::query;
|
||||
using namespace milvus::segcore;
|
||||
auto node = std::make_unique<FloatVectorANNS>();
|
||||
auto schema = std::make_shared<Schema>();
|
||||
int64_t num_queries = 100L;
|
||||
schema->AddField("fakevec", DataType::VECTOR_FLOAT, 16);
|
||||
auto raw_data = DataGen(schema, num_queries);
|
||||
auto& info = node->query_info_;
|
||||
info.metric_type_ = "L2";
|
||||
info.topK_ = 20;
|
||||
info.field_id_ = "fakevec";
|
||||
node->predicate_ = std::nullopt;
|
||||
ShowPlanNodeVisitor show_visitor;
|
||||
PlanNodePtr base(node.release());
|
||||
auto res = show_visitor.call_child(*base);
|
||||
auto dup = res;
|
||||
dup["data"] = "...collased...";
|
||||
std::cout << dup.dump(4);
|
||||
}
|
||||
@ -6,8 +6,6 @@
|
||||
#include "query/generated/PlanNodeVisitor.h"
|
||||
#include "test_utils/DataGen.h"
|
||||
#include "query/generated/ShowPlanNodeVisitor.h"
|
||||
#include "query/generated/ExecPlanNodeVisitor.h"
|
||||
#include "query/PlanImpl.h"
|
||||
|
||||
TEST(Query, Naive) {
|
||||
SUCCEED();
|
||||
@ -60,6 +58,7 @@ TEST(Query, ShowExecutor) {
|
||||
auto raw_data = DataGen(schema, num_queries);
|
||||
auto& info = node->query_info_;
|
||||
info.metric_type_ = "L2";
|
||||
info.num_queries_ = 10;
|
||||
info.topK_ = 20;
|
||||
info.field_id_ = "fakevec";
|
||||
node->predicate_ = std::nullopt;
|
||||
@ -67,87 +66,6 @@ TEST(Query, ShowExecutor) {
|
||||
PlanNodePtr base(node.release());
|
||||
auto res = show_visitor.call_child(*base);
|
||||
auto dup = res;
|
||||
dup["data"] = "...collased...";
|
||||
std::cout << dup.dump(4);
|
||||
}
|
||||
|
||||
TEST(Query, DSL) {
|
||||
using namespace milvus::query;
|
||||
using namespace milvus::segcore;
|
||||
ShowPlanNodeVisitor shower;
|
||||
|
||||
std::string dsl_string = R"(
|
||||
{
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"vector": {
|
||||
"Vec": {
|
||||
"metric_type": "L2",
|
||||
"params": {
|
||||
"nprobe": 10
|
||||
},
|
||||
"query": "$0",
|
||||
"topk": 10
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
})";
|
||||
auto plan = CreatePlan(dsl_string);
|
||||
auto res = shower.call_child(*plan->plan_node_);
|
||||
std::cout << res.dump(4) << std::endl;
|
||||
|
||||
std::string dsl_string2 = R"(
|
||||
{
|
||||
"bool": {
|
||||
"vector": {
|
||||
"Vec": {
|
||||
"metric_type": "L2",
|
||||
"params": {
|
||||
"nprobe": 10
|
||||
},
|
||||
"query": "$0",
|
||||
"topk": 10
|
||||
}
|
||||
}
|
||||
}
|
||||
})";
|
||||
auto plan2 = CreatePlan(dsl_string2);
|
||||
auto res2 = shower.call_child(*plan2->plan_node_);
|
||||
std::cout << res2.dump(4) << std::endl;
|
||||
ASSERT_EQ(res, res2);
|
||||
}
|
||||
|
||||
TEST(Query, ParsePlaceholderGroup) {
|
||||
using namespace milvus::query;
|
||||
using namespace milvus::segcore;
|
||||
namespace ser = milvus::proto::service;
|
||||
int num_queries = 10;
|
||||
int dim = 16;
|
||||
std::default_random_engine e;
|
||||
std::normal_distribution<double> dis(0, 1);
|
||||
ser::PlaceholderGroup raw_group;
|
||||
auto value = raw_group.add_placeholders();
|
||||
value->set_tag("$0");
|
||||
value->set_type(ser::PlaceholderType::VECTOR_FLOAT);
|
||||
for(int i = 0; i < num_queries; ++i) {
|
||||
std::vector<float> vec;
|
||||
for(int d = 0; d < dim; ++d) {
|
||||
vec.push_back(dis(e));
|
||||
}
|
||||
// std::string line((char*)vec.data(), (char*)vec.data() + vec.size() * sizeof(float));
|
||||
value->add_values(vec.data(), vec.size() * sizeof(float));
|
||||
}
|
||||
auto blob = raw_group.SerializeAsString();
|
||||
//ser::PlaceholderGroup new_group;
|
||||
//new_group.ParseFromString()
|
||||
auto fuck = ParsePlaceholderGroup(blob);
|
||||
int x = 1+1;
|
||||
}
|
||||
|
||||
|
||||
TEST(Query, Exec) {
|
||||
using namespace milvus::query;
|
||||
using namespace milvus::segcore;
|
||||
}
|
||||
}
|
||||
@ -3,17 +3,23 @@ package kv
|
||||
import (
|
||||
"context"
|
||||
"path"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil"
|
||||
"go.etcd.io/etcd/clientv3"
|
||||
)
|
||||
|
||||
func TestEtcdKV_Load(t *testing.T) {
|
||||
conf.LoadConfig("config.yaml")
|
||||
etcdPort := strconv.Itoa(int(conf.Config.Etcd.Port))
|
||||
err := gparams.GParams.LoadYaml("config.yaml")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
etcdPort, err := gparams.GParams.Load("etcd.port")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
cli, err := clientv3.New(clientv3.Config{Endpoints: []string{"127.0.0.1:" + etcdPort}})
|
||||
assert.Nil(t, err)
|
||||
rootpath := "/etcd/test/root"
|
||||
@ -68,8 +74,15 @@ func TestEtcdKV_Load(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEtcdKV_MultiSave(t *testing.T) {
|
||||
conf.LoadConfig("config.yaml")
|
||||
etcdPort := strconv.Itoa(int(conf.Config.Etcd.Port))
|
||||
err := gparams.GParams.LoadYaml("config.yaml")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
etcdPort, err := gparams.GParams.Load("etcd.port")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
cli, err := clientv3.New(clientv3.Config{Endpoints: []string{"127.0.0.1:" + etcdPort}})
|
||||
assert.Nil(t, err)
|
||||
rootpath := "/etcd/test/root"
|
||||
@ -97,8 +110,15 @@ func TestEtcdKV_MultiSave(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEtcdKV_Remove(t *testing.T) {
|
||||
conf.LoadConfig("config.yaml")
|
||||
etcdPort := strconv.Itoa(int(conf.Config.Etcd.Port))
|
||||
err := gparams.GParams.LoadYaml("config.yaml")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
etcdPort, err := gparams.GParams.Load("etcd.port")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
cli, err := clientv3.New(clientv3.Config{Endpoints: []string{"127.0.0.1:" + etcdPort}})
|
||||
assert.Nil(t, err)
|
||||
rootpath := "/etcd/test/root"
|
||||
@ -166,8 +186,15 @@ func TestEtcdKV_Remove(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEtcdKV_MultiSaveAndRemove(t *testing.T) {
|
||||
conf.LoadConfig("config.yaml")
|
||||
etcdPort := strconv.Itoa(int(conf.Config.Etcd.Port))
|
||||
err := gparams.GParams.LoadYaml("config.yaml")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
etcdPort, err := gparams.GParams.Load("etcd.port")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
cli, err := clientv3.New(clientv3.Config{Endpoints: []string{"127.0.0.1:" + etcdPort}})
|
||||
assert.Nil(t, err)
|
||||
rootpath := "/etcd/test/root"
|
||||
|
||||
@ -1,14 +1,14 @@
|
||||
package mockkv
|
||||
|
||||
import (
|
||||
"github.com/zilliztech/milvus-distributed/internal/kv"
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/kvutil"
|
||||
)
|
||||
|
||||
// use MemoryKV to mock EtcdKV
|
||||
func NewEtcdKV() *kv.MemoryKV {
|
||||
return kv.NewMemoryKV()
|
||||
func NewEtcdKV() *kvutil.MemoryKV {
|
||||
return kvutil.NewMemoryKV()
|
||||
}
|
||||
|
||||
func NewMemoryKV() *kv.MemoryKV {
|
||||
return kv.NewMemoryKV()
|
||||
func NewMemoryKV() *kvutil.MemoryKV {
|
||||
return kvutil.NewMemoryKV()
|
||||
}
|
||||
|
||||
@ -3,27 +3,32 @@ package master
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/masterpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/servicepb"
|
||||
gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil"
|
||||
"go.etcd.io/etcd/clientv3"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
func TestMaster_CreateCollectionTask(t *testing.T) {
|
||||
conf.LoadConfig("config.yaml")
|
||||
err := gparams.GParams.LoadYaml("config.yaml")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.TODO())
|
||||
defer cancel()
|
||||
|
||||
etcdPort := strconv.Itoa(int(conf.Config.Etcd.Port))
|
||||
etcdPort, err := gparams.GParams.Load("etcd.port")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
etcdAddr := "127.0.0.1:" + etcdPort
|
||||
|
||||
etcdCli, err := clientv3.New(clientv3.Config{Endpoints: []string{etcdAddr}})
|
||||
|
||||
@ -5,15 +5,13 @@ import (
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/master/id"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
"github.com/zilliztech/milvus-distributed/internal/kv"
|
||||
"github.com/zilliztech/milvus-distributed/internal/master/collection"
|
||||
"github.com/zilliztech/milvus-distributed/internal/master/id"
|
||||
"github.com/zilliztech/milvus-distributed/internal/master/segment"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
|
||||
gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil"
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
|
||||
)
|
||||
|
||||
type UniqueID = typeutil.UniqueID
|
||||
@ -68,7 +66,15 @@ func WriteCollection2Datastore(collectionMeta *schemapb.CollectionSchema, kvbase
|
||||
time.Now(), fieldMetas, []UniqueID{sID},
|
||||
[]string{"default"})
|
||||
cm := collection.GrpcMarshal(&c)
|
||||
s := segment.NewSegment(sID, cID, collectionMeta.Name, "default", 0, conf.Config.Pulsar.TopicNum, time.Now(), time.Unix(1<<46-1, 0))
|
||||
pulsarTopicNum, err := gparams.GParams.Load("pulsar.topicnum")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
topicNum, err := strconv.Atoi(pulsarTopicNum)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
s := segment.NewSegment(sID, cID, collectionMeta.Name, "default", 0, topicNum, time.Now(), time.Unix(1<<46-1, 0))
|
||||
collectionData, err := collection.Collection2JSON(*cm)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
|
||||
@ -5,24 +5,31 @@ import (
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/master/id"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
"github.com/zilliztech/milvus-distributed/internal/kv"
|
||||
"github.com/zilliztech/milvus-distributed/internal/master/collection"
|
||||
"github.com/zilliztech/milvus-distributed/internal/master/id"
|
||||
"github.com/zilliztech/milvus-distributed/internal/master/segment"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
|
||||
gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil"
|
||||
)
|
||||
|
||||
func ComputeCloseTime(ss internalpb.SegmentStats, kvbase *kv.EtcdKV) error {
|
||||
if int(ss.MemorySize) > int(conf.Config.Master.SegmentThreshole*0.8) {
|
||||
masterSegmentThreshole, err := gparams.GParams.Load("master.segmentthreshole")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
segmentThreshole, err := strconv.ParseFloat(masterSegmentThreshole, 32)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if int(ss.MemorySize) > int(segmentThreshole*0.8) {
|
||||
currentTime := time.Now()
|
||||
//memRate := int(ss.MemoryRate)
|
||||
memRate := 1 // to do
|
||||
if memRate == 0 {
|
||||
memRate = 1
|
||||
}
|
||||
sec := int(conf.Config.Master.SegmentThreshole*0.2) / memRate
|
||||
sec := int(segmentThreshole*0.2) / memRate
|
||||
data, err := kvbase.Load("segment/" + strconv.Itoa(int(ss.SegmentID)))
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@ -1,26 +1,34 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
"github.com/zilliztech/milvus-distributed/internal/kv"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
|
||||
gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil"
|
||||
"go.etcd.io/etcd/clientv3"
|
||||
)
|
||||
|
||||
func newKvBase() *kv.EtcdKV {
|
||||
//etcdAddr := conf.Config.Etcd.Address
|
||||
//etcdAddr += ":"
|
||||
//etcdAddr += strconv.FormatInt(int64(conf.Config.Etcd.Port), 10)
|
||||
etcdAddr := "127.0.0.1:" + strconv.Itoa(int(conf.Config.Etcd.Port))
|
||||
err := gparams.GParams.LoadYaml("config.yaml")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
etcdPort, err := gparams.GParams.Load("etcd.port")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
etcdAddr := "127.0.0.1:" + etcdPort
|
||||
cli, _ := clientv3.New(clientv3.Config{
|
||||
Endpoints: []string{etcdAddr},
|
||||
DialTimeout: 5 * time.Second,
|
||||
})
|
||||
kvbase := kv.NewEtcdKV(cli, conf.Config.Etcd.Rootpath)
|
||||
etcdRootPath, err := gparams.GParams.Load("etcd.rootpath")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
kvbase := kv.NewEtcdKV(cli, etcdRootPath)
|
||||
return kvbase
|
||||
}
|
||||
|
||||
|
||||
@ -2,26 +2,31 @@ package master
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/masterpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
|
||||
gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil"
|
||||
"go.etcd.io/etcd/clientv3"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
func TestMaster_CreateCollection(t *testing.T) {
|
||||
conf.LoadConfig("config.yaml")
|
||||
err := gparams.GParams.LoadYaml("config.yaml")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.TODO())
|
||||
defer cancel()
|
||||
|
||||
etcdPort := strconv.Itoa(int(conf.Config.Etcd.Port))
|
||||
etcdPort, err := gparams.GParams.Load("etcd.port")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
etcdAddr := "127.0.0.1:" + etcdPort
|
||||
|
||||
etcdCli, err := clientv3.New(clientv3.Config{Endpoints: []string{etcdAddr}})
|
||||
|
||||
@ -1,8 +1,8 @@
|
||||
package id
|
||||
|
||||
import (
|
||||
"github.com/zilliztech/milvus-distributed/internal/kv"
|
||||
"github.com/zilliztech/milvus-distributed/internal/master/tso"
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/kvutil"
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/tsoutil"
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
|
||||
)
|
||||
@ -20,12 +20,12 @@ func Init() {
|
||||
InitGlobalIDAllocator("idTimestamp", tsoutil.NewTSOKVBase("gid"))
|
||||
}
|
||||
|
||||
func InitGlobalIDAllocator(key string, base kv.Base) {
|
||||
func InitGlobalIDAllocator(key string, base kvutil.Base) {
|
||||
allocator = NewGlobalIDAllocator(key, base)
|
||||
allocator.Initialize()
|
||||
}
|
||||
|
||||
func NewGlobalIDAllocator(key string, base kv.Base) *GlobalIDAllocator {
|
||||
func NewGlobalIDAllocator(key string, base kvutil.Base) *GlobalIDAllocator {
|
||||
return &GlobalIDAllocator{
|
||||
allocator: tso.NewGlobalTSOAllocator(key, base),
|
||||
}
|
||||
|
||||
@ -6,14 +6,17 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil"
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/tsoutil"
|
||||
)
|
||||
|
||||
var GIdAllocator *GlobalIDAllocator
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
conf.LoadConfig("config.yaml")
|
||||
err := gparams.GParams.LoadYaml("config.yaml")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
GIdAllocator = NewGlobalIDAllocator("idTimestamp", tsoutil.NewTSOKVBase("gid"))
|
||||
exitCode := m.Run()
|
||||
os.Exit(exitCode)
|
||||
|
||||
@ -2,19 +2,22 @@ package informer
|
||||
|
||||
import (
|
||||
"log"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
|
||||
"github.com/apache/pulsar-client-go/pulsar"
|
||||
gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil"
|
||||
)
|
||||
|
||||
func NewPulsarClient() *PulsarClient {
|
||||
pulsarAddr := "pulsar://"
|
||||
pulsarAddr += conf.Config.Pulsar.Address
|
||||
pulsarAddr += ":"
|
||||
pulsarAddr += strconv.FormatInt(int64(conf.Config.Pulsar.Port), 10)
|
||||
pulsarAddr, err := gparams.GParams.Load("pulsar.address")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
pulsarPort, err := gparams.GParams.Load("pulsar.port")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
pulsarAddr = "pulsar://" + pulsarAddr + ":" + pulsarPort
|
||||
client, err := pulsar.NewClient(pulsar.ClientOptions{
|
||||
URL: pulsarAddr,
|
||||
OperationTimeout: 30 * time.Second,
|
||||
|
||||
@ -16,7 +16,6 @@ import (
|
||||
"go.etcd.io/etcd/clientv3"
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
"github.com/zilliztech/milvus-distributed/internal/errors"
|
||||
"github.com/zilliztech/milvus-distributed/internal/kv"
|
||||
"github.com/zilliztech/milvus-distributed/internal/master/controller"
|
||||
@ -25,6 +24,7 @@ import (
|
||||
"github.com/zilliztech/milvus-distributed/internal/master/tso"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/masterpb"
|
||||
gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil"
|
||||
)
|
||||
|
||||
// Server is the pd server.
|
||||
@ -236,8 +236,13 @@ func (s *Master) pulsarLoop() {
|
||||
|
||||
ctx, cancel := context.WithCancel(s.serverLoopCtx)
|
||||
|
||||
pulsarTopic, err := gparams.GParams.Load("master.pulsartopic")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
consumer, err := s.pc.Client.Subscribe(pulsar.ConsumerOptions{
|
||||
Topic: conf.Config.Master.PulsarTopic,
|
||||
Topic: pulsarTopic,
|
||||
SubscriptionName: "my-sub",
|
||||
Type: pulsar.Shared,
|
||||
})
|
||||
|
||||
@ -2,20 +2,25 @@ package master
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
"github.com/zilliztech/milvus-distributed/internal/kv"
|
||||
pb "github.com/zilliztech/milvus-distributed/internal/proto/etcdpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
|
||||
gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil"
|
||||
"go.etcd.io/etcd/clientv3"
|
||||
)
|
||||
|
||||
func TestMetaTable_Collection(t *testing.T) {
|
||||
conf.LoadConfig("config.yaml")
|
||||
etcdPort := strconv.Itoa(int(conf.Config.Etcd.Port))
|
||||
err := gparams.GParams.LoadYaml("config.yaml")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
etcdPort, err := gparams.GParams.Load("etcd.port")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
cli, err := clientv3.New(clientv3.Config{Endpoints: []string{"127.0.0.1:" + etcdPort}})
|
||||
assert.Nil(t, err)
|
||||
etcdKV := kv.NewEtcdKV(cli, "/etcd/test/root")
|
||||
@ -137,8 +142,15 @@ func TestMetaTable_Collection(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestMetaTable_DeletePartition(t *testing.T) {
|
||||
conf.LoadConfig("config.yaml")
|
||||
etcdPort := strconv.Itoa(int(conf.Config.Etcd.Port))
|
||||
err := gparams.GParams.LoadYaml("config.yaml")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
etcdPort, err := gparams.GParams.Load("etcd.port")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
cli, err := clientv3.New(clientv3.Config{Endpoints: []string{"127.0.0.1:" + etcdPort}})
|
||||
assert.Nil(t, err)
|
||||
etcdKV := kv.NewEtcdKV(cli, "/etcd/test/root")
|
||||
@ -220,8 +232,15 @@ func TestMetaTable_DeletePartition(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestMetaTable_Segment(t *testing.T) {
|
||||
conf.LoadConfig("config.yaml")
|
||||
etcdPort := strconv.Itoa(int(conf.Config.Etcd.Port))
|
||||
err := gparams.GParams.LoadYaml("config.yaml")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
etcdPort, err := gparams.GParams.Load("etcd.port")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
cli, err := clientv3.New(clientv3.Config{Endpoints: []string{"127.0.0.1:" + etcdPort}})
|
||||
assert.Nil(t, err)
|
||||
etcdKV := kv.NewEtcdKV(cli, "/etcd/test/root")
|
||||
|
||||
@ -3,6 +3,8 @@ package mockmaster
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/master/tso"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/master/id"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
|
||||
@ -73,7 +75,7 @@ func (s *Master) ShowPartitions(ctx context.Context, in *internalpb.ShowPartitio
|
||||
|
||||
func (s *Master) AllocTimestamp(ctx context.Context, request *internalpb.TsoRequest) (*internalpb.TsoResponse, error) {
|
||||
count := request.GetCount()
|
||||
ts, err := s.tsoAllocator.GenerateTSO(count)
|
||||
ts, err := tso.Alloc(count)
|
||||
|
||||
if err != nil {
|
||||
return &internalpb.TsoResponse{
|
||||
|
||||
@ -6,19 +6,18 @@ import (
|
||||
"log"
|
||||
"math/rand"
|
||||
"net"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
"github.com/zilliztech/milvus-distributed/internal/kv"
|
||||
"github.com/zilliztech/milvus-distributed/internal/kv/mockkv"
|
||||
"github.com/zilliztech/milvus-distributed/internal/master/id"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/masterpb"
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/kv/mockkv"
|
||||
"github.com/zilliztech/milvus-distributed/internal/master/id"
|
||||
"github.com/zilliztech/milvus-distributed/internal/master/tso"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/masterpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/kvutil"
|
||||
gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil"
|
||||
)
|
||||
|
||||
// Server is the pd server.
|
||||
@ -36,7 +35,7 @@ type Master struct {
|
||||
// for tso.
|
||||
tsoAllocator tso.Allocator
|
||||
|
||||
kvBase kv.Base
|
||||
kvBase kvutil.Base
|
||||
|
||||
// error chans
|
||||
grpcErr chan error
|
||||
@ -183,9 +182,15 @@ func (s *Master) checkReady(ctx context.Context, targetCh chan error) {
|
||||
|
||||
func (s *Master) grpcLoop() {
|
||||
defer s.serverLoopWg.Done()
|
||||
masterAddr := conf.Config.Etcd.Address
|
||||
masterAddr += ":"
|
||||
masterAddr += strconv.FormatInt(int64(conf.Config.Master.Port), 10)
|
||||
masterAddr, err := gparams.GParams.Load("master.address")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
masterPort, err := gparams.GParams.Load("master.port")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
masterAddr = masterAddr + ":" + masterPort
|
||||
lis, err := net.Listen("tcp", masterAddr)
|
||||
if err != nil {
|
||||
log.Printf("failed to listen: %v", err)
|
||||
|
||||
@ -6,25 +6,30 @@ import (
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/servicepb"
|
||||
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/masterpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/servicepb"
|
||||
gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil"
|
||||
"go.etcd.io/etcd/clientv3"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
func TestMaster_Partition(t *testing.T) {
|
||||
conf.LoadConfig("config.yaml")
|
||||
err := gparams.GParams.LoadYaml("config.yaml")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.TODO())
|
||||
defer cancel()
|
||||
|
||||
etcdPort := strconv.Itoa(int(conf.Config.Etcd.Port))
|
||||
etcdPort, err := gparams.GParams.Load("etcd.port")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
etcdAddr := "127.0.0.1:" + etcdPort
|
||||
|
||||
etcdCli, err := clientv3.New(clientv3.Config{Endpoints: []string{etcdAddr}})
|
||||
|
||||
@ -5,9 +5,8 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/kv"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/errors"
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/kvutil"
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/tsoutil"
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
|
||||
"go.uber.org/zap"
|
||||
@ -42,13 +41,13 @@ func Init() {
|
||||
InitGlobalTsoAllocator("timestamp", tsoutil.NewTSOKVBase("tso"))
|
||||
}
|
||||
|
||||
func InitGlobalTsoAllocator(key string, base kv.Base) {
|
||||
func InitGlobalTsoAllocator(key string, base kvutil.Base) {
|
||||
allocator = NewGlobalTSOAllocator(key, base)
|
||||
allocator.Initialize()
|
||||
}
|
||||
|
||||
// NewGlobalTSOAllocator creates a new global TSO allocator.
|
||||
func NewGlobalTSOAllocator(key string, kvBase kv.Base) *GlobalTSOAllocator {
|
||||
func NewGlobalTSOAllocator(key string, kvBase kvutil.Base) *GlobalTSOAllocator {
|
||||
var saveInterval = 3 * time.Second
|
||||
return &GlobalTSOAllocator{
|
||||
tso: ×tampOracle{
|
||||
|
||||
@ -7,14 +7,17 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil"
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/tsoutil"
|
||||
)
|
||||
|
||||
var GTsoAllocator Allocator
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
conf.LoadConfig("config.yaml")
|
||||
err := gparams.GParams.LoadYaml("config.yaml")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
GTsoAllocator = NewGlobalTSOAllocator("timestamp", tsoutil.NewTSOKVBase("tso"))
|
||||
|
||||
exitCode := m.Run()
|
||||
|
||||
@ -22,7 +22,7 @@ import (
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/errors"
|
||||
"github.com/zilliztech/milvus-distributed/internal/kv"
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/kvutil"
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/tsoutil"
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
|
||||
)
|
||||
@ -47,7 +47,7 @@ type atomicObject struct {
|
||||
// timestampOracle is used to maintain the logic of tso.
|
||||
type timestampOracle struct {
|
||||
key string
|
||||
kvBase kv.Base
|
||||
kvBase kvutil.Base
|
||||
|
||||
// TODO: remove saveInterval
|
||||
saveInterval time.Duration
|
||||
|
||||
@ -3,10 +3,8 @@ package proxy
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
|
||||
"github.com/golang/protobuf/proto"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/msgstream"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
|
||||
@ -28,27 +26,44 @@ func (p *Proxy) Insert(ctx context.Context, in *servicepb.RowBatch) (*servicepb.
|
||||
},
|
||||
},
|
||||
done: make(chan error),
|
||||
resultChan: make(chan *servicepb.IntegerRangeResponse),
|
||||
manipulationMsgStream: p.manipulationMsgStream,
|
||||
}
|
||||
|
||||
it.ctx, it.cancel = context.WithCancel(ctx)
|
||||
// TODO: req_id, segment_id, channel_id, proxy_id, timestamps, row_ids
|
||||
|
||||
defer it.cancel()
|
||||
|
||||
p.taskSch.DmQueue.Enqueue(it)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Print("insert timeout!")
|
||||
fn := func() error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return errors.New("insert timeout")
|
||||
default:
|
||||
return p.taskSch.DdQueue.Enqueue(it)
|
||||
}
|
||||
}
|
||||
err := fn()
|
||||
|
||||
if err != nil {
|
||||
return &servicepb.IntegerRangeResponse{
|
||||
Status: &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: "insert timeout!",
|
||||
Reason: err.Error(),
|
||||
},
|
||||
}, errors.New("insert timeout")
|
||||
case result := <-it.resultChan:
|
||||
return result, nil
|
||||
}, nil
|
||||
}
|
||||
|
||||
err = it.WaitToFinish()
|
||||
if err != nil {
|
||||
return &servicepb.IntegerRangeResponse{
|
||||
Status: &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: err.Error(),
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
return it.result, nil
|
||||
}
|
||||
|
||||
func (p *Proxy) CreateCollection(ctx context.Context, req *schemapb.CollectionSchema) (*commonpb.Status, error) {
|
||||
@ -60,24 +75,37 @@ func (p *Proxy) CreateCollection(ctx context.Context, req *schemapb.CollectionSc
|
||||
},
|
||||
masterClient: p.masterClient,
|
||||
done: make(chan error),
|
||||
resultChan: make(chan *commonpb.Status),
|
||||
}
|
||||
schemaBytes, _ := proto.Marshal(req)
|
||||
cct.CreateCollectionRequest.Schema.Value = schemaBytes
|
||||
cct.ctx, cct.cancel = context.WithCancel(ctx)
|
||||
defer cct.cancel()
|
||||
|
||||
p.taskSch.DdQueue.Enqueue(cct)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Print("create collection timeout!")
|
||||
fn := func() error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return errors.New("create collection timeout")
|
||||
default:
|
||||
return p.taskSch.DdQueue.Enqueue(cct)
|
||||
}
|
||||
}
|
||||
err := fn()
|
||||
if err != nil {
|
||||
return &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: "create collection timeout!",
|
||||
}, errors.New("create collection timeout")
|
||||
case result := <-cct.resultChan:
|
||||
return result, nil
|
||||
Reason: err.Error(),
|
||||
}, err
|
||||
}
|
||||
|
||||
err = cct.WaitToFinish()
|
||||
if err != nil {
|
||||
return &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: err.Error(),
|
||||
}, err
|
||||
}
|
||||
|
||||
return cct.result, nil
|
||||
}
|
||||
|
||||
func (p *Proxy) Search(ctx context.Context, req *servicepb.Query) (*servicepb.QueryResult, error) {
|
||||
@ -90,26 +118,41 @@ func (p *Proxy) Search(ctx context.Context, req *servicepb.Query) (*servicepb.Qu
|
||||
queryMsgStream: p.queryMsgStream,
|
||||
done: make(chan error),
|
||||
resultBuf: make(chan []*internalpb.SearchResult),
|
||||
resultChan: make(chan *servicepb.QueryResult),
|
||||
}
|
||||
qt.ctx, qt.cancel = context.WithCancel(ctx)
|
||||
queryBytes, _ := proto.Marshal(req)
|
||||
qt.SearchRequest.Query.Value = queryBytes
|
||||
defer qt.cancel()
|
||||
|
||||
p.taskSch.DqQueue.Enqueue(qt)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Print("query timeout!")
|
||||
fn := func() error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return errors.New("create collection timeout")
|
||||
default:
|
||||
return p.taskSch.DdQueue.Enqueue(qt)
|
||||
}
|
||||
}
|
||||
err := fn()
|
||||
if err != nil {
|
||||
return &servicepb.QueryResult{
|
||||
Status: &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: "query timeout!",
|
||||
Reason: err.Error(),
|
||||
},
|
||||
}, errors.New("query timeout")
|
||||
case result := <-qt.resultChan:
|
||||
return result, nil
|
||||
}, err
|
||||
}
|
||||
|
||||
err = qt.WaitToFinish()
|
||||
if err != nil {
|
||||
return &servicepb.QueryResult{
|
||||
Status: &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: err.Error(),
|
||||
},
|
||||
}, err
|
||||
}
|
||||
|
||||
return qt.result, nil
|
||||
}
|
||||
|
||||
func (p *Proxy) DropCollection(ctx context.Context, req *servicepb.CollectionName) (*commonpb.Status, error) {
|
||||
@ -121,22 +164,35 @@ func (p *Proxy) DropCollection(ctx context.Context, req *servicepb.CollectionNam
|
||||
},
|
||||
masterClient: p.masterClient,
|
||||
done: make(chan error),
|
||||
resultChan: make(chan *commonpb.Status),
|
||||
}
|
||||
dct.ctx, dct.cancel = context.WithCancel(ctx)
|
||||
defer dct.cancel()
|
||||
|
||||
p.taskSch.DdQueue.Enqueue(dct)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Print("create collection timeout!")
|
||||
fn := func() error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return errors.New("create collection timeout")
|
||||
default:
|
||||
return p.taskSch.DdQueue.Enqueue(dct)
|
||||
}
|
||||
}
|
||||
err := fn()
|
||||
if err != nil {
|
||||
return &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: "create collection timeout!",
|
||||
}, errors.New("create collection timeout")
|
||||
case result := <-dct.resultChan:
|
||||
return result, nil
|
||||
Reason: err.Error(),
|
||||
}, err
|
||||
}
|
||||
|
||||
err = dct.WaitToFinish()
|
||||
if err != nil {
|
||||
return &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: err.Error(),
|
||||
}, err
|
||||
}
|
||||
|
||||
return dct.result, nil
|
||||
}
|
||||
|
||||
func (p *Proxy) HasCollection(ctx context.Context, req *servicepb.CollectionName) (*servicepb.BoolResponse, error) {
|
||||
@ -148,25 +204,39 @@ func (p *Proxy) HasCollection(ctx context.Context, req *servicepb.CollectionName
|
||||
},
|
||||
masterClient: p.masterClient,
|
||||
done: make(chan error),
|
||||
resultChan: make(chan *servicepb.BoolResponse),
|
||||
}
|
||||
hct.ctx, hct.cancel = context.WithCancel(ctx)
|
||||
defer hct.cancel()
|
||||
|
||||
p.taskSch.DqQueue.Enqueue(hct)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Print("has collection timeout!")
|
||||
fn := func() error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return errors.New("create collection timeout")
|
||||
default:
|
||||
return p.taskSch.DdQueue.Enqueue(hct)
|
||||
}
|
||||
}
|
||||
err := fn()
|
||||
if err != nil {
|
||||
return &servicepb.BoolResponse{
|
||||
Status: &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: "has collection timeout!",
|
||||
Reason: err.Error(),
|
||||
},
|
||||
Value: false,
|
||||
}, errors.New("has collection timeout")
|
||||
case result := <-hct.resultChan:
|
||||
return result, nil
|
||||
}, err
|
||||
}
|
||||
|
||||
err = hct.WaitToFinish()
|
||||
if err != nil {
|
||||
return &servicepb.BoolResponse{
|
||||
Status: &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: err.Error(),
|
||||
},
|
||||
}, err
|
||||
}
|
||||
|
||||
return hct.result, nil
|
||||
}
|
||||
|
||||
func (p *Proxy) DescribeCollection(ctx context.Context, req *servicepb.CollectionName) (*servicepb.CollectionDescription, error) {
|
||||
@ -178,24 +248,39 @@ func (p *Proxy) DescribeCollection(ctx context.Context, req *servicepb.Collectio
|
||||
},
|
||||
masterClient: p.masterClient,
|
||||
done: make(chan error),
|
||||
resultChan: make(chan *servicepb.CollectionDescription),
|
||||
}
|
||||
dct.ctx, dct.cancel = context.WithCancel(ctx)
|
||||
defer dct.cancel()
|
||||
|
||||
p.taskSch.DqQueue.Enqueue(dct)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Print("has collection timeout!")
|
||||
fn := func() error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return errors.New("create collection timeout")
|
||||
default:
|
||||
return p.taskSch.DdQueue.Enqueue(dct)
|
||||
}
|
||||
}
|
||||
err := fn()
|
||||
if err != nil {
|
||||
return &servicepb.CollectionDescription{
|
||||
Status: &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: "describe collection timeout!",
|
||||
Reason: err.Error(),
|
||||
},
|
||||
}, errors.New("describe collection timeout")
|
||||
case result := <-dct.resultChan:
|
||||
return result, nil
|
||||
}, err
|
||||
}
|
||||
|
||||
err = dct.WaitToFinish()
|
||||
if err != nil {
|
||||
return &servicepb.CollectionDescription{
|
||||
Status: &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: err.Error(),
|
||||
},
|
||||
}, err
|
||||
}
|
||||
|
||||
return dct.result, nil
|
||||
}
|
||||
|
||||
func (p *Proxy) ShowCollections(ctx context.Context, req *commonpb.Empty) (*servicepb.StringListResponse, error) {
|
||||
@ -206,24 +291,39 @@ func (p *Proxy) ShowCollections(ctx context.Context, req *commonpb.Empty) (*serv
|
||||
},
|
||||
masterClient: p.masterClient,
|
||||
done: make(chan error),
|
||||
resultChan: make(chan *servicepb.StringListResponse),
|
||||
}
|
||||
sct.ctx, sct.cancel = context.WithCancel(ctx)
|
||||
defer sct.cancel()
|
||||
|
||||
p.taskSch.DqQueue.Enqueue(sct)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Print("show collections timeout!")
|
||||
fn := func() error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return errors.New("create collection timeout")
|
||||
default:
|
||||
return p.taskSch.DdQueue.Enqueue(sct)
|
||||
}
|
||||
}
|
||||
err := fn()
|
||||
if err != nil {
|
||||
return &servicepb.StringListResponse{
|
||||
Status: &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: "show collections timeout!",
|
||||
Reason: err.Error(),
|
||||
},
|
||||
}, errors.New("show collections timeout")
|
||||
case result := <-sct.resultChan:
|
||||
return result, nil
|
||||
}, err
|
||||
}
|
||||
|
||||
err = sct.WaitToFinish()
|
||||
if err != nil {
|
||||
return &servicepb.StringListResponse{
|
||||
Status: &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: err.Error(),
|
||||
},
|
||||
}, err
|
||||
}
|
||||
|
||||
return sct.result, nil
|
||||
}
|
||||
|
||||
func (p *Proxy) CreatePartition(ctx context.Context, in *servicepb.PartitionName) (*commonpb.Status, error) {
|
||||
|
||||
@ -5,9 +5,12 @@ import (
|
||||
"log"
|
||||
"math/rand"
|
||||
"net"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/allocator"
|
||||
@ -51,10 +54,30 @@ func CreateProxy(ctx context.Context) (*Proxy, error) {
|
||||
proxyLoopCtx: ctx1,
|
||||
proxyLoopCancel: cancel,
|
||||
}
|
||||
|
||||
// TODO: use config instead
|
||||
pulsarAddress := "pulsar://localhost:6650"
|
||||
bufSize := int64(1000)
|
||||
manipulationChannels := []string{"manipulation"}
|
||||
queryChannels := []string{"query"}
|
||||
queryResultChannels := []string{"QueryResult"}
|
||||
queryResultSubName := "QueryResultSubject"
|
||||
unmarshal := msgstream.NewUnmarshalDispatcher()
|
||||
|
||||
p.manipulationMsgStream = msgstream.NewPulsarMsgStream(p.proxyLoopCtx, bufSize)
|
||||
p.manipulationMsgStream.SetPulsarCient(pulsarAddress)
|
||||
p.manipulationMsgStream.CreatePulsarProducers(manipulationChannels)
|
||||
|
||||
p.queryMsgStream = msgstream.NewPulsarMsgStream(p.proxyLoopCtx, bufSize)
|
||||
p.queryMsgStream.SetPulsarCient(pulsarAddress)
|
||||
p.queryMsgStream.CreatePulsarProducers(queryChannels)
|
||||
|
||||
p.queryResultMsgStream = msgstream.NewPulsarMsgStream(p.proxyLoopCtx, bufSize)
|
||||
p.queryResultMsgStream.SetPulsarCient(pulsarAddress)
|
||||
p.queryResultMsgStream.CreatePulsarConsumers(queryResultChannels,
|
||||
queryResultSubName,
|
||||
unmarshal,
|
||||
bufSize)
|
||||
|
||||
idAllocator, err := allocator.NewIDAllocator(p.proxyLoopCtx)
|
||||
|
||||
@ -111,7 +134,7 @@ func (p *Proxy) grpcLoop() {
|
||||
defer p.proxyLoopWg.Done()
|
||||
|
||||
// TODO: use address in config instead
|
||||
lis, err := net.Listen("tcp", "5053")
|
||||
lis, err := net.Listen("tcp", ":5053")
|
||||
if err != nil {
|
||||
log.Fatalf("Proxy grpc server fatal error=%v", err)
|
||||
}
|
||||
@ -124,16 +147,18 @@ func (p *Proxy) grpcLoop() {
|
||||
}
|
||||
|
||||
func (p *Proxy) connectMaster() error {
|
||||
log.Printf("Connected to master, master_addr=%s", "127.0.0.1:5053")
|
||||
masterHost := conf.Config.Master.Address
|
||||
masterPort := conf.Config.Master.Port
|
||||
masterAddr := masterHost + ":" + strconv.FormatInt(int64(masterPort), 10)
|
||||
log.Printf("Proxy connected to master, master_addr=%s", masterAddr)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
|
||||
conn, err := grpc.DialContext(ctx, "127.0.0.1:5053", grpc.WithInsecure(), grpc.WithBlock())
|
||||
conn, err := grpc.DialContext(ctx, masterAddr, grpc.WithInsecure(), grpc.WithBlock())
|
||||
if err != nil {
|
||||
log.Printf("Connect to master failed, error= %v", err)
|
||||
log.Printf("Proxy connect to master failed, error= %v", err)
|
||||
return err
|
||||
}
|
||||
log.Printf("Connected to master, master_addr=%s", "127.0.0.1:5053")
|
||||
log.Printf("Proxy connected to master, master_addr=%s", masterAddr)
|
||||
p.masterConn = conn
|
||||
p.masterClient = masterpb.NewMasterClient(conn)
|
||||
return nil
|
||||
@ -141,28 +166,39 @@ func (p *Proxy) connectMaster() error {
|
||||
|
||||
func (p *Proxy) queryResultLoop() {
|
||||
defer p.proxyLoopWg.Done()
|
||||
defer p.proxyLoopCancel()
|
||||
|
||||
queryResultBuf := make(map[UniqueID][]*internalpb.SearchResult)
|
||||
|
||||
for {
|
||||
msgPack := p.queryResultMsgStream.Consume()
|
||||
if msgPack == nil {
|
||||
continue
|
||||
}
|
||||
tsMsg := msgPack.Msgs[0]
|
||||
searchResultMsg, _ := (*tsMsg).(*msgstream.SearchResultMsg)
|
||||
reqID := searchResultMsg.GetReqID()
|
||||
_, ok := queryResultBuf[reqID]
|
||||
if !ok {
|
||||
queryResultBuf[reqID] = make([]*internalpb.SearchResult, 0)
|
||||
}
|
||||
queryResultBuf[reqID] = append(queryResultBuf[reqID], &searchResultMsg.SearchResult)
|
||||
if len(queryResultBuf[reqID]) == 4 {
|
||||
// TODO: use the number of query node instead
|
||||
t := p.taskSch.getTaskByReqID(reqID)
|
||||
qt := t.(*QueryTask)
|
||||
qt.resultBuf <- queryResultBuf[reqID]
|
||||
delete(queryResultBuf, reqID)
|
||||
select {
|
||||
case msgPack, ok := <-p.queryResultMsgStream.Chan():
|
||||
if !ok {
|
||||
log.Print("buf chan closed")
|
||||
return
|
||||
}
|
||||
log.Print("Consume message from query result stream...")
|
||||
if msgPack == nil {
|
||||
continue
|
||||
}
|
||||
tsMsg := msgPack.Msgs[0]
|
||||
searchResultMsg, _ := (*tsMsg).(*msgstream.SearchResultMsg)
|
||||
reqID := searchResultMsg.GetReqID()
|
||||
_, ok = queryResultBuf[reqID]
|
||||
if !ok {
|
||||
queryResultBuf[reqID] = make([]*internalpb.SearchResult, 0)
|
||||
}
|
||||
queryResultBuf[reqID] = append(queryResultBuf[reqID], &searchResultMsg.SearchResult)
|
||||
if len(queryResultBuf[reqID]) == 4 {
|
||||
// TODO: use the number of query node instead
|
||||
t := p.taskSch.getTaskByReqID(reqID)
|
||||
qt := t.(*QueryTask)
|
||||
qt.resultBuf <- queryResultBuf[reqID]
|
||||
delete(queryResultBuf, reqID)
|
||||
}
|
||||
case <-p.proxyLoopCtx.Done():
|
||||
log.Print("proxy server is closed ...")
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -189,10 +225,15 @@ func (p *Proxy) stopProxyLoop() {
|
||||
p.grpcServer.GracefulStop()
|
||||
}
|
||||
p.tsoAllocator.Close()
|
||||
|
||||
p.idAllocator.Close()
|
||||
|
||||
p.taskSch.Close()
|
||||
|
||||
p.manipulationMsgStream.Close()
|
||||
|
||||
p.queryMsgStream.Close()
|
||||
|
||||
p.queryResultMsgStream.Close()
|
||||
|
||||
p.proxyLoopWg.Wait()
|
||||
|
||||
89
internal/proxy/proxy_test.go
Normal file
89
internal/proxy/proxy_test.go
Normal file
@ -0,0 +1,89 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
mockMaster "github.com/zilliztech/milvus-distributed/internal/master/mock"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/servicepb"
|
||||
"go.uber.org/zap"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
var ctx context.Context
|
||||
var cancel func()
|
||||
var proxyAddress = "127.0.0.1:5053"
|
||||
var proxyConn *grpc.ClientConn
|
||||
var proxyClient servicepb.MilvusServiceClient
|
||||
|
||||
var proxyServer *Proxy
|
||||
|
||||
var masterServer *mockMaster.Master
|
||||
|
||||
func startMaster(ctx context.Context) {
|
||||
//conf.LoadConfig("config.yaml")
|
||||
fmt.Println("THIS is test before.")
|
||||
svr, err := mockMaster.CreateServer(ctx)
|
||||
masterServer = svr
|
||||
if err != nil {
|
||||
log.Print("create server failed", zap.Error(err))
|
||||
}
|
||||
|
||||
if err := svr.Run(); err != nil {
|
||||
log.Fatal("run server failed", zap.Error(err))
|
||||
}
|
||||
|
||||
fmt.Println("Waiting for server!", svr.IsServing())
|
||||
|
||||
}
|
||||
|
||||
func startProxy(ctx context.Context) {
|
||||
|
||||
svr, err := CreateProxy(ctx)
|
||||
proxyServer = svr
|
||||
if err != nil {
|
||||
log.Print("create proxy failed", zap.Error(err))
|
||||
}
|
||||
|
||||
// TODO: change to wait until master is ready
|
||||
if err := svr.Run(); err != nil {
|
||||
log.Fatal("run proxy failed", zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
func setup() {
|
||||
conf.LoadConfig("config.yaml")
|
||||
ctx, cancel = context.WithCancel(context.Background())
|
||||
|
||||
startMaster(ctx)
|
||||
startProxy(ctx)
|
||||
|
||||
conn, err := grpc.DialContext(ctx, proxyAddress, grpc.WithInsecure(), grpc.WithBlock())
|
||||
if err != nil {
|
||||
log.Fatalf("Connect to proxy failed, error= %v", err)
|
||||
}
|
||||
proxyConn = conn
|
||||
proxyClient = servicepb.NewMilvusServiceClient(proxyConn)
|
||||
|
||||
}
|
||||
|
||||
func shutdown() {
|
||||
cancel()
|
||||
masterServer.Close()
|
||||
proxyServer.Close()
|
||||
}
|
||||
|
||||
func TestProxy_CreateCollection(t *testing.T) {
|
||||
fmt.Println(proxyClient)
|
||||
}
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
setup()
|
||||
code := m.Run()
|
||||
shutdown()
|
||||
os.Exit(code)
|
||||
}
|
||||
@ -31,7 +31,7 @@ type InsertTask struct {
|
||||
BaseInsertTask
|
||||
ts Timestamp
|
||||
done chan error
|
||||
resultChan chan *servicepb.IntegerRangeResponse
|
||||
result *servicepb.IntegerRangeResponse
|
||||
manipulationMsgStream *msgstream.PulsarMsgStream
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
@ -98,7 +98,7 @@ type CreateCollectionTask struct {
|
||||
internalpb.CreateCollectionRequest
|
||||
masterClient masterpb.MasterClient
|
||||
done chan error
|
||||
resultChan chan *commonpb.Status
|
||||
result *commonpb.Status
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
@ -131,12 +131,12 @@ func (cct *CreateCollectionTask) Execute() error {
|
||||
resp, err := cct.masterClient.CreateCollection(cct.ctx, &cct.CreateCollectionRequest)
|
||||
if err != nil {
|
||||
log.Printf("create collection failed, error= %v", err)
|
||||
cct.resultChan <- &commonpb.Status{
|
||||
cct.result = &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: err.Error(),
|
||||
}
|
||||
} else {
|
||||
cct.resultChan <- resp
|
||||
cct.result = resp
|
||||
}
|
||||
return err
|
||||
}
|
||||
@ -166,7 +166,7 @@ type DropCollectionTask struct {
|
||||
internalpb.DropCollectionRequest
|
||||
masterClient masterpb.MasterClient
|
||||
done chan error
|
||||
resultChan chan *commonpb.Status
|
||||
result *commonpb.Status
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
@ -199,12 +199,12 @@ func (dct *DropCollectionTask) Execute() error {
|
||||
resp, err := dct.masterClient.DropCollection(dct.ctx, &dct.DropCollectionRequest)
|
||||
if err != nil {
|
||||
log.Printf("drop collection failed, error= %v", err)
|
||||
dct.resultChan <- &commonpb.Status{
|
||||
dct.result = &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: err.Error(),
|
||||
}
|
||||
} else {
|
||||
dct.resultChan <- resp
|
||||
dct.result = resp
|
||||
}
|
||||
return err
|
||||
}
|
||||
@ -235,7 +235,7 @@ type QueryTask struct {
|
||||
queryMsgStream *msgstream.PulsarMsgStream
|
||||
done chan error
|
||||
resultBuf chan []*internalpb.SearchResult
|
||||
resultChan chan *servicepb.QueryResult
|
||||
result *servicepb.QueryResult
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
@ -312,12 +312,12 @@ func (qt *QueryTask) Notify(err error) {
|
||||
case searchResults := <-qt.resultBuf:
|
||||
rlen := len(searchResults) // query num
|
||||
if rlen <= 0 {
|
||||
qt.resultChan <- &servicepb.QueryResult{}
|
||||
qt.result = &servicepb.QueryResult{}
|
||||
return
|
||||
}
|
||||
n := len(searchResults[0].Hits) // n
|
||||
if n <= 0 {
|
||||
qt.resultChan <- &servicepb.QueryResult{}
|
||||
qt.result = &servicepb.QueryResult{}
|
||||
return
|
||||
}
|
||||
k := len(searchResults[0].Hits[0].IDs) // k
|
||||
@ -354,7 +354,7 @@ func (qt *QueryTask) Notify(err error) {
|
||||
}
|
||||
queryResult.Hits = append(queryResult.Hits, hits)
|
||||
}
|
||||
qt.resultChan <- queryResult
|
||||
qt.result = queryResult
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -363,7 +363,7 @@ type HasCollectionTask struct {
|
||||
internalpb.HasCollectionRequest
|
||||
masterClient masterpb.MasterClient
|
||||
done chan error
|
||||
resultChan chan *servicepb.BoolResponse
|
||||
result *servicepb.BoolResponse
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
@ -396,7 +396,7 @@ func (hct *HasCollectionTask) Execute() error {
|
||||
resp, err := hct.masterClient.HasCollection(hct.ctx, &hct.HasCollectionRequest)
|
||||
if err != nil {
|
||||
log.Printf("has collection failed, error= %v", err)
|
||||
hct.resultChan <- &servicepb.BoolResponse{
|
||||
hct.result = &servicepb.BoolResponse{
|
||||
Status: &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: "internal error",
|
||||
@ -404,7 +404,7 @@ func (hct *HasCollectionTask) Execute() error {
|
||||
Value: false,
|
||||
}
|
||||
} else {
|
||||
hct.resultChan <- resp
|
||||
hct.result = resp
|
||||
}
|
||||
return err
|
||||
}
|
||||
@ -434,7 +434,7 @@ type DescribeCollectionTask struct {
|
||||
internalpb.DescribeCollectionRequest
|
||||
masterClient masterpb.MasterClient
|
||||
done chan error
|
||||
resultChan chan *servicepb.CollectionDescription
|
||||
result *servicepb.CollectionDescription
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
@ -467,14 +467,14 @@ func (dct *DescribeCollectionTask) Execute() error {
|
||||
resp, err := dct.masterClient.DescribeCollection(dct.ctx, &dct.DescribeCollectionRequest)
|
||||
if err != nil {
|
||||
log.Printf("describe collection failed, error= %v", err)
|
||||
dct.resultChan <- &servicepb.CollectionDescription{
|
||||
dct.result = &servicepb.CollectionDescription{
|
||||
Status: &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: "internal error",
|
||||
},
|
||||
}
|
||||
} else {
|
||||
dct.resultChan <- resp
|
||||
dct.result = resp
|
||||
}
|
||||
return err
|
||||
}
|
||||
@ -504,7 +504,7 @@ type ShowCollectionsTask struct {
|
||||
internalpb.ShowCollectionRequest
|
||||
masterClient masterpb.MasterClient
|
||||
done chan error
|
||||
resultChan chan *servicepb.StringListResponse
|
||||
result *servicepb.StringListResponse
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
@ -537,14 +537,14 @@ func (sct *ShowCollectionsTask) Execute() error {
|
||||
resp, err := sct.masterClient.ShowCollections(sct.ctx, &sct.ShowCollectionRequest)
|
||||
if err != nil {
|
||||
log.Printf("show collections failed, error= %v", err)
|
||||
sct.resultChan <- &servicepb.StringListResponse{
|
||||
sct.result = &servicepb.StringListResponse{
|
||||
Status: &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: "internal error",
|
||||
},
|
||||
}
|
||||
} else {
|
||||
sct.resultChan <- resp
|
||||
sct.result = resp
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@ -3,73 +3,113 @@ package proxy
|
||||
import (
|
||||
"container/list"
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"sync"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/allocator"
|
||||
)
|
||||
|
||||
type TaskQueue interface {
|
||||
utChan() <-chan int
|
||||
utEmpty() bool
|
||||
utFull() bool
|
||||
addUnissuedTask(t task) error
|
||||
FrontUnissuedTask() task
|
||||
PopUnissuedTask() task
|
||||
AddActiveTask(t task)
|
||||
PopActiveTask(ts Timestamp) task
|
||||
getTaskByReqID(reqID UniqueID) task
|
||||
TaskDoneTest(ts Timestamp) bool
|
||||
Enqueue(t task) error
|
||||
}
|
||||
|
||||
type BaseTaskQueue struct {
|
||||
unissuedTasks *list.List
|
||||
activeTasks map[Timestamp]task
|
||||
utLock sync.Mutex
|
||||
atLock sync.Mutex
|
||||
|
||||
// maxTaskNum should keep still
|
||||
maxTaskNum int64
|
||||
|
||||
utBufChan chan int // to block scheduler
|
||||
}
|
||||
|
||||
func (queue *BaseTaskQueue) Empty() bool {
|
||||
queue.utLock.Lock()
|
||||
defer queue.utLock.Unlock()
|
||||
queue.atLock.Lock()
|
||||
defer queue.atLock.Unlock()
|
||||
return queue.unissuedTasks.Len() <= 0 && len(queue.activeTasks) <= 0
|
||||
func (queue *BaseTaskQueue) utChan() <-chan int {
|
||||
return queue.utBufChan
|
||||
}
|
||||
|
||||
func (queue *BaseTaskQueue) AddUnissuedTask(t task) {
|
||||
func (queue *BaseTaskQueue) utEmpty() bool {
|
||||
return queue.unissuedTasks.Len() == 0
|
||||
}
|
||||
|
||||
func (queue *BaseTaskQueue) utFull() bool {
|
||||
return int64(queue.unissuedTasks.Len()) >= queue.maxTaskNum
|
||||
}
|
||||
|
||||
func (queue *BaseTaskQueue) addUnissuedTask(t task) error {
|
||||
queue.utLock.Lock()
|
||||
defer queue.utLock.Unlock()
|
||||
|
||||
if queue.utFull() {
|
||||
return errors.New("task queue is full")
|
||||
}
|
||||
queue.unissuedTasks.PushBack(t)
|
||||
queue.utBufChan <- 1
|
||||
return nil
|
||||
}
|
||||
|
||||
func (queue *BaseTaskQueue) FrontUnissuedTask() task {
|
||||
queue.utLock.Lock()
|
||||
defer queue.utLock.Unlock()
|
||||
|
||||
if queue.unissuedTasks.Len() <= 0 {
|
||||
log.Fatal("sorry, but the unissued task list is empty!")
|
||||
log.Panic("sorry, but the unissued task list is empty!")
|
||||
return nil
|
||||
}
|
||||
|
||||
return queue.unissuedTasks.Front().Value.(task)
|
||||
}
|
||||
|
||||
func (queue *BaseTaskQueue) PopUnissuedTask() task {
|
||||
queue.utLock.Lock()
|
||||
defer queue.utLock.Unlock()
|
||||
|
||||
if queue.unissuedTasks.Len() <= 0 {
|
||||
log.Fatal("sorry, but the unissued task list is empty!")
|
||||
return nil
|
||||
}
|
||||
|
||||
ft := queue.unissuedTasks.Front()
|
||||
return queue.unissuedTasks.Remove(ft).(task)
|
||||
queue.unissuedTasks.Remove(ft)
|
||||
|
||||
return ft.Value.(task)
|
||||
}
|
||||
|
||||
func (queue *BaseTaskQueue) AddActiveTask(t task) {
|
||||
queue.atLock.Lock()
|
||||
defer queue.atLock.Lock()
|
||||
|
||||
ts := t.EndTs()
|
||||
_, ok := queue.activeTasks[ts]
|
||||
if ok {
|
||||
log.Fatalf("task with timestamp %v already in active task list!", ts)
|
||||
}
|
||||
|
||||
queue.activeTasks[ts] = t
|
||||
}
|
||||
|
||||
func (queue *BaseTaskQueue) PopActiveTask(ts Timestamp) task {
|
||||
queue.atLock.Lock()
|
||||
defer queue.atLock.Lock()
|
||||
|
||||
t, ok := queue.activeTasks[ts]
|
||||
if ok {
|
||||
delete(queue.activeTasks, ts)
|
||||
return t
|
||||
}
|
||||
|
||||
log.Fatalf("sorry, but the timestamp %d was not found in the active task list!", ts)
|
||||
return nil
|
||||
}
|
||||
@ -114,6 +154,11 @@ func (queue *BaseTaskQueue) TaskDoneTest(ts Timestamp) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (queue *BaseTaskQueue) Enqueue(t task) error {
|
||||
// TODO: set Ts, ReqId, ProxyId
|
||||
return queue.addUnissuedTask(t)
|
||||
}
|
||||
|
||||
type DdTaskQueue struct {
|
||||
BaseTaskQueue
|
||||
lock sync.Mutex
|
||||
@ -128,23 +173,13 @@ type DqTaskQueue struct {
|
||||
}
|
||||
|
||||
func (queue *DdTaskQueue) Enqueue(t task) error {
|
||||
|
||||
queue.lock.Lock()
|
||||
defer queue.lock.Unlock()
|
||||
// TODO: set Ts, ReqID, ProxyID
|
||||
queue.AddUnissuedTask(t)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (queue *DmTaskQueue) Enqueue(t task) error {
|
||||
// TODO: set Ts, ReqID, ProxyID
|
||||
queue.AddUnissuedTask(t)
|
||||
return nil
|
||||
}
|
||||
// TODO: set Ts, ReqId, ProxyId
|
||||
return queue.addUnissuedTask(t)
|
||||
|
||||
func (queue *DqTaskQueue) Enqueue(t task) error {
|
||||
// TODO: set Ts, ReqID, ProxyID
|
||||
queue.AddUnissuedTask(t)
|
||||
return nil
|
||||
}
|
||||
|
||||
func NewDdTaskQueue() *DdTaskQueue {
|
||||
@ -152,6 +187,8 @@ func NewDdTaskQueue() *DdTaskQueue {
|
||||
BaseTaskQueue: BaseTaskQueue{
|
||||
unissuedTasks: list.New(),
|
||||
activeTasks: make(map[Timestamp]task),
|
||||
maxTaskNum: 1024,
|
||||
utBufChan: make(chan int, 1024),
|
||||
},
|
||||
}
|
||||
}
|
||||
@ -161,6 +198,8 @@ func NewDmTaskQueue() *DmTaskQueue {
|
||||
BaseTaskQueue: BaseTaskQueue{
|
||||
unissuedTasks: list.New(),
|
||||
activeTasks: make(map[Timestamp]task),
|
||||
maxTaskNum: 1024,
|
||||
utBufChan: make(chan int, 1024),
|
||||
},
|
||||
}
|
||||
}
|
||||
@ -170,14 +209,16 @@ func NewDqTaskQueue() *DqTaskQueue {
|
||||
BaseTaskQueue: BaseTaskQueue{
|
||||
unissuedTasks: list.New(),
|
||||
activeTasks: make(map[Timestamp]task),
|
||||
maxTaskNum: 1024,
|
||||
utBufChan: make(chan int, 1024),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
type TaskScheduler struct {
|
||||
DdQueue *DdTaskQueue
|
||||
DmQueue *DmTaskQueue
|
||||
DqQueue *DqTaskQueue
|
||||
DdQueue TaskQueue
|
||||
DmQueue TaskQueue
|
||||
DqQueue TaskQueue
|
||||
|
||||
idAllocator *allocator.IDAllocator
|
||||
tsoAllocator *allocator.TimestampAllocator
|
||||
@ -229,116 +270,82 @@ func (sched *TaskScheduler) getTaskByReqID(collMeta UniqueID) task {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sched *TaskScheduler) processTask(t task, q TaskQueue) {
|
||||
|
||||
err := t.PreExecute()
|
||||
|
||||
defer func() {
|
||||
t.Notify(err)
|
||||
}()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
q.AddActiveTask(t)
|
||||
defer q.PopActiveTask(t.EndTs())
|
||||
|
||||
err = t.Execute()
|
||||
if err != nil {
|
||||
log.Printf("execute definition task failed, error = %v", err)
|
||||
return
|
||||
}
|
||||
err = t.PostExecute()
|
||||
}
|
||||
|
||||
func (sched *TaskScheduler) definitionLoop() {
|
||||
defer sched.wg.Done()
|
||||
defer sched.cancel()
|
||||
|
||||
for {
|
||||
if sched.DdQueue.Empty() {
|
||||
continue
|
||||
}
|
||||
|
||||
//sched.DdQueue.atLock.Lock()
|
||||
t := sched.scheduleDdTask()
|
||||
|
||||
err := t.PreExecute()
|
||||
if err != nil {
|
||||
select {
|
||||
case <-sched.ctx.Done():
|
||||
return
|
||||
case <-sched.DdQueue.utChan():
|
||||
if !sched.DdQueue.utEmpty() {
|
||||
t := sched.scheduleDdTask()
|
||||
sched.processTask(t, sched.DdQueue)
|
||||
}
|
||||
}
|
||||
err = t.Execute()
|
||||
if err != nil {
|
||||
log.Printf("execute definition task failed, error = %v", err)
|
||||
}
|
||||
t.Notify(err)
|
||||
|
||||
sched.DdQueue.AddActiveTask(t)
|
||||
|
||||
t.WaitToFinish()
|
||||
t.PostExecute()
|
||||
|
||||
sched.DdQueue.PopActiveTask(t.EndTs())
|
||||
}
|
||||
}
|
||||
|
||||
func (sched *TaskScheduler) manipulationLoop() {
|
||||
defer sched.wg.Done()
|
||||
defer sched.cancel()
|
||||
|
||||
for {
|
||||
if sched.DmQueue.Empty() {
|
||||
continue
|
||||
}
|
||||
|
||||
sched.DmQueue.atLock.Lock()
|
||||
t := sched.scheduleDmTask()
|
||||
|
||||
if err := t.PreExecute(); err != nil {
|
||||
select {
|
||||
case <-sched.ctx.Done():
|
||||
return
|
||||
}
|
||||
|
||||
go func() {
|
||||
err := t.Execute()
|
||||
if err != nil {
|
||||
log.Printf("execute manipulation task failed, error = %v", err)
|
||||
case <-sched.DmQueue.utChan():
|
||||
if !sched.DmQueue.utEmpty() {
|
||||
t := sched.scheduleDmTask()
|
||||
go sched.processTask(t, sched.DmQueue)
|
||||
}
|
||||
t.Notify(err)
|
||||
}()
|
||||
|
||||
sched.DmQueue.AddActiveTask(t)
|
||||
sched.DmQueue.atLock.Unlock()
|
||||
|
||||
go func() {
|
||||
t.WaitToFinish()
|
||||
t.PostExecute()
|
||||
|
||||
// remove from active list
|
||||
sched.DmQueue.PopActiveTask(t.EndTs())
|
||||
}()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (sched *TaskScheduler) queryLoop() {
|
||||
defer sched.wg.Done()
|
||||
defer sched.cancel()
|
||||
|
||||
for {
|
||||
if sched.DqQueue.Empty() {
|
||||
continue
|
||||
}
|
||||
|
||||
sched.DqQueue.atLock.Lock()
|
||||
t := sched.scheduleDqTask()
|
||||
|
||||
if err := t.PreExecute(); err != nil {
|
||||
select {
|
||||
case <-sched.ctx.Done():
|
||||
return
|
||||
}
|
||||
|
||||
go func() {
|
||||
err := t.Execute()
|
||||
if err != nil {
|
||||
log.Printf("execute query task failed, error = %v", err)
|
||||
case <-sched.DqQueue.utChan():
|
||||
if !sched.DqQueue.utEmpty() {
|
||||
t := sched.scheduleDqTask()
|
||||
go sched.processTask(t, sched.DqQueue)
|
||||
}
|
||||
t.Notify(err)
|
||||
}()
|
||||
|
||||
sched.DqQueue.AddActiveTask(t)
|
||||
sched.DqQueue.atLock.Unlock()
|
||||
|
||||
go func() {
|
||||
t.WaitToFinish()
|
||||
t.PostExecute()
|
||||
|
||||
// remove from active list
|
||||
sched.DqQueue.PopActiveTask(t.EndTs())
|
||||
}()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (sched *TaskScheduler) Start() error {
|
||||
sched.wg.Add(3)
|
||||
|
||||
sched.wg.Add(1)
|
||||
go sched.definitionLoop()
|
||||
|
||||
sched.wg.Add(1)
|
||||
go sched.manipulationLoop()
|
||||
|
||||
sched.wg.Add(1)
|
||||
go sched.queryLoop()
|
||||
|
||||
return nil
|
||||
|
||||
@ -3,9 +3,12 @@ package proxy
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/allocator"
|
||||
"github.com/zilliztech/milvus-distributed/internal/msgstream"
|
||||
|
||||
@ -14,10 +17,12 @@ import (
|
||||
"github.com/apache/pulsar-client-go/pulsar"
|
||||
)
|
||||
|
||||
type tickCheckFunc = func(Timestamp) bool
|
||||
|
||||
type timeTick struct {
|
||||
lastTick Timestamp
|
||||
currentTick Timestamp
|
||||
interval uint64
|
||||
interval int64
|
||||
|
||||
pulsarProducer pulsar.Producer
|
||||
|
||||
@ -30,21 +35,29 @@ type timeTick struct {
|
||||
cancel func()
|
||||
timer *time.Ticker
|
||||
|
||||
areRequestsDelivered func(ts Timestamp) bool
|
||||
checkFunc tickCheckFunc
|
||||
}
|
||||
|
||||
func newTimeTick(ctx context.Context, tsoAllocator *allocator.TimestampAllocator) *timeTick {
|
||||
func newTimeTick(ctx context.Context,
|
||||
tsoAllocator *allocator.TimestampAllocator,
|
||||
checkFunc tickCheckFunc) *timeTick {
|
||||
ctx1, cancel := context.WithCancel(ctx)
|
||||
t := &timeTick{
|
||||
ctx: ctx1,
|
||||
cancel: cancel,
|
||||
tsoAllocator: tsoAllocator,
|
||||
interval: 200,
|
||||
peerID: 1,
|
||||
checkFunc: checkFunc,
|
||||
}
|
||||
|
||||
bufSzie := int64(1000)
|
||||
t.tickMsgStream = msgstream.NewPulsarMsgStream(t.ctx, bufSzie)
|
||||
bufSize := int64(1000)
|
||||
t.tickMsgStream = msgstream.NewPulsarMsgStream(t.ctx, bufSize)
|
||||
pulsarAddress := "pulsar://"
|
||||
pulsarAddress += conf.Config.Pulsar.Address
|
||||
pulsarAddress += ":"
|
||||
pulsarAddress += strconv.FormatInt(int64(conf.Config.Pulsar.Port), 10)
|
||||
|
||||
pulsarAddress := "pulsar://localhost:6650"
|
||||
producerChannels := []string{"timeTick"}
|
||||
t.tickMsgStream.SetPulsarCient(pulsarAddress)
|
||||
t.tickMsgStream.CreatePulsarProducers(producerChannels)
|
||||
@ -61,7 +74,7 @@ func (tt *timeTick) tick() error {
|
||||
tt.currentTick = ts
|
||||
}
|
||||
|
||||
if !tt.areRequestsDelivered(tt.currentTick) {
|
||||
if !tt.checkFunc(tt.currentTick) {
|
||||
return nil
|
||||
}
|
||||
msgPack := msgstream.MsgPack{}
|
||||
|
||||
@ -2,84 +2,48 @@ package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/apache/pulsar-client-go/pulsar"
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/allocator"
|
||||
)
|
||||
|
||||
func TestTimeTick(t *testing.T) {
|
||||
client, err := pulsar.NewClient(pulsar.ClientOptions{URL: "pulsar://localhost:6650"})
|
||||
assert.Nil(t, err)
|
||||
var trueCnt = 0
|
||||
|
||||
producer, err := client.CreateProducer(pulsar.ProducerOptions{Topic: "timesync"})
|
||||
assert.Nil(t, err)
|
||||
|
||||
consumer, err := client.Subscribe(pulsar.ConsumerOptions{
|
||||
Topic: "timesync",
|
||||
SubscriptionName: "timesync_group",
|
||||
Type: pulsar.KeyShared,
|
||||
SubscriptionInitialPosition: pulsar.SubscriptionPositionEarliest,
|
||||
})
|
||||
assert.Nil(t, err)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second)
|
||||
defer cancel()
|
||||
|
||||
//var curTs Timestamp
|
||||
//curTs = 0
|
||||
tt := timeTick{
|
||||
interval: 200,
|
||||
pulsarProducer: producer,
|
||||
peerID: 1,
|
||||
ctx: ctx,
|
||||
areRequestsDelivered: func(ts Timestamp) bool { return true },
|
||||
func checkFunc(timestamp Timestamp) bool {
|
||||
ret := rand.Intn(2) == 1
|
||||
if ret {
|
||||
trueCnt++
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
func TestTimeTick_Start(t *testing.T) {
|
||||
fmt.Println("HHH")
|
||||
}
|
||||
|
||||
func TestTimeTick_Start2(t *testing.T) {
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
|
||||
tsoAllocator, err := allocator.NewTimestampAllocator(ctx)
|
||||
assert.Nil(t, err)
|
||||
err = tsoAllocator.Start()
|
||||
assert.Nil(t, err)
|
||||
|
||||
tt := newTimeTick(ctx, tsoAllocator, checkFunc)
|
||||
|
||||
defer func() {
|
||||
cancel()
|
||||
tsoAllocator.Close()
|
||||
tt.Close()
|
||||
}()
|
||||
|
||||
tt.Start()
|
||||
|
||||
ctx2, cancel2 := context.WithTimeout(context.Background(), time.Second*2)
|
||||
defer cancel2()
|
||||
<-ctx.Done()
|
||||
|
||||
isbreak := false
|
||||
for {
|
||||
if isbreak {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-ctx2.Done():
|
||||
isbreak = true
|
||||
break
|
||||
case cm, ok := <-consumer.Chan():
|
||||
if !ok {
|
||||
t.Fatalf("consumer closed")
|
||||
}
|
||||
consumer.AckID(cm.ID())
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
var lastTimestamp uint64 = 0
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case cm, ok := <-consumer.Chan():
|
||||
if ok == false {
|
||||
return
|
||||
}
|
||||
msg := cm.Message
|
||||
var tsm internalpb.TimeTickMsg
|
||||
if err := proto.Unmarshal(msg.Payload(), &tsm); err != nil {
|
||||
return
|
||||
}
|
||||
if tsm.Timestamp <= lastTimestamp {
|
||||
t.Fatalf("current = %d, last = %d", tsm.Timestamp, lastTimestamp)
|
||||
}
|
||||
t.Log("current = ", tsm.Timestamp)
|
||||
lastTimestamp = tsm.Timestamp
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
32
internal/proxy/util.go
Normal file
32
internal/proxy/util.go
Normal file
@ -0,0 +1,32 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"log"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Reference: https://blog.cyeam.com/golang/2018/08/27/retry
|
||||
|
||||
func Retry(attempts int, sleep time.Duration, fn func() error) error {
|
||||
if err := fn(); err != nil {
|
||||
if s, ok := err.(InterruptError); ok {
|
||||
return s.error
|
||||
}
|
||||
|
||||
if attempts--; attempts > 0 {
|
||||
log.Printf("retry func error: %s. attempts #%d after %s.", err.Error(), attempts, sleep)
|
||||
time.Sleep(sleep)
|
||||
return Retry(attempts, 2*sleep, fn)
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type InterruptError struct {
|
||||
error
|
||||
}
|
||||
|
||||
func NoRetryError(err error) InterruptError {
|
||||
return InterruptError{err}
|
||||
}
|
||||
@ -12,9 +12,9 @@ import (
|
||||
|
||||
"github.com/golang/protobuf/proto"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
"github.com/zilliztech/milvus-distributed/internal/kv"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/etcdpb"
|
||||
gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil"
|
||||
"go.etcd.io/etcd/clientv3"
|
||||
"go.etcd.io/etcd/mvcc/mvccpb"
|
||||
)
|
||||
@ -31,10 +31,19 @@ type metaService struct {
|
||||
}
|
||||
|
||||
func newMetaService(ctx context.Context, container *container) *metaService {
|
||||
ETCDAddr := "http://"
|
||||
ETCDAddr += conf.Config.Etcd.Address
|
||||
ETCDPort := conf.Config.Etcd.Port
|
||||
ETCDAddr = ETCDAddr + ":" + strconv.FormatInt(int64(ETCDPort), 10)
|
||||
ETCDAddr, err := gparams.GParams.Load("etcd.address")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
ETCDPort, err := gparams.GParams.Load("etcd.port")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
ETCDAddr = "http://" + ETCDAddr + ":" + ETCDPort
|
||||
ETCDRootPath, err := gparams.GParams.Load("etcd.rootpath")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
cli, _ := clientv3.New(clientv3.Config{
|
||||
Endpoints: []string{ETCDAddr},
|
||||
@ -43,7 +52,7 @@ func newMetaService(ctx context.Context, container *container) *metaService {
|
||||
|
||||
return &metaService{
|
||||
ctx: ctx,
|
||||
kvBase: kv.NewEtcdKV(cli, conf.Config.Etcd.Rootpath),
|
||||
kvBase: kv.NewEtcdKV(cli, ETCDRootPath),
|
||||
container: container,
|
||||
}
|
||||
}
|
||||
@ -74,17 +83,29 @@ func (mService *metaService) start() {
|
||||
}
|
||||
|
||||
func GetCollectionObjID(key string) string {
|
||||
prefix := path.Join(conf.Config.Etcd.Rootpath, CollectionPrefix) + "/"
|
||||
ETCDRootPath, err := gparams.GParams.Load("etcd.rootpath")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
prefix := path.Join(ETCDRootPath, CollectionPrefix) + "/"
|
||||
return strings.TrimPrefix(key, prefix)
|
||||
}
|
||||
|
||||
func GetSegmentObjID(key string) string {
|
||||
prefix := path.Join(conf.Config.Etcd.Rootpath, SegmentPrefix) + "/"
|
||||
ETCDRootPath, err := gparams.GParams.Load("etcd.rootpath")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
prefix := path.Join(ETCDRootPath, SegmentPrefix) + "/"
|
||||
return strings.TrimPrefix(key, prefix)
|
||||
}
|
||||
|
||||
func isCollectionObj(key string) bool {
|
||||
prefix := path.Join(conf.Config.Etcd.Rootpath, CollectionPrefix) + "/"
|
||||
ETCDRootPath, err := gparams.GParams.Load("etcd.rootpath")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
prefix := path.Join(ETCDRootPath, CollectionPrefix) + "/"
|
||||
prefix = strings.TrimSpace(prefix)
|
||||
index := strings.Index(key, prefix)
|
||||
|
||||
@ -92,7 +113,11 @@ func isCollectionObj(key string) bool {
|
||||
}
|
||||
|
||||
func isSegmentObj(key string) bool {
|
||||
prefix := path.Join(conf.Config.Etcd.Rootpath, SegmentPrefix) + "/"
|
||||
ETCDRootPath, err := gparams.GParams.Load("etcd.rootpath")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
prefix := path.Join(ETCDRootPath, SegmentPrefix) + "/"
|
||||
prefix = strings.TrimSpace(prefix)
|
||||
index := strings.Index(key, prefix)
|
||||
|
||||
@ -105,8 +130,24 @@ func isSegmentChannelRangeInQueryNodeChannelRange(segment *etcdpb.SegmentMeta) b
|
||||
return false
|
||||
}
|
||||
|
||||
var queryNodeChannelStart = conf.Config.Reader.TopicStart
|
||||
var queryNodeChannelEnd = conf.Config.Reader.TopicEnd
|
||||
readerTopicStart, err := gparams.GParams.Load("reader.topicstart")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
TopicStart, err := strconv.Atoi(readerTopicStart)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
readerTopicEnd, err := gparams.GParams.Load("reader.topicend")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
TopicEnd, err := strconv.Atoi(readerTopicEnd)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
var queryNodeChannelStart = TopicStart
|
||||
var queryNodeChannelEnd = TopicEnd
|
||||
|
||||
if segment.ChannelStart >= int32(queryNodeChannelStart) && segment.ChannelEnd <= int32(queryNodeChannelEnd) {
|
||||
return true
|
||||
|
||||
@ -3,16 +3,17 @@ package s3driver
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
. "github.com/zilliztech/milvus-distributed/internal/storage/type"
|
||||
gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil"
|
||||
)
|
||||
|
||||
var bucketName = conf.Config.Writer.Bucket
|
||||
var bucketName string
|
||||
|
||||
type S3Store struct {
|
||||
client *s3.S3
|
||||
@ -21,6 +22,16 @@ type S3Store struct {
|
||||
func NewS3Store(config aws.Config) (*S3Store, error) {
|
||||
sess := session.Must(session.NewSession(&config))
|
||||
service := s3.New(sess)
|
||||
err := gparams.GParams.LoadYaml("config.yaml")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
bucketName, err := gparams.GParams.Load("writer.bucket")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println(bucketName)
|
||||
|
||||
return &S3Store{
|
||||
client: service,
|
||||
}, nil
|
||||
|
||||
@ -7,12 +7,15 @@ import (
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/zilliztech/milvus-distributed/internal/storage/internal/minio/codec"
|
||||
storageType "github.com/zilliztech/milvus-distributed/internal/storage/type"
|
||||
gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil"
|
||||
)
|
||||
|
||||
type MinioDriver struct {
|
||||
driver *minioStore
|
||||
}
|
||||
|
||||
var bucketName string
|
||||
|
||||
func NewMinioDriver(ctx context.Context) (*MinioDriver, error) {
|
||||
// to-do read conf
|
||||
var endPoint = "localhost:9000"
|
||||
@ -20,6 +23,14 @@ func NewMinioDriver(ctx context.Context) (*MinioDriver, error) {
|
||||
var secretAccessKey = "testminio"
|
||||
var useSSL = false
|
||||
|
||||
err := gparams.GParams.LoadYaml("config.yaml")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
bucketName, err := gparams.GParams.Load("writer.bucket")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
minioClient, err := minio.New(endPoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
|
||||
Secure: useSSL,
|
||||
|
||||
@ -6,12 +6,9 @@ import (
|
||||
"io"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
. "github.com/zilliztech/milvus-distributed/internal/storage/type"
|
||||
)
|
||||
|
||||
var bucketName = conf.Config.Writer.Bucket
|
||||
|
||||
type minioStore struct {
|
||||
client *minio.Client
|
||||
}
|
||||
|
||||
@ -8,9 +8,9 @@ import (
|
||||
|
||||
"github.com/tikv/client-go/config"
|
||||
"github.com/tikv/client-go/rawkv"
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
. "github.com/zilliztech/milvus-distributed/internal/storage/internal/tikv/codec"
|
||||
. "github.com/zilliztech/milvus-distributed/internal/storage/type"
|
||||
gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil"
|
||||
)
|
||||
|
||||
func keyAddOne(key Key) Key {
|
||||
@ -88,11 +88,21 @@ type TikvStore struct {
|
||||
}
|
||||
|
||||
func NewTikvStore(ctx context.Context) (*TikvStore, error) {
|
||||
conf.LoadConfig("config.yaml")
|
||||
var pdAddress0 = conf.Config.Storage.Address + ":" + strconv.FormatInt(int64(conf.Config.Storage.Port), 10)
|
||||
pdAddrs := []string{pdAddress0}
|
||||
err := gparams.GParams.LoadYaml("config.yaml")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
pdAddress, err := gparams.GParams.Load("storage.address")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
pdPort, err := gparams.GParams.Load("storage.port")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
pdAddress = pdAddress + ":" + pdPort
|
||||
conf := config.Default()
|
||||
client, err := rawkv.NewClient(ctx, pdAddrs, conf)
|
||||
client, err := rawkv.NewClient(ctx, []string{pdAddress}, conf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"log"
|
||||
"math"
|
||||
"math/big"
|
||||
"math/rand"
|
||||
"sync"
|
||||
"testing"
|
||||
@ -137,7 +138,8 @@ func sendMsgFromCmd(ctx context.Context, fg *TimeTickedFlowGraph) {
|
||||
return
|
||||
}
|
||||
// assert result
|
||||
if res-(math.Pow(num, 2)+math.Sqrt(num)) > 0.000001 {
|
||||
expect := math.Pow(num, 2) + math.Sqrt(num)
|
||||
if big.NewFloat(res) != big.NewFloat(expect) {
|
||||
fmt.Println(res)
|
||||
fmt.Println(math.Pow(num, 2) + math.Sqrt(num))
|
||||
panic("wrong answer")
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
package kv
|
||||
package kvutil
|
||||
|
||||
type Base interface {
|
||||
Load(key string) (string, error)
|
||||
@ -1,4 +1,4 @@
|
||||
package kv
|
||||
package kvutil
|
||||
|
||||
import (
|
||||
"sync"
|
||||
20
internal/util/paramtableutil/kv.go
Normal file
20
internal/util/paramtableutil/kv.go
Normal file
@ -0,0 +1,20 @@
|
||||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
package paramtableutil
|
||||
|
||||
type Base interface {
|
||||
Load(key string) (string, error)
|
||||
LoadRange(key, endKey string, limit int) ([]string, []string, error)
|
||||
LoadYaml(fileName string) error
|
||||
Remove(key string) error
|
||||
Save(key, value string) error
|
||||
}
|
||||
66
internal/util/paramtableutil/paramtable.go
Normal file
66
internal/util/paramtableutil/paramtable.go
Normal file
@ -0,0 +1,66 @@
|
||||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
package paramtableutil
|
||||
|
||||
import (
|
||||
"path"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/kvutil"
|
||||
)
|
||||
|
||||
type GlobalParamsTable struct {
|
||||
params *kvutil.MemoryKV
|
||||
}
|
||||
|
||||
func NewGlobalParamsTable() Base {
|
||||
return &GlobalParamsTable{params: kvutil.NewMemoryKV()}
|
||||
}
|
||||
|
||||
var GParams = NewGlobalParamsTable()
|
||||
|
||||
func (gparams *GlobalParamsTable) Load(key string) (string, error) {
|
||||
return gparams.params.Load(strings.ToLower(key))
|
||||
}
|
||||
|
||||
func (gparams *GlobalParamsTable) LoadRange(key, endKey string, limit int) ([]string, []string, error) {
|
||||
return gparams.params.LoadRange(strings.ToLower(key), strings.ToLower(endKey), limit)
|
||||
}
|
||||
|
||||
func (gparams *GlobalParamsTable) LoadYaml(fileName string) error {
|
||||
config := viper.New()
|
||||
_, fpath, _, _ := runtime.Caller(0)
|
||||
configPath := path.Dir(fpath) + "/../../../configs/"
|
||||
config.SetConfigFile(configPath + fileName)
|
||||
if err := config.ReadInConfig(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
for _, key := range config.AllKeys() {
|
||||
err := gparams.params.Save(strings.ToLower(key), config.GetString(key))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (gparams *GlobalParamsTable) Remove(key string) error {
|
||||
return gparams.params.Remove(strings.ToLower(key))
|
||||
}
|
||||
|
||||
func (gparams *GlobalParamsTable) Save(key, value string) error {
|
||||
return gparams.params.Save(strings.ToLower(key), value)
|
||||
}
|
||||
114
internal/util/paramtableutil/paramtable_test.go
Normal file
114
internal/util/paramtableutil/paramtable_test.go
Normal file
@ -0,0 +1,114 @@
|
||||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
package paramtableutil
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
var param = NewGlobalParamsTable()
|
||||
|
||||
func TestGlobalParamsTable_SaveAndLoad(t *testing.T) {
|
||||
err1 := param.Save("int", "10")
|
||||
assert.Nil(t, err1)
|
||||
|
||||
err2 := param.Save("string", "testSaveAndLoad")
|
||||
assert.Nil(t, err2)
|
||||
|
||||
err3 := param.Save("float", "1.234")
|
||||
assert.Nil(t, err3)
|
||||
|
||||
r1, _ := param.Load("int")
|
||||
assert.Equal(t, "10", r1)
|
||||
|
||||
r2, _ := param.Load("string")
|
||||
assert.Equal(t, "testSaveAndLoad", r2)
|
||||
|
||||
r3, _ := param.Load("float")
|
||||
assert.Equal(t, "1.234", r3)
|
||||
|
||||
err4 := param.Remove("int")
|
||||
assert.Nil(t, err4)
|
||||
|
||||
err5 := param.Remove("string")
|
||||
assert.Nil(t, err5)
|
||||
|
||||
err6 := param.Remove("float")
|
||||
assert.Nil(t, err6)
|
||||
}
|
||||
|
||||
func TestGlobalParamsTable_LoadRange(t *testing.T) {
|
||||
_ = param.Save("abc", "10")
|
||||
_ = param.Save("fghz", "20")
|
||||
_ = param.Save("bcde", "1.1")
|
||||
_ = param.Save("abcd", "testSaveAndLoad")
|
||||
_ = param.Save("zhi", "12")
|
||||
|
||||
keys, values, err := param.LoadRange("a", "g", 10)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 4, len(keys))
|
||||
assert.Equal(t, "10", values[0])
|
||||
assert.Equal(t, "testSaveAndLoad", values[1])
|
||||
assert.Equal(t, "1.1", values[2])
|
||||
assert.Equal(t, "20", values[3])
|
||||
|
||||
_ = param.Remove("abc")
|
||||
_ = param.Remove("fghz")
|
||||
_ = param.Remove("bcde")
|
||||
_ = param.Remove("abcd")
|
||||
_ = param.Remove("zhi")
|
||||
}
|
||||
|
||||
func TestGlobalParamsTable_Remove(t *testing.T) {
|
||||
err1 := param.Save("RemoveInt", "10")
|
||||
assert.Nil(t, err1)
|
||||
|
||||
err2 := param.Save("RemoveString", "testRemove")
|
||||
assert.Nil(t, err2)
|
||||
|
||||
err3 := param.Save("RemoveFloat", "1.234")
|
||||
assert.Nil(t, err3)
|
||||
|
||||
err4 := param.Remove("RemoveInt")
|
||||
assert.Nil(t, err4)
|
||||
|
||||
err5 := param.Remove("RemoveString")
|
||||
assert.Nil(t, err5)
|
||||
|
||||
err6 := param.Remove("RemoveFloat")
|
||||
assert.Nil(t, err6)
|
||||
}
|
||||
|
||||
func TestGlobalParamsTable_LoadYaml(t *testing.T) {
|
||||
err := param.LoadYaml("config.yaml")
|
||||
assert.Nil(t, err)
|
||||
|
||||
value1, err1 := param.Load("etcd.address")
|
||||
value2, err2 := param.Load("pulsar.port")
|
||||
value3, err3 := param.Load("reader.topicend")
|
||||
value4, err4 := param.Load("proxy.pulsarTopics.readerTopicPrefix")
|
||||
value5, err5 := param.Load("proxy.network.address")
|
||||
|
||||
assert.Equal(t, value1, "localhost")
|
||||
assert.Equal(t, value2, "6650")
|
||||
assert.Equal(t, value3, "128")
|
||||
assert.Equal(t, value4, "milvusReader")
|
||||
assert.Equal(t, value5, "0.0.0.0")
|
||||
|
||||
assert.Nil(t, err1)
|
||||
assert.Nil(t, err2)
|
||||
assert.Nil(t, err3)
|
||||
assert.Nil(t, err4)
|
||||
assert.Nil(t, err5)
|
||||
}
|
||||
@ -3,11 +3,10 @@ package tsoutil
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
"github.com/zilliztech/milvus-distributed/internal/kv"
|
||||
gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil"
|
||||
"go.etcd.io/etcd/clientv3"
|
||||
)
|
||||
|
||||
@ -29,13 +28,24 @@ func ParseTS(ts uint64) (time.Time, uint64) {
|
||||
}
|
||||
|
||||
func NewTSOKVBase(subPath string) *kv.EtcdKV {
|
||||
etcdAddr := conf.Config.Etcd.Address
|
||||
etcdAddr += ":"
|
||||
etcdAddr += strconv.FormatInt(int64(conf.Config.Etcd.Port), 10)
|
||||
etcdAddr, err := gparams.GParams.Load("etcd.address")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
etcdPort, err := gparams.GParams.Load("etcd.port")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
etcdAddr = etcdAddr + ":" + etcdPort
|
||||
fmt.Println("etcdAddr ::: ", etcdAddr)
|
||||
client, _ := clientv3.New(clientv3.Config{
|
||||
Endpoints: []string{etcdAddr},
|
||||
DialTimeout: 5 * time.Second,
|
||||
})
|
||||
return kv.NewEtcdKV(client, path.Join(conf.Config.Etcd.Rootpath, subPath))
|
||||
|
||||
etcdRootPath, err := gparams.GParams.Load("etcd.rootpath")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return kv.NewEtcdKV(client, path.Join(etcdRootPath, subPath))
|
||||
}
|
||||
|
||||
@ -14,7 +14,7 @@ def gen_file(rootfile, template, output, **kwargs):
|
||||
|
||||
|
||||
def extract_extra_body(visitor_info, query_path):
|
||||
pattern = re.compile(r"class(.*){\n((.|\n)*?)\n};", re.MULTILINE)
|
||||
pattern = re.compile("class(.*){\n((.|\n)*?)\n};", re.MULTILINE)
|
||||
|
||||
for node, visitors in visitor_info.items():
|
||||
for visitor in visitors:
|
||||
@ -22,24 +22,11 @@ def extract_extra_body(visitor_info, query_path):
|
||||
vis_file = query_path + "visitors/" + vis_name + ".cpp"
|
||||
body = ' public:'
|
||||
|
||||
inc_pattern_str = r'^(#include(.|\n)*)\n#include "query/generated/{}.h"'.format(vis_name)
|
||||
inc_pattern = re.compile(inc_pattern_str, re.MULTILINE)
|
||||
|
||||
if os.path.exists(vis_file):
|
||||
content = readfile(vis_file)
|
||||
infos = pattern.findall(content)
|
||||
assert len(infos) <= 1
|
||||
infos = pattern.findall(readfile(vis_file))
|
||||
if len(infos) == 1:
|
||||
name, body, _ = infos[0]
|
||||
|
||||
extra_inc_infos = inc_pattern.findall(content)
|
||||
assert(len(extra_inc_infos) <= 1)
|
||||
print(extra_inc_infos)
|
||||
if len(extra_inc_infos) == 1:
|
||||
extra_inc_body, _ = extra_inc_infos[0]
|
||||
|
||||
visitor["ctor_and_member"] = body
|
||||
visitor["extra_inc"] = extra_inc_body
|
||||
|
||||
if __name__ == "__main__":
|
||||
query_path = "../../internal/core/src/query/"
|
||||
|
||||
@ -9,9 +9,7 @@
|
||||
#pragma once
|
||||
// Generated File
|
||||
// DO NOT EDIT
|
||||
@@extra_inc@@
|
||||
#include "@@base_visitor@@.h"
|
||||
|
||||
namespace @@namespace@@ {
|
||||
class @@visitor_name@@ : @@base_visitor@@ {
|
||||
public:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user