test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-01 19:39:21 +08:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

tikv integration (#26246)

Browse Source 
Signed-off-by: yiwangdr <yiwangdr@gmail.com>
This commit is contained in:
yiwangdr 2023-09-06 16:25:14 -07:00 committed by GitHub
parent 8b5b137207
commit 337edc321b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 3113 additions and 799 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
configs/milvus.yaml
View File

@ -45,9 +45,19 @@ etcd:
metastore:
# Default value: etcd
# Valid values: etcd
# Valid values: [etcd, tikv]
type: etcd
# Related configuration of tikv, used to store Milvus metadata.
# Notice that when TiKV is enabled for metastore, you still need to have etcd for service discovery.
# TiKV is a good option when the metadata size requires better horizontal scalability.
tikv:
# Note that the default pd port of tikv is 2379, which conflicts with etcd.
endpoints: 127.0.0.1:2389
rootPath: by-dev # The root path where data is stored
metaSubPath: meta # metaRootPath = rootPath + '/' + metaSubPath
kvSubPath: kv # kvRootPath = rootPath + '/' + kvSubPath
localStorage:
path: /var/lib/milvus/data/ # please adjust in embedded Milvus: /tmp/milvus/data/

21
go.mod
View File

@ -17,7 +17,7 @@ require (
github.com/gofrs/flock v0.8.1
github.com/gogo/protobuf v1.3.2
github.com/golang/protobuf v1.5.3
github.com/google/btree v1.0.1
github.com/google/btree v1.1.2
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/klauspost/compress v1.16.5
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d
@ -36,6 +36,7 @@ require (
github.com/stretchr/testify v1.8.3
github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c
github.com/tidwall/gjson v1.14.4
github.com/tikv/client-go/v2 v2.0.4
go.etcd.io/etcd/api/v3 v3.5.5
go.etcd.io/etcd/client/v3 v3.5.5
go.etcd.io/etcd/server/v3 v3.5.5
@ -43,8 +44,8 @@ require (
go.opentelemetry.io/otel v1.13.0
go.opentelemetry.io/otel/trace v1.13.0
go.uber.org/atomic v1.10.0
go.uber.org/multierr v1.6.0
go.uber.org/zap v1.17.0
go.uber.org/multierr v1.7.0
go.uber.org/zap v1.20.0
golang.org/x/crypto v0.9.0
golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17
golang.org/x/oauth2 v0.6.0
@ -69,6 +70,7 @@ require (
github.com/andybalholm/brotli v1.0.4 // indirect
github.com/apache/thrift v0.15.0 // indirect
github.com/ardielle/ardielle-go v1.5.2 // indirect
github.com/benbjohnson/clock v1.1.0 // indirect
github.com/benesch/cgosymbolizer v0.0.0-20190515212042-bec6fe6e597b // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.2.0 // indirect
@ -83,8 +85,10 @@ require (
github.com/containerd/cgroups v1.1.0 // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
github.com/danieljoos/wincred v1.1.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/dvsekhvalnov/jose2go v1.5.0 // indirect
@ -144,15 +148,22 @@ require (
github.com/nats-io/nkeys v0.4.4 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/opencontainers/runtime-spec v1.0.2 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/panjf2000/ants/v2 v2.7.2 // indirect
github.com/pelletier/go-toml v1.9.3 // indirect
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
github.com/pierrec/lz4 v2.5.2+incompatible // indirect
github.com/pierrec/lz4/v4 v4.1.12 // indirect
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 // indirect
github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 // indirect
github.com/pingcap/kvproto v0.0.0-20221129023506-621ec37aac7a // indirect
github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/prometheus/procfs v0.9.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
github.com/rogpeppe/go-internal v1.8.1 // indirect
github.com/rs/xid v1.5.0 // indirect
github.com/shirou/gopsutil/v3 v3.22.9 // indirect
@ -161,15 +172,19 @@ require (
github.com/spf13/afero v1.6.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stathat/consistent v1.0.0 // indirect
github.com/streamnative/pulsarctl v0.5.0 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07 // indirect
github.com/tklauser/go-sysconf v0.3.10 // indirect
github.com/tklauser/numcpus v0.4.0 // indirect
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/twmb/murmur3 v1.1.3 // indirect
github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
github.com/ugorji/go/codec v1.2.11 // indirect
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect

54
go.sum
View File

@ -83,6 +83,7 @@ github.com/aliyun/credentials-go v1.2.7 h1:gLtFylxLZ1TWi1pStIt1O6a53GFU1zkNwjtJi
github.com/aliyun/credentials-go v1.2.7/go.mod h1:/KowD1cfGSLrLsH28Jr8W+xwoId0ywIy5lNzDz6O1vw=
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e h1:GCzyKMDDjSGnlpl3clrdAK7I1AaVoaiKDOYkUzChZzg=
github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY=
@ -101,6 +102,8 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/aws/aws-sdk-go v1.32.6/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/benesch/cgosymbolizer v0.0.0-20190515212042-bec6fe6e597b h1:5JgaFtHFRnOPReItxvhMDXbvuBkjSWE+9glJyF466yw=
github.com/benesch/cgosymbolizer v0.0.0-20190515212042-bec6fe6e597b/go.mod h1:eMD2XUcPsHYbakFEocKrWZp47G0MRJYoC60qFblGjpA=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@ -183,6 +186,8 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso=
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM=
github.com/danieljoos/wincred v1.1.2 h1:QLdCxFs1/Yl4zduvBdcHB8goaYk9RARS2SgLLRuAyr0=
github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0=
github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -191,6 +196,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/dimfeld/httptreemux v5.0.1+incompatible h1:Qj3gVcDNoOthBAqftuD596rm4wg/adLLz5xh5CmpiCA=
@ -356,8 +362,9 @@ github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4=
github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
github.com/google/flatbuffers v2.0.5+incompatible h1:ANsW0idDAXIY+mNHzIHxWRfabV2x5LUEEIIWcwsYgB8=
github.com/google/flatbuffers v2.0.5+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@ -412,6 +419,7 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c=
github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0=
@ -641,6 +649,7 @@ github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9
github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0=
github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
github.com/panjf2000/ants/v2 v2.7.2 h1:2NUt9BaZFO5kQzrieOmK/wdb/tQ/K+QHaxN8sOgD63U=
github.com/panjf2000/ants/v2 v2.7.2/go.mod h1:KIBmYG9QQX5U2qzFP/yQJaq/nSb6rahS9iEHkrCMgM8=
@ -657,8 +666,19 @@ github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUM
github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pierrec/lz4/v4 v4.1.12 h1:44l88ehTZAUGW4VlO1QC4zkilL99M6Y9MXNwEs0uzP8=
github.com/pierrec/lz4/v4 v4.1.12/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4=
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg=
github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0=
github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew=
github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E=
github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw=
github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI=
github.com/pingcap/kvproto v0.0.0-20221129023506-621ec37aac7a h1:LzIZsQpXQlj8yF7+yvyOg680OaPq7bmPuDuszgXfHsw=
github.com/pingcap/kvproto v0.0.0-20221129023506-621ec37aac7a/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI=
github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 h1:URLoJ61DmmY++Sa/yyPEQHG2s/ZBeV1FbIswHEMrdoY=
github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@ -676,6 +696,7 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw=
github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
@ -702,6 +723,8 @@ github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/quasilyte/go-ruleguard/dsl v0.3.22 h1:wd8zkOhSNr+I+8Qeciml08ivDt1pSXe60+5DqOpCjPE=
github.com/quasilyte/go-ruleguard/dsl v0.3.22/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rivo/tview v0.0.0-20200219210816-cd38d7432498/go.mod h1:6lkG1x+13OShEf0EaOCaTQYyB7d5nSbb181KtjlS+84=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rogpeppe/clock v0.0.0-20190514195947-2896927a307a/go.mod h1:4r5QyqhjIWCcK8DO4KMclc5Iknq5qVBAlbYYzAbUScQ=
@ -730,6 +753,7 @@ github.com/sbinet/npyio v0.6.0/go.mod h1:/q3BNr6dJOy+t6h7RZchTJ0nwRJO52mivaem29W
github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/shirou/gopsutil/v3 v3.22.9 h1:yibtJhIVEMcdw+tCTbOPiF1VcsuDeTE4utJ8Dm4c5eA=
github.com/shirou/gopsutil/v3 v3.22.9/go.mod h1:bBYl1kjgEJpWpxeHmLI+dVHWtyAwfcmSBLDsp2TNT8A=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
@ -769,6 +793,8 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM
github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44=
github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns=
github.com/stathat/consistent v1.0.0 h1:ZFJ1QTRn8npNBKW065raSZ8xfOqhpb8vLOkfp4CcL/U=
github.com/stathat/consistent v1.0.0/go.mod h1:uajTPbgSygZBJ+V+0mY7meZ8i0XAcZs7AQ6V121XSxw=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
@ -793,12 +819,18 @@ github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M=
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW93SG+q0F8KI+yFrcIDT4c/RNoc4=
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM=
github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tikv/client-go/v2 v2.0.4 h1:cPtMXTExqjzk8L40qhrgB/mXiBXKP5LRU0vwjtI2Xxo=
github.com/tikv/client-go/v2 v2.0.4/go.mod h1:v52O5zDtv2BBus4lm5yrSQhxGW4Z4RaXWfg0U1Kuyqo=
github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07 h1:ckPpxKcl75mO2N6a4cJXiZH43hvcHPpqc9dh1TmH1nc=
github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07/go.mod h1:CipBxPfxPUME+BImx9MUYXCnAVLS3VJUr3mnSJwh40A=
github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw=
github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk=
github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o=
@ -808,6 +840,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
github.com/twmb/murmur3 v1.1.3 h1:D83U0XYKcHRYwYIpBKf3Pks91Z0Byda/9SJ8B6EMRcA=
github.com/twmb/murmur3 v1.1.3/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o=
github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
@ -909,20 +943,26 @@ go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.uber.org/automaxprocs v1.5.2 h1:2LxUOGiR3O6tw8ui5sZa2LAaHnsviZdVOUZw4fvbnME=
go.uber.org/automaxprocs v1.5.2/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0=
go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec=
go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ=
go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U=
go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
go.uber.org/zap v1.20.0 h1:N4oPlghZwYG55MlU6LXk/Zp00FVNE9X9wrYO8CEs4lc=
go.uber.org/zap v1.20.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k=
golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
@ -1018,6 +1058,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@ -1211,6 +1252,7 @@ golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@ -1250,6 +1292,7 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4=
@ -1307,6 +1350,7 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn
google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
@ -1353,6 +1397,7 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
@ -1373,6 +1418,7 @@ google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ
google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k=
google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag=

45
internal/datacoord/server.go
View File

@ -28,6 +28,7 @@ import (
"github.com/blang/semver/v4"
"github.com/cockroachdb/errors"
"github.com/tikv/client-go/v2/txnkv"
clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/zap"
@ -37,6 +38,7 @@ import (
rootcoordclient "github.com/milvus-io/milvus/internal/distributed/rootcoord/client"
"github.com/milvus-io/milvus/internal/kv"
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
"github.com/milvus-io/milvus/internal/kv/tikv"
"github.com/milvus-io/milvus/internal/metastore/kv/datacoord"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/storage"
@ -47,6 +49,7 @@ import (
"github.com/milvus-io/milvus/pkg/metrics"
"github.com/milvus-io/milvus/pkg/mq/msgstream"
"github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper"
"github.com/milvus-io/milvus/pkg/util"
"github.com/milvus-io/milvus/pkg/util/funcutil"
"github.com/milvus-io/milvus/pkg/util/logutil"
"github.com/milvus-io/milvus/pkg/util/merr"
@ -59,7 +62,7 @@ import (
)
const (
connEtcdMaxRetryTime = 100
connMetaMaxRetryTime = 100
allPartitionID = 0 // partitionID means no filtering
)
@ -102,8 +105,10 @@ type Server struct {
helper ServerHelper
etcdCli *clientv3.Client
tikvCli *txnkv.Client
address string
kvClient kv.WatchKV
watchClient kv.WatchKV
kv kv.MetaKv
meta *meta
segmentManager Manager
allocator allocator
@ -402,7 +407,7 @@ func (s *Server) initCluster() error {
}
var err error
s.channelManager, err = NewChannelManager(s.kvClient, s.handler, withMsgstreamFactory(s.factory),
s.channelManager, err = NewChannelManager(s.watchClient, s.handler, withMsgstreamFactory(s.factory),
withStateChecker(), withBgChecker())
if err != nil {
return err
@ -421,6 +426,10 @@ func (s *Server) SetEtcdClient(client *clientv3.Client) {
s.etcdCli = client
}
func (s *Server) SetTiKVClient(client *txnkv.Client) {
s.tikvCli = client
}
func (s *Server) SetRootCoord(rootCoord types.RootCoord) {
s.rootCoordClient = rootCoord
}
@ -532,19 +541,28 @@ func (s *Server) initMeta(chunkManager storage.ChunkManager) error {
if s.meta != nil {
return nil
}
etcdKV := etcdkv.NewEtcdKV(s.etcdCli, Params.EtcdCfg.MetaRootPath.GetValue())
s.watchClient = etcdkv.NewEtcdKV(s.etcdCli, Params.EtcdCfg.MetaRootPath.GetValue())
metaType := Params.MetaStoreCfg.MetaStoreType.GetValue()
log.Info("data coordinator connecting to metadata store", zap.String("metaType", metaType))
if metaType == util.MetaStoreTypeTiKV {
s.kv = tikv.NewTiKV(s.tikvCli, Params.TiKVCfg.MetaRootPath.GetValue())
} else if metaType == util.MetaStoreTypeEtcd {
s.kv = etcdkv.NewEtcdKV(s.etcdCli, Params.EtcdCfg.MetaRootPath.GetValue())
} else {
return retry.Unrecoverable(fmt.Errorf("not supported meta store: %s", metaType))
}
log.Info("data coordinator successfully connected to metadata store", zap.String("metaType", metaType))
s.kvClient = etcdKV
reloadEtcdFn := func() error {
var err error
catalog := datacoord.NewCatalog(etcdKV, chunkManager.RootPath(), Params.EtcdCfg.MetaRootPath.GetValue())
catalog := datacoord.NewCatalog(s.kv, chunkManager.RootPath(), Params.EtcdCfg.MetaRootPath.GetValue())
s.meta, err = newMeta(s.ctx, catalog, chunkManager)
if err != nil {
return err
}
return nil
}
return retry.Do(s.ctx, reloadEtcdFn, retry.Attempts(connEtcdMaxRetryTime))
return retry.Do(s.ctx, reloadEtcdFn, retry.Attempts(connMetaMaxRetryTime))
}
func (s *Server) initIndexBuilder(manager storage.ChunkManager) {
@ -991,8 +1009,17 @@ func (s *Server) Stop() error {
// CleanMeta only for test
func (s *Server) CleanMeta() error {
log.Debug("clean meta", zap.Any("kv", s.kvClient))
return s.kvClient.RemoveWithPrefix("")
log.Debug("clean meta", zap.Any("kv", s.kv))
err := s.kv.RemoveWithPrefix("")
err2 := s.watchClient.RemoveWithPrefix("")
if err2 != nil {
if err != nil {
err = fmt.Errorf("Failed to CleanMeta[metadata cleanup error: %w][watchdata cleanup error: %v]", err, err2)
} else {
err = err2
}
}
return err
}
func (s *Server) stopServerLoop() {

19
internal/datacoord/server_test.go
View File

@ -60,6 +60,7 @@ import (
"github.com/milvus-io/milvus/pkg/util/metautil"
"github.com/milvus-io/milvus/pkg/util/metricsinfo"
"github.com/milvus-io/milvus/pkg/util/paramtable"
"github.com/milvus-io/milvus/pkg/util/tikv"
"github.com/milvus-io/milvus/pkg/util/tsoutil"
"github.com/milvus-io/milvus/pkg/util/typeutil"
)
@ -79,7 +80,13 @@ func TestMain(m *testing.M) {
paramtable.Get().Save(Params.EtcdCfg.Endpoints.Key, strings.Join(addrs, ","))
rand.Seed(time.Now().UnixNano())
os.Exit(m.Run())
parameters := []string{"tikv", "etcd"}
var code int
for _, v := range parameters {
paramtable.Get().Save(paramtable.Get().MetaStoreCfg.MetaStoreType.Key, v)
code = m.Run()
}
os.Exit(code)
}
func TestGetSegmentInfoChannel(t *testing.T) {
@ -4002,6 +4009,8 @@ func TestDataCoordServer_UpdateChannelCheckpoint(t *testing.T) {
})
}
var global_test_tikv = tikv.SetupLocalTxn()
func newTestServer(t *testing.T, receiveCh chan any, opts ...Option) *Server {
var err error
paramtable.Get().Save(Params.CommonCfg.DataCoordTimeTick.Key, Params.CommonCfg.DataCoordTimeTick.GetValue()+strconv.Itoa(rand.Int()))
@ -4021,6 +4030,8 @@ func newTestServer(t *testing.T, receiveCh chan any, opts ...Option) *Server {
svr := CreateServer(context.TODO(), factory)
svr.SetEtcdClient(etcdCli)
svr.SetTiKVClient(global_test_tikv)
svr.dataNodeCreator = func(ctx context.Context, addr string, nodeID int64) (types.DataNode, error) {
return newMockDataNodeClient(0, receiveCh)
}
@ -4073,6 +4084,8 @@ func newTestServerWithMeta(t *testing.T, receiveCh chan any, meta *meta, opts ..
svr := CreateServer(context.TODO(), factory, opts...)
svr.SetEtcdClient(etcdCli)
svr.SetTiKVClient(global_test_tikv)
svr.dataNodeCreator = func(ctx context.Context, addr string, nodeID int64) (types.DataNode, error) {
return newMockDataNodeClient(0, receiveCh)
}
@ -4128,6 +4141,8 @@ func newTestServer2(t *testing.T, receiveCh chan any, opts ...Option) *Server {
svr := CreateServer(context.TODO(), factory, opts...)
svr.SetEtcdClient(etcdCli)
svr.SetTiKVClient(global_test_tikv)
svr.dataNodeCreator = func(ctx context.Context, addr string, nodeID int64) (types.DataNode, error) {
return newMockDataNodeClient(0, receiveCh)
}
@ -4320,6 +4335,8 @@ func testDataCoordBase(t *testing.T, opts ...Option) *Server {
svr := CreateServer(ctx, factory, opts...)
svr.SetEtcdClient(etcdCli)
svr.SetTiKVClient(global_test_tikv)
svr.SetDataNodeCreator(func(ctx context.Context, addr string, nodeID int64) (types.DataNode, error) {
return newMockDataNodeClient(0, nil)
})

4
internal/datanode/event_manager.go
View File

@ -109,7 +109,7 @@ type tickler struct {
progress *atomic.Int32
version int64
kv kv.MetaKv
kv kv.WatchKV
path string
watchInfo *datapb.ChannelWatchInfo
@ -182,7 +182,7 @@ func (t *tickler) stop() {
t.closeWg.Wait()
}
func newTickler(version int64, path string, watchInfo *datapb.ChannelWatchInfo, kv kv.MetaKv, interval time.Duration) *tickler {
func newTickler(version int64, path string, watchInfo *datapb.ChannelWatchInfo, kv kv.WatchKV, interval time.Duration) *tickler {
return &tickler{
progress: atomic.NewInt32(0),
path: path,

26
internal/distributed/datacoord/service.go
View File

@ -30,6 +30,7 @@ import (
"github.com/milvus-io/milvus/pkg/util/interceptor"
grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
"github.com/tikv/client-go/v2/txnkv"
clientv3 "go.etcd.io/etcd/client/v3"
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
"go.uber.org/atomic"
@ -44,10 +45,12 @@ import (
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/types"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/util"
"github.com/milvus-io/milvus/pkg/util/etcd"
"github.com/milvus-io/milvus/pkg/util/funcutil"
"github.com/milvus-io/milvus/pkg/util/logutil"
"github.com/milvus-io/milvus/pkg/util/paramtable"
"github.com/milvus-io/milvus/pkg/util/tikv"
)
// Server is the grpc server of datacoord
@ -61,6 +64,7 @@ type Server struct {
dataCoord types.DataCoordComponent
etcdCli *clientv3.Client
tikvCli *txnkv.Client
grpcErrChan chan error
grpcServer *grpc.Server
@ -79,9 +83,11 @@ func NewServer(ctx context.Context, factory dependency.Factory, opts ...datacoor
return s
}
var getTiKVClient = tikv.GetTiKVClient
func (s *Server) init() error {
etcdConfig := &paramtable.Get().EtcdCfg
Params := &paramtable.Get().DataCoordGrpcServerCfg
params := paramtable.Get()
etcdConfig := &params.EtcdCfg
etcdCli, err := etcd.GetEtcdClient(
etcdConfig.UseEmbedEtcd.GetAsBool(),
@ -97,7 +103,18 @@ func (s *Server) init() error {
}
s.etcdCli = etcdCli
s.dataCoord.SetEtcdClient(etcdCli)
s.dataCoord.SetAddress(Params.GetAddress())
s.dataCoord.SetAddress(params.DataCoordGrpcServerCfg.GetAddress())
if params.MetaStoreCfg.MetaStoreType.GetValue() == util.MetaStoreTypeTiKV {
log.Info("Connecting to tikv metadata storage.")
tikvCli, err := getTiKVClient(&paramtable.Get().TiKVCfg)
if err != nil {
log.Warn("DataCoord failed to connect to tikv", zap.Error(err))
return err
}
s.dataCoord.SetTiKVClient(tikvCli)
log.Info("Connected to tikv. Using tikv as metadata storage.")
}
err = s.startGrpc()
if err != nil {
@ -209,6 +226,9 @@ func (s *Server) Stop() error {
if s.etcdCli != nil {
defer s.etcdCli.Close()
}
if s.tikvCli != nil {
defer s.tikvCli.Close()
}
if s.grpcServer != nil {
log.Debug("Graceful stop grpc server...")
s.grpcServer.GracefulStop()

777
internal/distributed/datacoord/service_test.go
View File

@ -18,6 +18,7 @@ package grpcdatacoord
import (
"context"
"fmt"
"testing"
"github.com/cockroachdb/errors"
@ -29,7 +30,9 @@ import (
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/types"
"github.com/milvus-io/milvus/pkg/util/paramtable"
"github.com/milvus-io/milvus/pkg/util/tikv"
"github.com/stretchr/testify/assert"
"github.com/tikv/client-go/v2/txnkv"
clientv3 "go.etcd.io/etcd/client/v3"
)
@ -104,6 +107,9 @@ func (*MockDataCoord) SetAddress(address string) {
func (m *MockDataCoord) SetEtcdClient(etcdClient *clientv3.Client) {
}
func (m *MockDataCoord) SetTiKVClient(client *txnkv.Client) {
}
func (m *MockDataCoord) SetRootCoord(rootCoord types.RootCoord) {
}
@ -285,424 +291,447 @@ func (m *MockDataCoord) DropIndex(ctx context.Context, req *indexpb.DropIndexReq
func Test_NewServer(t *testing.T) {
paramtable.Init()
ctx := context.Background()
server := NewServer(ctx, nil)
assert.NotNil(t, server)
t.Run("Run", func(t *testing.T) {
server.dataCoord = &MockDataCoord{}
//indexCoord := mocks.NewMockIndexCoord(t)
//indexCoord.EXPECT().Init().Return(nil)
//server.indexCoord = indexCoord
err := server.Run()
assert.NoError(t, err)
})
t.Run("GetComponentStates", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
states: &milvuspb.ComponentStates{},
parameters := []string{"tikv", "etcd"}
for _, v := range parameters {
paramtable.Get().Save(paramtable.Get().MetaStoreCfg.MetaStoreType.Key, v)
ctx := context.Background()
getTiKVClient = func(cfg *paramtable.TiKVConfig) (*txnkv.Client, error) {
return tikv.SetupLocalTxn(), nil
}
states, err := server.GetComponentStates(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, states)
})
defer func() {
getTiKVClient = tikv.GetTiKVClient
}()
server := NewServer(ctx, nil)
assert.NotNil(t, server)
t.Run("GetTimeTickChannel", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
strResp: &milvuspb.StringResponse{},
}
resp, err := server.GetTimeTickChannel(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("Run", func(t *testing.T) {
server.dataCoord = &MockDataCoord{}
//indexCoord := mocks.NewMockIndexCoord(t)
//indexCoord.EXPECT().Init().Return(nil)
//server.indexCoord = indexCoord
t.Run("GetStatisticsChannel", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
strResp: &milvuspb.StringResponse{},
}
resp, err := server.GetStatisticsChannel(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
err := server.Run()
assert.NoError(t, err)
})
t.Run("GetSegmentInfo", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
infoResp: &datapb.GetSegmentInfoResponse{},
}
resp, err := server.GetSegmentInfo(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetComponentStates", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
states: &milvuspb.ComponentStates{},
}
states, err := server.GetComponentStates(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, states)
})
t.Run("Flush", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
flushResp: &datapb.FlushResponse{},
}
resp, err := server.Flush(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetTimeTickChannel", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
strResp: &milvuspb.StringResponse{},
}
resp, err := server.GetTimeTickChannel(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("AssignSegmentID", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
assignResp: &datapb.AssignSegmentIDResponse{},
}
resp, err := server.AssignSegmentID(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetStatisticsChannel", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
strResp: &milvuspb.StringResponse{},
}
resp, err := server.GetStatisticsChannel(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetSegmentStates", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
segStateResp: &datapb.GetSegmentStatesResponse{},
}
resp, err := server.GetSegmentStates(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetSegmentInfo", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
infoResp: &datapb.GetSegmentInfoResponse{},
}
resp, err := server.GetSegmentInfo(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetInsertBinlogPaths", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
binResp: &datapb.GetInsertBinlogPathsResponse{},
}
resp, err := server.GetInsertBinlogPaths(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("Flush", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
flushResp: &datapb.FlushResponse{},
}
resp, err := server.Flush(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetCollectionStatistics", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
colStatResp: &datapb.GetCollectionStatisticsResponse{},
}
resp, err := server.GetCollectionStatistics(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("AssignSegmentID", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
assignResp: &datapb.AssignSegmentIDResponse{},
}
resp, err := server.AssignSegmentID(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetPartitionStatistics", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
partStatResp: &datapb.GetPartitionStatisticsResponse{},
}
resp, err := server.GetPartitionStatistics(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetSegmentStates", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
segStateResp: &datapb.GetSegmentStatesResponse{},
}
resp, err := server.GetSegmentStates(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetSegmentInfoChannel", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
strResp: &milvuspb.StringResponse{},
}
resp, err := server.GetSegmentInfoChannel(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetInsertBinlogPaths", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
binResp: &datapb.GetInsertBinlogPathsResponse{},
}
resp, err := server.GetInsertBinlogPaths(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("SaveBinlogPaths", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
status: &commonpb.Status{},
}
resp, err := server.SaveBinlogPaths(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetCollectionStatistics", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
colStatResp: &datapb.GetCollectionStatisticsResponse{},
}
resp, err := server.GetCollectionStatistics(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetRecoveryInfo", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
recoverResp: &datapb.GetRecoveryInfoResponse{},
}
resp, err := server.GetRecoveryInfo(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetPartitionStatistics", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
partStatResp: &datapb.GetPartitionStatisticsResponse{},
}
resp, err := server.GetPartitionStatistics(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetFlushedSegments", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
flushSegResp: &datapb.GetFlushedSegmentsResponse{},
}
resp, err := server.GetFlushedSegments(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetSegmentInfoChannel", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
strResp: &milvuspb.StringResponse{},
}
resp, err := server.GetSegmentInfoChannel(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("ShowConfigurations", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
configResp: &internalpb.ShowConfigurationsResponse{},
}
resp, err := server.ShowConfigurations(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("SaveBinlogPaths", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
status: &commonpb.Status{},
}
resp, err := server.SaveBinlogPaths(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetMetrics", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
metricResp: &milvuspb.GetMetricsResponse{},
}
resp, err := server.GetMetrics(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetRecoveryInfo", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
recoverResp: &datapb.GetRecoveryInfoResponse{},
}
resp, err := server.GetRecoveryInfo(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("WatchChannels", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
watchChannelsResp: &datapb.WatchChannelsResponse{},
}
resp, err := server.WatchChannels(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetFlushedSegments", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
flushSegResp: &datapb.GetFlushedSegmentsResponse{},
}
resp, err := server.GetFlushedSegments(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetFlushState", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
getFlushStateResp: &milvuspb.GetFlushStateResponse{},
}
resp, err := server.GetFlushState(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("ShowConfigurations", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
configResp: &internalpb.ShowConfigurationsResponse{},
}
resp, err := server.ShowConfigurations(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetFlushAllState", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
getFlushAllStateResp: &milvuspb.GetFlushAllStateResponse{},
}
resp, err := server.GetFlushAllState(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetMetrics", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
metricResp: &milvuspb.GetMetricsResponse{},
}
resp, err := server.GetMetrics(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("DropVirtualChannel", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
dropVChanResp: &datapb.DropVirtualChannelResponse{},
}
resp, err := server.DropVirtualChannel(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("WatchChannels", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
watchChannelsResp: &datapb.WatchChannelsResponse{},
}
resp, err := server.WatchChannels(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("ManualCompaction", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
manualCompactionResp: &milvuspb.ManualCompactionResponse{},
}
resp, err := server.ManualCompaction(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetFlushState", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
getFlushStateResp: &milvuspb.GetFlushStateResponse{},
}
resp, err := server.GetFlushState(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetCompactionState", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
compactionStateResp: &milvuspb.GetCompactionStateResponse{},
}
resp, err := server.GetCompactionState(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetFlushAllState", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
getFlushAllStateResp: &milvuspb.GetFlushAllStateResponse{},
}
resp, err := server.GetFlushAllState(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetCompactionStateWithPlans", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
compactionPlansResp: &milvuspb.GetCompactionPlansResponse{},
}
resp, err := server.GetCompactionStateWithPlans(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("DropVirtualChannel", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
dropVChanResp: &datapb.DropVirtualChannelResponse{},
}
resp, err := server.DropVirtualChannel(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("set segment state", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
setSegmentStateResp: &datapb.SetSegmentStateResponse{},
}
resp, err := server.SetSegmentState(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("ManualCompaction", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
manualCompactionResp: &milvuspb.ManualCompactionResponse{},
}
resp, err := server.ManualCompaction(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("import", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
importResp: &datapb.ImportTaskResponse{
Status: &commonpb.Status{},
},
}
resp, err := server.Import(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetCompactionState", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
compactionStateResp: &milvuspb.GetCompactionStateResponse{},
}
resp, err := server.GetCompactionState(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("update seg stat", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
updateSegStatResp: &commonpb.Status{
ErrorCode: commonpb.ErrorCode_Success,
},
}
resp, err := server.UpdateSegmentStatistics(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetCompactionStateWithPlans", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
compactionPlansResp: &milvuspb.GetCompactionPlansResponse{},
}
resp, err := server.GetCompactionStateWithPlans(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("UpdateChannelCheckpoint", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
updateChanPos: &commonpb.Status{
ErrorCode: commonpb.ErrorCode_Success,
},
}
resp, err := server.UpdateChannelCheckpoint(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("set segment state", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
setSegmentStateResp: &datapb.SetSegmentStateResponse{},
}
resp, err := server.SetSegmentState(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("save import segment", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
addSegmentResp: &commonpb.Status{
ErrorCode: commonpb.ErrorCode_Success,
},
}
resp, err := server.SaveImportSegment(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("import", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
importResp: &datapb.ImportTaskResponse{
Status: &commonpb.Status{},
},
}
resp, err := server.Import(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("unset isImporting state", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
unsetIsImportingStateResp: &commonpb.Status{
ErrorCode: commonpb.ErrorCode_Success,
},
}
resp, err := server.UnsetIsImportingState(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("update seg stat", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
updateSegStatResp: &commonpb.Status{
ErrorCode: commonpb.ErrorCode_Success,
},
}
resp, err := server.UpdateSegmentStatistics(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("mark segments dropped", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
markSegmentsDroppedResp: &commonpb.Status{
ErrorCode: commonpb.ErrorCode_Success,
},
}
resp, err := server.MarkSegmentsDropped(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("UpdateChannelCheckpoint", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
updateChanPos: &commonpb.Status{
ErrorCode: commonpb.ErrorCode_Success,
},
}
resp, err := server.UpdateChannelCheckpoint(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("broadcast altered collection", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
broadCastResp: &commonpb.Status{},
}
resp, err := server.BroadcastAlteredCollection(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("save import segment", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
addSegmentResp: &commonpb.Status{
ErrorCode: commonpb.ErrorCode_Success,
},
}
resp, err := server.SaveImportSegment(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("CheckHealth", func(t *testing.T) {
server.dataCoord = &MockDataCoord{}
ret, err := server.CheckHealth(ctx, nil)
assert.NoError(t, err)
assert.Equal(t, true, ret.IsHealthy)
})
t.Run("unset isImporting state", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
unsetIsImportingStateResp: &commonpb.Status{
ErrorCode: commonpb.ErrorCode_Success,
},
}
resp, err := server.UnsetIsImportingState(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("CreateIndex", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
createIndexResp: &commonpb.Status{},
}
ret, err := server.CreateIndex(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, ret)
})
t.Run("mark segments dropped", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
markSegmentsDroppedResp: &commonpb.Status{
ErrorCode: commonpb.ErrorCode_Success,
},
}
resp, err := server.MarkSegmentsDropped(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("DescribeIndex", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
describeIndexResp: &indexpb.DescribeIndexResponse{},
}
ret, err := server.DescribeIndex(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, ret)
})
t.Run("broadcast altered collection", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
broadCastResp: &commonpb.Status{},
}
resp, err := server.BroadcastAlteredCollection(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetIndexStatistics", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
getIndexStatisticsResp: &indexpb.GetIndexStatisticsResponse{},
}
ret, err := server.GetIndexStatistics(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, ret)
})
t.Run("CheckHealth", func(t *testing.T) {
server.dataCoord = &MockDataCoord{}
ret, err := server.CheckHealth(ctx, nil)
assert.NoError(t, err)
assert.Equal(t, true, ret.IsHealthy)
})
t.Run("DropIndex", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
dropIndexResp: &commonpb.Status{},
}
ret, err := server.DropIndex(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, ret)
})
t.Run("CreateIndex", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
createIndexResp: &commonpb.Status{},
}
ret, err := server.CreateIndex(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, ret)
})
t.Run("GetIndexState", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
getIndexStateResp: &indexpb.GetIndexStateResponse{},
}
ret, err := server.GetIndexState(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, ret)
})
t.Run("DescribeIndex", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
describeIndexResp: &indexpb.DescribeIndexResponse{},
}
ret, err := server.DescribeIndex(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, ret)
})
t.Run("GetIndexBuildProgress", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
getIndexBuildProgressResp: &indexpb.GetIndexBuildProgressResponse{},
}
ret, err := server.GetIndexBuildProgress(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, ret)
})
t.Run("GetIndexStatistics", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
getIndexStatisticsResp: &indexpb.GetIndexStatisticsResponse{},
}
ret, err := server.GetIndexStatistics(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, ret)
})
t.Run("GetSegmentIndexState", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
getSegmentIndexStateResp: &indexpb.GetSegmentIndexStateResponse{},
}
ret, err := server.GetSegmentIndexState(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, ret)
})
t.Run("DropIndex", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
dropIndexResp: &commonpb.Status{},
}
ret, err := server.DropIndex(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, ret)
})
t.Run("GetIndexInfos", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
getIndexInfosResp: &indexpb.GetIndexInfoResponse{},
}
ret, err := server.GetIndexInfos(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, ret)
})
t.Run("GetIndexState", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
getIndexStateResp: &indexpb.GetIndexStateResponse{},
}
ret, err := server.GetIndexState(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, ret)
})
err := server.Stop()
assert.NoError(t, err)
t.Run("GetIndexBuildProgress", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
getIndexBuildProgressResp: &indexpb.GetIndexBuildProgressResponse{},
}
ret, err := server.GetIndexBuildProgress(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, ret)
})
t.Run("GetSegmentIndexState", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
getSegmentIndexStateResp: &indexpb.GetSegmentIndexStateResponse{},
}
ret, err := server.GetSegmentIndexState(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, ret)
})
t.Run("GetIndexInfos", func(t *testing.T) {
server.dataCoord = &MockDataCoord{
getIndexInfosResp: &indexpb.GetIndexInfoResponse{},
}
ret, err := server.GetIndexInfos(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, ret)
})
err := server.Stop()
assert.NoError(t, err)
}
}
func Test_Run(t *testing.T) {
ctx := context.Background()
server := NewServer(ctx, nil)
assert.NotNil(t, server)
paramtable.Init()
parameters := []string{"tikv", "etcd"}
for _, v := range parameters {
t.Run(fmt.Sprintf("Run server with %s as metadata storage", v), func(t *testing.T) {
paramtable.Get().Save(paramtable.Get().MetaStoreCfg.MetaStoreType.Key, v)
ctx := context.Background()
getTiKVClient = func(cfg *paramtable.TiKVConfig) (*txnkv.Client, error) {
return tikv.SetupLocalTxn(), nil
}
defer func() {
getTiKVClient = tikv.GetTiKVClient
}()
server := NewServer(ctx, nil)
assert.NotNil(t, server)
server.dataCoord = &MockDataCoord{
regErr: errors.New("error"),
server.dataCoord = &MockDataCoord{
regErr: errors.New("error"),
}
err := server.Run()
assert.Error(t, err)
server.dataCoord = &MockDataCoord{
startErr: errors.New("error"),
}
err = server.Run()
assert.Error(t, err)
server.dataCoord = &MockDataCoord{
initErr: errors.New("error"),
}
err = server.Run()
assert.Error(t, err)
server.dataCoord = &MockDataCoord{
stopErr: errors.New("error"),
}
err = server.Stop()
assert.Error(t, err)
})
}
err := server.Run()
assert.Error(t, err)
server.dataCoord = &MockDataCoord{
startErr: errors.New("error"),
}
err = server.Run()
assert.Error(t, err)
server.dataCoord = &MockDataCoord{
initErr: errors.New("error"),
}
err = server.Run()
assert.Error(t, err)
server.dataCoord = &MockDataCoord{
stopErr: errors.New("error"),
}
err = server.Stop()
assert.Error(t, err)
}

30
internal/distributed/querycoord/service.go
View File

@ -28,6 +28,7 @@ import (
"github.com/milvus-io/milvus/internal/util/dependency"
"github.com/milvus-io/milvus/pkg/tracer"
"github.com/milvus-io/milvus/pkg/util/interceptor"
"github.com/tikv/client-go/v2/txnkv"
clientv3 "go.etcd.io/etcd/client/v3"
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
"go.uber.org/atomic"
@ -44,10 +45,12 @@ import (
qc "github.com/milvus-io/milvus/internal/querycoordv2"
"github.com/milvus-io/milvus/internal/types"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/util"
"github.com/milvus-io/milvus/pkg/util/etcd"
"github.com/milvus-io/milvus/pkg/util/funcutil"
"github.com/milvus-io/milvus/pkg/util/logutil"
"github.com/milvus-io/milvus/pkg/util/paramtable"
"github.com/milvus-io/milvus/pkg/util/tikv"
)
// Server is the grpc server of QueryCoord.
@ -66,6 +69,7 @@ type Server struct {
factory dependency.Factory
etcdCli *clientv3.Client
tikvCli *txnkv.Client
dataCoord types.DataCoord
rootCoord types.RootCoord
@ -104,10 +108,13 @@ func (s *Server) Run() error {
return nil
}
var getTiKVClient = tikv.GetTiKVClient
// init initializes QueryCoord's grpc service.
func (s *Server) init() error {
etcdConfig := &paramtable.Get().EtcdCfg
Params := &paramtable.Get().QueryCoordGrpcServerCfg
params := paramtable.Get()
etcdConfig := &params.EtcdCfg
rpcParams := &params.QueryCoordGrpcServerCfg
etcdCli, err := etcd.GetEtcdClient(
etcdConfig.UseEmbedEtcd.GetAsBool(),
@ -123,10 +130,21 @@ func (s *Server) init() error {
}
s.etcdCli = etcdCli
s.SetEtcdClient(etcdCli)
s.queryCoord.SetAddress(Params.GetAddress())
s.queryCoord.SetAddress(rpcParams.GetAddress())
if params.MetaStoreCfg.MetaStoreType.GetValue() == util.MetaStoreTypeTiKV {
log.Info("Connecting to tikv metadata storage.")
s.tikvCli, err = getTiKVClient(&paramtable.Get().TiKVCfg)
if err != nil {
log.Warn("QueryCoord failed to connect to tikv", zap.Error(err))
return err
}
s.SetTiKVClient(s.tikvCli)
log.Info("Connected to tikv. Using tikv as metadata storage.")
}
s.wg.Add(1)
go s.startGrpcLoop(Params.Port.GetAsInt())
go s.startGrpcLoop(rpcParams.Port.GetAsInt())
// wait for grpc server loop start
err = <-s.grpcErrChan
if err != nil {
@ -287,6 +305,10 @@ func (s *Server) SetEtcdClient(etcdClient *clientv3.Client) {
s.queryCoord.SetEtcdClient(etcdClient)
}
func (s *Server) SetTiKVClient(client *txnkv.Client) {
s.queryCoord.SetTiKVClient(client)
}
// SetRootCoord sets the RootCoord's client for QueryCoord component.
func (s *Server) SetRootCoord(m types.RootCoord) error {
s.queryCoord.SetRootCoord(m)

576
internal/distributed/querycoord/service_test.go
View File

@ -31,8 +31,10 @@ import (
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/proto/querypb"
"github.com/milvus-io/milvus/pkg/util/tikv"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/tikv/client-go/v2/txnkv"
)
// /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@ -109,251 +111,282 @@ func TestMain(m *testing.M) {
}
func Test_NewServer(t *testing.T) {
ctx := context.Background()
server, err := NewServer(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, server)
mdc := &MockDataCoord{
stateErr: commonpb.ErrorCode_Success,
}
mrc := &MockRootCoord{
stateErr: commonpb.ErrorCode_Success,
}
mqc := getQueryCoord()
successStatus := &commonpb.Status{
ErrorCode: commonpb.ErrorCode_Success,
}
t.Run("Run", func(t *testing.T) {
server.queryCoord = mqc
server.dataCoord = mdc
server.rootCoord = mrc
err = server.Run()
assert.NoError(t, err)
})
t.Run("GetComponentStates", func(t *testing.T) {
mqc.EXPECT().GetComponentStates(mock.Anything).Return(&milvuspb.ComponentStates{
State: &milvuspb.ComponentInfo{
NodeID: 0,
Role: "MockQueryCoord",
StateCode: commonpb.StateCode_Healthy,
},
Status: successStatus,
}, nil)
req := &milvuspb.GetComponentStatesRequest{}
states, err := server.GetComponentStates(ctx, req)
assert.NoError(t, err)
assert.Equal(t, commonpb.StateCode_Healthy, states.State.StateCode)
})
t.Run("GetStatisticsChannel", func(t *testing.T) {
req := &internalpb.GetStatisticsChannelRequest{}
mqc.EXPECT().GetStatisticsChannel(mock.Anything).Return(
&milvuspb.StringResponse{
Status: successStatus,
}, nil,
)
resp, err := server.GetStatisticsChannel(ctx, req)
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
})
t.Run("GetTimeTickChannel", func(t *testing.T) {
req := &internalpb.GetTimeTickChannelRequest{}
mqc.EXPECT().GetTimeTickChannel(mock.Anything).Return(
&milvuspb.StringResponse{
Status: successStatus,
}, nil,
)
resp, err := server.GetTimeTickChannel(ctx, req)
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
})
t.Run("ShowCollections", func(t *testing.T) {
mqc.EXPECT().ShowCollections(mock.Anything, mock.Anything).Return(
&querypb.ShowCollectionsResponse{
Status: successStatus,
}, nil,
)
resp, err := server.ShowCollections(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
assert.Equal(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
})
t.Run("LoadCollection", func(t *testing.T) {
mqc.EXPECT().LoadCollection(mock.Anything, mock.Anything).Return(successStatus, nil)
resp, err := server.LoadCollection(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("ReleaseCollection", func(t *testing.T) {
mqc.EXPECT().ReleaseCollection(mock.Anything, mock.Anything).Return(successStatus, nil)
resp, err := server.ReleaseCollection(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("ShowPartitions", func(t *testing.T) {
mqc.EXPECT().ShowPartitions(mock.Anything, mock.Anything).Return(&querypb.ShowPartitionsResponse{Status: successStatus}, nil)
resp, err := server.ShowPartitions(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetPartitionStates", func(t *testing.T) {
mqc.EXPECT().GetPartitionStates(mock.Anything, mock.Anything).Return(&querypb.GetPartitionStatesResponse{Status: successStatus}, nil)
resp, err := server.GetPartitionStates(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("LoadPartitions", func(t *testing.T) {
mqc.EXPECT().LoadPartitions(mock.Anything, mock.Anything).Return(successStatus, nil)
resp, err := server.LoadPartitions(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("ReleasePartitions", func(t *testing.T) {
mqc.EXPECT().ReleasePartitions(mock.Anything, mock.Anything).Return(successStatus, nil)
resp, err := server.ReleasePartitions(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetTimeTickChannel", func(t *testing.T) {
mqc.EXPECT().GetTimeTickChannel(mock.Anything).Return(&milvuspb.StringResponse{Status: successStatus}, nil)
resp, err := server.GetTimeTickChannel(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetSegmentInfo", func(t *testing.T) {
req := &querypb.GetSegmentInfoRequest{}
mqc.EXPECT().GetSegmentInfo(mock.Anything, req).Return(&querypb.GetSegmentInfoResponse{Status: successStatus}, nil)
resp, err := server.GetSegmentInfo(ctx, req)
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
})
t.Run("LoadBalance", func(t *testing.T) {
req := &querypb.LoadBalanceRequest{}
mqc.EXPECT().LoadBalance(mock.Anything, req).Return(successStatus, nil)
resp, err := server.LoadBalance(ctx, req)
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.ErrorCode)
})
t.Run("GetMetrics", func(t *testing.T) {
req := &milvuspb.GetMetricsRequest{
Request: "",
parameters := []string{"tikv", "etcd"}
for _, v := range parameters {
paramtable.Get().Save(paramtable.Get().MetaStoreCfg.MetaStoreType.Key, v)
ctx := context.Background()
getTiKVClient = func(cfg *paramtable.TiKVConfig) (*txnkv.Client, error) {
return tikv.SetupLocalTxn(), nil
}
mqc.EXPECT().GetMetrics(mock.Anything, req).Return(&milvuspb.GetMetricsResponse{Status: successStatus}, nil)
resp, err := server.GetMetrics(ctx, req)
defer func() {
getTiKVClient = tikv.GetTiKVClient
}()
server, err := NewServer(ctx, nil)
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
})
assert.NotNil(t, server)
t.Run("CheckHealth", func(t *testing.T) {
mqc.EXPECT().CheckHealth(mock.Anything, mock.Anything).Return(
&milvuspb.CheckHealthResponse{Status: successStatus, IsHealthy: true}, nil)
ret, err := server.CheckHealth(ctx, nil)
mdc := &MockDataCoord{
stateErr: commonpb.ErrorCode_Success,
}
mrc := &MockRootCoord{
stateErr: commonpb.ErrorCode_Success,
}
mqc := getQueryCoord()
successStatus := &commonpb.Status{
ErrorCode: commonpb.ErrorCode_Success,
}
t.Run("Run", func(t *testing.T) {
server.queryCoord = mqc
server.dataCoord = mdc
server.rootCoord = mrc
err = server.Run()
assert.NoError(t, err)
})
t.Run("GetComponentStates", func(t *testing.T) {
mqc.EXPECT().GetComponentStates(mock.Anything).Return(&milvuspb.ComponentStates{
State: &milvuspb.ComponentInfo{
NodeID: 0,
Role: "MockQueryCoord",
StateCode: commonpb.StateCode_Healthy,
},
Status: successStatus,
}, nil)
req := &milvuspb.GetComponentStatesRequest{}
states, err := server.GetComponentStates(ctx, req)
assert.NoError(t, err)
assert.Equal(t, commonpb.StateCode_Healthy, states.State.StateCode)
})
t.Run("GetStatisticsChannel", func(t *testing.T) {
req := &internalpb.GetStatisticsChannelRequest{}
mqc.EXPECT().GetStatisticsChannel(mock.Anything).Return(
&milvuspb.StringResponse{
Status: successStatus,
}, nil,
)
resp, err := server.GetStatisticsChannel(ctx, req)
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
})
t.Run("GetTimeTickChannel", func(t *testing.T) {
req := &internalpb.GetTimeTickChannelRequest{}
mqc.EXPECT().GetTimeTickChannel(mock.Anything).Return(
&milvuspb.StringResponse{
Status: successStatus,
}, nil,
)
resp, err := server.GetTimeTickChannel(ctx, req)
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
})
t.Run("ShowCollections", func(t *testing.T) {
mqc.EXPECT().ShowCollections(mock.Anything, mock.Anything).Return(
&querypb.ShowCollectionsResponse{
Status: successStatus,
}, nil,
)
resp, err := server.ShowCollections(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
assert.Equal(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
})
t.Run("LoadCollection", func(t *testing.T) {
mqc.EXPECT().LoadCollection(mock.Anything, mock.Anything).Return(successStatus, nil)
resp, err := server.LoadCollection(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("ReleaseCollection", func(t *testing.T) {
mqc.EXPECT().ReleaseCollection(mock.Anything, mock.Anything).Return(successStatus, nil)
resp, err := server.ReleaseCollection(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("ShowPartitions", func(t *testing.T) {
mqc.EXPECT().ShowPartitions(mock.Anything, mock.Anything).Return(&querypb.ShowPartitionsResponse{Status: successStatus}, nil)
resp, err := server.ShowPartitions(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetPartitionStates", func(t *testing.T) {
mqc.EXPECT().GetPartitionStates(mock.Anything, mock.Anything).Return(&querypb.GetPartitionStatesResponse{Status: successStatus}, nil)
resp, err := server.GetPartitionStates(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("LoadPartitions", func(t *testing.T) {
mqc.EXPECT().LoadPartitions(mock.Anything, mock.Anything).Return(successStatus, nil)
resp, err := server.LoadPartitions(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("ReleasePartitions", func(t *testing.T) {
mqc.EXPECT().ReleasePartitions(mock.Anything, mock.Anything).Return(successStatus, nil)
resp, err := server.ReleasePartitions(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetTimeTickChannel", func(t *testing.T) {
mqc.EXPECT().GetTimeTickChannel(mock.Anything).Return(&milvuspb.StringResponse{Status: successStatus}, nil)
resp, err := server.GetTimeTickChannel(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, resp)
})
t.Run("GetSegmentInfo", func(t *testing.T) {
req := &querypb.GetSegmentInfoRequest{}
mqc.EXPECT().GetSegmentInfo(mock.Anything, req).Return(&querypb.GetSegmentInfoResponse{Status: successStatus}, nil)
resp, err := server.GetSegmentInfo(ctx, req)
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
})
t.Run("LoadBalance", func(t *testing.T) {
req := &querypb.LoadBalanceRequest{}
mqc.EXPECT().LoadBalance(mock.Anything, req).Return(successStatus, nil)
resp, err := server.LoadBalance(ctx, req)
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.ErrorCode)
})
t.Run("GetMetrics", func(t *testing.T) {
req := &milvuspb.GetMetricsRequest{
Request: "",
}
mqc.EXPECT().GetMetrics(mock.Anything, req).Return(&milvuspb.GetMetricsResponse{Status: successStatus}, nil)
resp, err := server.GetMetrics(ctx, req)
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
})
t.Run("CheckHealth", func(t *testing.T) {
mqc.EXPECT().CheckHealth(mock.Anything, mock.Anything).Return(
&milvuspb.CheckHealthResponse{Status: successStatus, IsHealthy: true}, nil)
ret, err := server.CheckHealth(ctx, nil)
assert.NoError(t, err)
assert.Equal(t, true, ret.IsHealthy)
})
t.Run("CreateResourceGroup", func(t *testing.T) {
mqc.EXPECT().CreateResourceGroup(mock.Anything, mock.Anything).Return(successStatus, nil)
resp, err := server.CreateResourceGroup(ctx, nil)
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.ErrorCode)
})
t.Run("DropResourceGroup", func(t *testing.T) {
mqc.EXPECT().DropResourceGroup(mock.Anything, mock.Anything).Return(successStatus, nil)
resp, err := server.DropResourceGroup(ctx, nil)
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.ErrorCode)
})
t.Run("TransferNode", func(t *testing.T) {
mqc.EXPECT().TransferNode(mock.Anything, mock.Anything).Return(successStatus, nil)
resp, err := server.TransferNode(ctx, nil)
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.ErrorCode)
})
t.Run("TransferReplica", func(t *testing.T) {
mqc.EXPECT().TransferReplica(mock.Anything, mock.Anything).Return(successStatus, nil)
resp, err := server.TransferReplica(ctx, nil)
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.ErrorCode)
})
t.Run("ListResourceGroups", func(t *testing.T) {
req := &milvuspb.ListResourceGroupsRequest{}
mqc.EXPECT().ListResourceGroups(mock.Anything, req).Return(&milvuspb.ListResourceGroupsResponse{Status: successStatus}, nil)
resp, err := server.ListResourceGroups(ctx, req)
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
})
t.Run("DescribeResourceGroup", func(t *testing.T) {
mqc.EXPECT().DescribeResourceGroup(mock.Anything, mock.Anything).Return(&querypb.DescribeResourceGroupResponse{Status: successStatus}, nil)
resp, err := server.DescribeResourceGroup(ctx, nil)
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
})
err = server.Stop()
assert.NoError(t, err)
assert.Equal(t, true, ret.IsHealthy)
})
t.Run("CreateResourceGroup", func(t *testing.T) {
mqc.EXPECT().CreateResourceGroup(mock.Anything, mock.Anything).Return(successStatus, nil)
resp, err := server.CreateResourceGroup(ctx, nil)
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.ErrorCode)
})
t.Run("DropResourceGroup", func(t *testing.T) {
mqc.EXPECT().DropResourceGroup(mock.Anything, mock.Anything).Return(successStatus, nil)
resp, err := server.DropResourceGroup(ctx, nil)
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.ErrorCode)
})
t.Run("TransferNode", func(t *testing.T) {
mqc.EXPECT().TransferNode(mock.Anything, mock.Anything).Return(successStatus, nil)
resp, err := server.TransferNode(ctx, nil)
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.ErrorCode)
})
t.Run("TransferReplica", func(t *testing.T) {
mqc.EXPECT().TransferReplica(mock.Anything, mock.Anything).Return(successStatus, nil)
resp, err := server.TransferReplica(ctx, nil)
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.ErrorCode)
})
t.Run("ListResourceGroups", func(t *testing.T) {
req := &milvuspb.ListResourceGroupsRequest{}
mqc.EXPECT().ListResourceGroups(mock.Anything, req).Return(&milvuspb.ListResourceGroupsResponse{Status: successStatus}, nil)
resp, err := server.ListResourceGroups(ctx, req)
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
})
t.Run("DescribeResourceGroup", func(t *testing.T) {
mqc.EXPECT().DescribeResourceGroup(mock.Anything, mock.Anything).Return(&querypb.DescribeResourceGroupResponse{Status: successStatus}, nil)
resp, err := server.DescribeResourceGroup(ctx, nil)
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode)
})
err = server.Stop()
assert.NoError(t, err)
}
}
// This test will no longer return error immediately.
func TestServer_Run1(t *testing.T) {
t.Skip()
ctx := context.Background()
server, err := NewServer(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, server)
parameters := []string{"tikv", "etcd"}
for _, v := range parameters {
paramtable.Get().Save(paramtable.Get().MetaStoreCfg.MetaStoreType.Key, v)
t.Skip()
ctx := context.Background()
getTiKVClient = func(cfg *paramtable.TiKVConfig) (*txnkv.Client, error) {
return tikv.SetupLocalTxn(), nil
}
defer func() {
getTiKVClient = tikv.GetTiKVClient
}()
server, err := NewServer(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, server)
mqc := getQueryCoord()
mqc.EXPECT().Start().Return(errors.New("error"))
server.queryCoord = mqc
err = server.Run()
assert.Error(t, err)
mqc := getQueryCoord()
mqc.EXPECT().Start().Return(errors.New("error"))
server.queryCoord = mqc
err = server.Run()
assert.Error(t, err)
err = server.Stop()
assert.NoError(t, err)
err = server.Stop()
assert.NoError(t, err)
}
}
func TestServer_Run2(t *testing.T) {
ctx := context.Background()
server, err := NewServer(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, server)
parameters := []string{"tikv", "etcd"}
for _, v := range parameters {
paramtable.Get().Save(paramtable.Get().MetaStoreCfg.MetaStoreType.Key, v)
ctx := context.Background()
getTiKVClient = func(cfg *paramtable.TiKVConfig) (*txnkv.Client, error) {
return tikv.SetupLocalTxn(), nil
}
defer func() {
getTiKVClient = tikv.GetTiKVClient
}()
server, err := NewServer(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, server)
server.queryCoord = getQueryCoord()
server.rootCoord = &MockRootCoord{
initErr: errors.New("error"),
server.queryCoord = getQueryCoord()
server.rootCoord = &MockRootCoord{
initErr: errors.New("error"),
}
assert.Panics(t, func() { server.Run() })
err = server.Stop()
assert.NoError(t, err)
}
assert.Panics(t, func() { server.Run() })
err = server.Stop()
assert.NoError(t, err)
}
func getQueryCoord() *mocks.MockQueryCoord {
mqc := &mocks.MockQueryCoord{}
mqc.EXPECT().Init().Return(nil)
mqc.EXPECT().SetEtcdClient(mock.Anything)
mqc.EXPECT().SetTiKVClient(mock.Anything)
mqc.EXPECT().SetAddress(mock.Anything)
mqc.EXPECT().SetRootCoord(mock.Anything).Return(nil)
mqc.EXPECT().SetDataCoord(mock.Anything).Return(nil)
@ -361,53 +394,82 @@ func getQueryCoord() *mocks.MockQueryCoord {
mqc.EXPECT().Register().Return(nil)
mqc.EXPECT().Start().Return(nil)
mqc.EXPECT().Stop().Return(nil)
return mqc
}
func TestServer_Run3(t *testing.T) {
ctx := context.Background()
server, err := NewServer(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, server)
server.queryCoord = getQueryCoord()
server.rootCoord = &MockRootCoord{
startErr: errors.New("error"),
parameters := []string{"tikv", "etcd"}
for _, v := range parameters {
paramtable.Get().Save(paramtable.Get().MetaStoreCfg.MetaStoreType.Key, v)
ctx := context.Background()
getTiKVClient = func(cfg *paramtable.TiKVConfig) (*txnkv.Client, error) {
return tikv.SetupLocalTxn(), nil
}
defer func() {
getTiKVClient = tikv.GetTiKVClient
}()
server, err := NewServer(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, server)
server.queryCoord = getQueryCoord()
server.rootCoord = &MockRootCoord{
startErr: errors.New("error"),
}
assert.Panics(t, func() { server.Run() })
err = server.Stop()
assert.NoError(t, err)
}
assert.Panics(t, func() { server.Run() })
err = server.Stop()
assert.NoError(t, err)
}
func TestServer_Run4(t *testing.T) {
ctx := context.Background()
server, err := NewServer(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, server)
parameters := []string{"tikv", "etcd"}
for _, v := range parameters {
paramtable.Get().Save(paramtable.Get().MetaStoreCfg.MetaStoreType.Key, v)
ctx := context.Background()
getTiKVClient = func(cfg *paramtable.TiKVConfig) (*txnkv.Client, error) {
return tikv.SetupLocalTxn(), nil
}
defer func() {
getTiKVClient = tikv.GetTiKVClient
}()
server, err := NewServer(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, server)
server.queryCoord = getQueryCoord()
server.rootCoord = &MockRootCoord{}
server.dataCoord = &MockDataCoord{
initErr: errors.New("error"),
server.queryCoord = getQueryCoord()
server.rootCoord = &MockRootCoord{}
server.dataCoord = &MockDataCoord{
initErr: errors.New("error"),
}
assert.Panics(t, func() { server.Run() })
err = server.Stop()
assert.NoError(t, err)
}
assert.Panics(t, func() { server.Run() })
err = server.Stop()
assert.NoError(t, err)
}
func TestServer_Run5(t *testing.T) {
ctx := context.Background()
server, err := NewServer(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, server)
parameters := []string{"tikv", "etcd"}
for _, v := range parameters {
paramtable.Get().Save(paramtable.Get().MetaStoreCfg.MetaStoreType.Key, v)
ctx := context.Background()
getTiKVClient = func(cfg *paramtable.TiKVConfig) (*txnkv.Client, error) {
return tikv.SetupLocalTxn(), nil
}
defer func() {
getTiKVClient = tikv.GetTiKVClient
}()
server, err := NewServer(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, server)
server.queryCoord = getQueryCoord()
server.rootCoord = &MockRootCoord{}
server.dataCoord = &MockDataCoord{
startErr: errors.New("error"),
server.queryCoord = getQueryCoord()
server.rootCoord = &MockRootCoord{}
server.dataCoord = &MockDataCoord{
startErr: errors.New("error"),
}
assert.Panics(t, func() { server.Run() })
err = server.Stop()
assert.NoError(t, err)
}
assert.Panics(t, func() { server.Run() })
err = server.Stop()
assert.NoError(t, err)
}

29
internal/distributed/rootcoord/service.go
View File

@ -27,6 +27,7 @@ import (
"github.com/milvus-io/milvus/internal/util/dependency"
"github.com/milvus-io/milvus/pkg/tracer"
"github.com/milvus-io/milvus/pkg/util/interceptor"
"github.com/tikv/client-go/v2/txnkv"
clientv3 "go.etcd.io/etcd/client/v3"
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
"go.uber.org/atomic"
@ -42,10 +43,12 @@ import (
"github.com/milvus-io/milvus/internal/rootcoord"
"github.com/milvus-io/milvus/internal/types"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/util"
"github.com/milvus-io/milvus/pkg/util/etcd"
"github.com/milvus-io/milvus/pkg/util/funcutil"
"github.com/milvus-io/milvus/pkg/util/logutil"
"github.com/milvus-io/milvus/pkg/util/paramtable"
"github.com/milvus-io/milvus/pkg/util/tikv"
dcc "github.com/milvus-io/milvus/internal/distributed/datacoord/client"
qcc "github.com/milvus-io/milvus/internal/distributed/querycoord/client"
@ -65,6 +68,7 @@ type Server struct {
serverID atomic.Int64
etcdCli *clientv3.Client
tikvCli *txnkv.Client
dataCoord types.DataCoord
queryCoord types.QueryCoord
@ -152,9 +156,12 @@ func (s *Server) Run() error {
return nil
}
var getTiKVClient = tikv.GetTiKVClient
func (s *Server) init() error {
etcdConfig := &paramtable.Get().EtcdCfg
Params := &paramtable.Get().RootCoordGrpcServerCfg
params := paramtable.Get()
etcdConfig := &params.EtcdCfg
rpcParams := &params.RootCoordGrpcServerCfg
log.Debug("init params done..")
etcdCli, err := etcd.GetEtcdClient(
@ -171,10 +178,21 @@ func (s *Server) init() error {
}
s.etcdCli = etcdCli
s.rootCoord.SetEtcdClient(s.etcdCli)
s.rootCoord.SetAddress(Params.GetAddress())
s.rootCoord.SetAddress(rpcParams.GetAddress())
log.Debug("etcd connect done ...")
err = s.startGrpc(Params.Port.GetAsInt())
if params.MetaStoreCfg.MetaStoreType.GetValue() == util.MetaStoreTypeTiKV {
log.Info("Connecting to tikv metadata storage.")
s.tikvCli, err = getTiKVClient(&paramtable.Get().TiKVCfg)
if err != nil {
log.Debug("RootCoord failed to connect to tikv", zap.Error(err))
return err
}
s.rootCoord.SetTiKVClient(s.tikvCli)
log.Info("Connected to tikv. Using tikv as metadata storage.")
}
err = s.startGrpc(rpcParams.Port.GetAsInt())
if err != nil {
return err
}
@ -305,6 +323,9 @@ func (s *Server) Stop() error {
if s.etcdCli != nil {
defer s.etcdCli.Close()
}
if s.tikvCli != nil {
defer s.tikvCli.Close()
}
if s.dataCoord != nil {
if err := s.dataCoord.Stop(); err != nil {
log.Error("Failed to close dataCoord client", zap.Error(err))

263
internal/distributed/rootcoord/service_test.go
View File

@ -27,6 +27,7 @@ import (
"github.com/cockroachdb/errors"
"github.com/milvus-io/milvus/internal/util/sessionutil"
"github.com/tikv/client-go/v2/txnkv"
clientv3 "go.etcd.io/etcd/client/v3"
"github.com/stretchr/testify/assert"
@ -37,6 +38,7 @@ import (
"github.com/milvus-io/milvus/internal/types"
"github.com/milvus-io/milvus/pkg/util/etcd"
"github.com/milvus-io/milvus/pkg/util/paramtable"
"github.com/milvus-io/milvus/pkg/util/tikv"
)
type mockCore struct {
@ -76,6 +78,9 @@ func (m *mockCore) SetAddress(address string) {
func (m *mockCore) SetEtcdClient(etcdClient *clientv3.Client) {
}
func (m *mockCore) SetTiKVClient(client *txnkv.Client) {
}
func (m *mockCore) SetDataCoord(types.DataCoord) error {
return nil
}
@ -154,141 +159,191 @@ func (m *mockQueryCoord) Stop() error {
func TestRun(t *testing.T) {
paramtable.Init()
ctx, cancel := context.WithCancel(context.Background())
svr := Server{
rootCoord: &mockCore{},
ctx: ctx,
cancel: cancel,
grpcErrChan: make(chan error),
}
rcServerConfig := &paramtable.Get().RootCoordGrpcServerCfg
paramtable.Get().Save(rcServerConfig.Port.Key, "1000000")
err := svr.Run()
assert.Error(t, err)
assert.EqualError(t, err, "listen tcp: address 1000000: invalid port")
parameters := []string{"tikv", "etcd"}
for _, v := range parameters {
paramtable.Get().Save(paramtable.Get().MetaStoreCfg.MetaStoreType.Key, v)
ctx, cancel := context.WithCancel(context.Background())
getTiKVClient = func(cfg *paramtable.TiKVConfig) (*txnkv.Client, error) {
return tikv.SetupLocalTxn(), nil
}
defer func() {
getTiKVClient = tikv.GetTiKVClient
}()
svr := Server{
rootCoord: &mockCore{},
ctx: ctx,
cancel: cancel,
grpcErrChan: make(chan error),
}
rcServerConfig := &paramtable.Get().RootCoordGrpcServerCfg
paramtable.Get().Save(rcServerConfig.Port.Key, "1000000")
err := svr.Run()
assert.Error(t, err)
assert.EqualError(t, err, "listen tcp: address 1000000: invalid port")
svr.newDataCoordClient = func(string, *clientv3.Client) types.DataCoord {
return &mockDataCoord{}
}
svr.newQueryCoordClient = func(string, *clientv3.Client) types.QueryCoord {
return &mockQueryCoord{}
}
svr.newDataCoordClient = func(string, *clientv3.Client) types.DataCoord {
return &mockDataCoord{}
}
svr.newQueryCoordClient = func(string, *clientv3.Client) types.QueryCoord {
return &mockQueryCoord{}
}
paramtable.Get().Save(rcServerConfig.Port.Key, fmt.Sprintf("%d", rand.Int()%100+10000))
etcdConfig := &paramtable.Get().EtcdCfg
paramtable.Get().Save(rcServerConfig.Port.Key, fmt.Sprintf("%d", rand.Int()%100+10000))
etcdConfig := &paramtable.Get().EtcdCfg
rand.Seed(time.Now().UnixNano())
randVal := rand.Int()
rootPath := fmt.Sprintf("/%d/test", randVal)
rootcoord.Params.Save("etcd.rootPath", rootPath)
rand.Seed(time.Now().UnixNano())
randVal := rand.Int()
rootPath := fmt.Sprintf("/%d/test", randVal)
rootcoord.Params.Save("etcd.rootPath", rootPath)
etcdCli, err := etcd.GetEtcdClient(
etcdConfig.UseEmbedEtcd.GetAsBool(),
etcdConfig.EtcdUseSSL.GetAsBool(),
etcdConfig.Endpoints.GetAsStrings(),
etcdConfig.EtcdTLSCert.GetValue(),
etcdConfig.EtcdTLSKey.GetValue(),
etcdConfig.EtcdTLSCACert.GetValue(),
etcdConfig.EtcdTLSMinVersion.GetValue())
assert.NoError(t, err)
sessKey := path.Join(rootcoord.Params.EtcdCfg.MetaRootPath.GetValue(), sessionutil.DefaultServiceRoot)
_, err = etcdCli.Delete(ctx, sessKey, clientv3.WithPrefix())
assert.NoError(t, err)
err = svr.Run()
assert.NoError(t, err)
t.Run("CheckHealth", func(t *testing.T) {
ret, err := svr.CheckHealth(ctx, nil)
etcdCli, err := etcd.GetEtcdClient(
etcdConfig.UseEmbedEtcd.GetAsBool(),
etcdConfig.EtcdUseSSL.GetAsBool(),
etcdConfig.Endpoints.GetAsStrings(),
etcdConfig.EtcdTLSCert.GetValue(),
etcdConfig.EtcdTLSKey.GetValue(),
etcdConfig.EtcdTLSCACert.GetValue(),
etcdConfig.EtcdTLSMinVersion.GetValue())
assert.NoError(t, err)
assert.Equal(t, true, ret.IsHealthy)
})
t.Run("RenameCollection", func(t *testing.T) {
_, err := svr.RenameCollection(ctx, nil)
sessKey := path.Join(rootcoord.Params.EtcdCfg.MetaRootPath.GetValue(), sessionutil.DefaultServiceRoot)
_, err = etcdCli.Delete(ctx, sessKey, clientv3.WithPrefix())
assert.NoError(t, err)
err = svr.Run()
assert.NoError(t, err)
})
t.Run("CreateDatabase", func(t *testing.T) {
ret, err := svr.CreateDatabase(ctx, nil)
assert.Nil(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, ret.ErrorCode)
})
t.Run("CheckHealth", func(t *testing.T) {
ret, err := svr.CheckHealth(ctx, nil)
assert.NoError(t, err)
assert.Equal(t, true, ret.IsHealthy)
})
t.Run("DropDatabase", func(t *testing.T) {
ret, err := svr.DropDatabase(ctx, nil)
assert.Nil(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, ret.ErrorCode)
})
t.Run("RenameCollection", func(t *testing.T) {
_, err := svr.RenameCollection(ctx, nil)
assert.NoError(t, err)
})
t.Run("ListDatabases", func(t *testing.T) {
ret, err := svr.ListDatabases(ctx, nil)
assert.Nil(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, ret.Status.ErrorCode)
})
err = svr.Stop()
assert.NoError(t, err)
t.Run("CreateDatabase", func(t *testing.T) {
ret, err := svr.CreateDatabase(ctx, nil)
assert.Nil(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, ret.ErrorCode)
})
t.Run("DropDatabase", func(t *testing.T) {
ret, err := svr.DropDatabase(ctx, nil)
assert.Nil(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, ret.ErrorCode)
})
t.Run("ListDatabases", func(t *testing.T) {
ret, err := svr.ListDatabases(ctx, nil)
assert.Nil(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, ret.Status.ErrorCode)
})
err = svr.Stop()
assert.NoError(t, err)
}
}
func TestServerRun_DataCoordClientInitErr(t *testing.T) {
paramtable.Init()
ctx := context.Background()
server, err := NewServer(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, server)
parameters := []string{"tikv", "etcd"}
for _, v := range parameters {
paramtable.Get().Save(paramtable.Get().MetaStoreCfg.MetaStoreType.Key, v)
ctx := context.Background()
getTiKVClient = func(cfg *paramtable.TiKVConfig) (*txnkv.Client, error) {
return tikv.SetupLocalTxn(), nil
}
defer func() {
getTiKVClient = tikv.GetTiKVClient
}()
server, err := NewServer(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, server)
server.newDataCoordClient = func(string, *clientv3.Client) types.DataCoord {
return &mockDataCoord{initErr: errors.New("mock datacoord init error")}
server.newDataCoordClient = func(string, *clientv3.Client) types.DataCoord {
return &mockDataCoord{initErr: errors.New("mock datacoord init error")}
}
assert.Panics(t, func() { server.Run() })
err = server.Stop()
assert.NoError(t, err)
}
assert.Panics(t, func() { server.Run() })
err = server.Stop()
assert.NoError(t, err)
}
func TestServerRun_DataCoordClientStartErr(t *testing.T) {
paramtable.Init()
ctx := context.Background()
server, err := NewServer(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, server)
parameters := []string{"tikv", "etcd"}
for _, v := range parameters {
paramtable.Get().Save(paramtable.Get().MetaStoreCfg.MetaStoreType.Key, v)
ctx := context.Background()
getTiKVClient = func(cfg *paramtable.TiKVConfig) (*txnkv.Client, error) {
return tikv.SetupLocalTxn(), nil
}
defer func() {
getTiKVClient = tikv.GetTiKVClient
}()
server, err := NewServer(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, server)
server.newDataCoordClient = func(string, *clientv3.Client) types.DataCoord {
return &mockDataCoord{startErr: errors.New("mock datacoord start error")}
server.newDataCoordClient = func(string, *clientv3.Client) types.DataCoord {
return &mockDataCoord{startErr: errors.New("mock datacoord start error")}
}
assert.Panics(t, func() { server.Run() })
err = server.Stop()
assert.NoError(t, err)
}
assert.Panics(t, func() { server.Run() })
err = server.Stop()
assert.NoError(t, err)
}
func TestServerRun_QueryCoordClientInitErr(t *testing.T) {
paramtable.Init()
ctx := context.Background()
server, err := NewServer(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, server)
parameters := []string{"tikv", "etcd"}
for _, v := range parameters {
paramtable.Get().Save(paramtable.Get().MetaStoreCfg.MetaStoreType.Key, v)
ctx := context.Background()
getTiKVClient = func(cfg *paramtable.TiKVConfig) (*txnkv.Client, error) {
return tikv.SetupLocalTxn(), nil
}
defer func() {
getTiKVClient = tikv.GetTiKVClient
}()
server, err := NewServer(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, server)
server.newQueryCoordClient = func(string, *clientv3.Client) types.QueryCoord {
return &mockQueryCoord{initErr: errors.New("mock querycoord init error")}
server.newQueryCoordClient = func(string, *clientv3.Client) types.QueryCoord {
return &mockQueryCoord{initErr: errors.New("mock querycoord init error")}
}
assert.Panics(t, func() { server.Run() })
err = server.Stop()
assert.NoError(t, err)
}
assert.Panics(t, func() { server.Run() })
err = server.Stop()
assert.NoError(t, err)
}
func TestServer_QueryCoordClientStartErr(t *testing.T) {
paramtable.Init()
ctx := context.Background()
server, err := NewServer(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, server)
parameters := []string{"tikv", "etcd"}
for _, v := range parameters {
paramtable.Get().Save(paramtable.Get().MetaStoreCfg.MetaStoreType.Key, v)
ctx := context.Background()
getTiKVClient = func(cfg *paramtable.TiKVConfig) (*txnkv.Client, error) {
return tikv.SetupLocalTxn(), nil
}
defer func() {
getTiKVClient = tikv.GetTiKVClient
}()
server, err := NewServer(ctx, nil)
assert.NoError(t, err)
assert.NotNil(t, server)
server.newQueryCoordClient = func(string, *clientv3.Client) types.QueryCoord {
return &mockQueryCoord{startErr: errors.New("mock querycoord start error")}
server.newQueryCoordClient = func(string, *clientv3.Client) types.QueryCoord {
return &mockQueryCoord{startErr: errors.New("mock querycoord start error")}
}
assert.Panics(t, func() { server.Run() })
err = server.Stop()
assert.NoError(t, err)
}
assert.Panics(t, func() { server.Run() })
err = server.Stop()
assert.NoError(t, err)
}

93
internal/kv/tikv/main_test.go Normal file
View File

@ -0,0 +1,93 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tikv
import (
"context"
"os"
"testing"
"github.com/tikv/client-go/v2/rawkv"
"github.com/tikv/client-go/v2/testutils"
tilib "github.com/tikv/client-go/v2/tikv"
"github.com/tikv/client-go/v2/txnkv"
"github.com/milvus-io/milvus/pkg/util/paramtable"
)
var txnClient *txnkv.Client
var rawClient *rawkv.Client
// creates a local TiKV Store for testing purpose.
func setupLocalTiKV() {
setupLocalTxn()
setupLocalRaw()
}
func setupLocalTxn() {
client, cluster, pdClient, err := testutils.NewMockTiKV("", nil)
if err != nil {
panic(err)
}
testutils.BootstrapWithSingleStore(cluster)
store, err := tilib.NewTestTiKVStore(client, pdClient, nil, nil, 0)
if err != nil {
panic(err)
}
txnClient = &txnkv.Client{KVStore: store}
}
func setupLocalRaw() {
client, cluster, pdClient, err := testutils.NewMockTiKV("", nil)
if err != nil {
panic(err)
}
testutils.BootstrapWithSingleStore(cluster)
rawClient = &rawkv.Client{}
p := rawkv.ClientProbe{Client: rawClient}
p.SetPDClient(pdClient)
p.SetRegionCache(tilib.NewRegionCache(pdClient))
p.SetRPCClient(client)
}
// Connects to a remote TiKV service for testing purpose. By default, it assumes the TiKV is from localhost.
func setupRemoteTiKV() {
pdsn := "127.0.0.1:2379"
var err error
txnClient, err = txnkv.NewClient([]string{pdsn})
if err != nil {
panic(err)
}
rawClient, err = rawkv.NewClientWithOpts(context.Background(), []string{pdsn})
if err != nil {
panic(err)
}
}
func setupTiKV(use_remote bool) {
if use_remote {
setupRemoteTiKV()
} else {
setupLocalTiKV()
}
}
func TestMain(m *testing.M) {
paramtable.Init()
setupTiKV(false)
code := m.Run()
os.Exit(code)
}

812
internal/kv/tikv/txn_tikv.go Normal file
View File

@ -0,0 +1,812 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tikv
import (
"bytes"
"context"
"fmt"
"math"
"path"
"time"
"github.com/cockroachdb/errors"
tikverr "github.com/tikv/client-go/v2/error"
tikv "github.com/tikv/client-go/v2/kv"
"github.com/tikv/client-go/v2/txnkv"
"github.com/tikv/client-go/v2/txnkv/transaction"
"github.com/tikv/client-go/v2/txnkv/txnsnapshot"
"go.uber.org/zap"
"github.com/milvus-io/milvus/internal/kv"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/metrics"
"github.com/milvus-io/milvus/pkg/util/paramtable"
"github.com/milvus-io/milvus/pkg/util/timerecord"
)
// A quick note is that we are using logging_error at our outermost scope in order to perform logging
const (
// We are using a Snapshot instead of transaction when doing read only operations due to the
// lower overhead (50% less overhead in small tests). In order to guarantee the latest values are
// grabbed at call, we can set the TS to be the max uint64.
MaxSnapshotTS = uint64(math.MaxUint64)
// Whether to enable transaction rollback if a transaction fails. Due to TiKV transactions being
// optimistic by default, a rollback does not need to be done and the transaction can just be
// discarded. Discarding saw a small bump in performance on small scale tests.
EnableRollback = false
// This empty value is what we are reserving within TiKv to represent an empty string value.
// TiKV does not allow storing empty values for keys which is something we do in Milvus, so
// to get over this we are using the reserved keyword as placeholder.
EmptyValueString = "__milvus_reserved_empty_tikv_value_DO_NOT_USE"
)
var Params *paramtable.ComponentParam = paramtable.Get()
// For reads by prefix we can customize the scan size to increase/decrease rpc calls.
var SnapshotScanSize int
// RequestTimeout is the default timeout for tikv request.
var RequestTimeout time.Duration
var EmptyValueByte = []byte(EmptyValueString)
func tiTxnBegin(txn *txnkv.Client) (*transaction.KVTxn, error) {
return txn.Begin()
}
func tiTxnCommit(txn *transaction.KVTxn, ctx context.Context) error {
return txn.Commit(ctx)
}
func tiTxnSnapshot(txn *txnkv.Client, paginationSize int) *txnsnapshot.KVSnapshot {
ss := txn.GetSnapshot(MaxSnapshotTS)
ss.SetScanBatchSize(paginationSize)
return ss
}
var beginTxn = tiTxnBegin
var commitTxn = tiTxnCommit
var getSnapshot = tiTxnSnapshot
// implementation assertion
var _ kv.MetaKv = (*txnTiKV)(nil)
// txnTiKV implements MetaKv and TxnKV interface. It supports processing multiple kvs within one transaction.
type txnTiKV struct {
txn *txnkv.Client
rootPath string
}
// NewTiKV creates a new txnTiKV client.
func NewTiKV(txn *txnkv.Client, rootPath string) *txnTiKV {
SnapshotScanSize = Params.TiKVCfg.SnapshotScanSize.GetAsInt()
RequestTimeout = Params.TiKVCfg.RequestTimeout.GetAsDuration(time.Millisecond)
kv := &txnTiKV{
txn: txn,
rootPath: rootPath,
}
return kv
}
// Close closes the connection to TiKV.
func (kv *txnTiKV) Close() {
log.Info("txnTiKV closed", zap.String("path", kv.rootPath))
}
// GetPath returns the path of the key/prefix.
func (kv *txnTiKV) GetPath(key string) string {
return path.Join(kv.rootPath, key)
}
// Log if error is not nil. We use error pointer as in most cases this function
// is Deferred. Deferred functions evaluate their args immediately.
func logWarnOnFailure(err *error, msg string, fields ...zap.Field) {
if *err != nil {
fields = append(fields, zap.Error(*err))
log.Warn(msg, fields...)
}
}
// Has returns if a key exists.
func (kv *txnTiKV) Has(key string) (bool, error) {
start := time.Now()
key = path.Join(kv.rootPath, key)
ctx, cancel := context.WithTimeout(context.Background(), RequestTimeout)
defer cancel()
var logging_error error
defer logWarnOnFailure(&logging_error, "txnTiKV Has() error", zap.String("key", key))
_, err := kv.getTiKVMeta(ctx, key)
if err != nil {
// Dont error out if not present unless failed call to tikv
if common.IsKeyNotExistError(err) {
return false, nil
} else {
logging_error = errors.Wrap(err, fmt.Sprintf("Failed to read key: %s", key))
return false, logging_error
}
}
CheckElapseAndWarn(start, "Slow txnTiKV Has() operation", zap.String("key", key))
return true, nil
}
func rollbackOnFailure(err *error, txn *transaction.KVTxn) {
if *err != nil && EnableRollback == true {
txn.Rollback()
}
}
// HasPrefix returns if a key prefix exists.
func (kv *txnTiKV) HasPrefix(prefix string) (bool, error) {
start := time.Now()
prefix = path.Join(kv.rootPath, prefix)
var logging_error error
defer logWarnOnFailure(&logging_error, "txnTiKV HasPrefix() error", zap.String("prefix", prefix))
ss := getSnapshot(kv.txn, SnapshotScanSize)
// Retrieve bounding keys for prefix
startKey := []byte(prefix)
endKey := tikv.PrefixNextKey([]byte(prefix))
iter, err := ss.Iter(startKey, endKey)
if err != nil {
logging_error = errors.Wrap(err, fmt.Sprintf("Failed to create iterator for prefix: %s", prefix))
return false, logging_error
}
defer iter.Close()
r := false
// Iterater only needs to check the first key-value pair
if iter.Valid() {
r = true
}
CheckElapseAndWarn(start, "Slow txnTiKV HasPrefix() operation", zap.String("prefix", prefix))
return r, nil
}
// Load returns value of the key.
func (kv *txnTiKV) Load(key string) (string, error) {
start := time.Now()
key = path.Join(kv.rootPath, key)
ctx, cancel := context.WithTimeout(context.Background(), RequestTimeout)
defer cancel()
var logging_error error
defer logWarnOnFailure(&logging_error, "txnTiKV Load() error", zap.String("key", key))
val, err := kv.getTiKVMeta(ctx, key)
if err != nil {
if common.IsKeyNotExistError(err) {
logging_error = err
} else {
logging_error = errors.Wrap(err, fmt.Sprintf("Failed to read key %s", key))
}
return "", logging_error
}
CheckElapseAndWarn(start, "Slow txnTiKV Load() operation", zap.String("key", key))
return val, nil
}
func batchConvertFromString(prefix string, keys []string) [][]byte {
output := make([][]byte, len(keys))
for i := 0; i < len(keys); i++ {
keys[i] = path.Join(prefix, keys[i])
output[i] = []byte(keys[i])
}
return output
}
// MultiLoad gets the values of input keys in a transaction.
func (kv *txnTiKV) MultiLoad(keys []string) ([]string, error) {
start := time.Now()
ctx, cancel := context.WithTimeout(context.Background(), RequestTimeout)
defer cancel()
var logging_error error
defer logWarnOnFailure(&logging_error, "txnTiKV MultiLoad() error", zap.Strings("keys", keys))
// Convert from []string to [][]byte
byte_keys := batchConvertFromString(kv.rootPath, keys)
// Since only reading, use Snapshot for less overhead
ss := getSnapshot(kv.txn, SnapshotScanSize)
key_map, err := ss.BatchGet(ctx, byte_keys)
if err != nil {
logging_error = errors.Wrap(err, "Failed ss.BatchGet() for MultiLoad")
return nil, logging_error
}
missing_values := []string{}
valid_values := []string{}
// Loop through keys and build valid/invalid slices
for _, k := range keys {
v, ok := key_map[k]
if !ok {
missing_values = append(missing_values, k)
}
// Check if empty value placeholder
str_val := convertEmptyByteToString(v)
valid_values = append(valid_values, str_val)
}
if len(missing_values) != 0 {
logging_error = fmt.Errorf("There are invalid keys: %s", missing_values)
}
CheckElapseAndWarn(start, "Slow txnTiKV MultiLoad() operation", zap.Any("keys", keys))
return valid_values, logging_error
}
// LoadWithPrefix returns all the keys and values for the given key prefix.
func (kv *txnTiKV) LoadWithPrefix(prefix string) ([]string, []string, error) {
start := time.Now()
prefix = path.Join(kv.rootPath, prefix)
var logging_error error
defer logWarnOnFailure(&logging_error, "txnTiKV LoadWithPrefix() error", zap.String("prefix", prefix))
ss := getSnapshot(kv.txn, SnapshotScanSize)
// Retrieve key-value pairs with the specified prefix
startKey := []byte(prefix)
endKey := tikv.PrefixNextKey([]byte(prefix))
iter, err := ss.Iter(startKey, endKey)
if err != nil {
logging_error = errors.Wrap(err, fmt.Sprintf("Failed to create iterater for LoadWithPrefix() for prefix: %s", prefix))
return nil, nil, logging_error
}
defer iter.Close()
var keys []string
var values []string
// Iterate over the key-value pairs
for iter.Valid() {
val := iter.Value()
// Check if empty value placeholder
str_val := convertEmptyByteToString(val)
keys = append(keys, string(iter.Key()))
values = append(values, str_val)
err = iter.Next()
if err != nil {
logging_error = errors.Wrap(err, fmt.Sprintf("Failed to iterate for LoadWithPrefix() for prefix: %s", prefix))
return nil, nil, logging_error
}
}
CheckElapseAndWarn(start, "Slow txnTiKV LoadWithPrefix() operation", zap.String("prefix", prefix))
return keys, values, nil
}
// Save saves the input key-value pair.
func (kv *txnTiKV) Save(key, value string) error {
key = path.Join(kv.rootPath, key)
ctx, cancel := context.WithTimeout(context.Background(), RequestTimeout)
defer cancel()
var logging_error error
defer logWarnOnFailure(&logging_error, "txnTiKV Save() error", zap.String("key", key), zap.String("value", value))
logging_error = kv.putTiKVMeta(ctx, key, value)
return logging_error
}
// MultiSave saves the input key-value pairs in transaction manner.
func (kv *txnTiKV) MultiSave(kvs map[string]string) error {
start := time.Now()
ctx, cancel := context.WithTimeout(context.Background(), RequestTimeout)
defer cancel()
var logging_error error
defer logWarnOnFailure(&logging_error, "txnTiKV MultiSave() error", zap.Any("kvs", kvs), zap.Int("len", len(kvs)))
txn, err := beginTxn(kv.txn)
if err != nil {
logging_error = errors.Wrap(err, "Failed to create txn for MultiSave")
return logging_error
}
// Defer a rollback only if the transaction hasn't been committed
defer rollbackOnFailure(&logging_error, txn)
for key, value := range kvs {
key = path.Join(kv.rootPath, key)
// Check if value is empty or taking reserved EmptyValue
byte_value, err := convertEmptyStringToByte(value)
if err != nil {
logging_error = errors.Wrap(err, fmt.Sprintf("Failed to cast to byte (%s:%s) for MultiSave()", key, value))
return logging_error
}
// Save the value within a transaction
err = txn.Set([]byte(key), byte_value)
if err != nil {
logging_error = errors.Wrap(err, fmt.Sprintf("Failed to set (%s:%s) for MultiSave()", key, value))
return logging_error
}
}
err = kv.executeTxn(txn, ctx)
if err != nil {
logging_error = errors.Wrap(err, "Failed to commit for MultiSave()")
return logging_error
}
CheckElapseAndWarn(start, "Slow txnTiKV MultiSave() operation", zap.Any("kvs", kvs))
return nil
}
// Remove removes the input key.
func (kv *txnTiKV) Remove(key string) error {
key = path.Join(kv.rootPath, key)
ctx, cancel := context.WithTimeout(context.Background(), RequestTimeout)
defer cancel()
var logging_error error
defer logWarnOnFailure(&logging_error, "txnTiKV Remove() error", zap.String("key", key))
logging_error = kv.removeTiKVMeta(ctx, key)
return logging_error
}
// MultiRemove removes the input keys in transaction manner.
func (kv *txnTiKV) MultiRemove(keys []string) error {
start := time.Now()
ctx, cancel := context.WithTimeout(context.Background(), RequestTimeout)
defer cancel()
var logging_error error
defer logWarnOnFailure(&logging_error, "txnTiKV MultiRemove() error", zap.Strings("keys", keys), zap.Int("len", len(keys)))
txn, err := beginTxn(kv.txn)
if err != nil {
logging_error = errors.Wrap(err, "Failed to create txn for MultiRemove")
return logging_error
}
// Defer a rollback only if the transaction hasn't been committed
defer rollbackOnFailure(&logging_error, txn)
for _, key := range keys {
key = path.Join(kv.rootPath, key)
logging_error = txn.Delete([]byte(key))
if err != nil {
logging_error = errors.Wrap(err, fmt.Sprintf("Failed to delete %s for MultiRemove", key))
return logging_error
}
}
err = kv.executeTxn(txn, ctx)
if err != nil {
logging_error = errors.Wrap(err, "Failed to commit for MultiRemove()")
return logging_error
}
CheckElapseAndWarn(start, "Slow txnTiKV MultiRemove() operation", zap.Strings("keys", keys))
return nil
}
// RemoveWithPrefix removes the keys for the given prefix.
func (kv *txnTiKV) RemoveWithPrefix(prefix string) error {
start := time.Now()
prefix = path.Join(kv.rootPath, prefix)
ctx, cancel := context.WithTimeout(context.Background(), RequestTimeout)
defer cancel()
var logging_error error
defer logWarnOnFailure(&logging_error, "txnTiKV RemoveWithPrefix() error", zap.String("prefix", prefix))
startKey := []byte(prefix)
endKey := tikv.PrefixNextKey(startKey)
_, err := kv.txn.DeleteRange(ctx, startKey, endKey, 1)
if err != nil {
logging_error = errors.Wrap(err, "Failed to DeleteRange for RemoveWithPrefix")
return logging_error
}
CheckElapseAndWarn(start, "Slow txnTiKV RemoveWithPrefix() operation", zap.String("prefix", prefix))
return nil
}
// MultiSaveAndRemove saves the key-value pairs and removes the keys in a transaction.
func (kv *txnTiKV) MultiSaveAndRemove(saves map[string]string, removals []string) error {
start := time.Now()
ctx, cancel := context.WithTimeout(context.Background(), RequestTimeout)
defer cancel()
var logging_error error
defer logWarnOnFailure(&logging_error, "txnTiKV MultiSaveAndRemove error", zap.Any("saves", saves), zap.Strings("removes", removals), zap.Int("saveLength", len(saves)), zap.Int("removeLength", len(removals)))
txn, err := beginTxn(kv.txn)
if err != nil {
logging_error = errors.Wrap(err, "Failed to create txn for MultiSaveAndRemove")
return logging_error
}
// Defer a rollback only if the transaction hasn't been committed
defer rollbackOnFailure(&logging_error, txn)
for key, value := range saves {
key = path.Join(kv.rootPath, key)
// Check if value is empty or taking reserved EmptyValue
byte_value, err := convertEmptyStringToByte(value)
if err != nil {
logging_error = errors.Wrap(err, fmt.Sprintf("Failed to cast to byte (%s:%s) for MultiSaveAndRemove", key, value))
return logging_error
}
err = txn.Set([]byte(key), byte_value)
if err != nil {
logging_error = errors.Wrap(err, fmt.Sprintf("Failed to set (%s:%s) for MultiSaveAndRemove", key, value))
return logging_error
}
}
for _, key := range removals {
key = path.Join(kv.rootPath, key)
if err = txn.Delete([]byte(key)); err != nil {
logging_error = errors.Wrap(err, fmt.Sprintf("Failed to delete %s for MultiSaveAndRemove", key))
return logging_error
}
}
err = kv.executeTxn(txn, ctx)
if err != nil {
logging_error = errors.Wrap(err, "Failed to commit for MultiSaveAndRemove")
return logging_error
}
CheckElapseAndWarn(start, "Slow txnTiKV MultiSaveAndRemove() operation", zap.Any("saves", saves), zap.Strings("removals", removals))
return nil
}
// MultiRemoveWithPrefix removes the keys with given prefix.
func (kv *txnTiKV) MultiRemoveWithPrefix(prefixes []string) error {
start := time.Now()
ctx, cancel := context.WithTimeout(context.Background(), RequestTimeout)
defer cancel()
var logging_error error
defer logWarnOnFailure(&logging_error, "txnTiKV MultiRemoveWithPrefix error", zap.Strings("keys", prefixes), zap.Int("len", len(prefixes)))
txn, err := beginTxn(kv.txn)
if err != nil {
logging_error = errors.Wrap(err, "Failed to create txn for MultiRemoveWithPrefix")
return logging_error
}
// Defer a rollback only if the transaction hasn't been committed
defer rollbackOnFailure(&logging_error, txn)
// Need in order for err to be propogated to the defered logger
// := within forloop will shadow the original variable, resulting in logger
// missing it
for _, prefix := range prefixes {
prefix = path.Join(kv.rootPath, prefix)
// Get the start and end keys for the prefix range
startKey := []byte(prefix)
endKey := tikv.PrefixNextKey([]byte(prefix))
// Use Scan to iterate over keys in the prefix range
iter, err := txn.Iter(startKey, endKey)
if err != nil {
logging_error = errors.Wrap(err, "Failed to create iterater for MultiRemoveWithPrefix")
return logging_error
}
// Iterate over keys and delete them
for iter.Valid() {
key := iter.Key()
err = txn.Delete(key)
if err != nil {
logging_error = errors.Wrap(err, fmt.Sprintf("Failed to delete %s for MultiRemoveWithPrefix", string(key)))
return logging_error
}
// Move the iterator to the next key
err = iter.Next()
if err != nil {
logging_error = errors.Wrap(err, fmt.Sprintf("Failed to move Iterator after key %s for MultiRemoveWithPrefix", string(key)))
return logging_error
}
}
}
err = kv.executeTxn(txn, ctx)
if err != nil {
logging_error = errors.Wrap(err, "Failed to commit for MultiRemoveWithPrefix")
return logging_error
}
CheckElapseAndWarn(start, "Slow txnTiKV MultiRemoveWithPrefix() operation", zap.Strings("keys", prefixes))
return nil
}
// MultiSaveAndRemoveWithPrefix saves kv in @saves and removes the keys with given prefix in @removals.
func (kv *txnTiKV) MultiSaveAndRemoveWithPrefix(saves map[string]string, removals []string) error {
start := time.Now()
ctx, cancel := context.WithTimeout(context.Background(), RequestTimeout)
defer cancel()
var logging_error error
defer logWarnOnFailure(&logging_error, "txnTiKV MultiSaveAndRemoveWithPrefix() error", zap.Any("saves", saves), zap.Strings("removes", removals), zap.Int("saveLength", len(saves)), zap.Int("removeLength", len(removals)))
txn, err := beginTxn(kv.txn)
if err != nil {
logging_error = errors.Wrap(err, "Failed to create txn for MultiSaveAndRemoveWithPrefix")
return logging_error
}
// Defer a rollback only if the transaction hasn't been committed
defer rollbackOnFailure(&logging_error, txn)
// Save key-value pairs
for key, value := range saves {
key = path.Join(kv.rootPath, key)
// Check if value is empty or taking reserved EmptyValue
byte_value, err := convertEmptyStringToByte(value)
if err != nil {
logging_error = errors.Wrap(err, fmt.Sprintf("Failed to cast to byte (%s:%s) for MultiSaveAndRemoveWithPrefix()", key, value))
return logging_error
}
err = txn.Set([]byte(key), byte_value)
if err != nil {
logging_error = errors.Wrap(err, fmt.Sprintf("Failed to set (%s:%s) for MultiSaveAndRemoveWithPrefix()", key, value))
return logging_error
}
}
// Remove keys with prefix
for _, prefix := range removals {
prefix = path.Join(kv.rootPath, prefix)
// Get the start and end keys for the prefix range
startKey := []byte(prefix)
endKey := tikv.PrefixNextKey([]byte(prefix))
// Use Scan to iterate over keys in the prefix range
iter, err := txn.Iter(startKey, endKey)
if err != nil {
logging_error = errors.Wrap(err, fmt.Sprintf("Failed to create iterater for %s during MultiSaveAndRemoveWithPrefix()", prefix))
return logging_error
}
// Iterate over keys and delete them
for iter.Valid() {
key := iter.Key()
err = txn.Delete(key)
if logging_error != nil {
logging_error = errors.Wrap(err, fmt.Sprintf("Failed to delete %s for MultiSaveAndRemoveWithPrefix", string(key)))
return logging_error
}
// Move the iterator to the next key
err = iter.Next()
if err != nil {
logging_error = errors.Wrap(err, fmt.Sprintf("Failed to move Iterator after key %s for MultiSaveAndRemoveWithPrefix", string(key)))
return logging_error
}
}
}
err = kv.executeTxn(txn, ctx)
if err != nil {
logging_error = errors.Wrap(err, "Failed to commit for MultiSaveAndRemoveWithPrefix")
return logging_error
}
CheckElapseAndWarn(start, "Slow txnTiKV MultiSaveAndRemoveWithPrefix() operation", zap.Any("saves", saves), zap.Strings("removals", removals))
return nil
}
// WalkWithPrefix visits each kv with input prefix and apply given fn to it.
func (kv *txnTiKV) WalkWithPrefix(prefix string, paginationSize int, fn func([]byte, []byte) error) error {
start := time.Now()
prefix = path.Join(kv.rootPath, prefix)
var logging_error error
defer logWarnOnFailure(&logging_error, "txnTiKV WalkWithPagination error", zap.String("prefix", prefix))
// Since only reading, use Snapshot for less overhead
ss := getSnapshot(kv.txn, paginationSize)
// Retrieve key-value pairs with the specified prefix
startKey := []byte(prefix)
endKey := tikv.PrefixNextKey([]byte(prefix))
iter, err := ss.Iter(startKey, endKey)
if err != nil {
logging_error = errors.Wrap(err, fmt.Sprintf("Failed to create iterater for %s during WalkWithPrefix", prefix))
return logging_error
}
defer iter.Close()
// Iterate over the key-value pairs
for iter.Valid() {
// Grab value for empty check
byte_val := iter.Value()
// Check if empty val and replace with placeholder
if isEmptyByte(byte_val) {
byte_val = []byte{}
}
err = fn(iter.Key(), byte_val)
if err != nil {
logging_error = errors.Wrap(err, fmt.Sprintf("Failed to apply fn to (%s;%s)", string(iter.Key()), string(byte_val)))
return logging_error
}
err = iter.Next()
if err != nil {
logging_error = errors.Wrap(err, fmt.Sprintf("Failed to move Iterator after key %s for WalkWithPrefix", string(iter.Key())))
return logging_error
}
}
CheckElapseAndWarn(start, "Slow txnTiKV WalkWithPagination() operation", zap.String("prefix", prefix))
return nil
}
func (kv *txnTiKV) executeTxn(txn *transaction.KVTxn, ctx context.Context) error {
start := timerecord.NewTimeRecorder("executeTxn")
elapsed := start.ElapseSpan()
metrics.MetaOpCounter.WithLabelValues(metrics.MetaTxnLabel, metrics.TotalLabel).Inc()
err := commitTxn(txn, ctx)
if err == nil {
metrics.MetaRequestLatency.WithLabelValues(metrics.MetaTxnLabel).Observe(float64(elapsed.Milliseconds()))
metrics.MetaOpCounter.WithLabelValues(metrics.MetaTxnLabel, metrics.SuccessLabel).Inc()
} else {
metrics.MetaOpCounter.WithLabelValues(metrics.MetaTxnLabel, metrics.FailLabel).Inc()
}
return err
}
func (kv *txnTiKV) getTiKVMeta(ctx context.Context, key string) (string, error) {
ctx1, cancel := context.WithTimeout(ctx, RequestTimeout)
defer cancel()
start := timerecord.NewTimeRecorder("getTiKVMeta")
ss := getSnapshot(kv.txn, SnapshotScanSize)
val, err := ss.Get(ctx1, []byte(key))
if err != nil {
// Log key read fail
metrics.MetaOpCounter.WithLabelValues(metrics.MetaGetLabel, metrics.FailLabel).Inc()
if err == tikverr.ErrNotExist {
// If key is missing
return "", common.NewKeyNotExistError(key)
} else {
// If call to tikv fails
return "", errors.Wrap(err, fmt.Sprintf("Failed to get value for key %s in getTiKVMeta", key))
}
}
// Check if value is the empty placeholder
str_val := convertEmptyByteToString(val)
elapsed := start.ElapseSpan()
metrics.MetaOpCounter.WithLabelValues(metrics.MetaGetLabel, metrics.TotalLabel).Inc()
metrics.MetaKvSize.WithLabelValues(metrics.MetaGetLabel).Observe(float64(len(val)))
metrics.MetaRequestLatency.WithLabelValues(metrics.MetaGetLabel).Observe(float64(elapsed.Milliseconds()))
metrics.MetaOpCounter.WithLabelValues(metrics.MetaGetLabel, metrics.SuccessLabel).Inc()
return str_val, nil
}
func (kv *txnTiKV) putTiKVMeta(ctx context.Context, key, val string) error {
ctx1, cancel := context.WithTimeout(ctx, RequestTimeout)
defer cancel()
start := timerecord.NewTimeRecorder("putTiKVMeta")
txn, err := beginTxn(kv.txn)
if err != nil {
return errors.Wrap(err, "Failed to build transaction for putTiKVMeta")
}
// Defer a rollback only if the transaction hasn't been committed
defer rollbackOnFailure(&err, txn)
// Check if the value being written needs to be empty plaeholder
byte_value, err := convertEmptyStringToByte(val)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("Failed to cast to byte (%s:%s) for putTiKVMeta", key, val))
}
err = txn.Set([]byte(key), byte_value)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("Failed to set value for key %s in putTiKVMeta", key))
}
err = commitTxn(txn, ctx1)
elapsed := start.ElapseSpan()
metrics.MetaOpCounter.WithLabelValues(metrics.MetaPutLabel, metrics.TotalLabel).Inc()
if err == nil {
metrics.MetaKvSize.WithLabelValues(metrics.MetaPutLabel).Observe(float64(len(byte_value)))
metrics.MetaRequestLatency.WithLabelValues(metrics.MetaPutLabel).Observe(float64(elapsed.Milliseconds()))
metrics.MetaOpCounter.WithLabelValues(metrics.MetaPutLabel, metrics.SuccessLabel).Inc()
} else {
metrics.MetaOpCounter.WithLabelValues(metrics.MetaPutLabel, metrics.FailLabel).Inc()
}
return err
}
func (kv *txnTiKV) removeTiKVMeta(ctx context.Context, key string) error {
ctx1, cancel := context.WithTimeout(ctx, RequestTimeout)
defer cancel()
start := timerecord.NewTimeRecorder("removeTiKVMeta")
txn, err := beginTxn(kv.txn)
if err != nil {
return errors.Wrap(err, "Failed to build transaction for removeTiKVMeta")
}
// Defer a rollback only if the transaction hasn't been committed
defer rollbackOnFailure(&err, txn)
err = txn.Delete([]byte(key))
if err != nil {
return errors.Wrap(err, fmt.Sprintf("Failed to remove key %s in removeTiKVMeta", key))
}
err = commitTxn(txn, ctx1)
elapsed := start.ElapseSpan()
metrics.MetaOpCounter.WithLabelValues(metrics.MetaRemoveLabel, metrics.TotalLabel).Inc()
if err == nil {
metrics.MetaRequestLatency.WithLabelValues(metrics.MetaRemoveLabel).Observe(float64(elapsed.Milliseconds()))
metrics.MetaOpCounter.WithLabelValues(metrics.MetaRemoveLabel, metrics.SuccessLabel).Inc()
} else {
metrics.MetaOpCounter.WithLabelValues(metrics.MetaRemoveLabel, metrics.FailLabel).Inc()
}
return err
}
func (kv *txnTiKV) CompareVersionAndSwap(key string, version int64, target string) (bool, error) {
err := fmt.Errorf("Unimplemented! CompareVersionAndSwap is under deprecation")
logWarnOnFailure(&err, "Unimplemented")
return false, err
}
// CheckElapseAndWarn checks the elapsed time and warns if it is too long.
func CheckElapseAndWarn(start time.Time, message string, fields ...zap.Field) bool {
elapsed := time.Since(start)
if elapsed.Milliseconds() > 2000 {
log.Warn(message, append([]zap.Field{zap.String("time spent", elapsed.String())}, fields...)...)
return true
}
return false
}
// Since TiKV cannot store empty key values, we assign them a placeholder held by EmptyValue.
// Upon loading, we need to check if the returned value is the placeholder.
func isEmptyByte(value []byte) bool {
return bytes.Equal(value, EmptyValueByte) || len(value) == 0
}
// Return an empty string if the value is the Empty placeholder, else return actual string value.
func convertEmptyByteToString(value []byte) string {
if isEmptyByte(value) {
return ""
} else {
return string(value)
}
}
// Convert string into EmptyValue if empty else cast to []byte. Will throw error if value is equal
// to the EmptyValueString.
func convertEmptyStringToByte(value string) ([]byte, error) {
if len(value) == 0 {
return EmptyValueByte, nil
} else if value == EmptyValueString {
return nil, fmt.Errorf("Value for key is reserved by EmptyValue: %s", EmptyValueString)
} else {
return []byte(value), nil
}
}

639
internal/kv/tikv/txn_tikv_test.go Normal file
View File

@ -0,0 +1,639 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tikv
import (
"context"
"fmt"
"sort"
"testing"
"time"
"github.com/cockroachdb/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/tikv/client-go/v2/txnkv"
"github.com/tikv/client-go/v2/txnkv/transaction"
"golang.org/x/exp/maps"
)
func TestTiKVLoad(te *testing.T) {
te.Run("kv SaveAndLoad", func(t *testing.T) {
rootPath := "/tikv/test/root/saveandload"
kv := NewTiKV(txnClient, rootPath)
err := kv.RemoveWithPrefix("")
require.NoError(t, err)
defer kv.Close()
defer kv.RemoveWithPrefix("")
saveAndLoadTests := []struct {
key string
value string
}{
{"test1", "value1"},
{"test2", "value2"},
{"test1/a", "value_a"},
{"test1/b", "value_b"},
}
for i, test := range saveAndLoadTests {
if i < 4 {
err = kv.Save(test.key, test.value)
assert.NoError(t, err)
}
val, err := kv.Load(test.key)
assert.NoError(t, err)
assert.Equal(t, test.value, val)
}
invalidLoadTests := []struct {
invalidKey string
}{
{"t"},
{"a"},
{"test1a"},
}
for _, test := range invalidLoadTests {
val, err := kv.Load(test.invalidKey)
assert.Error(t, err)
assert.Zero(t, val)
}
loadPrefixTests := []struct {
prefix string
expectedKeys []string
expectedValues []string
expectedError error
}{
{"test", []string{
kv.GetPath("test1"),
kv.GetPath("test2"),
kv.GetPath("test1/a"),
kv.GetPath("test1/b")}, []string{"value1", "value2", "value_a", "value_b"}, nil},
{"test1", []string{
kv.GetPath("test1"),
kv.GetPath("test1/a"),
kv.GetPath("test1/b")}, []string{"value1", "value_a", "value_b"}, nil},
{"test2", []string{kv.GetPath("test2")}, []string{"value2"}, nil},
{"", []string{
kv.GetPath("test1"),
kv.GetPath("test2"),
kv.GetPath("test1/a"),
kv.GetPath("test1/b")}, []string{"value1", "value2", "value_a", "value_b"}, nil},
{"test1/a", []string{kv.GetPath("test1/a")}, []string{"value_a"}, nil},
{"a", []string{}, []string{}, nil},
{"root", []string{}, []string{}, nil},
{"/tikv/test/root", []string{}, []string{}, nil},
}
for _, test := range loadPrefixTests {
actualKeys, actualValues, err := kv.LoadWithPrefix(test.prefix)
assert.ElementsMatch(t, test.expectedKeys, actualKeys)
assert.ElementsMatch(t, test.expectedValues, actualValues)
assert.Equal(t, test.expectedError, err)
}
removeTests := []struct {
validKey string
invalidKey string
}{
{"test1", "abc"},
{"test1/a", "test1/lskfjal"},
{"test1/b", "test1/b"},
{"test2", "-"},
}
for _, test := range removeTests {
err = kv.Remove(test.validKey)
assert.NoError(t, err)
_, err = kv.Load(test.validKey)
assert.Error(t, err)
err = kv.Remove(test.validKey)
assert.NoError(t, err)
err = kv.Remove(test.invalidKey)
assert.NoError(t, err)
}
})
te.Run("kv MultiSaveAndMultiLoad", func(t *testing.T) {
rootPath := "/tikv/test/root/multi_save_and_multi_load"
kv := NewTiKV(txnClient, rootPath)
defer kv.Close()
defer kv.RemoveWithPrefix("")
multiSaveTests := map[string]string{
"key_1": "value_1",
"key_2": "value_2",
"key_3/a": "value_3a",
"multikey_1": "multivalue_1",
"multikey_2": "multivalue_2",
"_": "other",
}
err := kv.MultiSave(multiSaveTests)
assert.NoError(t, err)
for k, v := range multiSaveTests {
actualV, err := kv.Load(k)
assert.NoError(t, err)
assert.Equal(t, v, actualV)
}
multiLoadTests := []struct {
inputKeys []string
expectedValues []string
}{
{[]string{"key_1"}, []string{"value_1"}},
{[]string{"key_1", "key_2", "key_3/a"}, []string{"value_1", "value_2", "value_3a"}},
{[]string{"multikey_1", "multikey_2"}, []string{"multivalue_1", "multivalue_2"}},
{[]string{"_"}, []string{"other"}},
}
for _, test := range multiLoadTests {
vs, err := kv.MultiLoad(test.inputKeys)
assert.NoError(t, err)
assert.Equal(t, test.expectedValues, vs)
}
invalidMultiLoad := []struct {
invalidKeys []string
expectedValues []string
}{
{[]string{"a", "key_1"}, []string{"", "value_1"}},
{[]string{".....", "key_1"}, []string{"", "value_1"}},
{[]string{"*********"}, []string{""}},
{[]string{"key_1", "1"}, []string{"value_1", ""}},
}
for _, test := range invalidMultiLoad {
vs, err := kv.MultiLoad(test.invalidKeys)
assert.Error(t, err)
assert.Equal(t, test.expectedValues, vs)
}
removeWithPrefixTests := []string{
"key_1",
"multi",
}
for _, k := range removeWithPrefixTests {
err = kv.RemoveWithPrefix(k)
assert.NoError(t, err)
ks, vs, err := kv.LoadWithPrefix(k)
assert.Empty(t, ks)
assert.Empty(t, vs)
assert.NoError(t, err)
}
multiRemoveTests := []string{
"key_2",
"key_3/a",
"multikey_2",
"_",
}
err = kv.MultiRemove(multiRemoveTests)
assert.NoError(t, err)
ks, vs, err := kv.LoadWithPrefix("")
assert.NoError(t, err)
assert.Empty(t, ks)
assert.Empty(t, vs)
multiSaveAndRemoveTests := []struct {
multiSaves map[string]string
multiRemoves []string
}{
{map[string]string{"key_1": "value_1"}, []string{}},
{map[string]string{"key_2": "value_2"}, []string{"key_1"}},
{map[string]string{"key_3/a": "value_3a"}, []string{"key_2"}},
{map[string]string{"multikey_1": "multivalue_1"}, []string{}},
{map[string]string{"multikey_2": "multivalue_2"}, []string{"multikey_1", "key_3/a"}},
{make(map[string]string), []string{"multikey_2"}},
}
for _, test := range multiSaveAndRemoveTests {
err = kv.MultiSaveAndRemove(test.multiSaves, test.multiRemoves)
assert.NoError(t, err)
}
ks, vs, err = kv.LoadWithPrefix("")
assert.NoError(t, err)
assert.Empty(t, ks)
assert.Empty(t, vs)
})
te.Run("kv MultiRemoveWithPrefix", func(t *testing.T) {
rootPath := "/tikv/test/root/multi_remove_with_prefix"
kv := NewTiKV(txnClient, rootPath)
defer kv.Close()
defer kv.RemoveWithPrefix("")
prepareTests := map[string]string{
"x/abc/1": "1",
"x/abc/2": "2",
"x/def/1": "10",
"x/def/2": "20",
"x/den/1": "100",
"x/den/2": "200",
}
err := kv.MultiSave(prepareTests)
require.NoError(t, err)
multiRemoveWithPrefixTests := []struct {
prefix []string
testKey string
expectedValue string
}{
{[]string{"x/abc"}, "x/abc/1", ""},
{[]string{}, "x/abc/2", ""},
{[]string{}, "x/def/1", "10"},
{[]string{}, "x/def/2", "20"},
{[]string{}, "x/den/1", "100"},
{[]string{}, "x/den/2", "200"},
{[]string{}, "not-exist", ""},
{[]string{"x/def", "x/den"}, "x/def/1", ""},
{[]string{}, "x/def/1", ""},
{[]string{}, "x/def/2", ""},
{[]string{}, "x/den/1", ""},
{[]string{}, "x/den/2", ""},
{[]string{}, "not-exist", ""},
}
for _, test := range multiRemoveWithPrefixTests {
if len(test.prefix) > 0 {
err = kv.MultiRemoveWithPrefix(test.prefix)
assert.NoError(t, err)
}
v, _ := kv.Load(test.testKey)
assert.Equal(t, test.expectedValue, v)
}
k, v, err := kv.LoadWithPrefix("/")
assert.NoError(t, err)
assert.Zero(t, len(k))
assert.Zero(t, len(v))
// MultiSaveAndRemoveWithPrefix
err = kv.MultiSave(prepareTests)
require.NoError(t, err)
multiSaveAndRemoveWithPrefixTests := []struct {
multiSave map[string]string
prefix []string
loadPrefix string
lengthBeforeRemove int
lengthAfterRemove int
}{
{map[string]string{}, []string{"x/abc", "x/def", "x/den"}, "x", 6, 0},
{map[string]string{"y/a": "vvv", "y/b": "vvv"}, []string{}, "y", 0, 2},
{map[string]string{"y/c": "vvv"}, []string{}, "y", 2, 3},
{map[string]string{"p/a": "vvv"}, []string{"y/a", "y"}, "y", 3, 0},
{map[string]string{}, []string{"p"}, "p", 1, 0},
}
for _, test := range multiSaveAndRemoveWithPrefixTests {
k, _, err = kv.LoadWithPrefix(test.loadPrefix)
assert.NoError(t, err)
assert.Equal(t, test.lengthBeforeRemove, len(k))
err = kv.MultiSaveAndRemoveWithPrefix(test.multiSave, test.prefix)
assert.NoError(t, err)
k, _, err = kv.LoadWithPrefix(test.loadPrefix)
assert.NoError(t, err)
assert.Equal(t, test.lengthAfterRemove, len(k))
}
})
te.Run("kv failed to start txn", func(t *testing.T) {
rootPath := "/tikv/test/root/start_exn"
kv := NewTiKV(txnClient, rootPath)
defer kv.Close()
beginTxn = func(txn *txnkv.Client) (*transaction.KVTxn, error) {
return nil, fmt.Errorf("bad txn!")
}
defer func() {
beginTxn = tiTxnBegin
}()
err := kv.Save("key1", "v1")
assert.Error(t, err)
err = kv.MultiSave(map[string]string{"A/100": "v1"})
assert.Error(t, err)
err = kv.Remove("key1")
assert.Error(t, err)
err = kv.MultiRemove([]string{"key_1", "key_2"})
assert.Error(t, err)
err = kv.MultiSaveAndRemove(map[string]string{"key_1": "value_1"}, []string{})
assert.Error(t, err)
err = kv.MultiSaveAndRemoveWithPrefix(map[string]string{"y/c": "vvv"}, []string{"/"})
assert.Error(t, err)
err = kv.MultiRemoveWithPrefix([]string{"x/def", "x/den"})
assert.Error(t, err)
})
te.Run("kv failed to commit txn", func(t *testing.T) {
rootPath := "/tikv/test/root/commit_txn"
kv := NewTiKV(txnClient, rootPath)
defer kv.Close()
commitTxn = func(txn *transaction.KVTxn, ctx context.Context) error {
return fmt.Errorf("bad txn commit!")
}
defer func() {
commitTxn = tiTxnCommit
}()
var err error
err = kv.Save("key1", "v1")
assert.Error(t, err)
err = kv.MultiSave(map[string]string{"A/100": "v1"})
assert.Error(t, err)
err = kv.Remove("key1")
assert.Error(t, err)
err = kv.MultiRemove([]string{"key_1", "key_2"})
assert.Error(t, err)
err = kv.MultiSaveAndRemove(map[string]string{"key_1": "value_1"}, []string{})
assert.Error(t, err)
err = kv.MultiSaveAndRemoveWithPrefix(map[string]string{"y/c": "vvv"}, []string{"/"})
assert.Error(t, err)
err = kv.MultiRemoveWithPrefix([]string{"x/def", "x/den"})
assert.Error(t, err)
})
}
func TestWalkWithPagination(t *testing.T) {
rootPath := "/tikv/test/root/pagination"
kv := NewTiKV(txnClient, rootPath)
defer kv.Close()
defer kv.RemoveWithPrefix("")
kvs := map[string]string{
"A/100": "v1",
"AA/100": "v2",
"AB/100": "v3",
"AB/2/100": "v4",
"B/100": "v5",
}
err := kv.MultiSave(kvs)
assert.NoError(t, err)
for k, v := range kvs {
actualV, err := kv.Load(k)
assert.NoError(t, err)
assert.Equal(t, v, actualV)
}
t.Run("apply function error ", func(t *testing.T) {
err = kv.WalkWithPrefix("A", 5, func(key []byte, value []byte) error {
return errors.New("error")
})
assert.Error(t, err)
})
t.Run("get with non-exist prefix ", func(t *testing.T) {
err = kv.WalkWithPrefix("non-exist-prefix", 5, func(key []byte, value []byte) error {
return nil
})
assert.NoError(t, err)
})
t.Run("with different pagination", func(t *testing.T) {
testFn := func(pagination int) {
expected := map[string]string{
"A/100": "v1",
"AA/100": "v2",
"AB/100": "v3",
"AB/2/100": "v4",
}
expectedKeys := maps.Keys(expected)
sort.Strings(expectedKeys)
ret := make(map[string]string)
actualKeys := make([]string, 0)
err = kv.WalkWithPrefix("A", pagination, func(key []byte, value []byte) error {
k := string(key)
k = k[len(rootPath)+1:]
ret[k] = string(value)
actualKeys = append(actualKeys, k)
return nil
})
assert.NoError(t, err)
assert.Equal(t, expected, ret, fmt.Errorf("pagination: %d", pagination))
// Ignore the order.
assert.ElementsMatch(t, expectedKeys, actualKeys, fmt.Errorf("pagination: %d", pagination))
}
for p := -1; p < 6; p++ {
testFn(p)
}
testFn(-100)
testFn(100)
})
}
func TestElapse(t *testing.T) {
start := time.Now()
isElapse := CheckElapseAndWarn(start, "err message")
assert.Equal(t, isElapse, false)
time.Sleep(2001 * time.Millisecond)
isElapse = CheckElapseAndWarn(start, "err message")
assert.Equal(t, isElapse, true)
}
func TestHas(t *testing.T) {
rootPath := "/tikv/test/root/pagination"
kv := NewTiKV(txnClient, rootPath)
err := kv.RemoveWithPrefix("")
require.NoError(t, err)
defer kv.Close()
defer kv.RemoveWithPrefix("")
has, err := kv.Has("key1")
assert.NoError(t, err)
assert.False(t, has)
err = kv.Save("key1", "value1")
assert.NoError(t, err)
err = kv.Save("key1", EmptyValueString)
assert.Error(t, err)
has, err = kv.Has("key1")
assert.NoError(t, err)
assert.True(t, has)
err = kv.Remove("key1")
assert.NoError(t, err)
has, err = kv.Has("key1")
assert.NoError(t, err)
assert.False(t, has)
}
func TestHasPrefix(t *testing.T) {
rootPath := "/etcd/test/root/hasprefix"
kv := NewTiKV(txnClient, rootPath)
err := kv.RemoveWithPrefix("")
require.NoError(t, err)
defer kv.Close()
defer kv.RemoveWithPrefix("")
has, err := kv.HasPrefix("key")
assert.NoError(t, err)
assert.False(t, has)
err = kv.Save("key1", "value1")
assert.NoError(t, err)
has, err = kv.HasPrefix("key")
assert.NoError(t, err)
assert.True(t, has)
err = kv.Remove("key1")
assert.NoError(t, err)
has, err = kv.HasPrefix("key")
assert.NoError(t, err)
assert.False(t, has)
}
func TestEmptyKey(t *testing.T) {
rootPath := "/etcd/test/root/loadempty"
kv := NewTiKV(txnClient, rootPath)
err := kv.RemoveWithPrefix("")
require.NoError(t, err)
defer kv.Close()
defer kv.RemoveWithPrefix("")
has, err := kv.HasPrefix("key")
assert.NoError(t, err)
assert.False(t, has)
err = kv.Save("key", "")
assert.NoError(t, err)
has, err = kv.HasPrefix("key")
assert.NoError(t, err)
assert.True(t, has)
val, err := kv.Load("key")
assert.NoError(t, err)
assert.Equal(t, val, "")
_, vals, err := kv.LoadWithPrefix("key")
assert.NoError(t, err)
assert.Equal(t, vals[0], "")
vals, err = kv.MultiLoad([]string{"key"})
assert.NoError(t, err)
assert.Equal(t, vals[0], "")
var res string
nothing := func(key, val []byte) error {
res = string(val)
return nil
}
err = kv.WalkWithPrefix("", 1, nothing)
assert.NoError(t, err)
assert.Equal(t, res, "")
multiSaveTests := map[string]string{
"key1": "",
}
err = kv.MultiSave(multiSaveTests)
assert.NoError(t, err)
val, err = kv.Load("key1")
assert.NoError(t, err)
assert.Equal(t, val, "")
multiSaveTests = map[string]string{
"key2": "",
}
err = kv.MultiSaveAndRemove(multiSaveTests, nil)
assert.NoError(t, err)
val, err = kv.Load("key2")
assert.NoError(t, err)
assert.Equal(t, val, "")
multiSaveTests = map[string]string{
"key3": "",
}
err = kv.MultiSaveAndRemoveWithPrefix(multiSaveTests, nil)
assert.NoError(t, err)
val, err = kv.Load("key3")
assert.NoError(t, err)
assert.Equal(t, val, "")
}
func TestScanSize(t *testing.T) {
var scan_size = SnapshotScanSize
kv := NewTiKV(txnClient, "/")
err := kv.RemoveWithPrefix("")
require.NoError(t, err)
defer kv.Close()
defer kv.RemoveWithPrefix("")
// Test total > scansize
key_map := map[string]string{}
for i := 1; i <= scan_size+100; i++ {
a := fmt.Sprintf("%v", i)
key_map[a] = a
}
err = kv.MultiSave(key_map)
assert.NoError(t, err)
keys, _, err := kv.LoadWithPrefix("")
assert.NoError(t, err)
assert.Equal(t, len(keys), scan_size+100)
err = kv.RemoveWithPrefix("")
require.NoError(t, err)
}
func TestTiKVUnimplemented(t *testing.T) {
kv := NewTiKV(txnClient, "/")
err := kv.RemoveWithPrefix("")
require.NoError(t, err)
defer kv.Close()
defer kv.RemoveWithPrefix("")
_, err = kv.CompareVersionAndSwap("k", 1, "target")
assert.Error(t, err)
}

35
internal/mocks/mock_datacoord.go
View File

@ -18,6 +18,8 @@ import (
mock "github.com/stretchr/testify/mock"
txnkv "github.com/tikv/client-go/v2/txnkv"
types "github.com/milvus-io/milvus/internal/types"
)
@ -2422,6 +2424,39 @@ func (_c *MockDataCoord_SetSegmentState_Call) RunAndReturn(run func(context.Cont
return _c
}
// SetTiKVClient provides a mock function with given fields: client
func (_m *MockDataCoord) SetTiKVClient(client *txnkv.Client) {
_m.Called(client)
}
// MockDataCoord_SetTiKVClient_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetTiKVClient'
type MockDataCoord_SetTiKVClient_Call struct {
*mock.Call
}
// SetTiKVClient is a helper method to define mock.On call
// - client *txnkv.Client
func (_e *MockDataCoord_Expecter) SetTiKVClient(client interface{}) *MockDataCoord_SetTiKVClient_Call {
return &MockDataCoord_SetTiKVClient_Call{Call: _e.mock.On("SetTiKVClient", client)}
}
func (_c *MockDataCoord_SetTiKVClient_Call) Run(run func(client *txnkv.Client)) *MockDataCoord_SetTiKVClient_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(*txnkv.Client))
})
return _c
}
func (_c *MockDataCoord_SetTiKVClient_Call) Return() *MockDataCoord_SetTiKVClient_Call {
_c.Call.Return()
return _c
}
func (_c *MockDataCoord_SetTiKVClient_Call) RunAndReturn(run func(*txnkv.Client)) *MockDataCoord_SetTiKVClient_Call {
_c.Call.Return(run)
return _c
}
// ShowConfigurations provides a mock function with given fields: ctx, req
func (_m *MockDataCoord) ShowConfigurations(ctx context.Context, req *internalpb.ShowConfigurationsRequest) (*internalpb.ShowConfigurationsResponse, error) {
ret := _m.Called(ctx, req)

35
internal/mocks/mock_querycoord.go
View File

@ -16,6 +16,8 @@ import (
querypb "github.com/milvus-io/milvus/internal/proto/querypb"
txnkv "github.com/tikv/client-go/v2/txnkv"
types "github.com/milvus-io/milvus/internal/types"
)
@ -1284,6 +1286,39 @@ func (_c *MockQueryCoord_SetRootCoord_Call) RunAndReturn(run func(types.RootCoor
return _c
}
// SetTiKVClient provides a mock function with given fields: client
func (_m *MockQueryCoord) SetTiKVClient(client *txnkv.Client) {
_m.Called(client)
}
// MockQueryCoord_SetTiKVClient_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetTiKVClient'
type MockQueryCoord_SetTiKVClient_Call struct {
*mock.Call
}
// SetTiKVClient is a helper method to define mock.On call
// - client *txnkv.Client
func (_e *MockQueryCoord_Expecter) SetTiKVClient(client interface{}) *MockQueryCoord_SetTiKVClient_Call {
return &MockQueryCoord_SetTiKVClient_Call{Call: _e.mock.On("SetTiKVClient", client)}
}
func (_c *MockQueryCoord_SetTiKVClient_Call) Run(run func(client *txnkv.Client)) *MockQueryCoord_SetTiKVClient_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(*txnkv.Client))
})
return _c
}
func (_c *MockQueryCoord_SetTiKVClient_Call) Return() *MockQueryCoord_SetTiKVClient_Call {
_c.Call.Return()
return _c
}
func (_c *MockQueryCoord_SetTiKVClient_Call) RunAndReturn(run func(*txnkv.Client)) *MockQueryCoord_SetTiKVClient_Call {
_c.Call.Return(run)
return _c
}
// ShowCollections provides a mock function with given fields: ctx, req
func (_m *MockQueryCoord) ShowCollections(ctx context.Context, req *querypb.ShowCollectionsRequest) (*querypb.ShowCollectionsResponse, error) {
ret := _m.Called(ctx, req)

28
internal/querycoordv2/server.go
View File

@ -25,6 +25,7 @@ import (
"time"
"github.com/cockroachdb/errors"
"github.com/tikv/client-go/v2/txnkv"
clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/atomic"
"go.uber.org/zap"
@ -34,6 +35,7 @@ import (
"github.com/milvus-io/milvus/internal/allocator"
"github.com/milvus-io/milvus/internal/kv"
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
"github.com/milvus-io/milvus/internal/kv/tikv"
"github.com/milvus-io/milvus/internal/metastore"
"github.com/milvus-io/milvus/internal/metastore/kv/querycoord"
"github.com/milvus-io/milvus/internal/querycoordv2/balance"
@ -52,6 +54,7 @@ import (
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/metrics"
"github.com/milvus-io/milvus/pkg/util"
"github.com/milvus-io/milvus/pkg/util/merr"
"github.com/milvus-io/milvus/pkg/util/metricsinfo"
"github.com/milvus-io/milvus/pkg/util/paramtable"
@ -70,6 +73,7 @@ type Server struct {
wg sync.WaitGroup
status atomic.Int32
etcdCli *clientv3.Client
tikvCli *txnkv.Client
address string
session *sessionutil.Session
kv kv.MetaKv
@ -204,13 +208,21 @@ func (s *Server) Init() error {
func (s *Server) initQueryCoord() error {
s.UpdateStateCode(commonpb.StateCode_Initializing)
log.Info("QueryCoord", zap.Any("State", commonpb.StateCode_Initializing))
// Init KV
etcdKV := etcdkv.NewEtcdKV(s.etcdCli, Params.EtcdCfg.MetaRootPath.GetValue())
s.kv = etcdKV
log.Info("query coordinator try to connect etcd success")
// Init KV and ID allocator
metaType := Params.MetaStoreCfg.MetaStoreType.GetValue()
var idAllocatorKV kv.TxnKV
log.Info(fmt.Sprintf("query coordinator connecting to %s.", metaType))
if metaType == util.MetaStoreTypeTiKV {
s.kv = tikv.NewTiKV(s.tikvCli, Params.TiKVCfg.MetaRootPath.GetValue())
idAllocatorKV = tsoutil.NewTSOTiKVBase(s.tikvCli, Params.TiKVCfg.KvRootPath.GetValue(), "querycoord-id-allocator")
} else if metaType == util.MetaStoreTypeEtcd {
s.kv = etcdkv.NewEtcdKV(s.etcdCli, Params.EtcdCfg.MetaRootPath.GetValue())
idAllocatorKV = tsoutil.NewTSOKVBase(s.etcdCli, Params.EtcdCfg.KvRootPath.GetValue(), "querycoord-id-allocator")
} else {
return fmt.Errorf("not supported meta store: %s", metaType)
}
log.Info(fmt.Sprintf("query coordinator successfully connected to %s.", metaType))
// Init ID allocator
idAllocatorKV := tsoutil.NewTSOKVBase(s.etcdCli, Params.EtcdCfg.KvRootPath.GetValue(), "querycoord-id-allocator")
idAllocator := allocator.NewGlobalIDAllocator("idTimestamp", idAllocatorKV)
err := idAllocator.Initialize()
if err != nil {
@ -539,6 +551,10 @@ func (s *Server) SetEtcdClient(etcdClient *clientv3.Client) {
s.etcdCli = etcdClient
}
func (s *Server) SetTiKVClient(client *txnkv.Client) {
s.tikvCli = client
}
// SetRootCoord sets root coordinator's client
func (s *Server) SetRootCoord(rootCoord types.RootCoord) error {
if rootCoord == nil {

21
internal/querycoordv2/server_test.go
View File

@ -26,6 +26,7 @@ import (
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
"github.com/tikv/client-go/v2/txnkv"
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
@ -47,6 +48,7 @@ import (
"github.com/milvus-io/milvus/pkg/util/etcd"
"github.com/milvus-io/milvus/pkg/util/merr"
"github.com/milvus-io/milvus/pkg/util/paramtable"
"github.com/milvus-io/milvus/pkg/util/tikv"
)
func TestMain(m *testing.M) {
@ -81,10 +83,13 @@ type ServerSuite struct {
// Mocks
broker *meta.MockBroker
server *Server
nodes []*mocks.MockQueryNode
tikvCli *txnkv.Client
server *Server
nodes []*mocks.MockQueryNode
}
var testMeta string
func (suite *ServerSuite) SetupSuite() {
paramtable.Init()
params.GenerateEtcdConfig()
@ -121,7 +126,10 @@ func (suite *ServerSuite) SetupSuite() {
func (suite *ServerSuite) SetupTest() {
var err error
paramtable.Get().Save(paramtable.Get().MetaStoreCfg.MetaStoreType.Key, testMeta)
suite.tikvCli = tikv.SetupLocalTxn()
suite.server, err = suite.newQueryCoord()
suite.Require().NoError(err)
suite.hackServer()
err = suite.server.Start()
@ -152,6 +160,7 @@ func (suite *ServerSuite) TearDownTest() {
node.Stop()
}
}
paramtable.Get().Reset(paramtable.Get().MetaStoreCfg.MetaStoreType.Key)
}
func (suite *ServerSuite) TestRecover() {
@ -563,6 +572,8 @@ func (suite *ServerSuite) newQueryCoord() (*Server, error) {
return nil, err
}
server.SetEtcdClient(etcdCli)
server.SetTiKVClient(suite.tikvCli)
server.SetQueryNodeCreator(session.DefaultQueryNodeCreator)
suite.hackBroker(server)
err = server.Init()
@ -570,5 +581,9 @@ func (suite *ServerSuite) newQueryCoord() (*Server, error) {
}
func TestServer(t *testing.T) {
suite.Run(t, new(ServerSuite))
parameters := []string{"tikv", "etcd"}
for _, v := range parameters {
testMeta = v
suite.Run(t, new(ServerSuite))
}
}

68
internal/rootcoord/root_coord.go
View File

@ -28,6 +28,7 @@ import (
"github.com/cockroachdb/errors"
"github.com/samber/lo"
"github.com/tikv/client-go/v2/txnkv"
clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/zap"
"golang.org/x/sync/errgroup"
@ -38,6 +39,7 @@ import (
"github.com/milvus-io/milvus/internal/allocator"
"github.com/milvus-io/milvus/internal/kv"
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
tikv "github.com/milvus-io/milvus/internal/kv/tikv"
"github.com/milvus-io/milvus/internal/metastore"
kvmetestore "github.com/milvus-io/milvus/internal/metastore/kv/rootcoord"
"github.com/milvus-io/milvus/internal/metastore/model"
@ -81,13 +83,7 @@ var Params *paramtable.ComponentParam = paramtable.Get()
type Opt func(*Core)
type metaKVCreator func(root string) (kv.MetaKv, error)
func defaultMetaKVCreator(etcdCli *clientv3.Client) metaKVCreator {
return func(root string) (kv.MetaKv, error) {
return etcdkv.NewEtcdKV(etcdCli, root), nil
}
}
type metaKVCreator func() (kv.MetaKv, error)
// Core root coordinator core
type Core struct {
@ -95,6 +91,7 @@ type Core struct {
cancel context.CancelFunc
wg sync.WaitGroup
etcdCli *clientv3.Client
tikvCli *txnkv.Client
address string
meta IMetaTable
scheduler IScheduler
@ -305,6 +302,11 @@ func (c *Core) SetEtcdClient(etcdClient *clientv3.Client) {
c.etcdCli = etcdClient
}
// SetTiKVClient sets the tikvCli of Core
func (c *Core) SetTiKVClient(client *txnkv.Client) {
c.tikvCli = client
}
func (c *Core) initSession() error {
c.session = sessionutil.NewSession(c.ctx, Params.EtcdCfg.MetaRootPath.GetValue(), c.etcdCli)
if c.session == nil {
@ -317,7 +319,15 @@ func (c *Core) initSession() error {
func (c *Core) initKVCreator() {
if c.metaKVCreator == nil {
c.metaKVCreator = defaultMetaKVCreator(c.etcdCli)
if Params.MetaStoreCfg.MetaStoreType.GetValue() == util.MetaStoreTypeTiKV {
c.metaKVCreator = func() (kv.MetaKv, error) {
return tikv.NewTiKV(c.tikvCli, Params.TiKVCfg.MetaRootPath.GetValue()), nil
}
} else {
c.metaKVCreator = func() (kv.MetaKv, error) {
return etcdkv.NewEtcdKV(c.etcdCli, Params.EtcdCfg.MetaRootPath.GetValue()), nil
}
}
}
}
@ -328,18 +338,32 @@ func (c *Core) initMetaTable() error {
switch Params.MetaStoreCfg.MetaStoreType.GetValue() {
case util.MetaStoreTypeEtcd:
log.Info("Using etcd as meta storage.")
var metaKV kv.MetaKv
var ss *kvmetestore.SuffixSnapshot
var err error
if metaKV, err = c.metaKVCreator(Params.EtcdCfg.MetaRootPath.GetValue()); err != nil {
if metaKV, err = c.metaKVCreator(); err != nil {
return err
}
if ss, err = kvmetestore.NewSuffixSnapshot(metaKV, kvmetestore.SnapshotsSep, Params.EtcdCfg.MetaRootPath.GetValue(), kvmetestore.SnapshotPrefix); err != nil {
return err
}
catalog = &kvmetestore.Catalog{Txn: metaKV, Snapshot: ss}
case util.MetaStoreTypeTiKV:
log.Info("Using tikv as meta storage.")
var metaKV kv.MetaKv
var ss *kvmetestore.SuffixSnapshot
var err error
if metaKV, err = c.metaKVCreator(); err != nil {
return err
}
if ss, err = kvmetestore.NewSuffixSnapshot(metaKV, kvmetestore.SnapshotsSep, Params.TiKVCfg.MetaRootPath.GetValue(), kvmetestore.SnapshotPrefix); err != nil {
return err
}
catalog = &kvmetestore.Catalog{Txn: metaKV, Snapshot: ss}
default:
return retry.Unrecoverable(fmt.Errorf("not supported meta store: %s", Params.MetaStoreCfg.MetaStoreType.GetValue()))
@ -356,7 +380,15 @@ func (c *Core) initMetaTable() error {
}
func (c *Core) initIDAllocator() error {
tsoKV := tsoutil2.NewTSOKVBase(c.etcdCli, Params.EtcdCfg.KvRootPath.GetValue(), globalIDAllocatorSubPath)
var tsoKV kv.TxnKV
var kvPath string
if Params.MetaStoreCfg.MetaStoreType.GetValue() == util.MetaStoreTypeTiKV {
kvPath = Params.TiKVCfg.KvRootPath.GetValue()
tsoKV = tsoutil2.NewTSOTiKVBase(c.tikvCli, kvPath, globalIDAllocatorSubPath)
} else {
kvPath = Params.EtcdCfg.KvRootPath.GetValue()
tsoKV = tsoutil2.NewTSOKVBase(c.etcdCli, kvPath, globalIDAllocatorSubPath)
}
idAllocator := allocator.NewGlobalIDAllocator(globalIDAllocatorKey, tsoKV)
if err := idAllocator.Initialize(); err != nil {
return err
@ -364,7 +396,7 @@ func (c *Core) initIDAllocator() error {
c.idAllocator = idAllocator
log.Info("id allocator initialized",
zap.String("root_path", Params.EtcdCfg.KvRootPath.GetValue()),
zap.String("root_path", kvPath),
zap.String("sub_path", globalIDAllocatorSubPath),
zap.String("key", globalIDAllocatorKey))
@ -372,7 +404,15 @@ func (c *Core) initIDAllocator() error {
}
func (c *Core) initTSOAllocator() error {
tsoKV := tsoutil2.NewTSOKVBase(c.etcdCli, Params.EtcdCfg.KvRootPath.GetValue(), globalTSOAllocatorSubPath)
var tsoKV kv.TxnKV
var kvPath string
if Params.MetaStoreCfg.MetaStoreType.GetValue() == util.MetaStoreTypeTiKV {
kvPath = Params.TiKVCfg.KvRootPath.GetValue()
tsoKV = tsoutil2.NewTSOTiKVBase(c.tikvCli, Params.TiKVCfg.KvRootPath.GetValue(), globalIDAllocatorSubPath)
} else {
kvPath = Params.EtcdCfg.KvRootPath.GetValue()
tsoKV = tsoutil2.NewTSOKVBase(c.etcdCli, Params.EtcdCfg.KvRootPath.GetValue(), globalIDAllocatorSubPath)
}
tsoAllocator := tso2.NewGlobalTSOAllocator(globalTSOAllocatorKey, tsoKV)
if err := tsoAllocator.Initialize(); err != nil {
return err
@ -380,7 +420,7 @@ func (c *Core) initTSOAllocator() error {
c.tsoAllocator = tsoAllocator
log.Info("tso allocator initialized",
zap.String("root_path", Params.EtcdCfg.KvRootPath.GetValue()),
zap.String("root_path", kvPath),
zap.String("sub_path", globalIDAllocatorSubPath),
zap.String("key", globalIDAllocatorKey))
@ -388,7 +428,7 @@ func (c *Core) initTSOAllocator() error {
}
func (c *Core) initImportManager() error {
impTaskKv, err := c.metaKVCreator(Params.EtcdCfg.KvRootPath.GetValue())
impTaskKv, err := c.metaKVCreator()
if err != nil {
return err
}

13
internal/rootcoord/root_coord_test.go
View File

@ -50,13 +50,20 @@ import (
"github.com/milvus-io/milvus/pkg/util/funcutil"
"github.com/milvus-io/milvus/pkg/util/metricsinfo"
"github.com/milvus-io/milvus/pkg/util/paramtable"
"github.com/milvus-io/milvus/pkg/util/tikv"
"github.com/milvus-io/milvus/pkg/util/typeutil"
)
func TestMain(m *testing.M) {
paramtable.Init()
rand.Seed(time.Now().UnixNano())
os.Exit(m.Run())
parameters := []string{"tikv", "etcd"}
var code int
for _, v := range parameters {
paramtable.Get().Save(paramtable.Get().MetaStoreCfg.MetaStoreType.Key, v)
code = m.Run()
}
os.Exit(code)
}
func TestRootCoord_CreateDatabase(t *testing.T) {
@ -1656,6 +1663,8 @@ func TestRootcoord_EnableActiveStandby(t *testing.T) {
core, err := NewCore(ctx, coreFactory)
core.etcdCli = etcdCli
assert.NoError(t, err)
core.SetTiKVClient(tikv.SetupLocalTxn())
err = core.Init()
assert.NoError(t, err)
assert.Equal(t, commonpb.StateCode_StandBy, core.stateCode.Load().(commonpb.StateCode))
@ -1704,6 +1713,8 @@ func TestRootcoord_DisableActiveStandby(t *testing.T) {
core, err := NewCore(ctx, coreFactory)
core.etcdCli = etcdCli
assert.NoError(t, err)
core.SetTiKVClient(tikv.SetupLocalTxn())
err = core.Init()
assert.NoError(t, err)
assert.Equal(t, commonpb.StateCode_Initializing, core.stateCode.Load().(commonpb.StateCode))

10
internal/types/types.go
View File

@ -19,6 +19,7 @@ package types
import (
"context"
"github.com/tikv/client-go/v2/txnkv"
clientv3 "go.etcd.io/etcd/client/v3"
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
@ -393,6 +394,9 @@ type DataCoordComponent interface {
// `etcdClient` is a client of etcd
SetEtcdClient(etcdClient *clientv3.Client)
// SetTiKVClient set TiKV client for QueryNode
SetTiKVClient(client *txnkv.Client)
SetRootCoord(rootCoord RootCoord)
// SetDataNodeCreator set DataNode client creator func for DataCoord
@ -820,6 +824,9 @@ type RootCoordComponent interface {
// `etcdClient` is a client of etcd
SetEtcdClient(etcdClient *clientv3.Client)
// SetTiKVClient set TiKV client for RootCoord
SetTiKVClient(client *txnkv.Client)
// UpdateStateCode updates state code for RootCoord
// State includes: Initializing, Healthy and Abnormal
UpdateStateCode(commonpb.StateCode)
@ -1519,6 +1526,9 @@ type QueryCoordComponent interface {
// SetEtcdClient set etcd client for QueryCoord
SetEtcdClient(etcdClient *clientv3.Client)
// SetTiKVClient set TiKV client for QueryCoord
SetTiKVClient(client *txnkv.Client)
// UpdateStateCode updates state code for QueryCoord
// `stateCode` is current statement of this QueryCoord, indicating whether it's healthy.
UpdateStateCode(stateCode commonpb.StateCode)

7
internal/util/tsoutil/tso.go
View File

@ -19,13 +19,20 @@ package tsoutil
import (
"path"
"github.com/tikv/client-go/v2/txnkv"
clientv3 "go.etcd.io/etcd/client/v3"
"github.com/milvus-io/milvus/internal/kv"
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
"github.com/milvus-io/milvus/internal/kv/tikv"
)
// NewTSOKVBase returns a kv.TxnKV object
func NewTSOKVBase(client *clientv3.Client, tsoRoot, subPath string) kv.TxnKV {
return etcdkv.NewEtcdKV(client, path.Join(tsoRoot, subPath))
}
// NewTSOTiKVBase returns a kv.TxnKV object
func NewTSOTiKVBase(client *txnkv.Client, tsoRoot, subPath string) kv.TxnKV {
return tikv.NewTiKV(client, path.Join(tsoRoot, subPath))
}

22
pkg/go.mod
View File

@ -25,6 +25,7 @@ require (
github.com/spf13/viper v1.8.1
github.com/streamnative/pulsarctl v0.5.0
github.com/stretchr/testify v1.8.3
github.com/tikv/client-go/v2 v2.0.4
github.com/uber/jaeger-client-go v2.30.0+incompatible
go.etcd.io/etcd/client/v3 v3.5.5
go.etcd.io/etcd/server/v3 v3.5.5
@ -37,7 +38,7 @@ require (
go.opentelemetry.io/otel/trace v1.13.0
go.uber.org/atomic v1.10.0
go.uber.org/automaxprocs v1.5.2
go.uber.org/zap v1.17.0
go.uber.org/zap v1.20.0
golang.org/x/crypto v0.9.0
golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17
golang.org/x/sync v0.1.0
@ -53,6 +54,7 @@ require (
github.com/BurntSushi/toml v1.2.1 // indirect
github.com/DataDog/zstd v1.5.0 // indirect
github.com/ardielle/ardielle-go v1.5.2 // indirect
github.com/benbjohnson/clock v1.1.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v4 v4.2.0 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
@ -60,8 +62,10 @@ require (
github.com/cockroachdb/redact v1.1.3 // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
github.com/danieljoos/wincred v1.1.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/dvsekhvalnov/jose2go v1.5.0 // indirect
@ -76,7 +80,8 @@ require (
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/btree v1.0.1 // indirect
github.com/google/btree v1.1.2 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect
github.com/gorilla/websocket v1.4.2 // indirect
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
@ -104,14 +109,21 @@ require (
github.com/nats-io/nkeys v0.4.4 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/opencontainers/runtime-spec v1.0.2 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pelletier/go-toml v1.9.3 // indirect
github.com/pierrec/lz4 v2.5.2+incompatible // indirect
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 // indirect
github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 // indirect
github.com/pingcap/kvproto v0.0.0-20221129023506-621ec37aac7a // indirect
github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.42.0 // indirect
github.com/prometheus/procfs v0.9.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
github.com/rogpeppe/go-internal v1.8.1 // indirect
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/smartystreets/assertions v1.1.0 // indirect
@ -119,11 +131,15 @@ require (
github.com/spf13/afero v1.6.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stathat/consistent v1.0.0 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07 // indirect
github.com/tklauser/go-sysconf v0.3.10 // indirect
github.com/tklauser/numcpus v0.4.0 // indirect
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
github.com/twmb/murmur3 v1.1.3 // indirect
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
go.etcd.io/bbolt v1.3.6 // indirect
@ -136,7 +152,7 @@ require (
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.13.0 // indirect
go.opentelemetry.io/otel/metric v0.35.0 // indirect
go.opentelemetry.io/proto/otlp v0.19.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
go.uber.org/multierr v1.7.0 // indirect
golang.org/x/net v0.10.0 // indirect
golang.org/x/oauth2 v0.6.0 // indirect
golang.org/x/sys v0.8.0 // indirect

57
pkg/go.sum
View File

@ -67,6 +67,7 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/ardielle/ardielle-go v1.5.2 h1:TilHTpHIQJ27R1Tl/iITBzMwiUGSlVfiVhwDNGM3Zj4=
github.com/ardielle/ardielle-go v1.5.2/go.mod h1:I4hy1n795cUhaVt/ojz83SNVCYIGsAFAONtv2Dr7HUI=
@ -77,6 +78,8 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/aws/aws-sdk-go v1.32.6/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/benesch/cgosymbolizer v0.0.0-20190515212042-bec6fe6e597b h1:5JgaFtHFRnOPReItxvhMDXbvuBkjSWE+9glJyF466yw=
github.com/benesch/cgosymbolizer v0.0.0-20190515212042-bec6fe6e597b/go.mod h1:eMD2XUcPsHYbakFEocKrWZp47G0MRJYoC60qFblGjpA=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@ -141,6 +144,8 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso=
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM=
github.com/danieljoos/wincred v1.1.2 h1:QLdCxFs1/Yl4zduvBdcHB8goaYk9RARS2SgLLRuAyr0=
github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -148,6 +153,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/dimfeld/httptreemux v5.0.1+incompatible h1:Qj3gVcDNoOthBAqftuD596rm4wg/adLLz5xh5CmpiCA=
@ -275,8 +281,9 @@ github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4=
github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.2.1-0.20190312032427-6f77996f0c42/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@ -329,6 +336,7 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c=
github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0=
@ -532,6 +540,7 @@ github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9
github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0=
github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
github.com/panjf2000/ants/v2 v2.7.2 h1:2NUt9BaZFO5kQzrieOmK/wdb/tQ/K+QHaxN8sOgD63U=
github.com/panjf2000/ants/v2 v2.7.2/go.mod h1:KIBmYG9QQX5U2qzFP/yQJaq/nSb6rahS9iEHkrCMgM8=
@ -542,8 +551,19 @@ github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI=
github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4=
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg=
github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0=
github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew=
github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E=
github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw=
github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI=
github.com/pingcap/kvproto v0.0.0-20221129023506-621ec37aac7a h1:LzIZsQpXQlj8yF7+yvyOg680OaPq7bmPuDuszgXfHsw=
github.com/pingcap/kvproto v0.0.0-20221129023506-621ec37aac7a/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI=
github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 h1:URLoJ61DmmY++Sa/yyPEQHG2s/ZBeV1FbIswHEMrdoY=
github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@ -560,6 +580,7 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw=
github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
@ -584,6 +605,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rogpeppe/clock v0.0.0-20190514195947-2896927a307a/go.mod h1:4r5QyqhjIWCcK8DO4KMclc5Iknq5qVBAlbYYzAbUScQ=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
@ -602,6 +625,7 @@ github.com/santhosh-tekuri/jsonschema/v5 v5.0.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H
github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/shirou/gopsutil/v3 v3.22.9 h1:yibtJhIVEMcdw+tCTbOPiF1VcsuDeTE4utJ8Dm4c5eA=
github.com/shirou/gopsutil/v3 v3.22.9/go.mod h1:bBYl1kjgEJpWpxeHmLI+dVHWtyAwfcmSBLDsp2TNT8A=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
@ -642,6 +666,8 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM
github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44=
github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns=
github.com/stathat/consistent v1.0.0 h1:ZFJ1QTRn8npNBKW065raSZ8xfOqhpb8vLOkfp4CcL/U=
github.com/stathat/consistent v1.0.0/go.mod h1:uajTPbgSygZBJ+V+0mY7meZ8i0XAcZs7AQ6V121XSxw=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
@ -664,6 +690,12 @@ github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M=
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW93SG+q0F8KI+yFrcIDT4c/RNoc4=
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM=
github.com/tikv/client-go/v2 v2.0.4 h1:cPtMXTExqjzk8L40qhrgB/mXiBXKP5LRU0vwjtI2Xxo=
github.com/tikv/client-go/v2 v2.0.4/go.mod h1:v52O5zDtv2BBus4lm5yrSQhxGW4Z4RaXWfg0U1Kuyqo=
github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07 h1:ckPpxKcl75mO2N6a4cJXiZH43hvcHPpqc9dh1TmH1nc=
github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07/go.mod h1:CipBxPfxPUME+BImx9MUYXCnAVLS3VJUr3mnSJwh40A=
github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw=
github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk=
github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o=
@ -671,6 +703,8 @@ github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hM
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA=
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/twmb/murmur3 v1.1.3 h1:D83U0XYKcHRYwYIpBKf3Pks91Z0Byda/9SJ8B6EMRcA=
github.com/twmb/murmur3 v1.1.3/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o=
github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
@ -760,18 +794,25 @@ go.opentelemetry.io/proto/otlp v0.9.0/go.mod h1:1vKfU9rv61e9EVGthD1zNvUbiwPcimSs
go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw=
go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.uber.org/automaxprocs v1.5.2 h1:2LxUOGiR3O6tw8ui5sZa2LAaHnsviZdVOUZw4fvbnME=
go.uber.org/automaxprocs v1.5.2/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0=
go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec=
go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U=
go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
go.uber.org/zap v1.20.0 h1:N4oPlghZwYG55MlU6LXk/Zp00FVNE9X9wrYO8CEs4lc=
go.uber.org/zap v1.20.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@ -845,6 +886,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@ -1026,6 +1068,8 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@ -1064,6 +1108,7 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@ -1107,6 +1152,7 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn
google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
@ -1151,6 +1197,7 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
@ -1171,6 +1218,7 @@ google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ
google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k=
google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag=
google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g=
@ -1244,3 +1292,4 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c=

1
pkg/util/constant.go
View File

@ -26,6 +26,7 @@ import (
// Meta Prefix consts
const (
MetaStoreTypeEtcd = "etcd"
MetaStoreTypeTiKV = "tikv"
SegmentMetaPrefix = "queryCoord-segmentMeta"
ChangeInfoMetaPrefix = "queryCoord-sealedSegmentChangeInfo"

2
pkg/util/paramtable/base_table.go
View File

@ -55,7 +55,7 @@ const (
// Const of Global Config List
func globalConfigPrefixs() []string {
return []string{"metastore", "localStorage", "etcd", "minio", "pulsar", "kafka", "rocksmq", "log", "grpc", "common", "quotaAndLimits"}
return []string{"metastore", "localStorage", "etcd", "tikv", "minio", "pulsar", "kafka", "rocksmq", "log", "grpc", "common", "quotaAndLimits"}
}
var defaultYaml = []string{"milvus.yaml"}

127
pkg/util/paramtable/service_param.go
View File

@ -45,6 +45,7 @@ type ServiceParam struct {
LocalStorageCfg LocalStorageConfig
MetaStoreCfg MetaStoreConfig
EtcdCfg EtcdConfig
TiKVCfg TiKVConfig
MQCfg MQConfig
PulsarCfg PulsarConfig
KafkaCfg KafkaConfig
@ -57,6 +58,7 @@ func (p *ServiceParam) init(bt *BaseTable) {
p.LocalStorageCfg.Init(bt)
p.MetaStoreCfg.Init(bt)
p.EtcdCfg.Init(bt)
p.TiKVCfg.Init(bt)
p.MQCfg.Init(bt)
p.PulsarCfg.Init(bt)
p.KafkaCfg.Init(bt)
@ -257,6 +259,129 @@ We recommend using version 1.2 and above.`,
p.EtcdTLSMinVersion.Init(base.mgr)
}
// /////////////////////////////////////////////////////////////////////////////
// --- tikv ---
type TiKVConfig struct {
Endpoints ParamItem `refreshable:"false"`
RootPath ParamItem `refreshable:"false"`
MetaSubPath ParamItem `refreshable:"false"`
KvSubPath ParamItem `refreshable:"false"`
MetaRootPath CompositeParamItem `refreshable:"false"`
KvRootPath CompositeParamItem `refreshable:"false"`
RequestTimeout ParamItem `refreshable:"true"`
SnapshotScanSize ParamItem `refreshable:"true"`
TiKVUseSSL ParamItem `refreshable:"false"`
TiKVTLSCert ParamItem `refreshable:"false"`
TiKVTLSKey ParamItem `refreshable:"false"`
TiKVTLSCACert ParamItem `refreshable:"false"`
}
func (p *TiKVConfig) Init(base *BaseTable) {
p.Endpoints = ParamItem{
Key: "tikv.endpoints",
Version: "2.3.0",
DefaultValue: "localhost:2379",
PanicIfEmpty: true,
Export: true,
}
p.Endpoints.Init(base.mgr)
p.RootPath = ParamItem{
Key: "tikv.rootPath",
Version: "2.3.0",
DefaultValue: "by-dev",
PanicIfEmpty: true,
Doc: "The root path where data is stored in tikv",
Export: true,
}
p.RootPath.Init(base.mgr)
p.MetaSubPath = ParamItem{
Key: "tikv.metaSubPath",
Version: "2.3.0",
DefaultValue: "meta",
PanicIfEmpty: true,
Doc: "metaRootPath = rootPath + '/' + metaSubPath",
Export: true,
}
p.MetaSubPath.Init(base.mgr)
p.MetaRootPath = CompositeParamItem{
Items: []*ParamItem{&p.RootPath, &p.MetaSubPath},
Format: func(kvs map[string]string) string {
return path.Join(kvs[p.RootPath.Key], kvs[p.MetaSubPath.Key])
},
}
p.KvSubPath = ParamItem{
Key: "tikv.kvSubPath",
Version: "2.3.0",
DefaultValue: "kv",
PanicIfEmpty: true,
Doc: "kvRootPath = rootPath + '/' + kvSubPath",
Export: true,
}
p.KvSubPath.Init(base.mgr)
p.KvRootPath = CompositeParamItem{
Items: []*ParamItem{&p.RootPath, &p.KvSubPath},
Format: func(kvs map[string]string) string {
return path.Join(kvs[p.RootPath.Key], kvs[p.KvSubPath.Key])
},
}
p.RequestTimeout = ParamItem{
Key: "tikv.requestTimeout",
Version: "2.3.0",
DefaultValue: "10000",
Doc: "ms, tikv request timeout",
Export: true,
}
p.RequestTimeout.Init(base.mgr)
p.SnapshotScanSize = ParamItem{
Key: "tikv.snapshotScanSize",
Version: "2.3.0",
DefaultValue: "256",
Doc: "batch size of tikv snapshot scan",
Export: true,
}
p.SnapshotScanSize.Init(base.mgr)
p.TiKVUseSSL = ParamItem{
Key: "tikv.ssl.enabled",
DefaultValue: "false",
Version: "2.3.0",
Doc: "Whether to support TiKV secure connection mode",
Export: true,
}
p.TiKVUseSSL.Init(base.mgr)
p.TiKVTLSCert = ParamItem{
Key: "tikv.ssl.tlsCert",
Version: "2.3.0",
Doc: "path to your cert file",
Export: true,
}
p.TiKVTLSCert.Init(base.mgr)
p.TiKVTLSKey = ParamItem{
Key: "tikv.ssl.tlsKey",
Version: "2.3.0",
Doc: "path to your key file",
Export: true,
}
p.TiKVTLSKey.Init(base.mgr)
p.TiKVTLSCACert = ParamItem{
Key: "tikv.ssl.tlsCACert",
Version: "2.3.0",
Doc: "path to your CACert file",
Export: true,
}
p.TiKVTLSCACert.Init(base.mgr)
}
type LocalStorageConfig struct {
Path ParamItem `refreshable:"false"`
}
@ -281,7 +406,7 @@ func (p *MetaStoreConfig) Init(base *BaseTable) {
Key: "metastore.type",
Version: "2.2.0",
DefaultValue: util.MetaStoreTypeEtcd,
Doc: `Default value: etcd, Valid values: etcd `,
Doc: `Default value: etcd, Valid values: [etcd, tikv] `,
Export: true,
}
p.MetaStoreType.Init(base.mgr)

16
pkg/util/paramtable/service_param_test.go
View File

@ -63,6 +63,22 @@ func TestServiceParam(t *testing.T) {
SParams.init(bt)
})
t.Run("test tikvConfig", func(t *testing.T) {
Params := &SParams.TiKVCfg
assert.NotZero(t, len(Params.Endpoints.GetAsStrings()))
t.Logf("tikv endpoints = %s", Params.Endpoints.GetAsStrings())
assert.NotEqual(t, Params.MetaRootPath, "")
t.Logf("meta root path = %s", Params.MetaRootPath.GetValue())
assert.NotEqual(t, Params.KvRootPath, "")
t.Logf("kv root path = %s", Params.KvRootPath.GetValue())
t.Setenv(metricsinfo.DeployModeEnvKey, metricsinfo.StandaloneDeployMode)
SParams.init(bt)
})
t.Run("test pulsarConfig", func(t *testing.T) {
// test default value
{

36
pkg/util/tikv/tikv_test_util.go Normal file
View File

@ -0,0 +1,36 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tikv
import (
"github.com/tikv/client-go/v2/testutils"
tilib "github.com/tikv/client-go/v2/tikv"
"github.com/tikv/client-go/v2/txnkv"
)
func SetupLocalTxn() *txnkv.Client {
client, cluster, pdClient, err := testutils.NewMockTiKV("", nil)
if err != nil {
panic(err)
}
testutils.BootstrapWithSingleStore(cluster)
store, err := tilib.NewTestTiKVStore(client, pdClient, nil, nil, 0)
if err != nil {
panic(err)
}
return &txnkv.Client{KVStore: store}
}

34
pkg/util/tikv/tikv_util.go Normal file
View File

@ -0,0 +1,34 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tikv
import (
"github.com/milvus-io/milvus/pkg/util/paramtable"
"github.com/tikv/client-go/v2/config"
"github.com/tikv/client-go/v2/txnkv"
)
func GetTiKVClient(cfg *paramtable.TiKVConfig) (*txnkv.Client, error) {
if cfg.TiKVUseSSL.GetAsBool() {
f := func(conf *config.Config) {
conf.Security = config.NewSecurity(cfg.TiKVTLSCACert.GetValue(), cfg.TiKVTLSCert.GetValue(), cfg.TiKVTLSKey.GetValue(), []string{})
}
config.UpdateGlobal(f)
return txnkv.NewClient([]string{cfg.Endpoints.GetValue()})
}
return txnkv.NewClient([]string{cfg.Endpoints.GetValue()})
}