mirror of
https://gitee.com/milvus-io/milvus.git
synced 2024-12-01 19:39:21 +08:00
Fix search test when partition not exists, add binary vector search validation
Signed-off-by: bigsheeper <yihao.dai@zilliz.com>
This commit is contained in:
parent
fa73f17f94
commit
e13822043d
@ -2,7 +2,7 @@
|
||||
"name": "Milvus Distributed Dev Container Definition",
|
||||
"dockerComposeFile": ["./docker-compose-vscode.yml"],
|
||||
"service": "ubuntu",
|
||||
"initializeCommand": "scripts/init_devcontainer.sh && docker-compose -f docker-compose-vscode.yml down || true && docker-compose -f docker-compose-vscode.yml pull --ignore-pull-failures ubuntu",
|
||||
"initializeCommand": "scripts/init_devcontainer.sh && docker-compose -f docker-compose-vscode.yml down || true",
|
||||
"workspaceFolder": "/go/src/github.com/zilliztech/milvus-distributed",
|
||||
"shutdownAction": "stopCompose",
|
||||
"extensions": [
|
||||
|
2
.gitignore
vendored
2
.gitignore
vendored
@ -11,7 +11,7 @@ pulsar/client-cpp/build/*
|
||||
# vscode generated files
|
||||
.vscode
|
||||
docker-compose-vscode.yml
|
||||
docker-compose-vscode.yml.bak
|
||||
docker-compose-vscode.yml.tmp
|
||||
|
||||
cmake-build-debug
|
||||
cmake-build-release
|
||||
|
@ -1,22 +1,20 @@
|
||||
timeout(time: 20, unit: 'MINUTES') {
|
||||
dir ("scripts") {
|
||||
sh '. ./before-install.sh && unset http_proxy && unset https_proxy && ./check_cache.sh -l $CCACHE_ARTFACTORY_URL --cache_dir=\$CCACHE_DIR -f ccache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz || echo \"Ccache artfactory files not found!\"'
|
||||
sh '. ./before-install.sh && unset http_proxy && unset https_proxy && ./check_cache.sh -l $GO_CACHE_ARTFACTORY_URL --cache_dir=\$(go env GOCACHE) -f go-cache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz || echo \"Go cache artfactory files not found!\"'
|
||||
sh '. ./before-install.sh && unset http_proxy && unset https_proxy && ./check_cache.sh -l $THIRDPARTY_ARTFACTORY_URL --cache_dir=$CUSTOM_THIRDPARTY_PATH -f thirdparty-download.tar.gz || echo \"Thirdparty artfactory files not found!\"'
|
||||
sh '. ./before-install.sh && unset http_proxy && unset https_proxy && ./check_cache.sh -l $GO_MOD_ARTFACTORY_URL --cache_dir=\$GOPATH/pkg/mod -f milvus-distributed-go-mod-cache.tar.gz || echo \"Go mod artfactory files not found!\"'
|
||||
}
|
||||
|
||||
sh '. ./scripts/before-install.sh && unset http_proxy && unset https_proxy && ./scripts/check_cache.sh -l $CCACHE_ARTFACTORY_URL --cache_dir=\$CCACHE_DIR -f ccache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz || echo \"Ccache artfactory files not found!\"'
|
||||
sh '. ./scripts/before-install.sh && unset http_proxy && unset https_proxy && ./scripts/check_cache.sh -l $GO_CACHE_ARTFACTORY_URL --cache_dir=\$(go env GOCACHE) -f go-cache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz || echo \"Go cache artfactory files not found!\"'
|
||||
sh '. ./scripts/before-install.sh && unset http_proxy && unset https_proxy && ./scripts/check_cache.sh -l $THIRDPARTY_ARTFACTORY_URL --cache_dir=$CUSTOM_THIRDPARTY_PATH -f thirdparty-download.tar.gz || echo \"Thirdparty artfactory files not found!\"'
|
||||
sh '. ./scripts/before-install.sh && unset http_proxy && unset https_proxy && ./scripts/check_cache.sh -l $GO_MOD_ARTFACTORY_URL --cache_dir=\$GOPATH/pkg/mod -f milvus-distributed-go-mod-\$(md5sum go.mod).tar.gz || echo \"Go mod artfactory files not found!\"'
|
||||
|
||||
// Zero the cache statistics (but not the configuration options)
|
||||
sh 'ccache -z'
|
||||
sh '. ./scripts/before-install.sh && make install'
|
||||
sh 'echo -e "===\n=== ccache statistics after build\n===" && ccache --show-stats'
|
||||
|
||||
dir ("scripts") {
|
||||
withCredentials([usernamePassword(credentialsId: "${env.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
|
||||
sh '. ./before-install.sh && unset http_proxy && unset https_proxy && ./update_cache.sh -l $CCACHE_ARTFACTORY_URL --cache_dir=\$CCACHE_DIR -f ccache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz -u ${USERNAME} -p ${PASSWORD}'
|
||||
sh '. ./before-install.sh && unset http_proxy && unset https_proxy && ./update_cache.sh -l $GO_CACHE_ARTFACTORY_URL --cache_dir=\$(go env GOCACHE) -f go-cache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz -u ${USERNAME} -p ${PASSWORD}'
|
||||
sh '. ./before-install.sh && unset http_proxy && unset https_proxy && ./update_cache.sh -l $THIRDPARTY_ARTFACTORY_URL --cache_dir=$CUSTOM_THIRDPARTY_PATH -f thirdparty-download.tar.gz -u ${USERNAME} -p ${PASSWORD}'
|
||||
sh '. ./before-install.sh && unset http_proxy && unset https_proxy && ./update_cache.sh -l $GO_MOD_ARTFACTORY_URL --cache_dir=\$GOPATH/pkg/mod -f milvus-distributed-go-mod-cache.tar.gz -u ${USERNAME} -p ${PASSWORD}'
|
||||
}
|
||||
|
||||
withCredentials([usernamePassword(credentialsId: "${env.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
|
||||
sh '. ./scripts/before-install.sh && unset http_proxy && unset https_proxy && ./scripts/update_cache.sh -l $CCACHE_ARTFACTORY_URL --cache_dir=\$CCACHE_DIR -f ccache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz -u ${USERNAME} -p ${PASSWORD}'
|
||||
sh '. ./scripts/before-install.sh && unset http_proxy && unset https_proxy && ./scripts/update_cache.sh -l $GO_CACHE_ARTFACTORY_URL --cache_dir=\$(go env GOCACHE) -f go-cache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz -u ${USERNAME} -p ${PASSWORD}'
|
||||
sh '. ./scripts/before-install.sh && unset http_proxy && unset https_proxy && ./scripts/update_cache.sh -l $THIRDPARTY_ARTFACTORY_URL --cache_dir=$CUSTOM_THIRDPARTY_PATH -f thirdparty-download.tar.gz -u ${USERNAME} -p ${PASSWORD}'
|
||||
sh '. ./scripts/before-install.sh && unset http_proxy && unset https_proxy && ./scripts/update_cache.sh -l $GO_MOD_ARTFACTORY_URL --cache_dir=\$GOPATH/pkg/mod -f milvus-distributed-go-mod-\$(md5sum go.mod).tar.gz -u ${USERNAME} -p ${PASSWORD}'
|
||||
}
|
||||
}
|
||||
|
14
Makefile
14
Makefile
@ -76,7 +76,7 @@ else
|
||||
@${GOPATH}/bin/ruleguard -rules ruleguard.rules.go ./tests/go/...
|
||||
endif
|
||||
|
||||
verifiers: getdeps cppcheck fmt static-check ruleguard
|
||||
verifiers: getdeps cppcheck fmt static-check
|
||||
|
||||
# Builds various components locally.
|
||||
build-go: build-cpp
|
||||
@ -127,7 +127,6 @@ install: all
|
||||
@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/master $(GOPATH)/bin/master
|
||||
@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/proxy $(GOPATH)/bin/proxy
|
||||
@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/writenode $(GOPATH)/bin/writenode
|
||||
@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/indexbuilder $(GOPATH)/bin/indexbuilder
|
||||
@mkdir -p $(LIBRARY_PATH) && cp -f $(PWD)/internal/core/output/lib/* $(LIBRARY_PATH)
|
||||
@echo "Installation successful."
|
||||
|
||||
@ -135,10 +134,7 @@ clean:
|
||||
@echo "Cleaning up all the generated files"
|
||||
@find . -name '*.test' | xargs rm -fv
|
||||
@find . -name '*~' | xargs rm -fv
|
||||
@rm -rf bin/
|
||||
@rm -rf lib/
|
||||
@rm -rf $(GOPATH)/bin/master
|
||||
@rm -rf $(GOPATH)/bin/proxy
|
||||
@rm -rf $(GOPATH)/bin/querynode
|
||||
@rm -rf $(GOPATH)/bin/writenode
|
||||
@rm -rf $(GOPATH)/bin/indexbuilder
|
||||
@rm -rvf querynode
|
||||
@rm -rvf master
|
||||
@rm -rvf proxy
|
||||
@rm -rvf writenode
|
||||
|
@ -6,4 +6,4 @@ PULSAR_ADDRESS=pulsar://pulsar:6650
|
||||
ETCD_ADDRESS=etcd:2379
|
||||
MASTER_ADDRESS=master:53100
|
||||
MINIO_ADDRESS=minio:9000
|
||||
INDEX_BUILDER_ADDRESS=indexbuider:31000
|
||||
INDEX_BUILDER_ADDRESS=indexbuilder:31000
|
||||
|
@ -9,12 +9,31 @@
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
# or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
FROM alpine:3.12.1
|
||||
FROM milvusdb/milvus-distributed-dev:amd64-ubuntu18.04-latest AS openblas
|
||||
|
||||
#FROM alpine
|
||||
FROM ubuntu:bionic-20200921
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends libtbb-dev gfortran
|
||||
|
||||
#RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories
|
||||
|
||||
#RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories \
|
||||
# && apk add --no-cache libtbb gfortran
|
||||
|
||||
COPY --from=openblas /usr/lib/libopenblas-r0.3.9.so /usr/lib/
|
||||
|
||||
RUN ln -s /usr/lib/libopenblas-r0.3.9.so /usr/lib/libopenblas.so.0 && \
|
||||
ln -s /usr/lib/libopenblas.so.0 /usr/lib/libopenblas.so
|
||||
|
||||
COPY ./bin/indexbuilder /milvus-distributed/bin/indexbuilder
|
||||
|
||||
COPY ./configs/ /milvus-distributed/configs/
|
||||
|
||||
COPY ./lib/ /milvus-distributed/lib/
|
||||
|
||||
ENV LD_LIBRARY_PATH=/milvus-distributed/lib:$LD_LIBRARY_PATH:/usr/lib
|
||||
|
||||
WORKDIR /milvus-distributed/
|
||||
|
||||
CMD ["./bin/indexbuilder"]
|
||||
|
@ -36,14 +36,6 @@ services:
|
||||
networks:
|
||||
- milvus
|
||||
|
||||
jaeger:
|
||||
image: jaegertracing/all-in-one:latest
|
||||
ports:
|
||||
- "6831:6831/udp"
|
||||
- "16686:16686"
|
||||
networks:
|
||||
- milvus
|
||||
|
||||
networks:
|
||||
milvus:
|
||||
|
||||
|
@ -86,10 +86,5 @@ services:
|
||||
networks:
|
||||
- milvus
|
||||
|
||||
jaeger:
|
||||
image: jaegertracing/all-in-one:latest
|
||||
networks:
|
||||
- milvus
|
||||
|
||||
networks:
|
||||
milvus:
|
||||
|
@ -1,108 +0,0 @@
|
||||
# Opentracing User Guide
|
||||
|
||||
This guide mainly describes the use of jaeger and the commonly used display meanings.
|
||||
|
||||
Jaeger, inspired by [Dapper](https://research.google.com/pubs/pub36356.html) and [OpenZipkin](https://zipkin.io/), is a distributed tracing platform created by [Uber Technologies](https://uber.github.io/) and donated to [Cloud Native Computing Foundation](https://cncf.io/). It can be used for monitoring microservices-based distributed systems:
|
||||
|
||||
- Distributed context propagation
|
||||
- Distributed transaction monitoring
|
||||
- Root cause analysis
|
||||
- Service dependency analysis
|
||||
- Performance / latency optimization
|
||||
|
||||
See also:
|
||||
|
||||
- Jaeger [documentation](https://jaegertracing.io/docs/) for getting started, operational details, and other information.
|
||||
- Blog post [Evolving Distributed Tracing at Uber](https://eng.uber.com/distributed-tracing/).
|
||||
- Tutorial / walkthrough [Take OpenTracing for a HotROD ride](https://medium.com/@YuriShkuro/take-opentracing-for-a-hotrod-ride-f6e3141f7941).
|
||||
|
||||
|
||||
|
||||
We mainly use jaeger as a implementation of opentracing.
|
||||
|
||||
Two request: **Insert Request** and **Search Request** in milvus-distributed system is traced at this stage.
|
||||
|
||||
|
||||
|
||||
## Jaeger Home page
|
||||
|
||||
![](./figs/jaeger_home_page.png)
|
||||
|
||||
|
||||
|
||||
### Lookup by Trace ID
|
||||
|
||||
The use of the search box requires configuration of the log collection system. For example, if the log collection system collects the log, if it is an error log, find the Trace ID. Search in jaeger to quickly locate the error. So as to quickly solve the problem
|
||||
|
||||
### Search
|
||||
|
||||
### Service
|
||||
|
||||
Filter with service name
|
||||
|
||||
### Operation
|
||||
|
||||
Operation in Service, eg request name, function name
|
||||
|
||||
### Tags
|
||||
|
||||
Set tag to facilitate search. Tag is defined in code
|
||||
|
||||
### Lookback
|
||||
|
||||
Filter with time.
|
||||
|
||||
### Min Duraton Max Duration
|
||||
|
||||
The minimum and maximum request duration, you can use this condition to find the time-consuming and short-term requests
|
||||
|
||||
### Limit Result
|
||||
|
||||
The max number of result
|
||||
|
||||
## Search result
|
||||
|
||||
You can search through the above conditions, and those that meet the conditions will appear in the right half.
|
||||
|
||||
![](./figs/jaeger_single_search_result.png)
|
||||
|
||||
The detailed information of search result.
|
||||
|
||||
1. The upper left corner identifies the service name: tracing, Root span name: Insert grpc received and The first half of the Trace ID 46874e2.
|
||||
2. The duration of the entire request is shown in the upper right corner.
|
||||
3. 10675 Span means that there are 10675 operations, see the number of operations for each service in the middle. And the trace time is shown in the right.
|
||||
4. The user can select multiple trace with the box in the upper left, and then compare with them to find something different. For example, different insert request may take a different time. At this time, you can select two for comparison, and the comparison can be very easy to know which trace went wrong
|
||||
|
||||
|
||||
|
||||
## Detailed trace information
|
||||
|
||||
Click the search result. You can analyze the detail trace information.
|
||||
|
||||
![](./figs/jaeger_detailed_trace_info.png)
|
||||
|
||||
1. Duration: Total time consume.
|
||||
2. Service: The number of service called.
|
||||
3. Depth: Call chain depth.
|
||||
4. Total Spans: This call consists of 10 spans
|
||||
5. To enter the details, look at the left half first, showing the call chain of the entire request. The black represents the service name, and the gray kid represents the span name defined in the code.
|
||||
6. The duration of the right half of the code call. The length represents the time consumed by the Span in the entire call chain.
|
||||
|
||||
|
||||
|
||||
### Span Detail information
|
||||
|
||||
Click the Span to see the detailed span information such as the last span in the picture above.
|
||||
|
||||
1. Tags contains a series of custom tags. You can mark in the code what type of call this Span is, request method, call result, call, etc. All the information it contains can be filtered by the Tags on the homepage.
|
||||
2. Process can locate which specific server processing this data.
|
||||
3. Logs are the logs printed by this span during the call.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## More
|
||||
|
||||
More usage guides will be updated in the future
|
||||
|
Binary file not shown.
Before Width: | Height: | Size: 403 KiB |
Binary file not shown.
Before Width: | Height: | Size: 336 KiB |
Binary file not shown.
Before Width: | Height: | Size: 183 KiB |
17
go.mod
17
go.mod
@ -4,17 +4,14 @@ go 1.15
|
||||
|
||||
require (
|
||||
code.cloudfoundry.org/bytefmt v0.0.0-20200131002437-cf55d5288a48 // indirect
|
||||
github.com/HdrHistogram/hdrhistogram-go v1.0.1 // indirect
|
||||
github.com/apache/pulsar-client-go v0.1.1
|
||||
github.com/apache/thrift v0.13.0
|
||||
github.com/aws/aws-sdk-go v1.30.8 // indirect
|
||||
github.com/aws/aws-sdk-go v1.30.8
|
||||
github.com/coreos/etcd v3.3.25+incompatible // indirect
|
||||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
|
||||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548
|
||||
github.com/frankban/quicktest v1.10.2 // indirect
|
||||
github.com/fsnotify/fsnotify v1.4.9 // indirect
|
||||
github.com/git-hooks/git-hooks v1.3.1 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
|
||||
github.com/golang/mock v1.3.1
|
||||
github.com/golang/protobuf v1.3.2
|
||||
github.com/google/btree v1.0.0
|
||||
github.com/klauspost/compress v1.10.11 // indirect
|
||||
@ -23,12 +20,12 @@ require (
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
|
||||
github.com/onsi/ginkgo v1.12.1 // indirect
|
||||
github.com/onsi/gomega v1.10.0 // indirect
|
||||
github.com/opentracing/opentracing-go v1.2.0
|
||||
github.com/opentracing/opentracing-go v1.2.0 // indirect
|
||||
github.com/pierrec/lz4 v2.5.2+incompatible // indirect
|
||||
github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 // indirect
|
||||
github.com/pingcap/errors v0.11.4 // indirect
|
||||
github.com/pingcap/log v0.0.0-20200828042413-fce0951f1463 // indirect
|
||||
github.com/pivotal-golang/bytefmt v0.0.0-20200131002437-cf55d5288a48 // indirect
|
||||
github.com/pivotal-golang/bytefmt v0.0.0-20200131002437-cf55d5288a48
|
||||
github.com/prometheus/client_golang v1.5.1 // indirect
|
||||
github.com/prometheus/common v0.10.0 // indirect
|
||||
github.com/prometheus/procfs v0.1.3 // indirect
|
||||
@ -38,9 +35,7 @@ require (
|
||||
github.com/spf13/cast v1.3.0
|
||||
github.com/spf13/viper v1.7.1
|
||||
github.com/stretchr/testify v1.6.1
|
||||
github.com/tikv/client-go v0.0.0-20200824032810-95774393107b // indirect
|
||||
github.com/uber/jaeger-client-go v2.25.0+incompatible
|
||||
github.com/uber/jaeger-lib v2.4.0+incompatible // indirect
|
||||
github.com/tikv/client-go v0.0.0-20200824032810-95774393107b
|
||||
github.com/urfave/cli v1.22.5 // indirect
|
||||
github.com/yahoo/athenz v1.9.16 // indirect
|
||||
go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738
|
||||
@ -55,7 +50,7 @@ require (
|
||||
google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150 // indirect
|
||||
google.golang.org/grpc v1.31.0
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
|
||||
gopkg.in/yaml.v2 v2.3.0 // indirect
|
||||
gopkg.in/yaml.v2 v2.3.0
|
||||
honnef.co/go/tools v0.0.1-2020.1.4 // indirect
|
||||
sigs.k8s.io/yaml v1.2.0 // indirect
|
||||
)
|
||||
|
13
go.sum
13
go.sum
@ -15,8 +15,6 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7
|
||||
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/HdrHistogram/hdrhistogram-go v1.0.1 h1:GX8GAYDuhlFQnI2fRDHQhTlkHMz8bEn0jTI6LJU0mpw=
|
||||
github.com/HdrHistogram/hdrhistogram-go v1.0.1/go.mod h1:BWJ+nMSHY3L41Zj7CA3uXnloDp7xxV0YvstAE7nKTaM=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
|
||||
@ -26,8 +24,6 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1C
|
||||
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/apache/pulsar-client-go v0.1.1 h1:v/kU+2ZCC6yFIcbZrFtWa9/nvVzVr18L+xYJUvZSxEQ=
|
||||
github.com/apache/pulsar-client-go v0.1.1/go.mod h1:mlxC65KL1BLhGO2bnT9zWMttVzR2czVPb27D477YpyU=
|
||||
github.com/apache/thrift v0.13.0 h1:5hryIiq9gtn+MiLVn0wP37kb/uTeRZgN08WoCsAhIhI=
|
||||
github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
|
||||
github.com/ardielle/ardielle-go v1.5.2 h1:TilHTpHIQJ27R1Tl/iITBzMwiUGSlVfiVhwDNGM3Zj4=
|
||||
github.com/ardielle/ardielle-go v1.5.2/go.mod h1:I4hy1n795cUhaVt/ojz83SNVCYIGsAFAONtv2Dr7HUI=
|
||||
github.com/ardielle/ardielle-tools v1.5.4/go.mod h1:oZN+JRMnqGiIhrzkRN9l26Cej9dEx4jeNG6A+AdkShk=
|
||||
@ -121,7 +117,6 @@ github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18h
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s=
|
||||
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
|
||||
github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
@ -173,7 +168,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.8.1/go.mod h1:vNeuVxBJEsws4ogUvrchl83t
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
|
||||
github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw=
|
||||
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
|
||||
github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
@ -349,7 +343,6 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx
|
||||
github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8=
|
||||
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
|
||||
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
|
||||
github.com/protocolbuffers/protobuf v3.14.0+incompatible h1:8r0H76h/Q/lEnFFY60AuM23NOnaDMi6bd7zuboSYM+o=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446 h1:/NRJ5vAYoqz+7sG51ubIDHXeWO8DlTSrToPu6q11ziA=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
|
||||
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
|
||||
@ -410,12 +403,6 @@ github.com/tikv/client-go v0.0.0-20200824032810-95774393107b/go.mod h1:K0NcdVNrX
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
github.com/uber/jaeger-client-go v1.6.0 h1:3+zLlq+4npI5fg8IsgAje3YsP7TcEdNzJScyqFIzxEQ=
|
||||
github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U=
|
||||
github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
|
||||
github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo=
|
||||
github.com/uber/jaeger-lib v2.4.0+incompatible h1:fY7QsGQWiCt8pajv4r7JEvmATdCVaWxXbjwyYwsNaLQ=
|
||||
github.com/uber/jaeger-lib v2.4.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
|
||||
github.com/ugorji/go v1.1.2/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=
|
||||
github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA=
|
||||
github.com/unrolled/render v1.0.0 h1:XYtvhA3UkpB7PqkvhUFYmpKD55OudoIeygcfus4vcd4=
|
||||
|
@ -34,6 +34,9 @@ IndexWrapper::IndexWrapper(const char* serialized_type_params, const char* seria
|
||||
auto mode = get_config_by_name<std::string>("index_mode");
|
||||
auto index_mode = mode.has_value() ? mode_map[mode.value()] : knowhere::IndexMode::MODE_CPU;
|
||||
|
||||
auto index_type = get_index_type();
|
||||
auto metric_type = get_metric_type();
|
||||
AssertInfo(!is_unsupported(index_type, metric_type), index_type + " doesn't support metric: " + metric_type);
|
||||
index_ = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(get_index_type(), index_mode);
|
||||
Assert(index_ != nullptr);
|
||||
}
|
||||
@ -263,6 +266,21 @@ IndexWrapper::get_index_type() {
|
||||
return type.has_value() ? type.value() : knowhere::IndexEnum::INDEX_FAISS_IVFPQ;
|
||||
}
|
||||
|
||||
std::string
|
||||
IndexWrapper::get_metric_type() {
|
||||
auto type = get_config_by_name<std::string>(knowhere::Metric::TYPE);
|
||||
if (type.has_value()) {
|
||||
return type.value();
|
||||
} else {
|
||||
auto index_type = get_index_type();
|
||||
if (is_in_bin_list(index_type)) {
|
||||
return knowhere::Metric::JACCARD;
|
||||
} else {
|
||||
return knowhere::Metric::L2;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<IndexWrapper::QueryResult>
|
||||
IndexWrapper::Query(const knowhere::DatasetPtr& dataset) {
|
||||
return std::move(QueryImpl(dataset, config_));
|
||||
|
@ -59,6 +59,9 @@ class IndexWrapper {
|
||||
std::string
|
||||
get_index_type();
|
||||
|
||||
std::string
|
||||
get_metric_type();
|
||||
|
||||
template <typename T>
|
||||
std::optional<T>
|
||||
get_config_by_name(std::string name);
|
||||
|
@ -14,6 +14,7 @@
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <algorithm>
|
||||
#include <tuple>
|
||||
|
||||
#include "index/knowhere/knowhere/index/IndexType.h"
|
||||
|
||||
@ -57,6 +58,14 @@ Need_BuildAll_list() {
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::vector<std::tuple<std::string, std::string>>
|
||||
unsupported_index_combinations() {
|
||||
static std::vector<std::tuple<std::string, std::string>> ret{
|
||||
std::make_tuple(std::string(knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT), std::string(knowhere::Metric::L2)),
|
||||
};
|
||||
return ret;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
bool
|
||||
is_in_list(const T& t, std::function<std::vector<T>()> list_func) {
|
||||
@ -84,5 +93,11 @@ is_in_need_id_list(const milvus::knowhere::IndexType& index_type) {
|
||||
return is_in_list<std::string>(index_type, Need_ID_List);
|
||||
}
|
||||
|
||||
bool
|
||||
is_unsupported(const milvus::knowhere::IndexType& index_type, const milvus::knowhere::MetricType& metric_type) {
|
||||
return is_in_list<std::tuple<std::string, std::string>>(std::make_tuple(index_type, metric_type),
|
||||
unsupported_index_combinations);
|
||||
}
|
||||
|
||||
} // namespace indexbuilder
|
||||
} // namespace milvus
|
||||
|
@ -61,6 +61,17 @@ InferIndexType(const Json& search_params) {
|
||||
PanicInfo("failed to infer index type");
|
||||
}
|
||||
|
||||
static knowhere::IndexType
|
||||
InferBinaryIndexType(const Json& search_params) {
|
||||
namespace ip = knowhere::IndexParams;
|
||||
namespace ie = knowhere::IndexEnum;
|
||||
if (search_params.contains(ip::nprobe)) {
|
||||
return ie::INDEX_FAISS_BIN_IVFFLAT;
|
||||
} else {
|
||||
return ie::INDEX_FAISS_BIN_IDMAP;
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
VerifyPlanNodeVisitor::visit(FloatVectorANNS& node) {
|
||||
auto& search_params = node.query_info_.search_params_;
|
||||
@ -79,7 +90,18 @@ VerifyPlanNodeVisitor::visit(FloatVectorANNS& node) {
|
||||
|
||||
void
|
||||
VerifyPlanNodeVisitor::visit(BinaryVectorANNS& node) {
|
||||
// TODO
|
||||
auto& search_params = node.query_info_.search_params_;
|
||||
auto inferred_type = InferBinaryIndexType(search_params);
|
||||
auto adapter = knowhere::AdapterMgr::GetInstance().GetAdapter(inferred_type);
|
||||
auto index_mode = knowhere::IndexMode::MODE_CPU;
|
||||
|
||||
// mock the api, topk will be passed from placeholder
|
||||
auto params_copy = search_params;
|
||||
params_copy[knowhere::meta::TOPK] = 10;
|
||||
|
||||
// NOTE: the second parameter is not checked in knowhere, may be redundant
|
||||
auto passed = adapter->CheckSearch(params_copy, inferred_type, index_mode);
|
||||
AssertInfo(passed, "invalid search params");
|
||||
}
|
||||
|
||||
} // namespace milvus::query
|
||||
|
@ -2,6 +2,10 @@ package indexbuilderclient
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/zilliztech/milvus-distributed/internal/errors"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
@ -54,20 +58,59 @@ func (c *Client) BuildIndexWithoutID(columnDataPaths []string, typeParams map[st
|
||||
if c.tryConnect() != nil {
|
||||
panic("BuildIndexWithoutID: failed to connect index builder")
|
||||
}
|
||||
parseMap := func(mStr string) (map[string]string, error) {
|
||||
buffer := make(map[string]interface{})
|
||||
err := json.Unmarshal([]byte(mStr), &buffer)
|
||||
if err != nil {
|
||||
return nil, errors.New("Unmarshal params failed")
|
||||
}
|
||||
ret := make(map[string]string)
|
||||
for key, value := range buffer {
|
||||
valueStr := fmt.Sprintf("%v", value)
|
||||
ret[key] = valueStr
|
||||
}
|
||||
return ret, nil
|
||||
}
|
||||
var typeParamsKV []*commonpb.KeyValuePair
|
||||
for typeParam := range typeParams {
|
||||
typeParamsKV = append(typeParamsKV, &commonpb.KeyValuePair{
|
||||
Key: typeParam,
|
||||
Value: typeParams[typeParam],
|
||||
})
|
||||
for key := range typeParams {
|
||||
if key == "params" {
|
||||
mapParams, err := parseMap(typeParams[key])
|
||||
if err != nil {
|
||||
log.Println("parse params error: ", err)
|
||||
}
|
||||
for pk, pv := range mapParams {
|
||||
typeParamsKV = append(typeParamsKV, &commonpb.KeyValuePair{
|
||||
Key: pk,
|
||||
Value: pv,
|
||||
})
|
||||
}
|
||||
} else {
|
||||
typeParamsKV = append(typeParamsKV, &commonpb.KeyValuePair{
|
||||
Key: key,
|
||||
Value: typeParams[key],
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
var indexParamsKV []*commonpb.KeyValuePair
|
||||
for indexParam := range indexParams {
|
||||
indexParamsKV = append(indexParamsKV, &commonpb.KeyValuePair{
|
||||
Key: indexParam,
|
||||
Value: indexParams[indexParam],
|
||||
})
|
||||
for key := range indexParams {
|
||||
if key == "params" {
|
||||
mapParams, err := parseMap(indexParams[key])
|
||||
if err != nil {
|
||||
log.Println("parse params error: ", err)
|
||||
}
|
||||
for pk, pv := range mapParams {
|
||||
indexParamsKV = append(indexParamsKV, &commonpb.KeyValuePair{
|
||||
Key: pk,
|
||||
Value: pv,
|
||||
})
|
||||
}
|
||||
} else {
|
||||
indexParamsKV = append(indexParamsKV, &commonpb.KeyValuePair{
|
||||
Key: key,
|
||||
Value: indexParams[key],
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
ctx := context.TODO()
|
||||
|
@ -14,6 +14,7 @@ package indexbuilder
|
||||
import "C"
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"unsafe"
|
||||
|
||||
@ -142,6 +143,8 @@ func (index *CIndex) Delete() error {
|
||||
}
|
||||
|
||||
func NewCIndex(typeParams, indexParams map[string]string) (Index, error) {
|
||||
fmt.Println("NNNNNNNNNNNNNNNNNNNNNNNNNNN typeParams: ", typeParams)
|
||||
fmt.Println("NNNNNNNNNNNNNNNNNNNNNNNNNNN indexParams: ", indexParams)
|
||||
protoTypeParams := &indexcgopb.TypeParams{
|
||||
Params: make([]*commonpb.KeyValuePair, 0),
|
||||
}
|
||||
@ -168,10 +171,14 @@ func NewCIndex(typeParams, indexParams map[string]string) (Index, error) {
|
||||
CIndex* res_index);
|
||||
*/
|
||||
var indexPtr C.CIndex
|
||||
fmt.Println("before create index ........................................")
|
||||
status := C.CreateIndex(typeParamsPointer, indexParamsPointer, &indexPtr)
|
||||
fmt.Println("after create index ........................................")
|
||||
errorCode := status.error_code
|
||||
fmt.Println("EEEEEEEEEEEEEEEEEEEEEEEEEE error code: ", errorCode)
|
||||
if errorCode != 0 {
|
||||
errorMsg := C.GoString(status.error_msg)
|
||||
fmt.Println("EEEEEEEEEEEEEEEEEEEEEEEEEE error msg: ", errorMsg)
|
||||
defer C.free(unsafe.Pointer(status.error_msg))
|
||||
return nil, errors.New(" failed, C runtime error detected, error code = " + strconv.Itoa(int(errorCode)) + ", error msg = " + errorMsg)
|
||||
}
|
||||
|
@ -2,6 +2,7 @@ package indexbuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"strconv"
|
||||
"time"
|
||||
@ -171,10 +172,12 @@ func (it *IndexBuildTask) Execute() error {
|
||||
indexParams[key] = value
|
||||
}
|
||||
|
||||
fmt.Println("before NewCIndex ..........................")
|
||||
it.index, err = NewCIndex(typeParams, indexParams)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println("after NewCIndex ..........................")
|
||||
|
||||
getKeyByPathNaive := func(path string) string {
|
||||
// splitElements := strings.Split(path, "/")
|
||||
@ -223,6 +226,7 @@ func (it *IndexBuildTask) Execute() error {
|
||||
|
||||
for _, value := range insertData.Data {
|
||||
// TODO: BinaryVectorFieldData
|
||||
fmt.Println("before build index ..................................")
|
||||
floatVectorFieldData, fOk := value.(*storage.FloatVectorFieldData)
|
||||
if fOk {
|
||||
err = it.index.BuildFloatVecIndexWithoutIds(floatVectorFieldData.Data)
|
||||
@ -238,12 +242,15 @@ func (it *IndexBuildTask) Execute() error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
fmt.Println("after build index ..................................")
|
||||
|
||||
if !fOk && !bOk {
|
||||
return errors.New("we expect FloatVectorFieldData or BinaryVectorFieldData")
|
||||
}
|
||||
|
||||
fmt.Println("before serialize .............................................")
|
||||
indexBlobs, err := it.index.Serialize()
|
||||
fmt.Println("after serialize .............................................")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -24,11 +24,6 @@ func (task *createIndexTask) Ts() (Timestamp, error) {
|
||||
}
|
||||
|
||||
func (task *createIndexTask) Execute() error {
|
||||
// modify schema
|
||||
if err := task.mt.UpdateFieldIndexParams(task.req.CollectionName, task.req.FieldName, task.req.ExtraParams); err != nil {
|
||||
return err
|
||||
}
|
||||
// check if closed segment has the same index build history
|
||||
collMeta, err := task.mt.GetCollectionByName(task.req.CollectionName)
|
||||
if err != nil {
|
||||
return err
|
||||
@ -44,6 +39,20 @@ func (task *createIndexTask) Execute() error {
|
||||
return fmt.Errorf("can not find field name %s", task.req.FieldName)
|
||||
}
|
||||
|
||||
// pre checks
|
||||
isIndexable, err := task.mt.IsIndexable(collMeta.ID, fieldID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !isIndexable {
|
||||
return fmt.Errorf("field %s is not vector", task.req.FieldName)
|
||||
}
|
||||
|
||||
// modify schema
|
||||
if err := task.mt.UpdateFieldIndexParams(task.req.CollectionName, task.req.FieldName, task.req.ExtraParams); err != nil {
|
||||
return err
|
||||
}
|
||||
// check if closed segment has the same index build history
|
||||
for _, segID := range collMeta.SegmentIDs {
|
||||
segMeta, err := task.mt.GetSegmentByID(segID)
|
||||
if err != nil {
|
||||
|
@ -218,7 +218,6 @@ func CreateServer(ctx context.Context) (*Master, error) {
|
||||
|
||||
m.grpcServer = grpc.NewServer()
|
||||
masterpb.RegisterMasterServer(m.grpcServer, m)
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
|
@ -110,7 +110,6 @@ func TestMaster(t *testing.T) {
|
||||
|
||||
conn, err := grpc.DialContext(ctx, Params.Address, grpc.WithInsecure(), grpc.WithBlock())
|
||||
require.Nil(t, err)
|
||||
|
||||
cli := masterpb.NewMasterClient(conn)
|
||||
|
||||
t.Run("TestConfigTask", func(t *testing.T) {
|
||||
@ -887,6 +886,12 @@ func TestMaster(t *testing.T) {
|
||||
var k2sMsgstream ms.MsgStream = k2sMs
|
||||
assert.True(t, receiveTimeTickMsg(&k2sMsgstream))
|
||||
|
||||
conn, err := grpc.DialContext(ctx, Params.Address, grpc.WithInsecure(), grpc.WithBlock())
|
||||
assert.Nil(t, err)
|
||||
defer conn.Close()
|
||||
|
||||
cli := masterpb.NewMasterClient(conn)
|
||||
|
||||
sch := schemapb.CollectionSchema{
|
||||
Name: "name" + strconv.FormatUint(rand.Uint64(), 10),
|
||||
Description: "test collection",
|
||||
|
@ -126,7 +126,7 @@ func TestSegmentManager_AssignSegment(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
time.Sleep(time.Duration(Params.SegIDAssignExpiration))
|
||||
time.Sleep(time.Duration(Params.SegIDAssignExpiration) * time.Millisecond)
|
||||
timestamp, err := globalTsoAllocator()
|
||||
assert.Nil(t, err)
|
||||
err = mt.UpdateSegment(&pb.SegmentMeta{
|
||||
@ -156,3 +156,122 @@ func TestSegmentManager_AssignSegment(t *testing.T) {
|
||||
assert.Nil(t, err)
|
||||
assert.NotEqualValues(t, 0, segMeta.CloseTime)
|
||||
}
|
||||
|
||||
func TestSegmentManager_SycnWritenode(t *testing.T) {
|
||||
ctx, cancelFunc := context.WithCancel(context.TODO())
|
||||
defer cancelFunc()
|
||||
|
||||
Init()
|
||||
Params.TopicNum = 5
|
||||
Params.QueryNodeNum = 3
|
||||
Params.SegmentSize = 536870912 / 1024 / 1024
|
||||
Params.SegmentSizeFactor = 0.75
|
||||
Params.DefaultRecordSize = 1024
|
||||
Params.MinSegIDAssignCnt = 1048576 / 1024
|
||||
Params.SegIDAssignExpiration = 2000
|
||||
etcdAddress := Params.EtcdAddress
|
||||
cli, err := clientv3.New(clientv3.Config{Endpoints: []string{etcdAddress}})
|
||||
assert.Nil(t, err)
|
||||
rootPath := "/test/root"
|
||||
_, err = cli.Delete(ctx, rootPath, clientv3.WithPrefix())
|
||||
assert.Nil(t, err)
|
||||
|
||||
kvBase := etcdkv.NewEtcdKV(cli, rootPath)
|
||||
defer kvBase.Close()
|
||||
mt, err := NewMetaTable(kvBase)
|
||||
assert.Nil(t, err)
|
||||
|
||||
collName := "segmgr_test_coll"
|
||||
var collID int64 = 1001
|
||||
partitionTag := "test_part"
|
||||
schema := &schemapb.CollectionSchema{
|
||||
Name: collName,
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{FieldID: 1, Name: "f1", IsPrimaryKey: false, DataType: schemapb.DataType_INT32},
|
||||
{FieldID: 2, Name: "f2", IsPrimaryKey: false, DataType: schemapb.DataType_VECTOR_FLOAT, TypeParams: []*commonpb.KeyValuePair{
|
||||
{Key: "dim", Value: "128"},
|
||||
}},
|
||||
},
|
||||
}
|
||||
err = mt.AddCollection(&pb.CollectionMeta{
|
||||
ID: collID,
|
||||
Schema: schema,
|
||||
CreateTime: 0,
|
||||
SegmentIDs: []UniqueID{},
|
||||
PartitionTags: []string{},
|
||||
})
|
||||
assert.Nil(t, err)
|
||||
err = mt.AddPartition(collID, partitionTag)
|
||||
assert.Nil(t, err)
|
||||
|
||||
var cnt int64
|
||||
globalIDAllocator := func() (UniqueID, error) {
|
||||
val := atomic.AddInt64(&cnt, 1)
|
||||
return val, nil
|
||||
}
|
||||
globalTsoAllocator := func() (Timestamp, error) {
|
||||
val := atomic.AddInt64(&cnt, 1)
|
||||
phy := time.Now().UnixNano() / int64(time.Millisecond)
|
||||
ts := tsoutil.ComposeTS(phy, val)
|
||||
return ts, nil
|
||||
}
|
||||
syncWriteChan := make(chan *msgstream.TimeTickMsg)
|
||||
syncProxyChan := make(chan *msgstream.TimeTickMsg)
|
||||
|
||||
segAssigner := NewSegmentAssigner(ctx, mt, globalTsoAllocator, syncProxyChan)
|
||||
mockScheduler := &MockFlushScheduler{}
|
||||
segManager, err := NewSegmentManager(ctx, mt, globalIDAllocator, globalTsoAllocator, syncWriteChan, mockScheduler, segAssigner)
|
||||
assert.Nil(t, err)
|
||||
|
||||
segManager.Start()
|
||||
defer segManager.Close()
|
||||
sizePerRecord, err := typeutil.EstimateSizePerRecord(schema)
|
||||
assert.Nil(t, err)
|
||||
maxCount := uint32(Params.SegmentSize * 1024 * 1024 / float64(sizePerRecord))
|
||||
|
||||
req := []*internalpb.SegIDRequest{
|
||||
{Count: maxCount, ChannelID: 1, CollName: collName, PartitionTag: partitionTag},
|
||||
{Count: maxCount, ChannelID: 2, CollName: collName, PartitionTag: partitionTag},
|
||||
{Count: maxCount, ChannelID: 3, CollName: collName, PartitionTag: partitionTag},
|
||||
}
|
||||
assignSegment, err := segManager.AssignSegment(req)
|
||||
assert.Nil(t, err)
|
||||
timestamp, err := globalTsoAllocator()
|
||||
assert.Nil(t, err)
|
||||
for i := 0; i < len(assignSegment); i++ {
|
||||
assert.EqualValues(t, maxCount, assignSegment[i].Count)
|
||||
assert.EqualValues(t, i+1, assignSegment[i].ChannelID)
|
||||
|
||||
err = mt.UpdateSegment(&pb.SegmentMeta{
|
||||
SegmentID: assignSegment[i].SegID,
|
||||
CollectionID: collID,
|
||||
PartitionTag: partitionTag,
|
||||
ChannelStart: 0,
|
||||
ChannelEnd: 1,
|
||||
CloseTime: timestamp,
|
||||
NumRows: int64(maxCount),
|
||||
MemSize: 500000,
|
||||
})
|
||||
assert.Nil(t, err)
|
||||
}
|
||||
|
||||
time.Sleep(time.Duration(Params.SegIDAssignExpiration) * time.Millisecond)
|
||||
|
||||
timestamp, err = globalTsoAllocator()
|
||||
assert.Nil(t, err)
|
||||
tsMsg := &msgstream.TimeTickMsg{
|
||||
BaseMsg: msgstream.BaseMsg{
|
||||
BeginTimestamp: timestamp, EndTimestamp: timestamp, HashValues: []uint32{},
|
||||
},
|
||||
TimeTickMsg: internalpb.TimeTickMsg{
|
||||
MsgType: internalpb.MsgType_kTimeTick,
|
||||
PeerID: 1,
|
||||
Timestamp: timestamp,
|
||||
},
|
||||
}
|
||||
syncWriteChan <- tsMsg
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
|
||||
status := segManager.collStatus[collID]
|
||||
assert.Empty(t, status.segments)
|
||||
}
|
||||
|
@ -1,8 +1,6 @@
|
||||
package msgstream
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/golang/protobuf/proto"
|
||||
internalPb "github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
|
||||
)
|
||||
@ -10,8 +8,6 @@ import (
|
||||
type MsgType = internalPb.MsgType
|
||||
|
||||
type TsMsg interface {
|
||||
GetMsgContext() context.Context
|
||||
SetMsgContext(context.Context)
|
||||
BeginTs() Timestamp
|
||||
EndTs() Timestamp
|
||||
Type() MsgType
|
||||
@ -21,7 +17,6 @@ type TsMsg interface {
|
||||
}
|
||||
|
||||
type BaseMsg struct {
|
||||
MsgCtx context.Context
|
||||
BeginTimestamp Timestamp
|
||||
EndTimestamp Timestamp
|
||||
HashValues []uint32
|
||||
@ -49,14 +44,6 @@ func (it *InsertMsg) Type() MsgType {
|
||||
return it.MsgType
|
||||
}
|
||||
|
||||
func (it *InsertMsg) GetMsgContext() context.Context {
|
||||
return it.MsgCtx
|
||||
}
|
||||
|
||||
func (it *InsertMsg) SetMsgContext(ctx context.Context) {
|
||||
it.MsgCtx = ctx
|
||||
}
|
||||
|
||||
func (it *InsertMsg) Marshal(input TsMsg) ([]byte, error) {
|
||||
insertMsg := input.(*InsertMsg)
|
||||
insertRequest := &insertMsg.InsertRequest
|
||||
@ -101,13 +88,6 @@ func (fl *FlushMsg) Type() MsgType {
|
||||
return fl.GetMsgType()
|
||||
}
|
||||
|
||||
func (fl *FlushMsg) GetMsgContext() context.Context {
|
||||
return fl.MsgCtx
|
||||
}
|
||||
func (fl *FlushMsg) SetMsgContext(ctx context.Context) {
|
||||
fl.MsgCtx = ctx
|
||||
}
|
||||
|
||||
func (fl *FlushMsg) Marshal(input TsMsg) ([]byte, error) {
|
||||
flushMsgTask := input.(*FlushMsg)
|
||||
flushMsg := &flushMsgTask.FlushMsg
|
||||
@ -141,14 +121,6 @@ func (dt *DeleteMsg) Type() MsgType {
|
||||
return dt.MsgType
|
||||
}
|
||||
|
||||
func (dt *DeleteMsg) GetMsgContext() context.Context {
|
||||
return dt.MsgCtx
|
||||
}
|
||||
|
||||
func (dt *DeleteMsg) SetMsgContext(ctx context.Context) {
|
||||
dt.MsgCtx = ctx
|
||||
}
|
||||
|
||||
func (dt *DeleteMsg) Marshal(input TsMsg) ([]byte, error) {
|
||||
deleteTask := input.(*DeleteMsg)
|
||||
deleteRequest := &deleteTask.DeleteRequest
|
||||
@ -193,14 +165,6 @@ func (st *SearchMsg) Type() MsgType {
|
||||
return st.MsgType
|
||||
}
|
||||
|
||||
func (st *SearchMsg) GetMsgContext() context.Context {
|
||||
return st.MsgCtx
|
||||
}
|
||||
|
||||
func (st *SearchMsg) SetMsgContext(ctx context.Context) {
|
||||
st.MsgCtx = ctx
|
||||
}
|
||||
|
||||
func (st *SearchMsg) Marshal(input TsMsg) ([]byte, error) {
|
||||
searchTask := input.(*SearchMsg)
|
||||
searchRequest := &searchTask.SearchRequest
|
||||
@ -234,14 +198,6 @@ func (srt *SearchResultMsg) Type() MsgType {
|
||||
return srt.MsgType
|
||||
}
|
||||
|
||||
func (srt *SearchResultMsg) GetMsgContext() context.Context {
|
||||
return srt.MsgCtx
|
||||
}
|
||||
|
||||
func (srt *SearchResultMsg) SetMsgContext(ctx context.Context) {
|
||||
srt.MsgCtx = ctx
|
||||
}
|
||||
|
||||
func (srt *SearchResultMsg) Marshal(input TsMsg) ([]byte, error) {
|
||||
searchResultTask := input.(*SearchResultMsg)
|
||||
searchResultRequest := &searchResultTask.SearchResult
|
||||
@ -275,14 +231,6 @@ func (tst *TimeTickMsg) Type() MsgType {
|
||||
return tst.MsgType
|
||||
}
|
||||
|
||||
func (tst *TimeTickMsg) GetMsgContext() context.Context {
|
||||
return tst.MsgCtx
|
||||
}
|
||||
|
||||
func (tst *TimeTickMsg) SetMsgContext(ctx context.Context) {
|
||||
tst.MsgCtx = ctx
|
||||
}
|
||||
|
||||
func (tst *TimeTickMsg) Marshal(input TsMsg) ([]byte, error) {
|
||||
timeTickTask := input.(*TimeTickMsg)
|
||||
timeTick := &timeTickTask.TimeTickMsg
|
||||
@ -316,14 +264,6 @@ func (qs *QueryNodeStatsMsg) Type() MsgType {
|
||||
return qs.MsgType
|
||||
}
|
||||
|
||||
func (qs *QueryNodeStatsMsg) GetMsgContext() context.Context {
|
||||
return qs.MsgCtx
|
||||
}
|
||||
|
||||
func (qs *QueryNodeStatsMsg) SetMsgContext(ctx context.Context) {
|
||||
qs.MsgCtx = ctx
|
||||
}
|
||||
|
||||
func (qs *QueryNodeStatsMsg) Marshal(input TsMsg) ([]byte, error) {
|
||||
queryNodeSegStatsTask := input.(*QueryNodeStatsMsg)
|
||||
queryNodeSegStats := &queryNodeSegStatsTask.QueryNodeStats
|
||||
@ -365,14 +305,6 @@ func (cc *CreateCollectionMsg) Type() MsgType {
|
||||
return cc.MsgType
|
||||
}
|
||||
|
||||
func (cc *CreateCollectionMsg) GetMsgContext() context.Context {
|
||||
return cc.MsgCtx
|
||||
}
|
||||
|
||||
func (cc *CreateCollectionMsg) SetMsgContext(ctx context.Context) {
|
||||
cc.MsgCtx = ctx
|
||||
}
|
||||
|
||||
func (cc *CreateCollectionMsg) Marshal(input TsMsg) ([]byte, error) {
|
||||
createCollectionMsg := input.(*CreateCollectionMsg)
|
||||
createCollectionRequest := &createCollectionMsg.CreateCollectionRequest
|
||||
@ -405,13 +337,6 @@ type DropCollectionMsg struct {
|
||||
func (dc *DropCollectionMsg) Type() MsgType {
|
||||
return dc.MsgType
|
||||
}
|
||||
func (dc *DropCollectionMsg) GetMsgContext() context.Context {
|
||||
return dc.MsgCtx
|
||||
}
|
||||
|
||||
func (dc *DropCollectionMsg) SetMsgContext(ctx context.Context) {
|
||||
dc.MsgCtx = ctx
|
||||
}
|
||||
|
||||
func (dc *DropCollectionMsg) Marshal(input TsMsg) ([]byte, error) {
|
||||
dropCollectionMsg := input.(*DropCollectionMsg)
|
||||
@ -436,20 +361,111 @@ func (dc *DropCollectionMsg) Unmarshal(input []byte) (TsMsg, error) {
|
||||
return dropCollectionMsg, nil
|
||||
}
|
||||
|
||||
/////////////////////////////////////////HasCollection//////////////////////////////////////////
|
||||
type HasCollectionMsg struct {
|
||||
BaseMsg
|
||||
internalPb.HasCollectionRequest
|
||||
}
|
||||
|
||||
func (hc *HasCollectionMsg) Type() MsgType {
|
||||
return hc.MsgType
|
||||
}
|
||||
|
||||
func (hc *HasCollectionMsg) Marshal(input TsMsg) ([]byte, error) {
|
||||
hasCollectionMsg := input.(*HasCollectionMsg)
|
||||
hasCollectionRequest := &hasCollectionMsg.HasCollectionRequest
|
||||
mb, err := proto.Marshal(hasCollectionRequest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return mb, nil
|
||||
}
|
||||
|
||||
func (hc *HasCollectionMsg) Unmarshal(input []byte) (TsMsg, error) {
|
||||
hasCollectionRequest := internalPb.HasCollectionRequest{}
|
||||
err := proto.Unmarshal(input, &hasCollectionRequest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hasCollectionMsg := &HasCollectionMsg{HasCollectionRequest: hasCollectionRequest}
|
||||
hasCollectionMsg.BeginTimestamp = hasCollectionMsg.Timestamp
|
||||
hasCollectionMsg.EndTimestamp = hasCollectionMsg.Timestamp
|
||||
|
||||
return hasCollectionMsg, nil
|
||||
}
|
||||
|
||||
/////////////////////////////////////////DescribeCollection//////////////////////////////////////////
|
||||
type DescribeCollectionMsg struct {
|
||||
BaseMsg
|
||||
internalPb.DescribeCollectionRequest
|
||||
}
|
||||
|
||||
func (dc *DescribeCollectionMsg) Type() MsgType {
|
||||
return dc.MsgType
|
||||
}
|
||||
|
||||
func (dc *DescribeCollectionMsg) Marshal(input TsMsg) ([]byte, error) {
|
||||
describeCollectionMsg := input.(*DescribeCollectionMsg)
|
||||
describeCollectionRequest := &describeCollectionMsg.DescribeCollectionRequest
|
||||
mb, err := proto.Marshal(describeCollectionRequest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return mb, nil
|
||||
}
|
||||
|
||||
func (dc *DescribeCollectionMsg) Unmarshal(input []byte) (TsMsg, error) {
|
||||
describeCollectionRequest := internalPb.DescribeCollectionRequest{}
|
||||
err := proto.Unmarshal(input, &describeCollectionRequest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
describeCollectionMsg := &DescribeCollectionMsg{DescribeCollectionRequest: describeCollectionRequest}
|
||||
describeCollectionMsg.BeginTimestamp = describeCollectionMsg.Timestamp
|
||||
describeCollectionMsg.EndTimestamp = describeCollectionMsg.Timestamp
|
||||
|
||||
return describeCollectionMsg, nil
|
||||
}
|
||||
|
||||
/////////////////////////////////////////ShowCollection//////////////////////////////////////////
|
||||
type ShowCollectionMsg struct {
|
||||
BaseMsg
|
||||
internalPb.ShowCollectionRequest
|
||||
}
|
||||
|
||||
func (sc *ShowCollectionMsg) Type() MsgType {
|
||||
return sc.MsgType
|
||||
}
|
||||
|
||||
func (sc *ShowCollectionMsg) Marshal(input TsMsg) ([]byte, error) {
|
||||
showCollectionMsg := input.(*ShowCollectionMsg)
|
||||
showCollectionRequest := &showCollectionMsg.ShowCollectionRequest
|
||||
mb, err := proto.Marshal(showCollectionRequest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return mb, nil
|
||||
}
|
||||
|
||||
func (sc *ShowCollectionMsg) Unmarshal(input []byte) (TsMsg, error) {
|
||||
showCollectionRequest := internalPb.ShowCollectionRequest{}
|
||||
err := proto.Unmarshal(input, &showCollectionRequest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
showCollectionMsg := &ShowCollectionMsg{ShowCollectionRequest: showCollectionRequest}
|
||||
showCollectionMsg.BeginTimestamp = showCollectionMsg.Timestamp
|
||||
showCollectionMsg.EndTimestamp = showCollectionMsg.Timestamp
|
||||
|
||||
return showCollectionMsg, nil
|
||||
}
|
||||
|
||||
/////////////////////////////////////////CreatePartition//////////////////////////////////////////
|
||||
type CreatePartitionMsg struct {
|
||||
BaseMsg
|
||||
internalPb.CreatePartitionRequest
|
||||
}
|
||||
|
||||
func (cc *CreatePartitionMsg) GetMsgContext() context.Context {
|
||||
return cc.MsgCtx
|
||||
}
|
||||
|
||||
func (cc *CreatePartitionMsg) SetMsgContext(ctx context.Context) {
|
||||
cc.MsgCtx = ctx
|
||||
}
|
||||
|
||||
func (cc *CreatePartitionMsg) Type() MsgType {
|
||||
return cc.MsgType
|
||||
}
|
||||
@ -483,14 +499,6 @@ type DropPartitionMsg struct {
|
||||
internalPb.DropPartitionRequest
|
||||
}
|
||||
|
||||
func (dc *DropPartitionMsg) GetMsgContext() context.Context {
|
||||
return dc.MsgCtx
|
||||
}
|
||||
|
||||
func (dc *DropPartitionMsg) SetMsgContext(ctx context.Context) {
|
||||
dc.MsgCtx = ctx
|
||||
}
|
||||
|
||||
func (dc *DropPartitionMsg) Type() MsgType {
|
||||
return dc.MsgType
|
||||
}
|
||||
@ -518,6 +526,105 @@ func (dc *DropPartitionMsg) Unmarshal(input []byte) (TsMsg, error) {
|
||||
return dropPartitionMsg, nil
|
||||
}
|
||||
|
||||
/////////////////////////////////////////HasPartition//////////////////////////////////////////
|
||||
type HasPartitionMsg struct {
|
||||
BaseMsg
|
||||
internalPb.HasPartitionRequest
|
||||
}
|
||||
|
||||
func (hc *HasPartitionMsg) Type() MsgType {
|
||||
return hc.MsgType
|
||||
}
|
||||
|
||||
func (hc *HasPartitionMsg) Marshal(input TsMsg) ([]byte, error) {
|
||||
hasPartitionMsg := input.(*HasPartitionMsg)
|
||||
hasPartitionRequest := &hasPartitionMsg.HasPartitionRequest
|
||||
mb, err := proto.Marshal(hasPartitionRequest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return mb, nil
|
||||
}
|
||||
|
||||
func (hc *HasPartitionMsg) Unmarshal(input []byte) (TsMsg, error) {
|
||||
hasPartitionRequest := internalPb.HasPartitionRequest{}
|
||||
err := proto.Unmarshal(input, &hasPartitionRequest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hasPartitionMsg := &HasPartitionMsg{HasPartitionRequest: hasPartitionRequest}
|
||||
hasPartitionMsg.BeginTimestamp = hasPartitionMsg.Timestamp
|
||||
hasPartitionMsg.EndTimestamp = hasPartitionMsg.Timestamp
|
||||
|
||||
return hasPartitionMsg, nil
|
||||
}
|
||||
|
||||
/////////////////////////////////////////DescribePartition//////////////////////////////////////////
|
||||
type DescribePartitionMsg struct {
|
||||
BaseMsg
|
||||
internalPb.DescribePartitionRequest
|
||||
}
|
||||
|
||||
func (dc *DescribePartitionMsg) Type() MsgType {
|
||||
return dc.MsgType
|
||||
}
|
||||
|
||||
func (dc *DescribePartitionMsg) Marshal(input TsMsg) ([]byte, error) {
|
||||
describePartitionMsg := input.(*DescribePartitionMsg)
|
||||
describePartitionRequest := &describePartitionMsg.DescribePartitionRequest
|
||||
mb, err := proto.Marshal(describePartitionRequest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return mb, nil
|
||||
}
|
||||
|
||||
func (dc *DescribePartitionMsg) Unmarshal(input []byte) (TsMsg, error) {
|
||||
describePartitionRequest := internalPb.DescribePartitionRequest{}
|
||||
err := proto.Unmarshal(input, &describePartitionRequest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
describePartitionMsg := &DescribePartitionMsg{DescribePartitionRequest: describePartitionRequest}
|
||||
describePartitionMsg.BeginTimestamp = describePartitionMsg.Timestamp
|
||||
describePartitionMsg.EndTimestamp = describePartitionMsg.Timestamp
|
||||
|
||||
return describePartitionMsg, nil
|
||||
}
|
||||
|
||||
/////////////////////////////////////////ShowPartition//////////////////////////////////////////
|
||||
type ShowPartitionMsg struct {
|
||||
BaseMsg
|
||||
internalPb.ShowPartitionRequest
|
||||
}
|
||||
|
||||
func (sc *ShowPartitionMsg) Type() MsgType {
|
||||
return sc.MsgType
|
||||
}
|
||||
|
||||
func (sc *ShowPartitionMsg) Marshal(input TsMsg) ([]byte, error) {
|
||||
showPartitionMsg := input.(*ShowPartitionMsg)
|
||||
showPartitionRequest := &showPartitionMsg.ShowPartitionRequest
|
||||
mb, err := proto.Marshal(showPartitionRequest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return mb, nil
|
||||
}
|
||||
|
||||
func (sc *ShowPartitionMsg) Unmarshal(input []byte) (TsMsg, error) {
|
||||
showPartitionRequest := internalPb.ShowPartitionRequest{}
|
||||
err := proto.Unmarshal(input, &showPartitionRequest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
showPartitionMsg := &ShowPartitionMsg{ShowPartitionRequest: showPartitionRequest}
|
||||
showPartitionMsg.BeginTimestamp = showPartitionMsg.Timestamp
|
||||
showPartitionMsg.EndTimestamp = showPartitionMsg.Timestamp
|
||||
|
||||
return showPartitionMsg, nil
|
||||
}
|
||||
|
||||
/////////////////////////////////////////LoadIndex//////////////////////////////////////////
|
||||
type LoadIndexMsg struct {
|
||||
BaseMsg
|
||||
@ -528,14 +635,6 @@ func (lim *LoadIndexMsg) Type() MsgType {
|
||||
return lim.MsgType
|
||||
}
|
||||
|
||||
func (lim *LoadIndexMsg) GetMsgContext() context.Context {
|
||||
return lim.MsgCtx
|
||||
}
|
||||
|
||||
func (lim *LoadIndexMsg) SetMsgContext(ctx context.Context) {
|
||||
lim.MsgCtx = ctx
|
||||
}
|
||||
|
||||
func (lim *LoadIndexMsg) Marshal(input TsMsg) ([]byte, error) {
|
||||
loadIndexMsg := input.(*LoadIndexMsg)
|
||||
loadIndexRequest := &loadIndexMsg.LoadIndex
|
||||
|
@ -4,15 +4,12 @@ import (
|
||||
"context"
|
||||
"log"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/apache/pulsar-client-go/pulsar"
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/opentracing/opentracing-go"
|
||||
"github.com/opentracing/opentracing-go/ext"
|
||||
oplog "github.com/opentracing/opentracing-go/log"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/errors"
|
||||
commonPb "github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
|
||||
internalPb "github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
|
||||
@ -154,29 +151,6 @@ func (ms *PulsarMsgStream) Close() {
|
||||
}
|
||||
}
|
||||
|
||||
type propertiesReaderWriter struct {
|
||||
ppMap map[string]string
|
||||
}
|
||||
|
||||
func (ppRW *propertiesReaderWriter) Set(key, val string) {
|
||||
// The GRPC HPACK implementation rejects any uppercase keys here.
|
||||
//
|
||||
// As such, since the HTTP_HEADERS format is case-insensitive anyway, we
|
||||
// blindly lowercase the key (which is guaranteed to work in the
|
||||
// Inject/Extract sense per the OpenTracing spec).
|
||||
key = strings.ToLower(key)
|
||||
ppRW.ppMap[key] = val
|
||||
}
|
||||
|
||||
func (ppRW *propertiesReaderWriter) ForeachKey(handler func(key, val string) error) error {
|
||||
for k, val := range ppRW.ppMap {
|
||||
if err := handler(k, val); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ms *PulsarMsgStream) Produce(msgPack *MsgPack) error {
|
||||
tsMsgs := msgPack.Msgs
|
||||
if len(tsMsgs) <= 0 {
|
||||
@ -226,51 +200,12 @@ func (ms *PulsarMsgStream) Produce(msgPack *MsgPack) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
msg := &pulsar.ProducerMessage{Payload: mb}
|
||||
var child opentracing.Span
|
||||
if v.Msgs[i].Type() == internalPb.MsgType_kInsert ||
|
||||
v.Msgs[i].Type() == internalPb.MsgType_kSearch ||
|
||||
v.Msgs[i].Type() == internalPb.MsgType_kSearchResult {
|
||||
tracer := opentracing.GlobalTracer()
|
||||
ctx := v.Msgs[i].GetMsgContext()
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
|
||||
if parent := opentracing.SpanFromContext(ctx); parent != nil {
|
||||
child = tracer.StartSpan("start send pulsar msg",
|
||||
opentracing.FollowsFrom(parent.Context()))
|
||||
} else {
|
||||
child = tracer.StartSpan("start send pulsar msg")
|
||||
}
|
||||
child.SetTag("hash keys", v.Msgs[i].HashKeys())
|
||||
child.SetTag("start time", v.Msgs[i].BeginTs())
|
||||
child.SetTag("end time", v.Msgs[i].EndTs())
|
||||
child.SetTag("msg type", v.Msgs[i].Type())
|
||||
msg.Properties = make(map[string]string)
|
||||
err = tracer.Inject(child.Context(), opentracing.TextMap, &propertiesReaderWriter{msg.Properties})
|
||||
if err != nil {
|
||||
child.LogFields(oplog.Error(err))
|
||||
child.Finish()
|
||||
return err
|
||||
}
|
||||
child.LogFields(oplog.String("inject success", "inject success"))
|
||||
}
|
||||
|
||||
if _, err := (*ms.producers[k]).Send(
|
||||
context.Background(),
|
||||
msg,
|
||||
&pulsar.ProducerMessage{Payload: mb},
|
||||
); err != nil {
|
||||
if child != nil {
|
||||
child.LogFields(oplog.Error(err))
|
||||
child.Finish()
|
||||
}
|
||||
return err
|
||||
}
|
||||
if child != nil {
|
||||
child.Finish()
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
@ -283,50 +218,14 @@ func (ms *PulsarMsgStream) Broadcast(msgPack *MsgPack) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := &pulsar.ProducerMessage{Payload: mb}
|
||||
var child opentracing.Span
|
||||
if v.Type() == internalPb.MsgType_kInsert ||
|
||||
v.Type() == internalPb.MsgType_kSearch ||
|
||||
v.Type() == internalPb.MsgType_kSearchResult {
|
||||
tracer := opentracing.GlobalTracer()
|
||||
ctx := v.GetMsgContext()
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if parent := opentracing.SpanFromContext(ctx); parent != nil {
|
||||
child = tracer.StartSpan("start send pulsar msg",
|
||||
opentracing.FollowsFrom(parent.Context()))
|
||||
} else {
|
||||
child = tracer.StartSpan("start send pulsar msg, start time: %d")
|
||||
}
|
||||
child.SetTag("hash keys", v.HashKeys())
|
||||
child.SetTag("start time", v.BeginTs())
|
||||
child.SetTag("end time", v.EndTs())
|
||||
child.SetTag("msg type", v.Type())
|
||||
msg.Properties = make(map[string]string)
|
||||
err = tracer.Inject(child.Context(), opentracing.TextMap, &propertiesReaderWriter{msg.Properties})
|
||||
if err != nil {
|
||||
child.LogFields(oplog.Error(err))
|
||||
child.Finish()
|
||||
return err
|
||||
}
|
||||
child.LogFields(oplog.String("inject success", "inject success"))
|
||||
}
|
||||
for i := 0; i < producerLen; i++ {
|
||||
if _, err := (*ms.producers[i]).Send(
|
||||
context.Background(),
|
||||
msg,
|
||||
&pulsar.ProducerMessage{Payload: mb},
|
||||
); err != nil {
|
||||
if child != nil {
|
||||
child.LogFields(oplog.Error(err))
|
||||
child.Finish()
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
if child != nil {
|
||||
child.Finish()
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -359,7 +258,6 @@ func (ms *PulsarMsgStream) bufMsgPackToChannel() {
|
||||
for {
|
||||
select {
|
||||
case <-ms.ctx.Done():
|
||||
log.Println("done")
|
||||
return
|
||||
default:
|
||||
tsMsgList := make([]TsMsg, 0)
|
||||
@ -372,7 +270,6 @@ func (ms *PulsarMsgStream) bufMsgPackToChannel() {
|
||||
}
|
||||
|
||||
pulsarMsg, ok := value.Interface().(pulsar.ConsumerMessage)
|
||||
|
||||
if !ok {
|
||||
log.Printf("type assertion failed, not consumer message type")
|
||||
continue
|
||||
@ -386,23 +283,6 @@ func (ms *PulsarMsgStream) bufMsgPackToChannel() {
|
||||
continue
|
||||
}
|
||||
tsMsg, err := ms.unmarshal.Unmarshal(pulsarMsg.Payload(), headerMsg.MsgType)
|
||||
if tsMsg.Type() == internalPb.MsgType_kSearch ||
|
||||
tsMsg.Type() == internalPb.MsgType_kSearchResult {
|
||||
tracer := opentracing.GlobalTracer()
|
||||
spanContext, err := tracer.Extract(opentracing.HTTPHeaders, &propertiesReaderWriter{pulsarMsg.Properties()})
|
||||
if err != nil {
|
||||
log.Println("extract message err")
|
||||
log.Println(err.Error())
|
||||
}
|
||||
span := opentracing.StartSpan("pulsar msg received",
|
||||
ext.RPCServerOption(spanContext))
|
||||
span.SetTag("msg type", tsMsg.Type())
|
||||
span.SetTag("hash keys", tsMsg.HashKeys())
|
||||
span.SetTag("start time", tsMsg.BeginTs())
|
||||
span.SetTag("end time", tsMsg.EndTs())
|
||||
tsMsg.SetMsgContext(opentracing.ContextWithSpan(context.Background(), span))
|
||||
span.Finish()
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("Failed to unmarshal tsMsg, error = %v", err)
|
||||
continue
|
||||
@ -466,8 +346,6 @@ func (ms *PulsarTtMsgStream) bufMsgPackToChannel() {
|
||||
ms.inputBuf = make([]TsMsg, 0)
|
||||
isChannelReady := make([]bool, len(ms.consumers))
|
||||
eofMsgTimeStamp := make(map[int]Timestamp)
|
||||
spans := make(map[Timestamp]opentracing.Span)
|
||||
ctxs := make(map[Timestamp]context.Context)
|
||||
for {
|
||||
select {
|
||||
case <-ms.ctx.Done():
|
||||
@ -493,22 +371,8 @@ func (ms *PulsarTtMsgStream) bufMsgPackToChannel() {
|
||||
ms.inputBuf = append(ms.inputBuf, ms.unsolvedBuf...)
|
||||
ms.unsolvedBuf = ms.unsolvedBuf[:0]
|
||||
for _, v := range ms.inputBuf {
|
||||
var ctx context.Context
|
||||
var span opentracing.Span
|
||||
if v.Type() == internalPb.MsgType_kInsert {
|
||||
if _, ok := spans[v.BeginTs()]; !ok {
|
||||
span, ctx = opentracing.StartSpanFromContext(v.GetMsgContext(), "after find time tick")
|
||||
ctxs[v.BeginTs()] = ctx
|
||||
spans[v.BeginTs()] = span
|
||||
}
|
||||
}
|
||||
if v.EndTs() <= timeStamp {
|
||||
timeTickBuf = append(timeTickBuf, v)
|
||||
if v.Type() == internalPb.MsgType_kInsert {
|
||||
v.SetMsgContext(ctxs[v.BeginTs()])
|
||||
spans[v.BeginTs()].Finish()
|
||||
delete(spans, v.BeginTs())
|
||||
}
|
||||
} else {
|
||||
ms.unsolvedBuf = append(ms.unsolvedBuf, v)
|
||||
}
|
||||
@ -556,24 +420,6 @@ func (ms *PulsarTtMsgStream) findTimeTick(channelIndex int,
|
||||
if err != nil {
|
||||
log.Printf("Failed to unmarshal, error = %v", err)
|
||||
}
|
||||
|
||||
if tsMsg.Type() == internalPb.MsgType_kInsert {
|
||||
tracer := opentracing.GlobalTracer()
|
||||
spanContext, err := tracer.Extract(opentracing.HTTPHeaders, &propertiesReaderWriter{pulsarMsg.Properties()})
|
||||
if err != nil {
|
||||
log.Println("extract message err")
|
||||
log.Println(err.Error())
|
||||
}
|
||||
span := opentracing.StartSpan("pulsar msg received",
|
||||
ext.RPCServerOption(spanContext))
|
||||
span.SetTag("hash keys", tsMsg.HashKeys())
|
||||
span.SetTag("start time", tsMsg.BeginTs())
|
||||
span.SetTag("end time", tsMsg.EndTs())
|
||||
span.SetTag("msg type", tsMsg.Type())
|
||||
tsMsg.SetMsgContext(opentracing.ContextWithSpan(context.Background(), span))
|
||||
span.Finish()
|
||||
}
|
||||
|
||||
if headerMsg.MsgType == internalPb.MsgType_kTimeTick {
|
||||
eofMsgMap[channelIndex] = tsMsg.(*TimeTickMsg).Timestamp
|
||||
return
|
||||
@ -654,7 +500,7 @@ func insertRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, e
|
||||
result := make(map[int32]*MsgPack)
|
||||
for i, request := range tsMsgs {
|
||||
if request.Type() != internalPb.MsgType_kInsert {
|
||||
return nil, errors.New("msg's must be Insert")
|
||||
return nil, errors.New(string("msg's must be Insert"))
|
||||
}
|
||||
insertRequest := request.(*InsertMsg)
|
||||
keys := hashKeys[i]
|
||||
@ -665,7 +511,7 @@ func insertRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, e
|
||||
keysLen := len(keys)
|
||||
|
||||
if keysLen != timestampLen || keysLen != rowIDLen || keysLen != rowDataLen {
|
||||
return nil, errors.New("the length of hashValue, timestamps, rowIDs, RowData are not equal")
|
||||
return nil, errors.New(string("the length of hashValue, timestamps, rowIDs, RowData are not equal"))
|
||||
}
|
||||
for index, key := range keys {
|
||||
_, ok := result[key]
|
||||
@ -688,9 +534,6 @@ func insertRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, e
|
||||
}
|
||||
|
||||
insertMsg := &InsertMsg{
|
||||
BaseMsg: BaseMsg{
|
||||
MsgCtx: request.GetMsgContext(),
|
||||
},
|
||||
InsertRequest: sliceRequest,
|
||||
}
|
||||
result[key].Msgs = append(result[key].Msgs, insertMsg)
|
||||
@ -703,7 +546,7 @@ func deleteRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, e
|
||||
result := make(map[int32]*MsgPack)
|
||||
for i, request := range tsMsgs {
|
||||
if request.Type() != internalPb.MsgType_kDelete {
|
||||
return nil, errors.New("msg's must be Delete")
|
||||
return nil, errors.New(string("msg's must be Delete"))
|
||||
}
|
||||
deleteRequest := request.(*DeleteMsg)
|
||||
keys := hashKeys[i]
|
||||
@ -713,7 +556,7 @@ func deleteRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, e
|
||||
keysLen := len(keys)
|
||||
|
||||
if keysLen != timestampLen || keysLen != primaryKeysLen {
|
||||
return nil, errors.New("the length of hashValue, timestamps, primaryKeys are not equal")
|
||||
return nil, errors.New(string("the length of hashValue, timestamps, primaryKeys are not equal"))
|
||||
}
|
||||
|
||||
for index, key := range keys {
|
||||
@ -747,7 +590,7 @@ func defaultRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack,
|
||||
for i, request := range tsMsgs {
|
||||
keys := hashKeys[i]
|
||||
if len(keys) != 1 {
|
||||
return nil, errors.New("len(msg.hashValue) must equal 1")
|
||||
return nil, errors.New(string("len(msg.hashValue) must equal 1"))
|
||||
}
|
||||
key := keys[0]
|
||||
_, ok := result[key]
|
||||
|
@ -6,7 +6,6 @@ import (
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/opentracing/opentracing-go"
|
||||
"github.com/zilliztech/milvus-distributed/internal/msgstream"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
|
||||
@ -19,13 +18,8 @@ const (
|
||||
)
|
||||
|
||||
func (p *Proxy) Insert(ctx context.Context, in *servicepb.RowBatch) (*servicepb.IntegerRangeResponse, error) {
|
||||
span, ctx := opentracing.StartSpanFromContext(ctx, "insert grpc received")
|
||||
defer span.Finish()
|
||||
span.SetTag("collection name", in.CollectionName)
|
||||
span.SetTag("partition tag", in.PartitionTag)
|
||||
log.Println("insert into: ", in.CollectionName)
|
||||
it := &InsertTask{
|
||||
ctx: ctx,
|
||||
Condition: NewTaskCondition(ctx),
|
||||
BaseInsertTask: BaseInsertTask{
|
||||
BaseMsg: msgstream.BaseMsg{
|
||||
@ -125,14 +119,8 @@ func (p *Proxy) CreateCollection(ctx context.Context, req *schemapb.CollectionSc
|
||||
}
|
||||
|
||||
func (p *Proxy) Search(ctx context.Context, req *servicepb.Query) (*servicepb.QueryResult, error) {
|
||||
span, ctx := opentracing.StartSpanFromContext(ctx, "search grpc received")
|
||||
defer span.Finish()
|
||||
span.SetTag("collection name", req.CollectionName)
|
||||
span.SetTag("partition tag", req.PartitionTags)
|
||||
span.SetTag("dsl", req.Dsl)
|
||||
log.Println("search: ", req.CollectionName, req.Dsl)
|
||||
qt := &QueryTask{
|
||||
ctx: ctx,
|
||||
Condition: NewTaskCondition(ctx),
|
||||
SearchRequest: internalpb.SearchRequest{
|
||||
ProxyID: Params.ProxyID(),
|
||||
|
@ -2,8 +2,6 @@ package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"math/rand"
|
||||
"net"
|
||||
@ -11,10 +9,6 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/opentracing/opentracing-go"
|
||||
"github.com/uber/jaeger-client-go"
|
||||
"github.com/uber/jaeger-client-go/config"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/allocator"
|
||||
@ -45,9 +39,6 @@ type Proxy struct {
|
||||
manipulationMsgStream *msgstream.PulsarMsgStream
|
||||
queryMsgStream *msgstream.PulsarMsgStream
|
||||
|
||||
tracer opentracing.Tracer
|
||||
closer io.Closer
|
||||
|
||||
// Add callback functions at different stages
|
||||
startCallbacks []func()
|
||||
closeCallbacks []func()
|
||||
@ -60,28 +51,11 @@ func Init() {
|
||||
func CreateProxy(ctx context.Context) (*Proxy, error) {
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
ctx1, cancel := context.WithCancel(ctx)
|
||||
var err error
|
||||
p := &Proxy{
|
||||
proxyLoopCtx: ctx1,
|
||||
proxyLoopCancel: cancel,
|
||||
}
|
||||
|
||||
cfg := &config.Configuration{
|
||||
ServiceName: "proxy",
|
||||
Sampler: &config.SamplerConfig{
|
||||
Type: "const",
|
||||
Param: 1,
|
||||
},
|
||||
Reporter: &config.ReporterConfig{
|
||||
LogSpans: true,
|
||||
},
|
||||
}
|
||||
p.tracer, p.closer, err = cfg.NewTracer(config.Logger(jaeger.StdLogger))
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("ERROR: cannot init Jaeger: %v\n", err))
|
||||
}
|
||||
opentracing.SetGlobalTracer(p.tracer)
|
||||
|
||||
pulsarAddress := Params.PulsarAddress()
|
||||
|
||||
p.queryMsgStream = msgstream.NewPulsarMsgStream(p.proxyLoopCtx, Params.MsgStreamSearchBufSize())
|
||||
@ -224,17 +198,12 @@ func (p *Proxy) stopProxyLoop() {
|
||||
p.tick.Close()
|
||||
|
||||
p.proxyLoopWg.Wait()
|
||||
|
||||
}
|
||||
|
||||
// Close closes the server.
|
||||
func (p *Proxy) Close() {
|
||||
p.stopProxyLoop()
|
||||
|
||||
if p.closer != nil {
|
||||
p.closer.Close()
|
||||
}
|
||||
|
||||
for _, cb := range p.closeCallbacks {
|
||||
cb()
|
||||
}
|
||||
|
@ -488,7 +488,9 @@ func TestProxy_CreateIndex(t *testing.T) {
|
||||
go func(group *sync.WaitGroup) {
|
||||
defer group.Done()
|
||||
createCollection(t, collName)
|
||||
createIndex(t, collName, fieldName)
|
||||
if i%2 == 0 {
|
||||
createIndex(t, collName, fieldName)
|
||||
}
|
||||
dropCollection(t, collName)
|
||||
// dropIndex(t, collectionName, fieldName, indexName)
|
||||
}(&wg)
|
||||
@ -510,7 +512,9 @@ func TestProxy_DescribeIndex(t *testing.T) {
|
||||
go func(group *sync.WaitGroup) {
|
||||
defer group.Done()
|
||||
createCollection(t, collName)
|
||||
createIndex(t, collName, fieldName)
|
||||
if i%2 == 0 {
|
||||
createIndex(t, collName, fieldName)
|
||||
}
|
||||
req := &servicepb.DescribeIndexRequest{
|
||||
CollectionName: collName,
|
||||
FieldName: fieldName,
|
||||
@ -539,7 +543,9 @@ func TestProxy_DescribeIndexProgress(t *testing.T) {
|
||||
go func(group *sync.WaitGroup) {
|
||||
defer group.Done()
|
||||
createCollection(t, collName)
|
||||
createIndex(t, collName, fieldName)
|
||||
if i%2 == 0 {
|
||||
createIndex(t, collName, fieldName)
|
||||
}
|
||||
req := &servicepb.DescribeIndexProgressRequest{
|
||||
CollectionName: collName,
|
||||
FieldName: fieldName,
|
||||
|
@ -182,7 +182,6 @@ func insertRepackFunc(tsMsgs []msgstream.TsMsg,
|
||||
insertMsg := &msgstream.InsertMsg{
|
||||
InsertRequest: sliceRequest,
|
||||
}
|
||||
insertMsg.SetMsgContext(request.GetMsgContext())
|
||||
if together { // all rows with same hash value are accumulated to only one message
|
||||
if len(result[key].Msgs) <= 0 {
|
||||
result[key].Msgs = append(result[key].Msgs, insertMsg)
|
||||
|
@ -7,9 +7,6 @@ import (
|
||||
"math"
|
||||
"strconv"
|
||||
|
||||
"github.com/opentracing/opentracing-go"
|
||||
oplog "github.com/opentracing/opentracing-go/log"
|
||||
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/zilliztech/milvus-distributed/internal/allocator"
|
||||
"github.com/zilliztech/milvus-distributed/internal/msgstream"
|
||||
@ -77,21 +74,12 @@ func (it *InsertTask) Type() internalpb.MsgType {
|
||||
}
|
||||
|
||||
func (it *InsertTask) PreExecute() error {
|
||||
span, ctx := opentracing.StartSpanFromContext(it.ctx, "InsertTask preExecute")
|
||||
defer span.Finish()
|
||||
it.ctx = ctx
|
||||
span.SetTag("hash keys", it.ReqID)
|
||||
span.SetTag("start time", it.BeginTs())
|
||||
collectionName := it.BaseInsertTask.CollectionName
|
||||
if err := ValidateCollectionName(collectionName); err != nil {
|
||||
span.LogFields(oplog.Error(err))
|
||||
span.Finish()
|
||||
return err
|
||||
}
|
||||
partitionTag := it.BaseInsertTask.PartitionTag
|
||||
if err := ValidatePartitionTag(partitionTag, true); err != nil {
|
||||
span.LogFields(oplog.Error(err))
|
||||
span.Finish()
|
||||
return err
|
||||
}
|
||||
|
||||
@ -99,36 +87,22 @@ func (it *InsertTask) PreExecute() error {
|
||||
}
|
||||
|
||||
func (it *InsertTask) Execute() error {
|
||||
span, ctx := opentracing.StartSpanFromContext(it.ctx, "InsertTask Execute")
|
||||
defer span.Finish()
|
||||
it.ctx = ctx
|
||||
span.SetTag("hash keys", it.ReqID)
|
||||
span.SetTag("start time", it.BeginTs())
|
||||
collectionName := it.BaseInsertTask.CollectionName
|
||||
span.LogFields(oplog.String("collection_name", collectionName))
|
||||
if !globalMetaCache.Hit(collectionName) {
|
||||
err := globalMetaCache.Sync(collectionName)
|
||||
if err != nil {
|
||||
span.LogFields(oplog.Error(err))
|
||||
span.Finish()
|
||||
return err
|
||||
}
|
||||
}
|
||||
description, err := globalMetaCache.Get(collectionName)
|
||||
if err != nil || description == nil {
|
||||
span.LogFields(oplog.Error(err))
|
||||
span.Finish()
|
||||
return err
|
||||
}
|
||||
autoID := description.Schema.AutoID
|
||||
span.LogFields(oplog.Bool("auto_id", autoID))
|
||||
var rowIDBegin UniqueID
|
||||
var rowIDEnd UniqueID
|
||||
rowNums := len(it.BaseInsertTask.RowData)
|
||||
rowIDBegin, rowIDEnd, _ = it.rowIDAllocator.Alloc(uint32(rowNums))
|
||||
span.LogFields(oplog.Int("rowNums", rowNums),
|
||||
oplog.Int("rowIDBegin", int(rowIDBegin)),
|
||||
oplog.Int("rowIDEnd", int(rowIDEnd)))
|
||||
it.BaseInsertTask.RowIDs = make([]UniqueID, rowNums)
|
||||
for i := rowIDBegin; i < rowIDEnd; i++ {
|
||||
offset := i - rowIDBegin
|
||||
@ -151,8 +125,6 @@ func (it *InsertTask) Execute() error {
|
||||
EndTs: it.EndTs(),
|
||||
Msgs: make([]msgstream.TsMsg, 1),
|
||||
}
|
||||
tsMsg.SetMsgContext(ctx)
|
||||
span.LogFields(oplog.String("send msg", "send msg"))
|
||||
msgPack.Msgs[0] = tsMsg
|
||||
err = it.manipulationMsgStream.Produce(msgPack)
|
||||
|
||||
@ -166,14 +138,11 @@ func (it *InsertTask) Execute() error {
|
||||
if err != nil {
|
||||
it.result.Status.ErrorCode = commonpb.ErrorCode_UNEXPECTED_ERROR
|
||||
it.result.Status.Reason = err.Error()
|
||||
span.LogFields(oplog.Error(err))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (it *InsertTask) PostExecute() error {
|
||||
span, _ := opentracing.StartSpanFromContext(it.ctx, "InsertTask postExecute")
|
||||
defer span.Finish()
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -383,38 +352,24 @@ func (qt *QueryTask) SetTs(ts Timestamp) {
|
||||
}
|
||||
|
||||
func (qt *QueryTask) PreExecute() error {
|
||||
span, ctx := opentracing.StartSpanFromContext(qt.ctx, "QueryTask preExecute")
|
||||
defer span.Finish()
|
||||
qt.ctx = ctx
|
||||
span.SetTag("hash keys", qt.ReqID)
|
||||
span.SetTag("start time", qt.BeginTs())
|
||||
|
||||
collectionName := qt.query.CollectionName
|
||||
if !globalMetaCache.Hit(collectionName) {
|
||||
err := globalMetaCache.Sync(collectionName)
|
||||
if err != nil {
|
||||
span.LogFields(oplog.Error(err))
|
||||
span.Finish()
|
||||
return err
|
||||
}
|
||||
}
|
||||
_, err := globalMetaCache.Get(collectionName)
|
||||
if err != nil { // err is not nil if collection not exists
|
||||
span.LogFields(oplog.Error(err))
|
||||
span.Finish()
|
||||
return err
|
||||
}
|
||||
|
||||
if err := ValidateCollectionName(qt.query.CollectionName); err != nil {
|
||||
span.LogFields(oplog.Error(err))
|
||||
span.Finish()
|
||||
return err
|
||||
}
|
||||
|
||||
for _, tag := range qt.query.PartitionTags {
|
||||
if err := ValidatePartitionTag(tag, false); err != nil {
|
||||
span.LogFields(oplog.Error(err))
|
||||
span.Finish()
|
||||
return err
|
||||
}
|
||||
}
|
||||
@ -424,8 +379,6 @@ func (qt *QueryTask) PreExecute() error {
|
||||
}
|
||||
queryBytes, err := proto.Marshal(qt.query)
|
||||
if err != nil {
|
||||
span.LogFields(oplog.Error(err))
|
||||
span.Finish()
|
||||
return err
|
||||
}
|
||||
qt.Query = &commonpb.Blob{
|
||||
@ -435,11 +388,6 @@ func (qt *QueryTask) PreExecute() error {
|
||||
}
|
||||
|
||||
func (qt *QueryTask) Execute() error {
|
||||
span, ctx := opentracing.StartSpanFromContext(qt.ctx, "QueryTask Execute")
|
||||
defer span.Finish()
|
||||
qt.ctx = ctx
|
||||
span.SetTag("hash keys", qt.ReqID)
|
||||
span.SetTag("start time", qt.BeginTs())
|
||||
var tsMsg msgstream.TsMsg = &msgstream.SearchMsg{
|
||||
SearchRequest: qt.SearchRequest,
|
||||
BaseMsg: msgstream.BaseMsg{
|
||||
@ -453,31 +401,22 @@ func (qt *QueryTask) Execute() error {
|
||||
EndTs: qt.Timestamp,
|
||||
Msgs: make([]msgstream.TsMsg, 1),
|
||||
}
|
||||
tsMsg.SetMsgContext(ctx)
|
||||
msgPack.Msgs[0] = tsMsg
|
||||
err := qt.queryMsgStream.Produce(msgPack)
|
||||
log.Printf("[Proxy] length of searchMsg: %v", len(msgPack.Msgs))
|
||||
if err != nil {
|
||||
span.LogFields(oplog.Error(err))
|
||||
span.Finish()
|
||||
log.Printf("[Proxy] send search request failed: %v", err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (qt *QueryTask) PostExecute() error {
|
||||
span, _ := opentracing.StartSpanFromContext(qt.ctx, "QueryTask postExecute")
|
||||
defer span.Finish()
|
||||
span.SetTag("hash keys", qt.ReqID)
|
||||
span.SetTag("start time", qt.BeginTs())
|
||||
for {
|
||||
select {
|
||||
case <-qt.ctx.Done():
|
||||
log.Print("wait to finish failed, timeout!")
|
||||
span.LogFields(oplog.String("wait to finish failed, timeout", "wait to finish failed, timeout"))
|
||||
return errors.New("wait to finish failed, timeout")
|
||||
case searchResults := <-qt.resultBuf:
|
||||
span.LogFields(oplog.String("receive result", "receive result"))
|
||||
filterSearchResult := make([]*internalpb.SearchResult, 0)
|
||||
var filterReason string
|
||||
for _, partialSearchResult := range searchResults {
|
||||
@ -496,7 +435,6 @@ func (qt *QueryTask) PostExecute() error {
|
||||
Reason: filterReason,
|
||||
},
|
||||
}
|
||||
span.LogFields(oplog.Error(errors.New(filterReason)))
|
||||
return errors.New(filterReason)
|
||||
}
|
||||
|
||||
@ -588,7 +526,6 @@ func (qt *QueryTask) PostExecute() error {
|
||||
reducedHitsBs, err := proto.Marshal(reducedHits)
|
||||
if err != nil {
|
||||
log.Println("marshal error")
|
||||
span.LogFields(oplog.Error(err))
|
||||
return err
|
||||
}
|
||||
qt.result.Hits = append(qt.result.Hits, reducedHitsBs)
|
||||
@ -700,10 +637,7 @@ func (dct *DescribeCollectionTask) PreExecute() error {
|
||||
func (dct *DescribeCollectionTask) Execute() error {
|
||||
var err error
|
||||
dct.result, err = dct.masterClient.DescribeCollection(dct.ctx, &dct.DescribeCollectionRequest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = globalMetaCache.Update(dct.CollectionName.CollectionName, dct.result)
|
||||
globalMetaCache.Update(dct.CollectionName.CollectionName, dct.result)
|
||||
return err
|
||||
}
|
||||
|
||||
|
@ -1,11 +1,9 @@
|
||||
package querynode
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"math"
|
||||
|
||||
"github.com/opentracing/opentracing-go"
|
||||
"github.com/zilliztech/milvus-distributed/internal/msgstream"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
|
||||
internalPb "github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
|
||||
@ -34,28 +32,6 @@ func (fdmNode *filterDmNode) Operate(in []*Msg) []*Msg {
|
||||
// TODO: add error handling
|
||||
}
|
||||
|
||||
var childs []opentracing.Span
|
||||
tracer := opentracing.GlobalTracer()
|
||||
if tracer != nil && msgStreamMsg != nil {
|
||||
for _, msg := range msgStreamMsg.TsMessages() {
|
||||
if msg.Type() == internalPb.MsgType_kInsert || msg.Type() == internalPb.MsgType_kSearch {
|
||||
var child opentracing.Span
|
||||
ctx := msg.GetMsgContext()
|
||||
if parent := opentracing.SpanFromContext(ctx); parent != nil {
|
||||
child = tracer.StartSpan("pass filter node",
|
||||
opentracing.FollowsFrom(parent.Context()))
|
||||
} else {
|
||||
child = tracer.StartSpan("pass filter node")
|
||||
}
|
||||
child.SetTag("hash keys", msg.HashKeys())
|
||||
child.SetTag("start time", msg.BeginTs())
|
||||
child.SetTag("end time", msg.EndTs())
|
||||
msg.SetMsgContext(opentracing.ContextWithSpan(ctx, child))
|
||||
childs = append(childs, child)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ddMsg, ok := (*in[1]).(*ddMsg)
|
||||
if !ok {
|
||||
log.Println("type assertion failed for ddMsg")
|
||||
@ -70,20 +46,11 @@ func (fdmNode *filterDmNode) Operate(in []*Msg) []*Msg {
|
||||
timestampMax: msgStreamMsg.TimestampMax(),
|
||||
},
|
||||
}
|
||||
for key, msg := range msgStreamMsg.TsMessages() {
|
||||
for _, msg := range msgStreamMsg.TsMessages() {
|
||||
switch msg.Type() {
|
||||
case internalPb.MsgType_kInsert:
|
||||
var ctx2 context.Context
|
||||
if childs != nil {
|
||||
if childs[key] != nil {
|
||||
ctx2 = opentracing.ContextWithSpan(msg.GetMsgContext(), childs[key])
|
||||
} else {
|
||||
ctx2 = context.Background()
|
||||
}
|
||||
}
|
||||
resMsg := fdmNode.filterInvalidInsertMessage(msg.(*msgstream.InsertMsg))
|
||||
if resMsg != nil {
|
||||
resMsg.SetMsgContext(ctx2)
|
||||
iMsg.insertMessages = append(iMsg.insertMessages, resMsg)
|
||||
}
|
||||
// case internalPb.MsgType_kDelete:
|
||||
@ -95,10 +62,6 @@ func (fdmNode *filterDmNode) Operate(in []*Msg) []*Msg {
|
||||
|
||||
iMsg.gcRecord = ddMsg.gcRecord
|
||||
var res Msg = &iMsg
|
||||
|
||||
for _, child := range childs {
|
||||
child.Finish()
|
||||
}
|
||||
return []*Msg{&res}
|
||||
}
|
||||
|
||||
|
@ -1,15 +1,11 @@
|
||||
package querynode
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"sync"
|
||||
|
||||
"github.com/opentracing/opentracing-go"
|
||||
oplog "github.com/opentracing/opentracing-go/log"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
|
||||
internalPb "github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
|
||||
)
|
||||
|
||||
type insertNode struct {
|
||||
@ -18,7 +14,6 @@ type insertNode struct {
|
||||
}
|
||||
|
||||
type InsertData struct {
|
||||
insertContext map[int64]context.Context
|
||||
insertIDs map[UniqueID][]UniqueID
|
||||
insertTimestamps map[UniqueID][]Timestamp
|
||||
insertRecords map[UniqueID][]*commonpb.Blob
|
||||
@ -43,30 +38,7 @@ func (iNode *insertNode) Operate(in []*Msg) []*Msg {
|
||||
// TODO: add error handling
|
||||
}
|
||||
|
||||
var childs []opentracing.Span
|
||||
tracer := opentracing.GlobalTracer()
|
||||
if tracer != nil && iMsg != nil {
|
||||
for _, msg := range iMsg.insertMessages {
|
||||
if msg.Type() == internalPb.MsgType_kInsert || msg.Type() == internalPb.MsgType_kSearch {
|
||||
var child opentracing.Span
|
||||
ctx := msg.GetMsgContext()
|
||||
if parent := opentracing.SpanFromContext(ctx); parent != nil {
|
||||
child = tracer.StartSpan("pass insert node",
|
||||
opentracing.FollowsFrom(parent.Context()))
|
||||
} else {
|
||||
child = tracer.StartSpan("pass insert node")
|
||||
}
|
||||
child.SetTag("hash keys", msg.HashKeys())
|
||||
child.SetTag("start time", msg.BeginTs())
|
||||
child.SetTag("end time", msg.EndTs())
|
||||
msg.SetMsgContext(opentracing.ContextWithSpan(ctx, child))
|
||||
childs = append(childs, child)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
insertData := InsertData{
|
||||
insertContext: make(map[int64]context.Context),
|
||||
insertIDs: make(map[int64][]int64),
|
||||
insertTimestamps: make(map[int64][]uint64),
|
||||
insertRecords: make(map[int64][]*commonpb.Blob),
|
||||
@ -75,7 +47,6 @@ func (iNode *insertNode) Operate(in []*Msg) []*Msg {
|
||||
|
||||
// 1. hash insertMessages to insertData
|
||||
for _, task := range iMsg.insertMessages {
|
||||
insertData.insertContext[task.SegmentID] = task.GetMsgContext()
|
||||
insertData.insertIDs[task.SegmentID] = append(insertData.insertIDs[task.SegmentID], task.RowIDs...)
|
||||
insertData.insertTimestamps[task.SegmentID] = append(insertData.insertTimestamps[task.SegmentID], task.Timestamps...)
|
||||
insertData.insertRecords[task.SegmentID] = append(insertData.insertRecords[task.SegmentID], task.RowData...)
|
||||
@ -114,7 +85,7 @@ func (iNode *insertNode) Operate(in []*Msg) []*Msg {
|
||||
wg := sync.WaitGroup{}
|
||||
for segmentID := range insertData.insertRecords {
|
||||
wg.Add(1)
|
||||
go iNode.insert(insertData.insertContext[segmentID], &insertData, segmentID, &wg)
|
||||
go iNode.insert(&insertData, segmentID, &wg)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
@ -122,21 +93,15 @@ func (iNode *insertNode) Operate(in []*Msg) []*Msg {
|
||||
gcRecord: iMsg.gcRecord,
|
||||
timeRange: iMsg.timeRange,
|
||||
}
|
||||
for _, child := range childs {
|
||||
child.Finish()
|
||||
}
|
||||
return []*Msg{&res}
|
||||
}
|
||||
|
||||
func (iNode *insertNode) insert(ctx context.Context, insertData *InsertData, segmentID int64, wg *sync.WaitGroup) {
|
||||
span, _ := opentracing.StartSpanFromContext(ctx, "insert node insert function")
|
||||
defer span.Finish()
|
||||
func (iNode *insertNode) insert(insertData *InsertData, segmentID int64, wg *sync.WaitGroup) {
|
||||
var targetSegment, err = iNode.replica.getSegmentByID(segmentID)
|
||||
if err != nil {
|
||||
log.Println("cannot find segment:", segmentID)
|
||||
// TODO: add error handling
|
||||
wg.Done()
|
||||
span.LogFields(oplog.Error(err))
|
||||
return
|
||||
}
|
||||
|
||||
@ -150,7 +115,6 @@ func (iNode *insertNode) insert(ctx context.Context, insertData *InsertData, seg
|
||||
log.Println(err)
|
||||
// TODO: add error handling
|
||||
wg.Done()
|
||||
span.LogFields(oplog.Error(err))
|
||||
return
|
||||
}
|
||||
|
||||
|
@ -100,6 +100,7 @@ func (lis *loadIndexService) start() {
|
||||
continue
|
||||
}
|
||||
// 1. use msg's index paths to get index bytes
|
||||
fmt.Println("start load index")
|
||||
var indexBuffer [][]byte
|
||||
var err error
|
||||
fn := func() error {
|
||||
@ -138,6 +139,13 @@ func (lis *loadIndexService) start() {
|
||||
}
|
||||
}
|
||||
|
||||
func (lis *loadIndexService) close() {
|
||||
if lis.loadIndexMsgStream != nil {
|
||||
lis.loadIndexMsgStream.Close()
|
||||
}
|
||||
lis.cancel()
|
||||
}
|
||||
|
||||
func (lis *loadIndexService) printIndexParams(index []*commonpb.KeyValuePair) {
|
||||
fmt.Println("=================================================")
|
||||
for i := 0; i < len(index); i++ {
|
||||
|
@ -22,26 +22,29 @@ import (
|
||||
"github.com/zilliztech/milvus-distributed/internal/querynode/client"
|
||||
)
|
||||
|
||||
func TestLoadIndexService(t *testing.T) {
|
||||
func TestLoadIndexService_FloatVector(t *testing.T) {
|
||||
node := newQueryNode()
|
||||
collectionID := rand.Int63n(1000000)
|
||||
segmentID := rand.Int63n(1000000)
|
||||
initTestMeta(t, node, "collection0", collectionID, segmentID)
|
||||
|
||||
// loadIndexService and statsService
|
||||
suffix := "-test-search" + strconv.FormatInt(rand.Int63n(1000000), 10)
|
||||
oldSearchChannelNames := Params.SearchChannelNames
|
||||
var newSearchChannelNames []string
|
||||
for _, channel := range oldSearchChannelNames {
|
||||
newSearchChannelNames = append(newSearchChannelNames, channel+"new")
|
||||
}
|
||||
newSearchChannelNames := makeNewChannelNames(oldSearchChannelNames, suffix)
|
||||
Params.SearchChannelNames = newSearchChannelNames
|
||||
|
||||
oldSearchResultChannelNames := Params.SearchChannelNames
|
||||
var newSearchResultChannelNames []string
|
||||
for _, channel := range oldSearchResultChannelNames {
|
||||
newSearchResultChannelNames = append(newSearchResultChannelNames, channel+"new")
|
||||
}
|
||||
newSearchResultChannelNames := makeNewChannelNames(oldSearchResultChannelNames, suffix)
|
||||
Params.SearchResultChannelNames = newSearchResultChannelNames
|
||||
|
||||
oldLoadIndexChannelNames := Params.LoadIndexChannelNames
|
||||
newLoadIndexChannelNames := makeNewChannelNames(oldLoadIndexChannelNames, suffix)
|
||||
Params.LoadIndexChannelNames = newLoadIndexChannelNames
|
||||
|
||||
oldStatsChannelName := Params.StatsChannelName
|
||||
newStatsChannelNames := makeNewChannelNames([]string{oldStatsChannelName}, suffix)
|
||||
Params.StatsChannelName = newStatsChannelNames[0]
|
||||
go node.Start()
|
||||
|
||||
//generate insert data
|
||||
@ -328,9 +331,319 @@ func TestLoadIndexService(t *testing.T) {
|
||||
}
|
||||
Params.SearchChannelNames = oldSearchChannelNames
|
||||
Params.SearchResultChannelNames = oldSearchResultChannelNames
|
||||
Params.LoadIndexChannelNames = oldLoadIndexChannelNames
|
||||
Params.StatsChannelName = oldStatsChannelName
|
||||
fmt.Println("loadIndex floatVector test Done!")
|
||||
|
||||
defer assert.Equal(t, findFiledStats, true)
|
||||
<-node.queryNodeLoopCtx.Done()
|
||||
node.Close()
|
||||
}
|
||||
|
||||
func TestLoadIndexService_BinaryVector(t *testing.T) {
|
||||
node := newQueryNode()
|
||||
collectionID := rand.Int63n(1000000)
|
||||
segmentID := rand.Int63n(1000000)
|
||||
initTestMeta(t, node, "collection0", collectionID, segmentID, true)
|
||||
|
||||
// loadIndexService and statsService
|
||||
suffix := "-test-search-binary" + strconv.FormatInt(rand.Int63n(1000000), 10)
|
||||
oldSearchChannelNames := Params.SearchChannelNames
|
||||
newSearchChannelNames := makeNewChannelNames(oldSearchChannelNames, suffix)
|
||||
Params.SearchChannelNames = newSearchChannelNames
|
||||
|
||||
oldSearchResultChannelNames := Params.SearchChannelNames
|
||||
newSearchResultChannelNames := makeNewChannelNames(oldSearchResultChannelNames, suffix)
|
||||
Params.SearchResultChannelNames = newSearchResultChannelNames
|
||||
|
||||
oldLoadIndexChannelNames := Params.LoadIndexChannelNames
|
||||
newLoadIndexChannelNames := makeNewChannelNames(oldLoadIndexChannelNames, suffix)
|
||||
Params.LoadIndexChannelNames = newLoadIndexChannelNames
|
||||
|
||||
oldStatsChannelName := Params.StatsChannelName
|
||||
newStatsChannelNames := makeNewChannelNames([]string{oldStatsChannelName}, suffix)
|
||||
Params.StatsChannelName = newStatsChannelNames[0]
|
||||
go node.Start()
|
||||
|
||||
const msgLength = 1000
|
||||
const receiveBufSize = 1024
|
||||
const DIM = 128
|
||||
|
||||
// generator index data
|
||||
var indexRowData []byte
|
||||
for n := 0; n < msgLength; n++ {
|
||||
for i := 0; i < DIM/8; i++ {
|
||||
indexRowData = append(indexRowData, byte(rand.Intn(8)))
|
||||
}
|
||||
}
|
||||
|
||||
//generator insert data
|
||||
var insertRowBlob []*commonpb.Blob
|
||||
var timestamps []uint64
|
||||
var rowIDs []int64
|
||||
var hashValues []uint32
|
||||
offset := 0
|
||||
for n := 0; n < msgLength; n++ {
|
||||
rowData := make([]byte, 0)
|
||||
rowData = append(rowData, indexRowData[offset:offset+(DIM/8)]...)
|
||||
offset += DIM / 8
|
||||
age := make([]byte, 4)
|
||||
binary.LittleEndian.PutUint32(age, 1)
|
||||
rowData = append(rowData, age...)
|
||||
blob := &commonpb.Blob{
|
||||
Value: rowData,
|
||||
}
|
||||
insertRowBlob = append(insertRowBlob, blob)
|
||||
timestamps = append(timestamps, uint64(n))
|
||||
rowIDs = append(rowIDs, int64(n))
|
||||
hashValues = append(hashValues, uint32(n))
|
||||
}
|
||||
|
||||
var insertMsg msgstream.TsMsg = &msgstream.InsertMsg{
|
||||
BaseMsg: msgstream.BaseMsg{
|
||||
HashValues: hashValues,
|
||||
},
|
||||
InsertRequest: internalpb.InsertRequest{
|
||||
MsgType: internalpb.MsgType_kInsert,
|
||||
ReqID: 0,
|
||||
CollectionName: "collection0",
|
||||
PartitionTag: "default",
|
||||
SegmentID: segmentID,
|
||||
ChannelID: int64(0),
|
||||
ProxyID: int64(0),
|
||||
Timestamps: timestamps,
|
||||
RowIDs: rowIDs,
|
||||
RowData: insertRowBlob,
|
||||
},
|
||||
}
|
||||
insertMsgPack := msgstream.MsgPack{
|
||||
BeginTs: 0,
|
||||
EndTs: math.MaxUint64,
|
||||
Msgs: []msgstream.TsMsg{insertMsg},
|
||||
}
|
||||
|
||||
// generate timeTick
|
||||
timeTickMsg := &msgstream.TimeTickMsg{
|
||||
BaseMsg: msgstream.BaseMsg{
|
||||
BeginTimestamp: 0,
|
||||
EndTimestamp: 0,
|
||||
HashValues: []uint32{0},
|
||||
},
|
||||
TimeTickMsg: internalpb.TimeTickMsg{
|
||||
MsgType: internalpb.MsgType_kTimeTick,
|
||||
PeerID: UniqueID(0),
|
||||
Timestamp: math.MaxUint64,
|
||||
},
|
||||
}
|
||||
timeTickMsgPack := &msgstream.MsgPack{
|
||||
Msgs: []msgstream.TsMsg{timeTickMsg},
|
||||
}
|
||||
|
||||
// pulsar produce
|
||||
insertChannels := Params.InsertChannelNames
|
||||
ddChannels := Params.DDChannelNames
|
||||
|
||||
insertStream := msgstream.NewPulsarMsgStream(node.queryNodeLoopCtx, receiveBufSize)
|
||||
insertStream.SetPulsarClient(Params.PulsarAddress)
|
||||
insertStream.CreatePulsarProducers(insertChannels)
|
||||
ddStream := msgstream.NewPulsarMsgStream(node.queryNodeLoopCtx, receiveBufSize)
|
||||
ddStream.SetPulsarClient(Params.PulsarAddress)
|
||||
ddStream.CreatePulsarProducers(ddChannels)
|
||||
|
||||
var insertMsgStream msgstream.MsgStream = insertStream
|
||||
insertMsgStream.Start()
|
||||
var ddMsgStream msgstream.MsgStream = ddStream
|
||||
ddMsgStream.Start()
|
||||
|
||||
err := insertMsgStream.Produce(&insertMsgPack)
|
||||
assert.NoError(t, err)
|
||||
err = insertMsgStream.Broadcast(timeTickMsgPack)
|
||||
assert.NoError(t, err)
|
||||
err = ddMsgStream.Broadcast(timeTickMsgPack)
|
||||
assert.NoError(t, err)
|
||||
|
||||
//generate search data and send search msg
|
||||
searchRowData := indexRowData[42*(DIM/8) : 43*(DIM/8)]
|
||||
dslString := "{\"bool\": { \n\"vector\": {\n \"vec\": {\n \"metric_type\": \"JACCARD\", \n \"params\": {\n \"nprobe\": 10 \n},\n \"query\": \"$0\",\"topk\": 10 \n } \n } \n } \n }"
|
||||
placeholderValue := servicepb.PlaceholderValue{
|
||||
Tag: "$0",
|
||||
Type: servicepb.PlaceholderType_VECTOR_BINARY,
|
||||
Values: [][]byte{searchRowData},
|
||||
}
|
||||
placeholderGroup := servicepb.PlaceholderGroup{
|
||||
Placeholders: []*servicepb.PlaceholderValue{&placeholderValue},
|
||||
}
|
||||
placeGroupByte, err := proto.Marshal(&placeholderGroup)
|
||||
if err != nil {
|
||||
log.Print("marshal placeholderGroup failed")
|
||||
}
|
||||
query := servicepb.Query{
|
||||
CollectionName: "collection0",
|
||||
PartitionTags: []string{"default"},
|
||||
Dsl: dslString,
|
||||
PlaceholderGroup: placeGroupByte,
|
||||
}
|
||||
queryByte, err := proto.Marshal(&query)
|
||||
if err != nil {
|
||||
log.Print("marshal query failed")
|
||||
}
|
||||
blob := commonpb.Blob{
|
||||
Value: queryByte,
|
||||
}
|
||||
fn := func(n int64) *msgstream.MsgPack {
|
||||
searchMsg := &msgstream.SearchMsg{
|
||||
BaseMsg: msgstream.BaseMsg{
|
||||
HashValues: []uint32{0},
|
||||
},
|
||||
SearchRequest: internalpb.SearchRequest{
|
||||
MsgType: internalpb.MsgType_kSearch,
|
||||
ReqID: n,
|
||||
ProxyID: int64(1),
|
||||
Timestamp: uint64(msgLength),
|
||||
ResultChannelID: int64(0),
|
||||
Query: &blob,
|
||||
},
|
||||
}
|
||||
return &msgstream.MsgPack{
|
||||
Msgs: []msgstream.TsMsg{searchMsg},
|
||||
}
|
||||
}
|
||||
searchStream := msgstream.NewPulsarMsgStream(node.queryNodeLoopCtx, receiveBufSize)
|
||||
searchStream.SetPulsarClient(Params.PulsarAddress)
|
||||
searchStream.CreatePulsarProducers(newSearchChannelNames)
|
||||
searchStream.Start()
|
||||
err = searchStream.Produce(fn(1))
|
||||
assert.NoError(t, err)
|
||||
|
||||
//get search result
|
||||
searchResultStream := msgstream.NewPulsarMsgStream(node.queryNodeLoopCtx, receiveBufSize)
|
||||
searchResultStream.SetPulsarClient(Params.PulsarAddress)
|
||||
unmarshalDispatcher := msgstream.NewUnmarshalDispatcher()
|
||||
searchResultStream.CreatePulsarConsumers(newSearchResultChannelNames, "loadIndexTestSubSearchResult2", unmarshalDispatcher, receiveBufSize)
|
||||
searchResultStream.Start()
|
||||
searchResult := searchResultStream.Consume()
|
||||
assert.NotNil(t, searchResult)
|
||||
unMarshaledHit := servicepb.Hits{}
|
||||
err = proto.Unmarshal(searchResult.Msgs[0].(*msgstream.SearchResultMsg).Hits[0], &unMarshaledHit)
|
||||
assert.Nil(t, err)
|
||||
|
||||
// gen load index message pack
|
||||
indexParams := make(map[string]string)
|
||||
indexParams["index_type"] = "BIN_IVF_FLAT"
|
||||
indexParams["index_mode"] = "cpu"
|
||||
indexParams["dim"] = "128"
|
||||
indexParams["k"] = "10"
|
||||
indexParams["nlist"] = "100"
|
||||
indexParams["nprobe"] = "10"
|
||||
indexParams["m"] = "4"
|
||||
indexParams["nbits"] = "8"
|
||||
indexParams["metric_type"] = "JACCARD"
|
||||
indexParams["SLICE_SIZE"] = "4"
|
||||
|
||||
var indexParamsKV []*commonpb.KeyValuePair
|
||||
for key, value := range indexParams {
|
||||
indexParamsKV = append(indexParamsKV, &commonpb.KeyValuePair{
|
||||
Key: key,
|
||||
Value: value,
|
||||
})
|
||||
}
|
||||
|
||||
// generator index
|
||||
typeParams := make(map[string]string)
|
||||
typeParams["dim"] = "128"
|
||||
index, err := indexbuilder.NewCIndex(typeParams, indexParams)
|
||||
assert.Nil(t, err)
|
||||
err = index.BuildBinaryVecIndexWithoutIds(indexRowData)
|
||||
assert.Equal(t, err, nil)
|
||||
|
||||
option := &minioKV.Option{
|
||||
Address: Params.MinioEndPoint,
|
||||
AccessKeyID: Params.MinioAccessKeyID,
|
||||
SecretAccessKeyID: Params.MinioSecretAccessKey,
|
||||
UseSSL: Params.MinioUseSSLStr,
|
||||
BucketName: Params.MinioBucketName,
|
||||
CreateBucket: true,
|
||||
}
|
||||
|
||||
minioKV, err := minioKV.NewMinIOKV(node.queryNodeLoopCtx, option)
|
||||
assert.Equal(t, err, nil)
|
||||
//save index to minio
|
||||
binarySet, err := index.Serialize()
|
||||
assert.Equal(t, err, nil)
|
||||
indexPaths := make([]string, 0)
|
||||
for _, index := range binarySet {
|
||||
path := strconv.Itoa(int(segmentID)) + "/" + index.Key
|
||||
indexPaths = append(indexPaths, path)
|
||||
minioKV.Save(path, string(index.Value))
|
||||
}
|
||||
|
||||
//test index search result
|
||||
indexResult, err := index.QueryOnBinaryVecIndexWithParam(searchRowData, indexParams)
|
||||
assert.Equal(t, err, nil)
|
||||
|
||||
// create loadIndexClient
|
||||
fieldID := UniqueID(100)
|
||||
loadIndexChannelNames := Params.LoadIndexChannelNames
|
||||
client := client.NewLoadIndexClient(node.queryNodeLoopCtx, Params.PulsarAddress, loadIndexChannelNames)
|
||||
client.LoadIndex(indexPaths, segmentID, fieldID, "vec", indexParams)
|
||||
|
||||
// init message stream consumer and do checks
|
||||
statsMs := msgstream.NewPulsarMsgStream(node.queryNodeLoopCtx, Params.StatsReceiveBufSize)
|
||||
statsMs.SetPulsarClient(Params.PulsarAddress)
|
||||
statsMs.CreatePulsarConsumers([]string{Params.StatsChannelName}, Params.MsgChannelSubName, msgstream.NewUnmarshalDispatcher(), Params.StatsReceiveBufSize)
|
||||
statsMs.Start()
|
||||
|
||||
findFiledStats := false
|
||||
for {
|
||||
receiveMsg := msgstream.MsgStream(statsMs).Consume()
|
||||
assert.NotNil(t, receiveMsg)
|
||||
assert.NotEqual(t, len(receiveMsg.Msgs), 0)
|
||||
|
||||
for _, msg := range receiveMsg.Msgs {
|
||||
statsMsg, ok := msg.(*msgstream.QueryNodeStatsMsg)
|
||||
if statsMsg.FieldStats == nil || len(statsMsg.FieldStats) == 0 {
|
||||
continue
|
||||
}
|
||||
findFiledStats = true
|
||||
assert.Equal(t, ok, true)
|
||||
assert.Equal(t, len(statsMsg.FieldStats), 1)
|
||||
fieldStats0 := statsMsg.FieldStats[0]
|
||||
assert.Equal(t, fieldStats0.FieldID, fieldID)
|
||||
assert.Equal(t, fieldStats0.CollectionID, collectionID)
|
||||
assert.Equal(t, len(fieldStats0.IndexStats), 1)
|
||||
indexStats0 := fieldStats0.IndexStats[0]
|
||||
params := indexStats0.IndexParams
|
||||
// sort index params by key
|
||||
sort.Slice(indexParamsKV, func(i, j int) bool { return indexParamsKV[i].Key < indexParamsKV[j].Key })
|
||||
indexEqual := node.loadIndexService.indexParamsEqual(params, indexParamsKV)
|
||||
assert.Equal(t, indexEqual, true)
|
||||
}
|
||||
|
||||
if findFiledStats {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
err = searchStream.Produce(fn(2))
|
||||
assert.NoError(t, err)
|
||||
searchResult = searchResultStream.Consume()
|
||||
assert.NotNil(t, searchResult)
|
||||
err = proto.Unmarshal(searchResult.Msgs[0].(*msgstream.SearchResultMsg).Hits[0], &unMarshaledHit)
|
||||
assert.Nil(t, err)
|
||||
|
||||
idsIndex := indexResult.IDs()
|
||||
idsSegment := unMarshaledHit.IDs
|
||||
assert.Equal(t, len(idsIndex), len(idsSegment))
|
||||
for i := 0; i < len(idsIndex); i++ {
|
||||
assert.Equal(t, idsIndex[i], idsSegment[i])
|
||||
}
|
||||
Params.SearchChannelNames = oldSearchChannelNames
|
||||
Params.SearchResultChannelNames = oldSearchResultChannelNames
|
||||
Params.LoadIndexChannelNames = oldLoadIndexChannelNames
|
||||
Params.StatsChannelName = oldStatsChannelName
|
||||
fmt.Println("loadIndex binaryVector test Done!")
|
||||
|
||||
defer assert.Equal(t, findFiledStats, true)
|
||||
<-node.queryNodeLoopCtx.Done()
|
||||
node.Close()
|
||||
}
|
||||
|
@ -14,12 +14,6 @@ import "C"
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/opentracing/opentracing-go"
|
||||
"github.com/uber/jaeger-client-go"
|
||||
"github.com/uber/jaeger-client-go/config"
|
||||
)
|
||||
|
||||
type QueryNode struct {
|
||||
@ -36,10 +30,6 @@ type QueryNode struct {
|
||||
searchService *searchService
|
||||
loadIndexService *loadIndexService
|
||||
statsService *statsService
|
||||
|
||||
//opentracing
|
||||
tracer opentracing.Tracer
|
||||
closer io.Closer
|
||||
}
|
||||
|
||||
func Init() {
|
||||
@ -49,47 +39,31 @@ func Init() {
|
||||
func NewQueryNode(ctx context.Context, queryNodeID uint64) *QueryNode {
|
||||
|
||||
ctx1, cancel := context.WithCancel(ctx)
|
||||
q := &QueryNode{
|
||||
queryNodeLoopCtx: ctx1,
|
||||
queryNodeLoopCancel: cancel,
|
||||
QueryNodeID: queryNodeID,
|
||||
|
||||
dataSyncService: nil,
|
||||
metaService: nil,
|
||||
searchService: nil,
|
||||
statsService: nil,
|
||||
}
|
||||
|
||||
var err error
|
||||
cfg := &config.Configuration{
|
||||
ServiceName: "query_node",
|
||||
Sampler: &config.SamplerConfig{
|
||||
Type: "const",
|
||||
Param: 1,
|
||||
},
|
||||
Reporter: &config.ReporterConfig{
|
||||
LogSpans: true,
|
||||
},
|
||||
}
|
||||
q.tracer, q.closer, err = cfg.NewTracer(config.Logger(jaeger.StdLogger))
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("ERROR: cannot init Jaeger: %v\n", err))
|
||||
}
|
||||
opentracing.SetGlobalTracer(q.tracer)
|
||||
|
||||
segmentsMap := make(map[int64]*Segment)
|
||||
collections := make([]*Collection, 0)
|
||||
|
||||
tSafe := newTSafe()
|
||||
|
||||
q.replica = &collectionReplicaImpl{
|
||||
var replica collectionReplica = &collectionReplicaImpl{
|
||||
collections: collections,
|
||||
segments: segmentsMap,
|
||||
|
||||
tSafe: tSafe,
|
||||
}
|
||||
|
||||
return q
|
||||
return &QueryNode{
|
||||
queryNodeLoopCtx: ctx1,
|
||||
queryNodeLoopCancel: cancel,
|
||||
QueryNodeID: queryNodeID,
|
||||
|
||||
replica: replica,
|
||||
|
||||
dataSyncService: nil,
|
||||
metaService: nil,
|
||||
searchService: nil,
|
||||
statsService: nil,
|
||||
}
|
||||
}
|
||||
|
||||
func (node *QueryNode) Start() error {
|
||||
@ -123,11 +97,10 @@ func (node *QueryNode) Close() {
|
||||
if node.searchService != nil {
|
||||
node.searchService.close()
|
||||
}
|
||||
if node.loadIndexService != nil {
|
||||
node.loadIndexService.close()
|
||||
}
|
||||
if node.statsService != nil {
|
||||
node.statsService.close()
|
||||
}
|
||||
if node.closer != nil {
|
||||
node.closer.Close()
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -35,7 +35,7 @@ func genTestCollectionMeta(collectionName string, collectionID UniqueID, isBinar
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: "dim",
|
||||
Value: "16",
|
||||
Value: "128",
|
||||
},
|
||||
},
|
||||
IndexParams: []*commonpb.KeyValuePair{
|
||||
@ -92,8 +92,12 @@ func genTestCollectionMeta(collectionName string, collectionID UniqueID, isBinar
|
||||
return &collectionMeta
|
||||
}
|
||||
|
||||
func initTestMeta(t *testing.T, node *QueryNode, collectionName string, collectionID UniqueID, segmentID UniqueID) {
|
||||
collectionMeta := genTestCollectionMeta(collectionName, collectionID, false)
|
||||
func initTestMeta(t *testing.T, node *QueryNode, collectionName string, collectionID UniqueID, segmentID UniqueID, optional ...bool) {
|
||||
isBinary := false
|
||||
if len(optional) > 0 {
|
||||
isBinary = optional[0]
|
||||
}
|
||||
collectionMeta := genTestCollectionMeta(collectionName, collectionID, isBinary)
|
||||
|
||||
schemaBlob := proto.MarshalTextString(collectionMeta.Schema)
|
||||
assert.NotEqual(t, "", schemaBlob)
|
||||
|
@ -5,8 +5,6 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/opentracing/opentracing-go"
|
||||
oplog "github.com/opentracing/opentracing-go/log"
|
||||
"log"
|
||||
"sync"
|
||||
|
||||
@ -135,27 +133,22 @@ func (ss *searchService) receiveSearchMsg() {
|
||||
}
|
||||
searchMsg := make([]msgstream.TsMsg, 0)
|
||||
serverTime := ss.getServiceableTime()
|
||||
for i, msg := range msgPack.Msgs {
|
||||
if msg.BeginTs() > serverTime {
|
||||
ss.msgBuffer <- msg
|
||||
for i := range msgPack.Msgs {
|
||||
if msgPack.Msgs[i].BeginTs() > serverTime {
|
||||
ss.msgBuffer <- msgPack.Msgs[i]
|
||||
continue
|
||||
}
|
||||
searchMsg = append(searchMsg, msgPack.Msgs[i])
|
||||
}
|
||||
for _, msg := range searchMsg {
|
||||
span, ctx := opentracing.StartSpanFromContext(msg.GetMsgContext(), "receive search msg")
|
||||
msg.SetMsgContext(ctx)
|
||||
err := ss.search(msg)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
span.LogFields(oplog.Error(err))
|
||||
err2 := ss.publishFailedSearchResult(msg, err.Error())
|
||||
if err2 != nil {
|
||||
span.LogFields(oplog.Error(err2))
|
||||
log.Println("publish FailedSearchResult failed, error message: ", err2)
|
||||
}
|
||||
}
|
||||
span.Finish()
|
||||
}
|
||||
log.Println("ReceiveSearchMsg, do search done, num of searchMsg = ", len(searchMsg))
|
||||
}
|
||||
@ -217,12 +210,8 @@ func (ss *searchService) doUnsolvedMsgSearch() {
|
||||
// TODO:: cache map[dsl]plan
|
||||
// TODO: reBatched search requests
|
||||
func (ss *searchService) search(msg msgstream.TsMsg) error {
|
||||
span, ctx := opentracing.StartSpanFromContext(msg.GetMsgContext(), "do search")
|
||||
defer span.Finish()
|
||||
msg.SetMsgContext(ctx)
|
||||
searchMsg, ok := msg.(*msgstream.SearchMsg)
|
||||
if !ok {
|
||||
span.LogFields(oplog.Error(errors.New("invalid request type = " + string(msg.Type()))))
|
||||
return errors.New("invalid request type = " + string(msg.Type()))
|
||||
}
|
||||
|
||||
@ -231,27 +220,23 @@ func (ss *searchService) search(msg msgstream.TsMsg) error {
|
||||
query := servicepb.Query{}
|
||||
err := proto.Unmarshal(queryBlob, &query)
|
||||
if err != nil {
|
||||
span.LogFields(oplog.Error(err))
|
||||
return errors.New("unmarshal query failed")
|
||||
}
|
||||
collectionName := query.CollectionName
|
||||
partitionTags := query.PartitionTags
|
||||
collection, err := ss.replica.getCollectionByName(collectionName)
|
||||
if err != nil {
|
||||
span.LogFields(oplog.Error(err))
|
||||
return err
|
||||
}
|
||||
collectionID := collection.ID()
|
||||
dsl := query.Dsl
|
||||
plan, err := createPlan(*collection, dsl)
|
||||
if err != nil {
|
||||
span.LogFields(oplog.Error(err))
|
||||
return err
|
||||
}
|
||||
placeHolderGroupBlob := query.PlaceholderGroup
|
||||
placeholderGroup, err := parserPlaceholderGroup(plan, placeHolderGroupBlob)
|
||||
if err != nil {
|
||||
span.LogFields(oplog.Error(err))
|
||||
return err
|
||||
}
|
||||
placeholderGroups := make([]*PlaceholderGroup, 0)
|
||||
@ -261,22 +246,16 @@ func (ss *searchService) search(msg msgstream.TsMsg) error {
|
||||
matchedSegments := make([]*Segment, 0)
|
||||
|
||||
for _, partitionTag := range partitionTags {
|
||||
hasPartition := ss.replica.hasPartition(collectionID, partitionTag)
|
||||
if !hasPartition {
|
||||
span.LogFields(oplog.Error(errors.New("search Failed, invalid partitionTag")))
|
||||
return errors.New("search Failed, invalid partitionTag")
|
||||
partition, err := ss.replica.getPartitionByTag(collectionID, partitionTag)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
for _, partitionTag := range partitionTags {
|
||||
partition, _ := ss.replica.getPartitionByTag(collectionID, partitionTag)
|
||||
for _, segment := range partition.segments {
|
||||
//fmt.Println("dsl = ", dsl)
|
||||
|
||||
searchResult, err := segment.segmentSearch(plan, placeholderGroups, []Timestamp{searchTimestamp})
|
||||
|
||||
if err != nil {
|
||||
span.LogFields(oplog.Error(err))
|
||||
return err
|
||||
}
|
||||
searchResults = append(searchResults, searchResult)
|
||||
@ -296,18 +275,13 @@ func (ss *searchService) search(msg msgstream.TsMsg) error {
|
||||
Hits: nil,
|
||||
}
|
||||
searchResultMsg := &msgstream.SearchResultMsg{
|
||||
BaseMsg: msgstream.BaseMsg{
|
||||
MsgCtx: searchMsg.MsgCtx,
|
||||
HashValues: []uint32{uint32(searchMsg.ResultChannelID)},
|
||||
},
|
||||
BaseMsg: msgstream.BaseMsg{HashValues: []uint32{uint32(searchMsg.ResultChannelID)}},
|
||||
SearchResult: results,
|
||||
}
|
||||
err = ss.publishSearchResult(searchResultMsg)
|
||||
if err != nil {
|
||||
span.LogFields(oplog.Error(err))
|
||||
return err
|
||||
}
|
||||
span.LogFields(oplog.String("publish search research success", "publish search research success"))
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -315,22 +289,18 @@ func (ss *searchService) search(msg msgstream.TsMsg) error {
|
||||
numSegment := int64(len(searchResults))
|
||||
err2 := reduceSearchResults(searchResults, numSegment, inReduced)
|
||||
if err2 != nil {
|
||||
span.LogFields(oplog.Error(err2))
|
||||
return err2
|
||||
}
|
||||
err = fillTargetEntry(plan, searchResults, matchedSegments, inReduced)
|
||||
if err != nil {
|
||||
span.LogFields(oplog.Error(err))
|
||||
return err
|
||||
}
|
||||
marshaledHits, err := reorganizeQueryResults(plan, placeholderGroups, searchResults, numSegment, inReduced)
|
||||
if err != nil {
|
||||
span.LogFields(oplog.Error(err))
|
||||
return err
|
||||
}
|
||||
hitsBlob, err := marshaledHits.getHitsBlob()
|
||||
if err != nil {
|
||||
span.LogFields(oplog.Error(err))
|
||||
return err
|
||||
}
|
||||
|
||||
@ -365,14 +335,11 @@ func (ss *searchService) search(msg msgstream.TsMsg) error {
|
||||
MetricType: plan.getMetricType(),
|
||||
}
|
||||
searchResultMsg := &msgstream.SearchResultMsg{
|
||||
BaseMsg: msgstream.BaseMsg{
|
||||
MsgCtx: searchMsg.MsgCtx,
|
||||
HashValues: []uint32{uint32(searchMsg.ResultChannelID)}},
|
||||
BaseMsg: msgstream.BaseMsg{HashValues: []uint32{uint32(searchMsg.ResultChannelID)}},
|
||||
SearchResult: results,
|
||||
}
|
||||
err = ss.publishSearchResult(searchResultMsg)
|
||||
if err != nil {
|
||||
span.LogFields(oplog.Error(err))
|
||||
return err
|
||||
}
|
||||
}
|
||||
@ -385,9 +352,6 @@ func (ss *searchService) search(msg msgstream.TsMsg) error {
|
||||
}
|
||||
|
||||
func (ss *searchService) publishSearchResult(msg msgstream.TsMsg) error {
|
||||
span, ctx := opentracing.StartSpanFromContext(msg.GetMsgContext(), "publish search result")
|
||||
defer span.Finish()
|
||||
msg.SetMsgContext(ctx)
|
||||
fmt.Println("Public SearchResult", msg.HashKeys())
|
||||
msgPack := msgstream.MsgPack{}
|
||||
msgPack.Msgs = append(msgPack.Msgs, msg)
|
||||
@ -396,9 +360,6 @@ func (ss *searchService) publishSearchResult(msg msgstream.TsMsg) error {
|
||||
}
|
||||
|
||||
func (ss *searchService) publishFailedSearchResult(msg msgstream.TsMsg, errMsg string) error {
|
||||
span, ctx := opentracing.StartSpanFromContext(msg.GetMsgContext(), "receive search msg")
|
||||
defer span.Finish()
|
||||
msg.SetMsgContext(ctx)
|
||||
msgPack := msgstream.MsgPack{}
|
||||
searchMsg, ok := msg.(*msgstream.SearchMsg)
|
||||
if !ok {
|
||||
|
@ -1,12 +1,8 @@
|
||||
package flowgraph
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/opentracing/opentracing-go"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/msgstream"
|
||||
)
|
||||
|
||||
@ -29,33 +25,11 @@ func (inNode *InputNode) InStream() *msgstream.MsgStream {
|
||||
}
|
||||
|
||||
// empty input and return one *Msg
|
||||
func (inNode *InputNode) Operate([]*Msg) []*Msg {
|
||||
func (inNode *InputNode) Operate(in []*Msg) []*Msg {
|
||||
//fmt.Println("Do InputNode operation")
|
||||
|
||||
msgPack := (*inNode.inStream).Consume()
|
||||
|
||||
var childs []opentracing.Span
|
||||
tracer := opentracing.GlobalTracer()
|
||||
if tracer != nil && msgPack != nil {
|
||||
for _, msg := range msgPack.Msgs {
|
||||
if msg.Type() == internalpb.MsgType_kInsert {
|
||||
var child opentracing.Span
|
||||
ctx := msg.GetMsgContext()
|
||||
if parent := opentracing.SpanFromContext(ctx); parent != nil {
|
||||
child = tracer.StartSpan(fmt.Sprintf("through msg input node, start time = %d", msg.BeginTs()),
|
||||
opentracing.FollowsFrom(parent.Context()))
|
||||
} else {
|
||||
child = tracer.StartSpan(fmt.Sprintf("through msg input node, start time = %d", msg.BeginTs()))
|
||||
}
|
||||
child.SetTag("hash keys", msg.HashKeys())
|
||||
child.SetTag("start time", msg.BeginTs())
|
||||
child.SetTag("end time", msg.EndTs())
|
||||
msg.SetMsgContext(opentracing.ContextWithSpan(ctx, child))
|
||||
childs = append(childs, child)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: add status
|
||||
if msgPack == nil {
|
||||
log.Println("null msg pack")
|
||||
@ -68,10 +42,6 @@ func (inNode *InputNode) Operate([]*Msg) []*Msg {
|
||||
timestampMax: msgPack.EndTs,
|
||||
}
|
||||
|
||||
for _, child := range childs {
|
||||
child.Finish()
|
||||
}
|
||||
|
||||
return []*Msg{&msgStreamMsg}
|
||||
}
|
||||
|
||||
|
@ -103,8 +103,6 @@ func (ddNode *ddNode) Operate(in []*Msg) []*Msg {
|
||||
return tsMessages[i].BeginTs() < tsMessages[j].BeginTs()
|
||||
})
|
||||
|
||||
var flush bool = false
|
||||
var flushSegID UniqueID
|
||||
// do dd tasks
|
||||
for _, msg := range tsMessages {
|
||||
switch msg.Type() {
|
||||
@ -118,100 +116,102 @@ func (ddNode *ddNode) Operate(in []*Msg) []*Msg {
|
||||
ddNode.dropPartition(msg.(*msgstream.DropPartitionMsg))
|
||||
case internalPb.MsgType_kFlush:
|
||||
fMsg := msg.(*msgstream.FlushMsg)
|
||||
flush = true
|
||||
flushSegID = fMsg.SegmentID
|
||||
flushSegID := fMsg.SegmentID
|
||||
ddMsg.flushMessages = append(ddMsg.flushMessages, fMsg)
|
||||
ddNode.flush()
|
||||
|
||||
log.Println(".. manual flush completed ...")
|
||||
ddlFlushMsg := &ddlFlushSyncMsg{
|
||||
flushCompleted: true,
|
||||
ddlBinlogPathMsg: ddlBinlogPathMsg{
|
||||
segID: flushSegID,
|
||||
},
|
||||
}
|
||||
|
||||
ddNode.outCh <- ddlFlushMsg
|
||||
|
||||
default:
|
||||
log.Println("Non supporting message type:", msg.Type())
|
||||
}
|
||||
}
|
||||
|
||||
// generate binlog
|
||||
if ddNode.ddBuffer.full() || flush {
|
||||
log.Println(". dd buffer full or receive Flush msg ...")
|
||||
ddCodec := &storage.DataDefinitionCodec{}
|
||||
for collectionID, data := range ddNode.ddBuffer.ddData {
|
||||
// buffer data to binlog
|
||||
binLogs, err := ddCodec.Serialize(data.timestamps, data.ddRequestString, data.eventTypes)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
continue
|
||||
}
|
||||
if len(binLogs) != 2 {
|
||||
log.Println("illegal binLogs")
|
||||
continue
|
||||
}
|
||||
|
||||
// binLogs -> minIO/S3
|
||||
if len(data.ddRequestString) != len(data.timestamps) ||
|
||||
len(data.timestamps) != len(data.eventTypes) {
|
||||
log.Println("illegal ddBuffer, failed to save binlog")
|
||||
continue
|
||||
} else {
|
||||
log.Println(".. dd buffer flushing ...")
|
||||
// Blob key example:
|
||||
// ${tenant}/data_definition_log/${collection_id}/ts/${log_idx}
|
||||
// ${tenant}/data_definition_log/${collection_id}/ddl/${log_idx}
|
||||
keyCommon := path.Join(Params.DdLogRootPath, strconv.FormatInt(collectionID, 10))
|
||||
|
||||
// save ts binlog
|
||||
timestampLogIdx, err := ddNode.idAllocator.AllocOne()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
timestampKey := path.Join(keyCommon, binLogs[0].GetKey(), strconv.FormatInt(timestampLogIdx, 10))
|
||||
err = ddNode.kv.Save(timestampKey, string(binLogs[0].GetValue()))
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
log.Println("save ts binlog, key = ", timestampKey)
|
||||
|
||||
// save dd binlog
|
||||
ddLogIdx, err := ddNode.idAllocator.AllocOne()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
ddKey := path.Join(keyCommon, binLogs[1].GetKey(), strconv.FormatInt(ddLogIdx, 10))
|
||||
err = ddNode.kv.Save(ddKey, string(binLogs[1].GetValue()))
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
log.Println("save dd binlog, key = ", ddKey)
|
||||
|
||||
ddlFlushMsg := &ddlFlushSyncMsg{
|
||||
flushCompleted: false,
|
||||
ddlBinlogPathMsg: ddlBinlogPathMsg{
|
||||
collID: collectionID,
|
||||
paths: []string{timestampKey, ddKey},
|
||||
},
|
||||
}
|
||||
|
||||
ddNode.outCh <- ddlFlushMsg
|
||||
}
|
||||
|
||||
}
|
||||
// clear buffer
|
||||
ddNode.ddBuffer.ddData = make(map[UniqueID]*ddData)
|
||||
}
|
||||
|
||||
if flush {
|
||||
|
||||
log.Println(".. manual flush completed ...")
|
||||
ddlFlushMsg := &ddlFlushSyncMsg{
|
||||
flushCompleted: true,
|
||||
ddlBinlogPathMsg: ddlBinlogPathMsg{
|
||||
segID: flushSegID,
|
||||
},
|
||||
}
|
||||
|
||||
ddNode.outCh <- ddlFlushMsg
|
||||
|
||||
if ddNode.ddBuffer.full() {
|
||||
ddNode.flush()
|
||||
}
|
||||
|
||||
var res Msg = ddNode.ddMsg
|
||||
return []*Msg{&res}
|
||||
}
|
||||
|
||||
func (ddNode *ddNode) flush() {
|
||||
// generate binlog
|
||||
log.Println(". dd buffer full or receive Flush msg ...")
|
||||
ddCodec := &storage.DataDefinitionCodec{}
|
||||
for collectionID, data := range ddNode.ddBuffer.ddData {
|
||||
// buffer data to binlog
|
||||
binLogs, err := ddCodec.Serialize(data.timestamps, data.ddRequestString, data.eventTypes)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
continue
|
||||
}
|
||||
if len(binLogs) != 2 {
|
||||
log.Println("illegal binLogs")
|
||||
continue
|
||||
}
|
||||
|
||||
// binLogs -> minIO/S3
|
||||
if len(data.ddRequestString) != len(data.timestamps) ||
|
||||
len(data.timestamps) != len(data.eventTypes) {
|
||||
log.Println("illegal ddBuffer, failed to save binlog")
|
||||
continue
|
||||
} else {
|
||||
log.Println(".. dd buffer flushing ...")
|
||||
// Blob key example:
|
||||
// ${tenant}/data_definition_log/${collection_id}/ts/${log_idx}
|
||||
// ${tenant}/data_definition_log/${collection_id}/ddl/${log_idx}
|
||||
keyCommon := path.Join(Params.DdLogRootPath, strconv.FormatInt(collectionID, 10))
|
||||
|
||||
// save ts binlog
|
||||
timestampLogIdx, err := ddNode.idAllocator.AllocOne()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
timestampKey := path.Join(keyCommon, binLogs[0].GetKey(), strconv.FormatInt(timestampLogIdx, 10))
|
||||
err = ddNode.kv.Save(timestampKey, string(binLogs[0].GetValue()))
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
log.Println("save ts binlog, key = ", timestampKey)
|
||||
|
||||
// save dd binlog
|
||||
ddLogIdx, err := ddNode.idAllocator.AllocOne()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
ddKey := path.Join(keyCommon, binLogs[1].GetKey(), strconv.FormatInt(ddLogIdx, 10))
|
||||
err = ddNode.kv.Save(ddKey, string(binLogs[1].GetValue()))
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
log.Println("save dd binlog, key = ", ddKey)
|
||||
|
||||
ddlFlushMsg := &ddlFlushSyncMsg{
|
||||
flushCompleted: false,
|
||||
ddlBinlogPathMsg: ddlBinlogPathMsg{
|
||||
collID: collectionID,
|
||||
paths: []string{timestampKey, ddKey},
|
||||
},
|
||||
}
|
||||
|
||||
ddNode.outCh <- ddlFlushMsg
|
||||
}
|
||||
|
||||
}
|
||||
// clear buffer
|
||||
ddNode.ddBuffer.ddData = make(map[UniqueID]*ddData)
|
||||
}
|
||||
|
||||
func (ddNode *ddNode) createCollection(msg *msgstream.CreateCollectionMsg) {
|
||||
collectionID := msg.CollectionID
|
||||
|
||||
|
@ -1,12 +1,9 @@
|
||||
package writenode
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"math"
|
||||
|
||||
"github.com/opentracing/opentracing-go"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/msgstream"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
|
||||
internalPb "github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
|
||||
@ -35,34 +32,11 @@ func (fdmNode *filterDmNode) Operate(in []*Msg) []*Msg {
|
||||
// TODO: add error handling
|
||||
}
|
||||
|
||||
var childs []opentracing.Span
|
||||
tracer := opentracing.GlobalTracer()
|
||||
if tracer != nil {
|
||||
for _, msg := range msgStreamMsg.TsMessages() {
|
||||
if msg.Type() == internalPb.MsgType_kInsert {
|
||||
var child opentracing.Span
|
||||
ctx := msg.GetMsgContext()
|
||||
if parent := opentracing.SpanFromContext(ctx); parent != nil {
|
||||
child = tracer.StartSpan("pass filter node",
|
||||
opentracing.FollowsFrom(parent.Context()))
|
||||
} else {
|
||||
child = tracer.StartSpan("pass filter node")
|
||||
}
|
||||
child.SetTag("hash keys", msg.HashKeys())
|
||||
child.SetTag("start time", msg.BeginTs())
|
||||
child.SetTag("end time", msg.EndTs())
|
||||
msg.SetMsgContext(opentracing.ContextWithSpan(ctx, child))
|
||||
childs = append(childs, child)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ddMsg, ok := (*in[1]).(*ddMsg)
|
||||
if !ok {
|
||||
log.Println("type assertion failed for ddMsg")
|
||||
// TODO: add error handling
|
||||
}
|
||||
|
||||
fdmNode.ddMsg = ddMsg
|
||||
|
||||
var iMsg = insertMsg{
|
||||
@ -83,20 +57,11 @@ func (fdmNode *filterDmNode) Operate(in []*Msg) []*Msg {
|
||||
}
|
||||
}
|
||||
|
||||
for key, msg := range msgStreamMsg.TsMessages() {
|
||||
for _, msg := range msgStreamMsg.TsMessages() {
|
||||
switch msg.Type() {
|
||||
case internalPb.MsgType_kInsert:
|
||||
var ctx2 context.Context
|
||||
if childs != nil {
|
||||
if childs[key] != nil {
|
||||
ctx2 = opentracing.ContextWithSpan(msg.GetMsgContext(), childs[key])
|
||||
} else {
|
||||
ctx2 = context.Background()
|
||||
}
|
||||
}
|
||||
resMsg := fdmNode.filterInvalidInsertMessage(msg.(*msgstream.InsertMsg))
|
||||
if resMsg != nil {
|
||||
resMsg.SetMsgContext(ctx2)
|
||||
iMsg.insertMessages = append(iMsg.insertMessages, resMsg)
|
||||
}
|
||||
// case internalPb.MsgType_kDelete:
|
||||
@ -108,9 +73,6 @@ func (fdmNode *filterDmNode) Operate(in []*Msg) []*Msg {
|
||||
|
||||
iMsg.gcRecord = ddMsg.gcRecord
|
||||
var res Msg = &iMsg
|
||||
for _, child := range childs {
|
||||
child.Finish()
|
||||
}
|
||||
return []*Msg{&res}
|
||||
}
|
||||
|
||||
|
@ -4,15 +4,11 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"log"
|
||||
"path"
|
||||
"strconv"
|
||||
"unsafe"
|
||||
|
||||
"github.com/opentracing/opentracing-go"
|
||||
oplog "github.com/opentracing/opentracing-go/log"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/allocator"
|
||||
"github.com/zilliztech/milvus-distributed/internal/kv"
|
||||
miniokv "github.com/zilliztech/milvus-distributed/internal/kv/minio"
|
||||
@ -100,23 +96,12 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
|
||||
// iMsg is insertMsg
|
||||
// 1. iMsg -> buffer
|
||||
for _, msg := range iMsg.insertMessages {
|
||||
ctx := msg.GetMsgContext()
|
||||
var span opentracing.Span
|
||||
if ctx != nil {
|
||||
span, _ = opentracing.StartSpanFromContext(ctx, fmt.Sprintf("insert buffer node, start time = %d", msg.BeginTs()))
|
||||
} else {
|
||||
span = opentracing.StartSpan(fmt.Sprintf("insert buffer node, start time = %d", msg.BeginTs()))
|
||||
}
|
||||
span.SetTag("hash keys", msg.HashKeys())
|
||||
span.SetTag("start time", msg.BeginTs())
|
||||
span.SetTag("end time", msg.EndTs())
|
||||
if len(msg.RowIDs) != len(msg.Timestamps) || len(msg.RowIDs) != len(msg.RowData) {
|
||||
log.Println("Error: misaligned messages detected")
|
||||
continue
|
||||
}
|
||||
currentSegID := msg.GetSegmentID()
|
||||
collectionName := msg.GetCollectionName()
|
||||
span.LogFields(oplog.Int("segment id", int(currentSegID)))
|
||||
|
||||
idata, ok := ibNode.insertBuffer.insertData[currentSegID]
|
||||
if !ok {
|
||||
@ -125,21 +110,6 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
|
||||
}
|
||||
}
|
||||
|
||||
// Timestamps
|
||||
_, ok = idata.Data[1].(*storage.Int64FieldData)
|
||||
if !ok {
|
||||
idata.Data[1] = &storage.Int64FieldData{
|
||||
Data: []int64{},
|
||||
NumRows: 0,
|
||||
}
|
||||
}
|
||||
tsData := idata.Data[1].(*storage.Int64FieldData)
|
||||
for _, ts := range msg.Timestamps {
|
||||
tsData.Data = append(tsData.Data, int64(ts))
|
||||
}
|
||||
tsData.NumRows += len(msg.Timestamps)
|
||||
span.LogFields(oplog.Int("tsData numRows", tsData.NumRows))
|
||||
|
||||
// 1.1 Get CollectionMeta from etcd
|
||||
collection, err := ibNode.replica.getCollectionByName(collectionName)
|
||||
//collSchema, err := ibNode.getCollectionSchemaByName(collectionName)
|
||||
@ -388,11 +358,9 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
|
||||
|
||||
// 1.3 store in buffer
|
||||
ibNode.insertBuffer.insertData[currentSegID] = idata
|
||||
span.LogFields(oplog.String("store in buffer", "store in buffer"))
|
||||
|
||||
// 1.4 if full
|
||||
// 1.4.1 generate binlogs
|
||||
span.LogFields(oplog.String("generate binlogs", "generate binlogs"))
|
||||
if ibNode.insertBuffer.full(currentSegID) {
|
||||
log.Printf(". Insert Buffer full, auto flushing (%v) rows of data...", ibNode.insertBuffer.size(currentSegID))
|
||||
// partitionTag -> partitionID
|
||||
@ -461,7 +429,6 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
|
||||
ibNode.outCh <- inBinlogMsg
|
||||
}
|
||||
}
|
||||
span.Finish()
|
||||
}
|
||||
|
||||
if len(iMsg.insertMessages) > 0 {
|
||||
|
@ -112,6 +112,7 @@ func (fService *flushSyncService) start() {
|
||||
fService.completeInsertFlush(insertFlushMsg.segID)
|
||||
|
||||
if fService.FlushCompleted(insertFlushMsg.segID) {
|
||||
log.Printf("Seg(%d) flush completed.", insertFlushMsg.segID)
|
||||
fService.metaTable.CompleteFlush(insertFlushMsg.ts, insertFlushMsg.segID)
|
||||
}
|
||||
}
|
||||
|
@ -2,12 +2,6 @@ package writenode
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/opentracing/opentracing-go"
|
||||
"github.com/uber/jaeger-client-go"
|
||||
"github.com/uber/jaeger-client-go/config"
|
||||
)
|
||||
|
||||
type WriteNode struct {
|
||||
@ -17,8 +11,6 @@ type WriteNode struct {
|
||||
flushSyncService *flushSyncService
|
||||
metaService *metaService
|
||||
replica collectionReplica
|
||||
tracer opentracing.Tracer
|
||||
closer io.Closer
|
||||
}
|
||||
|
||||
func NewWriteNode(ctx context.Context, writeNodeID uint64) *WriteNode {
|
||||
@ -46,22 +38,6 @@ func Init() {
|
||||
}
|
||||
|
||||
func (node *WriteNode) Start() error {
|
||||
cfg := &config.Configuration{
|
||||
ServiceName: "write_node",
|
||||
Sampler: &config.SamplerConfig{
|
||||
Type: "const",
|
||||
Param: 1,
|
||||
},
|
||||
Reporter: &config.ReporterConfig{
|
||||
LogSpans: true,
|
||||
},
|
||||
}
|
||||
var err error
|
||||
node.tracer, node.closer, err = cfg.NewTracer(config.Logger(jaeger.StdLogger))
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("ERROR: cannot init Jaeger: %v\n", err))
|
||||
}
|
||||
opentracing.SetGlobalTracer(node.tracer)
|
||||
|
||||
// TODO GOOSE Init Size??
|
||||
chanSize := 100
|
||||
@ -85,9 +61,4 @@ func (node *WriteNode) Close() {
|
||||
if node.dataSyncService != nil {
|
||||
(*node.dataSyncService).close()
|
||||
}
|
||||
|
||||
if node.closer != nil {
|
||||
node.closer.Close()
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -8,6 +8,15 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
|
||||
done
|
||||
ROOT_DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
|
||||
|
||||
unameOut="$(uname -s)"
|
||||
case "${unameOut}" in
|
||||
Linux*) machine=Linux;;
|
||||
Darwin*) machine=Mac;;
|
||||
CYGWIN*) machine=Cygwin;;
|
||||
MINGW*) machine=MinGw;;
|
||||
*) machine="UNKNOWN:${unameOut}"
|
||||
esac
|
||||
|
||||
# Attempt to run in the container with the same UID/GID as we have on the host,
|
||||
# as this results in the correct permissions on files created in the shared
|
||||
# volumes. This isn't always possible, however, as IDs less than 100 are
|
||||
@ -21,8 +30,14 @@ gid=$(id -g)
|
||||
[ "$uid" -lt 500 ] && uid=501
|
||||
[ "$gid" -lt 500 ] && gid=$uid
|
||||
|
||||
awk 'c&&c--{sub(/^/,"#")} /# Build devcontainer/{c=5} 1' $ROOT_DIR/docker-compose.yml > $ROOT_DIR/docker-compose-vscode.yml.bak
|
||||
awk 'c&&c--{sub(/^/,"#")} /# Build devcontainer/{c=5} 1' $ROOT_DIR/docker-compose.yml > $ROOT_DIR/docker-compose-vscode.yml.tmp
|
||||
|
||||
awk 'c&&c--{sub(/^/,"#")} /# Command/{c=3} 1' $ROOT_DIR/docker-compose-vscode.yml.bak > $ROOT_DIR/docker-compose-vscode.yml
|
||||
awk 'c&&c--{sub(/^/,"#")} /# Command/{c=3} 1' $ROOT_DIR/docker-compose-vscode.yml.tmp > $ROOT_DIR/docker-compose-vscode.yml
|
||||
|
||||
sed -i '.bak' "s/# user: {{ CURRENT_ID }}/user: \"$uid:$gid\"/g" $ROOT_DIR/docker-compose-vscode.yml
|
||||
rm $ROOT_DIR/docker-compose-vscode.yml.tmp
|
||||
|
||||
if [ "${machine}" == "Mac" ];then
|
||||
sed -i '' "s/# user: {{ CURRENT_ID }}/user: \"$uid:$gid\"/g" $ROOT_DIR/docker-compose-vscode.yml
|
||||
else
|
||||
sed -i "s/# user: {{ CURRENT_ID }}/user: \"$uid:$gid\"/g" $ROOT_DIR/docker-compose-vscode.yml
|
||||
fi
|
@ -4,5 +4,5 @@ numpy==1.18.1
|
||||
pytest==5.3.4
|
||||
pytest-cov==2.8.1
|
||||
pytest-timeout==1.3.4
|
||||
pymilvus-distributed==0.0.10
|
||||
pymilvus-distributed==0.0.14
|
||||
sklearn==0.0
|
||||
|
@ -101,7 +101,6 @@ class TestInsertBase:
|
||||
connect.flush([collection])
|
||||
connect.drop_collection(collection)
|
||||
|
||||
@pytest.mark.skip("create index")
|
||||
@pytest.mark.timeout(ADD_TIMEOUT)
|
||||
def test_insert_create_index(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
@ -119,7 +118,6 @@ class TestInsertBase:
|
||||
if field["name"] == field_name:
|
||||
assert field["indexes"][0] == get_simple_index
|
||||
|
||||
@pytest.mark.skip("create index")
|
||||
@pytest.mark.timeout(ADD_TIMEOUT)
|
||||
def test_insert_create_index_new(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
@ -137,7 +135,6 @@ class TestInsertBase:
|
||||
if field["name"] == field_name:
|
||||
assert field["indexes"][0] == get_simple_index
|
||||
|
||||
@pytest.mark.skip("create index")
|
||||
@pytest.mark.timeout(ADD_TIMEOUT)
|
||||
def test_insert_after_create_index(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
@ -154,7 +151,6 @@ class TestInsertBase:
|
||||
if field["name"] == field_name:
|
||||
assert field["indexes"][0] == get_simple_index
|
||||
|
||||
# @pytest.mark.skip(" later ")
|
||||
@pytest.mark.timeout(ADD_TIMEOUT)
|
||||
def test_insert_search(self, connect, collection):
|
||||
'''
|
||||
@ -645,7 +641,6 @@ class TestInsertBinary:
|
||||
connect.flush([binary_collection])
|
||||
assert connect.count_entities(binary_collection) == default_nb
|
||||
|
||||
@pytest.mark.skip("create index")
|
||||
def test_insert_binary_after_create_index(self, connect, binary_collection, get_binary_index):
|
||||
'''
|
||||
target: test insert binary entities after build index
|
||||
@ -662,7 +657,6 @@ class TestInsertBinary:
|
||||
if field["name"] == binary_field_name:
|
||||
assert field["indexes"][0] == get_binary_index
|
||||
|
||||
@pytest.mark.skip("create index")
|
||||
@pytest.mark.timeout(ADD_TIMEOUT)
|
||||
def test_insert_binary_create_index(self, connect, binary_collection, get_binary_index):
|
||||
'''
|
||||
@ -863,7 +857,6 @@ class TestInsertMultiCollections:
|
||||
connect.flush([collection_name])
|
||||
assert len(ids) == 1
|
||||
|
||||
@pytest.mark.skip("create index")
|
||||
@pytest.mark.timeout(ADD_TIMEOUT)
|
||||
def test_create_index_insert_vector_another(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
@ -877,7 +870,7 @@ class TestInsertMultiCollections:
|
||||
ids = connect.bulk_insert(collection, default_entity)
|
||||
connect.drop_collection(collection_name)
|
||||
|
||||
@pytest.mark.skip("create index")
|
||||
@pytest.mark.skip("count entities")
|
||||
@pytest.mark.timeout(ADD_TIMEOUT)
|
||||
def test_insert_vector_create_index_another(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
@ -892,7 +885,7 @@ class TestInsertMultiCollections:
|
||||
count = connect.count_entities(collection_name)
|
||||
assert count == 0
|
||||
|
||||
@pytest.mark.skip("create index")
|
||||
@pytest.mark.skip("count entities")
|
||||
@pytest.mark.timeout(ADD_TIMEOUT)
|
||||
def test_insert_vector_sleep_create_index_another(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
|
@ -101,7 +101,6 @@ class TestInsertBase:
|
||||
connect.flush([collection])
|
||||
connect.drop_collection(collection)
|
||||
|
||||
@pytest.mark.skip("create_index")
|
||||
@pytest.mark.timeout(ADD_TIMEOUT)
|
||||
def test_insert_create_index(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
@ -119,7 +118,6 @@ class TestInsertBase:
|
||||
if field["name"] == field_name:
|
||||
assert field["indexes"][0] == get_simple_index
|
||||
|
||||
@pytest.mark.skip("create_index")
|
||||
@pytest.mark.timeout(ADD_TIMEOUT)
|
||||
def test_insert_after_create_index(self, connect, collection, get_simple_index):
|
||||
'''
|
||||
@ -136,7 +134,6 @@ class TestInsertBase:
|
||||
if field["name"] == field_name:
|
||||
assert field["indexes"][0] == get_simple_index
|
||||
|
||||
@pytest.mark.skip(" todo fix search")
|
||||
@pytest.mark.timeout(ADD_TIMEOUT)
|
||||
def test_insert_search(self, connect, collection):
|
||||
'''
|
||||
@ -313,7 +310,6 @@ class TestInsertBinary:
|
||||
connect.flush([binary_collection])
|
||||
assert connect.count_entities(binary_collection) == default_nb
|
||||
|
||||
@pytest.mark.skip("create index")
|
||||
def test_insert_binary_after_create_index(self, connect, binary_collection, get_binary_index):
|
||||
'''
|
||||
target: test insert binary entities after build index
|
||||
@ -330,7 +326,6 @@ class TestInsertBinary:
|
||||
if field["name"] == binary_field_name:
|
||||
assert field["indexes"][0] == get_binary_index
|
||||
|
||||
@pytest.mark.skip("create index")
|
||||
@pytest.mark.timeout(ADD_TIMEOUT)
|
||||
def test_insert_binary_create_index(self, connect, binary_collection, get_binary_index):
|
||||
'''
|
||||
|
@ -255,6 +255,7 @@ class TestSearchBase:
|
||||
assert res2[0][0].id == res[0][1].id
|
||||
assert res2[0][0].entity.get("int64") == res[0][1].entity.get("int64")
|
||||
|
||||
# pass
|
||||
@pytest.mark.skip("search_after_index")
|
||||
@pytest.mark.level(2)
|
||||
def test_search_after_index(self, connect, collection, get_simple_index, get_top_k, get_nq):
|
||||
@ -333,6 +334,7 @@ class TestSearchBase:
|
||||
res = connect.search(collection, query, partition_tags=[default_tag])
|
||||
assert len(res) == nq
|
||||
|
||||
# pass
|
||||
@pytest.mark.skip("search_index_partition_B")
|
||||
@pytest.mark.level(2)
|
||||
def test_search_index_partition_B(self, connect, collection, get_simple_index, get_top_k, get_nq):
|
||||
@ -451,7 +453,7 @@ class TestSearchBase:
|
||||
assert res[0]._distances[0] < epsilon
|
||||
assert res[1]._distances[0] < epsilon
|
||||
|
||||
#
|
||||
# pass
|
||||
# test for ip metric
|
||||
#
|
||||
# TODO: reopen after we supporting ip flat
|
||||
@ -477,6 +479,7 @@ class TestSearchBase:
|
||||
with pytest.raises(Exception) as e:
|
||||
res = connect.search(collection, query)
|
||||
|
||||
# pass
|
||||
@pytest.mark.skip("search_ip_after_index")
|
||||
@pytest.mark.level(2)
|
||||
def test_search_ip_after_index(self, connect, collection, get_simple_index, get_top_k, get_nq):
|
||||
@ -618,7 +621,7 @@ class TestSearchBase:
|
||||
res = connect.search(collection, query)
|
||||
assert abs(np.sqrt(res[0]._distances[0]) - min(distance_0, distance_1)) <= gen_inaccuracy(res[0]._distances[0])
|
||||
|
||||
@pytest.mark.skip("search_distance_l2_after_index")
|
||||
@pytest.mark.skip("test_search_distance_l2_after_index")
|
||||
def test_search_distance_l2_after_index(self, connect, id_collection, get_simple_index):
|
||||
'''
|
||||
target: search collection, and check the result: distance
|
||||
@ -672,6 +675,7 @@ class TestSearchBase:
|
||||
res = connect.search(collection, query)
|
||||
assert abs(res[0]._distances[0] - max(distance_0, distance_1)) <= epsilon
|
||||
|
||||
# pass
|
||||
@pytest.mark.skip("search_distance_ip_after_index")
|
||||
def test_search_distance_ip_after_index(self, connect, id_collection, get_simple_index):
|
||||
'''
|
||||
@ -1741,7 +1745,7 @@ class TestSearchInvalid(object):
|
||||
def get_search_params(self, request):
|
||||
yield request.param
|
||||
|
||||
# TODO: reopen after we supporting create index
|
||||
# pass
|
||||
@pytest.mark.skip("search_with_invalid_params")
|
||||
@pytest.mark.level(2)
|
||||
def test_search_with_invalid_params(self, connect, collection, get_simple_index, get_search_params):
|
||||
@ -1763,7 +1767,7 @@ class TestSearchInvalid(object):
|
||||
with pytest.raises(Exception) as e:
|
||||
res = connect.search(collection, query)
|
||||
|
||||
# TODO: reopen after we supporting binary type
|
||||
# pass
|
||||
@pytest.mark.skip("search_with_invalid_params_binary")
|
||||
@pytest.mark.level(2)
|
||||
def test_search_with_invalid_params_binary(self, connect, binary_collection):
|
||||
@ -1783,6 +1787,7 @@ class TestSearchInvalid(object):
|
||||
with pytest.raises(Exception) as e:
|
||||
res = connect.search(binary_collection, query)
|
||||
|
||||
# pass
|
||||
@pytest.mark.skip("search_with_empty_params")
|
||||
@pytest.mark.level(2)
|
||||
def test_search_with_empty_params(self, connect, collection, args, get_simple_index):
|
||||
|
@ -55,7 +55,7 @@ default_index_params = [
|
||||
{"nlist": 128},
|
||||
{"nlist": 128},
|
||||
{"nlist": 128},
|
||||
{"nlist": 128, "m": 16},
|
||||
{"nlist": 128, "m": 16, "nbits": 8},
|
||||
{"M": 48, "efConstruction": 500},
|
||||
# {"search_length": 50, "out_degree": 40, "candidate_pool_size": 100, "knng": 50},
|
||||
{"n_trees": 50},
|
||||
@ -281,9 +281,9 @@ def gen_entities(nb, is_normal=False):
|
||||
def gen_entities_new(nb, is_normal=False):
|
||||
vectors = gen_vectors(nb, default_dim, is_normal)
|
||||
entities = [
|
||||
{"name": "int64", "values": [i for i in range(nb)]},
|
||||
{"name": "float", "values": [float(i) for i in range(nb)]},
|
||||
{"name": default_float_vec_field_name, "values": vectors}
|
||||
{"name": "int64", "type": DataType.INT64, "values": [i for i in range(nb)]},
|
||||
{"name": "float", "type": DataType.FLOAT, "values": [float(i) for i in range(nb)]},
|
||||
{"name": default_float_vec_field_name, "type": DataType.FLOAT_VECTOR, "values": vectors}
|
||||
]
|
||||
return entities
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user