Add dockerfile for index

Signed-off-by: cai.zhang <cai.zhang@zilliz.com>

.jenkins/modules/Publish/Publish.groovy

@@ -17,10 +17,6 @@ dir ('build/docker/deploy') {
             sh 'docker-compose build --force-rm master'
             sh 'docker-compose push master'
 
-            sh 'docker pull ${SOURCE_REPO}/indexbuilder:${SOURCE_TAG} || true'
-            sh 'docker-compose build --force-rm indexbuilder'
-            sh 'docker-compose push indexbuilder'
-
             sh 'docker pull ${SOURCE_REPO}/proxyservice:${SOURCE_TAG} || true'
             sh 'docker-compose build --force-rm proxyservice'
             sh 'docker-compose push proxyservice'
@@ -47,3 +43,25 @@ dir ('build/docker/deploy') {
         sh 'docker-compose down --rmi all'
     }
 }
+
+dir ('build/docker/deploy/distributed') {
+	try {
+        withCredentials([usernamePassword(credentialsId: "${env.DOCKER_CREDENTIALS_ID}", usernameVariable: 'DOCKER_USERNAME', passwordVariable: 'DOCKER_PASSWORD')]) {
+            sh 'docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD} ${DOKCER_REGISTRY_URL}'
+
+            sh 'docker pull ${SOURCE_REPO}/indexservice:${SOURCE_TAG} || true'
+            sh 'docker-compose build --force-rm indexservice'
+            sh 'docker-compose push indexservice'
+
+            sh 'docker pull ${SOURCE_REPO}/indexnode:${SOURCE_TAG} || true'
+            sh 'docker-compose build --force-rm indexnode'
+            sh 'docker-compose push indexnode'
+        }
+    } catch (exc) {
+        throw exc
+    } finally {
+        sh 'docker logout ${DOKCER_REGISTRY_URL}'
+        sh "docker rmi -f \$(docker images | grep '<none>' | awk '{print \$3}') || true"
+        sh 'docker-compose down --rmi all'
+    }
+}

.jenkins/modules/Regression/PythonRegression.groovy

@@ -6,7 +6,6 @@ try {
     dir ('build/docker/deploy') {
         sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} pull'
         sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d master'
-        sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d indexbuilder'
         sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d proxyservice'
         sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d proxynode'
         sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} run -e QUERY_NODE_ID=1 -d querynode'
@@ -14,6 +13,12 @@ try {
         sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} run -e WRITE_NODE_ID=3 -d writenode'
     }
 
+    dir ('build/docker/deploy/distributed') {
+        sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} pull'
+        sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d indexservice'
+        sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME} up -d indexnode'
+    }
+
     dir ('build/docker/test') {
         sh 'docker pull ${SOURCE_REPO}/pytest:${SOURCE_TAG} || true'
         sh 'docker-compose build --force-rm regression'

Makefile

@@ -121,8 +121,6 @@ build-go: build-cpp
 	@mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="0" && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/proxynode $(PWD)/cmd/proxy/node/proxy_node.go 1>/dev/null
 	@echo "Building query node ..."
 	@mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/querynode $(PWD)/cmd/querynode/query_node.go 1>/dev/null
-	@echo "Building indexbuilder ..."
-	@mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/indexbuilder $(PWD)/cmd/indexbuilder/indexbuilder.go 1>/dev/null
 	@echo "Building write node ..."
 	@mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && GO111MODULE=on $(GO) build -o $(INSTALL_PATH)/writenode $(PWD)/cmd/writenode/writenode.go 1>/dev/null
 	@echo "Building binlog ..."
@@ -173,7 +171,6 @@ install: all
 	@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/proxynode $(GOPATH)/bin/proxynode
 	@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/proxyservice $(GOPATH)/bin/proxyservice
 	@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/writenode $(GOPATH)/bin/writenode
-	@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/indexbuilder $(GOPATH)/bin/indexbuilder
 	@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/bin/singlenode $(GOPATH)/bin/singlenode
 	@mkdir -p $(LIBRARY_PATH) && cp -f $(PWD)/internal/core/output/lib/* $(LIBRARY_PATH)
 	@echo "Installation successful."
@@ -189,5 +186,4 @@ clean:
 	@rm -rf $(GOPATH)/bin/proxyservice
 	@rm -rf $(GOPATH)/bin/querynode
 	@rm -rf $(GOPATH)/bin/writenode
-	@rm -rf $(GOPATH)/bin/indexbuilder
 	@rm -rf $(GOPATH)/bin/singlenode

build/docker/deploy/.env

@@ -6,6 +6,6 @@ PULSAR_ADDRESS=pulsar://pulsar:6650
 ETCD_ADDRESS=etcd:2379
 MASTER_ADDRESS=master:53100
 MINIO_ADDRESS=minio:9000
-INDEX_BUILDER_ADDRESS=indexbuilder:31000
 PROXY_NODE_HOST=proxynode
 PROXY_SERVICE_ADDRESS=proxyservice:19530
+INDEX_SERVICE_ADDRESS=indexservice:31000

build/docker/deploy/distributed/.env

@@ -0,0 +1,9 @@
+SOURCE_REPO=milvusdb
+TARGET_REPO=milvusdb
+SOURCE_TAG=latest
+TARGET_TAG=latest
+PULSAR_ADDRESS=pulsar://pulsar:6650
+ETCD_ADDRESS=etcd:2379
+MASTER_ADDRESS=master:53100
+MINIO_ADDRESS=minio:9000
+INDEX_SERVICE_ADDRESS=indexservice:31000

build/docker/deploy/distributed/docker-compose.yml

@@ -0,0 +1,36 @@
+version: '3.5'
+
+services:
+  indexservice:
+    image: ${TARGET_REPO}/indexservice:${TARGET_TAG}
+    build:
+      context: ../../../../
+      dockerfile: build/docker/deploy/distributed/indexservice/DockerFile
+      cache_from:
+        - ${SOURCE_REPO}/indexservice:${SOURCE_TAG}
+    environment:
+      MASTER_ADDRESS: ${MASTER_ADDRESS}
+      ETCD_ADDRESS: ${ETCD_ADDRESS}
+      MINIO_ADDRESS: ${MINIO_ADDRESS}
+
+    networks:
+      - milvus
+
+  indexnode:
+    image: ${TARGET_REPO}/indexnode:${TARGET_TAG}
+    build:
+      context: ../../../../
+      dockerfile: build/docker/deploy/distributed/indexnode/DockerFile
+      cache_from:
+        - ${SOURCE_REPO}/indexnode:${SOURCE_TAG}
+    environment:
+      INDEX_SERVICE_ADDRESS: ${INDEX_SERVICE_ADDRESS}
+      MINIO_ADDRESS: ${MINIO_ADDRESS}
+    depends_on:
+      - "indexservice"
+
+    networks:
+      - milvus
+
+networks:
+  milvus:

build/docker/deploy/distributed/indexnode/DockerFile

@@ -0,0 +1,40 @@
+# Copyright (C) 2019-2020 Zilliz. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied. See the License for the specific language governing permissions and limitations under the License.
+
+FROM milvusdb/milvus-distributed-dev:amd64-ubuntu18.04-latest AS openblas
+
+#FROM alpine
+FROM ubuntu:bionic-20200921
+
+RUN apt-get update && apt-get install -y --no-install-recommends libtbb-dev gfortran
+
+#RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories
+
+#RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories \
+#      && apk add --no-cache libtbb gfortran
+
+COPY --from=openblas /usr/lib/libopenblas-r0.3.9.so /usr/lib/
+
+RUN ln -s /usr/lib/libopenblas-r0.3.9.so /usr/lib/libopenblas.so.0 && \
+    ln -s /usr/lib/libopenblas.so.0 /usr/lib/libopenblas.so
+
+COPY ./bin/distributed/indexnode /milvus-distributed/bin/distributed/indexnode
+
+COPY ./configs/ /milvus-distributed/configs/
+
+COPY ./lib/ /milvus-distributed/lib/
+
+ENV LD_LIBRARY_PATH=/milvus-distributed/lib:$LD_LIBRARY_PATH:/usr/lib
+
+WORKDIR /milvus-distributed/
+
+CMD ["./bin/distributed/indexnode"]
+

build/docker/deploy/distributed/indexservice/DockerFile

@@ -0,0 +1,41 @@
+# Copyright (C) 2019-2020 Zilliz. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied. See the License for the specific language governing permissions and limitations under the License.
+
+FROM milvusdb/milvus-distributed-dev:amd64-ubuntu18.04-latest AS openblas
+
+#FROM alpine
+FROM ubuntu:bionic-20200921
+
+RUN apt-get update && apt-get install -y --no-install-recommends libtbb-dev gfortran
+
+#RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories
+
+#RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories \
+#      && apk add --no-cache libtbb gfortran
+
+COPY --from=openblas /usr/lib/libopenblas-r0.3.9.so /usr/lib/
+
+RUN ln -s /usr/lib/libopenblas-r0.3.9.so /usr/lib/libopenblas.so.0 && \
+    ln -s /usr/lib/libopenblas.so.0 /usr/lib/libopenblas.so
+
+COPY ./bin/distributed/indexservice /milvus-distributed/bin/distributed/indexservice
+
+COPY ./configs/ /milvus-distributed/configs/
+
+COPY ./lib/ /milvus-distributed/lib/
+
+ENV LD_LIBRARY_PATH=/milvus-distributed/lib:$LD_LIBRARY_PATH:/usr/lib
+
+WORKDIR /milvus-distributed/
+
+CMD ["./bin/distributed/indexservice"]
+
+EXPOSE 21118

build/docker/deploy/docker-compose.yml

@@ -11,7 +11,7 @@ services:
     environment: 
       PULSAR_ADDRESS: ${PULSAR_ADDRESS}
       ETCD_ADDRESS: ${ETCD_ADDRESS}
-      INDEX_BUILDER_ADDRESS: ${INDEX_BUILDER_ADDRESS}
+      INDEX_SERVICE_ADDRESS: ${INDEX_SERVICE_ADDRESS}
     networks: 
       - milvus
 

cmd/distributed/indexnode/main.go

@@ -24,6 +24,7 @@ import (
 
 func main() {
 	ctx, cancel := context.WithCancel(context.Background())
+	grpcindexnode.Init()
 	svr, err := grpcindexnode.CreateIndexNode(ctx)
 	if err != nil {
 		log.Print("create server failed", zap.Error(err))
@@ -49,7 +50,7 @@ func main() {
 	<-ctx.Done()
 	log.Print("Got signal to exit", zap.String("signal", sig.String()))
 
-	svr.Close()
+	svr.Stop()
 	switch sig {
 	case syscall.SIGTERM:
 		exit(0)

cmd/distributed/indexservice/main.go

@@ -24,6 +24,7 @@ import (
 
 func main() {
 	ctx, cancel := context.WithCancel(context.Background())
+	grpcindexserver.Init()
 	svr, err := grpcindexserver.CreateIndexServer(ctx)
 	if err != nil {
 		log.Print("create server failed", zap.Error(err))
@@ -49,7 +50,7 @@ func main() {
 	<-ctx.Done()
 	log.Print("Got signal to exit", zap.String("signal", sig.String()))
 
-	svr.Close()
+	svr.Stop()
 	switch sig {
 	case syscall.SIGTERM:
 		exit(0)

cmd/indexbuilder/indexbuilder.go

@@ -1,53 +1,53 @@
 package main
 
-import (
-	"context"
-	"log"
-	"os"
-	"os/signal"
-	"syscall"
-
-	"github.com/zilliztech/milvus-distributed/internal/indexnode"
-	"go.uber.org/zap"
-)
-
-func main() {
-	indexnode.Init()
-	ctx, cancel := context.WithCancel(context.Background())
-	svr, err := indexnode.CreateBuilder(ctx)
-	if err != nil {
-		log.Print("create server failed", zap.Error(err))
-	}
-
-	sc := make(chan os.Signal, 1)
-	signal.Notify(sc,
-		syscall.SIGHUP,
-		syscall.SIGINT,
-		syscall.SIGTERM,
-		syscall.SIGQUIT)
-
-	var sig os.Signal
-	go func() {
-		sig = <-sc
-		cancel()
-	}()
-
-	if err := svr.Start(); err != nil {
-		log.Fatal("run builder server failed", zap.Error(err))
-	}
-
-	<-ctx.Done()
-	log.Print("Got signal to exit", zap.String("signal", sig.String()))
-
-	svr.Close()
-	switch sig {
-	case syscall.SIGTERM:
-		exit(0)
-	default:
-		exit(1)
-	}
-}
-
-func exit(code int) {
-	os.Exit(code)
-}
+//import (
+//	"context"
+//	"log"
+//	"os"
+//	"os/signal"
+//	"syscall"
+//
+//	"github.com/zilliztech/milvus-distributed/internal/indexnode"
+//	"go.uber.org/zap"
+//)
+//
+//func main() {
+//	indexnode.Init()
+//	ctx, cancel := context.WithCancel(context.Background())
+//	svr, err := indexnode.CreateIndexNode(ctx)
+//	if err != nil {
+//		log.Print("create server failed", zap.Error(err))
+//	}
+//
+//	sc := make(chan os.Signal, 1)
+//	signal.Notify(sc,
+//		syscall.SIGHUP,
+//		syscall.SIGINT,
+//		syscall.SIGTERM,
+//		syscall.SIGQUIT)
+//
+//	var sig os.Signal
+//	go func() {
+//		sig = <-sc
+//		cancel()
+//	}()
+//
+//	if err := svr.Start(); err != nil {
+//		log.Fatal("run builder server failed", zap.Error(err))
+//	}
+//
+//	<-ctx.Done()
+//	log.Print("Got signal to exit", zap.String("signal", sig.String()))
+//
+//	svr.Stop()
+//	switch sig {
+//	case syscall.SIGTERM:
+//		exit(0)
+//	default:
+//		exit(1)
+//	}
+//}
+//
+//func exit(code int) {
+//	os.Exit(code)
+//}

cmd/singlenode/main.go

@@ -142,7 +142,7 @@ func InitIndexBuilder(wg *sync.WaitGroup) {
 	defer wg.Done()
 	indexnode.Init()
 	ctx, cancel := context.WithCancel(context.Background())
-	svr, err := indexnode.CreateBuilder(ctx)
+	svr, err := indexnode.CreateIndexNode(ctx)
 	if err != nil {
 		log.Print("create server failed", zap.Error(err))
 	}
@@ -167,7 +167,7 @@ func InitIndexBuilder(wg *sync.WaitGroup) {
 	<-ctx.Done()
 	log.Print("Got signal to exit", zap.String("signal", sig.String()))
 
-	svr.Close()
+	svr.Stop()
 	switch sig {
 	case syscall.SIGTERM:
 		exit(0)

configs/milvus.yaml

@@ -66,13 +66,9 @@ indexBuilder:
     address: localhost
     port: 31000
 
-indexNode:
-  address: localhost
-  port: 21116
-
 indexServer:
   address: localhost
-  port: 21118
+  port: 31000
 
 dataNode:
   address: localhost

internal/core/src/query/CMakeLists.txt

@@ -8,6 +8,8 @@ set(MILVUS_QUERY_SRCS
         visitors/ExecExprVisitor.cpp
         visitors/VerifyPlanNodeVisitor.cpp
         visitors/VerifyExprVisitor.cpp
+        visitors/ExtractInfoPlanNodeVisitor.cpp
+        visitors/ExtractInfoExprVisitor.cpp
         Plan.cpp
         SearchOnGrowing.cpp
         SearchOnSealed.cpp

internal/core/src/query/Plan.cpp

@@ -22,6 +22,7 @@
 #include <boost/algorithm/string.hpp>
 #include <algorithm>
 #include "query/generated/VerifyPlanNodeVisitor.h"
+#include "query/generated/ExtractInfoPlanNodeVisitor.h"
 
 namespace milvus::query {
 
@@ -142,9 +143,14 @@ Parser::CreatePlanImpl(const std::string& dsl_str) {
     VerifyPlanNodeVisitor verifier;
     vec_node->accept(verifier);
 
+    ExtractedPlanInfo plan_info(schema.size());
+    ExtractInfoPlanNodeVisitor extractor(plan_info);
+    vec_node->accept(extractor);
+
     auto plan = std::make_unique<Plan>(schema);
     plan->tag2field_ = std::move(tag2field_);
     plan->plan_node_ = std::move(vec_node);
+    plan->extra_info_opt_ = std::move(plan_info);
     return plan;
 }
 

internal/core/src/query/PlanImpl.h

@@ -20,11 +20,28 @@
 #include <map>
 #include <string>
 #include <vector>
+#include <optional>
+#include <boost/dynamic_bitset.hpp>
 
 namespace milvus::query {
 using Json = nlohmann::json;
 
 // class definitions
+
+struct ExtractedPlanInfo {
+ public:
+    explicit ExtractedPlanInfo(int64_t size) : involved_fields_(size) {
+    }
+
+    void
+    add_involved_field(FieldOffset field_offset) {
+        involved_fields_.set(field_offset.get());
+    }
+
+ public:
+    boost::dynamic_bitset<> involved_fields_;
+};
+
 struct Plan {
  public:
     explicit Plan(const Schema& schema) : schema_(schema) {
@@ -35,7 +52,9 @@ struct Plan {
     std::unique_ptr<VectorPlanNode> plan_node_;
     std::map<std::string, FieldOffset> tag2field_;  // PlaceholderName -> FieldOffset
     std::vector<FieldOffset> target_entries_;
-    std::vector<FieldOffset> referred_fields_;
+
+ public:
+    std::optional<ExtractedPlanInfo> extra_info_opt_;
     // TODO: add move extra info
 };
 

internal/core/src/query/generated/.gitignore

@@ -0,0 +1,3 @@
+!.gitignore
+*PlanNodeVisitor.cpp
+*ExprVisitor.cpp

internal/core/src/query/generated/ExecPlanNodeVisitor.cpp

@@ -1,26 +0,0 @@
-// Copyright (C) 2019-2020 Zilliz. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software distributed under the License
-// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-// or implied. See the License for the specific language governing permissions and limitations under the License
-
-#error TODO: copy this file out, and modify the content.
-#include "query/generated/ExecPlanNodeVisitor.h"
-
-namespace milvus::query {
-void
-ExecPlanNodeVisitor::visit(FloatVectorANNS& node) {
-    // TODO
-}
-
-void
-ExecPlanNodeVisitor::visit(BinaryVectorANNS& node) {
-    // TODO
-}
-
-}  // namespace milvus::query

internal/core/src/query/generated/ExtractInfoExprVisitor.h

@@ -9,28 +9,32 @@
 // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 // or implied. See the License for the specific language governing permissions and limitations under the License
 
-#error TODO: copy this file out, and modify the content.
-#include "query/generated/VerifyExprVisitor.h"
+#pragma once
+// Generated File
+// DO NOT EDIT
+#include "query/Plan.h"
+#include "ExprVisitor.h"
 
 namespace milvus::query {
-void
-VerifyExprVisitor::visit(BoolUnaryExpr& expr) {
-    // TODO
-}
+class ExtractInfoExprVisitor : public ExprVisitor {
+ public:
+    void
+    visit(BoolUnaryExpr& expr) override;
 
-void
-VerifyExprVisitor::visit(BoolBinaryExpr& expr) {
-    // TODO
-}
+    void
+    visit(BoolBinaryExpr& expr) override;
 
-void
-VerifyExprVisitor::visit(TermExpr& expr) {
-    // TODO
-}
+    void
+    visit(TermExpr& expr) override;
 
-void
-VerifyExprVisitor::visit(RangeExpr& expr) {
-    // TODO
-}
+    void
+    visit(RangeExpr& expr) override;
 
+ public:
+    explicit ExtractInfoExprVisitor(ExtractedPlanInfo& plan_info) : plan_info_(plan_info) {
+    }
+
+ private:
+    ExtractedPlanInfo& plan_info_;
+};
 }  // namespace milvus::query

internal/core/src/query/generated/ExtractInfoPlanNodeVisitor.h

@@ -9,28 +9,26 @@
 // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 // or implied. See the License for the specific language governing permissions and limitations under the License
 
-#error TODO: copy this file out, and modify the content.
-#include "query/generated/ExecExprVisitor.h"
+#pragma once
+// Generated File
+// DO NOT EDIT
+#include "query/Plan.h"
+#include "PlanNodeVisitor.h"
 
 namespace milvus::query {
-void
-ExecExprVisitor::visit(BoolUnaryExpr& expr) {
-    // TODO
-}
+class ExtractInfoPlanNodeVisitor : public PlanNodeVisitor {
+ public:
+    void
+    visit(FloatVectorANNS& node) override;
 
-void
-ExecExprVisitor::visit(BoolBinaryExpr& expr) {
-    // TODO
-}
+    void
+    visit(BinaryVectorANNS& node) override;
 
-void
-ExecExprVisitor::visit(TermExpr& expr) {
-    // TODO
-}
-
-void
-ExecExprVisitor::visit(RangeExpr& expr) {
-    // TODO
-}
+ public:
+    explicit ExtractInfoPlanNodeVisitor(ExtractedPlanInfo& plan_info) : plan_info_(plan_info) {
+    }
 
+ private:
+    ExtractedPlanInfo& plan_info_;
+};
 }  // namespace milvus::query

internal/core/src/query/generated/ShowExprVisitor.cpp

@@ -1,36 +0,0 @@
-// Copyright (C) 2019-2020 Zilliz. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software distributed under the License
-// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-// or implied. See the License for the specific language governing permissions and limitations under the License
-
-#error TODO: copy this file out, and modify the content.
-#include "query/generated/ShowExprVisitor.h"
-
-namespace milvus::query {
-void
-ShowExprVisitor::visit(BoolUnaryExpr& expr) {
-    // TODO
-}
-
-void
-ShowExprVisitor::visit(BoolBinaryExpr& expr) {
-    // TODO
-}
-
-void
-ShowExprVisitor::visit(TermExpr& expr) {
-    // TODO
-}
-
-void
-ShowExprVisitor::visit(RangeExpr& expr) {
-    // TODO
-}
-
-}  // namespace milvus::query

internal/core/src/query/generated/ShowPlanNodeVisitor.cpp

@@ -1,26 +0,0 @@
-// Copyright (C) 2019-2020 Zilliz. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software distributed under the License
-// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-// or implied. See the License for the specific language governing permissions and limitations under the License
-
-#error TODO: copy this file out, and modify the content.
-#include "query/generated/ShowPlanNodeVisitor.h"
-
-namespace milvus::query {
-void
-ShowPlanNodeVisitor::visit(FloatVectorANNS& node) {
-    // TODO
-}
-
-void
-ShowPlanNodeVisitor::visit(BinaryVectorANNS& node) {
-    // TODO
-}
-
-}  // namespace milvus::query

internal/core/src/query/generated/VerifyPlanNodeVisitor.cpp

@@ -1,26 +0,0 @@
-// Copyright (C) 2019-2020 Zilliz. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software distributed under the License
-// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-// or implied. See the License for the specific language governing permissions and limitations under the License
-
-#error TODO: copy this file out, and modify the content.
-#include "query/generated/VerifyPlanNodeVisitor.h"
-
-namespace milvus::query {
-void
-VerifyPlanNodeVisitor::visit(FloatVectorANNS& node) {
-    // TODO
-}
-
-void
-VerifyPlanNodeVisitor::visit(BinaryVectorANNS& node) {
-    // TODO
-}
-
-}  // namespace milvus::query

internal/core/src/query/visitors/ExtractInfoExprVisitor.cpp

@@ -0,0 +1,52 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License
+
+#include "query/Plan.h"
+#include "query/generated/ExtractInfoExprVisitor.h"
+
+namespace milvus::query {
+#if 1
+namespace impl {
+// THIS CONTAINS EXTRA BODY FOR VISITOR
+// WILL BE USED BY GENERATOR UNDER suvlim/core_gen/
+class ExtractInfoExprVisitor : ExprVisitor {
+ public:
+    explicit ExtractInfoExprVisitor(ExtractedPlanInfo& plan_info) : plan_info_(plan_info) {
+    }
+
+ private:
+    ExtractedPlanInfo& plan_info_;
+};
+}  // namespace impl
+#endif
+
+void
+ExtractInfoExprVisitor::visit(BoolUnaryExpr& expr) {
+    expr.child_->accept(*this);
+}
+
+void
+ExtractInfoExprVisitor::visit(BoolBinaryExpr& expr) {
+    expr.left_->accept(*this);
+    expr.right_->accept(*this);
+}
+
+void
+ExtractInfoExprVisitor::visit(TermExpr& expr) {
+    plan_info_.add_involved_field(expr.field_offset_);
+}
+
+void
+ExtractInfoExprVisitor::visit(RangeExpr& expr) {
+    plan_info_.add_involved_field(expr.field_offset_);
+}
+
+}  // namespace milvus::query

internal/core/src/query/visitors/ExtractInfoPlanNodeVisitor.cpp

@@ -0,0 +1,51 @@
+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License
+
+#include "query/Plan.h"
+#include "query/generated/ExtractInfoPlanNodeVisitor.h"
+#include "query/generated/ExtractInfoExprVisitor.h"
+
+namespace milvus::query {
+
+#if 1
+namespace impl {
+// THIS CONTAINS EXTRA BODY FOR VISITOR
+// WILL BE USED BY GENERATOR UNDER suvlim/core_gen/
+class ExtractInfoPlanNodeVisitor : PlanNodeVisitor {
+ public:
+    explicit ExtractInfoPlanNodeVisitor(ExtractedPlanInfo& plan_info) : plan_info_(plan_info) {
+    }
+
+ private:
+    ExtractedPlanInfo& plan_info_;
+};
+}  // namespace impl
+#endif
+
+void
+ExtractInfoPlanNodeVisitor::visit(FloatVectorANNS& node) {
+    plan_info_.add_involved_field(node.query_info_.field_offset_);
+    if (node.predicate_.has_value()) {
+        ExtractInfoExprVisitor expr_visitor(plan_info_);
+        node.predicate_.value()->accept(expr_visitor);
+    }
+}
+
+void
+ExtractInfoPlanNodeVisitor::visit(BinaryVectorANNS& node) {
+    plan_info_.add_involved_field(node.query_info_.field_offset_);
+    if (node.predicate_.has_value()) {
+        ExtractInfoExprVisitor expr_visitor(plan_info_);
+        node.predicate_.value()->accept(expr_visitor);
+    }
+}
+
+}  // namespace milvus::query

internal/core/src/segcore/SealedIndexingRecord.h

@@ -46,6 +46,12 @@ struct SealedIndexingRecord {
         return field_indexings_.at(field_offset).get();
     }
 
+    void
+    drop_field_indexing(FieldOffset field_offset) {
+        std::unique_lock lck(mutex_);
+        field_indexings_.erase(field_offset);
+    }
+
     bool
     is_ready(FieldOffset field_offset) const {
         std::shared_lock lck(mutex_);

internal/core/src/segcore/SegmentGrowingImpl.h

@@ -158,9 +158,6 @@ class SegmentGrowingImpl : public SegmentGrowing {
         bulk_subscript_impl<int64_t>(*vec_ptr, seg_offsets, count, output);
     }
 
-    int64_t
-    num_chunk() const override;
-
     Status
     LoadIndexing(const LoadIndexInfo& info) override;
 
@@ -188,9 +185,17 @@ class SegmentGrowingImpl : public SegmentGrowing {
     get_deleted_bitmap(int64_t del_barrier, Timestamp query_timestamp, int64_t insert_barrier, bool force = false);
 
  protected:
+    int64_t
+    num_chunk() const override;
+
     SpanBase
     chunk_data_impl(FieldOffset field_offset, int64_t chunk_id) const override;
 
+    void
+    check_search(const query::Plan* plan) const override {
+        Assert(plan);
+    }
+
  private:
     int64_t size_per_chunk_;
     SchemaPtr schema_;

internal/core/src/segcore/SegmentInterface.cpp

@@ -48,6 +48,7 @@ SegmentInternalInterface::Search(const query::Plan* plan,
                                  const Timestamp* timestamps,
                                  int64_t num_groups) const {
     std::shared_lock lck(mutex_);
+    check_search(plan);
     Assert(num_groups == 1);
     query::ExecPlanNodeVisitor visitor(*this, timestamps[0], *placeholder_groups[0]);
     auto results = visitor.get_moved_result(*plan->plan_node_);

internal/core/src/segcore/SegmentInterface.h

@@ -117,6 +117,9 @@ class SegmentInternalInterface : public SegmentInterface {
     virtual void
     bulk_subscript(FieldOffset field_offset, const int64_t* seg_offsets, int64_t count, void* output) const = 0;
 
+    virtual void
+    check_search(const query::Plan* plan) const = 0;
+
  protected:
     mutable std::shared_mutex mutex_;
 };

internal/core/src/segcore/SegmentSealedImpl.cpp

@@ -31,7 +31,7 @@ SegmentSealedImpl::LoadIndex(const LoadIndexInfo& info) {
     }
     Assert(!vec_indexings_.is_ready(field_offset));
     vec_indexings_.append_field_indexing(field_offset, GetMetricType(metric_type_str), info.index);
-    ++ready_count_;
+    set_field_ready(field_offset, true);
 }
 
 void
@@ -70,10 +70,10 @@ SegmentSealedImpl::LoadFieldData(const LoadFieldDataInfo& info) {
         // write data under lock
         std::unique_lock lck(mutex_);
         update_row_count(info.row_count);
-        AssertInfo(columns_data_[field_offset.get()].empty(), "already exists");
-        columns_data_[field_offset.get()] = std::move(vec_data);
+        AssertInfo(field_datas_[field_offset.get()].empty(), "already exists");
+        field_datas_[field_offset.get()] = std::move(vec_data);
 
-        ++ready_count_;
+        set_field_ready(field_offset, true);
     }
 }
 
@@ -96,10 +96,10 @@ SegmentSealedImpl::size_per_chunk() const {
 SpanBase
 SegmentSealedImpl::chunk_data_impl(FieldOffset field_offset, int64_t chunk_id) const {
     std::shared_lock lck(mutex_);
+    Assert(is_field_ready(field_offset));
     auto& field_meta = schema_->operator[](field_offset);
     auto element_sizeof = field_meta.get_sizeof();
-    Assert(is_all_ready());
-    SpanBase base(columns_data_[field_offset.get()].data(), row_count_opt_.value(), element_sizeof);
+    SpanBase base(field_datas_[field_offset.get()].data(), row_count_opt_.value(), element_sizeof);
     return base;
 }
 
@@ -143,13 +143,39 @@ SegmentSealedImpl::vector_search(int64_t vec_count,
 }
 void
 SegmentSealedImpl::DropFieldData(const FieldId field_id) {
-    std::unique_lock lck(mutex_);
-    PanicInfo("unimplemented");
+    if (SystemProperty::Instance().IsSystem(field_id)) {
+        auto system_field_type = SystemProperty::Instance().GetSystemFieldType(field_id);
+        Assert(system_field_type == SystemFieldType::RowId);
+
+        std::unique_lock lck(mutex_);
+        --system_ready_count_;
+        auto row_ids = std::move(row_ids_);
+        lck.unlock();
+
+        row_ids.clear();
+    } else {
+        auto field_offset = schema_->get_offset(field_id);
+        auto& field_meta = schema_->operator[](field_offset);
+        Assert(!field_meta.is_vector());
+
+        std::unique_lock lck(mutex_);
+        set_field_ready(field_offset, false);
+        auto vec = std::move(field_datas_[field_offset.get()]);
+        lck.unlock();
+
+        vec.clear();
+    }
 }
+
 void
 SegmentSealedImpl::DropIndex(const FieldId field_id) {
+    Assert(!SystemProperty::Instance().IsSystem(field_id));
+    auto field_offset = schema_->get_offset(field_id);
+    auto& field_meta = schema_->operator[](field_offset);
+    Assert(field_meta.is_vector());
+
     std::unique_lock lck(mutex_);
-    PanicInfo("unimplemented");
+    vec_indexings_.drop_field_indexing(field_offset);
 }
 
 SegmentSealedPtr

internal/core/src/segcore/SegmentSealedImpl.h

@@ -18,7 +18,8 @@
 namespace milvus::segcore {
 class SegmentSealedImpl : public SegmentSealed {
  public:
-    explicit SegmentSealedImpl(SchemaPtr schema) : schema_(schema), columns_data_(schema->size()) {
+    explicit SegmentSealedImpl(SchemaPtr schema)
+        : schema_(schema), field_datas_(schema->size()), field_ready_bitset_(schema->size()) {
     }
     void
     LoadIndex(const LoadIndexInfo& info) override;
@@ -65,7 +66,7 @@ class SegmentSealedImpl : public SegmentSealed {
                    const int64_t* seg_offsets,
                    int64_t count,
                    void* output) const override {
-        Assert(is_all_ready());
+        Assert(is_system_field_ready());
         Assert(system_type == SystemFieldType::RowId);
         bulk_subscript_impl<int64_t>(row_ids_.data(), seg_offsets, count, output);
     }
@@ -74,10 +75,29 @@ class SegmentSealedImpl : public SegmentSealed {
     // where Vec is determined from field_offset
     void
     bulk_subscript(FieldOffset field_offset, const int64_t* seg_offsets, int64_t count, void* output) const override {
-        Assert(is_all_ready());
+        Assert(is_field_ready(field_offset));
         auto& field_meta = schema_->operator[](field_offset);
         Assert(field_meta.get_data_type() == DataType::INT64);
-        bulk_subscript_impl<int64_t>(columns_data_[field_offset.get()].data(), seg_offsets, count, output);
+        bulk_subscript_impl<int64_t>(field_datas_[field_offset.get()].data(), seg_offsets, count, output);
+    }
+
+    void
+    check_search(const query::Plan* plan) const override {
+        Assert(plan);
+        Assert(plan->extra_info_opt_.has_value());
+
+        if (!is_system_field_ready()) {
+            PanicInfo("System Field RowID is not loaded");
+        }
+
+        auto& request_fields = plan->extra_info_opt_.value().involved_fields_;
+        Assert(request_fields.size() == field_ready_bitset_.size());
+        auto absent_fields = request_fields - field_ready_bitset_;
+        if (absent_fields.any()) {
+            auto field_offset = FieldOffset(absent_fields.find_first());
+            auto& field_meta = schema_->operator[](field_offset);
+            PanicInfo("User Field(" + field_meta.get_name().get() + ") is not loaded");
+        }
     }
 
  private:
@@ -116,25 +136,25 @@ class SegmentSealedImpl : public SegmentSealed {
     }
 
     bool
-    is_all_ready() const {
-        // TODO: optimize here
-        // NOTE: including row_ids
-        if (!is_system_field_ready()) {
-            return false;
-        }
-        return ready_count_ == schema_->size();
+    is_field_ready(FieldOffset field_offset) const {
+        return field_ready_bitset_.test(field_offset.get());
+    }
+
+    void
+    set_field_ready(FieldOffset field_offset, bool flag = true) {
+        field_ready_bitset_[field_offset.get()] = flag;
     }
 
  private:
     // segment loading state
-    std::atomic<int> ready_count_ = 0;
+    boost::dynamic_bitset<> field_ready_bitset_;
     std::atomic<int> system_ready_count_ = 0;
     // segment datas
     // TODO: generate index for scalar
     std::optional<int64_t> row_count_opt_;
     std::map<FieldOffset, knowhere::IndexPtr> scalar_indexings_;
     SealedIndexingRecord vec_indexings_;
-    std::vector<aligned_vector<char>> columns_data_;
+    std::vector<aligned_vector<char>> field_datas_;
     aligned_vector<idx_t> row_ids_;
     SchemaPtr schema_;
 };

internal/core/unittest/test_sealed.cpp

@@ -249,8 +249,9 @@ TEST(Sealed, LoadFieldData) {
     auto metric_type = MetricType::METRIC_L2;
     auto schema = std::make_shared<Schema>();
     auto fakevec_id = schema->AddDebugField("fakevec", DataType::VECTOR_FLOAT, dim, metric_type);
-    schema->AddDebugField("counter", DataType::INT64);
-    schema->AddDebugField("double", DataType::DOUBLE);
+    auto counter_id = schema->AddDebugField("counter", DataType::INT64);
+    auto double_id = schema->AddDebugField("double", DataType::DOUBLE);
+    auto nothing_id = schema->AddDebugField("nothing", DataType::INT32);
 
     auto dataset = DataGen(schema, N);
 
@@ -268,24 +269,6 @@ TEST(Sealed, LoadFieldData) {
     indexing->AddWithoutIds(database, conf);
 
     auto segment = CreateSealedSegment(schema);
-    SealedLoader(dataset, *segment);
-    {
-        LoadIndexInfo vec_info;
-        vec_info.field_id = fakevec_id.get();
-        vec_info.field_name = "fakevec";
-        vec_info.index = indexing;
-        vec_info.index_params["metric_type"] = milvus::knowhere::Metric::L2;
-        segment->LoadIndex(vec_info);
-    }
-    ASSERT_EQ(segment->num_chunk(), 1);
-    auto chunk_span1 = segment->chunk_data<int64_t>(FieldOffset(1), 0);
-    auto chunk_span2 = segment->chunk_data<double>(FieldOffset(2), 0);
-    auto ref1 = dataset.get_col<int64_t>(1);
-    auto ref2 = dataset.get_col<double>(2);
-    for (int i = 0; i < N; ++i) {
-        ASSERT_EQ(chunk_span1[i], ref1[i]);
-        ASSERT_EQ(chunk_span2[i], ref2[i]);
-    }
     std::string dsl = R"({
         "bool": {
             "must": [
@@ -313,14 +296,47 @@ TEST(Sealed, LoadFieldData) {
         }
     })";
 
+    Timestamp time = 1000000;
     auto plan = CreatePlan(*schema, dsl);
     auto num_queries = 5;
     auto ph_group_raw = CreatePlaceholderGroup(num_queries, 16, 1024);
     auto ph_group = ParsePlaceholderGroup(plan.get(), ph_group_raw.SerializeAsString());
-    Timestamp time = 1000000;
     std::vector<const PlaceholderGroup*> ph_group_arr = {ph_group.get()};
 
+    ASSERT_ANY_THROW(segment->Search(plan.get(), ph_group_arr.data(), &time, 1));
+
+    SealedLoader(dataset, *segment);
+    segment->DropFieldData(nothing_id);
+
+    ASSERT_ANY_THROW(segment->Search(plan.get(), ph_group_arr.data(), &time, 1));
+
+    LoadIndexInfo vec_info;
+    vec_info.field_id = fakevec_id.get();
+    vec_info.field_name = "fakevec";
+    vec_info.index = indexing;
+    vec_info.index_params["metric_type"] = milvus::knowhere::Metric::L2;
+    segment->LoadIndex(vec_info);
+
+    ASSERT_EQ(segment->num_chunk(), 1);
+    auto chunk_span1 = segment->chunk_data<int64_t>(FieldOffset(1), 0);
+    auto chunk_span2 = segment->chunk_data<double>(FieldOffset(2), 0);
+    auto ref1 = dataset.get_col<int64_t>(1);
+    auto ref2 = dataset.get_col<double>(2);
+    for (int i = 0; i < N; ++i) {
+        ASSERT_EQ(chunk_span1[i], ref1[i]);
+        ASSERT_EQ(chunk_span2[i], ref2[i]);
+    }
+
     auto qr = segment->Search(plan.get(), ph_group_arr.data(), &time, 1);
     auto json = QueryResultToJson(qr);
     std::cout << json.dump(1);
+
+    segment->DropIndex(fakevec_id);
+    ASSERT_ANY_THROW(segment->Search(plan.get(), ph_group_arr.data(), &time, 1));
+    segment->LoadIndex(vec_info);
+    auto qr2 = segment->Search(plan.get(), ph_group_arr.data(), &time, 1);
+    auto json2 = QueryResultToJson(qr);
+    ASSERT_EQ(json.dump(-2), json2.dump(-2));
+    segment->DropFieldData(double_id);
+    ASSERT_ANY_THROW(segment->Search(plan.get(), ph_group_arr.data(), &time, 1));
 }

internal/distributed/indexnode/client/client.go

@@ -11,25 +11,48 @@ import (
 )
 
 type Client struct {
-	grpcClient indexpb.IndexNodeClient
+	grpcClient  indexpb.IndexNodeClient
+	nodeAddress string
 }
 
-func (c Client) BuildIndex(req *indexpb.BuildIndexCmd) (*commonpb.Status, error) {
+func (c Client) Init() error {
+	return nil
+}
+
+func (c Client) Start() error {
+	return nil
+}
+
+func (c Client) Stop() error {
+	return nil
+}
+
+func (c *Client) tryConnect() error {
+	if c.grpcClient != nil {
+		return nil
+	}
+	ctx1, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	conn, err := grpc.DialContext(ctx1, c.nodeAddress, grpc.WithInsecure(), grpc.WithBlock())
+	if err != nil {
+		log.Printf("Connect to IndexNode failed, error= %v", err)
+		return err
+	}
+	c.grpcClient = indexpb.NewIndexNodeClient(conn)
+	return nil
+}
+
+func (c *Client) BuildIndex(req *indexpb.BuildIndexCmd) (*commonpb.Status, error) {
 
 	ctx := context.TODO()
+	c.tryConnect()
 
 	return c.grpcClient.BuildIndex(ctx, req)
 }
 
-func NewClient(nodeAddress string) *Client {
-	ctx1, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	conn, err := grpc.DialContext(ctx1, nodeAddress, grpc.WithInsecure(), grpc.WithBlock())
-	if err != nil {
-		log.Printf("Connect to IndexNode failed, error= %v", err)
-	}
-	log.Printf("Connected to IndexService, IndexService=%s", nodeAddress)
+func NewClient(nodeAddress string) (*Client, error) {
+
 	return &Client{
-		grpcClient: indexpb.NewIndexNodeClient(conn),
-	}
+		nodeAddress: nodeAddress,
+	}, nil
 }

internal/distributed/indexnode/service.go

@@ -11,31 +11,41 @@ import (
 	"github.com/zilliztech/milvus-distributed/internal/indexnode"
 	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
 	"github.com/zilliztech/milvus-distributed/internal/proto/indexpb"
+	"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
 	"google.golang.org/grpc"
 )
 
 type Server struct {
-	node indexnode.Interface
+	node typeutil.IndexNodeInterface
 
-	grpcServer *grpc.Server
-
-	loopCtx    context.Context
-	loopCancel func()
-	loopWg     sync.WaitGroup
+	grpcServer   *grpc.Server
+	serverClient typeutil.IndexServiceInterface
+	loopCtx      context.Context
+	loopCancel   func()
+	loopWg       sync.WaitGroup
 }
 
-func NewGrpcServer(ctx context.Context, nodeID int64) *Server {
+func NewGrpcServer(ctx context.Context) *Server {
 	ctx1, cancel := context.WithCancel(ctx)
+	indexServiceClient := serviceclient.NewClient(indexnode.Params.ServiceAddress)
+
+	node, err := indexnode.CreateIndexNode(ctx1)
+	if err != nil {
+		defer cancel()
+		return nil
+	}
+
+	node.SetServiceClient(indexServiceClient)
+
 	return &Server{
-		loopCtx:    ctx1,
-		loopCancel: cancel,
-		node:       indexnode.NewIndexNode(ctx, nodeID),
+		loopCtx:      ctx1,
+		loopCancel:   cancel,
+		node:         node,
+		serverClient: indexServiceClient,
 	}
 }
 
-func registerNode() error {
-
-	indexServiceClient := serviceclient.NewClient(indexnode.Params.ServiceAddress)
+func (s *Server) registerNode() error {
 
 	log.Printf("Registering node. IP = %s, Port = %d", indexnode.Params.NodeIP, indexnode.Params.NodePort)
 
@@ -46,7 +56,7 @@ func registerNode() error {
 			Port: int64(indexnode.Params.NodePort),
 		},
 	}
-	resp, err := indexServiceClient.RegisterNode(request)
+	resp, err := s.serverClient.RegisterNode(request)
 	if err != nil {
 		log.Printf("IndexNode connect to IndexService failed, error= %v", err)
 		return err
@@ -81,16 +91,16 @@ func (s *Server) startIndexNode() error {
 
 	log.Println("IndexNode grpc server start successfully")
 
-	err := registerNode()
+	err := s.registerNode()
 	if err != nil {
 		return err
 	}
 
 	indexnode.Params.Init()
-	return nil
+	return s.node.Start()
 }
 
-func (s *Server) Init() {
+func Init() error {
 	indexnode.Params.Init()
 
 	//Get native ip
@@ -111,34 +121,38 @@ func (s *Server) Init() {
 	//Generate random and available port
 	listener, err := net.Listen("tcp", ":0")
 	if err != nil {
-		panic(err)
+		return err
 	}
 
 	indexnode.Params.NodePort = listener.Addr().(*net.TCPAddr).Port
 	listener.Close()
 	indexnode.Params.NodeAddress = indexnode.Params.NodeIP + ":" + strconv.FormatInt(int64(indexnode.Params.NodePort), 10)
 	log.Println("IndexNode init successfully, nodeAddress=", indexnode.Params.NodeAddress)
+
+	return nil
 }
 
 func CreateIndexNode(ctx context.Context) (*Server, error) {
 
-	return NewGrpcServer(ctx, indexnode.Params.NodeID), nil
+	return NewGrpcServer(ctx), nil
 }
 
 func (s *Server) Start() error {
-	s.Init()
 	return s.startIndexNode()
 }
 
-func (s *Server) Stop() {
+func (s *Server) Stop() error {
+	s.node.Stop()
+	s.loopCancel()
+	if s.grpcServer != nil {
+		s.grpcServer.GracefulStop()
+	}
 	s.loopWg.Wait()
-}
 
-func (s *Server) Close() {
-
-	s.Stop()
+	return nil
 }
 
 func (s *Server) BuildIndex(ctx context.Context, req *indexpb.BuildIndexCmd) (*commonpb.Status, error) {
+	log.Println("distributed build index")
 	return s.node.BuildIndex(req)
 }

internal/distributed/indexservice/client/client.go

@@ -7,57 +7,217 @@ import (
 
 	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
 	"github.com/zilliztech/milvus-distributed/internal/proto/indexpb"
+	"github.com/zilliztech/milvus-distributed/internal/proto/internalpb2"
+	"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
 	"google.golang.org/grpc"
 )
 
+type UniqueID = typeutil.UniqueID
+
 type Client struct {
 	grpcClient indexpb.IndexServiceClient
+	address    string
 }
 
-func (g Client) RegisterNode(req *indexpb.RegisterNodeRequest) (*indexpb.RegisterNodeResponse, error) {
+func (g *Client) Init() error {
+	return nil
+}
+
+func (g *Client) Start() error {
+	return nil
+}
+
+func (g *Client) Stop() error {
+	return nil
+}
+
+//func (g *Client) BuildIndex2(columnDataPaths []string, typeParams map[string]string, indexParams map[string]string) (UniqueID, error) {
+//
+//	parseMap := func(mStr string) (map[string]string, error) {
+//		buffer := make(map[string]interface{})
+//		err := json.Unmarshal([]byte(mStr), &buffer)
+//		if err != nil {
+//			return nil, errors.New("Unmarshal params failed")
+//		}
+//		ret := make(map[string]string)
+//		for key, value := range buffer {
+//			valueStr := fmt.Sprintf("%v", value)
+//			ret[key] = valueStr
+//		}
+//		return ret, nil
+//	}
+//	var typeParamsKV []*commonpb.KeyValuePair
+//	for key := range typeParams {
+//		if key == "params" {
+//			mapParams, err := parseMap(typeParams[key])
+//			if err != nil {
+//				log.Println("parse params error: ", err)
+//			}
+//			for pk, pv := range mapParams {
+//				typeParamsKV = append(typeParamsKV, &commonpb.KeyValuePair{
+//					Key:   pk,
+//					Value: pv,
+//				})
+//			}
+//		} else {
+//			typeParamsKV = append(typeParamsKV, &commonpb.KeyValuePair{
+//				Key:   key,
+//				Value: typeParams[key],
+//			})
+//		}
+//	}
+//
+//	var indexParamsKV []*commonpb.KeyValuePair
+//	for key := range indexParams {
+//		if key == "params" {
+//			mapParams, err := parseMap(indexParams[key])
+//			if err != nil {
+//				log.Println("parse params error: ", err)
+//			}
+//			for pk, pv := range mapParams {
+//				indexParamsKV = append(indexParamsKV, &commonpb.KeyValuePair{
+//					Key:   pk,
+//					Value: pv,
+//				})
+//			}
+//		} else {
+//			indexParamsKV = append(indexParamsKV, &commonpb.KeyValuePair{
+//				Key:   key,
+//				Value: indexParams[key],
+//			})
+//		}
+//	}
+//
+//	requset := &indexpb.BuildIndexRequest{
+//		DataPaths:   columnDataPaths,
+//		TypeParams:  typeParamsKV,
+//		IndexParams: indexParamsKV,
+//	}
+//	response, err := g.BuildIndex(requset)
+//	if err != nil {
+//		return 0, err
+//	}
+//
+//	indexID := response.IndexID
+//
+//	return indexID, nil
+//}
+//
+//func (g *Client) GetIndexStates2(indexIDs []UniqueID) (*indexpb.IndexStatesResponse, error) {
+//
+//	request := &indexpb.IndexStatesRequest{
+//		IndexIDs: indexIDs,
+//	}
+//
+//	response, err := g.GetIndexStates(request)
+//	return response, err
+//}
+//
+//func (g *Client) GetIndexFilePaths2(indexIDs []UniqueID) ([][]string, error) {
+//
+//	request := &indexpb.IndexFilePathsRequest{
+//		IndexIDs: indexIDs,
+//	}
+//
+//	response, err := g.GetIndexFilePaths(request)
+//	if err != nil {
+//		return nil, err
+//	}
+//
+//	var filePaths [][]string
+//	for _, indexID := range indexIDs {
+//		for _, filePathInfo := range response.FilePaths {
+//			if indexID == filePathInfo.IndexID {
+//				filePaths = append(filePaths, filePathInfo.IndexFilePaths)
+//				break
+//			}
+//		}
+//	}
+//
+//	return filePaths, nil
+//}
+
+func (g *Client) GetComponentStates() (*internalpb2.ComponentStates, error) {
+	return nil, nil
+}
+
+func (g *Client) GetTimeTickChannel() (string, error) {
+	return "", nil
+}
+
+func (g *Client) GetStatisticsChannel() (string, error) {
+	return "", nil
+}
+
+func (g *Client) tryConnect() error {
+	if g.grpcClient != nil {
+		return nil
+	}
+	ctx1, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	log.Println("indexservice address = ", g.address)
+	conn, err := grpc.DialContext(ctx1, g.address, grpc.WithInsecure(), grpc.WithBlock())
+	if err != nil {
+		log.Printf("Connect to IndexService failed, error= %v", err)
+		return err
+	}
+	g.grpcClient = indexpb.NewIndexServiceClient(conn)
+	return nil
+}
+
+func (g *Client) RegisterNode(req *indexpb.RegisterNodeRequest) (*indexpb.RegisterNodeResponse, error) {
+	err := g.tryConnect()
+	if err != nil {
+		return nil, err
+	}
 
 	ctx := context.TODO()
 	return g.grpcClient.RegisterNode(ctx, req)
 }
 
-func (g Client) BuildIndex(req *indexpb.BuildIndexRequest) (*indexpb.BuildIndexResponse, error) {
+func (g *Client) BuildIndex(req *indexpb.BuildIndexRequest) (*indexpb.BuildIndexResponse, error) {
+	err := g.tryConnect()
+	if err != nil {
+		return nil, err
+	}
 
 	ctx := context.TODO()
-
 	return g.grpcClient.BuildIndex(ctx, req)
 }
 
-func (g Client) GetIndexStates(req *indexpb.IndexStatesRequest) (*indexpb.IndexStatesResponse, error) {
+func (g *Client) GetIndexStates(req *indexpb.IndexStatesRequest) (*indexpb.IndexStatesResponse, error) {
+	err := g.tryConnect()
+	if err != nil {
+		return nil, err
+	}
 
 	ctx := context.TODO()
-
 	return g.grpcClient.GetIndexStates(ctx, req)
 }
-
-func (g Client) GetIndexFilePaths(req *indexpb.IndexFilePathsRequest) (*indexpb.IndexFilePathsResponse, error) {
+func (g *Client) GetIndexFilePaths(req *indexpb.IndexFilePathsRequest) (*indexpb.IndexFilePathsResponse, error) {
+	err := g.tryConnect()
+	if err != nil {
+		return nil, err
+	}
 
 	ctx := context.TODO()
-
 	return g.grpcClient.GetIndexFilePaths(ctx, req)
 }
 
-func (g Client) NotifyBuildIndex(nty *indexpb.BuildIndexNotification) (*commonpb.Status, error) {
-	ctx := context.TODO()
+func (g *Client) NotifyBuildIndex(nty *indexpb.BuildIndexNotification) (*commonpb.Status, error) {
+	err := g.tryConnect()
+	if err != nil {
+		return nil, err
+	}
 
+	ctx := context.TODO()
 	return g.grpcClient.NotifyBuildIndex(ctx, nty)
 }
 
 func NewClient(address string) *Client {
 
-	ctx1, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	conn, err := grpc.DialContext(ctx1, address, grpc.WithInsecure(), grpc.WithBlock())
-	if err != nil {
-		log.Printf("Connect to IndexService failed, error= %v", err)
-	}
-	log.Printf("Connected to IndexService, IndexService=%s", address)
-
+	log.Println("new indexservice, address = ", address)
 	return &Client{
-		grpcClient: indexpb.NewIndexServiceClient(conn),
+		address: address,
 	}
 }

internal/distributed/indexservice/service.go

@@ -19,7 +19,7 @@ type UniqueID = typeutil.UniqueID
 type Timestamp = typeutil.Timestamp
 
 type Server struct {
-	server indexservice.Interface
+	server typeutil.IndexServiceInterface
 
 	grpcServer *grpc.Server
 
@@ -28,17 +28,22 @@ type Server struct {
 	loopWg     sync.WaitGroup
 }
 
-func (s *Server) Init() error {
+func Init() error {
 	indexservice.Params.Init()
 	return nil
 }
 
 func (s *Server) Start() error {
-	s.Init()
 	return s.startIndexServer()
 }
 
 func (s *Server) Stop() error {
+	s.server.Stop()
+	s.loopCancel()
+	if s.grpcServer != nil {
+		s.grpcServer.GracefulStop()
+	}
+
 	s.loopWg.Wait()
 	return nil
 }
@@ -57,12 +62,13 @@ func (s *Server) GetStatisticsChannel() (string, error) {
 
 func (s *Server) RegisterNode(ctx context.Context, req *indexpb.RegisterNodeRequest) (*indexpb.RegisterNodeResponse, error) {
 
-	log.Println("Register IndexNode starting...")
+	log.Println("Register IndexNode starting..., node address = ", req.Address)
 	return s.server.RegisterNode(req)
 }
 
 func (s *Server) BuildIndex(ctx context.Context, req *indexpb.BuildIndexRequest) (*indexpb.BuildIndexResponse, error) {
 
+	log.Println("Build Index ...")
 	return s.server.BuildIndex(req)
 }
 
@@ -78,6 +84,7 @@ func (s *Server) GetIndexFilePaths(ctx context.Context, req *indexpb.IndexFilePa
 
 func (s *Server) NotifyBuildIndex(ctx context.Context, nty *indexpb.BuildIndexNotification) (*commonpb.Status, error) {
 
+	log.Println("build index finished.")
 	return s.server.NotifyBuildIndex(nty)
 }
 
@@ -112,23 +119,23 @@ func (s *Server) startIndexServer() error {
 	go s.grpcLoop()
 	log.Println("IndexServer grpc server start successfully")
 
-	return nil
+	return s.server.Start()
 }
 
 func CreateIndexServer(ctx context.Context) (*Server, error) {
 
 	ctx1, cancel := context.WithCancel(ctx)
+	serverImp, err := indexservice.CreateIndexService(ctx)
+	if err != nil {
+		defer cancel()
+		return nil, err
+	}
 	s := &Server{
 		loopCtx:    ctx1,
 		loopCancel: cancel,
 
-		server: indexservice.NewIndexServiceImpl(ctx),
+		server: serverImp,
 	}
 
 	return s, nil
 }
-
-func (s *Server) Close() {
-
-	s.Stop()
-}

internal/indexnode/client/client.go

@@ -1,161 +0,0 @@
-package indexnodeclient
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"log"
-	"time"
-
-	"github.com/zilliztech/milvus-distributed/internal/errors"
-	"google.golang.org/grpc"
-
-	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
-	"github.com/zilliztech/milvus-distributed/internal/proto/indexpb"
-	"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
-)
-
-type UniqueID = typeutil.UniqueID
-
-type Client struct {
-	client  indexpb.IndexServiceClient
-	address string
-	ctx     context.Context
-}
-
-func NewBuildIndexClient(ctx context.Context, address string) (*Client, error) {
-	return &Client{
-		address: address,
-		ctx:     ctx,
-	}, nil
-}
-
-func parseTS(t int64) time.Time {
-	return time.Unix(0, t)
-}
-
-func (c *Client) tryConnect() error {
-	if c.client != nil {
-		return nil
-	}
-	conn, err := grpc.DialContext(c.ctx, c.address, grpc.WithInsecure(), grpc.WithBlock())
-	if err != nil {
-		return err
-	}
-	c.client = indexpb.NewIndexServiceClient(conn)
-	return nil
-}
-
-func (c *Client) BuildIndex(columnDataPaths []string, typeParams map[string]string, indexParams map[string]string) (UniqueID, error) {
-	if c.tryConnect() != nil {
-		panic("BuildIndexWithoutID: failed to connect index builder")
-	}
-	parseMap := func(mStr string) (map[string]string, error) {
-		buffer := make(map[string]interface{})
-		err := json.Unmarshal([]byte(mStr), &buffer)
-		if err != nil {
-			return nil, errors.New("Unmarshal params failed")
-		}
-		ret := make(map[string]string)
-		for key, value := range buffer {
-			valueStr := fmt.Sprintf("%v", value)
-			ret[key] = valueStr
-		}
-		return ret, nil
-	}
-	var typeParamsKV []*commonpb.KeyValuePair
-	for key := range typeParams {
-		if key == "params" {
-			mapParams, err := parseMap(typeParams[key])
-			if err != nil {
-				log.Println("parse params error: ", err)
-			}
-			for pk, pv := range mapParams {
-				typeParamsKV = append(typeParamsKV, &commonpb.KeyValuePair{
-					Key:   pk,
-					Value: pv,
-				})
-			}
-		} else {
-			typeParamsKV = append(typeParamsKV, &commonpb.KeyValuePair{
-				Key:   key,
-				Value: typeParams[key],
-			})
-		}
-	}
-
-	var indexParamsKV []*commonpb.KeyValuePair
-	for key := range indexParams {
-		if key == "params" {
-			mapParams, err := parseMap(indexParams[key])
-			if err != nil {
-				log.Println("parse params error: ", err)
-			}
-			for pk, pv := range mapParams {
-				indexParamsKV = append(indexParamsKV, &commonpb.KeyValuePair{
-					Key:   pk,
-					Value: pv,
-				})
-			}
-		} else {
-			indexParamsKV = append(indexParamsKV, &commonpb.KeyValuePair{
-				Key:   key,
-				Value: indexParams[key],
-			})
-		}
-	}
-
-	ctx := context.TODO()
-	requset := &indexpb.BuildIndexRequest{
-		DataPaths:   columnDataPaths,
-		TypeParams:  typeParamsKV,
-		IndexParams: indexParamsKV,
-	}
-	response, err := c.client.BuildIndex(ctx, requset)
-	if err != nil {
-		return 0, err
-	}
-
-	indexID := response.IndexID
-	return indexID, err
-}
-
-func (c *Client) GetIndexStates(indexIDs []UniqueID) (*indexpb.IndexStatesResponse, error) {
-	if c.tryConnect() != nil {
-		panic("DescribeIndex: failed to connect index builder")
-	}
-	ctx := context.TODO()
-	request := &indexpb.IndexStatesRequest{
-		IndexID: indexIDs,
-	}
-
-	response, err := c.client.GetIndexStates(ctx, request)
-	return response, err
-}
-
-func (c *Client) GetIndexFilePaths(indexIDs []UniqueID) ([][]string, error) {
-	if c.tryConnect() != nil {
-		panic("GetIndexFilePaths: failed to connect index builder")
-	}
-	ctx := context.TODO()
-	request := &indexpb.IndexFilePathsRequest{
-		IndexIDs: indexIDs,
-	}
-
-	response, err := c.client.GetIndexFilePaths(ctx, request)
-	if err != nil {
-		return nil, err
-	}
-
-	var filePaths [][]string
-	for _, indexID := range indexIDs {
-		for _, filePathInfo := range response.FilePaths {
-			if indexID == filePathInfo.IndexID {
-				filePaths = append(filePaths, filePathInfo.IndexFilePaths)
-				break
-			}
-		}
-	}
-
-	return filePaths, nil
-}

internal/indexnode/grpc_service.go

@@ -1,100 +0,0 @@
-package indexnode
-
-import (
-	"context"
-	"errors"
-	"log"
-	"time"
-
-	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
-	"github.com/zilliztech/milvus-distributed/internal/proto/indexpb"
-)
-
-const (
-	reqTimeoutInterval = time.Second * 10
-)
-
-func (b *Builder) BuildIndex(ctx context.Context, request *indexpb.BuildIndexRequest) (*indexpb.BuildIndexResponse, error) {
-	t := NewIndexAddTask()
-	t.req = request
-	t.idAllocator = b.idAllocator
-	t.buildQueue = b.sched.IndexBuildQueue
-	t.table = b.metaTable
-	t.kv = b.kv
-	var cancel func()
-	t.ctx, cancel = context.WithTimeout(ctx, reqTimeoutInterval)
-	defer cancel()
-
-	fn := func() error {
-		select {
-		case <-ctx.Done():
-			return errors.New("insert timeout")
-		default:
-			return b.sched.IndexAddQueue.Enqueue(t)
-		}
-	}
-	ret := &indexpb.BuildIndexResponse{
-		Status: &commonpb.Status{
-			ErrorCode: commonpb.ErrorCode_SUCCESS,
-		},
-	}
-
-	err := fn()
-	if err != nil {
-		ret.Status.ErrorCode = commonpb.ErrorCode_UNEXPECTED_ERROR
-		ret.Status.Reason = err.Error()
-		return ret, nil
-	}
-
-	err = t.WaitToFinish()
-	if err != nil {
-		ret.Status.ErrorCode = commonpb.ErrorCode_UNEXPECTED_ERROR
-		ret.Status.Reason = err.Error()
-		return ret, nil
-	}
-	ret.IndexID = t.indexID
-	return ret, nil
-}
-
-func (b *Builder) GetIndexStates(ctx context.Context, request *indexpb.IndexStatesRequest) (*indexpb.IndexStatesResponse, error) {
-	var indexStates []*indexpb.IndexInfo
-	for _, indexID := range request.IndexID {
-		indexState, err := b.metaTable.GetIndexStates(indexID)
-		if err != nil {
-			log.Println("GetIndexStates error, err=", err)
-		}
-		indexStates = append(indexStates, indexState)
-	}
-	ret := &indexpb.IndexStatesResponse{
-		Status: &commonpb.Status{
-			ErrorCode: commonpb.ErrorCode_SUCCESS,
-			Reason:    "",
-		},
-		States: indexStates,
-	}
-
-	return ret, nil
-}
-
-func (b *Builder) GetIndexFilePaths(ctx context.Context, request *indexpb.IndexFilePathsRequest) (*indexpb.IndexFilePathsResponse, error) {
-	ret := &indexpb.IndexFilePathsResponse{
-		Status: &commonpb.Status{ErrorCode: commonpb.ErrorCode_SUCCESS},
-	}
-	var filePathInfos []*indexpb.IndexFilePathInfo
-	for _, indexID := range request.IndexIDs {
-		filePathInfo := &indexpb.IndexFilePathInfo{
-			Status:  &commonpb.Status{ErrorCode: commonpb.ErrorCode_SUCCESS},
-			IndexID: indexID,
-		}
-
-		filePaths, err := b.metaTable.GetIndexFilePaths(indexID)
-		if err != nil {
-			filePathInfo.Status.ErrorCode = commonpb.ErrorCode_UNEXPECTED_ERROR
-			filePathInfo.Status.Reason = err.Error()
-		}
-		filePathInfo.IndexFilePaths = filePaths
-		filePathInfos = append(filePathInfos, filePathInfo)
-	}
-	ret.FilePaths = filePathInfos
-	return ret, nil
-}

internal/indexnode/index.go

@@ -12,6 +12,7 @@ package indexnode
 
 */
 import "C"
+
 import (
 	"errors"
 	"fmt"
@@ -27,6 +28,91 @@ import (
 // TODO: use storage.Blob instead later
 type Blob = storage.Blob
 
+// just for debugging
+type QueryResult interface {
+	Delete() error
+	NQ() int64
+	TOPK() int64
+	IDs() []int64
+	Distances() []float32
+}
+
+type CQueryResult struct {
+	ptr C.CIndexQueryResult
+}
+
+type CFunc func() C.CStatus
+
+func TryCatch(fn CFunc) error {
+	status := fn()
+	errorCode := status.error_code
+	if errorCode != 0 {
+		errorMsg := C.GoString(status.error_msg)
+		defer C.free(unsafe.Pointer(status.error_msg))
+		return errors.New("error code = " + strconv.Itoa(int(errorCode)) + ", error msg = " + errorMsg)
+	}
+	return nil
+}
+
+func CreateQueryResult() (QueryResult, error) {
+	var ptr C.CIndexQueryResult
+	fn := func() C.CStatus {
+		return C.CreateQueryResult(&ptr)
+	}
+	err := TryCatch(fn)
+	if err != nil {
+		return nil, err
+	}
+	return &CQueryResult{
+		ptr: ptr,
+	}, nil
+}
+
+func (qs *CQueryResult) Delete() error {
+	fn := func() C.CStatus {
+		return C.DeleteQueryResult(qs.ptr)
+	}
+	return TryCatch(fn)
+}
+
+func (qs *CQueryResult) NQ() int64 {
+	return int64(C.NqOfQueryResult(qs.ptr))
+}
+
+func (qs *CQueryResult) TOPK() int64 {
+	return int64(C.TopkOfQueryResult(qs.ptr))
+}
+
+func (qs *CQueryResult) IDs() []int64 {
+	nq := qs.NQ()
+	topk := qs.TOPK()
+
+	if nq <= 0 || topk <= 0 {
+		return []int64{}
+	}
+
+	// TODO: how could we avoid memory copy every time when this called
+	ids := make([]int64, nq*topk)
+	C.GetIdsOfQueryResult(qs.ptr, (*C.int64_t)(&ids[0]))
+
+	return ids
+}
+
+func (qs *CQueryResult) Distances() []float32 {
+	nq := qs.NQ()
+	topk := qs.TOPK()
+
+	if nq <= 0 || topk <= 0 {
+		return []float32{}
+	}
+
+	// TODO: how could we avoid memory copy every time when this called
+	distances := make([]float32, nq*topk)
+	C.GetDistancesOfQueryResult(qs.ptr, (*C.float)(&distances[0]))
+
+	return distances
+}
+
 type Index interface {
 	Serialize() ([]*Blob, error)
 	Load([]*Blob) error
@@ -146,8 +232,6 @@ func (index *CIndex) Delete() error {
 }
 
 func NewCIndex(typeParams, indexParams map[string]string) (Index, error) {
-	fmt.Println("NNNNNNNNNNNNNNNNNNNNNNNNNNN typeParams: ", typeParams)
-	fmt.Println("NNNNNNNNNNNNNNNNNNNNNNNNNNN indexParams: ", indexParams)
 	protoTypeParams := &indexcgopb.TypeParams{
 		Params: make([]*commonpb.KeyValuePair, 0),
 	}

internal/indexnode/indexnode.go

@@ -4,21 +4,19 @@ import (
 	"context"
 	"log"
 	"math/rand"
-	"net"
-	"strconv"
-	"sync"
 	"time"
 
-	"github.com/zilliztech/milvus-distributed/internal/allocator"
+	"github.com/zilliztech/milvus-distributed/internal/errors"
 	"github.com/zilliztech/milvus-distributed/internal/kv"
-	etcdkv "github.com/zilliztech/milvus-distributed/internal/kv/etcd"
 	miniokv "github.com/zilliztech/milvus-distributed/internal/kv/minio"
 	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
 	"github.com/zilliztech/milvus-distributed/internal/proto/indexpb"
-	"github.com/zilliztech/milvus-distributed/internal/proto/internalpb2"
+	"github.com/zilliztech/milvus-distributed/internal/util/retry"
 	"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
-	"go.etcd.io/etcd/clientv3"
-	"google.golang.org/grpc"
+)
+
+const (
+	reqTimeoutInterval = time.Second * 10
 )
 
 type UniqueID = typeutil.UniqueID
@@ -27,97 +25,16 @@ type Timestamp = typeutil.Timestamp
 type IndexNode struct {
 	loopCtx    context.Context
 	loopCancel func()
-	loopWg     sync.WaitGroup
 
-	grpcServer *grpc.Server
-	sched      *TaskScheduler
-
-	idAllocator *allocator.IDAllocator
+	sched *TaskScheduler
 
 	kv kv.Base
 
-	metaTable *metaTable
+	serviceClient typeutil.IndexServiceInterface // method factory
+
 	// Add callback functions at different stages
 	startCallbacks []func()
 	closeCallbacks []func()
-
-	indexNodeID int64
-	//serviceClient indexservice.Interface // method factory
-}
-
-func (i *IndexNode) Init() error {
-	panic("implement me")
-}
-
-func (i *IndexNode) Start() error {
-	panic("implement me")
-}
-
-func (i *IndexNode) Stop() error {
-	panic("implement me")
-}
-
-func (i *IndexNode) GetComponentStates() (*internalpb2.ComponentStates, error) {
-	panic("implement me")
-}
-
-func (i *IndexNode) GetTimeTickChannel() (string, error) {
-	panic("implement me")
-}
-
-func (i *IndexNode) GetStatisticsChannel() (string, error) {
-	panic("implement me")
-}
-
-func (i *IndexNode) BuildIndex(req *indexpb.BuildIndexCmd) (*commonpb.Status, error) {
-
-	log.Println("Create index with indexID=", req.IndexID)
-	return &commonpb.Status{
-		ErrorCode: commonpb.ErrorCode_SUCCESS,
-		Reason:    "",
-	}, nil
-}
-
-func CreateIndexNode(ctx context.Context) (*IndexNode, error) {
-	return &IndexNode{}, nil
-}
-
-func NewIndexNode(ctx context.Context, nodeID int64) *IndexNode {
-	ctx1, cancel := context.WithCancel(ctx)
-	in := &IndexNode{
-		loopCtx:    ctx1,
-		loopCancel: cancel,
-
-		indexNodeID: nodeID,
-	}
-
-	return in
-}
-
-type Builder struct {
-	loopCtx    context.Context
-	loopCancel func()
-	loopWg     sync.WaitGroup
-
-	grpcServer *grpc.Server
-	sched      *TaskScheduler
-
-	idAllocator *allocator.IDAllocator
-
-	kv kv.Base
-
-	metaTable *metaTable
-	// Add callback functions at different stages
-	startCallbacks []func()
-	closeCallbacks []func()
-}
-
-func (b *Builder) RegisterNode(ctx context.Context, request *indexpb.RegisterNodeRequest) (*indexpb.RegisterNodeResponse, error) {
-	panic("implement me")
-}
-
-func (b *Builder) NotifyBuildIndex(ctx context.Context, notification *indexpb.BuildIndexNotification) (*commonpb.Status, error) {
-	panic("implement me")
 }
 
 func Init() {
@@ -125,35 +42,13 @@ func Init() {
 	Params.Init()
 }
 
-func CreateBuilder(ctx context.Context) (*Builder, error) {
+func CreateIndexNode(ctx context.Context) (*IndexNode, error) {
 	ctx1, cancel := context.WithCancel(ctx)
-	b := &Builder{
+	b := &IndexNode{
 		loopCtx:    ctx1,
 		loopCancel: cancel,
 	}
 
-	connectEtcdFn := func() error {
-		etcdAddress := Params.EtcdAddress
-		etcdClient, err := clientv3.New(clientv3.Config{Endpoints: []string{etcdAddress}})
-		if err != nil {
-			return err
-		}
-		etcdKV := etcdkv.NewEtcdKV(etcdClient, Params.MetaRootPath)
-		metakv, err := NewMetaTable(etcdKV)
-		if err != nil {
-			return err
-		}
-		b.metaTable = metakv
-		return nil
-	}
-	err := Retry(10, time.Millisecond*200, connectEtcdFn)
-	if err != nil {
-		return nil, err
-	}
-
-	idAllocator, err := allocator.NewIDAllocator(b.loopCtx, Params.MasterAddress)
-	b.idAllocator = idAllocator
-
 	connectMinIOFn := func() error {
 		option := &miniokv.Option{
 			Address:           Params.MinIOAddress,
@@ -163,19 +58,22 @@ func CreateBuilder(ctx context.Context) (*Builder, error) {
 			BucketName:        Params.MinioBucketName,
 			CreateBucket:      true,
 		}
-
+		var err error
 		b.kv, err = miniokv.NewMinIOKV(b.loopCtx, option)
 		if err != nil {
 			return err
 		}
 		return nil
 	}
-	err = Retry(10, time.Millisecond*200, connectMinIOFn)
+
+	err := retry.Retry(10, time.Millisecond*200, connectMinIOFn)
 	if err != nil {
 		return nil, err
 	}
 
-	b.sched, err = NewTaskScheduler(b.loopCtx, b.idAllocator, b.kv, b.metaTable)
+	log.Println("loopctx = ", b.loopCtx)
+	b.sched, err = NewTaskScheduler(b.loopCtx, b.kv)
+	log.Println("err = ", err)
 	if err != nil {
 		return nil, err
 	}
@@ -184,70 +82,86 @@ func CreateBuilder(ctx context.Context) (*Builder, error) {
 }
 
 // AddStartCallback adds a callback in the startServer phase.
-func (b *Builder) AddStartCallback(callbacks ...func()) {
-	b.startCallbacks = append(b.startCallbacks, callbacks...)
+func (i *IndexNode) AddStartCallback(callbacks ...func()) {
+	i.startCallbacks = append(i.startCallbacks, callbacks...)
 }
 
-func (b *Builder) startBuilder() error {
+// AddCloseCallback adds a callback in the Close phase.
+func (i *IndexNode) AddCloseCallback(callbacks ...func()) {
+	i.closeCallbacks = append(i.closeCallbacks, callbacks...)
+}
 
-	b.sched.Start()
+func (i *IndexNode) Init() error {
+	return nil
+}
+
+func (i *IndexNode) Start() error {
+	i.sched.Start()
 
 	// Start callbacks
-	for _, cb := range b.startCallbacks {
+	for _, cb := range i.startCallbacks {
 		cb()
 	}
 
-	b.idAllocator.Start()
-
-	b.loopWg.Add(1)
-	go b.grpcLoop()
-
 	return nil
 }
 
-// AddCloseCallback adds a callback in the Close phase.
-func (b *Builder) AddCloseCallback(callbacks ...func()) {
-	b.closeCallbacks = append(b.closeCallbacks, callbacks...)
-}
+func (i *IndexNode) stopIndexNodeLoop() {
+	i.loopCancel()
 
-func (b *Builder) grpcLoop() {
-	defer b.loopWg.Done()
-
-	lis, err := net.Listen("tcp", ":"+strconv.Itoa(Params.Port))
-	if err != nil {
-		log.Fatalf("Builder grpc server fatal error=%v", err)
-	}
-
-	b.grpcServer = grpc.NewServer()
-	indexpb.RegisterIndexServiceServer(b.grpcServer, b)
-	if err = b.grpcServer.Serve(lis); err != nil {
-		log.Fatalf("Builder grpc server fatal error=%v", err)
-	}
-}
-
-func (b *Builder) Start() error {
-	return b.startBuilder()
-}
-
-func (b *Builder) stopBuilderLoop() {
-	b.loopCancel()
-
-	b.idAllocator.Close()
-
-	if b.grpcServer != nil {
-		b.grpcServer.GracefulStop()
-	}
-
-	b.sched.Close()
-	b.loopWg.Wait()
+	i.sched.Close()
 }
 
 // Close closes the server.
-func (b *Builder) Close() {
-	b.stopBuilderLoop()
+func (i *IndexNode) Stop() error {
+	i.stopIndexNodeLoop()
 
-	for _, cb := range b.closeCallbacks {
+	for _, cb := range i.closeCallbacks {
 		cb()
 	}
-	log.Print("builder  closed.")
+	log.Print("IndexNode  closed.")
+	return nil
+}
+
+func (i *IndexNode) SetServiceClient(serviceClient typeutil.IndexServiceInterface) {
+	i.serviceClient = serviceClient
+}
+
+func (i *IndexNode) BuildIndex(request *indexpb.BuildIndexCmd) (*commonpb.Status, error) {
+	t := newIndexBuildTask()
+	t.cmd = request
+	t.kv = i.kv
+	t.serviceClient = i.serviceClient
+	log.Println("t.serviceClient = ", t.serviceClient)
+	t.nodeID = Params.NodeID
+	ctx, cancel := context.WithTimeout(context.Background(), reqTimeoutInterval)
+	defer cancel()
+
+	fn := func() error {
+		select {
+		case <-ctx.Done():
+			return errors.New("Enqueue BuildQueue timeout")
+		default:
+			return i.sched.IndexBuildQueue.Enqueue(t)
+		}
+	}
+	ret := &commonpb.Status{
+		ErrorCode: commonpb.ErrorCode_SUCCESS,
+	}
+
+	err := fn()
+	if err != nil {
+		ret.ErrorCode = commonpb.ErrorCode_UNEXPECTED_ERROR
+		ret.Reason = err.Error()
+		return ret, nil
+	}
+	log.Println("index scheduler successfully with indexID = ", request.IndexID)
+	err = t.WaitToFinish()
+	log.Println("build index finish ...err = ", err)
+	if err != nil {
+		ret.ErrorCode = commonpb.ErrorCode_UNEXPECTED_ERROR
+		ret.Reason = err.Error()
+		return ret, nil
+	}
+	return ret, nil
 }

internal/indexnode/indexnode_test.go

@@ -7,24 +7,16 @@ import (
 	"os"
 	"strconv"
 	"testing"
-	"time"
 
-	"go.etcd.io/etcd/clientv3"
-
-	"go.uber.org/zap"
-
-	"github.com/stretchr/testify/assert"
-	indexnodeclient "github.com/zilliztech/milvus-distributed/internal/indexnode/client"
 	"github.com/zilliztech/milvus-distributed/internal/master"
-	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
+	"go.etcd.io/etcd/clientv3"
+	"go.uber.org/zap"
 )
 
 var ctx context.Context
 var cancel func()
 
-var buildClient *indexnodeclient.Client
-
-var builderServer *Builder
+var buildClient *IndexNode
 
 var masterPort = 53101
 var masterServer *master.Master
@@ -70,13 +62,13 @@ func startMaster(ctx context.Context) {
 
 func startBuilder(ctx context.Context) {
 	var err error
-	builderServer, err = CreateBuilder(ctx)
+	buildClient, err = CreateIndexNode(ctx)
 	if err != nil {
 		log.Print("create builder failed", zap.Error(err))
 	}
 
 	// TODO: change to wait until master is ready
-	if err := builderServer.Start(); err != nil {
+	if err := buildClient.Start(); err != nil {
 		log.Fatal("run builder failed", zap.Error(err))
 	}
 }
@@ -86,18 +78,11 @@ func setup() {
 	ctx, cancel = context.WithCancel(context.Background())
 	startMaster(ctx)
 	startBuilder(ctx)
-	addr := Params.Address
-	var err error
-	buildClient, err = indexnodeclient.NewBuildIndexClient(ctx, addr)
-	if err != nil {
-		panic("Create buildClient Failed!")
-	}
-
 }
 
 func shutdown() {
 	cancel()
-	builderServer.Close()
+	buildClient.Stop()
 	masterServer.Close()
 }
 
@@ -108,23 +93,23 @@ func TestMain(m *testing.M) {
 	os.Exit(code)
 }
 
-func TestBuilder_GRPC(t *testing.T) {
-	typeParams := make(map[string]string)
-	typeParams["a"] = "1"
-	indexParams := make(map[string]string)
-	indexParams["b"] = "2"
-	columnDataPaths := []string{"dataA", "dataB"}
-	indexID, err := buildClient.BuildIndex(columnDataPaths, typeParams, indexParams)
-	assert.Nil(t, err)
-
-	time.Sleep(time.Second * 3)
-
-	description, err := buildClient.GetIndexStates([]UniqueID{indexID})
-	assert.Nil(t, err)
-	assert.Equal(t, commonpb.IndexState_INPROGRESS, description.States[0].State)
-	assert.Equal(t, indexID, description.States[0].IndexID)
-
-	indexDataPaths, err := buildClient.GetIndexFilePaths([]UniqueID{indexID})
-	assert.Nil(t, err)
-	assert.Nil(t, indexDataPaths[0])
-}
+//func TestBuilder_GRPC(t *testing.T) {
+//	typeParams := make(map[string]string)
+//	typeParams["a"] = "1"
+//	indexParams := make(map[string]string)
+//	indexParams["b"] = "2"
+//	columnDataPaths := []string{"dataA", "dataB"}
+//	indexID, err := buildClient.BuildIndex(columnDataPaths, typeParams, indexParams)
+//	assert.Nil(t, err)
+//
+//	time.Sleep(time.Second * 3)
+//
+//	description, err := buildClient.GetIndexStates([]UniqueID{indexID})
+//	assert.Nil(t, err)
+//	assert.Equal(t, commonpb.IndexState_INPROGRESS, description.States[0].State)
+//	assert.Equal(t, indexID, description.States[0].IndexID)
+//
+//	indexDataPaths, err := buildClient.GetIndexFilePaths([]UniqueID{indexID})
+//	assert.Nil(t, err)
+//	assert.Nil(t, indexDataPaths[0])
+//}

internal/indexnode/interface.go

@@ -1,13 +0,0 @@
-package indexnode
-
-import (
-	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
-	"github.com/zilliztech/milvus-distributed/internal/proto/indexpb"
-	"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
-)
-
-type ServiceBase = typeutil.Component
-
-type Interface interface {
-	BuildIndex(req *indexpb.BuildIndexCmd) (*commonpb.Status, error)
-}

internal/indexnode/meta_table.go

@@ -1,171 +0,0 @@
-package indexnode
-
-import (
-	"fmt"
-	"strconv"
-	"sync"
-	"time"
-
-	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
-
-	"github.com/golang/protobuf/proto"
-	"github.com/zilliztech/milvus-distributed/internal/errors"
-	"github.com/zilliztech/milvus-distributed/internal/kv"
-	pb "github.com/zilliztech/milvus-distributed/internal/proto/indexpb"
-)
-
-type metaTable struct {
-	client       kv.TxnBase                // client of a reliable kv service, i.e. etcd client
-	indexID2Meta map[UniqueID]pb.IndexMeta // index id to index meta
-
-	lock sync.RWMutex
-}
-
-func NewMetaTable(kv kv.TxnBase) (*metaTable, error) {
-	mt := &metaTable{
-		client: kv,
-		lock:   sync.RWMutex{},
-	}
-	err := mt.reloadFromKV()
-	if err != nil {
-		return nil, err
-	}
-	return mt, nil
-}
-
-func (mt *metaTable) reloadFromKV() error {
-	mt.indexID2Meta = make(map[UniqueID]pb.IndexMeta)
-
-	_, values, err := mt.client.LoadWithPrefix("indexes")
-	if err != nil {
-		return err
-	}
-
-	for _, value := range values {
-		indexMeta := pb.IndexMeta{}
-		err = proto.UnmarshalText(value, &indexMeta)
-		if err != nil {
-			return err
-		}
-		mt.indexID2Meta[indexMeta.IndexID] = indexMeta
-	}
-	return nil
-}
-
-// metaTable.lock.Lock() before call this function
-func (mt *metaTable) saveIndexMeta(meta *pb.IndexMeta) error {
-	value := proto.MarshalTextString(meta)
-
-	mt.indexID2Meta[meta.IndexID] = *meta
-
-	return mt.client.Save("/indexes/"+strconv.FormatInt(meta.IndexID, 10), value)
-}
-
-func (mt *metaTable) AddIndex(indexID UniqueID, req *pb.BuildIndexRequest) error {
-	mt.lock.Lock()
-	defer mt.lock.Unlock()
-	_, ok := mt.indexID2Meta[indexID]
-	if ok {
-		return errors.Errorf("index already exists with ID = " + strconv.FormatInt(indexID, 10))
-	}
-	meta := &pb.IndexMeta{
-		State:   commonpb.IndexState_UNISSUED,
-		IndexID: indexID,
-		Req:     req,
-	}
-	mt.saveIndexMeta(meta)
-	return nil
-}
-
-func (mt *metaTable) UpdateIndexState(indexID UniqueID, state commonpb.IndexState) error {
-	mt.lock.Lock()
-	defer mt.lock.Unlock()
-	meta, ok := mt.indexID2Meta[indexID]
-	if !ok {
-		return errors.Errorf("index not exists with ID = " + strconv.FormatInt(indexID, 10))
-	}
-	meta.State = state
-	mt.saveIndexMeta(&meta)
-	return nil
-}
-
-func (mt *metaTable) UpdateIndexEnqueTime(indexID UniqueID, t time.Time) error {
-	mt.lock.Lock()
-	defer mt.lock.Unlock()
-	meta, ok := mt.indexID2Meta[indexID]
-	if !ok {
-		return errors.Errorf("index not exists with ID = " + strconv.FormatInt(indexID, 10))
-	}
-	meta.EnqueTime = t.UnixNano()
-	mt.saveIndexMeta(&meta)
-	return nil
-}
-
-func (mt *metaTable) UpdateIndexScheduleTime(indexID UniqueID, t time.Time) error {
-	mt.lock.Lock()
-	defer mt.lock.Unlock()
-	meta, ok := mt.indexID2Meta[indexID]
-	if !ok {
-		return errors.Errorf("index not exists with ID = " + strconv.FormatInt(indexID, 10))
-	}
-	meta.ScheduleTime = t.UnixNano()
-	mt.saveIndexMeta(&meta)
-	return nil
-}
-
-func (mt *metaTable) CompleteIndex(indexID UniqueID, dataPaths []string) error {
-	mt.lock.Lock()
-	defer mt.lock.Unlock()
-	meta, ok := mt.indexID2Meta[indexID]
-	if !ok {
-		return errors.Errorf("index not exists with ID = " + strconv.FormatInt(indexID, 10))
-	}
-	meta.State = commonpb.IndexState_FINISHED
-	meta.IndexFilePaths = dataPaths
-	meta.BuildCompleteTime = time.Now().UnixNano()
-	mt.saveIndexMeta(&meta)
-	return nil
-}
-
-func (mt *metaTable) GetIndexStates(indexID UniqueID) (*pb.IndexInfo, error) {
-	mt.lock.Lock()
-	defer mt.lock.Unlock()
-	ret := &pb.IndexInfo{
-		IndexID: indexID,
-		Reason:  "",
-	}
-	meta, ok := mt.indexID2Meta[indexID]
-	if !ok {
-		ret.Reason = "index not exists with ID = " + strconv.FormatInt(indexID, 10)
-		ret.State = commonpb.IndexState_NONE
-		return ret, nil
-	}
-
-	ret.State = meta.State
-	return ret, nil
-}
-
-func (mt *metaTable) GetIndexFilePaths(indexID UniqueID) ([]string, error) {
-	mt.lock.Lock()
-	defer mt.lock.Unlock()
-
-	meta, ok := mt.indexID2Meta[indexID]
-	if !ok {
-		return nil, errors.Errorf("index not exists with ID = " + strconv.FormatInt(indexID, 10))
-	}
-	return meta.IndexFilePaths, nil
-}
-
-func (mt *metaTable) DeleteIndex(indexID UniqueID) error {
-	mt.lock.Lock()
-	defer mt.lock.Unlock()
-
-	indexMeta, ok := mt.indexID2Meta[indexID]
-	if !ok {
-		return errors.Errorf("can't find index. id = " + strconv.FormatInt(indexID, 10))
-	}
-	fmt.Print(indexMeta)
-
-	return nil
-
-}

internal/indexnode/paramtable.go

@@ -53,7 +53,7 @@ func (pt *ParamTable) Init() {
 }
 
 func (pt *ParamTable) initAddress() {
-	addr, err := pt.Load("indexBuilder.address")
+	addr, err := pt.Load("indexServer.address")
 	if err != nil {
 		panic(err)
 	}
@@ -65,7 +65,7 @@ func (pt *ParamTable) initAddress() {
 		}
 	}
 
-	port, err := pt.Load("indexBuilder.port")
+	port, err := pt.Load("indexServer.port")
 	if err != nil {
 		panic(err)
 	}
@@ -78,7 +78,7 @@ func (pt *ParamTable) initAddress() {
 }
 
 func (pt *ParamTable) initPort() {
-	pt.Port = pt.ParseInt("indexBuilder.port")
+	pt.Port = pt.ParseInt("indexServer.port")
 }
 
 func (pt *ParamTable) initIndexServerAddress() {

internal/indexnode/query_result.go

@@ -1,104 +0,0 @@
-package indexnode
-
-/*
-
-#cgo CFLAGS: -I${SRCDIR}/../core/output/include
-
-#cgo LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_indexbuilder -Wl,-rpath=${SRCDIR}/../core/output/lib
-
-#include <stdlib.h>	// free
-#include "segcore/collection_c.h"
-#include "indexbuilder/index_c.h"
-
-*/
-import "C"
-import (
-	"strconv"
-	"unsafe"
-
-	"github.com/zilliztech/milvus-distributed/internal/errors"
-)
-
-type QueryResult interface {
-	Delete() error
-	NQ() int64
-	TOPK() int64
-	IDs() []int64
-	Distances() []float32
-}
-
-type CQueryResult struct {
-	ptr C.CIndexQueryResult
-}
-
-type CFunc func() C.CStatus
-
-func TryCatch(fn CFunc) error {
-	status := fn()
-	errorCode := status.error_code
-	if errorCode != 0 {
-		errorMsg := C.GoString(status.error_msg)
-		defer C.free(unsafe.Pointer(status.error_msg))
-		return errors.New("error code = " + strconv.Itoa(int(errorCode)) + ", error msg = " + errorMsg)
-	}
-	return nil
-}
-
-func CreateQueryResult() (QueryResult, error) {
-	var ptr C.CIndexQueryResult
-	fn := func() C.CStatus {
-		return C.CreateQueryResult(&ptr)
-	}
-	err := TryCatch(fn)
-	if err != nil {
-		return nil, err
-	}
-	return &CQueryResult{
-		ptr: ptr,
-	}, nil
-}
-
-func (qs *CQueryResult) Delete() error {
-	fn := func() C.CStatus {
-		return C.DeleteQueryResult(qs.ptr)
-	}
-	return TryCatch(fn)
-}
-
-func (qs *CQueryResult) NQ() int64 {
-	return int64(C.NqOfQueryResult(qs.ptr))
-}
-
-func (qs *CQueryResult) TOPK() int64 {
-	return int64(C.TopkOfQueryResult(qs.ptr))
-}
-
-func (qs *CQueryResult) IDs() []int64 {
-	nq := qs.NQ()
-	topk := qs.TOPK()
-
-	if nq <= 0 || topk <= 0 {
-		return []int64{}
-	}
-
-	// TODO: how could we avoid memory copy every time when this called
-	ids := make([]int64, nq*topk)
-	C.GetIdsOfQueryResult(qs.ptr, (*C.int64_t)(&ids[0]))
-
-	return ids
-}
-
-func (qs *CQueryResult) Distances() []float32 {
-	nq := qs.NQ()
-	topk := qs.TOPK()
-
-	if nq <= 0 || topk <= 0 {
-		return []float32{}
-	}
-
-	// TODO: how could we avoid memory copy every time when this called
-	distances := make([]float32, nq*topk)
-	C.GetDistancesOfQueryResult(qs.ptr, (*C.float)(&distances[0]))
-
-	return distances
-}

internal/indexnode/task.go

@@ -5,12 +5,12 @@ import (
 	"fmt"
 	"log"
 	"strconv"
-	"time"
 
-	"github.com/zilliztech/milvus-distributed/internal/allocator"
+	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
+	"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
+
 	"github.com/zilliztech/milvus-distributed/internal/errors"
 	"github.com/zilliztech/milvus-distributed/internal/kv"
-	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
 	"github.com/zilliztech/milvus-distributed/internal/proto/indexpb"
 	"github.com/zilliztech/milvus-distributed/internal/storage"
 )
@@ -27,10 +27,9 @@ type task interface {
 }
 
 type BaseTask struct {
-	done  chan error
-	ctx   context.Context
-	id    UniqueID
-	table *metaTable
+	done chan error
+	ctx  context.Context
+	id   UniqueID
 }
 
 func (bt *BaseTask) ID() UniqueID {
@@ -54,83 +53,22 @@ func (bt *BaseTask) Notify(err error) {
 	bt.done <- err
 }
 
-type IndexAddTask struct {
-	BaseTask
-	req         *indexpb.BuildIndexRequest
-	indexID     UniqueID
-	idAllocator *allocator.IDAllocator
-	buildQueue  TaskQueue
-	kv          kv.Base
-}
-
-func (it *IndexAddTask) SetID(ID UniqueID) {
-	it.BaseTask.setID(ID)
-}
-
-func (it *IndexAddTask) OnEnqueue() error {
-	var err error
-	it.indexID, err = it.idAllocator.AllocOne()
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (it *IndexAddTask) PreExecute() error {
-	log.Println("pretend to check Index Req")
-	err := it.table.AddIndex(it.indexID, it.req)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (it *IndexAddTask) Execute() error {
-	t := newIndexBuildTask()
-	t.table = it.table
-	t.indexID = it.indexID
-	t.kv = it.kv
-	t.req = it.req
-	var cancel func()
-	t.ctx, cancel = context.WithTimeout(it.ctx, reqTimeoutInterval)
-	defer cancel()
-
-	fn := func() error {
-		select {
-		case <-t.ctx.Done():
-			return errors.New("index add timeout")
-		default:
-			return it.buildQueue.Enqueue(t)
-		}
-	}
-	return fn()
-}
-
-func (it *IndexAddTask) PostExecute() error {
-	return nil
-}
-
-func NewIndexAddTask() *IndexAddTask {
-	return &IndexAddTask{
-		BaseTask: BaseTask{
-			done: make(chan error),
-		},
-	}
-}
-
 type IndexBuildTask struct {
 	BaseTask
-	index     Index
-	indexID   UniqueID
-	kv        kv.Base
-	savePaths []string
-	req       *indexpb.BuildIndexRequest
+	index         Index
+	kv            kv.Base
+	savePaths     []string
+	cmd           *indexpb.BuildIndexCmd
+	serviceClient typeutil.IndexServiceInterface
+	nodeID        UniqueID
 }
 
 func newIndexBuildTask() *IndexBuildTask {
+	ctx := context.Background()
 	return &IndexBuildTask{
 		BaseTask: BaseTask{
-			done: make(chan error, 1), // intend to do this
+			ctx:  ctx,
+			done: make(chan error), // intend to do this
 		},
 	}
 }
@@ -140,21 +78,70 @@ func (it *IndexBuildTask) SetID(ID UniqueID) {
 }
 
 func (it *IndexBuildTask) OnEnqueue() error {
-	return it.table.UpdateIndexEnqueTime(it.indexID, time.Now())
+	it.SetID(it.cmd.IndexID)
+	log.Printf("[IndexBuilderTask] Enqueue TaskID: %v", it.ID())
+	return nil
 }
 
 func (it *IndexBuildTask) PreExecute() error {
-	return it.table.UpdateIndexScheduleTime(it.indexID, time.Now())
+	log.Println("preExecute...")
+	return nil
 }
 
-func (it *IndexBuildTask) Execute() error {
-	err := it.table.UpdateIndexState(it.indexID, commonpb.IndexState_INPROGRESS)
-	if err != nil {
+func (it *IndexBuildTask) PostExecute() error {
+	log.Println("PostExecute...")
+	var err error
+	defer func() {
+		if err != nil {
+			it.Rollback()
+		}
+	}()
+
+	if it.serviceClient == nil {
+		err = errors.New("IndexBuildTask, serviceClient is nil")
 		return err
 	}
 
+	nty := &indexpb.BuildIndexNotification{
+		Status: &commonpb.Status{
+			ErrorCode: commonpb.ErrorCode_SUCCESS,
+		},
+		IndexID:        it.cmd.IndexID,
+		NodeID:         it.nodeID,
+		IndexFilePaths: it.savePaths,
+	}
+
+	resp, err := it.serviceClient.NotifyBuildIndex(nty)
+	if err != nil {
+		log.Println("IndexBuildTask notify err:", err.Error())
+		return err
+	}
+
+	if resp.ErrorCode != commonpb.ErrorCode_SUCCESS {
+		err = errors.New(resp.Reason)
+	}
+	return err
+}
+
+func (it *IndexBuildTask) Rollback() error {
+
+	if it.savePaths == nil {
+		return nil
+	}
+
+	err := it.kv.MultiRemove(it.savePaths)
+	if err != nil {
+		log.Println("IndexBuildTask Rollback Failed:", err.Error())
+		return err
+	}
+	return nil
+}
+
+func (it *IndexBuildTask) Execute() error {
+	log.Println("start build index ...")
+	var err error
 	typeParams := make(map[string]string)
-	for _, kvPair := range it.req.GetTypeParams() {
+	for _, kvPair := range it.cmd.Req.GetTypeParams() {
 		key, value := kvPair.GetKey(), kvPair.GetValue()
 		_, ok := typeParams[key]
 		if ok {
@@ -164,7 +151,7 @@ func (it *IndexBuildTask) Execute() error {
 	}
 
 	indexParams := make(map[string]string)
-	for _, kvPair := range it.req.GetIndexParams() {
+	for _, kvPair := range it.cmd.Req.GetIndexParams() {
 		key, value := kvPair.GetKey(), kvPair.GetValue()
 		_, ok := indexParams[key]
 		if ok {
@@ -205,7 +192,7 @@ func (it *IndexBuildTask) Execute() error {
 		return blobs
 	}
 
-	toLoadDataPaths := it.req.GetDataPaths()
+	toLoadDataPaths := it.cmd.Req.GetDataPaths()
 	keys := make([]string, 0)
 	blobs := make([]*Blob, 0)
 	for _, path := range toLoadDataPaths {
@@ -267,7 +254,7 @@ func (it *IndexBuildTask) Execute() error {
 
 		getSavePathByKey := func(key string) string {
 			// TODO: fix me, use more reasonable method
-			return strconv.Itoa(int(it.indexID)) + "/" + strconv.Itoa(int(partitionID)) + "/" + strconv.Itoa(int(segmentID)) + "/" + key
+			return strconv.Itoa(int(it.cmd.IndexID)) + "/" + strconv.Itoa(int(partitionID)) + "/" + strconv.Itoa(int(segmentID)) + "/" + key
 		}
 		saveBlob := func(path string, value []byte) error {
 			return it.kv.Save(path, string(value))
@@ -284,10 +271,9 @@ func (it *IndexBuildTask) Execute() error {
 			it.savePaths = append(it.savePaths, savePath)
 		}
 	}
-	it.index.Delete()
+	err = it.index.Delete()
+	if err != nil {
+		log.Print("CIndexDelete Failed")
+	}
 	return nil
 }
-
-func (it *IndexBuildTask) PostExecute() error {
-	return it.table.CompleteIndex(it.indexID, it.savePaths)
-}

internal/indexnode/task_scheduler.go

@@ -7,7 +7,6 @@ import (
 	"log"
 	"sync"
 
-	"github.com/zilliztech/milvus-distributed/internal/allocator"
 	"github.com/zilliztech/milvus-distributed/internal/kv"
 )
 
@@ -66,7 +65,7 @@ func (queue *BaseTaskQueue) FrontUnissuedTask() task {
 	defer queue.utLock.Unlock()
 
 	if queue.unissuedTasks.Len() <= 0 {
-		log.Panic("sorry, but the unissued task list is empty!")
+		log.Println("FrontUnissuedTask sorry, but the unissued task list is empty!")
 		return nil
 	}
 
@@ -78,7 +77,7 @@ func (queue *BaseTaskQueue) PopUnissuedTask() task {
 	defer queue.utLock.Unlock()
 
 	if queue.unissuedTasks.Len() <= 0 {
-		log.Fatal("sorry, but the unissued task list is empty!")
+		log.Println("PopUnissuedtask sorry, but the unissued task list is empty!")
 		return nil
 	}
 
@@ -115,9 +114,6 @@ func (queue *BaseTaskQueue) PopActiveTask(tID UniqueID) task {
 }
 
 func (queue *BaseTaskQueue) Enqueue(t task) error {
-	tID, _ := queue.sched.idAllocator.AllocOne()
-	// log.Printf("[Builder] allocate reqID: %v", tID)
-	t.SetID(tID)
 	err := t.OnEnqueue()
 	if err != nil {
 		return err
@@ -125,33 +121,10 @@ func (queue *BaseTaskQueue) Enqueue(t task) error {
 	return queue.addUnissuedTask(t)
 }
 
-type IndexAddTaskQueue struct {
-	BaseTaskQueue
-	lock sync.Mutex
-}
-
 type IndexBuildTaskQueue struct {
 	BaseTaskQueue
 }
 
-func (queue *IndexAddTaskQueue) Enqueue(t task) error {
-	queue.lock.Lock()
-	defer queue.lock.Unlock()
-	return queue.BaseTaskQueue.Enqueue(t)
-}
-
-func NewIndexAddTaskQueue(sched *TaskScheduler) *IndexAddTaskQueue {
-	return &IndexAddTaskQueue{
-		BaseTaskQueue: BaseTaskQueue{
-			unissuedTasks: list.New(),
-			activeTasks:   make(map[UniqueID]task),
-			maxTaskNum:    1024,
-			utBufChan:     make(chan int, 1024),
-			sched:         sched,
-		},
-	}
-}
-
 func NewIndexBuildTaskQueue(sched *TaskScheduler) *IndexBuildTaskQueue {
 	return &IndexBuildTaskQueue{
 		BaseTaskQueue: BaseTaskQueue{
@@ -165,42 +138,47 @@ func NewIndexBuildTaskQueue(sched *TaskScheduler) *IndexBuildTaskQueue {
 }
 
 type TaskScheduler struct {
-	IndexAddQueue   TaskQueue
 	IndexBuildQueue TaskQueue
 
-	idAllocator *allocator.IDAllocator
-	metaTable   *metaTable
-	kv          kv.Base
-
-	wg     sync.WaitGroup
-	ctx    context.Context
-	cancel context.CancelFunc
+	buildParallel int
+	kv            kv.Base
+	wg            sync.WaitGroup
+	ctx           context.Context
+	cancel        context.CancelFunc
 }
 
 func NewTaskScheduler(ctx context.Context,
-	idAllocator *allocator.IDAllocator,
-	kv kv.Base,
-	table *metaTable) (*TaskScheduler, error) {
+	kv kv.Base) (*TaskScheduler, error) {
 	ctx1, cancel := context.WithCancel(ctx)
 	s := &TaskScheduler{
-		idAllocator: idAllocator,
-		metaTable:   table,
-		kv:          kv,
-		ctx:         ctx1,
-		cancel:      cancel,
+		kv:            kv,
+		ctx:           ctx1,
+		cancel:        cancel,
+		buildParallel: 1, // default value
 	}
-	s.IndexAddQueue = NewIndexAddTaskQueue(s)
 	s.IndexBuildQueue = NewIndexBuildTaskQueue(s)
 
 	return s, nil
 }
 
-func (sched *TaskScheduler) scheduleIndexAddTask() task {
-	return sched.IndexAddQueue.PopUnissuedTask()
+func (sched *TaskScheduler) setParallelism(parallel int) {
+	if parallel <= 0 {
+		log.Println("can not set parallelism to less than zero!")
+		return
+	}
+	sched.buildParallel = parallel
 }
 
-func (sched *TaskScheduler) scheduleIndexBuildTask() task {
-	return sched.IndexBuildQueue.PopUnissuedTask()
+func (sched *TaskScheduler) scheduleIndexBuildTask() []task {
+	ret := make([]task, 0)
+	for i := 0; i < sched.buildParallel; i++ {
+		t := sched.IndexBuildQueue.PopUnissuedTask()
+		if t == nil {
+			return ret
+		}
+		ret = append(ret, t)
+	}
+	return ret
 }
 
 func (sched *TaskScheduler) processTask(t task, q TaskQueue) {
@@ -233,30 +211,26 @@ func (sched *TaskScheduler) processTask(t task, q TaskQueue) {
 }
 
 func (sched *TaskScheduler) indexBuildLoop() {
+	log.Println("index build loop ...")
 	defer sched.wg.Done()
 	for {
 		select {
 		case <-sched.ctx.Done():
+			log.Println("id Done?")
 			return
 		case <-sched.IndexBuildQueue.utChan():
+			log.Println("index build loop ...")
 			if !sched.IndexBuildQueue.utEmpty() {
-				t := sched.scheduleIndexBuildTask()
-				sched.processTask(t, sched.IndexBuildQueue)
-			}
-		}
-	}
-}
-
-func (sched *TaskScheduler) indexAddLoop() {
-	defer sched.wg.Done()
-	for {
-		select {
-		case <-sched.ctx.Done():
-			return
-		case <-sched.IndexAddQueue.utChan():
-			if !sched.IndexAddQueue.utEmpty() {
-				t := sched.scheduleIndexAddTask()
-				go sched.processTask(t, sched.IndexAddQueue)
+				tasks := sched.scheduleIndexBuildTask()
+				var wg sync.WaitGroup
+				for _, t := range tasks {
+					wg.Add(1)
+					go func(group *sync.WaitGroup, t task) {
+						defer group.Done()
+						sched.processTask(t, sched.IndexBuildQueue)
+					}(&wg, t)
+				}
+				wg.Wait()
 			}
 		}
 	}
@@ -264,9 +238,6 @@ func (sched *TaskScheduler) indexAddLoop() {
 
 func (sched *TaskScheduler) Start() error {
 
-	sched.wg.Add(1)
-	go sched.indexAddLoop()
-
 	sched.wg.Add(1)
 	go sched.indexBuildLoop()
 	return nil

internal/indexservice/global_allocator.go

@@ -0,0 +1,118 @@
+package indexservice
+
+import (
+	"log"
+	"sync/atomic"
+	"time"
+
+	"github.com/zilliztech/milvus-distributed/internal/errors"
+	"github.com/zilliztech/milvus-distributed/internal/kv"
+	"github.com/zilliztech/milvus-distributed/internal/util/tsoutil"
+	"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
+	"go.uber.org/zap"
+)
+
+// Allocator is a Timestamp Oracle allocator.
+type Allocator interface {
+	// Initialize is used to initialize a TSO allocator.
+	// It will synchronize TSO with etcd and initialize the
+	// memory for later allocation work.
+	Initialize() error
+	// UpdateTSO is used to update the TSO in memory and the time window in etcd.
+	UpdateTSO() error
+	// SetTSO sets the physical part with given tso. It's mainly used for BR restore
+	// and can not forcibly set the TSO smaller than now.
+	SetTSO(tso uint64) error
+	// GenerateTSO is used to generate a given number of TSOs.
+	// Make sure you have initialized the TSO allocator before calling.
+	GenerateTSO(count uint32) (uint64, error)
+	// Reset is used to reset the TSO allocator.
+	Reset()
+}
+
+// GlobalTSOAllocator is the global single point TSO allocator.
+type GlobalTSOAllocator struct {
+	tso *timestampOracle
+}
+
+// NewGlobalTSOAllocator creates a new global TSO allocator.
+func NewGlobalTSOAllocator(key string, kvBase kv.TxnBase) *GlobalTSOAllocator {
+	var saveInterval = 3 * time.Second
+	return &GlobalTSOAllocator{
+		tso: &timestampOracle{
+			kvBase:        kvBase,
+			saveInterval:  saveInterval,
+			maxResetTSGap: func() time.Duration { return 3 * time.Second },
+			key:           key,
+		},
+	}
+}
+
+// Initialize will initialize the created global TSO allocator.
+func (gta *GlobalTSOAllocator) Initialize() error {
+	return gta.tso.InitTimestamp()
+}
+
+// UpdateTSO is used to update the TSO in memory and the time window in etcd.
+func (gta *GlobalTSOAllocator) UpdateTSO() error {
+	return gta.tso.UpdateTimestamp()
+}
+
+// SetTSO sets the physical part with given tso.
+func (gta *GlobalTSOAllocator) SetTSO(tso uint64) error {
+	return gta.tso.ResetUserTimestamp(tso)
+}
+
+// GenerateTSO is used to generate a given number of TSOs.
+// Make sure you have initialized the TSO allocator before calling.
+func (gta *GlobalTSOAllocator) GenerateTSO(count uint32) (uint64, error) {
+	var physical, logical int64
+	if count == 0 {
+		return 0, errors.New("tso count should be positive")
+	}
+
+	maxRetryCount := 10
+
+	for i := 0; i < maxRetryCount; i++ {
+		current := (*atomicObject)(atomic.LoadPointer(&gta.tso.TSO))
+		if current == nil || current.physical.Equal(typeutil.ZeroTime) {
+			// If it's leader, maybe SyncTimestamp hasn't completed yet
+			log.Println("sync hasn't completed yet, wait for a while")
+			time.Sleep(200 * time.Millisecond)
+			continue
+		}
+
+		physical = current.physical.UnixNano() / int64(time.Millisecond)
+		logical = atomic.AddInt64(&current.logical, int64(count))
+		if logical >= maxLogical {
+			log.Println("logical part outside of max logical interval, please check ntp time",
+				zap.Int("retry-count", i))
+			time.Sleep(UpdateTimestampStep)
+			continue
+		}
+		return tsoutil.ComposeTS(physical, logical), nil
+	}
+	return 0, errors.New("can not get timestamp")
+}
+
+func (gta *GlobalTSOAllocator) Alloc(count uint32) (typeutil.Timestamp, error) {
+	//return gta.tso.SyncTimestamp()
+	start, err := gta.GenerateTSO(count)
+	if err != nil {
+		return typeutil.ZeroTimestamp, err
+	}
+	//ret := make([]typeutil.Timestamp, count)
+	//for i:=uint32(0); i < count; i++{
+	//	ret[i] = start + uint64(i)
+	//}
+	return start, err
+}
+
+func (gta *GlobalTSOAllocator) AllocOne() (typeutil.Timestamp, error) {
+	return gta.GenerateTSO(1)
+}
+
+// Reset is used to reset the TSO allocator.
+func (gta *GlobalTSOAllocator) Reset() {
+	gta.tso.ResetTimestamp()
+}

internal/indexservice/id.go

@@ -0,0 +1,52 @@
+package indexservice
+
+import (
+	"github.com/zilliztech/milvus-distributed/internal/kv"
+)
+
+type IDAllocator interface {
+	Alloc(count uint32) (UniqueID, UniqueID, error)
+	AllocOne() (UniqueID, error)
+	UpdateID() error
+}
+
+// GlobalTSOAllocator is the global single point TSO allocator.
+type GlobalIDAllocator struct {
+	allocator Allocator
+}
+
+func NewGlobalIDAllocator(key string, base kv.TxnBase) *GlobalIDAllocator {
+	return &GlobalIDAllocator{
+		allocator: NewGlobalTSOAllocator(key, base),
+	}
+}
+
+// Initialize will initialize the created global TSO allocator.
+func (gia *GlobalIDAllocator) Initialize() error {
+	return gia.allocator.Initialize()
+}
+
+// GenerateTSO is used to generate a given number of TSOs.
+// Make sure you have initialized the TSO allocator before calling.
+func (gia *GlobalIDAllocator) Alloc(count uint32) (UniqueID, UniqueID, error) {
+	timestamp, err := gia.allocator.GenerateTSO(count)
+	if err != nil {
+		return 0, 0, err
+	}
+	idStart := UniqueID(timestamp)
+	idEnd := idStart + int64(count)
+	return idStart, idEnd, nil
+}
+
+func (gia *GlobalIDAllocator) AllocOne() (UniqueID, error) {
+	timestamp, err := gia.allocator.GenerateTSO(1)
+	if err != nil {
+		return 0, err
+	}
+	idStart := UniqueID(timestamp)
+	return idStart, nil
+}
+
+func (gia *GlobalIDAllocator) UpdateID() error {
+	return gia.allocator.UpdateTSO()
+}

internal/indexservice/indexservice.go

@@ -3,41 +3,45 @@ package indexservice
 import (
 	"context"
 	"log"
-	"strconv"
 	"sync"
 	"time"
 
-	"github.com/zilliztech/milvus-distributed/internal/allocator"
-	grpcindexnodeclient "github.com/zilliztech/milvus-distributed/internal/distributed/indexnode/client"
 	"github.com/zilliztech/milvus-distributed/internal/errors"
-	"github.com/zilliztech/milvus-distributed/internal/indexnode"
 	"github.com/zilliztech/milvus-distributed/internal/kv"
 	etcdkv "github.com/zilliztech/milvus-distributed/internal/kv/etcd"
+	miniokv "github.com/zilliztech/milvus-distributed/internal/kv/minio"
 	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
 	"github.com/zilliztech/milvus-distributed/internal/proto/indexpb"
 	"github.com/zilliztech/milvus-distributed/internal/proto/internalpb2"
+	"github.com/zilliztech/milvus-distributed/internal/util/retry"
+	"github.com/zilliztech/milvus-distributed/internal/util/tsoutil"
 	"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
+
 	"go.etcd.io/etcd/clientv3"
-	"google.golang.org/grpc"
+)
+
+const (
+	reqTimeoutInterval = time.Second * 10
 )
 
 type IndexService struct {
-	// implement Service
+	nodeClients *PriorityQueue
 
-	nodeClients []indexnode.Interface
-	// factory method
+	//factory method
 	loopCtx    context.Context
 	loopCancel func()
 	loopWg     sync.WaitGroup
 
-	grpcServer *grpc.Server
-	nodeNum    int64
+	sched *TaskScheduler
 
-	idAllocator *allocator.IDAllocator
+	idAllocator *GlobalIDAllocator
 
 	kv kv.Base
 
 	metaTable *metaTable
+
+	nodeLock sync.RWMutex
+
 	// Add callback functions at different stages
 	startCallbacks []func()
 	closeCallbacks []func()
@@ -46,141 +50,17 @@ type IndexService struct {
 type UniqueID = typeutil.UniqueID
 type Timestamp = typeutil.Timestamp
 
-func (i *IndexService) Init() error {
-	panic("implement me")
-}
-
-func (i *IndexService) Start() error {
-	panic("implement me")
-}
-
-func (i *IndexService) Stop() error {
-	panic("implement me")
-}
-
-func (i *IndexService) GetComponentStates() (*internalpb2.ComponentStates, error) {
-	panic("implement me")
-}
-
-func (i *IndexService) GetTimeTickChannel() (string, error) {
-	panic("implement me")
-}
-
-func (i *IndexService) GetStatisticsChannel() (string, error) {
-	panic("implement me")
-}
-
-func (i *IndexService) RegisterNode(req *indexpb.RegisterNodeRequest) (*indexpb.RegisterNodeResponse, error) {
-	nodeID := i.nodeNum + 1
-
-	//TODO: update meta table
-	_, ok := i.metaTable.nodeID2Address[nodeID]
-	if ok {
-		log.Fatalf("Register IndexNode fatal, IndexNode has already exists with nodeID=%d", nodeID)
-	}
-
-	log.Println("this is register indexNode func")
-	i.metaTable.nodeID2Address[nodeID] = req.Address
-
-	var params []*commonpb.KeyValuePair
-	params = append(params, &commonpb.KeyValuePair{Key: "minio.address", Value: Params.MinIOAddress})
-	params = append(params, &commonpb.KeyValuePair{Key: "minio.accessKeyID", Value: Params.MinIOAccessKeyID})
-	params = append(params, &commonpb.KeyValuePair{Key: "minio.secretAccessKey", Value: Params.MinIOSecretAccessKey})
-	params = append(params, &commonpb.KeyValuePair{Key: "minio.useSSL", Value: strconv.FormatBool(Params.MinIOUseSSL)})
-	params = append(params, &commonpb.KeyValuePair{Key: "minio.bucketName", Value: Params.MinioBucketName})
-
-	i.nodeNum++
-
-	nodeAddress := req.Address.Ip + ":" + strconv.FormatInt(req.Address.Port, 10)
-	log.Println(nodeAddress)
-	nodeClient := grpcindexnodeclient.NewClient(nodeAddress)
-	i.nodeClients = append(i.nodeClients, nodeClient)
-
-	return &indexpb.RegisterNodeResponse{
-		InitParams: &internalpb2.InitParams{
-			NodeID:      nodeID,
-			StartParams: params,
-		},
-	}, nil
-}
-
-func (i *IndexService) BuildIndex(req *indexpb.BuildIndexRequest) (*indexpb.BuildIndexResponse, error) {
-
-	//TODO: Allocator ID
-	indexID := int64(0)
-	log.Println("Build index, indexID = ", indexID)
-	nodeClient := i.nodeClients[0]
-	request := &indexpb.BuildIndexCmd{
-		IndexID: indexID,
-		Req:     req,
-	}
-	status, err := nodeClient.BuildIndex(request)
-	return &indexpb.BuildIndexResponse{
-		Status:  status,
-		IndexID: indexID,
-	}, err
-}
-
-func (i *IndexService) GetIndexStates(req *indexpb.IndexStatesRequest) (*indexpb.IndexStatesResponse, error) {
-	var indexStates []*indexpb.IndexInfo
-	for _, indexID := range req.IndexID {
-		indexState := &indexpb.IndexInfo{
-			IndexID: indexID,
-			State:   commonpb.IndexState_NONE,
-			Reason:  "",
-		}
-		meta, ok := i.metaTable.indexID2Meta[indexID]
-		if !ok {
-			indexState.State = commonpb.IndexState_NONE
-			indexState.Reason = "index does not exists with ID = " + strconv.FormatInt(indexID, 10)
-		} else {
-			indexState.State = meta.State
-		}
-		indexStates = append(indexStates, indexState)
-	}
-	ret := &indexpb.IndexStatesResponse{
-		Status: &commonpb.Status{
-			ErrorCode: commonpb.ErrorCode_SUCCESS,
-			Reason:    "",
-		},
-		States: indexStates,
-	}
-
-	return ret, nil
-}
-
-func (i *IndexService) GetIndexFilePaths(req *indexpb.IndexFilePathsRequest) (*indexpb.IndexFilePathsResponse, error) {
-	panic("implement me")
-}
-
-func (i *IndexService) NotifyBuildIndex(nty *indexpb.BuildIndexNotification) (*commonpb.Status, error) {
-	//TODO: Multiple indexes are building successfully at same time.
-	meta, ok := i.metaTable.indexID2Meta[nty.IndexID]
-	if !ok {
-		return &commonpb.Status{
-			ErrorCode: commonpb.ErrorCode_BUILD_INDEX_ERROR,
-			Reason:    "index already exists with ID = " + strconv.FormatInt(nty.IndexID, 10),
-		}, errors.Errorf("index already exists with ID = " + strconv.FormatInt(nty.IndexID, 10))
-	}
-
-	meta.State = commonpb.IndexState_FINISHED
-	meta.IndexFilePaths = nty.IndexFilePaths
-	return &commonpb.Status{
-		ErrorCode: commonpb.ErrorCode_SUCCESS,
-		Reason:    "",
-	}, nil
-}
-
-func NewIndexServiceImpl(ctx context.Context) *IndexService {
-	Params.Init()
+func CreateIndexService(ctx context.Context) (*IndexService, error) {
 	ctx1, cancel := context.WithCancel(ctx)
-	s := &IndexService{
-		loopCtx:    ctx1,
-		loopCancel: cancel,
+	i := &IndexService{
+		loopCtx:     ctx1,
+		loopCancel:  cancel,
+		nodeClients: &PriorityQueue{},
 	}
 
+	etcdAddress := Params.EtcdAddress
+	log.Println("etcd address = ", etcdAddress)
 	connectEtcdFn := func() error {
-		etcdAddress := Params.EtcdAddress
 		etcdClient, err := clientv3.New(clientv3.Config{Endpoints: []string{etcdAddress}})
 		if err != nil {
 			return err
@@ -190,14 +70,178 @@ func NewIndexServiceImpl(ctx context.Context) *IndexService {
 		if err != nil {
 			return err
 		}
-		s.metaTable = metakv
+		i.metaTable = metakv
 		return nil
 	}
-	err := Retry(10, time.Millisecond*200, connectEtcdFn)
+	err := retry.Retry(10, time.Millisecond*200, connectEtcdFn)
 	if err != nil {
-		return nil
+		return nil, err
 	}
 
-	s.nodeNum = 0
-	return s
+	//init idAllocator
+	kvRootPath := Params.KvRootPath
+	i.idAllocator = NewGlobalIDAllocator("idTimestamp", tsoutil.NewTSOKVBase([]string{etcdAddress}, kvRootPath, "index_gid"))
+	if err := i.idAllocator.Initialize(); err != nil {
+		return nil, err
+	}
+
+	connectMinIOFn := func() error {
+		option := &miniokv.Option{
+			Address:           Params.MinIOAddress,
+			AccessKeyID:       Params.MinIOAccessKeyID,
+			SecretAccessKeyID: Params.MinIOSecretAccessKey,
+			UseSSL:            Params.MinIOUseSSL,
+			BucketName:        Params.MinioBucketName,
+			CreateBucket:      true,
+		}
+
+		i.kv, err = miniokv.NewMinIOKV(i.loopCtx, option)
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+	err = retry.Retry(10, time.Millisecond*200, connectMinIOFn)
+	if err != nil {
+		return nil, err
+	}
+
+	i.sched, err = NewTaskScheduler(i.loopCtx, i.idAllocator, i.kv, i.metaTable)
+	if err != nil {
+		return nil, err
+	}
+
+	return i, nil
+}
+
+func (i *IndexService) Init() error {
+	return nil
+}
+
+func (i *IndexService) Start() error {
+	i.sched.Start()
+	// Start callbacks
+	for _, cb := range i.startCallbacks {
+		cb()
+	}
+	log.Print("IndexService  closed.")
+
+	return nil
+}
+
+func (i *IndexService) Stop() error {
+	i.loopCancel()
+	i.sched.Close()
+	for _, cb := range i.closeCallbacks {
+		cb()
+	}
+	return nil
+}
+
+func (i *IndexService) GetComponentStates() (*internalpb2.ComponentStates, error) {
+	panic("implement me")
+}
+
+func (i *IndexService) GetTimeTickChannel() (string, error) {
+	return "", nil
+}
+
+func (i *IndexService) GetStatisticsChannel() (string, error) {
+	panic("implement me")
+}
+
+func (i *IndexService) BuildIndex(req *indexpb.BuildIndexRequest) (*indexpb.BuildIndexResponse, error) {
+	ret := &indexpb.BuildIndexResponse{
+		Status: &commonpb.Status{
+			ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
+		},
+	}
+	t := NewIndexAddTask()
+	t.req = req
+	t.idAllocator = i.idAllocator
+	t.table = i.metaTable
+	t.kv = i.kv
+
+	if i.nodeClients == nil || i.nodeClients.Len() <= 0 {
+		ret.Status.Reason = "IndexBuilding Service not available"
+		return ret, nil
+	}
+	t.nodeClients = i.nodeClients
+
+	var cancel func()
+	ctx := context.Background()
+	t.ctx, cancel = context.WithTimeout(ctx, reqTimeoutInterval)
+	defer cancel()
+
+	fn := func() error {
+		select {
+		case <-ctx.Done():
+			return errors.New("IndexAddQueue enqueue timeout")
+		default:
+			return i.sched.IndexAddQueue.Enqueue(t)
+		}
+	}
+
+	err := fn()
+	if err != nil {
+		ret.Status.ErrorCode = commonpb.ErrorCode_UNEXPECTED_ERROR
+		ret.Status.Reason = err.Error()
+		return ret, nil
+	}
+
+	err = t.WaitToFinish()
+	if err != nil {
+		ret.Status.ErrorCode = commonpb.ErrorCode_UNEXPECTED_ERROR
+		ret.Status.Reason = err.Error()
+		return ret, nil
+	}
+	ret.IndexID = t.indexID
+	return ret, nil
+}
+
+func (i *IndexService) GetIndexStates(req *indexpb.IndexStatesRequest) (*indexpb.IndexStatesResponse, error) {
+	var indexStates []*indexpb.IndexInfo
+	for _, indexID := range req.IndexIDs {
+		indexState, err := i.metaTable.GetIndexState(indexID)
+		if err != nil {
+			indexState.Reason = err.Error()
+		}
+		indexStates = append(indexStates, indexState)
+	}
+	ret := &indexpb.IndexStatesResponse{
+		Status: &commonpb.Status{
+			ErrorCode: commonpb.ErrorCode_SUCCESS,
+		},
+		States: indexStates,
+	}
+	return ret, nil
+}
+
+func (i *IndexService) GetIndexFilePaths(req *indexpb.IndexFilePathsRequest) (*indexpb.IndexFilePathsResponse, error) {
+	var indexPaths []*indexpb.IndexFilePathInfo
+
+	for _, indexID := range req.IndexIDs {
+		indexPathInfo, _ := i.metaTable.GetIndexFilePathInfo(indexID)
+		indexPaths = append(indexPaths, indexPathInfo)
+	}
+
+	ret := &indexpb.IndexFilePathsResponse{
+		Status: &commonpb.Status{
+			ErrorCode: commonpb.ErrorCode_SUCCESS,
+		},
+		FilePaths: indexPaths,
+	}
+	return ret, nil
+}
+
+func (i *IndexService) NotifyBuildIndex(nty *indexpb.BuildIndexNotification) (*commonpb.Status, error) {
+	ret := &commonpb.Status{
+		ErrorCode: commonpb.ErrorCode_SUCCESS,
+	}
+	if err := i.metaTable.NotifyBuildIndex(nty); err != nil {
+		ret.ErrorCode = commonpb.ErrorCode_BUILD_INDEX_ERROR
+		ret.Reason = err.Error()
+	}
+	i.nodeClients.IncPriority(nty.NodeID, -1)
+	return ret, nil
 }

internal/indexservice/interface.go

@@ -1,18 +0,0 @@
-package indexservice
-
-import (
-	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
-	"github.com/zilliztech/milvus-distributed/internal/proto/indexpb"
-	"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
-)
-
-type ServiceBase = typeutil.Component
-
-type Interface interface {
-	ServiceBase
-	RegisterNode(req *indexpb.RegisterNodeRequest) (*indexpb.RegisterNodeResponse, error)
-	BuildIndex(req *indexpb.BuildIndexRequest) (*indexpb.BuildIndexResponse, error)
-	GetIndexStates(req *indexpb.IndexStatesRequest) (*indexpb.IndexStatesResponse, error)
-	GetIndexFilePaths(req *indexpb.IndexFilePathsRequest) (*indexpb.IndexFilePathsResponse, error)
-	NotifyBuildIndex(nty *indexpb.BuildIndexNotification) (*commonpb.Status, error)
-}

internal/indexservice/meta_table.go

@@ -15,20 +15,17 @@ import (
 	"fmt"
 	"strconv"
 	"sync"
-	"time"
 
 	"github.com/golang/protobuf/proto"
 	"github.com/zilliztech/milvus-distributed/internal/errors"
 	"github.com/zilliztech/milvus-distributed/internal/kv"
 	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
-	pb "github.com/zilliztech/milvus-distributed/internal/proto/indexpb"
+	"github.com/zilliztech/milvus-distributed/internal/proto/indexpb"
 )
 
 type metaTable struct {
-	client       kv.TxnBase                // client of a reliable kv service, i.e. etcd client
-	indexID2Meta map[UniqueID]pb.IndexMeta // index id to index meta
-
-	nodeID2Address map[int64]*commonpb.Address
+	client       kv.TxnBase                     // client of a reliable kv service, i.e. etcd client
+	indexID2Meta map[UniqueID]indexpb.IndexMeta // index id to index meta
 
 	lock sync.RWMutex
 }
@@ -43,12 +40,11 @@ func NewMetaTable(kv kv.TxnBase) (*metaTable, error) {
 		return nil, err
 	}
 
-	mt.nodeID2Address = make(map[int64]*commonpb.Address)
 	return mt, nil
 }
 
 func (mt *metaTable) reloadFromKV() error {
-	mt.indexID2Meta = make(map[UniqueID]pb.IndexMeta)
+	mt.indexID2Meta = make(map[UniqueID]indexpb.IndexMeta)
 
 	_, values, err := mt.client.LoadWithPrefix("indexes")
 	if err != nil {
@@ -56,7 +52,7 @@ func (mt *metaTable) reloadFromKV() error {
 	}
 
 	for _, value := range values {
-		indexMeta := pb.IndexMeta{}
+		indexMeta := indexpb.IndexMeta{}
 		err = proto.UnmarshalText(value, &indexMeta)
 		if err != nil {
 			return err
@@ -67,7 +63,7 @@ func (mt *metaTable) reloadFromKV() error {
 }
 
 // metaTable.lock.Lock() before call this function
-func (mt *metaTable) saveIndexMeta(meta *pb.IndexMeta) error {
+func (mt *metaTable) saveIndexMeta(meta *indexpb.IndexMeta) error {
 	value := proto.MarshalTextString(meta)
 
 	mt.indexID2Meta[meta.IndexID] = *meta
@@ -75,55 +71,42 @@ func (mt *metaTable) saveIndexMeta(meta *pb.IndexMeta) error {
 	return mt.client.Save("/indexes/"+strconv.FormatInt(meta.IndexID, 10), value)
 }
 
-func (mt *metaTable) AddIndex(indexID UniqueID, req *pb.BuildIndexRequest) error {
+func (mt *metaTable) AddIndex(indexID UniqueID, req *indexpb.BuildIndexRequest) error {
 	mt.lock.Lock()
 	defer mt.lock.Unlock()
 	_, ok := mt.indexID2Meta[indexID]
 	if ok {
 		return errors.Errorf("index already exists with ID = " + strconv.FormatInt(indexID, 10))
 	}
-	meta := &pb.IndexMeta{
-		State:   commonpb.IndexState_INPROGRESS,
+	meta := &indexpb.IndexMeta{
+		State:   commonpb.IndexState_UNISSUED,
 		IndexID: indexID,
 		Req:     req,
 	}
 	return mt.saveIndexMeta(meta)
 }
 
-func (mt *metaTable) UpdateIndexStatus(indexID UniqueID, status commonpb.IndexState) error {
+func (mt *metaTable) NotifyBuildIndex(nty *indexpb.BuildIndexNotification) error {
 	mt.lock.Lock()
 	defer mt.lock.Unlock()
+	indexID := nty.IndexID
 	meta, ok := mt.indexID2Meta[indexID]
 	if !ok {
 		return errors.Errorf("index not exists with ID = " + strconv.FormatInt(indexID, 10))
 	}
-	meta.State = status
-	return mt.saveIndexMeta(&meta)
-}
 
-func (mt *metaTable) UpdateIndexEnqueTime(indexID UniqueID, t time.Time) error {
-	mt.lock.Lock()
-	defer mt.lock.Unlock()
-	meta, ok := mt.indexID2Meta[indexID]
-	if !ok {
-		return errors.Errorf("index not exists with ID = " + strconv.FormatInt(indexID, 10))
+	if nty.Status.ErrorCode != commonpb.ErrorCode_SUCCESS {
+		meta.State = commonpb.IndexState_FAILED
+		meta.FailReason = nty.Status.Reason
+	} else {
+		meta.State = commonpb.IndexState_FINISHED
+		meta.IndexFilePaths = nty.IndexFilePaths
 	}
-	meta.EnqueTime = t.UnixNano()
+
 	return mt.saveIndexMeta(&meta)
 }
 
-func (mt *metaTable) UpdateIndexScheduleTime(indexID UniqueID, t time.Time) error {
-	mt.lock.Lock()
-	defer mt.lock.Unlock()
-	meta, ok := mt.indexID2Meta[indexID]
-	if !ok {
-		return errors.Errorf("index not exists with ID = " + strconv.FormatInt(indexID, 10))
-	}
-	meta.ScheduleTime = t.UnixNano()
-	return mt.saveIndexMeta(&meta)
-}
-
-func (mt *metaTable) NotifyBuildIndex(indexID UniqueID, dataPaths []string, state commonpb.IndexState) error {
+func (mt *metaTable) NotifyBuildIndex2(indexID UniqueID, dataPaths []string, state commonpb.IndexState) error {
 	mt.lock.Lock()
 	defer mt.lock.Unlock()
 
@@ -137,15 +120,35 @@ func (mt *metaTable) NotifyBuildIndex(indexID UniqueID, dataPaths []string, stat
 	return mt.saveIndexMeta(&meta)
 }
 
-func (mt *metaTable) GetIndexFilePaths(indexID UniqueID) ([]string, error) {
+func (mt *metaTable) GetIndexState(indexID UniqueID) (*indexpb.IndexInfo, error) {
 	mt.lock.Lock()
 	defer mt.lock.Unlock()
+	ret := &indexpb.IndexInfo{
+		IndexID: indexID,
+		State:   commonpb.IndexState_NONE,
+	}
+	meta, ok := mt.indexID2Meta[indexID]
+	if !ok {
+		return ret, errors.Errorf("index not exists with ID = " + strconv.FormatInt(indexID, 10))
+	}
+	ret.IndexID = meta.IndexID
+	ret.Reason = meta.FailReason
+	ret.State = meta.State
+	return ret, nil
+}
 
+func (mt *metaTable) GetIndexFilePathInfo(indexID UniqueID) (*indexpb.IndexFilePathInfo, error) {
+	mt.lock.Lock()
+	defer mt.lock.Unlock()
+	ret := &indexpb.IndexFilePathInfo{
+		IndexID: indexID,
+	}
 	meta, ok := mt.indexID2Meta[indexID]
 	if !ok {
 		return nil, errors.Errorf("index not exists with ID = " + strconv.FormatInt(indexID, 10))
 	}
-	return meta.IndexFilePaths, nil
+	ret.IndexFilePaths = meta.IndexFilePaths
+	return ret, nil
 }
 
 func (mt *metaTable) DeleteIndex(indexID UniqueID) error {

internal/indexservice/node_mgr.go

@@ -0,0 +1,79 @@
+package indexservice
+
+import (
+	"strconv"
+
+	grpcindexnodeclient "github.com/zilliztech/milvus-distributed/internal/distributed/indexnode/client"
+	"github.com/zilliztech/milvus-distributed/internal/errors"
+	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
+	"github.com/zilliztech/milvus-distributed/internal/proto/indexpb"
+	"github.com/zilliztech/milvus-distributed/internal/proto/internalpb2"
+)
+
+func (i *IndexService) removeNode(nodeID UniqueID) {
+	i.nodeLock.Lock()
+	defer i.nodeLock.Unlock()
+	i.nodeClients.Remove(nodeID)
+}
+
+func (i *IndexService) addNode(nodeID UniqueID, req *indexpb.RegisterNodeRequest) error {
+	i.nodeLock.Lock()
+	defer i.nodeLock.Unlock()
+
+	if i.nodeClients.CheckAddressExist(req.Address) {
+		errMsg := "Register IndexNode fatal, address conflict with nodeID:%d 's address" + strconv.FormatInt(nodeID, 10)
+		return errors.New(errMsg)
+	}
+
+	nodeAddress := req.Address.Ip + ":" + strconv.FormatInt(req.Address.Port, 10)
+	nodeClient, err := grpcindexnodeclient.NewClient(nodeAddress)
+	if err != nil {
+		return err
+	}
+	item := &PQItem{
+		value:    nodeClient,
+		key:      nodeID,
+		addr:     req.Address,
+		priority: 0,
+	}
+	i.nodeClients.Push(item)
+	return nil
+}
+
+func (i *IndexService) prepareNodeInitParams() []*commonpb.KeyValuePair {
+	var params []*commonpb.KeyValuePair
+	params = append(params, &commonpb.KeyValuePair{Key: "minio.address", Value: Params.MinIOAddress})
+	params = append(params, &commonpb.KeyValuePair{Key: "minio.accessKeyID", Value: Params.MinIOAccessKeyID})
+	params = append(params, &commonpb.KeyValuePair{Key: "minio.secretAccessKey", Value: Params.MinIOSecretAccessKey})
+	params = append(params, &commonpb.KeyValuePair{Key: "minio.useSSL", Value: strconv.FormatBool(Params.MinIOUseSSL)})
+	params = append(params, &commonpb.KeyValuePair{Key: "minio.bucketName", Value: Params.MinioBucketName})
+	return params
+}
+
+func (i *IndexService) RegisterNode(req *indexpb.RegisterNodeRequest) (*indexpb.RegisterNodeResponse, error) {
+	ret := &indexpb.RegisterNodeResponse{
+		Status: &commonpb.Status{
+			ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
+		},
+	}
+
+	nodeID, err := i.idAllocator.AllocOne()
+	if err != nil {
+		ret.Status.Reason = "IndexService:RegisterNode Failed to acquire NodeID"
+		return ret, nil
+	}
+
+	err = i.addNode(nodeID, req)
+	if err != nil {
+		ret.Status.Reason = err.Error()
+		return ret, nil
+	}
+
+	ret.Status.ErrorCode = commonpb.ErrorCode_SUCCESS
+	params := i.prepareNodeInitParams()
+	ret.InitParams = &internalpb2.InitParams{
+		NodeID:      nodeID,
+		StartParams: params,
+	}
+	return ret, nil
+}

internal/indexservice/paramtable.go

@@ -16,6 +16,7 @@ type ParamTable struct {
 	MasterAddress string
 
 	EtcdAddress  string
+	KvRootPath   string
 	MetaRootPath string
 
 	MinIOAddress         string
@@ -34,6 +35,7 @@ func (pt *ParamTable) Init() {
 	pt.initEtcdAddress()
 	pt.initMasterAddress()
 	pt.initMetaRootPath()
+	pt.initKvRootPath()
 	pt.initMinIOAddress()
 	pt.initMinIOAccessKeyID()
 	pt.initMinIOSecretAccessKey()
@@ -90,6 +92,18 @@ func (pt *ParamTable) initMetaRootPath() {
 	pt.MetaRootPath = rootPath + "/" + subPath
 }
 
+func (pt *ParamTable) initKvRootPath() {
+	rootPath, err := pt.Load("etcd.rootPath")
+	if err != nil {
+		panic(err)
+	}
+	subPath, err := pt.Load("etcd.kvSubPath")
+	if err != nil {
+		panic(err)
+	}
+	pt.KvRootPath = rootPath + "/" + subPath
+}
+
 func (pt *ParamTable) initMasterAddress() {
 	ret, err := pt.Load("_MasterAddress")
 	if err != nil {

internal/indexservice/priority_queue.go

@@ -0,0 +1,133 @@
+package indexservice
+
+import (
+	"container/heap"
+	"sync"
+
+	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
+	"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
+)
+
+// An Item is something we manage in a priority queue.
+type PQItem struct {
+	value    typeutil.IndexNodeInterface // The value of the item; arbitrary.
+	key      UniqueID
+	addr     *commonpb.Address
+	priority int // The priority of the item in the queue.
+	// The index is needed by update and is maintained by the heap.Interface methods.
+	index int // The index of the item in the heap.
+}
+
+// A PriorityQueue implements heap.Interface and holds Items.
+
+type PriorityQueue struct {
+	items []*PQItem
+	lock  sync.RWMutex
+}
+
+func (pq *PriorityQueue) Len() int {
+	return len(pq.items)
+}
+
+func (pq *PriorityQueue) Less(i, j int) bool {
+	// We want Pop to give us the highest, not lowest, priority so we use greater than here.
+	return pq.items[i].priority < pq.items[j].priority
+}
+
+func (pq *PriorityQueue) Swap(i, j int) {
+	pq.items[i], pq.items[j] = pq.items[j], pq.items[i]
+	pq.items[i].index = i
+	pq.items[j].index = j
+}
+
+func (pq *PriorityQueue) Push(x interface{}) {
+	pq.lock.Lock()
+	defer pq.lock.Unlock()
+	n := (*pq).Len()
+	item := x.(*PQItem)
+	item.index = n
+	pq.items = append(pq.items, item)
+}
+
+// do not call this directly.
+func (pq *PriorityQueue) Pop() interface{} {
+	old := pq.items
+	n := len(old)
+	item := old[n-1]
+	item.index = -1 // for safety
+	pq.items = old[0 : n-1]
+	return item
+}
+
+func (pq *PriorityQueue) CheckAddressExist(addr *commonpb.Address) bool {
+	pq.lock.RLock()
+	defer pq.lock.RUnlock()
+
+	for _, item := range pq.items {
+		if CompareAddress(addr, item.addr) {
+			return true
+		}
+	}
+	return false
+}
+
+func (pq *PriorityQueue) getItemByKey(key UniqueID) interface{} {
+	var ret interface{} = nil
+	for _, item := range pq.items {
+		if item.key == key {
+			ret = item
+			break
+		}
+	}
+	return ret
+}
+
+// update modifies the priority and value of an Item in the queue.
+func (pq *PriorityQueue) IncPriority(key UniqueID, priority int) {
+	pq.lock.Lock()
+	defer pq.lock.Unlock()
+	item := pq.getItemByKey(key)
+	if item != nil {
+		item.(*PQItem).priority += priority
+		heap.Fix(pq, item.(*PQItem).index)
+	}
+}
+
+// update modifies the priority and value of an Item in the queue.
+func (pq *PriorityQueue) UpdatePriority(key UniqueID, priority int) {
+	pq.lock.Lock()
+	defer pq.lock.Unlock()
+	item := pq.getItemByKey(key)
+	if item != nil {
+		item.(*PQItem).priority = priority
+		heap.Fix(pq, item.(*PQItem).index)
+	}
+}
+
+func (pq *PriorityQueue) Remove(key UniqueID) {
+	pq.lock.Lock()
+	defer pq.lock.Unlock()
+	item := pq.getItemByKey(key)
+	if item != nil {
+		heap.Remove(pq, item.(*PQItem).index)
+	}
+}
+
+func (pq *PriorityQueue) Peek() interface{} {
+	pq.lock.RLock()
+	defer pq.lock.RUnlock()
+	if pq.Len() == 0 {
+		return nil
+	}
+	return pq.items[0]
+	//item := pq.items[0]
+	//return item.value
+}
+
+func (pq *PriorityQueue) PeekClient() (UniqueID, typeutil.IndexNodeInterface) {
+	item := pq.Peek()
+	if item == nil {
+		return UniqueID(-1), nil
+	}
+	return item.(*PQItem).key, item.(*PQItem).value
+}

internal/indexservice/priority_queue_test.go

@@ -0,0 +1,74 @@
+package indexservice
+
+import (
+	"container/heap"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const QueueLen = 10
+
+func newPriorityQueue() *PriorityQueue {
+	ret := &PriorityQueue{}
+	for i := 0; i < QueueLen; i++ {
+		item := &PQItem{
+			value:    nil,
+			key:      UniqueID(i),
+			priority: i,
+			index:    i,
+		}
+		ret.items = append(ret.items, item)
+	}
+	heap.Init(ret)
+	return ret
+}
+
+func TestPriorityQueue_Len(t *testing.T) {
+	pq := newPriorityQueue()
+
+	assert.Equal(t, QueueLen, pq.Len())
+	pq = nil
+}
+
+func TestPriorityQueue_Push(t *testing.T) {
+	pq := newPriorityQueue()
+	for i := 1; i <= QueueLen; i++ {
+		item := &PQItem{
+			key:      UniqueID(i),
+			priority: i,
+			index:    i,
+		}
+		pq.Push(item)
+		assert.Equal(t, i+QueueLen, pq.Len())
+	}
+}
+
+func TestPriorityQueue_Remove(t *testing.T) {
+	pq := newPriorityQueue()
+	cnt := 0
+	for i := 0; i < QueueLen; i++ {
+		if i%2 == 0 {
+			continue
+		}
+		pq.Remove(UniqueID(i))
+		cnt++
+	}
+	assert.Equal(t, QueueLen-cnt, pq.Len())
+}
+
+func TestPriorityQueue_UpdatePriority(t *testing.T) {
+	pq := newPriorityQueue()
+	key := UniqueID(pq.Len() / 2)
+	pq.UpdatePriority(key, -pq.Len())
+	item := pq.Peek()
+	assert.Equal(t, key, item.(*PQItem).key)
+}
+
+func TestPriorityQueue_IncPriority(t *testing.T) {
+	pq := newPriorityQueue()
+	key := UniqueID(pq.Len() / 2)
+	pq.IncPriority(key, -pq.Len())
+	item := pq.Peek()
+	assert.Equal(t, key, item.(*PQItem).key)
+}

internal/indexservice/retry.go

@@ -1,40 +0,0 @@
-package indexservice
-
-import (
-	"log"
-	"time"
-)
-
-// Reference: https://blog.cyeam.com/golang/2018/08/27/retry
-
-func RetryImpl(attempts int, sleep time.Duration, fn func() error, maxSleepTime time.Duration) error {
-	if err := fn(); err != nil {
-		if s, ok := err.(InterruptError); ok {
-			return s.error
-		}
-
-		if attempts--; attempts > 0 {
-			log.Printf("retry func error: %s. attempts #%d after %s.", err.Error(), attempts, sleep)
-			time.Sleep(sleep)
-			if sleep < maxSleepTime {
-				return RetryImpl(attempts, 2*sleep, fn, maxSleepTime)
-			}
-			return RetryImpl(attempts, maxSleepTime, fn, maxSleepTime)
-		}
-		return err
-	}
-	return nil
-}
-
-func Retry(attempts int, sleep time.Duration, fn func() error) error {
-	maxSleepTime := time.Millisecond * 1000
-	return RetryImpl(attempts, sleep, fn, maxSleepTime)
-}
-
-type InterruptError struct {
-	error
-}
-
-func NoRetryError(err error) InterruptError {
-	return InterruptError{err}
-}

internal/indexservice/task.go

@@ -0,0 +1,122 @@
+package indexservice
+
+import (
+	"context"
+	"log"
+
+	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
+	"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
+
+	"github.com/zilliztech/milvus-distributed/internal/errors"
+	"github.com/zilliztech/milvus-distributed/internal/kv"
+	"github.com/zilliztech/milvus-distributed/internal/proto/indexpb"
+)
+
+type task interface {
+	ID() UniqueID       // return ReqID
+	SetID(uid UniqueID) // set ReqID
+	PreExecute() error
+	Execute() error
+	PostExecute() error
+	WaitToFinish() error
+	Notify(err error)
+	OnEnqueue() error
+}
+
+type BaseTask struct {
+	done  chan error
+	ctx   context.Context
+	id    UniqueID
+	table *metaTable
+}
+
+func (bt *BaseTask) ID() UniqueID {
+	return bt.id
+}
+
+func (bt *BaseTask) setID(id UniqueID) {
+	bt.id = id
+}
+
+func (bt *BaseTask) WaitToFinish() error {
+	select {
+	case <-bt.ctx.Done():
+		return errors.New("Task wait to finished timeout")
+	case err := <-bt.done:
+		return err
+	}
+}
+
+func (bt *BaseTask) Notify(err error) {
+	bt.done <- err
+}
+
+type IndexAddTask struct {
+	BaseTask
+	req               *indexpb.BuildIndexRequest
+	indexID           UniqueID
+	idAllocator       *GlobalIDAllocator
+	buildQueue        TaskQueue
+	kv                kv.Base
+	builderClient     typeutil.IndexNodeInterface
+	nodeClients       *PriorityQueue
+	buildClientNodeID UniqueID
+}
+
+func (it *IndexAddTask) SetID(ID UniqueID) {
+	it.BaseTask.setID(ID)
+}
+
+func (it *IndexAddTask) OnEnqueue() error {
+	var err error
+	it.indexID, err = it.idAllocator.AllocOne()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (it *IndexAddTask) PreExecute() error {
+	log.Println("pretend to check Index Req")
+	nodeID, builderClient := it.nodeClients.PeekClient()
+	if builderClient == nil {
+		return errors.New("IndexAddTask Service not available")
+	}
+	it.builderClient = builderClient
+	it.buildClientNodeID = nodeID
+	err := it.table.AddIndex(it.indexID, it.req)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (it *IndexAddTask) Execute() error {
+	cmd := &indexpb.BuildIndexCmd{
+		IndexID: it.indexID,
+		Req:     it.req,
+	}
+	log.Println("before index ...")
+	resp, err := it.builderClient.BuildIndex(cmd)
+	if err != nil {
+		return err
+	}
+	log.Println("build index finish, err = ", err)
+	if resp.ErrorCode != commonpb.ErrorCode_SUCCESS {
+		return errors.New(resp.Reason)
+	}
+	it.nodeClients.IncPriority(it.buildClientNodeID, 1)
+	return nil
+}
+
+func (it *IndexAddTask) PostExecute() error {
+	return nil
+}
+
+func NewIndexAddTask() *IndexAddTask {
+	return &IndexAddTask{
+		BaseTask: BaseTask{
+			done: make(chan error),
+		},
+	}
+}

internal/indexservice/task_scheduler.go

@@ -0,0 +1,241 @@
+package indexservice
+
+import (
+	"container/list"
+	"context"
+	"errors"
+	"log"
+	"sync"
+
+	"github.com/zilliztech/milvus-distributed/internal/kv"
+)
+
+type TaskQueue interface {
+	utChan() <-chan int
+	utEmpty() bool
+	utFull() bool
+	addUnissuedTask(t task) error
+	FrontUnissuedTask() task
+	PopUnissuedTask() task
+	AddActiveTask(t task)
+	PopActiveTask(tID UniqueID) task
+	Enqueue(t task) error
+}
+
+type BaseTaskQueue struct {
+	unissuedTasks *list.List
+	activeTasks   map[UniqueID]task
+	utLock        sync.Mutex
+	atLock        sync.Mutex
+
+	// maxTaskNum should keep still
+	maxTaskNum int64
+
+	utBufChan chan int // to block scheduler
+
+	sched *TaskScheduler
+}
+
+func (queue *BaseTaskQueue) utChan() <-chan int {
+	return queue.utBufChan
+}
+
+func (queue *BaseTaskQueue) utEmpty() bool {
+	return queue.unissuedTasks.Len() == 0
+}
+
+func (queue *BaseTaskQueue) utFull() bool {
+	return int64(queue.unissuedTasks.Len()) >= queue.maxTaskNum
+}
+
+func (queue *BaseTaskQueue) addUnissuedTask(t task) error {
+	queue.utLock.Lock()
+	defer queue.utLock.Unlock()
+
+	if queue.utFull() {
+		return errors.New("task queue is full")
+	}
+	queue.unissuedTasks.PushBack(t)
+	queue.utBufChan <- 1
+	return nil
+}
+
+func (queue *BaseTaskQueue) FrontUnissuedTask() task {
+	queue.utLock.Lock()
+	defer queue.utLock.Unlock()
+
+	if queue.unissuedTasks.Len() <= 0 {
+		log.Panic("sorry, but the unissued task list is empty!")
+		return nil
+	}
+
+	return queue.unissuedTasks.Front().Value.(task)
+}
+
+func (queue *BaseTaskQueue) PopUnissuedTask() task {
+	queue.utLock.Lock()
+	defer queue.utLock.Unlock()
+
+	if queue.unissuedTasks.Len() <= 0 {
+		log.Fatal("sorry, but the unissued task list is empty!")
+		return nil
+	}
+
+	ft := queue.unissuedTasks.Front()
+	queue.unissuedTasks.Remove(ft)
+
+	return ft.Value.(task)
+}
+
+func (queue *BaseTaskQueue) AddActiveTask(t task) {
+	queue.atLock.Lock()
+	defer queue.atLock.Unlock()
+
+	tID := t.ID()
+	_, ok := queue.activeTasks[tID]
+	if ok {
+		log.Fatalf("task with ID %v already in active task list!", tID)
+	}
+
+	queue.activeTasks[tID] = t
+}
+
+func (queue *BaseTaskQueue) PopActiveTask(tID UniqueID) task {
+	queue.atLock.Lock()
+	defer queue.atLock.Unlock()
+
+	t, ok := queue.activeTasks[tID]
+	if ok {
+		delete(queue.activeTasks, tID)
+		return t
+	}
+	log.Fatalf("sorry, but the ID %d was not found in the active task list!", tID)
+	return nil
+}
+
+func (queue *BaseTaskQueue) Enqueue(t task) error {
+	tID, _ := queue.sched.idAllocator.AllocOne()
+	log.Printf("[Builder] allocate reqID: %v", tID)
+	t.SetID(tID)
+	err := t.OnEnqueue()
+	if err != nil {
+		return err
+	}
+	return queue.addUnissuedTask(t)
+}
+
+type IndexAddTaskQueue struct {
+	BaseTaskQueue
+	lock sync.Mutex
+}
+
+func (queue *IndexAddTaskQueue) Enqueue(t task) error {
+	queue.lock.Lock()
+	defer queue.lock.Unlock()
+	return queue.BaseTaskQueue.Enqueue(t)
+}
+
+func NewIndexAddTaskQueue(sched *TaskScheduler) *IndexAddTaskQueue {
+	return &IndexAddTaskQueue{
+		BaseTaskQueue: BaseTaskQueue{
+			unissuedTasks: list.New(),
+			activeTasks:   make(map[UniqueID]task),
+			maxTaskNum:    1024,
+			utBufChan:     make(chan int, 1024),
+			sched:         sched,
+		},
+	}
+}
+
+type TaskScheduler struct {
+	IndexAddQueue TaskQueue
+
+	idAllocator *GlobalIDAllocator
+	metaTable   *metaTable
+	kv          kv.Base
+
+	wg     sync.WaitGroup
+	ctx    context.Context
+	cancel context.CancelFunc
+}
+
+func NewTaskScheduler(ctx context.Context,
+	idAllocator *GlobalIDAllocator,
+	kv kv.Base,
+	table *metaTable) (*TaskScheduler, error) {
+	ctx1, cancel := context.WithCancel(ctx)
+	s := &TaskScheduler{
+		idAllocator: idAllocator,
+		metaTable:   table,
+		kv:          kv,
+		ctx:         ctx1,
+		cancel:      cancel,
+	}
+	s.IndexAddQueue = NewIndexAddTaskQueue(s)
+	return s, nil
+}
+
+func (sched *TaskScheduler) scheduleIndexAddTask() task {
+	return sched.IndexAddQueue.PopUnissuedTask()
+}
+
+//func (sched *TaskScheduler) scheduleIndexBuildClient() indexnode.Interface {
+//	return sched.IndexAddQueue.PopUnissuedTask()
+//}
+
+func (sched *TaskScheduler) processTask(t task, q TaskQueue) {
+
+	err := t.PreExecute()
+
+	defer func() {
+		t.Notify(err)
+		log.Printf("notify with error: %v", err)
+	}()
+	if err != nil {
+		return
+	}
+
+	q.AddActiveTask(t)
+	log.Printf("task add to active list ...")
+	defer func() {
+		q.PopActiveTask(t.ID())
+		log.Printf("pop from active list ...")
+	}()
+
+	err = t.Execute()
+	if err != nil {
+		log.Printf("execute definition task failed, error = %v", err)
+		return
+	}
+	log.Printf("task execution done ...")
+	err = t.PostExecute()
+	log.Printf("post execute task done ...")
+}
+
+func (sched *TaskScheduler) indexAddLoop() {
+	defer sched.wg.Done()
+	for {
+		select {
+		case <-sched.ctx.Done():
+			return
+		case <-sched.IndexAddQueue.utChan():
+			if !sched.IndexAddQueue.utEmpty() {
+				t := sched.scheduleIndexAddTask()
+				go sched.processTask(t, sched.IndexAddQueue)
+			}
+		}
+	}
+}
+
+func (sched *TaskScheduler) Start() error {
+
+	sched.wg.Add(1)
+	go sched.indexAddLoop()
+
+	return nil
+}
+
+func (sched *TaskScheduler) Close() {
+	sched.cancel()
+	sched.wg.Wait()
+}

internal/indexservice/tso.go

@@ -0,0 +1,202 @@
+// Copyright 2016 TiKV Project Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package indexservice
+
+import (
+	"log"
+	"sync/atomic"
+	"time"
+	"unsafe"
+
+	"go.uber.org/zap"
+
+	"github.com/zilliztech/milvus-distributed/internal/errors"
+	"github.com/zilliztech/milvus-distributed/internal/kv"
+	"github.com/zilliztech/milvus-distributed/internal/util/tsoutil"
+	"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
+)
+
+const (
+	// UpdateTimestampStep is used to update timestamp.
+	UpdateTimestampStep = 50 * time.Millisecond
+	// updateTimestampGuard is the min timestamp interval.
+	updateTimestampGuard = time.Millisecond
+	// maxLogical is the max upper limit for logical time.
+	// When a TSO's logical time reaches this limit,
+	// the physical time will be forced to increase.
+	maxLogical = int64(1 << 18)
+)
+
+// atomicObject is used to store the current TSO in memory.
+type atomicObject struct {
+	physical time.Time
+	logical  int64
+}
+
+// timestampOracle is used to maintain the logic of tso.
+type timestampOracle struct {
+	key    string
+	kvBase kv.TxnBase
+
+	// TODO: remove saveInterval
+	saveInterval  time.Duration
+	maxResetTSGap func() time.Duration
+	// For tso, set after the PD becomes a leader.
+	TSO           unsafe.Pointer
+	lastSavedTime atomic.Value
+}
+
+func (t *timestampOracle) loadTimestamp() (time.Time, error) {
+	strData, err := t.kvBase.Load(t.key)
+
+	var binData []byte = []byte(strData)
+
+	if err != nil {
+		return typeutil.ZeroTime, err
+	}
+	if len(binData) == 0 {
+		return typeutil.ZeroTime, nil
+	}
+	return typeutil.ParseTimestamp(binData)
+}
+
+// save timestamp, if lastTs is 0, we think the timestamp doesn't exist, so create it,
+// otherwise, update it.
+func (t *timestampOracle) saveTimestamp(ts time.Time) error {
+	data := typeutil.Uint64ToBytes(uint64(ts.UnixNano()))
+	err := t.kvBase.Save(t.key, string(data))
+	if err != nil {
+		return errors.WithStack(err)
+	}
+	t.lastSavedTime.Store(ts)
+	return nil
+}
+
+func (t *timestampOracle) InitTimestamp() error {
+
+	//last, err := t.loadTimestamp()
+	//if err != nil {
+	//	return err
+	//}
+
+	next := time.Now()
+
+	// If the current system time minus the saved etcd timestamp is less than `updateTimestampGuard`,
+	// the timestamp allocation will start from the saved etcd timestamp temporarily.
+	//if typeutil.SubTimeByWallClock(next, last) < updateTimestampGuard {
+	//	next = last.Add(updateTimestampGuard)
+	//}
+
+	save := next.Add(t.saveInterval)
+	if err := t.saveTimestamp(save); err != nil {
+		return err
+	}
+
+	//log.Print("sync and save timestamp", zap.Time("last", last), zap.Time("save", save), zap.Time("next", next))
+
+	current := &atomicObject{
+		physical: next,
+	}
+	atomic.StorePointer(&t.TSO, unsafe.Pointer(current))
+
+	return nil
+}
+
+// ResetUserTimestamp update the physical part with specified tso.
+func (t *timestampOracle) ResetUserTimestamp(tso uint64) error {
+	physical, _ := tsoutil.ParseTS(tso)
+	next := physical.Add(time.Millisecond)
+	prev := (*atomicObject)(atomic.LoadPointer(&t.TSO))
+
+	// do not update
+	if typeutil.SubTimeByWallClock(next, prev.physical) <= 3*updateTimestampGuard {
+		return errors.New("the specified ts too small than now")
+	}
+
+	if typeutil.SubTimeByWallClock(next, prev.physical) >= t.maxResetTSGap() {
+		return errors.New("the specified ts too large than now")
+	}
+
+	save := next.Add(t.saveInterval)
+	if err := t.saveTimestamp(save); err != nil {
+		return err
+	}
+	update := &atomicObject{
+		physical: next,
+	}
+	atomic.CompareAndSwapPointer(&t.TSO, unsafe.Pointer(prev), unsafe.Pointer(update))
+	return nil
+}
+
+// UpdateTimestamp is used to update the timestamp.
+// This function will do two things:
+// 1. When the logical time is going to be used up, increase the current physical time.
+// 2. When the time window is not big enough, which means the saved etcd time minus the next physical time
+//    will be less than or equal to `updateTimestampGuard`, then the time window needs to be updated and
+//    we also need to save the next physical time plus `TsoSaveInterval` into etcd.
+//
+// Here is some constraints that this function must satisfy:
+// 1. The saved time is monotonically increasing.
+// 2. The physical time is monotonically increasing.
+// 3. The physical time is always less than the saved timestamp.
+func (t *timestampOracle) UpdateTimestamp() error {
+	prev := (*atomicObject)(atomic.LoadPointer(&t.TSO))
+	now := time.Now()
+
+	jetLag := typeutil.SubTimeByWallClock(now, prev.physical)
+	if jetLag > 3*UpdateTimestampStep {
+		log.Print("clock offset", zap.Duration("jet-lag", jetLag), zap.Time("prev-physical", prev.physical), zap.Time("now", now))
+	}
+
+	var next time.Time
+	prevLogical := atomic.LoadInt64(&prev.logical)
+	// If the system time is greater, it will be synchronized with the system time.
+	if jetLag > updateTimestampGuard {
+		next = now
+	} else if prevLogical > maxLogical/2 {
+		// The reason choosing maxLogical/2 here is that it's big enough for common cases.
+		// Because there is enough timestamp can be allocated before next update.
+		log.Print("the logical time may be not enough", zap.Int64("prev-logical", prevLogical))
+		next = prev.physical.Add(time.Millisecond)
+	} else {
+		// It will still use the previous physical time to alloc the timestamp.
+		return nil
+	}
+
+	// It is not safe to increase the physical time to `next`.
+	// The time window needs to be updated and saved to etcd.
+	if typeutil.SubTimeByWallClock(t.lastSavedTime.Load().(time.Time), next) <= updateTimestampGuard {
+		save := next.Add(t.saveInterval)
+		if err := t.saveTimestamp(save); err != nil {
+			return err
+		}
+	}
+
+	current := &atomicObject{
+		physical: next,
+		logical:  0,
+	}
+
+	atomic.StorePointer(&t.TSO, unsafe.Pointer(current))
+
+	return nil
+}
+
+// ResetTimestamp is used to reset the timestamp.
+func (t *timestampOracle) ResetTimestamp() {
+	zero := &atomicObject{
+		physical: time.Now(),
+	}
+	atomic.StorePointer(&t.TSO, unsafe.Pointer(zero))
+}

internal/indexservice/util.go

@@ -0,0 +1,10 @@
+package indexservice
+
+import "github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
+
+func CompareAddress(a *commonpb.Address, b *commonpb.Address) bool {
+	if a == b {
+		return true
+	}
+	return a.Ip == b.Ip && a.Port == b.Port
+}

internal/master/client.go

@@ -63,21 +63,26 @@ func (m *MockWriteNodeClient) GetInsertBinlogPaths(segmentID UniqueID) (map[Uniq
 }
 
 type BuildIndexClient interface {
-	BuildIndex(columnDataPaths []string, typeParams map[string]string, indexParams map[string]string) (UniqueID, error)
-	GetIndexStates(indexIDs []UniqueID) (*indexpb.IndexStatesResponse, error)
-	GetIndexFilePaths(indexID []UniqueID) ([][]string, error)
+	BuildIndex(req *indexpb.BuildIndexRequest) (*indexpb.BuildIndexResponse, error)
+	GetIndexStates(req *indexpb.IndexStatesRequest) (*indexpb.IndexStatesResponse, error)
+	GetIndexFilePaths(req *indexpb.IndexFilePathsRequest) (*indexpb.IndexFilePathsResponse, error)
 }
 
 type MockBuildIndexClient struct {
 	buildTime time.Time
 }
 
-func (m *MockBuildIndexClient) BuildIndex(columnDataPaths []string, typeParams map[string]string, indexParams map[string]string) (UniqueID, error) {
+func (m *MockBuildIndexClient) BuildIndex(req *indexpb.BuildIndexRequest) (*indexpb.BuildIndexResponse, error) {
 	m.buildTime = time.Now()
-	return 1, nil
+	return &indexpb.BuildIndexResponse{
+		Status: &commonpb.Status{
+			ErrorCode: commonpb.ErrorCode_SUCCESS,
+		},
+		IndexID: int64(1),
+	}, nil
 }
 
-func (m *MockBuildIndexClient) GetIndexStates(indexIDs []UniqueID) (*indexpb.IndexStatesResponse, error) {
+func (m *MockBuildIndexClient) GetIndexStates(req *indexpb.IndexStatesRequest) (*indexpb.IndexStatesResponse, error) {
 	now := time.Now()
 	ret := &indexpb.IndexStatesResponse{
 		Status: &commonpb.Status{
@@ -86,7 +91,7 @@ func (m *MockBuildIndexClient) GetIndexStates(indexIDs []UniqueID) (*indexpb.Ind
 	}
 	var indexStates []*indexpb.IndexInfo
 	if now.Sub(m.buildTime).Seconds() > 2 {
-		for _, indexID := range indexIDs {
+		for _, indexID := range req.IndexIDs {
 			indexState := &indexpb.IndexInfo{
 				State:   commonpb.IndexState_FINISHED,
 				IndexID: indexID,
@@ -96,7 +101,7 @@ func (m *MockBuildIndexClient) GetIndexStates(indexIDs []UniqueID) (*indexpb.Ind
 		ret.States = indexStates
 		return ret, nil
 	}
-	for _, indexID := range indexIDs {
+	for _, indexID := range req.IndexIDs {
 		indexState := &indexpb.IndexInfo{
 			State:   commonpb.IndexState_INPROGRESS,
 			IndexID: indexID,
@@ -107,8 +112,25 @@ func (m *MockBuildIndexClient) GetIndexStates(indexIDs []UniqueID) (*indexpb.Ind
 	return ret, nil
 }
 
-func (m *MockBuildIndexClient) GetIndexFilePaths(indexIDs []UniqueID) ([][]string, error) {
-	return [][]string{{"/binlog/index/file_1", "/binlog/index/file_2", "/binlog/index/file_3"}}, nil
+func (m *MockBuildIndexClient) GetIndexFilePaths(req *indexpb.IndexFilePathsRequest) (*indexpb.IndexFilePathsResponse, error) {
+	var filePathInfos []*indexpb.IndexFilePathInfo
+	for _, indexID := range req.IndexIDs {
+		filePaths := &indexpb.IndexFilePathInfo{
+			Status: &commonpb.Status{
+				ErrorCode: commonpb.ErrorCode_SUCCESS,
+			},
+			IndexID:        indexID,
+			IndexFilePaths: []string{"/binlog/index/file_1", "/binlog/index/file_2", "/binlog/index/file_3"},
+		}
+		filePathInfos = append(filePathInfos, filePaths)
+	}
+
+	return &indexpb.IndexFilePathsResponse{
+		Status: &commonpb.Status{
+			ErrorCode: commonpb.ErrorCode_SUCCESS,
+		},
+		FilePaths: filePathInfos,
+	}, nil
 }
 
 type LoadIndexClient interface {

internal/master/index_builder_scheduler.go

@@ -2,10 +2,13 @@ package master
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"log"
 	"time"
 
+	"github.com/zilliztech/milvus-distributed/internal/proto/indexpb"
+
 	"github.com/zilliztech/milvus-distributed/internal/errors"
 	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
 	"github.com/zilliztech/milvus-distributed/internal/proto/etcdpb"
@@ -74,11 +77,73 @@ func (scheduler *IndexBuildScheduler) schedule(info interface{}) error {
 		indexParamsMap[kv.Key] = kv.Value
 	}
 
-	indexID, err := scheduler.client.BuildIndex(indexBuildInfo.binlogFilePath, typeParamsMap, indexParamsMap)
+	parseMap := func(mStr string) (map[string]string, error) {
+		buffer := make(map[string]interface{})
+		err := json.Unmarshal([]byte(mStr), &buffer)
+		if err != nil {
+			return nil, errors.New("Unmarshal params failed")
+		}
+		ret := make(map[string]string)
+		for key, value := range buffer {
+			valueStr := fmt.Sprintf("%v", value)
+			ret[key] = valueStr
+		}
+		return ret, nil
+	}
+	var typeParamsKV []*commonpb.KeyValuePair
+	for key := range typeParamsMap {
+		if key == "params" {
+			mapParams, err := parseMap(typeParamsMap[key])
+			if err != nil {
+				log.Println("parse params error: ", err)
+			}
+			for pk, pv := range mapParams {
+				typeParamsKV = append(typeParamsKV, &commonpb.KeyValuePair{
+					Key:   pk,
+					Value: pv,
+				})
+			}
+		} else {
+			typeParamsKV = append(typeParamsKV, &commonpb.KeyValuePair{
+				Key:   key,
+				Value: typeParamsMap[key],
+			})
+		}
+	}
+
+	var indexParamsKV []*commonpb.KeyValuePair
+	for key := range indexParamsMap {
+		if key == "params" {
+			mapParams, err := parseMap(indexParamsMap[key])
+			if err != nil {
+				log.Println("parse params error: ", err)
+			}
+			for pk, pv := range mapParams {
+				indexParamsKV = append(indexParamsKV, &commonpb.KeyValuePair{
+					Key:   pk,
+					Value: pv,
+				})
+			}
+		} else {
+			indexParamsKV = append(indexParamsKV, &commonpb.KeyValuePair{
+				Key:   key,
+				Value: indexParamsMap[key],
+			})
+		}
+	}
+
+	requset := &indexpb.BuildIndexRequest{
+		DataPaths:   indexBuildInfo.binlogFilePath,
+		TypeParams:  typeParamsKV,
+		IndexParams: indexParamsKV,
+	}
+
+	indexResp, err := scheduler.client.BuildIndex(requset)
 	if err != nil {
 		log.Printf("build index for segment %d field %d, failed:%s", indexBuildInfo.segmentID, indexBuildInfo.fieldID, err.Error())
 		return err
 	}
+	indexID := indexResp.IndexID
 
 	err = scheduler.metaTable.AddFieldIndexMeta(&etcdpb.FieldIndexMeta{
 		SegmentID:   indexBuildInfo.segmentID,
@@ -112,17 +177,34 @@ func (scheduler *IndexBuildScheduler) describe() error {
 			indexID := channelInfo.id
 			indexBuildInfo := channelInfo.info
 			for {
-				description, err := scheduler.client.GetIndexStates([]UniqueID{channelInfo.id})
+				indexIDs := []UniqueID{channelInfo.id}
+				request := &indexpb.IndexStatesRequest{
+					IndexIDs: indexIDs,
+				}
+				description, err := scheduler.client.GetIndexStates(request)
 				if err != nil {
 					return err
 				}
 				if description.States[0].State == commonpb.IndexState_FINISHED {
 					log.Printf("build index for segment %d field %d is finished", indexBuildInfo.segmentID, indexBuildInfo.fieldID)
-					filesPaths, err := scheduler.client.GetIndexFilePaths([]UniqueID{indexID})
+					request := &indexpb.IndexFilePathsRequest{
+						IndexIDs: indexIDs,
+					}
+
+					response, err := scheduler.client.GetIndexFilePaths(request)
 					if err != nil {
 						return err
 					}
-					filePaths := filesPaths[0]
+					var filePathsInfos [][]string
+					for _, indexID := range indexIDs {
+						for _, filePathInfo := range response.FilePaths {
+							if indexID == filePathInfo.IndexID {
+								filePathsInfos = append(filePathsInfos, filePathInfo.IndexFilePaths)
+								break
+							}
+						}
+					}
+					filePaths := filePathsInfos[0]
 
 					//TODO: remove fileName
 					var fieldName string

internal/master/master.go

@@ -10,19 +10,16 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/zilliztech/milvus-distributed/internal/querynode/client"
-
-	indexnodeclient "github.com/zilliztech/milvus-distributed/internal/indexnode/client"
-
-	writerclient "github.com/zilliztech/milvus-distributed/internal/writenode/client"
-
+	grpcindexserviceclient "github.com/zilliztech/milvus-distributed/internal/distributed/indexservice/client"
 	etcdkv "github.com/zilliztech/milvus-distributed/internal/kv/etcd"
 	ms "github.com/zilliztech/milvus-distributed/internal/msgstream"
 	"github.com/zilliztech/milvus-distributed/internal/msgstream/pulsarms"
 	"github.com/zilliztech/milvus-distributed/internal/msgstream/util"
 	"github.com/zilliztech/milvus-distributed/internal/proto/masterpb"
+	"github.com/zilliztech/milvus-distributed/internal/querynode/client"
 	"github.com/zilliztech/milvus-distributed/internal/util/tsoutil"
 	"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
+	writerclient "github.com/zilliztech/milvus-distributed/internal/writenode/client"
 	"go.etcd.io/etcd/clientv3"
 	"google.golang.org/grpc"
 )
@@ -187,10 +184,7 @@ func CreateServer(ctx context.Context) (*Master, error) {
 	if err != nil {
 		return nil, err
 	}
-	buildIndexClient, err := indexnodeclient.NewBuildIndexClient(ctx, Params.IndexBuilderAddress)
-	if err != nil {
-		return nil, err
-	}
+	buildIndexClient := grpcindexserviceclient.NewClient(Params.IndexBuilderAddress)
 	queryNodeClient := client.NewQueryNodeClient(ctx, Params.PulsarAddress, Params.LoadIndexChannelNames)
 
 	m.indexLoadSch = NewIndexLoadScheduler(ctx, queryNodeClient, m.metaTable)

internal/master/param_table.go

@@ -132,7 +132,7 @@ func (p *ParamTable) initPulsarAddress() {
 }
 
 func (p *ParamTable) initIndexBuilderAddress() {
-	ret, err := p.Load("_IndexBuilderAddress")
+	ret, err := p.Load("IndexServiceAddress")
 	if err != nil {
 		panic(err)
 	}

internal/master/param_table_test.go

@@ -31,7 +31,7 @@ func TestParamTable_KVRootPath(t *testing.T) {
 	assert.Equal(t, path, "by-dev/kv")
 }
 
-func TestParamTable_IndexBuilderAddress(t *testing.T) {
+func TestParamTableIndexServiceAddress(t *testing.T) {
 	path := Params.IndexBuilderAddress
 	assert.Equal(t, path, "localhost:31000")
 }

internal/master/persistence_scheduler_test.go

@@ -5,6 +5,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/zilliztech/milvus-distributed/internal/proto/indexpb"
+
 	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
 
 	"github.com/zilliztech/milvus-distributed/internal/proto/etcdpb"
@@ -96,14 +98,20 @@ func TestPersistenceScheduler(t *testing.T) {
 
 	//wait flush segment request sent to build index node
 	time.Sleep(100 * time.Microsecond)
-	idxDes, err := buildIndexClient.GetIndexStates([]UniqueID{UniqueID(1)})
+	req := &indexpb.IndexStatesRequest{
+		IndexIDs: []UniqueID{UniqueID(1)},
+	}
+	idxDes, err := buildIndexClient.GetIndexStates(req)
 	assert.Nil(t, err)
 	assert.Equal(t, commonpb.IndexState_INPROGRESS, idxDes.States[0].State)
 
 	//wait build index to finish
 	time.Sleep(3 * time.Second)
 
-	idxDes, err = buildIndexClient.GetIndexStates([]UniqueID{UniqueID(1)})
+	req2 := &indexpb.IndexStatesRequest{
+		IndexIDs: []UniqueID{UniqueID(1)},
+	}
+	idxDes, err = buildIndexClient.GetIndexStates(req2)
 	assert.Nil(t, err)
 	assert.Equal(t, commonpb.IndexState_FINISHED, idxDes.States[0].State)
 

internal/msgstream/rmqms/rmq_msgstream.go

@@ -1,4 +1,4 @@
-package rmqmsgstream
+package rmqms
 
 import (
 	"context"

internal/proto/index_service.proto

@@ -19,7 +19,7 @@ message RegisterNodeResponse {
 }
 
 message IndexStatesRequest {
-  repeated int64 indexID = 1;
+  repeated int64 indexIDs = 1;
 }
 
 message IndexInfo {
@@ -44,7 +44,6 @@ message BuildIndexResponse {
   int64 indexID = 2;
 }
 
-
 message BuildIndexCmd {
   int64 indexID = 1;
   BuildIndexRequest req = 2;
@@ -54,6 +53,7 @@ message BuildIndexNotification {
   common.Status status = 1;
   int64 indexID = 2;
   repeated string index_file_paths = 3;
+  int64 nodeID = 4;
 }
 
 message IndexFilePathsRequest {
@@ -74,11 +74,9 @@ message IndexFilePathsResponse {
 message IndexMeta {
   common.IndexState state = 1;
   int64   indexID = 2;
-  int64 enque_time = 3;
-  int64 schedule_time = 4;
-  int64 build_complete_time = 5;
-  BuildIndexRequest req = 6;
-  repeated string index_file_paths = 7;
+  string fail_reason = 3;
+  BuildIndexRequest req = 4;
+  repeated string index_file_paths = 5;
 }
 
 /****************IndexService************************/

internal/proto/indexpb/index_service.pb.go

@@ -121,7 +121,7 @@ func (m *RegisterNodeResponse) GetInitParams() *internalpb2.InitParams {
 }
 
 type IndexStatesRequest struct {
-	IndexID              []int64  `protobuf:"varint,1,rep,packed,name=indexID,proto3" json:"indexID,omitempty"`
+	IndexIDs             []int64  `protobuf:"varint,1,rep,packed,name=indexIDs,proto3" json:"indexIDs,omitempty"`
 	XXX_NoUnkeyedLiteral struct{} `json:"-"`
 	XXX_unrecognized     []byte   `json:"-"`
 	XXX_sizecache        int32    `json:"-"`
@@ -152,9 +152,9 @@ func (m *IndexStatesRequest) XXX_DiscardUnknown() {
 
 var xxx_messageInfo_IndexStatesRequest proto.InternalMessageInfo
 
-func (m *IndexStatesRequest) GetIndexID() []int64 {
+func (m *IndexStatesRequest) GetIndexIDs() []int64 {
 	if m != nil {
-		return m.IndexID
+		return m.IndexIDs
 	}
 	return nil
 }
@@ -414,6 +414,7 @@ type BuildIndexNotification struct {
 	Status               *commonpb.Status `protobuf:"bytes,1,opt,name=status,proto3" json:"status,omitempty"`
 	IndexID              int64            `protobuf:"varint,2,opt,name=indexID,proto3" json:"indexID,omitempty"`
 	IndexFilePaths       []string         `protobuf:"bytes,3,rep,name=index_file_paths,json=indexFilePaths,proto3" json:"index_file_paths,omitempty"`
+	NodeID               int64            `protobuf:"varint,4,opt,name=nodeID,proto3" json:"nodeID,omitempty"`
 	XXX_NoUnkeyedLiteral struct{}         `json:"-"`
 	XXX_unrecognized     []byte           `json:"-"`
 	XXX_sizecache        int32            `json:"-"`
@@ -465,6 +466,13 @@ func (m *BuildIndexNotification) GetIndexFilePaths() []string {
 	return nil
 }
 
+func (m *BuildIndexNotification) GetNodeID() int64 {
+	if m != nil {
+		return m.NodeID
+	}
+	return 0
+}
+
 type IndexFilePathsRequest struct {
 	IndexIDs             []int64  `protobuf:"varint,1,rep,packed,name=indexIDs,proto3" json:"indexIDs,omitempty"`
 	XXX_NoUnkeyedLiteral struct{} `json:"-"`
@@ -609,11 +617,9 @@ func (m *IndexFilePathsResponse) GetFilePaths() []*IndexFilePathInfo {
 type IndexMeta struct {
 	State                commonpb.IndexState `protobuf:"varint,1,opt,name=state,proto3,enum=milvus.proto.common.IndexState" json:"state,omitempty"`
 	IndexID              int64               `protobuf:"varint,2,opt,name=indexID,proto3" json:"indexID,omitempty"`
-	EnqueTime            int64               `protobuf:"varint,3,opt,name=enque_time,json=enqueTime,proto3" json:"enque_time,omitempty"`
-	ScheduleTime         int64               `protobuf:"varint,4,opt,name=schedule_time,json=scheduleTime,proto3" json:"schedule_time,omitempty"`
-	BuildCompleteTime    int64               `protobuf:"varint,5,opt,name=build_complete_time,json=buildCompleteTime,proto3" json:"build_complete_time,omitempty"`
-	Req                  *BuildIndexRequest  `protobuf:"bytes,6,opt,name=req,proto3" json:"req,omitempty"`
-	IndexFilePaths       []string            `protobuf:"bytes,7,rep,name=index_file_paths,json=indexFilePaths,proto3" json:"index_file_paths,omitempty"`
+	FailReason           string              `protobuf:"bytes,3,opt,name=fail_reason,json=failReason,proto3" json:"fail_reason,omitempty"`
+	Req                  *BuildIndexRequest  `protobuf:"bytes,4,opt,name=req,proto3" json:"req,omitempty"`
+	IndexFilePaths       []string            `protobuf:"bytes,5,rep,name=index_file_paths,json=indexFilePaths,proto3" json:"index_file_paths,omitempty"`
 	XXX_NoUnkeyedLiteral struct{}            `json:"-"`
 	XXX_unrecognized     []byte              `json:"-"`
 	XXX_sizecache        int32               `json:"-"`
@@ -658,25 +664,11 @@ func (m *IndexMeta) GetIndexID() int64 {
 	return 0
 }
 
-func (m *IndexMeta) GetEnqueTime() int64 {
+func (m *IndexMeta) GetFailReason() string {
 	if m != nil {
-		return m.EnqueTime
+		return m.FailReason
 	}
-	return 0
-}
-
-func (m *IndexMeta) GetScheduleTime() int64 {
-	if m != nil {
-		return m.ScheduleTime
-	}
-	return 0
-}
-
-func (m *IndexMeta) GetBuildCompleteTime() int64 {
-	if m != nil {
-		return m.BuildCompleteTime
-	}
-	return 0
+	return ""
 }
 
 func (m *IndexMeta) GetReq() *BuildIndexRequest {
@@ -712,57 +704,55 @@ func init() {
 func init() { proto.RegisterFile("index_service.proto", fileDescriptor_a5d2036b4df73e0a) }
 
 var fileDescriptor_a5d2036b4df73e0a = []byte{
-	// 793 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x56, 0x5d, 0x4f, 0xe3, 0x46,
-	0x14, 0xc5, 0x18, 0x42, 0x73, 0x13, 0x22, 0x32, 0x69, 0x51, 0x94, 0x16, 0x15, 0x5c, 0x01, 0x29,
-	0x52, 0x9d, 0x2a, 0x88, 0xf6, 0xb1, 0x22, 0xa0, 0x56, 0x51, 0x05, 0x42, 0xd3, 0xaa, 0x0f, 0xad,
-	0xaa, 0xc8, 0xb1, 0x6f, 0xc8, 0x48, 0xfe, 0x08, 0x9e, 0x31, 0x5a, 0x78, 0x59, 0xad, 0xb4, 0x8f,
-	0x2b, 0xad, 0x56, 0xfb, 0x53, 0xf6, 0x75, 0x7f, 0xdc, 0xca, 0xe3, 0xb1, 0x63, 0x43, 0x48, 0x58,
-	0x81, 0xb4, 0x6f, 0x99, 0xeb, 0x73, 0x3f, 0xe6, 0x9c, 0x33, 0x33, 0x81, 0x06, 0xf3, 0x1d, 0x7c,
-	0x31, 0xe0, 0x18, 0x5e, 0x33, 0x1b, 0xcd, 0x49, 0x18, 0x88, 0x80, 0x10, 0x8f, 0xb9, 0xd7, 0x11,
-	0x4f, 0x56, 0xa6, 0x44, 0xb4, 0xaa, 0x76, 0xe0, 0x79, 0x81, 0x9f, 0xc4, 0x5a, 0x35, 0xe6, 0x0b,
-	0x0c, 0x7d, 0xcb, 0x4d, 0xd6, 0xc6, 0x4b, 0x68, 0x50, 0xbc, 0x64, 0x5c, 0x60, 0x78, 0x1e, 0x38,
-	0x48, 0xf1, 0x2a, 0x42, 0x2e, 0xc8, 0xcf, 0xb0, 0x32, 0xb4, 0x38, 0x36, 0xb5, 0x6d, 0xad, 0x5d,
-	0xe9, 0x7e, 0x67, 0x16, 0xea, 0xaa, 0x82, 0x67, 0xfc, 0xb2, 0x67, 0x71, 0xa4, 0x12, 0x49, 0x7e,
-	0x81, 0x35, 0xcb, 0x71, 0x42, 0xe4, 0xbc, 0xb9, 0x3c, 0x27, 0xe9, 0x38, 0xc1, 0xd0, 0x14, 0x6c,
-	0xbc, 0xd5, 0xe0, 0xeb, 0xe2, 0x04, 0x7c, 0x12, 0xf8, 0x1c, 0xc9, 0x21, 0x94, 0xb8, 0xb0, 0x44,
-	0xc4, 0xd5, 0x10, 0xdf, 0xce, 0xac, 0xf7, 0x97, 0x84, 0x50, 0x05, 0x25, 0x3d, 0xa8, 0x30, 0x9f,
-	0x89, 0xc1, 0xc4, 0x0a, 0x2d, 0x2f, 0x9d, 0x64, 0xc7, 0xbc, 0x43, 0x8b, 0x62, 0xa0, 0xef, 0x33,
-	0x71, 0x21, 0x81, 0x14, 0x58, 0xf6, 0xdb, 0x30, 0x81, 0xf4, 0x63, 0xe6, 0xe2, 0xd2, 0xc8, 0x53,
-	0x46, 0x9a, 0xb0, 0x26, 0xf9, 0xec, 0x9f, 0x36, 0xb5, 0x6d, 0xbd, 0xad, 0xd3, 0x74, 0x69, 0x08,
-	0x28, 0x4b, 0x7c, 0xdf, 0x1f, 0x05, 0xe4, 0x08, 0x56, 0xe3, 0x51, 0x12, 0xe6, 0x6a, 0xdd, 0xef,
-	0x67, 0x0e, 0x3d, 0x2d, 0x4f, 0x13, 0x74, 0xbe, 0x7a, 0x3c, 0xf3, 0xb4, 0x3a, 0xd9, 0x84, 0x12,
-	0x45, 0x8b, 0x07, 0x7e, 0x53, 0xdf, 0xd6, 0xda, 0x65, 0xaa, 0x56, 0xc6, 0x2b, 0x0d, 0x1a, 0x85,
-	0x31, 0x9f, 0x42, 0xdb, 0x51, 0x92, 0x84, 0x31, 0x63, 0x7a, 0xbb, 0xd2, 0xdd, 0x32, 0xef, 0x1b,
-	0xc9, 0xcc, 0x36, 0x49, 0x15, 0xd8, 0xf8, 0xa8, 0x41, 0xbd, 0x17, 0x31, 0xd7, 0x91, 0x9f, 0x52,
-	0xa6, 0xb6, 0x00, 0x1c, 0x4b, 0x58, 0x83, 0x89, 0x25, 0xc6, 0x5c, 0x92, 0x55, 0xa6, 0xe5, 0x38,
-	0x72, 0x11, 0x07, 0x62, 0x89, 0xc4, 0xcd, 0x04, 0xa7, 0x12, 0xe9, 0xf7, 0x25, 0x52, 0x53, 0xfe,
-	0x89, 0x37, 0xff, 0x58, 0x6e, 0x84, 0x17, 0x16, 0x0b, 0x29, 0xc4, 0x59, 0x89, 0x44, 0xe4, 0x14,
-	0xaa, 0x89, 0xfd, 0x55, 0x11, 0xfd, 0xb1, 0x45, 0x2a, 0x32, 0x4d, 0x09, 0x6d, 0x03, 0xc9, 0x4f,
-	0xff, 0x14, 0x02, 0x1f, 0xd4, 0xcf, 0x18, 0xc2, 0xfa, 0xb4, 0xc9, 0x89, 0xe7, 0x14, 0x8d, 0x54,
-	0x90, 0xfa, 0x57, 0xd0, 0x43, 0xbc, 0x52, 0xa6, 0xdd, 0x9d, 0x25, 0xc1, 0x3d, 0xb2, 0x69, 0x9c,
-	0x61, 0xbc, 0xd3, 0x60, 0x73, 0xfa, 0xe9, 0x3c, 0x10, 0x6c, 0xc4, 0x6c, 0x4b, 0xb0, 0xc0, 0x7f,
-	0xe6, 0xdd, 0x90, 0x36, 0x6c, 0x24, 0xc4, 0x8f, 0x98, 0x8b, 0x4a, 0x61, 0x5d, 0x2a, 0x5c, 0x93,
-	0xf1, 0xdf, 0x99, 0x8b, 0x52, 0x66, 0xe3, 0x10, 0xbe, 0xe9, 0x17, 0x22, 0xa9, 0x3d, 0x5a, 0xf0,
-	0x95, 0xaa, 0xc6, 0xd5, 0x49, 0xca, 0xd6, 0xc6, 0x1b, 0x0d, 0xea, 0x85, 0x2c, 0x79, 0xa6, 0xbe,
-	0xd8, 0x1e, 0xde, 0x6b, 0xb0, 0x79, 0x77, 0x13, 0x4f, 0x71, 0xc9, 0x29, 0x40, 0xae, 0x67, 0xe2,
-	0xfc, 0xdd, 0x07, 0x8f, 0x5a, 0x9e, 0x03, 0x5a, 0x1e, 0x65, 0x53, 0x7d, 0x58, 0x56, 0x17, 0xce,
-	0x19, 0x0a, 0xeb, 0xf9, 0x2f, 0x9c, 0x2d, 0x00, 0xf4, 0xaf, 0x22, 0x1c, 0x08, 0xe6, 0xa1, 0xbc,
-	0x74, 0x74, 0x5a, 0x96, 0x91, 0xbf, 0x99, 0x87, 0xe4, 0x07, 0x58, 0xe7, 0xf6, 0x18, 0x9d, 0xc8,
-	0x55, 0x88, 0x15, 0x89, 0xa8, 0xa6, 0x41, 0x09, 0x32, 0xa1, 0x31, 0x8c, 0xfd, 0x38, 0xb0, 0x03,
-	0x6f, 0xe2, 0xa2, 0x50, 0xd0, 0x55, 0x09, 0xad, 0xcb, 0x4f, 0x27, 0xea, 0x8b, 0xc4, 0x2b, 0xe7,
-	0x97, 0x3e, 0xd7, 0xf9, 0x33, 0xb5, 0x5c, 0x9b, 0xa5, 0x65, 0xf7, 0xf5, 0x0a, 0x54, 0x13, 0x1a,
-	0x92, 0x17, 0x93, 0xd8, 0x50, 0xcd, 0xbf, 0x3b, 0x64, 0x7f, 0x56, 0xdb, 0x19, 0x6f, 0x63, 0xab,
-	0xbd, 0x18, 0x98, 0x98, 0xc4, 0x58, 0x22, 0xff, 0x03, 0x4c, 0x27, 0x27, 0x8f, 0xdb, 0x59, 0x6b,
-	0x6f, 0x11, 0x2c, 0x2b, 0x6f, 0x43, 0xed, 0x0f, 0x14, 0xb9, 0x67, 0x80, 0xec, 0x3d, 0x68, 0xa7,
-	0xc2, 0x73, 0xd6, 0xda, 0x5f, 0x88, 0xcb, 0x9a, 0xb8, 0x50, 0x4f, 0x9b, 0x64, 0x74, 0x92, 0x1f,
-	0x17, 0xda, 0x36, 0x6b, 0x75, 0xf0, 0x18, 0x68, 0x8e, 0xb1, 0x0d, 0x79, 0x81, 0xdd, 0xe4, 0x78,
-	0x3b, 0x98, 0x4f, 0x48, 0xfe, 0xc2, 0x6b, 0xcd, 0x3b, 0x88, 0xc6, 0x52, 0xf7, 0x3f, 0x75, 0x76,
-	0xa4, 0xe4, 0xe7, 0x05, 0x75, 0x76, 0xe6, 0x77, 0x39, 0xf1, 0x9c, 0x05, 0xc5, 0x7b, 0xc7, 0xff,
-	0xfe, 0x76, 0xc9, 0xc4, 0x38, 0x1a, 0xc6, 0x5f, 0x3a, 0xb7, 0xcc, 0x75, 0xd9, 0xad, 0x40, 0x7b,
-	0xdc, 0x49, 0xb2, 0x7e, 0x72, 0x18, 0x17, 0x21, 0x1b, 0x46, 0x02, 0x9d, 0x4e, 0xfa, 0x2f, 0xa4,
-	0x23, 0x4b, 0x75, 0x64, 0xb7, 0xc9, 0x70, 0x58, 0x92, 0xcb, 0xc3, 0x4f, 0x01, 0x00, 0x00, 0xff,
-	0xff, 0x10, 0x1a, 0x0a, 0xe9, 0xdf, 0x09, 0x00, 0x00,
+	// 757 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x56, 0x5f, 0x4b, 0x1b, 0x4b,
+	0x14, 0x77, 0xef, 0xc6, 0x78, 0x73, 0x92, 0x1b, 0x74, 0xbc, 0x37, 0x84, 0xdc, 0x8a, 0xba, 0xa0,
+	0xa6, 0x42, 0x13, 0x89, 0xd8, 0x3e, 0x16, 0x63, 0x68, 0x09, 0x45, 0x91, 0x29, 0xf4, 0xa1, 0xa5,
+	0x84, 0x49, 0x76, 0xa2, 0x03, 0x9b, 0xdd, 0xb8, 0x33, 0x2b, 0xd5, 0x97, 0x52, 0xe8, 0x63, 0xa1,
+	0x0f, 0xfd, 0x14, 0xfd, 0x0e, 0xfd, 0x30, 0xfd, 0x28, 0x65, 0x67, 0x66, 0x37, 0xbb, 0xba, 0x26,
+	0x29, 0x0a, 0x7d, 0xcb, 0x99, 0xfd, 0x9d, 0x3f, 0xf3, 0xfb, 0x9d, 0x73, 0x26, 0xb0, 0xca, 0x5c,
+	0x9b, 0x7e, 0xe8, 0x71, 0xea, 0x5f, 0xb2, 0x01, 0x6d, 0x8c, 0x7d, 0x4f, 0x78, 0x08, 0x8d, 0x98,
+	0x73, 0x19, 0x70, 0x65, 0x35, 0x24, 0xa2, 0x56, 0x1a, 0x78, 0xa3, 0x91, 0xe7, 0xaa, 0xb3, 0x5a,
+	0x99, 0xb9, 0x82, 0xfa, 0x2e, 0x71, 0x94, 0x6d, 0x7d, 0x84, 0x55, 0x4c, 0xcf, 0x18, 0x17, 0xd4,
+	0x3f, 0xf1, 0x6c, 0x8a, 0xe9, 0x45, 0x40, 0xb9, 0x40, 0x7b, 0x90, 0xeb, 0x13, 0x4e, 0xab, 0xc6,
+	0x86, 0x51, 0x2f, 0xb6, 0x1e, 0x35, 0x52, 0x71, 0x75, 0xc0, 0x63, 0x7e, 0xd6, 0x26, 0x9c, 0x62,
+	0x89, 0x44, 0x4f, 0x61, 0x89, 0xd8, 0xb6, 0x4f, 0x39, 0xaf, 0xfe, 0x35, 0xc5, 0xe9, 0x50, 0x61,
+	0x70, 0x04, 0xb6, 0xbe, 0x1a, 0xf0, 0x6f, 0xba, 0x02, 0x3e, 0xf6, 0x5c, 0x4e, 0xd1, 0x3e, 0xe4,
+	0xb9, 0x20, 0x22, 0xe0, 0xba, 0x88, 0xff, 0x33, 0xe3, 0xbd, 0x96, 0x10, 0xac, 0xa1, 0xa8, 0x0d,
+	0x45, 0xe6, 0x32, 0xd1, 0x1b, 0x13, 0x9f, 0x8c, 0xa2, 0x4a, 0x36, 0x1b, 0x37, 0x68, 0xd1, 0x0c,
+	0x74, 0x5d, 0x26, 0x4e, 0x25, 0x10, 0x03, 0x8b, 0x7f, 0x5b, 0x7b, 0x80, 0xba, 0x21, 0x73, 0x61,
+	0x68, 0xca, 0x23, 0x46, 0x6a, 0xf0, 0xb7, 0xe4, 0xb3, 0xdb, 0x09, 0x0b, 0x32, 0xeb, 0x26, 0x8e,
+	0x6d, 0x4b, 0x40, 0x41, 0x7a, 0x74, 0xdd, 0xa1, 0x87, 0x0e, 0x60, 0x31, 0x2c, 0x46, 0x71, 0x57,
+	0x6e, 0xad, 0x67, 0x96, 0x3d, 0x49, 0x80, 0x15, 0x1a, 0x55, 0x61, 0x49, 0xc7, 0x93, 0x55, 0x9b,
+	0x38, 0x32, 0x51, 0x05, 0xf2, 0x98, 0x12, 0xee, 0xb9, 0x55, 0x73, 0xc3, 0xa8, 0x17, 0xb0, 0xb6,
+	0xac, 0x4f, 0x06, 0xac, 0xa6, 0x0a, 0xbd, 0x0f, 0x71, 0x07, 0xca, 0x89, 0x86, 0x9c, 0x99, 0xf5,
+	0x62, 0x6b, 0xad, 0x71, 0xbb, 0x95, 0x1a, 0xf1, 0x25, 0xb1, 0x06, 0x5b, 0x3f, 0x0c, 0x58, 0x69,
+	0x07, 0xcc, 0xb1, 0xe5, 0xa7, 0x88, 0xab, 0x35, 0x00, 0x9b, 0x08, 0xd2, 0x1b, 0x13, 0x71, 0xae,
+	0xd8, 0x2a, 0xe0, 0x42, 0x78, 0x72, 0x1a, 0x1e, 0x84, 0x22, 0x89, 0xab, 0x31, 0x9d, 0x88, 0x64,
+	0xde, 0x16, 0x49, 0x57, 0xf9, 0x8a, 0x5e, 0xbd, 0x21, 0x4e, 0x40, 0x4f, 0x09, 0xf3, 0x31, 0x84,
+	0x5e, 0x4a, 0x24, 0xd4, 0x81, 0x92, 0x1a, 0x00, 0x1d, 0xc4, 0x9c, 0x37, 0x48, 0x51, 0xba, 0x69,
+	0xa9, 0x07, 0x80, 0x92, 0xd5, 0xdf, 0x87, 0xc0, 0x3b, 0xf5, 0xb3, 0xfa, 0xf0, 0xcf, 0x24, 0xc9,
+	0xd1, 0xc8, 0x4e, 0x42, 0x8d, 0xb4, 0xd4, 0xcf, 0xc0, 0xf4, 0xe9, 0x85, 0x6e, 0xdb, 0xad, 0x2c,
+	0x09, 0x6e, 0x91, 0x8d, 0x43, 0x0f, 0xeb, 0xbb, 0x01, 0x95, 0xc9, 0xa7, 0x13, 0x4f, 0xb0, 0x21,
+	0x1b, 0x10, 0xc1, 0x3c, 0xf7, 0x81, 0x6f, 0x83, 0xea, 0xb0, 0xac, 0x88, 0x1f, 0x32, 0x87, 0x6a,
+	0x85, 0x4d, 0xa9, 0x70, 0x59, 0x9e, 0xbf, 0x60, 0x0e, 0x55, 0x32, 0x57, 0x20, 0xef, 0x7a, 0x36,
+	0xed, 0x76, 0xaa, 0x39, 0x19, 0x42, 0x5b, 0xd6, 0x3e, 0xfc, 0xd7, 0x4d, 0x21, 0xe7, 0x19, 0xb1,
+	0x2f, 0x06, 0xac, 0xa4, 0xbc, 0xe4, 0xac, 0xfd, 0xa9, 0xbb, 0x59, 0xdf, 0x0c, 0xa8, 0xdc, 0xbc,
+	0xc4, 0x7d, 0xba, 0xa7, 0x03, 0x90, 0xc8, 0xa9, 0x26, 0x62, 0xeb, 0xce, 0x11, 0x4c, 0x72, 0x80,
+	0x0b, 0xc3, 0xb8, 0xaa, 0x9f, 0x86, 0x5e, 0x44, 0xc7, 0x54, 0x90, 0x87, 0x5f, 0x44, 0xeb, 0x50,
+	0x1c, 0x12, 0xe6, 0xf4, 0xfc, 0xe4, 0x36, 0x82, 0xf0, 0x48, 0x6d, 0xa4, 0xa8, 0x7d, 0x73, 0xbf,
+	0xdb, 0xbe, 0x99, 0xc4, 0x2f, 0x66, 0x11, 0xdf, 0xfa, 0x9c, 0x83, 0x92, 0xaa, 0x59, 0x3d, 0x7c,
+	0x68, 0x00, 0xa5, 0xe4, 0xf3, 0x81, 0x76, 0xb2, 0xd2, 0x66, 0x3c, 0x71, 0xb5, 0xfa, 0x6c, 0xa0,
+	0x52, 0xd4, 0x5a, 0x40, 0xef, 0x01, 0x26, 0x95, 0xa3, 0xf9, 0x6e, 0x56, 0xdb, 0x9e, 0x05, 0x8b,
+	0xc3, 0x0f, 0xa0, 0xfc, 0x92, 0x8a, 0xc4, 0x2e, 0x47, 0xdb, 0x77, 0x6a, 0x9f, 0x7a, 0x95, 0x6a,
+	0x3b, 0x33, 0x71, 0x71, 0x12, 0x07, 0x56, 0xa2, 0x24, 0x93, 0x19, 0x7d, 0x3c, 0xb3, 0xc7, 0xe2,
+	0x54, 0xbb, 0xf3, 0x40, 0x13, 0x8c, 0x2d, 0xcb, 0x2d, 0x74, 0x95, 0xe0, 0x6d, 0x77, 0x3a, 0x21,
+	0xc9, 0xad, 0x55, 0x9b, 0x36, 0x35, 0xd6, 0x42, 0xeb, 0x9d, 0x6e, 0x74, 0x29, 0xf9, 0x49, 0x4a,
+	0x9d, 0xcd, 0xe9, 0x59, 0x8e, 0x46, 0xf6, 0x8c, 0xe0, 0xed, 0xc3, 0xb7, 0xcf, 0xcf, 0x98, 0x38,
+	0x0f, 0xfa, 0xe1, 0x97, 0xe6, 0x35, 0x73, 0x1c, 0x76, 0x2d, 0xe8, 0xe0, 0xbc, 0xa9, 0xbc, 0x9e,
+	0xd8, 0x8c, 0x0b, 0x9f, 0xf5, 0x03, 0x41, 0xed, 0x66, 0xf4, 0x67, 0xa2, 0x29, 0x43, 0x35, 0x65,
+	0xb6, 0x71, 0xbf, 0x9f, 0x97, 0xe6, 0xfe, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x5c, 0xbe, 0xea,
+	0x0e, 0xa6, 0x09, 0x00, 0x00,
 }
 
 // Reference imports to suppress errors if they are not otherwise used.

internal/querynode/collection_replica.go

@@ -54,6 +54,9 @@ type collectionReplica interface {
 	getPartitionByTag(collectionID UniqueID, partitionTag string) (*Partition, error)
 	getPartitionByID(collectionID UniqueID, partitionID UniqueID) (*Partition, error)
 	hasPartition(collectionID UniqueID, partitionTag string) bool
+	enablePartitionDM(collectionID UniqueID, partitionID UniqueID) error
+	disablePartitionDM(collectionID UniqueID, partitionID UniqueID) error
+	getEnablePartitionDM(collectionID UniqueID, partitionID UniqueID) (bool, error)
 
 	// segment
 	getSegmentNum() int
@@ -362,6 +365,43 @@ func (colReplica *collectionReplicaImpl) hasPartition(collectionID UniqueID, par
 	return false
 }
 
+func (colReplica *collectionReplicaImpl) enablePartitionDM(collectionID UniqueID, partitionID UniqueID) error {
+	colReplica.mu.Lock()
+	defer colReplica.mu.Unlock()
+
+	partition, err := colReplica.getPartitionByIDPrivate(collectionID, partitionID)
+	if err != nil {
+		return err
+	}
+
+	partition.enableDM = true
+	return nil
+}
+
+func (colReplica *collectionReplicaImpl) disablePartitionDM(collectionID UniqueID, partitionID UniqueID) error {
+	colReplica.mu.Lock()
+	defer colReplica.mu.Unlock()
+
+	partition, err := colReplica.getPartitionByIDPrivate(collectionID, partitionID)
+	if err != nil {
+		return err
+	}
+
+	partition.enableDM = false
+	return nil
+}
+
+func (colReplica *collectionReplicaImpl) getEnablePartitionDM(collectionID UniqueID, partitionID UniqueID) (bool, error) {
+	colReplica.mu.Lock()
+	defer colReplica.mu.Unlock()
+
+	partition, err := colReplica.getPartitionByIDPrivate(collectionID, partitionID)
+	if err != nil {
+		return false, err
+	}
+	return partition.enableDM, nil
+}
+
 //----------------------------------------------------------------------------------------------------- segment
 func (colReplica *collectionReplicaImpl) getSegmentNum() int {
 	colReplica.mu.RLock()

internal/querynode/data_sync_service.go

@@ -19,17 +19,18 @@ type dataSyncService struct {
 }
 
 func newDataSyncService(ctx context.Context, replica collectionReplica) *dataSyncService {
-
-	return &dataSyncService{
+	service := &dataSyncService{
 		ctx: ctx,
 		fg:  nil,
 
 		replica: replica,
 	}
+
+	service.initNodes()
+	return service
 }
 
 func (dsService *dataSyncService) start() {
-	dsService.initNodes()
 	dsService.fg.Start()
 }
 
@@ -47,7 +48,7 @@ func (dsService *dataSyncService) initNodes() {
 	var dmStreamNode node = dsService.newDmInputNode(dsService.ctx)
 	var ddStreamNode node = dsService.newDDInputNode(dsService.ctx)
 
-	var filterDmNode node = newFilteredDmNode()
+	var filterDmNode node = newFilteredDmNode(dsService.replica)
 	var ddNode node = newDDNode(dsService.replica)
 
 	var insertNode node = newInsertNode(dsService.replica)

internal/querynode/flow_graph_filter_dm_node.go

@@ -12,7 +12,8 @@ import (
 
 type filterDmNode struct {
 	baseNode
-	ddMsg *ddMsg
+	ddMsg   *ddMsg
+	replica collectionReplica
 }
 
 func (fdmNode *filterDmNode) Name() string {
@@ -102,6 +103,12 @@ func (fdmNode *filterDmNode) Operate(in []*Msg) []*Msg {
 }
 
 func (fdmNode *filterDmNode) filterInvalidInsertMessage(msg *msgstream.InsertMsg) *msgstream.InsertMsg {
+	// TODO: open this check
+	// check if partition dm enable
+	//if enable, _ := fdmNode.replica.getEnablePartitionDM(msg.CollectionID, msg.PartitionID); !enable {
+	//	return nil
+	//}
+
 	// No dd record, do all insert requests.
 	records, ok := fdmNode.ddMsg.collectionRecords[msg.CollectionName]
 	if !ok {
@@ -154,7 +161,7 @@ func (fdmNode *filterDmNode) filterInvalidInsertMessage(msg *msgstream.InsertMsg
 	return msg
 }
 
-func newFilteredDmNode() *filterDmNode {
+func newFilteredDmNode(replica collectionReplica) *filterDmNode {
 	maxQueueLength := Params.FlowGraphMaxQueueLength
 	maxParallelism := Params.FlowGraphMaxParallelism
 
@@ -164,5 +171,6 @@ func newFilteredDmNode() *filterDmNode {
 
 	return &filterDmNode{
 		baseNode: baseNode,
+		replica:  replica,
 	}
 }

internal/querynode/param_table.go

@@ -12,9 +12,11 @@ import (
 type ParamTable struct {
 	paramtable.BaseTable
 
-	PulsarAddress string
-	ETCDAddress   string
-	MetaRootPath  string
+	PulsarAddress         string
+	ETCDAddress           string
+	MetaRootPath          string
+	WriteNodeSegKvSubPath string
+	IndexBuilderAddress   string
 
 	QueryNodeIP                 string
 	QueryNodePort               int64
@@ -131,6 +133,8 @@ func (p *ParamTable) Init() {
 	p.initPulsarAddress()
 	p.initETCDAddress()
 	p.initMetaRootPath()
+	p.initWriteNodeSegKvSubPath()
+	p.initIndexBuilderAddress()
 
 	p.initGracefulTime()
 	p.initMsgChannelSubName()
@@ -246,6 +250,14 @@ func (p *ParamTable) initPulsarAddress() {
 	p.PulsarAddress = url
 }
 
+func (p *ParamTable) initIndexBuilderAddress() {
+	ret, err := p.Load("_IndexBuilderAddress")
+	if err != nil {
+		panic(err)
+	}
+	p.IndexBuilderAddress = ret
+}
+
 func (p *ParamTable) initInsertChannelRange() {
 	insertChannelRange, err := p.Load("msgChannel.channelRange.insert")
 	if err != nil {
@@ -338,6 +350,14 @@ func (p *ParamTable) initMetaRootPath() {
 	p.MetaRootPath = rootPath + "/" + subPath
 }
 
+func (p *ParamTable) initWriteNodeSegKvSubPath() {
+	subPath, err := p.Load("etcd.writeNodeSegKvSubPath")
+	if err != nil {
+		panic(err)
+	}
+	p.WriteNodeSegKvSubPath = subPath + "/"
+}
+
 func (p *ParamTable) initGracefulTime() {
 	p.GracefulTime = p.ParseInt64("queryNode.gracefulTime")
 }

internal/querynode/partition.go

@@ -16,6 +16,7 @@ type Partition struct {
 	partitionTag string
 	id           UniqueID
 	segments     []*Segment
+	enableDM     bool
 }
 
 func (p *Partition) ID() UniqueID {
@@ -33,6 +34,7 @@ func (p *Partition) Segments() *[]*Segment {
 func newPartition2(partitionTag string) *Partition {
 	var newPartition = &Partition{
 		partitionTag: partitionTag,
+		enableDM:     false,
 	}
 
 	return newPartition
@@ -40,7 +42,8 @@ func newPartition2(partitionTag string) *Partition {
 
 func newPartition(partitionID UniqueID) *Partition {
 	var newPartition = &Partition{
-		id: partitionID,
+		id:       partitionID,
+		enableDM: false,
 	}
 
 	return newPartition

internal/querynode/query_node.go

@@ -136,7 +136,7 @@ func (node *QueryNode) Start() error {
 	node.metaService = newMetaService(node.queryNodeLoopCtx, node.replica)
 	node.loadIndexService = newLoadIndexService(node.queryNodeLoopCtx, node.replica)
 	node.statsService = newStatsService(node.queryNodeLoopCtx, node.replica, node.loadIndexService.fieldStatsChan)
-	node.segManager = newSegmentManager(node.queryNodeLoopCtx, node.replica, node.loadIndexService.loadIndexReqChan)
+	node.segManager = newSegmentManager(node.queryNodeLoopCtx, node.replica, node.dataSyncService.dmStream, node.loadIndexService.loadIndexReqChan)
 
 	// start services
 	go node.dataSyncService.start()
@@ -344,14 +344,31 @@ func (node *QueryNode) LoadSegments(in *queryPb.LoadSegmentRequest) (*commonpb.S
 	segmentIDs := in.SegmentIDs
 	fieldIDs := in.FieldIDs
 
+	err := node.replica.enablePartitionDM(collectionID, partitionID)
+	if err != nil {
+		status := &commonpb.Status{
+			ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
+			Reason:    err.Error(),
+		}
+		return status, err
+	}
+
 	// segments are ordered before LoadSegments calling
 	if in.LastSegmentState.State == datapb.SegmentState_SegmentGrowing {
 		segmentNum := len(segmentIDs)
-		node.segManager.seekSegment(segmentIDs[segmentNum-1])
+		positions := in.LastSegmentState.StartPositions
+		err = node.segManager.seekSegment(positions)
+		if err != nil {
+			status := &commonpb.Status{
+				ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
+				Reason:    err.Error(),
+			}
+			return status, err
+		}
 		segmentIDs = segmentIDs[:segmentNum-1]
 	}
 
-	err := node.segManager.loadSegment(collectionID, partitionID, segmentIDs, fieldIDs)
+	err = node.segManager.loadSegment(collectionID, partitionID, segmentIDs, fieldIDs)
 	if err != nil {
 		status := &commonpb.Status{
 			ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
@@ -363,6 +380,17 @@ func (node *QueryNode) LoadSegments(in *queryPb.LoadSegmentRequest) (*commonpb.S
 }
 
 func (node *QueryNode) ReleaseSegments(in *queryPb.ReleaseSegmentRequest) (*commonpb.Status, error) {
+	for _, id := range in.PartitionIDs {
+		err := node.replica.enablePartitionDM(in.CollectionID, id)
+		if err != nil {
+			status := &commonpb.Status{
+				ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
+				Reason:    err.Error(),
+			}
+			return status, err
+		}
+	}
+
 	// release all fields in the segments
 	for _, id := range in.SegmentIDs {
 		err := node.segManager.releaseSegment(id)

internal/querynode/segment_manager.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"strconv"
 
+	indexnodeclient "github.com/zilliztech/milvus-distributed/internal/indexnode/client"
 	"github.com/zilliztech/milvus-distributed/internal/kv"
 	miniokv "github.com/zilliztech/milvus-distributed/internal/kv/minio"
 	"github.com/zilliztech/milvus-distributed/internal/msgstream"
@@ -13,52 +14,31 @@ import (
 	"github.com/zilliztech/milvus-distributed/internal/proto/indexpb"
 	internalPb "github.com/zilliztech/milvus-distributed/internal/proto/internalpb2"
 	"github.com/zilliztech/milvus-distributed/internal/storage"
+	writerclient "github.com/zilliztech/milvus-distributed/internal/writenode/client"
 )
 
 type segmentManager struct {
 	replica collectionReplica
 
+	dmStream         msgstream.MsgStream
 	loadIndexReqChan chan []msgstream.TsMsg
 
-	// TODO: replace by client instead of grpc client
-	dataClient         datapb.DataServiceClient
-	indexBuilderClient indexpb.IndexServiceClient
+	dataClient  *writerclient.Client
+	indexClient *indexnodeclient.Client
 
 	kv     kv.Base // minio kv
 	iCodec *storage.InsertCodec
 }
 
-func newSegmentManager(ctx context.Context, replica collectionReplica, loadIndexReqChan chan []msgstream.TsMsg) *segmentManager {
-	bucketName := Params.MinioBucketName
-	option := &miniokv.Option{
-		Address:           Params.MinioEndPoint,
-		AccessKeyID:       Params.MinioAccessKeyID,
-		SecretAccessKeyID: Params.MinioSecretAccessKey,
-		UseSSL:            Params.MinioUseSSLStr,
-		BucketName:        bucketName,
-		CreateBucket:      true,
-	}
-
-	minioKV, err := miniokv.NewMinIOKV(ctx, option)
-	if err != nil {
-		panic(err)
-	}
-
-	return &segmentManager{
-		replica:          replica,
-		loadIndexReqChan: loadIndexReqChan,
-
-		// TODO: init clients
-		dataClient:         nil,
-		indexBuilderClient: nil,
-
-		kv:     minioKV,
-		iCodec: &storage.InsertCodec{},
-	}
-}
-
-func (s *segmentManager) seekSegment(segmentID UniqueID) {
-	// TODO: impl
+func (s *segmentManager) seekSegment(positions []*internalPb.MsgPosition) error {
+	// TODO: open seek
+	//for _, position := range positions {
+	//	err := s.dmStream.Seek(position)
+	//	if err != nil {
+	//		return err
+	//	}
+	//}
+	return nil
 }
 
 func (s *segmentManager) loadSegment(collectionID UniqueID, partitionID UniqueID, segmentIDs []UniqueID, fieldIDs []int64) error {
@@ -81,7 +61,11 @@ func (s *segmentManager) loadSegment(collectionID UniqueID, partitionID UniqueID
 		}
 
 		targetFields := s.filterOutNeedlessFields(paths, srcFieldIDs, fieldIDs)
-		// create segment
+		// replace segment
+		err = s.replica.removeSegment(segmentID)
+		if err != nil {
+			return err
+		}
 		err = s.replica.addSegment(segmentID, partitionID, collectionID, segTypeSealed)
 		if err != nil {
 			return err
@@ -118,16 +102,25 @@ func (s *segmentManager) getInsertBinlogPaths(segmentID UniqueID) ([]*internalPb
 		SegmentID: segmentID,
 	}
 
-	pathResponse, err := s.dataClient.GetInsertBinlogPaths(context.TODO(), insertBinlogPathRequest)
+	pathResponse, err := s.dataClient.GetInsertBinlogPaths(insertBinlogPathRequest.SegmentID)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	if len(pathResponse.FieldIDs) != len(pathResponse.Paths) {
-		return nil, nil, errors.New("illegal InsertBinlogPathsResponse")
+	//if len(pathResponse.FieldIDs) != len(pathResponse.Paths) {
+	//	return nil, nil, errors.New("illegal InsertBinlogPathsResponse")
+	//}
+
+	fieldIDs := make([]int64, 0)
+	paths := make([]*internalPb.StringList, 0)
+	for k, v := range pathResponse {
+		fieldIDs = append(fieldIDs, k)
+		paths = append(paths, &internalPb.StringList{
+			Values: v,
+		})
 	}
 
-	return pathResponse.Paths, pathResponse.FieldIDs, nil
+	return paths, fieldIDs, nil
 }
 
 func (s *segmentManager) filterOutNeedlessFields(paths []*internalPb.StringList, srcFieldIDS []int64, dstFields []int64) map[int64]*internalPb.StringList {
@@ -234,12 +227,15 @@ func (s *segmentManager) getIndexPaths(indexID UniqueID) ([]string, error) {
 	indexFilePathRequest := &indexpb.IndexFilePathsRequest{
 		IndexIDs: []UniqueID{indexID},
 	}
-	pathResponse, err := s.indexBuilderClient.GetIndexFilePaths(context.TODO(), indexFilePathRequest)
-	if err != nil || pathResponse.Status.ErrorCode != commonpb.ErrorCode_SUCCESS {
+	pathResponse, err := s.indexClient.GetIndexFilePaths(indexFilePathRequest.IndexIDs)
+	//if err != nil || pathResponse.Status.ErrorCode != commonpb.ErrorCode_SUCCESS {
+	//	return nil, err
+	//}
+	if err != nil {
 		return nil, err
 	}
 
-	return pathResponse.FilePaths[0].IndexFilePaths, nil
+	return pathResponse[0], nil
 }
 
 func (s *segmentManager) getIndexParam() (indexParam, error) {
@@ -293,3 +289,42 @@ func (s *segmentManager) sendLoadIndex(indexPaths []string,
 	messages := []msgstream.TsMsg{loadIndexMsg}
 	s.loadIndexReqChan <- messages
 }
+
+func newSegmentManager(ctx context.Context, replica collectionReplica, dmStream msgstream.MsgStream, loadIndexReqChan chan []msgstream.TsMsg) *segmentManager {
+	bucketName := Params.MinioBucketName
+	option := &miniokv.Option{
+		Address:           Params.MinioEndPoint,
+		AccessKeyID:       Params.MinioAccessKeyID,
+		SecretAccessKeyID: Params.MinioSecretAccessKey,
+		UseSSL:            Params.MinioUseSSLStr,
+		BucketName:        bucketName,
+		CreateBucket:      true,
+	}
+
+	minioKV, err := miniokv.NewMinIOKV(ctx, option)
+	if err != nil {
+		panic(err)
+	}
+
+	dataClient, err := writerclient.NewWriterClient(Params.ETCDAddress, Params.MetaRootPath, Params.WriteNodeSegKvSubPath, nil)
+	if err != nil {
+		panic(err)
+	}
+
+	indexClient, err := indexnodeclient.NewBuildIndexClient(ctx, Params.IndexBuilderAddress)
+	if err != nil {
+		panic(err)
+	}
+
+	return &segmentManager{
+		replica:          replica,
+		dmStream:         dmStream,
+		loadIndexReqChan: loadIndexReqChan,
+
+		dataClient:  dataClient,
+		indexClient: indexClient,
+
+		kv:     minioKV,
+		iCodec: &storage.InsertCodec{},
+	}
+}

internal/querynode/segment_manager_test.go

@@ -16,6 +16,9 @@ import (
 
 	"github.com/zilliztech/milvus-distributed/internal/indexnode"
 	minioKV "github.com/zilliztech/milvus-distributed/internal/kv/minio"
+	"github.com/zilliztech/milvus-distributed/internal/msgstream"
+	"github.com/zilliztech/milvus-distributed/internal/msgstream/pulsarms"
+	"github.com/zilliztech/milvus-distributed/internal/msgstream/util"
 	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
 	internalPb "github.com/zilliztech/milvus-distributed/internal/proto/internalpb2"
 	"github.com/zilliztech/milvus-distributed/internal/proto/milvuspb"
@@ -23,7 +26,7 @@ import (
 	"github.com/zilliztech/milvus-distributed/internal/storage"
 )
 
-func generateInsertBinLog(collectionID UniqueID, partitionID UniqueID, segmentID UniqueID) ([]*internalPb.StringList, []int64, error) {
+func generateInsertBinLog(collectionID UniqueID, partitionID UniqueID, segmentID UniqueID, keyPrefix string) ([]*internalPb.StringList, []int64, error) {
 	const (
 		msgLength = 1000
 		DIM       = 16
@@ -108,10 +111,8 @@ func generateInsertBinLog(collectionID UniqueID, partitionID UniqueID, segmentID
 	}
 
 	// binLogs -> minIO/S3
-	collIDStr := strconv.FormatInt(collectionID, 10)
-	partitionIDStr := strconv.FormatInt(partitionID, 10)
 	segIDStr := strconv.FormatInt(segmentID, 10)
-	keyPrefix := path.Join("query-node-seg-manager-test-minio-prefix", collIDStr, partitionIDStr, segIDStr)
+	keyPrefix = path.Join(keyPrefix, segIDStr)
 
 	paths := make([]*internalPb.StringList, 0)
 	fieldIDs := make([]int64, 0)
@@ -214,18 +215,197 @@ func generateIndex(segmentID UniqueID) ([]string, indexParam, error) {
 	return indexPaths, indexParams, nil
 }
 
+func doInsert(ctx context.Context, collectionName string, partitionTag string, segmentID UniqueID) error {
+	const msgLength = 1000
+	const DIM = 16
+
+	var vec = [DIM]float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
+	var rawData []byte
+	for _, ele := range vec {
+		buf := make([]byte, 4)
+		binary.LittleEndian.PutUint32(buf, math.Float32bits(ele))
+		rawData = append(rawData, buf...)
+	}
+	bs := make([]byte, 4)
+	binary.LittleEndian.PutUint32(bs, 1)
+	rawData = append(rawData, bs...)
+
+	timeRange := TimeRange{
+		timestampMin: 0,
+		timestampMax: math.MaxUint64,
+	}
+
+	// messages generate
+	insertMessages := make([]msgstream.TsMsg, 0)
+	for i := 0; i < msgLength; i++ {
+		var msg msgstream.TsMsg = &msgstream.InsertMsg{
+			BaseMsg: msgstream.BaseMsg{
+				HashValues: []uint32{
+					uint32(i),
+				},
+			},
+			InsertRequest: internalPb.InsertRequest{
+				Base: &commonpb.MsgBase{
+					MsgType:   commonpb.MsgType_kInsert,
+					MsgID:     0,
+					Timestamp: uint64(i + 1000),
+					SourceID:  0,
+				},
+				CollectionName: collectionName,
+				PartitionName:  partitionTag,
+				SegmentID:      segmentID,
+				ChannelID:      "0",
+				Timestamps:     []uint64{uint64(i + 1000)},
+				RowIDs:         []int64{int64(i)},
+				RowData: []*commonpb.Blob{
+					{Value: rawData},
+				},
+			},
+		}
+		insertMessages = append(insertMessages, msg)
+	}
+
+	msgPack := msgstream.MsgPack{
+		BeginTs: timeRange.timestampMin,
+		EndTs:   timeRange.timestampMax,
+		Msgs:    insertMessages,
+	}
+
+	// generate timeTick
+	timeTickMsgPack := msgstream.MsgPack{}
+	baseMsg := msgstream.BaseMsg{
+		BeginTimestamp: 1000,
+		EndTimestamp:   1500,
+		HashValues:     []uint32{0},
+	}
+	timeTickResult := internalPb.TimeTickMsg{
+		Base: &commonpb.MsgBase{
+			MsgType:   commonpb.MsgType_kTimeTick,
+			MsgID:     0,
+			Timestamp: 1000,
+			SourceID:  0,
+		},
+	}
+	timeTickMsg := &msgstream.TimeTickMsg{
+		BaseMsg:     baseMsg,
+		TimeTickMsg: timeTickResult,
+	}
+	timeTickMsgPack.Msgs = append(timeTickMsgPack.Msgs, timeTickMsg)
+
+	// pulsar produce
+	const receiveBufSize = 1024
+	insertChannels := Params.InsertChannelNames
+	ddChannels := Params.DDChannelNames
+	pulsarURL := Params.PulsarAddress
+
+	insertStream := pulsarms.NewPulsarMsgStream(ctx, receiveBufSize)
+	insertStream.SetPulsarClient(pulsarURL)
+	insertStream.CreatePulsarProducers(insertChannels)
+	unmarshalDispatcher := util.NewUnmarshalDispatcher()
+	insertStream.CreatePulsarConsumers(insertChannels, Params.MsgChannelSubName, unmarshalDispatcher, receiveBufSize)
+
+	ddStream := pulsarms.NewPulsarMsgStream(ctx, receiveBufSize)
+	ddStream.SetPulsarClient(pulsarURL)
+	ddStream.CreatePulsarProducers(ddChannels)
+
+	var insertMsgStream msgstream.MsgStream = insertStream
+	insertMsgStream.Start()
+
+	var ddMsgStream msgstream.MsgStream = ddStream
+	ddMsgStream.Start()
+
+	err := insertMsgStream.Produce(&msgPack)
+	if err != nil {
+		return err
+	}
+
+	err = insertMsgStream.Broadcast(&timeTickMsgPack)
+	if err != nil {
+		return err
+	}
+	err = ddMsgStream.Broadcast(&timeTickMsgPack)
+	if err != nil {
+		return err
+	}
+
+	//messages := insertStream.Consume()
+	//for _, msg := range messages.Msgs {
+	//
+	//}
+
+	return nil
+}
+
+func sentTimeTick(ctx context.Context) error {
+	timeTickMsgPack := msgstream.MsgPack{}
+	baseMsg := msgstream.BaseMsg{
+		BeginTimestamp: 1500,
+		EndTimestamp:   2000,
+		HashValues:     []uint32{0},
+	}
+	timeTickResult := internalPb.TimeTickMsg{
+		Base: &commonpb.MsgBase{
+			MsgType:   commonpb.MsgType_kTimeTick,
+			MsgID:     0,
+			Timestamp: math.MaxUint64,
+			SourceID:  0,
+		},
+	}
+	timeTickMsg := &msgstream.TimeTickMsg{
+		BaseMsg:     baseMsg,
+		TimeTickMsg: timeTickResult,
+	}
+	timeTickMsgPack.Msgs = append(timeTickMsgPack.Msgs, timeTickMsg)
+
+	// pulsar produce
+	const receiveBufSize = 1024
+	insertChannels := Params.InsertChannelNames
+	ddChannels := Params.DDChannelNames
+	pulsarURL := Params.PulsarAddress
+
+	insertStream := pulsarms.NewPulsarMsgStream(ctx, receiveBufSize)
+	insertStream.SetPulsarClient(pulsarURL)
+	insertStream.CreatePulsarProducers(insertChannels)
+	unmarshalDispatcher := util.NewUnmarshalDispatcher()
+	insertStream.CreatePulsarConsumers(insertChannels, Params.MsgChannelSubName, unmarshalDispatcher, receiveBufSize)
+
+	ddStream := pulsarms.NewPulsarMsgStream(ctx, receiveBufSize)
+	ddStream.SetPulsarClient(pulsarURL)
+	ddStream.CreatePulsarProducers(ddChannels)
+
+	var insertMsgStream msgstream.MsgStream = insertStream
+	insertMsgStream.Start()
+
+	var ddMsgStream msgstream.MsgStream = ddStream
+	ddMsgStream.Start()
+
+	err := insertMsgStream.Broadcast(&timeTickMsgPack)
+	if err != nil {
+		return err
+	}
+	err = ddMsgStream.Broadcast(&timeTickMsgPack)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func TestSegmentManager_load_release_and_search(t *testing.T) {
 	collectionID := UniqueID(0)
 	partitionID := UniqueID(1)
 	segmentID := UniqueID(2)
 	fieldIDs := []int64{0, 101}
 
+	// mock write insert bin log
+	keyPrefix := path.Join("query-node-seg-manager-test-minio-prefix", strconv.FormatInt(collectionID, 10), strconv.FormatInt(partitionID, 10))
+	Params.WriteNodeSegKvSubPath = keyPrefix
+
 	node := newQueryNodeMock()
 	defer node.Stop()
 
 	ctx := node.queryNodeLoopCtx
 	node.loadIndexService = newLoadIndexService(ctx, node.replica)
-	node.segManager = newSegmentManager(ctx, node.replica, node.loadIndexService.loadIndexReqChan)
+	node.segManager = newSegmentManager(ctx, node.replica, nil, node.loadIndexService.loadIndexReqChan)
 	go node.loadIndexService.start()
 
 	collectionName := "collection0"
@@ -237,7 +417,7 @@ func TestSegmentManager_load_release_and_search(t *testing.T) {
 	err = node.replica.addSegment(segmentID, partitionID, collectionID, segTypeSealed)
 	assert.NoError(t, err)
 
-	paths, srcFieldIDs, err := generateInsertBinLog(collectionID, partitionID, segmentID)
+	paths, srcFieldIDs, err := generateInsertBinLog(collectionID, partitionID, segmentID, keyPrefix)
 	assert.NoError(t, err)
 
 	fieldsMap := node.segManager.filterOutNeedlessFields(paths, srcFieldIDs, fieldIDs)
@@ -299,3 +479,111 @@ func TestSegmentManager_load_release_and_search(t *testing.T) {
 
 	<-ctx.Done()
 }
+
+//// NOTE: start pulsar before test
+//func TestSegmentManager_with_seek(t *testing.T) {
+//	collectionID := UniqueID(0)
+//	partitionID := UniqueID(1)
+//	//segmentID := UniqueID(2)
+//	fieldIDs := []int64{0, 101}
+//
+//	//// mock write insert bin log
+//	//keyPrefix := path.Join("query-node-seg-manager-test-minio-prefix", strconv.FormatInt(collectionID, 10), strconv.FormatInt(partitionID, 10))
+//	//Params.WriteNodeSegKvSubPath = keyPrefix + "/"
+//	node := newQueryNodeMock()
+//
+//	ctx := node.queryNodeLoopCtx
+//	go node.Start()
+//
+//	collectionName := "collection0"
+//	initTestMeta(t, node, collectionName, collectionID, 0)
+//
+//	err := node.replica.addPartition(collectionID, partitionID)
+//	assert.NoError(t, err)
+//
+//	//err = node.replica.addSegment(segmentID, partitionID, collectionID, segTypeSealed)
+//	//assert.NoError(t, err)
+//
+//	//paths, srcFieldIDs, err := generateInsertBinLog(collectionID, partitionID, segmentID, keyPrefix)
+//	//assert.NoError(t, err)
+//
+//	//fieldsMap := node.segManager.filterOutNeedlessFields(paths, srcFieldIDs, fieldIDs)
+//	//assert.Equal(t, len(fieldsMap), 2)
+//
+//	segmentIDToInsert := UniqueID(3)
+//	err = doInsert(ctx, collectionName, "default", segmentIDToInsert)
+//	assert.NoError(t, err)
+//
+//	startPositions := make([]*internalPb.MsgPosition, 0)
+//	for _, ch := range Params.InsertChannelNames {
+//		startPositions = append(startPositions, &internalPb.MsgPosition{
+//			ChannelName: ch,
+//		})
+//	}
+//	var positions []*internalPb.MsgPosition
+//	lastSegStates := &datapb.SegmentStatesResponse{
+//		State:          datapb.SegmentState_SegmentGrowing,
+//		StartPositions: positions,
+//	}
+//	loadReq := &querypb.LoadSegmentRequest{
+//		CollectionID:     collectionID,
+//		PartitionID:      partitionID,
+//		SegmentIDs:       []UniqueID{segmentIDToInsert},
+//		FieldIDs:         fieldIDs,
+//		LastSegmentState: lastSegStates,
+//	}
+//	_, err = node.LoadSegments(loadReq)
+//	assert.NoError(t, err)
+//
+//	err = sentTimeTick(ctx)
+//	assert.NoError(t, err)
+//
+//	// do search
+//	dslString := "{\"bool\": { \n\"vector\": {\n \"vec\": {\n \"metric_type\": \"L2\", \n \"params\": {\n \"nprobe\": 10 \n},\n \"query\": \"$0\",\"topk\": 10 \n } \n } \n } \n }"
+//
+//	const DIM = 16
+//	var searchRawData []byte
+//	var vec = [DIM]float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
+//	for _, ele := range vec {
+//		buf := make([]byte, 4)
+//		binary.LittleEndian.PutUint32(buf, math.Float32bits(ele))
+//		searchRawData = append(searchRawData, buf...)
+//	}
+//	placeholderValue := milvuspb.PlaceholderValue{
+//		Tag:    "$0",
+//		Type:   milvuspb.PlaceholderType_VECTOR_FLOAT,
+//		Values: [][]byte{searchRawData},
+//	}
+//
+//	placeholderGroup := milvuspb.PlaceholderGroup{
+//		Placeholders: []*milvuspb.PlaceholderValue{&placeholderValue},
+//	}
+//
+//	placeHolderGroupBlob, err := proto.Marshal(&placeholderGroup)
+//	assert.NoError(t, err)
+//
+//	//searchTimestamp := Timestamp(1020)
+//	collection, err := node.replica.getCollectionByID(collectionID)
+//	assert.NoError(t, err)
+//	plan, err := createPlan(*collection, dslString)
+//	assert.NoError(t, err)
+//	holder, err := parserPlaceholderGroup(plan, placeHolderGroupBlob)
+//	assert.NoError(t, err)
+//	placeholderGroups := make([]*PlaceholderGroup, 0)
+//	placeholderGroups = append(placeholderGroups, holder)
+//
+//	// wait for segment building index
+//	time.Sleep(3 * time.Second)
+//
+//	//segment, err := node.replica.getSegmentByID(segmentIDToInsert)
+//	//assert.NoError(t, err)
+//	//_, err = segment.segmentSearch(plan, placeholderGroups, []Timestamp{searchTimestamp})
+//	//assert.Nil(t, err)
+//
+//	plan.delete()
+//	holder.delete()
+//
+//	<-ctx.Done()
+//	err = node.Stop()
+//	assert.NoError(t, err)
+//}

internal/util/paramtable/paramtable.go

@@ -145,7 +145,7 @@ func (gp *BaseTable) tryloadFromEnv() {
 		panic(err)
 	}
 
-	indexBuilderAddress := os.Getenv("INDEX_BUILDER_ADDRESS")
+	indexBuilderAddress := os.Getenv("INDEX_SERVICE_ADDRESS")
 	if indexBuilderAddress == "" {
 		indexBuilderHost, err := gp.Load("indexBuilder.address")
 		if err != nil {
@@ -157,7 +157,7 @@ func (gp *BaseTable) tryloadFromEnv() {
 		}
 		indexBuilderAddress = indexBuilderHost + ":" + port
 	}
-	err = gp.Save("_IndexBuilderAddress", indexBuilderAddress)
+	err = gp.Save("IndexServiceAddress", indexBuilderAddress)
 	if err != nil {
 		panic(err)
 	}

internal/util/retry/retry.go

@@ -1,4 +1,4 @@
-package indexnode
+package retry
 
 import (
 	"log"
@@ -7,7 +7,7 @@ import (
 
 // Reference: https://blog.cyeam.com/golang/2018/08/27/retry
 
-func RetryImpl(attempts int, sleep time.Duration, fn func() error, maxSleepTime time.Duration) error {
+func Impl(attempts int, sleep time.Duration, fn func() error, maxSleepTime time.Duration) error {
 	if err := fn(); err != nil {
 		if s, ok := err.(InterruptError); ok {
 			return s.error
@@ -17,9 +17,9 @@ func RetryImpl(attempts int, sleep time.Duration, fn func() error, maxSleepTime
 			log.Printf("retry func error: %s. attempts #%d after %s.", err.Error(), attempts, sleep)
 			time.Sleep(sleep)
 			if sleep < maxSleepTime {
-				return RetryImpl(attempts, 2*sleep, fn, maxSleepTime)
+				return Impl(attempts, 2*sleep, fn, maxSleepTime)
 			}
-			return RetryImpl(attempts, maxSleepTime, fn, maxSleepTime)
+			return Impl(attempts, maxSleepTime, fn, maxSleepTime)
 		}
 		return err
 	}
@@ -28,7 +28,7 @@ func RetryImpl(attempts int, sleep time.Duration, fn func() error, maxSleepTime
 
 func Retry(attempts int, sleep time.Duration, fn func() error) error {
 	maxSleepTime := time.Millisecond * 1000
-	return RetryImpl(attempts, sleep, fn, maxSleepTime)
+	return Impl(attempts, sleep, fn, maxSleepTime)
 }
 
 type InterruptError struct {

internal/util/rocksmq/global_allocator.go

@@ -0,0 +1,167 @@
+package rocksmq
+
+import (
+	"errors"
+	"log"
+	"sync/atomic"
+	"time"
+
+	"github.com/zilliztech/milvus-distributed/internal/kv"
+	"github.com/zilliztech/milvus-distributed/internal/util/tsoutil"
+	"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
+	"go.uber.org/zap"
+)
+
+// Allocator is a Timestamp Oracle allocator.
+type Allocator interface {
+	// Initialize is used to initialize a TSO allocator.
+	// It will synchronize TSO with etcd and initialize the
+	// memory for later allocation work.
+	Initialize() error
+	// UpdateTSO is used to update the TSO in memory and the time window in etcd.
+	UpdateTSO() error
+	// SetTSO sets the physical part with given tso. It's mainly used for BR restore
+	// and can not forcibly set the TSO smaller than now.
+	SetTSO(tso uint64) error
+	// GenerateTSO is used to generate a given number of TSOs.
+	// Make sure you have initialized the TSO allocator before calling.
+	GenerateTSO(count uint32) (uint64, error)
+	// Reset is used to reset the TSO allocator.
+	Reset()
+}
+
+// GlobalTSOAllocator is the global single point TSO allocator.
+type GlobalTSOAllocator struct {
+	tso *timestampOracle
+}
+
+// NewGlobalTSOAllocator creates a new global TSO allocator.
+func NewGlobalTSOAllocator(key string, kvBase kv.TxnBase) *GlobalTSOAllocator {
+	var saveInterval = 3 * time.Second
+	return &GlobalTSOAllocator{
+		tso: &timestampOracle{
+			kvBase:        kvBase,
+			saveInterval:  saveInterval,
+			maxResetTSGap: func() time.Duration { return 3 * time.Second },
+			key:           key,
+		},
+	}
+}
+
+// Initialize will initialize the created global TSO allocator.
+func (gta *GlobalTSOAllocator) Initialize() error {
+	return gta.tso.InitTimestamp()
+}
+
+// UpdateTSO is used to update the TSO in memory and the time window in etcd.
+func (gta *GlobalTSOAllocator) UpdateTSO() error {
+	return gta.tso.UpdateTimestamp()
+}
+
+// SetTSO sets the physical part with given tso.
+func (gta *GlobalTSOAllocator) SetTSO(tso uint64) error {
+	return gta.tso.ResetUserTimestamp(tso)
+}
+
+// GenerateTSO is used to generate a given number of TSOs.
+// Make sure you have initialized the TSO allocator before calling.
+func (gta *GlobalTSOAllocator) GenerateTSO(count uint32) (uint64, error) {
+	var physical, logical int64
+	if count == 0 {
+		return 0, errors.New("tso count should be positive")
+	}
+
+	maxRetryCount := 10
+
+	for i := 0; i < maxRetryCount; i++ {
+		current := (*atomicObject)(atomic.LoadPointer(&gta.tso.TSO))
+		if current == nil || current.physical.Equal(typeutil.ZeroTime) {
+			// If it's leader, maybe SyncTimestamp hasn't completed yet
+			log.Println("sync hasn't completed yet, wait for a while")
+			time.Sleep(200 * time.Millisecond)
+			continue
+		}
+
+		physical = current.physical.UnixNano() / int64(time.Millisecond)
+		logical = atomic.AddInt64(&current.logical, int64(count))
+		if logical >= maxLogical {
+			log.Println("logical part outside of max logical interval, please check ntp time",
+				zap.Int("retry-count", i))
+			time.Sleep(UpdateTimestampStep)
+			continue
+		}
+		return tsoutil.ComposeTS(physical, logical), nil
+	}
+	return 0, errors.New("can not get timestamp")
+}
+
+func (gta *GlobalTSOAllocator) Alloc(count uint32) (typeutil.Timestamp, error) {
+	//return gta.tso.SyncTimestamp()
+	start, err := gta.GenerateTSO(count)
+	if err != nil {
+		return typeutil.ZeroTimestamp, err
+	}
+	//ret := make([]typeutil.Timestamp, count)
+	//for i:=uint32(0); i < count; i++{
+	//	ret[i] = start + uint64(i)
+	//}
+	return start, err
+}
+
+func (gta *GlobalTSOAllocator) AllocOne() (typeutil.Timestamp, error) {
+	return gta.GenerateTSO(1)
+}
+
+// Reset is used to reset the TSO allocator.
+func (gta *GlobalTSOAllocator) Reset() {
+	gta.tso.ResetTimestamp()
+}
+
+///////////////////////////////////////////////////////////////////////
+
+type IDAllocator interface {
+	Alloc(count uint32) (UniqueID, UniqueID, error)
+	AllocOne() (UniqueID, error)
+	UpdateID() error
+}
+
+// GlobalTSOAllocator is the global single point TSO allocator.
+type GlobalIDAllocator struct {
+	allocator Allocator
+}
+
+func NewGlobalIDAllocator(key string, base kv.TxnBase) *GlobalIDAllocator {
+	return &GlobalIDAllocator{
+		allocator: NewGlobalTSOAllocator(key, base),
+	}
+}
+
+// Initialize will initialize the created global TSO allocator.
+func (gia *GlobalIDAllocator) Initialize() error {
+	return gia.allocator.Initialize()
+}
+
+// GenerateTSO is used to generate a given number of TSOs.
+// Make sure you have initialized the TSO allocator before calling.
+func (gia *GlobalIDAllocator) Alloc(count uint32) (UniqueID, UniqueID, error) {
+	timestamp, err := gia.allocator.GenerateTSO(count)
+	if err != nil {
+		return 0, 0, err
+	}
+	idStart := UniqueID(timestamp)
+	idEnd := idStart + int64(count)
+	return idStart, idEnd, nil
+}
+
+func (gia *GlobalIDAllocator) AllocOne() (UniqueID, error) {
+	timestamp, err := gia.allocator.GenerateTSO(1)
+	if err != nil {
+		return 0, err
+	}
+	idStart := UniqueID(timestamp)
+	return idStart, nil
+}
+
+func (gia *GlobalIDAllocator) UpdateID() error {
+	return gia.allocator.UpdateTSO()
+}

internal/util/rocksmq/global_rmq.go

@@ -0,0 +1,13 @@
+package rocksmq
+
+var rmq *RocksMQ
+
+func InitRmq(rocksdbName string, idAllocator IDAllocator) error {
+	var err error
+	rmq, err = NewRocksMQ(rocksdbName, idAllocator)
+	return err
+}
+
+func GetRmq() *RocksMQ {
+	return rmq
+}

internal/util/rocksmq/rocksmq.go

@@ -7,7 +7,6 @@ import (
 	"github.com/tecbot/gorocksdb"
 	"github.com/zilliztech/milvus-distributed/internal/errors"
 	"github.com/zilliztech/milvus-distributed/internal/kv"
-	"github.com/zilliztech/milvus-distributed/internal/master"
 	"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
 
 	memkv "github.com/zilliztech/milvus-distributed/internal/kv/mem"
@@ -73,7 +72,7 @@ type RocksMQ struct {
 	kv          kv.Base
 	channels    map[string]*Channel
 	cgCtxs      map[string]ConsumerGroupContext
-	idAllocator master.IDAllocator
+	idAllocator IDAllocator
 	produceMu   sync.Mutex
 	consumeMu   sync.Mutex
 	//ctx              context.Context
@@ -85,7 +84,7 @@ type RocksMQ struct {
 	//tsoTicker *time.Ticker
 }
 
-func NewRocksMQ(name string, idAllocator master.IDAllocator) (*RocksMQ, error) {
+func NewRocksMQ(name string, idAllocator IDAllocator) (*RocksMQ, error) {
 	bbto := gorocksdb.NewDefaultBlockBasedTableOptions()
 	bbto.SetBlockCache(gorocksdb.NewLRUCache(RocksDBLRUCacheCapacity))
 	opts := gorocksdb.NewDefaultOptions()

internal/util/rocksmq/rocksmq_test.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	etcdkv "github.com/zilliztech/milvus-distributed/internal/kv/etcd"
-	master "github.com/zilliztech/milvus-distributed/internal/master"
 	"go.etcd.io/etcd/clientv3"
 )
 
@@ -20,14 +19,15 @@ func TestFixChannelName(t *testing.T) {
 }
 
 func TestRocksMQ(t *testing.T) {
-	master.Init()
-
-	etcdAddr := master.Params.EtcdAddress
+	etcdAddr := os.Getenv("ETCD_ADDRESS")
+	if etcdAddr == "" {
+		etcdAddr = "localhost:2379"
+	}
 	cli, err := clientv3.New(clientv3.Config{Endpoints: []string{etcdAddr}})
 	assert.Nil(t, err)
 	etcdKV := etcdkv.NewEtcdKV(cli, "/etcd/test/root")
 	defer etcdKV.Close()
-	idAllocator := master.NewGlobalIDAllocator("dummy", etcdKV)
+	idAllocator := NewGlobalIDAllocator("dummy", etcdKV)
 	_ = idAllocator.Initialize()
 
 	name := "/tmp/rocksmq"
@@ -76,14 +76,15 @@ func TestRocksMQ(t *testing.T) {
 }
 
 func TestRocksMQ_Loop(t *testing.T) {
-	master.Init()
-
-	etcdAddr := master.Params.EtcdAddress
+	etcdAddr := os.Getenv("ETCD_ADDRESS")
+	if etcdAddr == "" {
+		etcdAddr = "localhost:2379"
+	}
 	cli, err := clientv3.New(clientv3.Config{Endpoints: []string{etcdAddr}})
 	assert.Nil(t, err)
 	etcdKV := etcdkv.NewEtcdKV(cli, "/etcd/test/root")
 	defer etcdKV.Close()
-	idAllocator := master.NewGlobalIDAllocator("dummy", etcdKV)
+	idAllocator := NewGlobalIDAllocator("dummy", etcdKV)
 	_ = idAllocator.Initialize()
 
 	name := "/tmp/rocksmq_1"
@@ -143,14 +144,15 @@ func TestRocksMQ_Loop(t *testing.T) {
 }
 
 func TestRocksMQ_Goroutines(t *testing.T) {
-	master.Init()
-
-	etcdAddr := master.Params.EtcdAddress
+	etcdAddr := os.Getenv("ETCD_ADDRESS")
+	if etcdAddr == "" {
+		etcdAddr = "localhost:2379"
+	}
 	cli, err := clientv3.New(clientv3.Config{Endpoints: []string{etcdAddr}})
 	assert.Nil(t, err)
 	etcdKV := etcdkv.NewEtcdKV(cli, "/etcd/test/root")
 	defer etcdKV.Close()
-	idAllocator := master.NewGlobalIDAllocator("dummy", etcdKV)
+	idAllocator := NewGlobalIDAllocator("dummy", etcdKV)
 	_ = idAllocator.Initialize()
 
 	name := "/tmp/rocksmq_2"

internal/util/rocksmq/tso.go

@@ -0,0 +1,202 @@
+// Copyright 2016 TiKV Project Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rocksmq
+
+import (
+	"log"
+	"sync/atomic"
+	"time"
+	"unsafe"
+
+	"go.uber.org/zap"
+
+	"github.com/zilliztech/milvus-distributed/internal/errors"
+	"github.com/zilliztech/milvus-distributed/internal/kv"
+	"github.com/zilliztech/milvus-distributed/internal/util/tsoutil"
+	"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
+)
+
+const (
+	// UpdateTimestampStep is used to update timestamp.
+	UpdateTimestampStep = 50 * time.Millisecond
+	// updateTimestampGuard is the min timestamp interval.
+	updateTimestampGuard = time.Millisecond
+	// maxLogical is the max upper limit for logical time.
+	// When a TSO's logical time reaches this limit,
+	// the physical time will be forced to increase.
+	maxLogical = int64(1 << 18)
+)
+
+// atomicObject is used to store the current TSO in memory.
+type atomicObject struct {
+	physical time.Time
+	logical  int64
+}
+
+// timestampOracle is used to maintain the logic of tso.
+type timestampOracle struct {
+	key    string
+	kvBase kv.TxnBase
+
+	// TODO: remove saveInterval
+	saveInterval  time.Duration
+	maxResetTSGap func() time.Duration
+	// For tso, set after the PD becomes a leader.
+	TSO           unsafe.Pointer
+	lastSavedTime atomic.Value
+}
+
+func (t *timestampOracle) loadTimestamp() (time.Time, error) {
+	strData, err := t.kvBase.Load(t.key)
+
+	var binData []byte = []byte(strData)
+
+	if err != nil {
+		return typeutil.ZeroTime, err
+	}
+	if len(binData) == 0 {
+		return typeutil.ZeroTime, nil
+	}
+	return typeutil.ParseTimestamp(binData)
+}
+
+// save timestamp, if lastTs is 0, we think the timestamp doesn't exist, so create it,
+// otherwise, update it.
+func (t *timestampOracle) saveTimestamp(ts time.Time) error {
+	data := typeutil.Uint64ToBytes(uint64(ts.UnixNano()))
+	err := t.kvBase.Save(t.key, string(data))
+	if err != nil {
+		return errors.WithStack(err)
+	}
+	t.lastSavedTime.Store(ts)
+	return nil
+}
+
+func (t *timestampOracle) InitTimestamp() error {
+
+	//last, err := t.loadTimestamp()
+	//if err != nil {
+	//	return err
+	//}
+
+	next := time.Now()
+
+	// If the current system time minus the saved etcd timestamp is less than `updateTimestampGuard`,
+	// the timestamp allocation will start from the saved etcd timestamp temporarily.
+	//if typeutil.SubTimeByWallClock(next, last) < updateTimestampGuard {
+	//	next = last.Add(updateTimestampGuard)
+	//}
+
+	save := next.Add(t.saveInterval)
+	if err := t.saveTimestamp(save); err != nil {
+		return err
+	}
+
+	//log.Print("sync and save timestamp", zap.Time("last", last), zap.Time("save", save), zap.Time("next", next))
+
+	current := &atomicObject{
+		physical: next,
+	}
+	atomic.StorePointer(&t.TSO, unsafe.Pointer(current))
+
+	return nil
+}
+
+// ResetUserTimestamp update the physical part with specified tso.
+func (t *timestampOracle) ResetUserTimestamp(tso uint64) error {
+	physical, _ := tsoutil.ParseTS(tso)
+	next := physical.Add(time.Millisecond)
+	prev := (*atomicObject)(atomic.LoadPointer(&t.TSO))
+
+	// do not update
+	if typeutil.SubTimeByWallClock(next, prev.physical) <= 3*updateTimestampGuard {
+		return errors.New("the specified ts too small than now")
+	}
+
+	if typeutil.SubTimeByWallClock(next, prev.physical) >= t.maxResetTSGap() {
+		return errors.New("the specified ts too large than now")
+	}
+
+	save := next.Add(t.saveInterval)
+	if err := t.saveTimestamp(save); err != nil {
+		return err
+	}
+	update := &atomicObject{
+		physical: next,
+	}
+	atomic.CompareAndSwapPointer(&t.TSO, unsafe.Pointer(prev), unsafe.Pointer(update))
+	return nil
+}
+
+// UpdateTimestamp is used to update the timestamp.
+// This function will do two things:
+// 1. When the logical time is going to be used up, increase the current physical time.
+// 2. When the time window is not big enough, which means the saved etcd time minus the next physical time
+//    will be less than or equal to `updateTimestampGuard`, then the time window needs to be updated and
+//    we also need to save the next physical time plus `TsoSaveInterval` into etcd.
+//
+// Here is some constraints that this function must satisfy:
+// 1. The saved time is monotonically increasing.
+// 2. The physical time is monotonically increasing.
+// 3. The physical time is always less than the saved timestamp.
+func (t *timestampOracle) UpdateTimestamp() error {
+	prev := (*atomicObject)(atomic.LoadPointer(&t.TSO))
+	now := time.Now()
+
+	jetLag := typeutil.SubTimeByWallClock(now, prev.physical)
+	if jetLag > 3*UpdateTimestampStep {
+		log.Print("clock offset", zap.Duration("jet-lag", jetLag), zap.Time("prev-physical", prev.physical), zap.Time("now", now))
+	}
+
+	var next time.Time
+	prevLogical := atomic.LoadInt64(&prev.logical)
+	// If the system time is greater, it will be synchronized with the system time.
+	if jetLag > updateTimestampGuard {
+		next = now
+	} else if prevLogical > maxLogical/2 {
+		// The reason choosing maxLogical/2 here is that it's big enough for common cases.
+		// Because there is enough timestamp can be allocated before next update.
+		log.Print("the logical time may be not enough", zap.Int64("prev-logical", prevLogical))
+		next = prev.physical.Add(time.Millisecond)
+	} else {
+		// It will still use the previous physical time to alloc the timestamp.
+		return nil
+	}
+
+	// It is not safe to increase the physical time to `next`.
+	// The time window needs to be updated and saved to etcd.
+	if typeutil.SubTimeByWallClock(t.lastSavedTime.Load().(time.Time), next) <= updateTimestampGuard {
+		save := next.Add(t.saveInterval)
+		if err := t.saveTimestamp(save); err != nil {
+			return err
+		}
+	}
+
+	current := &atomicObject{
+		physical: next,
+		logical:  0,
+	}
+
+	atomic.StorePointer(&t.TSO, unsafe.Pointer(current))
+
+	return nil
+}
+
+// ResetTimestamp is used to reset the timestamp.
+func (t *timestampOracle) ResetTimestamp() {
+	zero := &atomicObject{
+		physical: time.Now(),
+	}
+	atomic.StorePointer(&t.TSO, unsafe.Pointer(zero))
+}

internal/util/typeutil/interface.go

@@ -2,6 +2,7 @@ package typeutil
 
 import (
 	"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
+	"github.com/zilliztech/milvus-distributed/internal/proto/indexpb"
 	"github.com/zilliztech/milvus-distributed/internal/proto/internalpb2"
 	"github.com/zilliztech/milvus-distributed/internal/proto/querypb"
 )
@@ -18,6 +19,21 @@ type Component interface {
 	GetStatisticsChannel() (string, error)
 }
 
+type IndexNodeInterface interface {
+	Service
+	BuildIndex(req *indexpb.BuildIndexCmd) (*commonpb.Status, error)
+}
+
+type IndexServiceInterface interface {
+	Service
+	Component
+	RegisterNode(req *indexpb.RegisterNodeRequest) (*indexpb.RegisterNodeResponse, error)
+	BuildIndex(req *indexpb.BuildIndexRequest) (*indexpb.BuildIndexResponse, error)
+	GetIndexStates(req *indexpb.IndexStatesRequest) (*indexpb.IndexStatesResponse, error)
+	GetIndexFilePaths(req *indexpb.IndexFilePathsRequest) (*indexpb.IndexFilePathsResponse, error)
+	NotifyBuildIndex(nty *indexpb.BuildIndexNotification) (*commonpb.Status, error)
+}
+
 type QueryServiceInterface interface {
 	Service
 	Component

tools/core_gen/all_generate.py

@@ -62,6 +62,10 @@ if __name__ == "__main__":
                 'visitor_name': "VerifyExprVisitor",
                 "parameter_name": 'expr',
             },
+            {
+                'visitor_name': "ExtractInfoExprVisitor",
+                "parameter_name": 'expr',
+            },
         ],
         'PlanNode': [
             {
@@ -76,6 +80,10 @@ if __name__ == "__main__":
                 'visitor_name': "VerifyPlanNodeVisitor",
                 "parameter_name": 'node',
             },
+            {
+                'visitor_name': "ExtractInfoPlanNodeVisitor",
+                "parameter_name": 'node',
+            },
         ]
     }
     extract_extra_body(visitor_info, query_path)