2021-12-02 22:32:42 +08:00
|
|
|
// Licensed to the LF AI & Data foundation under one
|
|
|
|
// or more contributor license agreements. See the NOTICE file
|
|
|
|
// distributed with this work for additional information
|
|
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
|
|
// to you under the Apache License, Version 2.0 (the
|
|
|
|
// "License"); you may not use this file except in compliance
|
2021-05-28 10:26:30 +08:00
|
|
|
// with the License. You may obtain a copy of the License at
|
|
|
|
//
|
2021-12-02 22:32:42 +08:00
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
2021-05-28 10:26:30 +08:00
|
|
|
//
|
2021-12-02 22:32:42 +08:00
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
2021-05-28 10:26:30 +08:00
|
|
|
|
|
|
|
package querynode
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
2021-06-15 12:41:40 +08:00
|
|
|
"errors"
|
|
|
|
"fmt"
|
2021-07-14 11:15:54 +08:00
|
|
|
"sync"
|
2021-05-28 10:26:30 +08:00
|
|
|
|
2021-06-17 16:56:04 +08:00
|
|
|
"go.uber.org/zap"
|
|
|
|
|
2021-12-24 18:06:30 +08:00
|
|
|
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
|
|
|
|
|
2021-06-17 16:56:04 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/log"
|
2021-07-14 11:15:54 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/proto/querypb"
|
2021-08-14 11:18:10 +08:00
|
|
|
"github.com/milvus-io/milvus/internal/proto/segcorepb"
|
|
|
|
"github.com/milvus-io/milvus/internal/storage"
|
2021-07-14 11:15:54 +08:00
|
|
|
)
|
|
|
|
|
2021-09-26 20:19:58 +08:00
|
|
|
// historical is in charge of historical data in query node
|
2021-05-28 10:26:30 +08:00
|
|
|
type historical struct {
|
2021-07-14 11:15:54 +08:00
|
|
|
ctx context.Context
|
|
|
|
|
2021-05-28 10:26:30 +08:00
|
|
|
replica ReplicaInterface
|
2021-11-06 11:02:58 +08:00
|
|
|
tSafeReplica TSafeReplicaInterface
|
2021-06-19 11:45:09 +08:00
|
|
|
|
2021-07-14 11:15:54 +08:00
|
|
|
mu sync.Mutex // guards globalSealedSegments
|
|
|
|
globalSealedSegments map[UniqueID]*querypb.SegmentInfo
|
|
|
|
|
|
|
|
etcdKV *etcdkv.EtcdKV
|
2021-05-28 10:26:30 +08:00
|
|
|
}
|
|
|
|
|
2021-10-01 09:45:12 +08:00
|
|
|
// newHistorical returns a new historical
|
2021-05-28 10:26:30 +08:00
|
|
|
func newHistorical(ctx context.Context,
|
2021-11-06 11:02:58 +08:00
|
|
|
replica ReplicaInterface,
|
|
|
|
etcdKV *etcdkv.EtcdKV,
|
|
|
|
tSafeReplica TSafeReplicaInterface) *historical {
|
2021-05-28 10:26:30 +08:00
|
|
|
|
|
|
|
return &historical{
|
2021-07-14 11:15:54 +08:00
|
|
|
ctx: ctx,
|
|
|
|
replica: replica,
|
|
|
|
globalSealedSegments: make(map[UniqueID]*querypb.SegmentInfo),
|
|
|
|
etcdKV: etcdKV,
|
2021-11-06 11:02:58 +08:00
|
|
|
tSafeReplica: tSafeReplica,
|
2021-05-28 10:26:30 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (h *historical) start() {
|
|
|
|
}
|
|
|
|
|
2021-12-14 21:55:06 +08:00
|
|
|
// close would release all resources in historical
|
2021-05-28 10:26:30 +08:00
|
|
|
func (h *historical) close() {
|
|
|
|
// free collectionReplica
|
|
|
|
h.replica.freeAll()
|
|
|
|
}
|
2021-06-15 12:41:40 +08:00
|
|
|
|
2021-07-14 11:15:54 +08:00
|
|
|
func (h *historical) getGlobalSegmentIDsByCollectionID(collectionID UniqueID) []UniqueID {
|
|
|
|
h.mu.Lock()
|
|
|
|
defer h.mu.Unlock()
|
|
|
|
resIDs := make([]UniqueID, 0)
|
|
|
|
for _, v := range h.globalSealedSegments {
|
|
|
|
if v.CollectionID == collectionID {
|
|
|
|
resIDs = append(resIDs, v.SegmentID)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return resIDs
|
|
|
|
}
|
|
|
|
|
|
|
|
func (h *historical) getGlobalSegmentIDsByPartitionIds(partitionIDs []UniqueID) []UniqueID {
|
|
|
|
h.mu.Lock()
|
|
|
|
defer h.mu.Unlock()
|
|
|
|
resIDs := make([]UniqueID, 0)
|
|
|
|
for _, v := range h.globalSealedSegments {
|
|
|
|
for _, partitionID := range partitionIDs {
|
|
|
|
if v.PartitionID == partitionID {
|
|
|
|
resIDs = append(resIDs, v.SegmentID)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return resIDs
|
|
|
|
}
|
|
|
|
|
|
|
|
func (h *historical) removeGlobalSegmentIDsByCollectionID(collectionID UniqueID) {
|
|
|
|
h.mu.Lock()
|
|
|
|
defer h.mu.Unlock()
|
|
|
|
for _, v := range h.globalSealedSegments {
|
|
|
|
if v.CollectionID == collectionID {
|
|
|
|
delete(h.globalSealedSegments, v.SegmentID)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (h *historical) removeGlobalSegmentIDsByPartitionIds(partitionIDs []UniqueID) {
|
|
|
|
h.mu.Lock()
|
|
|
|
defer h.mu.Unlock()
|
|
|
|
for _, v := range h.globalSealedSegments {
|
|
|
|
for _, partitionID := range partitionIDs {
|
|
|
|
if v.PartitionID == partitionID {
|
|
|
|
delete(h.globalSealedSegments, v.SegmentID)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-08-18 16:30:11 +08:00
|
|
|
func (h *historical) retrieve(collID UniqueID, partIDs []UniqueID, vcm storage.ChunkManager,
|
2021-12-10 18:51:08 +08:00
|
|
|
plan *RetrievePlan) ([]*segcorepb.RetrieveResults, []UniqueID, []UniqueID, error) {
|
2021-08-14 11:18:10 +08:00
|
|
|
|
|
|
|
retrieveResults := make([]*segcorepb.RetrieveResults, 0)
|
|
|
|
retrieveSegmentIDs := make([]UniqueID, 0)
|
|
|
|
|
|
|
|
// get historical partition ids
|
|
|
|
var retrievePartIDs []UniqueID
|
|
|
|
if len(partIDs) == 0 {
|
|
|
|
hisPartIDs, err := h.replica.getPartitionIDs(collID)
|
|
|
|
if err != nil {
|
2021-12-10 18:51:08 +08:00
|
|
|
return retrieveResults, retrieveSegmentIDs, retrievePartIDs, err
|
2021-08-14 11:18:10 +08:00
|
|
|
}
|
|
|
|
retrievePartIDs = hisPartIDs
|
|
|
|
} else {
|
|
|
|
for _, id := range partIDs {
|
|
|
|
_, err := h.replica.getPartitionByID(id)
|
|
|
|
if err == nil {
|
|
|
|
retrievePartIDs = append(retrievePartIDs, id)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, partID := range retrievePartIDs {
|
|
|
|
segIDs, err := h.replica.getSegmentIDs(partID)
|
|
|
|
if err != nil {
|
2021-12-10 18:51:08 +08:00
|
|
|
return retrieveResults, retrieveSegmentIDs, retrievePartIDs, err
|
2021-08-14 11:18:10 +08:00
|
|
|
}
|
|
|
|
for _, segID := range segIDs {
|
|
|
|
seg, err := h.replica.getSegmentByID(segID)
|
|
|
|
if err != nil {
|
2021-12-10 18:51:08 +08:00
|
|
|
return retrieveResults, retrieveSegmentIDs, retrievePartIDs, err
|
2021-08-14 11:18:10 +08:00
|
|
|
}
|
2021-11-10 17:15:37 +08:00
|
|
|
result, err := seg.retrieve(plan)
|
2021-08-14 11:18:10 +08:00
|
|
|
if err != nil {
|
2021-12-10 18:51:08 +08:00
|
|
|
return retrieveResults, retrieveSegmentIDs, retrievePartIDs, err
|
2021-08-14 11:18:10 +08:00
|
|
|
}
|
|
|
|
|
2021-08-18 16:30:11 +08:00
|
|
|
if err = seg.fillVectorFieldsData(collID, vcm, result); err != nil {
|
2021-12-10 18:51:08 +08:00
|
|
|
return retrieveResults, retrieveSegmentIDs, retrievePartIDs, err
|
2021-08-14 11:18:10 +08:00
|
|
|
}
|
|
|
|
retrieveResults = append(retrieveResults, result)
|
|
|
|
retrieveSegmentIDs = append(retrieveSegmentIDs, segID)
|
|
|
|
}
|
|
|
|
}
|
2021-12-10 18:51:08 +08:00
|
|
|
|
|
|
|
return retrieveResults, retrieveSegmentIDs, retrievePartIDs, nil
|
2021-08-14 11:18:10 +08:00
|
|
|
}
|
|
|
|
|
2021-09-28 22:38:13 +08:00
|
|
|
// search will search all the target segments in historical
|
2021-08-14 11:18:10 +08:00
|
|
|
func (h *historical) search(searchReqs []*searchRequest, collID UniqueID, partIDs []UniqueID, plan *SearchPlan,
|
2021-12-10 18:51:08 +08:00
|
|
|
searchTs Timestamp) ([]*SearchResult, []UniqueID, []UniqueID, error) {
|
2021-06-15 12:41:40 +08:00
|
|
|
|
|
|
|
searchResults := make([]*SearchResult, 0)
|
2021-08-14 11:18:10 +08:00
|
|
|
searchSegmentIDs := make([]UniqueID, 0)
|
2021-06-15 12:41:40 +08:00
|
|
|
|
|
|
|
// get historical partition ids
|
|
|
|
var searchPartIDs []UniqueID
|
|
|
|
if len(partIDs) == 0 {
|
|
|
|
hisPartIDs, err := h.replica.getPartitionIDs(collID)
|
|
|
|
if err != nil {
|
2021-12-10 18:51:08 +08:00
|
|
|
return searchResults, searchSegmentIDs, searchPartIDs, err
|
2021-06-15 12:41:40 +08:00
|
|
|
}
|
2021-06-17 16:56:04 +08:00
|
|
|
log.Debug("no partition specified, search all partitions",
|
|
|
|
zap.Any("collectionID", collID),
|
|
|
|
zap.Any("all partitions", hisPartIDs),
|
|
|
|
)
|
2021-06-15 12:41:40 +08:00
|
|
|
searchPartIDs = hisPartIDs
|
|
|
|
} else {
|
|
|
|
for _, id := range partIDs {
|
|
|
|
_, err := h.replica.getPartitionByID(id)
|
|
|
|
if err == nil {
|
2021-06-17 16:56:04 +08:00
|
|
|
log.Debug("append search partition id",
|
|
|
|
zap.Any("collectionID", collID),
|
|
|
|
zap.Any("partitionID", id),
|
|
|
|
)
|
2021-06-15 12:41:40 +08:00
|
|
|
searchPartIDs = append(searchPartIDs, id)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-06-19 18:38:07 +08:00
|
|
|
col, err := h.replica.getCollectionByID(collID)
|
|
|
|
if err != nil {
|
2021-12-10 18:51:08 +08:00
|
|
|
return searchResults, searchSegmentIDs, searchPartIDs, err
|
2021-06-19 18:38:07 +08:00
|
|
|
}
|
|
|
|
|
2021-06-15 12:41:40 +08:00
|
|
|
// all partitions have been released
|
2021-06-19 18:38:07 +08:00
|
|
|
if len(searchPartIDs) == 0 && col.getLoadType() == loadTypePartition {
|
2021-12-10 18:51:08 +08:00
|
|
|
return searchResults, searchSegmentIDs, searchPartIDs, errors.New("partitions have been released , collectionID = " +
|
2021-08-14 11:18:10 +08:00
|
|
|
fmt.Sprintln(collID) + "target partitionIDs = " + fmt.Sprintln(partIDs))
|
2021-06-15 12:41:40 +08:00
|
|
|
}
|
|
|
|
|
2021-06-19 18:38:07 +08:00
|
|
|
if len(searchPartIDs) == 0 && col.getLoadType() == loadTypeCollection {
|
|
|
|
if err = col.checkReleasedPartitions(partIDs); err != nil {
|
2021-12-10 18:51:08 +08:00
|
|
|
return searchResults, searchSegmentIDs, searchPartIDs, err
|
2021-06-19 18:38:07 +08:00
|
|
|
}
|
2021-12-10 18:51:08 +08:00
|
|
|
return searchResults, searchSegmentIDs, searchPartIDs, nil
|
2021-06-19 18:38:07 +08:00
|
|
|
}
|
|
|
|
|
2021-12-01 18:13:32 +08:00
|
|
|
var segmentLock sync.RWMutex
|
2021-06-15 12:41:40 +08:00
|
|
|
for _, partID := range searchPartIDs {
|
|
|
|
segIDs, err := h.replica.getSegmentIDs(partID)
|
|
|
|
if err != nil {
|
2021-12-10 18:51:08 +08:00
|
|
|
return searchResults, searchSegmentIDs, searchPartIDs, err
|
2021-06-15 12:41:40 +08:00
|
|
|
}
|
2021-11-27 20:05:16 +08:00
|
|
|
|
|
|
|
var err2 error
|
|
|
|
var wg sync.WaitGroup
|
2021-06-15 12:41:40 +08:00
|
|
|
for _, segID := range segIDs {
|
2021-11-27 20:05:16 +08:00
|
|
|
segID2 := segID
|
|
|
|
wg.Add(1)
|
|
|
|
go func() {
|
|
|
|
defer wg.Done()
|
|
|
|
seg, err := h.replica.getSegmentByID(segID2)
|
|
|
|
if err != nil {
|
|
|
|
err2 = err
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if !seg.getOnService() {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
searchResult, err := seg.search(plan, searchReqs, []Timestamp{searchTs})
|
|
|
|
if err != nil {
|
|
|
|
err2 = err
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
segmentLock.Lock()
|
|
|
|
searchResults = append(searchResults, searchResult)
|
|
|
|
searchSegmentIDs = append(searchSegmentIDs, seg.segmentID)
|
|
|
|
segmentLock.Unlock()
|
|
|
|
}()
|
|
|
|
|
|
|
|
}
|
|
|
|
wg.Wait()
|
|
|
|
if err2 != nil {
|
2021-12-10 18:51:08 +08:00
|
|
|
return searchResults, searchSegmentIDs, searchPartIDs, err2
|
2021-06-15 12:41:40 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-12-10 18:51:08 +08:00
|
|
|
return searchResults, searchSegmentIDs, searchPartIDs, nil
|
2021-06-15 12:41:40 +08:00
|
|
|
}
|