milvus/internal/datanode/flow_graph_delete_node.go

// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package datanode

import (
	"context"
	"fmt"
	"reflect"
	"sync"

	"github.com/opentracing/opentracing-go"
	"go.uber.org/zap"

	"github.com/milvus-io/milvus-proto/go-api/schemapb"
	"github.com/milvus-io/milvus/internal/common"
	"github.com/milvus-io/milvus/internal/log"
	"github.com/milvus-io/milvus/internal/metrics"
	"github.com/milvus-io/milvus/internal/mq/msgstream"
	"github.com/milvus-io/milvus/internal/storage"
	"github.com/milvus-io/milvus/internal/util/retry"
	"github.com/milvus-io/milvus/internal/util/trace"
)

// DeleteNode is to process delete msg, flush delete info into storage.
type deleteNode struct {
	BaseNode

	ctx          context.Context
	channelName  string
	delBuf       sync.Map // map[segmentID]*DelDataBuf
	channel      Channel
	idAllocator  allocatorInterface
	flushManager flushManager

	clearSignal chan<- string
}

func (dn *deleteNode) Name() string {
	return "deleteNode-" + dn.channelName
}

func (dn *deleteNode) Close() {
	log.Info("Flowgraph Delete Node closing")
}

func (dn *deleteNode) showDelBuf(segIDs []UniqueID, ts Timestamp) {
	for _, segID := range segIDs {
		if v, ok := dn.delBuf.Load(segID); ok {
			delDataBuf, _ := v.(*DelDataBuf)
			log.Debug("delta buffer status",
				zap.Uint64("timestamp", ts),
				zap.Int64("segment ID", segID),
				zap.Int64("entries", delDataBuf.GetEntriesNum()),
				zap.String("vChannel", dn.channelName))
		}
	}
}

// Operate implementing flowgraph.Node, performs delete data process
func (dn *deleteNode) Operate(in []Msg) []Msg {
	if in == nil {
		log.Debug("type assertion failed for flowGraphMsg because it's nil")
		return []Msg{}
	}

	if len(in) != 1 {
		log.Error("Invalid operate message input in deleteNode", zap.Int("input length", len(in)))
		return []Msg{}
	}

	fgMsg, ok := in[0].(*flowGraphMsg)
	if !ok {
		log.Warn("type assertion failed for flowGraphMsg", zap.String("name", reflect.TypeOf(in[0]).Name()))
		return []Msg{}
	}

	var spans []opentracing.Span
	for _, msg := range fgMsg.deleteMessages {
		sp, ctx := trace.StartSpanFromContext(msg.TraceCtx())
		spans = append(spans, sp)
		msg.SetTraceCtx(ctx)
	}

	// update compacted segment before operation
	if len(fgMsg.deleteMessages) > 0 || len(fgMsg.segmentsToFlush) > 0 {
		dn.updateCompactedSegments()
	}

	// process delete messages
	var segIDs []UniqueID
	for i, msg := range fgMsg.deleteMessages {
		traceID, _, _ := trace.InfoFromSpan(spans[i])
		log.Info("Buffer delete request in DataNode", zap.String("traceID", traceID))

		tmpSegIDs, err := dn.bufferDeleteMsg(msg, fgMsg.timeRange)
		if err != nil {
			// error occurs only when deleteMsg is misaligned, should not happen
			err = fmt.Errorf("buffer delete msg failed, err = %s", err)
			log.Error(err.Error())
			panic(err)
		}
		segIDs = append(segIDs, tmpSegIDs...)
	}

	// display changed segment's status in dn.delBuf of a certain ts
	if len(fgMsg.deleteMessages) != 0 {
		dn.showDelBuf(segIDs, fgMsg.timeRange.timestampMax)
	}

	// process flush messages
	if len(fgMsg.segmentsToFlush) > 0 {
		log.Debug("DeleteNode receives flush message",
			zap.Int64s("segIDs", fgMsg.segmentsToFlush),
			zap.String("vChannelName", dn.channelName))
		for _, segmentToFlush := range fgMsg.segmentsToFlush {
			buf, ok := dn.delBuf.Load(segmentToFlush)
			if !ok {
				// no related delta data to flush, send empty buf to complete flush life-cycle
				dn.flushManager.flushDelData(nil, segmentToFlush, fgMsg.endPositions[0])
			} else {
				err := retry.Do(dn.ctx, func() error {
					return dn.flushManager.flushDelData(buf.(*DelDataBuf), segmentToFlush, fgMsg.endPositions[0])
				}, getFlowGraphRetryOpt())
				if err != nil {
					err = fmt.Errorf("failed to flush delete data, err = %s", err)
					log.Error(err.Error())
					panic(err)
				}
				// remove delete buf
				dn.delBuf.Delete(segmentToFlush)
			}
		}
	}

	// process drop collection message, delete node shall notify flush manager all data are cleared and send signal to DataSyncService cleaner
	if fgMsg.dropCollection {
		dn.flushManager.notifyAllFlushed()
		log.Debug("DeleteNode notifies BackgroundGC to release vchannel", zap.String("vChannelName", dn.channelName))
		dn.clearSignal <- dn.channelName
	}

	for _, sp := range spans {
		sp.Finish()
	}
	return in
}

// update delBuf for compacted segments
func (dn *deleteNode) updateCompactedSegments() {
	compactedTo2From := dn.channel.listCompactedSegmentIDs()

	for compactedTo, compactedFrom := range compactedTo2From {
		// if the compactedTo segment has 0 numRows, remove all segments related
		if !dn.channel.hasSegment(compactedTo, true) {
			for _, segID := range compactedFrom {
				dn.delBuf.Delete(segID)
			}
			dn.channel.removeSegments(compactedFrom...)
			continue
		}

		var compactToDelBuff *DelDataBuf
		delBuf, loaded := dn.delBuf.Load(compactedTo)
		if !loaded {
			compactToDelBuff = newDelDataBuf()
		} else {
			compactToDelBuff = delBuf.(*DelDataBuf)
		}

		for _, segID := range compactedFrom {
			if value, loaded := dn.delBuf.LoadAndDelete(segID); loaded {
				compactToDelBuff.updateFromBuf(value.(*DelDataBuf))

				// only store delBuf if EntriesNum > 0
				if compactToDelBuff.EntriesNum > 0 {
					dn.delBuf.Store(compactedTo, compactToDelBuff)
				}
			}
		}
		log.Debug("update delBuf for compacted segments",
			zap.Int64("compactedTo segmentID", compactedTo),
			zap.Int64s("compactedFrom segmentIDs", compactedFrom),
		)
		dn.channel.removeSegments(compactedFrom...)
	}
}

func (dn *deleteNode) bufferDeleteMsg(msg *msgstream.DeleteMsg, tr TimeRange) ([]UniqueID, error) {
	log.Debug("bufferDeleteMsg", zap.Any("primary keys", msg.PrimaryKeys), zap.String("vChannelName", dn.channelName))

	primaryKeys := storage.ParseIDs2PrimaryKeys(msg.PrimaryKeys)
	segIDToPks, segIDToTss := dn.filterSegmentByPK(msg.PartitionID, primaryKeys, msg.Timestamps)

	segIDs := make([]UniqueID, 0, len(segIDToPks))
	for segID, pks := range segIDToPks {
		segIDs = append(segIDs, segID)

		rows := len(pks)
		tss, ok := segIDToTss[segID]
		if !ok || rows != len(tss) {
			return nil, fmt.Errorf("primary keys and timestamp's element num mis-match, segmentID = %d", segID)
		}

		var delDataBuf *DelDataBuf
		value, ok := dn.delBuf.Load(segID)
		if ok {
			delDataBuf = value.(*DelDataBuf)
		} else {
			delDataBuf = newDelDataBuf()
		}
		delData := delDataBuf.delData

		for i := 0; i < rows; i++ {
			delData.Pks = append(delData.Pks, pks[i])
			delData.Tss = append(delData.Tss, tss[i])
			// this log will influence the data node performance as it may be too many,
			// only use it when we focus on delete operations
			//log.Debug("delete",
			//	zap.Any("primary key", pks[i]),
			//	zap.Uint64("ts", tss[i]),
			//	zap.Int64("segmentID", segID),
			//	zap.String("vChannelName", dn.channelName))
		}

		// store
		delDataBuf.updateSize(int64(rows))
		metrics.DataNodeConsumeMsgRowsCount.WithLabelValues(fmt.Sprint(Params.DataNodeCfg.GetNodeID()), metrics.DeleteLabel).Add(float64(rows))
		delDataBuf.updateTimeRange(tr)
		dn.delBuf.Store(segID, delDataBuf)
	}

	return segIDs, nil
}

// filterSegmentByPK returns the bloom filter check result.
// If the key may exists in the segment, returns it in map.
// If the key not exists in the segment, the segment is filter out.
func (dn *deleteNode) filterSegmentByPK(partID UniqueID, pks []primaryKey, tss []Timestamp) (map[UniqueID][]primaryKey, map[UniqueID][]uint64) {
	segID2Pks := make(map[UniqueID][]primaryKey)
	segID2Tss := make(map[UniqueID][]uint64)
	buf := make([]byte, 8)
	segments := dn.channel.filterSegments(partID)
	for index, pk := range pks {
		for _, segment := range segments {
			segmentID := segment.segmentID
			exist := false
			switch pk.Type() {
			case schemapb.DataType_Int64:
				int64Pk := pk.(*int64PrimaryKey)
				common.Endian.PutUint64(buf, uint64(int64Pk.Value))
				exist = segment.pkStat.pkFilter.Test(buf)
			case schemapb.DataType_VarChar:
				varCharPk := pk.(*varCharPrimaryKey)
				exist = segment.pkStat.pkFilter.TestString(varCharPk.Value)
			default:
				//TODO::
			}
			if exist {
				segID2Pks[segmentID] = append(segID2Pks[segmentID], pk)
				segID2Tss[segmentID] = append(segID2Tss[segmentID], tss[index])
			}
		}
	}

	return segID2Pks, segID2Tss
}

func newDeleteNode(ctx context.Context, fm flushManager, sig chan<- string, config *nodeConfig) (*deleteNode, error) {
	baseNode := BaseNode{}
	baseNode.SetMaxQueueLength(config.maxQueueLength)
	baseNode.SetMaxParallelism(config.maxParallelism)

	return &deleteNode{
		ctx:      ctx,
		BaseNode: baseNode,
		delBuf:   sync.Map{},

		channel:      config.channel,
		idAllocator:  config.allocator,
		channelName:  config.vChannelName,
		flushManager: fm,
		clearSignal:  sig,
	}, nil
}