milvus/internal/rootcoord/undo.go

package rootcoord

import (
	"context"
	"fmt"
	"time"

	"github.com/milvus-io/milvus/internal/log"
	"go.uber.org/zap"
)

type baseUndoTask struct {
	todoStep []Step // steps to execute
	undoStep []Step // steps to undo
}

func newBaseUndoTask() *baseUndoTask {
	return &baseUndoTask{
		todoStep: make([]Step, 0),
		undoStep: make([]Step, 0),
	}
}

func (b *baseUndoTask) AddStep(todoStep, undoStep Step) {
	b.todoStep = append(b.todoStep, todoStep)
	b.undoStep = append(b.undoStep, undoStep)
}

func (b *baseUndoTask) undoFromLastFinished(lastFinished int) {
	// You cannot just use the ctx of task, since it will be canceled after response is returned.
	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
	defer cancel()
	for i := lastFinished; i >= 0; i-- {
		undo := b.undoStep[i]
		if err := undo.Execute(ctx); err != nil {
			// You depend on the collection meta to do other gc.
			// TODO: add ddl logger after other service can be idempotent enough, then you can do separate steps
			//		independently.
			log.Error("failed to execute step, garbage may be generated", zap.Error(err))
			return
		}
	}
}

func (b *baseUndoTask) Execute(ctx context.Context) error {
	if len(b.todoStep) != len(b.undoStep) {
		return fmt.Errorf("todo step and undo step length not equal")
	}
	for i := 0; i < len(b.todoStep); i++ {
		todoStep := b.todoStep[i]
		err := todoStep.Execute(ctx)
		if err != nil {
			go b.undoFromLastFinished(i - 1)
			log.Warn("failed to execute step, trying to undo", zap.Error(err))
			return err
		}
	}
	return nil
}