milvus/internal/querynode/data_sync_service_test.go

// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package querynode

import (
	"context"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestDataSyncService_DMLFlowGraphs(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	replica, err := genSimpleReplica()
	assert.NoError(t, err)

	fac := genFactory()
	assert.NoError(t, err)

	tSafe := newTSafeReplica()
	dataSyncService := newDataSyncService(ctx, replica, tSafe, fac)
	assert.NotNil(t, dataSyncService)

	t.Run("test DMLFlowGraphs", func(t *testing.T) {
		_, err = dataSyncService.addFlowGraphsForDMLChannels(defaultCollectionID, []Channel{defaultDMLChannel})
		assert.NoError(t, err)
		assert.Len(t, dataSyncService.dmlChannel2FlowGraph, 1)

		_, err = dataSyncService.addFlowGraphsForDMLChannels(defaultCollectionID, []Channel{defaultDMLChannel})
		assert.NoError(t, err)
		assert.Len(t, dataSyncService.dmlChannel2FlowGraph, 1)

		fg, err := dataSyncService.getFlowGraphByDMLChannel(defaultCollectionID, defaultDMLChannel)
		assert.NotNil(t, fg)
		assert.NoError(t, err)

		err = dataSyncService.startFlowGraphByDMLChannel(defaultCollectionID, defaultDMLChannel)
		assert.NoError(t, err)

		dataSyncService.removeFlowGraphsByDMLChannels([]Channel{defaultDMLChannel})
		assert.Len(t, dataSyncService.dmlChannel2FlowGraph, 0)

		fg, err = dataSyncService.getFlowGraphByDMLChannel(defaultCollectionID, defaultDMLChannel)
		assert.Nil(t, fg)
		assert.Error(t, err)

		_, err = dataSyncService.addFlowGraphsForDMLChannels(defaultCollectionID, []Channel{defaultDMLChannel})
		assert.NoError(t, err)
		assert.Len(t, dataSyncService.dmlChannel2FlowGraph, 1)

		dataSyncService.close()
		assert.Len(t, dataSyncService.dmlChannel2FlowGraph, 0)
	})

	t.Run("test DMLFlowGraphs invalid channel", func(t *testing.T) {
		fg, err := dataSyncService.getFlowGraphByDMLChannel(defaultCollectionID, "invalid-vChannel")
		assert.Nil(t, fg)
		assert.Error(t, err)

		err = dataSyncService.startFlowGraphByDMLChannel(defaultCollectionID, "invalid-vChannel")
		assert.Error(t, err)
	})

	t.Run("test addFlowGraphsForDMLChannels checkReplica Failed", func(t *testing.T) {
		err = dataSyncService.metaReplica.removeCollection(defaultCollectionID)
		assert.NoError(t, err)
		_, err = dataSyncService.addFlowGraphsForDMLChannels(defaultCollectionID, []Channel{defaultDMLChannel})
		assert.Error(t, err)
		dataSyncService.metaReplica.addCollection(defaultCollectionID, genTestCollectionSchema())
	})
}

func TestDataSyncService_DeltaFlowGraphs(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	replica, err := genSimpleReplica()
	assert.NoError(t, err)

	fac := genFactory()
	assert.NoError(t, err)

	tSafe := newTSafeReplica()
	dataSyncService := newDataSyncService(ctx, replica, tSafe, fac)
	assert.NotNil(t, dataSyncService)

	t.Run("test DeltaFlowGraphs", func(t *testing.T) {
		_, err = dataSyncService.addFlowGraphsForDeltaChannels(defaultCollectionID, []Channel{defaultDeltaChannel})
		assert.NoError(t, err)
		assert.Len(t, dataSyncService.deltaChannel2FlowGraph, 1)

		_, err = dataSyncService.addFlowGraphsForDeltaChannels(defaultCollectionID, []Channel{defaultDeltaChannel})
		assert.NoError(t, err)
		assert.Len(t, dataSyncService.deltaChannel2FlowGraph, 1)

		fg, err := dataSyncService.getFlowGraphByDeltaChannel(defaultCollectionID, defaultDeltaChannel)
		assert.NotNil(t, fg)
		assert.NoError(t, err)

		err = dataSyncService.startFlowGraphForDeltaChannel(defaultCollectionID, defaultDeltaChannel)
		assert.NoError(t, err)

		dataSyncService.removeFlowGraphsByDeltaChannels([]Channel{defaultDeltaChannel})
		assert.Len(t, dataSyncService.deltaChannel2FlowGraph, 0)

		fg, err = dataSyncService.getFlowGraphByDeltaChannel(defaultCollectionID, defaultDeltaChannel)
		assert.Nil(t, fg)
		assert.Error(t, err)

		_, err = dataSyncService.addFlowGraphsForDeltaChannels(defaultCollectionID, []Channel{defaultDMLChannel})
		assert.NoError(t, err)
		assert.Len(t, dataSyncService.deltaChannel2FlowGraph, 1)

		dataSyncService.close()
		assert.Len(t, dataSyncService.deltaChannel2FlowGraph, 0)
	})

	t.Run("test DeltaFlowGraphs invalid channel", func(t *testing.T) {
		fg, err := dataSyncService.getFlowGraphByDeltaChannel(defaultCollectionID, "invalid-vChannel")
		assert.Nil(t, fg)
		assert.Error(t, err)

		err = dataSyncService.startFlowGraphForDeltaChannel(defaultCollectionID, "invalid-vChannel")
		assert.Error(t, err)
	})

	t.Run("test addFlowGraphsForDeltaChannels checkReplica Failed", func(t *testing.T) {
		err = dataSyncService.metaReplica.removeCollection(defaultCollectionID)
		assert.NoError(t, err)
		_, err = dataSyncService.addFlowGraphsForDeltaChannels(defaultCollectionID, []Channel{defaultDMLChannel})
		assert.Error(t, err)
		dataSyncService.metaReplica.addCollection(defaultCollectionID, genTestCollectionSchema())
	})
}

func TestDataSyncService_checkReplica(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	replica, err := genSimpleReplica()
	assert.NoError(t, err)

	fac := genFactory()
	assert.NoError(t, err)

	tSafe := newTSafeReplica()
	dataSyncService := newDataSyncService(ctx, replica, tSafe, fac)
	assert.NotNil(t, dataSyncService)
	defer dataSyncService.close()

	t.Run("test checkReplica", func(t *testing.T) {
		err = dataSyncService.checkReplica(defaultCollectionID)
		assert.NoError(t, err)
	})

	t.Run("test collection doesn't exist", func(t *testing.T) {
		err = dataSyncService.metaReplica.removeCollection(defaultCollectionID)
		assert.NoError(t, err)
		err = dataSyncService.checkReplica(defaultCollectionID)
		assert.Error(t, err)
		coll := dataSyncService.metaReplica.addCollection(defaultCollectionID, genTestCollectionSchema())
		assert.NotNil(t, coll)
	})

	t.Run("test cannot find tSafe", func(t *testing.T) {
		coll, err := dataSyncService.metaReplica.getCollectionByID(defaultCollectionID)
		assert.NoError(t, err)
		coll.addVDeltaChannels([]Channel{defaultDeltaChannel})
		coll.addVChannels([]Channel{defaultDMLChannel})

		dataSyncService.tSafeReplica.addTSafe(defaultDeltaChannel)
		dataSyncService.tSafeReplica.addTSafe(defaultDMLChannel)

		dataSyncService.tSafeReplica.removeTSafe(defaultDeltaChannel)
		err = dataSyncService.checkReplica(defaultCollectionID)
		assert.Error(t, err)

		dataSyncService.tSafeReplica.removeTSafe(defaultDMLChannel)
		err = dataSyncService.checkReplica(defaultCollectionID)
		assert.Error(t, err)

		dataSyncService.tSafeReplica.addTSafe(defaultDeltaChannel)
		dataSyncService.tSafeReplica.addTSafe(defaultDMLChannel)
	})
}