Fix watch event timeout when IO Pool busy (#23293)

Signed-off-by: aoiasd <zhicheng.yue@zilliz.com>

configs/milvus.yaml

@@ -262,7 +262,7 @@ indexNode:
 
 dataCoord:
   channel:
-    watchTimeoutInterval: 30 # Timeout on watching channels (in seconds). Datanode tickler update watch progress will reset timeout timer.
+    watchTimeoutInterval: 120 # Timeout on watching channels (in seconds). Datanode tickler update watch progress will reset timeout timer.
     balanceSilentDuration: 300 # The duration before the channelBalancer on datacoord to run
     balanceInterval: 360 #The interval for the channelBalancer on datacoord to check balance status
   segment:

internal/datanode/data_sync_service.go

@@ -199,6 +199,9 @@ func (dsService *dataSyncService) initNodes(vchanInfo *datapb.VchannelInfo, tick
 		return err
 	}
 
+	//tickler will update addSegment progress to watchInfo
+	tickler.watch()
+	defer tickler.stop()
 	futures := make([]*conc.Future[any], 0, len(unflushedSegmentInfos)+len(flushedSegmentInfos))
 
 	for _, us := range unflushedSegmentInfos {
@@ -275,10 +278,6 @@ func (dsService *dataSyncService) initNodes(vchanInfo *datapb.VchannelInfo, tick
 		futures = append(futures, future)
 	}
 
-	//tickler will update addSegment progress to watchInfo
-	tickler.watch()
-	defer tickler.stop()
-
 	err = conc.AwaitAll(futures...)
 	if err != nil {
 		return err

pkg/util/paramtable/component_param.go

@@ -1797,7 +1797,7 @@ func (p *dataCoordConfig) init(base *BaseTable) {
 	p.WatchTimeoutInterval = ParamItem{
 		Key:          "dataCoord.channel.watchTimeoutInterval",
 		Version:      "2.2.3",
-		DefaultValue: "30",
+		DefaultValue: "120",
 		Doc:          "Timeout on watching channels (in seconds). Datanode tickler update watch progress will reset timeout timer.",
 		Export:       true,
 	}