milvus/internal/datanode/compaction_executor.go

118 lines
3.0 KiB
Go
Raw Normal View History

// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package datanode
import (
"context"
"sync"
"github.com/milvus-io/milvus/internal/log"
"go.uber.org/zap"
)
const (
maxTaskNum = 1024
)
var maxParallelCompactionNum = calculateParallel()
type compactionExecutor struct {
parallelCh chan struct{}
executing sync.Map // planID to compactor
taskCh chan compactor
dropped sync.Map // vchannel dropped
}
// 0.5*min(8, NumCPU/2)
func calculateParallel() int {
return 2
//cores := runtime.NumCPU()
//if cores < 16 {
//return 4
//}
//return cores / 2
}
func newCompactionExecutor() *compactionExecutor {
return &compactionExecutor{
parallelCh: make(chan struct{}, maxParallelCompactionNum),
executing: sync.Map{},
taskCh: make(chan compactor, maxTaskNum),
}
}
func (c *compactionExecutor) execute(task compactor) {
c.taskCh <- task
}
func (c *compactionExecutor) start(ctx context.Context) {
for {
select {
case <-ctx.Done():
return
case task := <-c.taskCh:
go c.executeTask(task)
}
}
}
func (c *compactionExecutor) executeTask(task compactor) {
c.parallelCh <- struct{}{}
defer func() {
<-c.parallelCh
}()
c.executing.Store(task.getPlanID(), task)
log.Info("start to execute compaction", zap.Int64("planID", task.getPlanID()))
err := task.compact()
if err != nil {
log.Warn("compaction task failed",
zap.Int64("planID", task.getPlanID()),
zap.Error(err),
)
}
c.executing.Delete(task.getPlanID())
log.Info("end to execute compaction", zap.Int64("planID", task.getPlanID()))
}
func (c *compactionExecutor) stopTask(planID UniqueID) {
task, loaded := c.executing.LoadAndDelete(planID)
if loaded {
log.Warn("compaction executor stop task", zap.Int64("planID", planID))
task.(compactor).stop()
}
}
func (c *compactionExecutor) channelValidateForCompaction(vChannelName string) bool {
// if vchannel marked dropped, compaction should not proceed
_, loaded := c.dropped.Load(vChannelName)
return !loaded
}
func (c *compactionExecutor) stopExecutingtaskByVChannelName(vChannelName string) {
c.dropped.Store(vChannelName, struct{}{})
c.executing.Range(func(key interface{}, value interface{}) bool {
if value.(compactor).getChannelName() == vChannelName {
c.stopTask(key.(UniqueID))
}
log.Warn(value.(compactor).getChannelName())
return true
})
}