mirror of
https://gitee.com/milvus-io/milvus.git
synced 2024-12-04 21:09:06 +08:00
4bd426dbe7
see also: https://github.com/milvus-io/milvus/issues/28509 Currently Minio latency monitoring for get operation only collects the duration of getting object (which just returns an io.Reader and does not really read from minio), this pr will correct this behavior. Signed-off-by: bigsheeper <yihao.dai@zilliz.com>
472 lines
16 KiB
Go
472 lines
16 KiB
Go
// Licensed to the LF AI & Data foundation under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package storage
|
|
|
|
import (
|
|
"bytes"
|
|
"container/list"
|
|
"context"
|
|
"io"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/cockroachdb/errors"
|
|
minio "github.com/minio/minio-go/v7"
|
|
"go.uber.org/zap"
|
|
"golang.org/x/exp/mmap"
|
|
"golang.org/x/sync/errgroup"
|
|
|
|
"github.com/milvus-io/milvus/pkg/log"
|
|
"github.com/milvus-io/milvus/pkg/metrics"
|
|
"github.com/milvus-io/milvus/pkg/util/merr"
|
|
"github.com/milvus-io/milvus/pkg/util/timerecord"
|
|
)
|
|
|
|
var CheckBucketRetryAttempts uint = 20
|
|
|
|
// MinioChunkManager is responsible for read and write data stored in minio.
|
|
type MinioChunkManager struct {
|
|
*minio.Client
|
|
|
|
// ctx context.Context
|
|
bucketName string
|
|
rootPath string
|
|
}
|
|
|
|
var _ ChunkManager = (*MinioChunkManager)(nil)
|
|
|
|
// NewMinioChunkManager create a new local manager object.
|
|
// Deprecated: Do not call this directly! Use factory.NewPersistentStorageChunkManager instead.
|
|
func NewMinioChunkManager(ctx context.Context, opts ...Option) (*MinioChunkManager, error) {
|
|
c := newDefaultConfig()
|
|
for _, opt := range opts {
|
|
opt(c)
|
|
}
|
|
|
|
return newMinioChunkManagerWithConfig(ctx, c)
|
|
}
|
|
|
|
func newMinioChunkManagerWithConfig(ctx context.Context, c *config) (*MinioChunkManager, error) {
|
|
minIOClient, err := newMinioClient(ctx, c)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
mcm := &MinioChunkManager{
|
|
Client: minIOClient,
|
|
bucketName: c.bucketName,
|
|
}
|
|
mcm.rootPath = mcm.normalizeRootPath(c.rootPath)
|
|
log.Info("minio chunk manager init success.", zap.String("bucketname", c.bucketName), zap.String("root", mcm.RootPath()))
|
|
return mcm, nil
|
|
}
|
|
|
|
// normalizeRootPath
|
|
func (mcm *MinioChunkManager) normalizeRootPath(rootPath string) string {
|
|
// no leading "/"
|
|
return strings.TrimLeft(rootPath, "/")
|
|
}
|
|
|
|
// SetVar set the variable value of mcm
|
|
func (mcm *MinioChunkManager) SetVar(bucketName string, rootPath string) {
|
|
log.Info("minio chunkmanager ", zap.String("bucketName", bucketName), zap.String("rootpath", rootPath))
|
|
mcm.bucketName = bucketName
|
|
mcm.rootPath = rootPath
|
|
}
|
|
|
|
// RootPath returns minio root path.
|
|
func (mcm *MinioChunkManager) RootPath() string {
|
|
return mcm.rootPath
|
|
}
|
|
|
|
// Path returns the path of minio data if exists.
|
|
func (mcm *MinioChunkManager) Path(ctx context.Context, filePath string) (string, error) {
|
|
exist, err := mcm.Exist(ctx, filePath)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if !exist {
|
|
return "", merr.WrapErrIoKeyNotFound(filePath)
|
|
}
|
|
return filePath, nil
|
|
}
|
|
|
|
// Reader returns the path of minio data if exists.
|
|
func (mcm *MinioChunkManager) Reader(ctx context.Context, filePath string) (FileReader, error) {
|
|
reader, err := mcm.getMinioObject(ctx, mcm.bucketName, filePath, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
log.Warn("failed to get object", zap.String("bucket", mcm.bucketName), zap.String("path", filePath), zap.Error(err))
|
|
return nil, err
|
|
}
|
|
return reader, nil
|
|
}
|
|
|
|
func (mcm *MinioChunkManager) Size(ctx context.Context, filePath string) (int64, error) {
|
|
objectInfo, err := mcm.statMinioObject(ctx, mcm.bucketName, filePath, minio.StatObjectOptions{})
|
|
if err != nil {
|
|
log.Warn("failed to stat object", zap.String("bucket", mcm.bucketName), zap.String("path", filePath), zap.Error(err))
|
|
return 0, err
|
|
}
|
|
|
|
return objectInfo.Size, nil
|
|
}
|
|
|
|
// Write writes the data to minio storage.
|
|
func (mcm *MinioChunkManager) Write(ctx context.Context, filePath string, content []byte) error {
|
|
_, err := mcm.putMinioObject(ctx, mcm.bucketName, filePath, bytes.NewReader(content), int64(len(content)), minio.PutObjectOptions{})
|
|
if err != nil {
|
|
log.Warn("failed to put object", zap.String("bucket", mcm.bucketName), zap.String("path", filePath), zap.Error(err))
|
|
return err
|
|
}
|
|
|
|
metrics.PersistentDataKvSize.WithLabelValues(metrics.DataPutLabel).Observe(float64(len(content)))
|
|
return nil
|
|
}
|
|
|
|
// MultiWrite saves multiple objects, the path is the key of @kvs.
|
|
// The object value is the value of @kvs.
|
|
func (mcm *MinioChunkManager) MultiWrite(ctx context.Context, kvs map[string][]byte) error {
|
|
errors := make([]error, 0, len(kvs))
|
|
for key, value := range kvs {
|
|
err := mcm.Write(ctx, key, value)
|
|
errors = append(errors, err)
|
|
}
|
|
return merr.Combine(errors...)
|
|
}
|
|
|
|
// Exist checks whether chunk is saved to minio storage.
|
|
func (mcm *MinioChunkManager) Exist(ctx context.Context, filePath string) (bool, error) {
|
|
_, err := mcm.statMinioObject(ctx, mcm.bucketName, filePath, minio.StatObjectOptions{})
|
|
if err != nil {
|
|
if errors.Is(err, merr.ErrIoKeyNotFound) {
|
|
return false, nil
|
|
}
|
|
log.Warn("failed to stat object",
|
|
zap.String("bucket", mcm.bucketName),
|
|
zap.String("path", filePath),
|
|
zap.Error(err),
|
|
)
|
|
return false, err
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
// Read reads the minio storage data if exists.
|
|
func (mcm *MinioChunkManager) Read(ctx context.Context, filePath string) ([]byte, error) {
|
|
start := time.Now()
|
|
object, err := mcm.getMinioObject(ctx, mcm.bucketName, filePath, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
log.Warn("failed to get object", zap.String("bucket", mcm.bucketName), zap.String("path", filePath), zap.Error(err))
|
|
return nil, err
|
|
}
|
|
defer object.Close()
|
|
|
|
// Prefetch object data
|
|
var empty []byte
|
|
_, err = object.Read(empty)
|
|
err = checkObjectStorageError(filePath, err)
|
|
if err != nil {
|
|
log.Warn("failed to read object", zap.String("path", filePath), zap.Error(err))
|
|
return nil, err
|
|
}
|
|
|
|
objectInfo, err := object.Stat()
|
|
err = checkObjectStorageError(filePath, err)
|
|
if err != nil {
|
|
log.Warn("failed to stat object", zap.String("bucket", mcm.bucketName), zap.String("path", filePath), zap.Error(err))
|
|
return nil, err
|
|
}
|
|
|
|
data, err := Read(object, objectInfo.Size)
|
|
err = checkObjectStorageError(filePath, err)
|
|
if err != nil {
|
|
log.Warn("failed to read object", zap.String("bucket", mcm.bucketName), zap.String("path", filePath), zap.Error(err))
|
|
return nil, err
|
|
}
|
|
metrics.PersistentDataKvSize.WithLabelValues(metrics.DataGetLabel).Observe(float64(objectInfo.Size))
|
|
metrics.PersistentDataRequestLatency.WithLabelValues(metrics.DataGetLabel).Observe(float64(time.Since(start).Milliseconds()))
|
|
return data, nil
|
|
}
|
|
|
|
func (mcm *MinioChunkManager) MultiRead(ctx context.Context, keys []string) ([][]byte, error) {
|
|
errors := make([]error, 0)
|
|
var objectsValues [][]byte
|
|
for _, key := range keys {
|
|
objectValue, err := mcm.Read(ctx, key)
|
|
if err != nil {
|
|
errors = append(errors, err)
|
|
}
|
|
objectsValues = append(objectsValues, objectValue)
|
|
}
|
|
|
|
return objectsValues, merr.Combine(errors...)
|
|
}
|
|
|
|
func (mcm *MinioChunkManager) ReadWithPrefix(ctx context.Context, prefix string) ([]string, [][]byte, error) {
|
|
objectsKeys, _, err := mcm.ListWithPrefix(ctx, prefix, true)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
objectsValues, err := mcm.MultiRead(ctx, objectsKeys)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
return objectsKeys, objectsValues, nil
|
|
}
|
|
|
|
func (mcm *MinioChunkManager) Mmap(ctx context.Context, filePath string) (*mmap.ReaderAt, error) {
|
|
return nil, merr.WrapErrServiceInternal("mmap not supported for MinIO chunk manager")
|
|
}
|
|
|
|
// ReadAt reads specific position data of minio storage if exists.
|
|
func (mcm *MinioChunkManager) ReadAt(ctx context.Context, filePath string, off int64, length int64) ([]byte, error) {
|
|
if off < 0 || length < 0 {
|
|
return nil, io.EOF
|
|
}
|
|
|
|
start := time.Now()
|
|
opts := minio.GetObjectOptions{}
|
|
err := opts.SetRange(off, off+length-1)
|
|
if err != nil {
|
|
log.Warn("failed to set range", zap.String("bucket", mcm.bucketName), zap.String("path", filePath), zap.Error(err))
|
|
return nil, merr.WrapErrParameterInvalidMsg("invalid range while reading %s: %v", filePath, err)
|
|
}
|
|
|
|
object, err := mcm.getMinioObject(ctx, mcm.bucketName, filePath, opts)
|
|
if err != nil {
|
|
log.Warn("failed to get object", zap.String("bucket", mcm.bucketName), zap.String("path", filePath), zap.Error(err))
|
|
return nil, err
|
|
}
|
|
defer object.Close()
|
|
|
|
data, err := Read(object, length)
|
|
if err != nil {
|
|
err = checkObjectStorageError(filePath, err)
|
|
log.Warn("failed to read object", zap.String("bucket", mcm.bucketName), zap.String("path", filePath), zap.Error(err))
|
|
return nil, err
|
|
}
|
|
metrics.PersistentDataKvSize.WithLabelValues(metrics.DataGetLabel).Observe(float64(length))
|
|
metrics.PersistentDataRequestLatency.WithLabelValues(metrics.DataGetLabel).Observe(float64(time.Since(start).Milliseconds()))
|
|
return data, nil
|
|
}
|
|
|
|
// Remove deletes an object with @key.
|
|
func (mcm *MinioChunkManager) Remove(ctx context.Context, filePath string) error {
|
|
err := mcm.removeMinioObject(ctx, mcm.bucketName, filePath, minio.RemoveObjectOptions{})
|
|
if err != nil {
|
|
log.Warn("failed to remove object", zap.String("bucket", mcm.bucketName), zap.String("path", filePath), zap.Error(err))
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// MultiRemove deletes a objects with @keys.
|
|
func (mcm *MinioChunkManager) MultiRemove(ctx context.Context, keys []string) error {
|
|
var el error
|
|
for _, key := range keys {
|
|
err := mcm.Remove(ctx, key)
|
|
if err != nil {
|
|
el = merr.Combine(el, errors.Wrapf(err, "failed to remove %s", key))
|
|
}
|
|
}
|
|
return el
|
|
}
|
|
|
|
// RemoveWithPrefix removes all objects with the same prefix @prefix from minio.
|
|
func (mcm *MinioChunkManager) RemoveWithPrefix(ctx context.Context, prefix string) error {
|
|
objects := mcm.listMinioObjects(ctx, mcm.bucketName, minio.ListObjectsOptions{Prefix: prefix, Recursive: true})
|
|
i := 0
|
|
maxGoroutine := 10
|
|
removeKeys := make([]string, 0, len(objects))
|
|
for object := range objects {
|
|
if object.Err != nil {
|
|
return object.Err
|
|
}
|
|
removeKeys = append(removeKeys, object.Key)
|
|
}
|
|
for i < len(removeKeys) {
|
|
runningGroup, groupCtx := errgroup.WithContext(ctx)
|
|
for j := 0; j < maxGoroutine && i < len(removeKeys); j++ {
|
|
key := removeKeys[i]
|
|
runningGroup.Go(func() error {
|
|
err := mcm.removeMinioObject(groupCtx, mcm.bucketName, key, minio.RemoveObjectOptions{})
|
|
if err != nil {
|
|
log.Warn("failed to remove object", zap.String("path", key), zap.Error(err))
|
|
return err
|
|
}
|
|
return nil
|
|
})
|
|
i++
|
|
}
|
|
if err := runningGroup.Wait(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ListWithPrefix returns objects with provided prefix.
|
|
// by default, if `recursive`=false, list object with return object with path under save level
|
|
// say minio has followinng objects: [a, ab, a/b, ab/c]
|
|
// calling `ListWithPrefix` with `prefix` = a && `recursive` = false will only returns [a, ab]
|
|
// If caller needs all objects without level limitation, `recursive` shall be true.
|
|
func (mcm *MinioChunkManager) ListWithPrefix(ctx context.Context, prefix string, recursive bool) ([]string, []time.Time, error) {
|
|
// cannot use ListObjects(ctx, bucketName, Opt{Prefix:prefix, Recursive:true})
|
|
// if minio has lots of objects under the provided path
|
|
// recursive = true may timeout during the recursive browsing the objects.
|
|
// See also: https://github.com/milvus-io/milvus/issues/19095
|
|
|
|
var objectsKeys []string
|
|
var modTimes []time.Time
|
|
|
|
tasks := list.New()
|
|
tasks.PushBack(prefix)
|
|
for tasks.Len() > 0 {
|
|
e := tasks.Front()
|
|
pre := e.Value.(string)
|
|
tasks.Remove(e)
|
|
|
|
// TODO add concurrent call if performance matters
|
|
// only return current level per call
|
|
objects := mcm.listMinioObjects(ctx, mcm.bucketName, minio.ListObjectsOptions{Prefix: pre, Recursive: false})
|
|
|
|
for object := range objects {
|
|
if object.Err != nil {
|
|
log.Warn("failed to list with prefix", zap.String("bucket", mcm.bucketName), zap.String("prefix", prefix), zap.Error(object.Err))
|
|
return nil, nil, object.Err
|
|
}
|
|
|
|
// with tailing "/", object is a "directory"
|
|
if strings.HasSuffix(object.Key, "/") && recursive {
|
|
// enqueue when recursive is true
|
|
if object.Key != pre {
|
|
tasks.PushBack(object.Key)
|
|
}
|
|
continue
|
|
}
|
|
objectsKeys = append(objectsKeys, object.Key)
|
|
modTimes = append(modTimes, object.LastModified)
|
|
}
|
|
}
|
|
|
|
return objectsKeys, modTimes, nil
|
|
}
|
|
|
|
// Learn from file.ReadFile
|
|
func Read(r io.Reader, size int64) ([]byte, error) {
|
|
data := make([]byte, 0, size)
|
|
for {
|
|
n, err := r.Read(data[len(data):cap(data)])
|
|
data = data[:len(data)+n]
|
|
if err != nil {
|
|
if err == io.EOF {
|
|
err = nil
|
|
}
|
|
return data, err
|
|
}
|
|
if len(data) == cap(data) {
|
|
return data, nil
|
|
}
|
|
}
|
|
}
|
|
|
|
func (mcm *MinioChunkManager) getMinioObject(ctx context.Context, bucketName, objectName string,
|
|
opts minio.GetObjectOptions,
|
|
) (*minio.Object, error) {
|
|
start := timerecord.NewTimeRecorder("getMinioObject")
|
|
|
|
reader, err := mcm.Client.GetObject(ctx, bucketName, objectName, opts)
|
|
metrics.PersistentDataOpCounter.WithLabelValues(metrics.DataGetLabel, metrics.TotalLabel).Inc()
|
|
if err != nil {
|
|
metrics.PersistentDataOpCounter.WithLabelValues(metrics.DataGetLabel, metrics.FailLabel).Inc()
|
|
return nil, checkObjectStorageError(objectName, err)
|
|
}
|
|
if reader == nil {
|
|
metrics.PersistentDataOpCounter.WithLabelValues(metrics.DataGetLabel, metrics.FailLabel).Inc()
|
|
return nil, nil
|
|
}
|
|
|
|
metrics.PersistentDataRequestLatency.WithLabelValues(metrics.DataGetLabel).Observe(float64(start.ElapseSpan().Milliseconds()))
|
|
metrics.PersistentDataOpCounter.WithLabelValues(metrics.DataGetLabel, metrics.SuccessLabel).Inc()
|
|
return reader, nil
|
|
}
|
|
|
|
func (mcm *MinioChunkManager) putMinioObject(ctx context.Context, bucketName, objectName string, reader io.Reader, objectSize int64,
|
|
opts minio.PutObjectOptions,
|
|
) (minio.UploadInfo, error) {
|
|
start := timerecord.NewTimeRecorder("putMinioObject")
|
|
|
|
info, err := mcm.Client.PutObject(ctx, bucketName, objectName, reader, objectSize, opts)
|
|
metrics.PersistentDataOpCounter.WithLabelValues(metrics.DataPutLabel, metrics.TotalLabel).Inc()
|
|
if err != nil {
|
|
metrics.PersistentDataOpCounter.WithLabelValues(metrics.MetaPutLabel, metrics.FailLabel).Inc()
|
|
return info, checkObjectStorageError(objectName, err)
|
|
}
|
|
|
|
metrics.PersistentDataRequestLatency.WithLabelValues(metrics.DataPutLabel).Observe(float64(start.ElapseSpan().Milliseconds()))
|
|
metrics.PersistentDataOpCounter.WithLabelValues(metrics.MetaPutLabel, metrics.SuccessLabel).Inc()
|
|
return info, nil
|
|
}
|
|
|
|
func (mcm *MinioChunkManager) statMinioObject(ctx context.Context, bucketName, objectName string,
|
|
opts minio.StatObjectOptions,
|
|
) (minio.ObjectInfo, error) {
|
|
start := timerecord.NewTimeRecorder("statMinioObject")
|
|
|
|
info, err := mcm.Client.StatObject(ctx, bucketName, objectName, opts)
|
|
metrics.PersistentDataOpCounter.WithLabelValues(metrics.DataStatLabel, metrics.TotalLabel).Inc()
|
|
if err != nil {
|
|
metrics.PersistentDataOpCounter.WithLabelValues(metrics.DataStatLabel, metrics.FailLabel).Inc()
|
|
err = checkObjectStorageError(objectName, err)
|
|
return info, err
|
|
}
|
|
|
|
metrics.PersistentDataRequestLatency.WithLabelValues(metrics.DataStatLabel).Observe(float64(start.ElapseSpan().Milliseconds()))
|
|
metrics.PersistentDataOpCounter.WithLabelValues(metrics.DataStatLabel, metrics.SuccessLabel).Inc()
|
|
return info, nil
|
|
}
|
|
|
|
func (mcm *MinioChunkManager) listMinioObjects(ctx context.Context, bucketName string,
|
|
opts minio.ListObjectsOptions,
|
|
) <-chan minio.ObjectInfo {
|
|
start := timerecord.NewTimeRecorder("listMinioObjects")
|
|
|
|
res := mcm.Client.ListObjects(ctx, bucketName, opts)
|
|
metrics.PersistentDataRequestLatency.WithLabelValues(metrics.DataListLabel).Observe(float64(start.ElapseSpan().Milliseconds()))
|
|
metrics.PersistentDataOpCounter.WithLabelValues(metrics.DataListLabel, metrics.TotalLabel).Inc()
|
|
metrics.PersistentDataOpCounter.WithLabelValues(metrics.DataListLabel, metrics.SuccessLabel).Inc()
|
|
|
|
return res
|
|
}
|
|
|
|
func (mcm *MinioChunkManager) removeMinioObject(ctx context.Context, bucketName, objectName string,
|
|
opts minio.RemoveObjectOptions,
|
|
) error {
|
|
start := timerecord.NewTimeRecorder("removeMinioObject")
|
|
|
|
err := mcm.Client.RemoveObject(ctx, bucketName, objectName, opts)
|
|
metrics.PersistentDataOpCounter.WithLabelValues(metrics.DataRemoveLabel, metrics.TotalLabel).Inc()
|
|
if err != nil {
|
|
metrics.PersistentDataOpCounter.WithLabelValues(metrics.DataRemoveLabel, metrics.FailLabel).Inc()
|
|
return checkObjectStorageError(objectName, err)
|
|
}
|
|
|
|
metrics.PersistentDataRequestLatency.WithLabelValues(metrics.DataRemoveLabel).Observe(float64(start.ElapseSpan().Milliseconds()))
|
|
metrics.PersistentDataOpCounter.WithLabelValues(metrics.DataRemoveLabel, metrics.SuccessLabel).Inc()
|
|
return nil
|
|
}
|