mirror of
https://gitee.com/milvus-io/milvus.git
synced 2024-12-04 04:49:08 +08:00
346e9cba4a
* Add minio FGet method Signed-off-by: godchen <qingxiang.chen@zilliz.com> * go fmt Signed-off-by: godchen <qingxiang.chen@zilliz.com> * change el append Signed-off-by: godchen <qingxiang.chen@zilliz.com>
282 lines
6.9 KiB
Go
282 lines
6.9 KiB
Go
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
|
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
|
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
|
|
|
package miniokv_test
|
|
|
|
import (
|
|
"context"
|
|
"io/ioutil"
|
|
"os"
|
|
"strconv"
|
|
"testing"
|
|
|
|
miniokv "github.com/milvus-io/milvus/internal/kv/minio"
|
|
"github.com/milvus-io/milvus/internal/util/paramtable"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
)
|
|
|
|
var Params paramtable.BaseTable
|
|
|
|
func newMinIOKVClient(ctx context.Context, bucketName string) (*miniokv.MinIOKV, error) {
|
|
endPoint, _ := Params.Load("_MinioAddress")
|
|
accessKeyID, _ := Params.Load("minio.accessKeyID")
|
|
secretAccessKey, _ := Params.Load("minio.secretAccessKey")
|
|
useSSLStr, _ := Params.Load("minio.useSSL")
|
|
useSSL, _ := strconv.ParseBool(useSSLStr)
|
|
option := &miniokv.Option{
|
|
Address: endPoint,
|
|
AccessKeyID: accessKeyID,
|
|
SecretAccessKeyID: secretAccessKey,
|
|
UseSSL: useSSL,
|
|
BucketName: bucketName,
|
|
CreateBucket: true,
|
|
}
|
|
client, err := miniokv.NewMinIOKV(ctx, option)
|
|
return client, err
|
|
}
|
|
|
|
func TestMinIOKV_Load(t *testing.T) {
|
|
Params.Init()
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
bucketName := "fantastic-tech-test"
|
|
MinIOKV, err := newMinIOKVClient(ctx, bucketName)
|
|
assert.Nil(t, err)
|
|
defer MinIOKV.RemoveWithPrefix("")
|
|
|
|
err = MinIOKV.Save("abc", "123")
|
|
assert.Nil(t, err)
|
|
err = MinIOKV.Save("abcd", "1234")
|
|
assert.Nil(t, err)
|
|
|
|
val, err := MinIOKV.Load("abc")
|
|
assert.Nil(t, err)
|
|
assert.Equal(t, val, "123")
|
|
|
|
keys, vals, err := MinIOKV.LoadWithPrefix("abc")
|
|
assert.Nil(t, err)
|
|
assert.Equal(t, len(keys), len(vals))
|
|
assert.Equal(t, len(keys), 2)
|
|
|
|
assert.Equal(t, vals[0], "123")
|
|
assert.Equal(t, vals[1], "1234")
|
|
|
|
err = MinIOKV.Save("key_1", "123")
|
|
assert.Nil(t, err)
|
|
err = MinIOKV.Save("key_2", "456")
|
|
assert.Nil(t, err)
|
|
err = MinIOKV.Save("key_3", "789")
|
|
assert.Nil(t, err)
|
|
|
|
keys = []string{"key_1", "key_100"}
|
|
|
|
vals, err = MinIOKV.MultiLoad(keys)
|
|
assert.NotNil(t, err)
|
|
assert.Equal(t, len(vals), len(keys))
|
|
assert.Equal(t, vals[0], "123")
|
|
assert.Empty(t, vals[1])
|
|
|
|
keys = []string{"key_1", "key_2"}
|
|
|
|
vals, err = MinIOKV.MultiLoad(keys)
|
|
assert.Nil(t, err)
|
|
assert.Equal(t, len(vals), len(keys))
|
|
assert.Equal(t, vals[0], "123")
|
|
assert.Equal(t, vals[1], "456")
|
|
|
|
}
|
|
|
|
func TestMinIOKV_MultiSave(t *testing.T) {
|
|
Params.Init()
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
bucketName := "fantastic-tech-test"
|
|
MinIOKV, err := newMinIOKVClient(ctx, bucketName)
|
|
assert.Nil(t, err)
|
|
|
|
defer MinIOKV.RemoveWithPrefix("")
|
|
|
|
err = MinIOKV.Save("key_1", "111")
|
|
assert.Nil(t, err)
|
|
|
|
kvs := map[string]string{
|
|
"key_1": "123",
|
|
"key_2": "456",
|
|
}
|
|
|
|
err = MinIOKV.MultiSave(kvs)
|
|
assert.Nil(t, err)
|
|
|
|
val, err := MinIOKV.Load("key_1")
|
|
assert.Nil(t, err)
|
|
assert.Equal(t, val, "123")
|
|
}
|
|
|
|
func TestMinIOKV_Remove(t *testing.T) {
|
|
Params.Init()
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
bucketName := "fantastic-tech-test"
|
|
MinIOKV, err := newMinIOKVClient(ctx, bucketName)
|
|
assert.Nil(t, err)
|
|
defer MinIOKV.RemoveWithPrefix("")
|
|
|
|
err = MinIOKV.Save("key_1", "123")
|
|
assert.Nil(t, err)
|
|
err = MinIOKV.Save("key_2", "456")
|
|
assert.Nil(t, err)
|
|
|
|
val, err := MinIOKV.Load("key_1")
|
|
assert.Nil(t, err)
|
|
assert.Equal(t, val, "123")
|
|
// delete "key_1"
|
|
err = MinIOKV.Remove("key_1")
|
|
assert.Nil(t, err)
|
|
val, err = MinIOKV.Load("key_1")
|
|
assert.Error(t, err)
|
|
assert.Empty(t, val)
|
|
|
|
val, err = MinIOKV.Load("key_2")
|
|
assert.Nil(t, err)
|
|
assert.Equal(t, val, "456")
|
|
|
|
keys, vals, err := MinIOKV.LoadWithPrefix("key")
|
|
assert.Nil(t, err)
|
|
assert.Equal(t, len(keys), len(vals))
|
|
assert.Equal(t, len(keys), 1)
|
|
|
|
assert.Equal(t, vals[0], "456")
|
|
|
|
// MultiRemove
|
|
err = MinIOKV.Save("key_1", "111")
|
|
assert.Nil(t, err)
|
|
|
|
kvs := map[string]string{
|
|
"key_1": "123",
|
|
"key_2": "456",
|
|
"key_3": "789",
|
|
"key_4": "012",
|
|
}
|
|
|
|
err = MinIOKV.MultiSave(kvs)
|
|
assert.Nil(t, err)
|
|
val, err = MinIOKV.Load("key_1")
|
|
assert.Nil(t, err)
|
|
assert.Equal(t, val, "123")
|
|
val, err = MinIOKV.Load("key_3")
|
|
assert.Nil(t, err)
|
|
assert.Equal(t, val, "789")
|
|
|
|
keys = []string{"key_1", "key_2", "key_3"}
|
|
err = MinIOKV.MultiRemove(keys)
|
|
assert.Nil(t, err)
|
|
|
|
val, err = MinIOKV.Load("key_1")
|
|
assert.Error(t, err)
|
|
assert.Empty(t, val)
|
|
}
|
|
|
|
func TestMinIOKV_FGetObject(t *testing.T) {
|
|
Params.Init()
|
|
path := "/tmp/milvus/data"
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
bucketName := "fantastic-tech-test"
|
|
MinIOKV, err := newMinIOKVClient(ctx, bucketName)
|
|
assert.Nil(t, err)
|
|
defer MinIOKV.RemoveWithPrefix("")
|
|
|
|
name1 := "31280791048324/4325023534/53443534/key_1"
|
|
value1 := "123"
|
|
err = MinIOKV.Save(name1, value1)
|
|
assert.Nil(t, err)
|
|
name2 := "312895849354/31205934503459/18948129301/key_2"
|
|
value2 := "333"
|
|
err = MinIOKV.Save(name2, value2)
|
|
assert.Nil(t, err)
|
|
|
|
err = MinIOKV.FGetObject(name1, path)
|
|
assert.Nil(t, err)
|
|
|
|
err = MinIOKV.FGetObject(name2, path)
|
|
assert.Nil(t, err)
|
|
|
|
err = MinIOKV.FGetObject("fail", path)
|
|
assert.NotNil(t, err)
|
|
|
|
file1, err := os.Open(path + name1)
|
|
assert.Nil(t, err)
|
|
content1, err := ioutil.ReadAll(file1)
|
|
assert.Nil(t, err)
|
|
assert.Equal(t, value1, string(content1))
|
|
defer file1.Close()
|
|
defer os.Remove(path + name1)
|
|
|
|
file2, err := os.Open(path + name2)
|
|
assert.Nil(t, err)
|
|
content2, err := ioutil.ReadAll(file2)
|
|
assert.Nil(t, err)
|
|
assert.Equal(t, value2, string(content2))
|
|
defer file1.Close()
|
|
defer os.Remove(path + name2)
|
|
}
|
|
|
|
func TestMinIOKV_FGetObjects(t *testing.T) {
|
|
Params.Init()
|
|
path := "/tmp/milvus/data"
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
bucketName := "fantastic-tech-test"
|
|
MinIOKV, err := newMinIOKVClient(ctx, bucketName)
|
|
assert.Nil(t, err)
|
|
defer MinIOKV.RemoveWithPrefix("")
|
|
|
|
name1 := "31280791048324/4325023534/53443534/key_1"
|
|
value1 := "123"
|
|
err = MinIOKV.Save(name1, value1)
|
|
assert.Nil(t, err)
|
|
name2 := "312895849354/31205934503459/18948129301/key_2"
|
|
value2 := "333"
|
|
err = MinIOKV.Save(name2, value2)
|
|
assert.Nil(t, err)
|
|
|
|
err = MinIOKV.FGetObjects([]string{name1, name2}, path)
|
|
assert.Nil(t, err)
|
|
|
|
err = MinIOKV.FGetObjects([]string{"fail1", "fail2"}, path)
|
|
assert.NotNil(t, err)
|
|
|
|
file1, err := os.Open(path + name1)
|
|
assert.Nil(t, err)
|
|
content1, err := ioutil.ReadAll(file1)
|
|
assert.Nil(t, err)
|
|
assert.Equal(t, value1, string(content1))
|
|
defer file1.Close()
|
|
defer os.Remove(path + name1)
|
|
|
|
file2, err := os.Open(path + name2)
|
|
assert.Nil(t, err)
|
|
content2, err := ioutil.ReadAll(file2)
|
|
assert.Nil(t, err)
|
|
assert.Equal(t, value2, string(content2))
|
|
defer file1.Close()
|
|
defer os.Remove(path + name2)
|
|
}
|