2023-09-19 10:01:23 +08:00
|
|
|
// Licensed to the LF AI & Data foundation under one
|
|
|
|
// or more contributor license agreements. See the NOTICE file
|
|
|
|
// distributed with this work for additional information
|
|
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
|
|
// to you under the Apache License, Version 2.0 (the
|
|
|
|
// "License"); you may not use this file except in compliance
|
|
|
|
// with the License. You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
|
|
|
package storage
|
|
|
|
|
|
|
|
import (
|
|
|
|
"bytes"
|
|
|
|
"context"
|
2023-11-01 11:34:14 +08:00
|
|
|
"fmt"
|
2023-09-19 10:01:23 +08:00
|
|
|
"io"
|
|
|
|
"os"
|
|
|
|
"testing"
|
|
|
|
|
|
|
|
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
)
|
|
|
|
|
|
|
|
func TestAzureObjectStorage(t *testing.T) {
|
|
|
|
ctx := context.Background()
|
|
|
|
bucketName := Params.MinioCfg.BucketName.GetValue()
|
|
|
|
config := config{
|
|
|
|
bucketName: bucketName,
|
|
|
|
createBucket: true,
|
|
|
|
useIAM: false,
|
|
|
|
cloudProvider: "azure",
|
|
|
|
}
|
|
|
|
|
|
|
|
t.Run("test initialize", func(t *testing.T) {
|
|
|
|
var err error
|
|
|
|
config.bucketName = ""
|
|
|
|
_, err = newAzureObjectStorageWithConfig(ctx, &config)
|
|
|
|
assert.Error(t, err)
|
|
|
|
config.bucketName = bucketName
|
|
|
|
_, err = newAzureObjectStorageWithConfig(ctx, &config)
|
|
|
|
assert.Equal(t, err, nil)
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("test load", func(t *testing.T) {
|
|
|
|
testCM, err := newAzureObjectStorageWithConfig(ctx, &config)
|
|
|
|
assert.Equal(t, err, nil)
|
|
|
|
defer testCM.DeleteContainer(ctx, config.bucketName, &azblob.DeleteContainerOptions{})
|
|
|
|
|
|
|
|
prepareTests := []struct {
|
|
|
|
key string
|
|
|
|
value []byte
|
|
|
|
}{
|
|
|
|
{"abc", []byte("123")},
|
|
|
|
{"abcd", []byte("1234")},
|
|
|
|
{"key_1", []byte("111")},
|
|
|
|
{"key_2", []byte("222")},
|
|
|
|
{"key_3", []byte("333")},
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, test := range prepareTests {
|
|
|
|
err := testCM.PutObject(ctx, config.bucketName, test.key, bytes.NewReader(test.value), int64(len(test.value)))
|
|
|
|
require.NoError(t, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
loadTests := []struct {
|
|
|
|
isvalid bool
|
|
|
|
loadKey string
|
|
|
|
expectedValue []byte
|
|
|
|
|
|
|
|
description string
|
|
|
|
}{
|
|
|
|
{true, "abc", []byte("123"), "load valid key abc"},
|
|
|
|
{true, "abcd", []byte("1234"), "load valid key abcd"},
|
|
|
|
{true, "key_1", []byte("111"), "load valid key key_1"},
|
|
|
|
{true, "key_2", []byte("222"), "load valid key key_2"},
|
|
|
|
{true, "key_3", []byte("333"), "load valid key key_3"},
|
|
|
|
{false, "key_not_exist", []byte(""), "load invalid key key_not_exist"},
|
|
|
|
{false, "/", []byte(""), "load leading slash"},
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, test := range loadTests {
|
|
|
|
t.Run(test.description, func(t *testing.T) {
|
|
|
|
if test.isvalid {
|
|
|
|
got, err := testCM.GetObject(ctx, config.bucketName, test.loadKey, 0, 1024)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
contentData, err := io.ReadAll(got)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, len(contentData), len(test.expectedValue))
|
|
|
|
assert.Equal(t, test.expectedValue, contentData)
|
|
|
|
statSize, err := testCM.StatObject(ctx, config.bucketName, test.loadKey)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, statSize, int64(len(contentData)))
|
|
|
|
_, err = testCM.GetObject(ctx, config.bucketName, test.loadKey, 1, 1023)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
} else {
|
|
|
|
if test.loadKey == "/" {
|
|
|
|
got, err := testCM.GetObject(ctx, config.bucketName, test.loadKey, 0, 1024)
|
|
|
|
assert.Error(t, err)
|
|
|
|
assert.Empty(t, got)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
got, err := testCM.GetObject(ctx, config.bucketName, test.loadKey, 0, 1024)
|
|
|
|
assert.Error(t, err)
|
|
|
|
assert.Empty(t, got)
|
|
|
|
}
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
loadWithPrefixTests := []struct {
|
|
|
|
isvalid bool
|
|
|
|
prefix string
|
|
|
|
expectedValue [][]byte
|
|
|
|
|
|
|
|
description string
|
|
|
|
}{
|
|
|
|
{true, "abc", [][]byte{[]byte("123"), []byte("1234")}, "load with valid prefix abc"},
|
|
|
|
{true, "key_", [][]byte{[]byte("111"), []byte("222"), []byte("333")}, "load with valid prefix key_"},
|
|
|
|
{true, "prefix", [][]byte{}, "load with valid but not exist prefix prefix"},
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, test := range loadWithPrefixTests {
|
|
|
|
t.Run(test.description, func(t *testing.T) {
|
2023-11-01 11:34:14 +08:00
|
|
|
gotk, _, err := testCM.ListObjects(ctx, config.bucketName, test.prefix, false)
|
2023-09-19 10:01:23 +08:00
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, len(test.expectedValue), len(gotk))
|
2023-11-01 11:34:14 +08:00
|
|
|
for _, key := range gotk {
|
2023-09-19 10:01:23 +08:00
|
|
|
err := testCM.RemoveObject(ctx, config.bucketName, key)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
}
|
|
|
|
})
|
|
|
|
}
|
|
|
|
})
|
|
|
|
|
2023-11-01 11:34:14 +08:00
|
|
|
t.Run("test list", func(t *testing.T) {
|
|
|
|
testCM, err := newAzureObjectStorageWithConfig(ctx, &config)
|
|
|
|
assert.Equal(t, err, nil)
|
|
|
|
defer testCM.DeleteContainer(ctx, config.bucketName, &azblob.DeleteContainerOptions{})
|
|
|
|
|
|
|
|
prepareTests := []struct {
|
|
|
|
valid bool
|
|
|
|
key string
|
|
|
|
value []byte
|
|
|
|
}{
|
|
|
|
{false, "abc/", []byte("123")},
|
|
|
|
{true, "abc/d", []byte("1234")},
|
|
|
|
{false, "abc/d/e", []byte("12345")},
|
|
|
|
{true, "abc/e/d", []byte("12354")},
|
|
|
|
{true, "key_/1/1", []byte("111")},
|
|
|
|
{true, "key_/1/2", []byte("222")},
|
|
|
|
{false, "key_/1/2/3", []byte("333")},
|
|
|
|
{true, "key_/2/3", []byte("333")},
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, test := range prepareTests {
|
|
|
|
err := testCM.PutObject(ctx, config.bucketName, test.key, bytes.NewReader(test.value), int64(len(test.value)))
|
|
|
|
require.Nil(t, err)
|
|
|
|
if !test.valid {
|
|
|
|
err := testCM.RemoveObject(ctx, config.bucketName, test.key)
|
|
|
|
require.Nil(t, err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
insertWithPrefixTests := []struct {
|
|
|
|
recursive bool
|
|
|
|
prefix string
|
|
|
|
expectedValue []string
|
|
|
|
}{
|
|
|
|
{true, "abc/", []string{"abc/d", "abc/e/d"}},
|
|
|
|
{true, "key_/", []string{"key_/1/1", "key_/1/2", "key_/2/3"}},
|
|
|
|
{false, "abc/", []string{"abc/d", "abc/e/"}},
|
|
|
|
{false, "key_/", []string{"key_/1/", "key_/2/"}},
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, test := range insertWithPrefixTests {
|
|
|
|
t.Run(fmt.Sprintf("prefix: %s, recursive: %t", test.prefix, test.recursive), func(t *testing.T) {
|
|
|
|
gotk, _, err := testCM.ListObjects(ctx, config.bucketName, test.prefix, test.recursive)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, len(test.expectedValue), len(gotk))
|
|
|
|
for _, key := range gotk {
|
|
|
|
assert.Contains(t, test.expectedValue, key)
|
|
|
|
}
|
|
|
|
})
|
|
|
|
}
|
|
|
|
})
|
|
|
|
|
2023-09-19 10:01:23 +08:00
|
|
|
t.Run("test useIAM", func(t *testing.T) {
|
|
|
|
var err error
|
|
|
|
config.useIAM = true
|
|
|
|
_, err = newAzureObjectStorageWithConfig(ctx, &config)
|
|
|
|
assert.Error(t, err)
|
|
|
|
os.Setenv("AZURE_CLIENT_ID", "00000000-0000-0000-0000-00000000000")
|
|
|
|
os.Setenv("AZURE_TENANT_ID", "00000000-0000-0000-0000-00000000000")
|
|
|
|
os.Setenv("AZURE_FEDERATED_TOKEN_FILE", "/var/run/secrets/tokens/azure-identity-token")
|
|
|
|
_, err = newAzureObjectStorageWithConfig(ctx, &config)
|
|
|
|
assert.Error(t, err)
|
|
|
|
config.useIAM = false
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("test key secret", func(t *testing.T) {
|
|
|
|
var err error
|
|
|
|
connectionString := os.Getenv("AZURE_STORAGE_CONNECTION_STRING")
|
|
|
|
os.Setenv("AZURE_STORAGE_CONNECTION_STRING", "")
|
|
|
|
config.accessKeyID = "devstoreaccount1"
|
|
|
|
config.secretAccessKeyID = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
|
|
|
|
config.address = "core.windows.net"
|
|
|
|
_, err = newAzureObjectStorageWithConfig(ctx, &config)
|
|
|
|
assert.Error(t, err)
|
|
|
|
os.Setenv("AZURE_STORAGE_CONNECTION_STRING", connectionString)
|
|
|
|
})
|
|
|
|
}
|
2024-01-04 20:50:46 +08:00
|
|
|
|
|
|
|
func TestReadFile(t *testing.T) {
|
|
|
|
ctx := context.Background()
|
|
|
|
bucketName := Params.MinioCfg.BucketName.GetValue()
|
|
|
|
c := &config{
|
|
|
|
bucketName: bucketName,
|
|
|
|
createBucket: true,
|
|
|
|
useIAM: false,
|
|
|
|
cloudProvider: "azure",
|
|
|
|
}
|
|
|
|
rcm, err := NewRemoteChunkManager(ctx, c)
|
|
|
|
|
|
|
|
t.Run("Read", func(t *testing.T) {
|
|
|
|
filePath := "test-Read"
|
|
|
|
data := []byte("Test data for Read.")
|
|
|
|
|
|
|
|
err = rcm.Write(ctx, filePath, data)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
defer rcm.Remove(ctx, filePath)
|
|
|
|
|
|
|
|
reader, err := rcm.Reader(ctx, filePath)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
|
|
|
|
buffer := make([]byte, 4)
|
|
|
|
n, err := reader.Read(buffer)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, 4, n)
|
|
|
|
assert.Equal(t, "Test", string(buffer))
|
|
|
|
|
|
|
|
buffer = make([]byte, 6)
|
|
|
|
n, err = reader.Read(buffer)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, 6, n)
|
|
|
|
assert.Equal(t, " data ", string(buffer))
|
|
|
|
|
|
|
|
buffer = make([]byte, 40)
|
|
|
|
n, err = reader.Read(buffer)
|
|
|
|
assert.Error(t, err)
|
|
|
|
assert.Equal(t, 9, n)
|
|
|
|
assert.Equal(t, "for Read.", string(buffer[:9]))
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("ReadAt", func(t *testing.T) {
|
|
|
|
filePath := "test-ReadAt"
|
|
|
|
data := []byte("Test data for ReadAt.")
|
|
|
|
|
|
|
|
err = rcm.Write(ctx, filePath, data)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
defer rcm.Remove(ctx, filePath)
|
|
|
|
|
|
|
|
reader, err := rcm.Reader(ctx, filePath)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
|
|
|
|
buffer := make([]byte, 4)
|
|
|
|
n, err := reader.ReadAt(buffer, 5)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, 4, n)
|
|
|
|
assert.Equal(t, "data", string(buffer))
|
|
|
|
|
|
|
|
buffer = make([]byte, 4)
|
|
|
|
n, err = reader.Read(buffer)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, 4, n)
|
|
|
|
assert.Equal(t, "Test", string(buffer))
|
|
|
|
|
|
|
|
buffer = make([]byte, 4)
|
|
|
|
n, err = reader.ReadAt(buffer, 20)
|
|
|
|
assert.Error(t, err)
|
|
|
|
assert.Equal(t, 1, n)
|
|
|
|
assert.Equal(t, ".", string(buffer[:1]))
|
|
|
|
|
|
|
|
buffer = make([]byte, 4)
|
|
|
|
n, err = reader.ReadAt(buffer, 25)
|
|
|
|
assert.Error(t, err)
|
|
|
|
assert.Equal(t, 0, n)
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("Seek start", func(t *testing.T) {
|
|
|
|
filePath := "test-SeekStart"
|
|
|
|
data := []byte("Test data for Seek start.")
|
|
|
|
|
|
|
|
err = rcm.Write(ctx, filePath, data)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
defer rcm.Remove(ctx, filePath)
|
|
|
|
|
|
|
|
reader, err := rcm.Reader(ctx, filePath)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
|
|
|
|
offset, err := reader.Seek(10, io.SeekStart)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, int64(10), offset)
|
|
|
|
|
|
|
|
buffer := make([]byte, 4)
|
|
|
|
n, err := reader.Read(buffer)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, 4, n)
|
|
|
|
assert.Equal(t, "for ", string(buffer))
|
|
|
|
|
|
|
|
offset, err = reader.Seek(40, io.SeekStart)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, int64(40), offset)
|
|
|
|
|
|
|
|
buffer = make([]byte, 4)
|
|
|
|
n, err = reader.Read(buffer)
|
|
|
|
assert.Error(t, err)
|
|
|
|
assert.Equal(t, 0, n)
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("Seek current", func(t *testing.T) {
|
|
|
|
filePath := "test-SeekStart"
|
|
|
|
data := []byte("Test data for Seek current.")
|
|
|
|
|
|
|
|
err = rcm.Write(ctx, filePath, data)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
defer rcm.Remove(ctx, filePath)
|
|
|
|
|
|
|
|
reader, err := rcm.Reader(ctx, filePath)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
|
|
|
|
buffer := make([]byte, 4)
|
|
|
|
n, err := reader.Read(buffer)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, 4, n)
|
|
|
|
assert.Equal(t, "Test", string(buffer))
|
|
|
|
|
|
|
|
offset, err := reader.Seek(10, io.SeekCurrent)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, int64(14), offset)
|
|
|
|
|
|
|
|
buffer = make([]byte, 4)
|
|
|
|
n, err = reader.Read(buffer)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, 4, n)
|
|
|
|
assert.Equal(t, "Seek", string(buffer))
|
|
|
|
|
|
|
|
offset, err = reader.Seek(40, io.SeekCurrent)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, int64(58), offset)
|
|
|
|
|
|
|
|
buffer = make([]byte, 4)
|
|
|
|
n, err = reader.Read(buffer)
|
|
|
|
assert.Error(t, err)
|
|
|
|
assert.Equal(t, 0, n)
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("Seek end", func(t *testing.T) {
|
|
|
|
filePath := "test-SeekEnd"
|
|
|
|
data := []byte("Test data for Seek end.")
|
|
|
|
|
|
|
|
err = rcm.Write(ctx, filePath, data)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
defer rcm.Remove(ctx, filePath)
|
|
|
|
|
|
|
|
reader, err := rcm.Reader(ctx, filePath)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
|
|
|
|
buffer := make([]byte, 4)
|
|
|
|
n, err := reader.Read(buffer)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, 4, n)
|
|
|
|
assert.Equal(t, "Test", string(buffer))
|
|
|
|
|
|
|
|
offset, err := reader.Seek(10, io.SeekEnd)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
assert.Equal(t, int64(33), offset)
|
|
|
|
|
|
|
|
buffer = make([]byte, 4)
|
|
|
|
n, err = reader.Read(buffer)
|
|
|
|
assert.Error(t, err)
|
|
|
|
assert.Equal(t, 0, n)
|
|
|
|
|
|
|
|
offset, err = reader.Seek(10, 3)
|
|
|
|
assert.Error(t, err)
|
|
|
|
assert.Equal(t, int64(0), offset)
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("Close", func(t *testing.T) {
|
|
|
|
filePath := "test-Close"
|
|
|
|
data := []byte("Test data for Close.")
|
|
|
|
|
|
|
|
err = rcm.Write(ctx, filePath, data)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
defer rcm.Remove(ctx, filePath)
|
|
|
|
|
|
|
|
reader, err := rcm.Reader(ctx, filePath)
|
|
|
|
assert.NoError(t, err)
|
|
|
|
|
|
|
|
err = reader.Close()
|
|
|
|
assert.NoError(t, err)
|
|
|
|
})
|
|
|
|
}
|