mirror of
https://gitee.com/milvus-io/milvus.git
synced 2024-12-05 05:18:52 +08:00
ebd0279d3f
Signed-off-by: yah01 <yang.cen@zilliz.com>
159 lines
5.3 KiB
Go
159 lines
5.3 KiB
Go
// Licensed to the LF AI & Data foundation under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package storage
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
|
|
"github.com/milvus-io/milvus/pkg/common"
|
|
"github.com/milvus-io/milvus/pkg/util/funcutil"
|
|
"github.com/milvus-io/milvus/pkg/util/uniquegenerator"
|
|
)
|
|
|
|
func TestIndexFileBinlogCodec(t *testing.T) {
|
|
indexBuildID := UniqueID(uniquegenerator.GetUniqueIntGeneratorIns().GetInt())
|
|
version := int64(uniquegenerator.GetUniqueIntGeneratorIns().GetInt())
|
|
collectionID := UniqueID(uniquegenerator.GetUniqueIntGeneratorIns().GetInt())
|
|
partitionID := UniqueID(uniquegenerator.GetUniqueIntGeneratorIns().GetInt())
|
|
segmentID := UniqueID(uniquegenerator.GetUniqueIntGeneratorIns().GetInt())
|
|
fieldID := UniqueID(uniquegenerator.GetUniqueIntGeneratorIns().GetInt())
|
|
indexName := funcutil.GenRandomStr()
|
|
indexID := UniqueID(uniquegenerator.GetUniqueIntGeneratorIns().GetInt())
|
|
indexParams := make(map[string]string)
|
|
indexParams[common.IndexTypeKey] = "IVF_FLAT"
|
|
datas := []*Blob{
|
|
{
|
|
Key: "ivf1",
|
|
Value: []byte{1, 2, 3},
|
|
},
|
|
{
|
|
Key: "ivf2",
|
|
Value: []byte{4, 5, 6},
|
|
},
|
|
{
|
|
Key: "large",
|
|
Value: funcutil.RandomBytes(maxLengthPerRowOfIndexFile + 1),
|
|
},
|
|
}
|
|
|
|
codec := NewIndexFileBinlogCodec()
|
|
|
|
serializedBlobs, err := codec.Serialize(indexBuildID, version, collectionID, partitionID, segmentID, fieldID, indexParams, indexName, indexID, datas)
|
|
assert.NoError(t, err)
|
|
|
|
idxBuildID, v, collID, parID, segID, fID, params, idxName, idxID, blobs, err := codec.DeserializeImpl(serializedBlobs)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, indexBuildID, idxBuildID)
|
|
assert.Equal(t, version, v)
|
|
assert.Equal(t, collectionID, collID)
|
|
assert.Equal(t, partitionID, parID)
|
|
assert.Equal(t, segmentID, segID)
|
|
assert.Equal(t, fieldID, fID)
|
|
assert.Equal(t, len(indexParams), len(params))
|
|
for key, value := range indexParams {
|
|
assert.Equal(t, value, params[key])
|
|
}
|
|
assert.Equal(t, indexName, idxName)
|
|
assert.Equal(t, indexID, idxID)
|
|
assert.ElementsMatch(t, datas, blobs)
|
|
|
|
blobs, indexParams, indexName, indexID, err = codec.Deserialize(serializedBlobs)
|
|
assert.NoError(t, err)
|
|
assert.ElementsMatch(t, datas, blobs)
|
|
for key, value := range indexParams {
|
|
assert.Equal(t, value, params[key])
|
|
}
|
|
assert.Equal(t, indexName, idxName)
|
|
assert.Equal(t, indexID, idxID)
|
|
|
|
// empty
|
|
_, _, _, _, _, _, _, _, _, _, err = codec.DeserializeImpl(nil)
|
|
assert.Error(t, err)
|
|
}
|
|
|
|
func TestIndexFileBinlogCodecError(t *testing.T) {
|
|
var err error
|
|
|
|
// failed to read binlog
|
|
codec := NewIndexFileBinlogCodec()
|
|
_, _, _, _, err = codec.Deserialize([]*Blob{{Key: "key", Value: []byte("not in binlog format")}})
|
|
assert.Error(t, err)
|
|
|
|
indexBuildID := UniqueID(uniquegenerator.GetUniqueIntGeneratorIns().GetInt())
|
|
version := int64(uniquegenerator.GetUniqueIntGeneratorIns().GetInt())
|
|
collectionID := UniqueID(uniquegenerator.GetUniqueIntGeneratorIns().GetInt())
|
|
partitionID := UniqueID(uniquegenerator.GetUniqueIntGeneratorIns().GetInt())
|
|
segmentID := UniqueID(uniquegenerator.GetUniqueIntGeneratorIns().GetInt())
|
|
fieldID := UniqueID(uniquegenerator.GetUniqueIntGeneratorIns().GetInt())
|
|
indexName := funcutil.GenRandomStr()
|
|
indexID := UniqueID(uniquegenerator.GetUniqueIntGeneratorIns().GetInt())
|
|
indexParams := make(map[string]string)
|
|
indexParams[common.IndexTypeKey] = "IVF_FLAT"
|
|
datas := []*Blob{
|
|
{
|
|
Key: "ivf1",
|
|
Value: []byte{1, 2, 3},
|
|
},
|
|
}
|
|
|
|
_, err = codec.Serialize(indexBuildID, version, collectionID, partitionID, segmentID, fieldID, indexParams, indexName, indexID, datas)
|
|
assert.NoError(t, err)
|
|
}
|
|
|
|
func TestIndexCodec(t *testing.T) {
|
|
indexCodec := NewIndexCodec()
|
|
blobs := []*Blob{
|
|
{
|
|
Key: "12345",
|
|
Value: []byte{1, 2, 3, 4, 5, 6, 7, 1, 2, 3, 4, 5, 6, 7},
|
|
Size: 14,
|
|
},
|
|
{
|
|
Key: "6666",
|
|
Value: []byte{6, 6, 6, 6, 6, 1, 2, 3, 4, 5, 6, 7},
|
|
Size: 12,
|
|
},
|
|
{
|
|
Key: "8885",
|
|
Value: []byte{8, 8, 8, 8, 8, 8, 8, 8, 2, 3, 4, 5, 6, 7},
|
|
Size: 14,
|
|
},
|
|
}
|
|
indexParams := map[string]string{
|
|
"k1": "v1", "k2": "v2",
|
|
}
|
|
blobsInput, err := indexCodec.Serialize(blobs, indexParams, "index_test_name", 1234)
|
|
assert.NoError(t, err)
|
|
assert.EqualValues(t, 4, len(blobsInput))
|
|
assert.EqualValues(t, IndexParamsKey, blobsInput[3].Key)
|
|
blobsOutput, indexParamsOutput, indexName, indexID, err := indexCodec.Deserialize(blobsInput)
|
|
assert.NoError(t, err)
|
|
assert.EqualValues(t, 3, len(blobsOutput))
|
|
for i := 0; i < 3; i++ {
|
|
assert.EqualValues(t, blobs[i], blobsOutput[i])
|
|
}
|
|
assert.EqualValues(t, indexParams, indexParamsOutput)
|
|
assert.EqualValues(t, "index_test_name", indexName)
|
|
assert.EqualValues(t, 1234, indexID)
|
|
|
|
blobs = []*Blob{}
|
|
_, _, _, _, err = indexCodec.Deserialize(blobs)
|
|
assert.Error(t, err)
|
|
}
|