test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-11-29 18:38:44 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

feat: Add e2e test cases for GoSDK (#33378)

Browse Source 
- Add e2e test cases for Go `milvusclient`
- Fix client SparseEmbedding to vector
issue: #33419

Signed-off-by: ThreadDao <yufen.zong@zilliz.com>
This commit is contained in:
ThreadDao 2024-06-04 10:57:47 +08:00 committed by GitHub
parent 7f4698f4a7
commit ac5e098e13
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 3858 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
client/entity/sparse.go
View File

@ -29,6 +29,7 @@ type SparseEmbedding interface {
Len() int // the actual items in this vector
Get(idx int) (pos uint32, value float32, ok bool)
Serialize() []byte
FieldType() FieldType
}
var (

240
tests/go_client/base/milvus_client.go Normal file
View File

@ -0,0 +1,240 @@
package base
import (
"context"
"encoding/json"
"strings"
"time"
"github.com/milvus-io/milvus/client/v2/entity"
"github.com/milvus-io/milvus/pkg/log"
"go.uber.org/zap"
"google.golang.org/grpc"
clientv2 "github.com/milvus-io/milvus/client/v2"
"github.com/milvus-io/milvus/client/v2/index"
)
func LoggingUnaryInterceptor() grpc.UnaryClientInterceptor {
return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
maxLogLength := 300
_method := strings.Split(method, "/")
_methodShotName := _method[len(_method)-1]
// Marshal req to json str
reqJSON, err := json.Marshal(req)
if err != nil {
log.Error("Failed to marshal request", zap.Error(err))
reqJSON = []byte("could not marshal request")
}
reqStr := string(reqJSON)
if len(reqStr) > maxLogLength {
reqStr = reqStr[:maxLogLength] + "..."
}
// log before
log.Info("Request", zap.String("method", _methodShotName), zap.Any("reqs", reqStr))
// invoker
start := time.Now()
errResp := invoker(ctx, method, req, reply, cc, opts...)
cost := time.Since(start)
// Marshal reply to json str
respJSON, err := json.Marshal(reply)
if err != nil {
log.Error("Failed to marshal response", zap.Error(err))
respJSON = []byte("could not marshal response")
}
respStr := string(respJSON)
if len(respStr) > maxLogLength {
respStr = respStr[:maxLogLength] + "..."
}
// log after
log.Info("Response", zap.String("method", _methodShotName), zap.Any("resp", respStr))
log.Debug("Cost", zap.String("method", _methodShotName), zap.Duration("cost", cost))
return errResp
}
}
type MilvusClient struct {
mClient *clientv2.Client
}
func NewMilvusClient(ctx context.Context, cfg *clientv2.ClientConfig) (*MilvusClient, error) {
cfg.DialOptions = append(cfg.DialOptions, grpc.WithUnaryInterceptor(LoggingUnaryInterceptor()))
mClient, err := clientv2.New(ctx, cfg)
return &MilvusClient{
mClient,
}, err
}
func (mc *MilvusClient) Close(ctx context.Context) error {
err := mc.mClient.Close(ctx)
return err
}
// -- database --
// UsingDatabase list all database in milvus cluster.
func (mc *MilvusClient) UsingDatabase(ctx context.Context, option clientv2.UsingDatabaseOption) error {
err := mc.mClient.UsingDatabase(ctx, option)
return err
}
// ListDatabases list all database in milvus cluster.
func (mc *MilvusClient) ListDatabases(ctx context.Context, option clientv2.ListDatabaseOption, callOptions ...grpc.CallOption) ([]string, error) {
databaseNames, err := mc.mClient.ListDatabase(ctx, option, callOptions...)
return databaseNames, err
}
// CreateDatabase create database with the given name.
func (mc *MilvusClient) CreateDatabase(ctx context.Context, option clientv2.CreateDatabaseOption, callOptions ...grpc.CallOption) error {
err := mc.mClient.CreateDatabase(ctx, option, callOptions...)
return err
}
// DropDatabase drop database with the given db name.
func (mc *MilvusClient) DropDatabase(ctx context.Context, option clientv2.DropDatabaseOption, callOptions ...grpc.CallOption) error {
err := mc.mClient.DropDatabase(ctx, option, callOptions...)
return err
}
// -- collection --
// CreateCollection Create Collection
func (mc *MilvusClient) CreateCollection(ctx context.Context, option clientv2.CreateCollectionOption, callOptions ...grpc.CallOption) error {
err := mc.mClient.CreateCollection(ctx, option, callOptions...)
return err
}
// ListCollections Create Collection
func (mc *MilvusClient) ListCollections(ctx context.Context, option clientv2.ListCollectionOption, callOptions ...grpc.CallOption) ([]string, error) {
collectionNames, err := mc.mClient.ListCollections(ctx, option, callOptions...)
return collectionNames, err
}
//DescribeCollection Describe collection
func (mc *MilvusClient) DescribeCollection(ctx context.Context, option clientv2.DescribeCollectionOption, callOptions ...grpc.CallOption) (*entity.Collection, error) {
collection, err := mc.mClient.DescribeCollection(ctx, option, callOptions...)
return collection, err
}
// HasCollection Has collection
func (mc *MilvusClient) HasCollection(ctx context.Context, option clientv2.HasCollectionOption, callOptions ...grpc.CallOption) (bool, error) {
has, err := mc.mClient.HasCollection(ctx, option, callOptions...)
return has, err
}
// DropCollection Drop Collection
func (mc *MilvusClient) DropCollection(ctx context.Context, option clientv2.DropCollectionOption, callOptions ...grpc.CallOption) error {
err := mc.mClient.DropCollection(ctx, option, callOptions...)
return err
}
// -- partition --
// CreatePartition Create Partition
func (mc *MilvusClient) CreatePartition(ctx context.Context, option clientv2.CreatePartitionOption, callOptions ...grpc.CallOption) error {
err := mc.mClient.CreatePartition(ctx, option, callOptions...)
return err
}
// DropPartition Drop Partition
func (mc *MilvusClient) DropPartition(ctx context.Context, option clientv2.DropPartitionOption, callOptions ...grpc.CallOption) error {
err := mc.mClient.DropPartition(ctx, option, callOptions...)
return err
}
// HasPartition Has Partition
func (mc *MilvusClient) HasPartition(ctx context.Context, option clientv2.HasPartitionOption, callOptions ...grpc.CallOption) (bool, error) {
has, err := mc.mClient.HasPartition(ctx, option, callOptions...)
return has, err
}
// ListPartitions List Partitions
func (mc *MilvusClient) ListPartitions(ctx context.Context, option clientv2.ListPartitionsOption, callOptions ...grpc.CallOption) ([]string, error) {
partitionNames, err := mc.mClient.ListPartitions(ctx, option, callOptions...)
return partitionNames, err
}
// LoadPartitions Load Partitions into memory
func (mc *MilvusClient) LoadPartitions(ctx context.Context, option clientv2.LoadPartitionsOption, callOptions ...grpc.CallOption) (clientv2.LoadTask, error) {
loadTask, err := mc.mClient.LoadPartitions(ctx, option, callOptions...)
return loadTask, err
}
// -- index --
// CreateIndex Create Index
func (mc *MilvusClient) CreateIndex(ctx context.Context, option clientv2.CreateIndexOption, callOptions ...grpc.CallOption) (*clientv2.CreateIndexTask, error) {
createIndexTask, err := mc.mClient.CreateIndex(ctx, option, callOptions...)
return createIndexTask, err
}
// ListIndexes List Indexes
func (mc *MilvusClient) ListIndexes(ctx context.Context, option clientv2.ListIndexOption, callOptions ...grpc.CallOption) ([]string, error) {
indexes, err := mc.mClient.ListIndexes(ctx, option, callOptions...)
return indexes, err
}
// DescribeIndex Describe Index
func (mc *MilvusClient) DescribeIndex(ctx context.Context, option clientv2.DescribeIndexOption, callOptions ...grpc.CallOption) (index.Index, error) {
index, err := mc.mClient.DescribeIndex(ctx, option, callOptions...)
return index, err
}
// DropIndex Drop Index
func (mc *MilvusClient) DropIndex(ctx context.Context, option clientv2.DropIndexOption, callOptions ...grpc.CallOption) error {
err := mc.mClient.DropIndex(ctx, option, callOptions...)
return err
}
// -- write --
// Insert insert data
func (mc *MilvusClient) Insert(ctx context.Context, option clientv2.InsertOption, callOptions ...grpc.CallOption) (clientv2.InsertResult, error) {
insertRes, err := mc.mClient.Insert(ctx, option, callOptions...)
log.Info("Insert", zap.Any("result", insertRes))
return insertRes, err
}
// Flush flush data
func (mc *MilvusClient) Flush(ctx context.Context, option clientv2.FlushOption, callOptions ...grpc.CallOption) (*clientv2.FlushTask, error) {
flushTask, err := mc.mClient.Flush(ctx, option, callOptions...)
return flushTask, err
}
// Delete deletes data
func (mc *MilvusClient) Delete(ctx context.Context, option clientv2.DeleteOption, callOptions ...grpc.CallOption) (clientv2.DeleteResult, error) {
deleteRes, err := mc.mClient.Delete(ctx, option, callOptions...)
return deleteRes, err
}
// Upsert upsert data
func (mc *MilvusClient) Upsert(ctx context.Context, option clientv2.UpsertOption, callOptions ...grpc.CallOption) (clientv2.UpsertResult, error) {
upsertRes, err := mc.mClient.Upsert(ctx, option, callOptions...)
return upsertRes, err
}
// -- read --
// LoadCollection Load Collection
func (mc *MilvusClient) LoadCollection(ctx context.Context, option clientv2.LoadCollectionOption, callOptions ...grpc.CallOption) (clientv2.LoadTask, error) {
loadTask, err := mc.mClient.LoadCollection(ctx, option, callOptions...)
return loadTask, err
}
// Search search from collection
func (mc *MilvusClient) Search(ctx context.Context, option clientv2.SearchOption, callOptions ...grpc.CallOption) ([]clientv2.ResultSet, error) {
resultSets, err := mc.mClient.Search(ctx, option, callOptions...)
return resultSets, err
}
// Query query from collection
func (mc *MilvusClient) Query(ctx context.Context, option clientv2.QueryOption, callOptions ...grpc.CallOption) (clientv2.ResultSet, error) {
resultSet, err := mc.mClient.Query(ctx, option, callOptions...)
return resultSet, err
}

67
tests/go_client/common/consts.go Normal file
View File

@ -0,0 +1,67 @@
package common
// cost default field name
const (
DefaultInt8FieldName = "int8"
DefaultInt16FieldName = "int16"
DefaultInt32FieldName = "int32"
DefaultInt64FieldName = "int64"
DefaultBoolFieldName = "bool"
DefaultFloatFieldName = "float"
DefaultDoubleFieldName = "double"
DefaultVarcharFieldName = "varchar"
DefaultJSONFieldName = "json"
DefaultArrayFieldName = "array"
DefaultFloatVecFieldName = "floatVec"
DefaultBinaryVecFieldName = "binaryVec"
DefaultFloat16VecFieldName = "fp16Vec"
DefaultBFloat16VecFieldName = "bf16Vec"
DefaultSparseVecFieldName = "sparseVec"
DefaultDynamicNumberField = "dynamicNumber"
DefaultDynamicStringField = "dynamicString"
DefaultDynamicBoolField = "dynamicBool"
DefaultDynamicListField = "dynamicList"
DefaultBoolArrayField = "boolArray"
DefaultInt8ArrayField = "int8Array"
DefaultInt16ArrayField = "int16Array"
DefaultInt32ArrayField = "int32Array"
DefaultInt64ArrayField = "int64Array"
DefaultFloatArrayField = "floatArray"
DefaultDoubleArrayField = "doubleArray"
DefaultVarcharArrayField = "varcharArray"
)
// cost for test cases
const (
RowCount = "row_count"
DefaultTimeout = 120
DefaultDim = 128
DefaultShards = int32(2)
DefaultNb = 3000
DefaultNq = 5
DefaultLimit = 10
TestCapacity = 100 // default array field capacity
TestMaxLen = 100 // default varchar field max length
)
// const default value from milvus config
const (
MaxPartitionNum = 4096
DefaultDynamicFieldName = "$meta"
QueryCountFieldName = "count(*)"
DefaultPartition = "_default"
DefaultIndexName = "_default_idx_102"
DefaultIndexNameBinary = "_default_idx_100"
DefaultRgName = "__default_resource_group"
DefaultDb = "default"
MaxDim = 32768
MaxLength = int64(65535)
MaxCollectionNameLen = 255
DefaultRgCapacity = 1000000
RetentionDuration = 40 // common.retentionDuration
MaxCapacity = 4096 // max array capacity
DefaultPartitionNum = 16 // default num_partitions
MaxTopK = 16384
MaxVectorFieldNum = 4
MaxShardNum = 16
)

44
tests/go_client/common/response_checker.go Normal file
View File

@ -0,0 +1,44 @@
package common
import (
"strings"
"testing"
"github.com/milvus-io/milvus/pkg/log"
"github.com/stretchr/testify/require"
clientv2 "github.com/milvus-io/milvus/client/v2"
)
func CheckErr(t *testing.T, actualErr error, expErrNil bool, expErrorMsg ...string) {
if expErrNil {
require.NoError(t, actualErr)
} else {
require.Error(t, actualErr)
switch len(expErrorMsg) {
case 0:
log.Fatal("expect error message should not be empty")
case 1:
require.ErrorContains(t, actualErr, expErrorMsg[0])
default:
contains := false
for i := 0; i < len(expErrorMsg); i++ {
if strings.Contains(actualErr.Error(), expErrorMsg[i]) {
contains = true
}
}
if !contains {
t.FailNow()
}
}
}
}
// CheckSearchResult check search result, check nq, topk, ids, score
func CheckSearchResult(t *testing.T, actualSearchResults []clientv2.ResultSet, expNq int, expTopK int) {
require.Equal(t, len(actualSearchResults), expNq)
require.Len(t, actualSearchResults, expNq)
for _, actualSearchResult := range actualSearchResults {
require.Equal(t, actualSearchResult.ResultCount, expTopK)
}
}

122
tests/go_client/common/utils.go Normal file
View File

@ -0,0 +1,122 @@
package common
import (
"encoding/binary"
"fmt"
"log"
"math"
"math/rand"
"strings"
"time"
"github.com/milvus-io/milvus/client/v2/entity"
"github.com/x448/float16"
)
var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
var r *rand.Rand
func init() {
r = rand.New(rand.NewSource(time.Now().UnixNano()))
}
func GenRandomString(prefix string, n int) string {
b := make([]rune, n)
for i := range b {
b[i] = letterRunes[r.Intn(len(letterRunes))]
}
str := fmt.Sprintf("%s_%s", prefix, string(b))
return str
}
// GenLongString gen invalid long string
func GenLongString(n int) string {
var builder strings.Builder
longString := "a"
for i := 0; i < n; i++ {
builder.WriteString(longString)
}
return builder.String()
}
func GenValidNames() []string {
return []string{
"a",
"_",
"_name",
"_123",
"name_",
"_coll_123_",
}
}
func GenInvalidNames() []string {
invalidNames := []string{
"",
" ",
"12-s",
"(mn)",
"中文",
"%$#",
"1",
"[10]",
"a b",
DefaultDynamicFieldName,
GenLongString(MaxCollectionNameLen + 1),
}
return invalidNames
}
func GenFloatVector(dim int) []float32 {
vector := make([]float32, 0, dim)
for j := 0; j < int(dim); j++ {
vector = append(vector, rand.Float32())
}
return vector
}
func GenFloat16Vector(dim int) []byte {
ret := make([]byte, dim*2)
for i := 0; i < int(dim); i++ {
v := float16.Fromfloat32(rand.Float32()).Bits()
binary.LittleEndian.PutUint16(ret[i*2:], v)
}
return ret
}
func GenBFloat16Vector(dim int) []byte {
ret16 := make([]uint16, 0, dim)
for i := 0; i < int(dim); i++ {
f := rand.Float32()
bits := math.Float32bits(f)
bits >>= 16
bits &= 0x7FFF
ret16 = append(ret16, uint16(bits))
}
ret := make([]byte, len(ret16)*2)
for i, value := range ret16 {
binary.LittleEndian.PutUint16(ret[i*2:], value)
}
return ret
}
func GenBinaryVector(dim int) []byte {
vector := make([]byte, dim/8)
rand.Read(vector)
return vector
}
func GenSparseVector(maxLen int) entity.SparseEmbedding {
length := 1 + rand.Intn(1+maxLen)
positions := make([]uint32, length)
values := make([]float32, length)
for i := 0; i < length; i++ {
positions[i] = uint32(2*i + 1)
values[i] = rand.Float32()
}
vector, err := entity.NewSliceSparseEmbedding(positions, values)
if err != nil {
log.Fatalf("Generate vector failed %s", err)
}
return vector
}

129
tests/go_client/go.mod Normal file
View File

@ -0,0 +1,129 @@
module github.com/milvus-io/milvus/tests/go_client
go 1.20
require (
github.com/milvus-io/milvus/client/v2 v2.0.0-20240521081339-017fd7bc25de
github.com/milvus-io/milvus/pkg v0.0.2-0.20240317152703-17b4938985f3
github.com/stretchr/testify v1.9.0
github.com/x448/float16 v0.8.4
go.uber.org/zap v1.27.0
google.golang.org/grpc v1.64.0
)
replace github.com/milvus-io/milvus/client/v2 v2.0.0-20240521081339-017fd7bc25de => ../../../milvus/client
require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/blang/semver/v4 v4.0.0 // indirect
github.com/cenkalti/backoff/v4 v4.2.0 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cilium/ebpf v0.11.0 // indirect
github.com/cockroachdb/errors v1.9.1 // indirect
github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect
github.com/cockroachdb/redact v1.1.3 // indirect
github.com/containerd/cgroups/v3 v3.0.3 // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
github.com/fsnotify/fsnotify v1.4.9 // indirect
github.com/getsentry/sentry-go v0.12.0 // indirect
github.com/go-logr/logr v1.3.0 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/godbus/dbus/v5 v5.0.4 // indirect
github.com/gogo/googleapis v1.4.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/gogo/status v1.1.0 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/btree v1.1.2 // indirect
github.com/gorilla/websocket v1.4.2 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/jonboulle/clockwork v0.2.2 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/magiconair/properties v1.8.5 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240430035521-259ae1d10016 // indirect
github.com/mitchellh/mapstructure v1.4.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/opencontainers/runtime-spec v1.0.2 // indirect
github.com/panjf2000/ants/v2 v2.7.2 // indirect
github.com/pelletier/go-toml v1.9.3 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/prometheus/client_golang v1.14.0 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.42.0 // indirect
github.com/prometheus/procfs v0.9.0 // indirect
github.com/rogpeppe/go-internal v1.10.0 // indirect
github.com/samber/lo v1.27.0 // indirect
github.com/shirou/gopsutil/v3 v3.22.9 // indirect
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/soheilhy/cmux v0.1.5 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/afero v1.6.0 // indirect
github.com/spf13/cast v1.3.1 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/spf13/viper v1.8.1 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
github.com/tidwall/gjson v1.17.1 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/tklauser/go-sysconf v0.3.10 // indirect
github.com/tklauser/numcpus v0.4.0 // indirect
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
go.etcd.io/bbolt v1.3.6 // indirect
go.etcd.io/etcd/api/v3 v3.5.5 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect
go.etcd.io/etcd/client/v2 v2.305.5 // indirect
go.etcd.io/etcd/client/v3 v3.5.5 // indirect
go.etcd.io/etcd/pkg/v3 v3.5.5 // indirect
go.etcd.io/etcd/raft/v3 v3.5.5 // indirect
go.etcd.io/etcd/server/v3 v3.5.5 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.38.0 // indirect
go.opentelemetry.io/otel v1.13.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.13.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.13.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.13.0 // indirect
go.opentelemetry.io/otel/metric v0.35.0 // indirect
go.opentelemetry.io/otel/sdk v1.13.0 // indirect
go.opentelemetry.io/otel/trace v1.13.0 // indirect
go.opentelemetry.io/proto/otlp v0.19.0 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/automaxprocs v1.5.2 // indirect
go.uber.org/multierr v1.10.0 // indirect
golang.org/x/crypto v0.22.0 // indirect
golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 // indirect
golang.org/x/net v0.24.0 // indirect
golang.org/x/sync v0.6.0 // indirect
golang.org/x/sys v0.20.0 // indirect
golang.org/x/text v0.15.0 // indirect
golang.org/x/time v0.3.0 // indirect
google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.62.0 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/apimachinery v0.28.6 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)

1113
tests/go_client/go.sum Normal file
View File

File diff suppressed because it is too large Load Diff

92
tests/go_client/testcases/client_test.go Normal file
View File

@ -0,0 +1,92 @@
///go:build L0
package testcases
import (
"strings"
"testing"
"time"
"github.com/milvus-io/milvus/tests/go_client/testcases/helper"
clientv2 "github.com/milvus-io/milvus/client/v2"
"github.com/milvus-io/milvus/tests/go_client/base"
"github.com/milvus-io/milvus/tests/go_client/common"
)
// test connect and close, connect again
func TestConnectClose(t *testing.T) {
// connect
ctx := helper.CreateContext(t, time.Second*common.DefaultTimeout)
mc, errConnect := base.NewMilvusClient(ctx, &defaultCfg)
common.CheckErr(t, errConnect, true)
// verify that connect success
listOpt := clientv2.NewListCollectionOption()
_, errList := mc.ListCollections(ctx, listOpt)
common.CheckErr(t, errList, true)
// close connect and verify
err := mc.Close(ctx)
common.CheckErr(t, err, true)
_, errList2 := mc.ListCollections(ctx, listOpt)
common.CheckErr(t, errList2, false, "service not ready[SDK=0]: not connected")
// connect again
mc, errConnect2 := base.NewMilvusClient(ctx, &defaultCfg)
common.CheckErr(t, errConnect2, true)
_, errList3 := mc.ListCollections(ctx, listOpt)
common.CheckErr(t, errList3, true)
}
func genInvalidClientConfig() []clientv2.ClientConfig {
invalidClientConfigs := []clientv2.ClientConfig{
{Address: "aaa"}, // not exist address
{Address: strings.Split(*addr, ":")[0]}, // Address=localhost
{Address: strings.Split(*addr, ":")[1]}, // Address=19530
{Address: *addr, Username: "aaa"}, // not exist username
{Address: *addr, Username: "root", Password: "aaa"}, // wrong password
{Address: *addr, DBName: "aaa"}, // not exist db
}
return invalidClientConfigs
}
// test connect with timeout and invalid addr
func TestConnectInvalidAddr(t *testing.T) {
// connect
ctx := helper.CreateContext(t, time.Second*5)
for _, invalidCfg := range genInvalidClientConfig() {
_, errConnect := base.NewMilvusClient(ctx, &invalidCfg)
common.CheckErr(t, errConnect, false, "context deadline exceeded")
}
}
// test connect repeatedly
func TestConnectRepeat(t *testing.T) {
// connect
ctx := helper.CreateContext(t, time.Second*10)
_, errConnect := base.NewMilvusClient(ctx, &defaultCfg)
common.CheckErr(t, errConnect, true)
// connect again
mc, errConnect2 := base.NewMilvusClient(ctx, &defaultCfg)
common.CheckErr(t, errConnect2, true)
_, err := mc.ListCollections(ctx, clientv2.NewListCollectionOption())
common.CheckErr(t, err, true)
}
// test close repeatedly
func TestCloseRepeat(t *testing.T) {
// connect
ctx := helper.CreateContext(t, time.Second*10)
mc, errConnect2 := base.NewMilvusClient(ctx, &defaultCfg)
common.CheckErr(t, errConnect2, true)
// close and again
err := mc.Close(ctx)
common.CheckErr(t, err, true)
err = mc.Close(ctx)
common.CheckErr(t, err, true)
}

950
tests/go_client/testcases/collection_test.go Normal file
View File

@ -0,0 +1,950 @@
package testcases
import (
"fmt"
"testing"
"time"
hp "github.com/milvus-io/milvus/tests/go_client/testcases/helper"
"github.com/stretchr/testify/require"
"github.com/milvus-io/milvus/pkg/log"
"go.uber.org/zap"
clientv2 "github.com/milvus-io/milvus/client/v2"
"github.com/milvus-io/milvus/client/v2/entity"
"github.com/milvus-io/milvus/tests/go_client/common"
)
var prefix = "collection"
// test create default floatVec and binaryVec collection
func TestCreateCollection(t *testing.T) {
t.Parallel()
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
for _, collectionFieldsType := range []hp.CollectionFieldsType{hp.Int64Vec, hp.VarcharBinary, hp.Int64VarcharSparseVec, hp.AllFields} {
fields := hp.FieldsFact.GenFieldsForCollection(collectionFieldsType, hp.TNewFieldsOption())
schema := hp.GenSchema(hp.TNewSchemaOption().TWithFields(fields))
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(schema.CollectionName, schema))
common.CheckErr(t, err, true)
// has collections and verify
has, err := mc.HasCollection(ctx, clientv2.NewHasCollectionOption(schema.CollectionName))
common.CheckErr(t, err, true)
require.True(t, has)
// list collections and verify
collections, err := mc.ListCollections(ctx, clientv2.NewListCollectionOption())
common.CheckErr(t, err, true)
require.Contains(t, collections, schema.CollectionName)
}
}
//func TestCreateCollection(t *testing.T) {}
func TestCreateAutoIdCollectionField(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
vecField := entity.NewField().WithName(common.DefaultFloatVecFieldName).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim)
int64Field := entity.NewField().WithName(common.DefaultInt64FieldName).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true).WithIsAutoID(true)
varcharField := entity.NewField().WithName(common.DefaultVarcharFieldName).WithDataType(entity.FieldTypeVarChar).WithIsPrimaryKey(true).WithIsAutoID(true).WithMaxLength(common.MaxLength)
for _, pkField := range []*entity.Field{int64Field, varcharField} {
// pk field with name
collName := common.GenRandomString(prefix, 6)
schema := entity.NewSchema().WithName(collName).WithField(pkField).WithField(vecField)
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, true)
// verify field name
coll, err := mc.DescribeCollection(ctx, clientv2.NewDescribeCollectionOption(collName))
common.CheckErr(t, err, true)
require.True(t, coll.Schema.AutoID)
require.True(t, coll.Schema.Fields[0].AutoID)
// insert
vecColumn := hp.GenColumnData(common.DefaultNb, vecField.DataType, *hp.TNewColumnOption())
_, err = mc.Insert(ctx, clientv2.NewColumnBasedInsertOption(schema.CollectionName, vecColumn))
common.CheckErr(t, err, true)
}
}
// create collection and specify shard num
func TestCreateCollectionShards(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
vecField := entity.NewField().WithName(common.DefaultFloatVecFieldName).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim)
int64Field := entity.NewField().WithName(common.DefaultInt64FieldName).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true).WithIsAutoID(true)
for _, shard := range []int32{-1, 0, 2, 16} {
// pk field with name
collName := common.GenRandomString(prefix, 6)
schema := entity.NewSchema().WithName(collName).WithField(int64Field).WithField(vecField)
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema).WithShardNum(shard))
common.CheckErr(t, err, true)
// verify field name
coll, err := mc.DescribeCollection(ctx, clientv2.NewDescribeCollectionOption(collName))
common.CheckErr(t, err, true)
if shard < 1 {
shard = 1
}
require.Equal(t, shard, coll.ShardNum)
}
}
// test create auto collection with schema
func TestCreateAutoIdCollectionSchema(t *testing.T) {
t.Skip("waiting for valid AutoId from schema params")
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
collName := common.GenRandomString(prefix, 6)
vecField := entity.NewField().WithName(common.DefaultFloatVecFieldName).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim)
for _, pkFieldType := range []entity.FieldType{entity.FieldTypeVarChar, entity.FieldTypeInt64} {
pkField := entity.NewField().WithName("pk").WithDataType(pkFieldType).WithIsPrimaryKey(true).WithMaxLength(common.MaxLength)
// pk field with name
schema := entity.NewSchema().WithName(collName).WithField(pkField).WithField(vecField).WithAutoID(true)
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, true)
// verify field name
coll, err := mc.DescribeCollection(ctx, clientv2.NewDescribeCollectionOption(collName))
common.CheckErr(t, err, true)
log.Info("schema autoID", zap.Bool("schemaAuto", coll.Schema.AutoID))
log.Info("field autoID", zap.Bool("fieldAuto", coll.Schema.Fields[0].AutoID))
// insert
vecColumn := hp.GenColumnData(common.DefaultNb, vecField.DataType, *hp.TNewColumnOption())
_, err = mc.Insert(ctx, clientv2.NewColumnBasedInsertOption(schema.CollectionName, vecColumn))
common.CheckErr(t, err, false, "field pk not passed")
}
}
// test create auto collection with collection option
func TestCreateAutoIdCollection(t *testing.T) {
t.Skip("waiting for valid AutoId from collection option")
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
collName := common.GenRandomString(prefix, 6)
vecField := entity.NewField().WithName(common.DefaultFloatVecFieldName).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim)
for _, pkFieldType := range []entity.FieldType{entity.FieldTypeVarChar, entity.FieldTypeInt64} {
pkField := entity.NewField().WithName("pk").WithDataType(pkFieldType).WithIsPrimaryKey(true).WithMaxLength(common.MaxLength)
// pk field with name
schema := entity.NewSchema().WithName(collName).WithField(pkField).WithField(vecField)
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema).WithAutoID(true))
common.CheckErr(t, err, true)
// verify field name
coll, err := mc.DescribeCollection(ctx, clientv2.NewDescribeCollectionOption(collName))
common.CheckErr(t, err, true)
log.Info("schema autoID", zap.Bool("schemaAuto", coll.Schema.AutoID))
log.Info("field autoID", zap.Bool("fieldAuto", coll.Schema.Fields[0].AutoID))
// insert
vecColumn := hp.GenColumnData(common.DefaultNb, vecField.DataType, *hp.TNewColumnOption())
_, err = mc.Insert(ctx, clientv2.NewColumnBasedInsertOption(schema.CollectionName, vecColumn))
common.CheckErr(t, err, false, "field pk not passed")
}
}
func TestCreateJsonCollection(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
collName := common.GenRandomString(prefix, 6)
vecField := entity.NewField().WithName(common.DefaultFloatVecFieldName).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim)
pkField := entity.NewField().WithName(common.DefaultVarcharFieldName).WithDataType(entity.FieldTypeVarChar).WithIsPrimaryKey(true).WithMaxLength(common.MaxLength)
jsonField := entity.NewField().WithName(common.DefaultJSONFieldName).WithDataType(entity.FieldTypeJSON)
// pk field with name
schema := entity.NewSchema().WithName(collName).WithField(pkField).WithField(vecField).WithField(jsonField)
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, true)
// verify field name
has, err := mc.HasCollection(ctx, clientv2.NewHasCollectionOption(schema.CollectionName))
common.CheckErr(t, err, true)
require.True(t, has)
}
func TestCreateArrayCollections(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
collName := common.GenRandomString(prefix, 6)
vecField := entity.NewField().WithName(common.DefaultFloatVecFieldName).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim)
pkField := entity.NewField().WithName(common.DefaultVarcharFieldName).WithDataType(entity.FieldTypeVarChar).WithIsPrimaryKey(true).WithMaxLength(common.MaxLength)
schema := entity.NewSchema().WithName(collName).WithField(pkField).WithField(vecField)
for _, eleType := range hp.GetAllArrayElementType() {
arrayField := entity.NewField().WithName(hp.GetFieldNameByElementType(eleType)).WithDataType(entity.FieldTypeArray).WithElementType(eleType).WithMaxCapacity(common.MaxCapacity)
if eleType == entity.FieldTypeVarChar {
arrayField.WithMaxLength(common.MaxLength)
}
schema.WithField(arrayField)
}
// pk field with name
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, true)
// verify field name
has, err := mc.HasCollection(ctx, clientv2.NewHasCollectionOption(schema.CollectionName))
common.CheckErr(t, err, true)
require.True(t, has)
}
// test create collection with partition key not supported field type
func TestCreateCollectionPartitionKey(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
int64Field := entity.NewField().WithName(common.DefaultInt64FieldName).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)
vecField := entity.NewField().WithName(common.DefaultFloatVecFieldName).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim)
t.Parallel()
for _, fieldType := range []entity.FieldType{entity.FieldTypeVarChar, entity.FieldTypeInt64} {
partitionKeyField := entity.NewField().WithName("par_key").WithDataType(fieldType).WithIsPartitionKey(true).WithMaxLength(common.TestMaxLen)
collName := common.GenRandomString(prefix, 6)
schema := entity.NewSchema().WithName(collName).WithField(int64Field).WithField(vecField).WithField(partitionKeyField)
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, true)
coll, err := mc.DescribeCollection(ctx, clientv2.NewDescribeCollectionOption(collName))
common.CheckErr(t, err, true)
for _, field := range coll.Schema.Fields {
if field.Name == "par_key" {
require.True(t, field.IsPartitionKey)
}
}
// verify partitions
partitions, err := mc.ListPartitions(ctx, clientv2.NewListPartitionOption(collName))
require.Len(t, partitions, common.DefaultPartitionNum)
}
}
// test create partition key collection WithPartitionNum
func TestCreateCollectionPartitionKeyNumPartition(t *testing.T) {
t.Skip("Waiting for WithPartitionNum")
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
int64Field := entity.NewField().WithName(common.DefaultInt64FieldName).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)
vecField := entity.NewField().WithName(common.DefaultFloatVecFieldName).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim)
partitionKeyField := entity.NewField().WithName("par_key").WithDataType(entity.FieldTypeInt64).WithIsPartitionKey(true)
t.Parallel()
for _, numPartition := range []int64{1, 128, 64, 4096} {
collName := common.GenRandomString(prefix, 6)
schema := entity.NewSchema().WithName(collName).WithField(int64Field).WithField(vecField).WithField(partitionKeyField)
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, true)
// verify partitions num
partitions, err := mc.ListPartitions(ctx, clientv2.NewListPartitionOption(collName))
require.Len(t, partitions, int(numPartition))
}
}
func TestCreateCollectionDynamicSchema(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
collName := common.GenRandomString(prefix, 6)
vecField := entity.NewField().WithName(common.DefaultFloatVecFieldName).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim)
pkField := entity.NewField().WithName(common.DefaultVarcharFieldName).WithDataType(entity.FieldTypeVarChar).WithIsPrimaryKey(true).WithMaxLength(common.MaxLength)
schema := entity.NewSchema().WithName(collName).WithField(pkField).WithField(vecField).WithDynamicFieldEnabled(true)
// pk field with name
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, true)
// verify field name
has, err := mc.HasCollection(ctx, clientv2.NewHasCollectionOption(schema.CollectionName))
common.CheckErr(t, err, true)
require.True(t, has)
coll, err := mc.DescribeCollection(ctx, clientv2.NewDescribeCollectionOption(schema.CollectionName))
require.True(t, coll.Schema.EnableDynamicField)
// insert dynamic
columnOption := *hp.TNewColumnOption()
varcharColumn := hp.GenColumnData(common.DefaultNb, entity.FieldTypeVarChar, columnOption)
vecColumn := hp.GenColumnData(common.DefaultNb, entity.FieldTypeFloatVector, columnOption)
dynamicData := hp.GenDynamicFieldData(0, common.DefaultNb)
_, err = mc.Insert(ctx, clientv2.NewColumnBasedInsertOption(schema.CollectionName, varcharColumn, vecColumn).WithColumns(dynamicData...))
common.CheckErr(t, err, true)
}
func TestCreateCollectionDynamic(t *testing.T) {
t.Skip("waiting for dynamicField alignment")
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
collName := common.GenRandomString(prefix, 6)
vecField := entity.NewField().WithName(common.DefaultFloatVecFieldName).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim)
pkField := entity.NewField().WithName(common.DefaultVarcharFieldName).WithDataType(entity.FieldTypeVarChar).WithIsPrimaryKey(true).WithMaxLength(common.MaxLength)
schema := entity.NewSchema().WithName(collName).WithField(pkField).WithField(vecField)
// pk field with name
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema).WithDynamicSchema(true))
common.CheckErr(t, err, true)
// verify field name
has, err := mc.HasCollection(ctx, clientv2.NewHasCollectionOption(schema.CollectionName))
common.CheckErr(t, err, true)
require.True(t, has)
coll, err := mc.DescribeCollection(ctx, clientv2.NewDescribeCollectionOption(schema.CollectionName))
log.Info("collection dynamic", zap.Bool("collectionSchema", coll.Schema.EnableDynamicField))
//require.True(t, coll.Schema.Fields[0].IsDynamic)
// insert dynamic
columnOption := *hp.TNewColumnOption()
varcharColumn := hp.GenColumnData(common.DefaultNb, entity.FieldTypeVarChar, columnOption)
vecColumn := hp.GenColumnData(common.DefaultNb, entity.FieldTypeFloatVector, columnOption)
dynamicData := hp.GenDynamicFieldData(0, common.DefaultNb)
_, err = mc.Insert(ctx, clientv2.NewColumnBasedInsertOption(schema.CollectionName, varcharColumn, vecColumn).WithColumns(dynamicData...))
common.CheckErr(t, err, false, "field dynamicNumber does not exist")
}
func TestCreateCollectionAllFields(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
collName := common.GenRandomString(prefix, 6)
schema := entity.NewSchema().WithName(collName)
// gen all fields except sparse vector
fields := hp.FieldsFactory{}.GenFieldsForCollection(hp.AllFields, hp.TNewFieldsOption())
for _, field := range fields {
schema.WithField(field)
}
// pk field with name
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, true)
// verify field name
has, err := mc.HasCollection(ctx, clientv2.NewHasCollectionOption(schema.CollectionName))
common.CheckErr(t, err, true)
require.True(t, has)
}
func TestCreateCollectionSparseVector(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
collName := common.GenRandomString(prefix, 6)
sparseVecField := entity.NewField().WithName(common.DefaultSparseVecFieldName).WithDataType(entity.FieldTypeSparseVector)
pkField := entity.NewField().WithName(common.DefaultVarcharFieldName).WithDataType(entity.FieldTypeVarChar).WithIsPrimaryKey(true).WithMaxLength(common.MaxLength)
schema := entity.NewSchema().WithName(collName).WithField(pkField).WithField(sparseVecField)
// pk field with name
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema).WithDynamicSchema(true))
common.CheckErr(t, err, true)
// verify field name
has, err := mc.HasCollection(ctx, clientv2.NewHasCollectionOption(schema.CollectionName))
common.CheckErr(t, err, true)
require.True(t, has)
}
func TestCreateCollectionWithValidFieldName(t *testing.T) {
t.Parallel()
// connect
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// create collection with valid field name
for _, name := range common.GenValidNames() {
collName := common.GenRandomString(prefix, 6)
// pk field with name
pkField := entity.NewField().WithName(name).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)
vecField := entity.NewField().WithName(common.DefaultFloatVecFieldName).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim)
schema := entity.NewSchema().WithName(collName).WithField(pkField).WithField(vecField)
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, true)
// verify field name
coll, err := mc.DescribeCollection(ctx, clientv2.NewDescribeCollectionOption(collName))
common.CheckErr(t, err, true)
require.Equal(t, name, coll.Schema.Fields[0].Name)
}
}
func genDefaultSchema() *entity.Schema {
int64Pk := entity.NewField().WithName(common.DefaultInt64FieldName).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)
varchar := entity.NewField().WithName(common.DefaultVarcharFieldName).WithDataType(entity.FieldTypeVarChar).WithMaxLength(common.TestMaxLen)
floatVec := entity.NewField().WithName(common.DefaultFloatVecFieldName).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim)
binaryVec := entity.NewField().WithName(common.DefaultBinaryVecFieldName).WithDataType(entity.FieldTypeBinaryVector).WithDim(common.DefaultDim)
schema := entity.NewSchema().WithField(int64Pk).WithField(varchar).WithField(floatVec).WithField(binaryVec)
return schema
}
// create collection with valid name
func TestCreateCollectionWithValidName(t *testing.T) {
t.Parallel()
// connect
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
for _, name := range common.GenValidNames() {
schema := genDefaultSchema().WithName(name)
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(name, schema))
common.CheckErr(t, err, true)
collections, err := mc.ListCollections(ctx, clientv2.NewListCollectionOption())
common.CheckErr(t, err, true)
require.Contains(t, collections, name)
err = mc.DropCollection(ctx, clientv2.NewDropCollectionOption(name))
common.CheckErr(t, err, true)
}
}
// create collection with invalid field name
func TestCreateCollectionWithInvalidFieldName(t *testing.T) {
t.Parallel()
// connect
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// create collection with invalid field name
for _, invalidName := range common.GenInvalidNames() {
log.Debug("TestCreateCollectionWithInvalidFieldName", zap.String("fieldName", invalidName))
pkField := entity.NewField().WithName(invalidName).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)
schema := entity.NewSchema().WithName("aaa").WithField(pkField)
collOpt := clientv2.NewCreateCollectionOption("aaa", schema)
err := mc.CreateCollection(ctx, collOpt)
common.CheckErr(t, err, false, "field name should not be empty",
"The first character of a field name must be an underscore or letter",
"Field name cannot only contain numbers, letters, and underscores",
"The length of a field name must be less than 255 characters")
}
}
// create collection with invalid collection name: invalid str, schemaName isn't equal to collectionName, schema name is empty
func TestCreateCollectionWithInvalidCollectionName(t *testing.T) {
t.Parallel()
// connect
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
collName := common.GenRandomString(prefix, 6)
// create collection and schema no name
schema := genDefaultSchema()
err2 := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err2, false, "collection name should not be empty")
// create collection with invalid schema name
for _, invalidName := range common.GenInvalidNames() {
log.Debug("TestCreateCollectionWithInvalidCollectionName", zap.String("collectionName", invalidName))
// schema has invalid name
schema.WithName(invalidName)
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, false, "collection name should not be empty",
"the first character of a collection name must be an underscore or letter",
"collection name can only contain numbers, letters and underscores",
"the length of a collection name must be less than 255 characters")
// collection option has invalid name
schema.WithName(collName)
err2 := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(invalidName, schema))
common.CheckErr(t, err2, false, "collection name matches schema name")
}
// collection name not equal to schema name
schema.WithName(collName)
err3 := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(common.GenRandomString("pre", 4), schema))
common.CheckErr(t, err3, false, "collection name matches schema name")
}
// create collection missing pk field or vector field
func TestCreateCollectionInvalidFields(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
type invalidFieldsStruct struct {
fields []*entity.Field
errMsg string
}
pkField := entity.NewField().WithName(common.DefaultInt64FieldName).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)
pkField2 := entity.NewField().WithName("pk").WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)
varcharField := entity.NewField().WithName(common.DefaultVarcharFieldName).WithDataType(entity.FieldTypeVarChar)
stringField := entity.NewField().WithName("str").WithDataType(entity.FieldTypeString)
vecField := entity.NewField().WithName(common.DefaultFloatVecFieldName).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim)
noneField := entity.NewField().WithName("none").WithDataType(entity.FieldTypeNone)
invalidFields := []invalidFieldsStruct{
// TODO https://github.com/milvus-io/milvus/issues/33199
//{fields: []*entity.Field{pkField}, errMsg: "vector field not set"},
{fields: []*entity.Field{vecField}, errMsg: "primary key is not specified"},
{fields: []*entity.Field{pkField, pkField2, vecField}, errMsg: "there are more than one primary key"},
{fields: []*entity.Field{pkField, vecField, noneField}, errMsg: "data type None is not valid"},
{fields: []*entity.Field{pkField, vecField, stringField}, errMsg: "string data type not supported yet, please use VarChar type instead"},
{fields: []*entity.Field{pkField, vecField, varcharField}, errMsg: "type param(max_length) should be specified for varChar field"},
}
collName := common.GenRandomString(prefix, 6)
for _, invalidField := range invalidFields {
schema := entity.NewSchema().WithName(collName)
for _, field := range invalidField.fields {
schema.WithField(field)
}
collOpt := clientv2.NewCreateCollectionOption(collName, schema)
err := mc.CreateCollection(ctx, collOpt)
common.CheckErr(t, err, false, invalidField.errMsg)
}
}
// create autoID or not collection with non-int64 and non-varchar field
func TestCreateCollectionInvalidAutoPkField(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
t.Parallel()
// create collection with autoID true or not
collName := common.GenRandomString(prefix, 6)
for _, autoId := range []bool{true, false} {
vecField := entity.NewField().WithName(common.DefaultFloatVecFieldName).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim)
// pk field type: non-int64 and non-varchar
for _, fieldType := range hp.GetInvalidPkFieldType() {
invalidPkField := entity.NewField().WithName("pk").WithDataType(fieldType).WithIsPrimaryKey(true)
schema := entity.NewSchema().WithName(collName).WithField(vecField).WithField(invalidPkField).WithAutoID(autoId)
errNonInt64Field := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, errNonInt64Field, false, "the data type of primary key should be Int64 or VarChar")
}
}
}
// test create collection with duplicate field name
func TestCreateCollectionDuplicateField(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// duplicate field
pkField := entity.NewField().WithName("id").WithDataType(entity.FieldTypeVarChar).WithIsPrimaryKey(true)
pkField2 := entity.NewField().WithName("id").WithDataType(entity.FieldTypeVarChar)
vecField := entity.NewField().WithName(common.DefaultFloatVecFieldName).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim)
// two vector fields have same name
collName := common.GenRandomString(prefix, 6)
schema := entity.NewSchema().WithName(collName).WithField(pkField).WithField(vecField).WithField(vecField)
errDupField := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, errDupField, false, "duplicated field name")
// two named "id" fields, one is pk field and other is scalar field
schema2 := entity.NewSchema().WithName(collName).WithField(pkField).WithField(pkField2).WithField(vecField).WithAutoID(true)
errDupField2 := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema2))
common.CheckErr(t, errDupField2, false, "duplicated field name")
}
// test create collection with partition key not supported field type
func TestCreateCollectionInvalidPartitionKeyType(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
int64Field := entity.NewField().WithName(common.DefaultInt64FieldName).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)
vecField := entity.NewField().WithName(common.DefaultFloatVecFieldName).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim)
collName := common.GenRandomString(prefix, 6)
t.Parallel()
for _, fieldType := range hp.GetInvalidPartitionKeyFieldType() {
log.Debug("TestCreateCollectionInvalidPartitionKeyType", zap.Any("partitionKeyFieldType", fieldType))
partitionKeyField := entity.NewField().WithName("parKey").WithDataType(fieldType).WithIsPartitionKey(true)
if fieldType == entity.FieldTypeArray {
partitionKeyField.WithElementType(entity.FieldTypeInt64)
}
schema := entity.NewSchema().WithName(collName).WithField(int64Field).WithField(vecField).WithField(partitionKeyField)
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, false, "the data type of partition key should be Int64 or VarChar")
}
}
// partition key field cannot be primary field, d can only be one partition key field
func TestCreateCollectionPartitionKeyPk(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
int64Field := entity.NewField().WithName(common.DefaultInt64FieldName).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true).WithIsPartitionKey(true)
vecField := entity.NewField().WithName(common.DefaultFloatVecFieldName).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim)
collName := common.GenRandomString(prefix, 6)
schema := entity.NewSchema().WithName(collName).WithField(int64Field).WithField(vecField)
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, false, "the partition key field must not be primary field")
}
// can only be one partition key field
func TestCreateCollectionPartitionKeyNum(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
int64Field := entity.NewField().WithName(common.DefaultInt64FieldName).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)
vecField := entity.NewField().WithName(common.DefaultFloatVecFieldName).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim)
collName := common.GenRandomString(prefix, 6)
pkField1 := entity.NewField().WithName("pk_1").WithDataType(entity.FieldTypeInt64).WithIsPartitionKey(true)
pkField2 := entity.NewField().WithName("pk_2").WithDataType(entity.FieldTypeVarChar).WithMaxLength(common.TestMaxLen).WithIsPartitionKey(true)
schema := entity.NewSchema().WithName(collName).WithField(int64Field).WithField(vecField).WithField(pkField1).WithField(pkField2)
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, false, "there are more than one partition key")
}
func TestPartitionKeyInvalidNumPartition(t *testing.T) {
t.Skip("Waiting for num partition")
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// prepare field and schema
int64Field := entity.NewField().WithName(common.DefaultInt64FieldName).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)
vecField := entity.NewField().WithName(common.DefaultFloatVecFieldName).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim)
pkField1 := entity.NewField().WithName("partitionKeyField").WithDataType(entity.FieldTypeInt64).WithIsPartitionKey(true)
// schema
collName := common.GenRandomString(prefix, 6)
schema := entity.NewSchema().WithName(collName).WithField(int64Field).WithField(vecField).WithField(pkField1)
invalidNumPartitionStruct := []struct {
numPartitions int64
errMsg string
}{
{common.MaxPartitionNum + 1, "exceeds max configuration (4096)"},
{-1, "the specified partitions should be greater than 0 if partition key is used"},
}
for _, npStruct := range invalidNumPartitionStruct {
// create collection with num partitions
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, false, npStruct.errMsg)
}
}
// test create collection with multi auto id
func TestCreateCollectionMultiAutoId(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
collName := common.GenRandomString(prefix, 6)
schema := entity.NewSchema().WithField(
entity.NewField().WithName(common.DefaultInt64FieldName).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true).WithIsAutoID(true)).WithField(
entity.NewField().WithName("dupInt").WithDataType(entity.FieldTypeInt64).WithIsAutoID(true)).WithField(
entity.NewField().WithName(common.DefaultFloatVecFieldName).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim),
).WithName(collName)
errMultiAuto := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, errMultiAuto, false, "only one field can speficy AutoID with true")
}
// test create collection with different autoId between pk field and schema
func TestCreateCollectionInconsistentAutoId(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
for _, autoId := range []bool{true, false} {
log.Debug("TestCreateCollectionInconsistentAutoId", zap.Bool("autoId", autoId))
collName := common.GenRandomString(prefix, 6)
// field and schema have opposite autoID
schema := entity.NewSchema().WithField(
entity.NewField().WithName(common.DefaultInt64FieldName).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true).WithIsAutoID(autoId)).WithField(
entity.NewField().WithName(common.DefaultFloatVecFieldName).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim),
).WithName(collName).WithAutoID(!autoId)
// create collection
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, true)
// describe collection
coll, err := mc.DescribeCollection(ctx, clientv2.NewDescribeCollectionOption(collName))
common.CheckErr(t, err, true)
require.EqualValues(t, autoId, coll.Schema.AutoID)
for _, field := range coll.Schema.Fields {
if field.Name == common.DefaultInt64FieldName {
require.EqualValues(t, autoId, coll.Schema.Fields[0].AutoID)
}
}
}
}
// create collection with field or schema description
func TestCreateCollectionDescription(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// gen field with description
pkDesc := "This is pk field"
schemaDesc := "This is schema"
collName := common.GenRandomString(prefix, 6)
pkField := entity.NewField().WithName(common.DefaultInt64FieldName).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true).WithDescription(pkDesc)
vecField := entity.NewField().WithName(common.DefaultFloatVecFieldName).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim)
schema := entity.NewSchema().WithName(collName).WithField(pkField).WithField(vecField).WithDescription(schemaDesc)
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, true)
coll, err := mc.DescribeCollection(ctx, clientv2.NewDescribeCollectionOption(collName))
common.CheckErr(t, err, true)
require.EqualValues(t, schemaDesc, coll.Schema.Description)
for _, field := range coll.Schema.Fields {
if field.Name == common.DefaultInt64FieldName {
require.Equal(t, pkDesc, field.Description)
} else {
require.Empty(t, field.Description)
}
}
}
// test invalid dim of binary field
func TestCreateBinaryCollectionInvalidDim(t *testing.T) {
t.Parallel()
type invalidDimStruct struct {
dim int64
errMsg string
}
invalidDims := []invalidDimStruct{
{dim: 10, errMsg: "should be multiple of 8"},
{dim: 0, errMsg: "should be in range 2 ~ 32768"},
{dim: 1, errMsg: "should be in range 2 ~ 32768"},
{dim: common.MaxDim * 9, errMsg: "binary vector dimension should be in range 2 ~ 262144"},
{dim: common.MaxDim*8 + 1, errMsg: "binary vector dimension should be multiple of 8"},
}
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
for _, invalidDim := range invalidDims {
log.Debug("TestCreateBinaryCollectionInvalidDim", zap.Int64("dim", invalidDim.dim))
collName := common.GenRandomString(prefix, 6)
// field and schema have opposite autoID
schema := entity.NewSchema().WithField(
entity.NewField().WithName(common.DefaultInt64FieldName).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)).WithField(
entity.NewField().WithName(common.DefaultBinaryVecFieldName).WithDataType(entity.FieldTypeBinaryVector).WithDim(invalidDim.dim),
).WithName(collName)
// create collection
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, false, invalidDim.errMsg)
}
}
// test invalid dim of float vector
func TestCreateFloatCollectionInvalidDim(t *testing.T) {
t.Parallel()
type invalidDimStruct struct {
dim string
errMsg string
}
invalidDims := []invalidDimStruct{
{dim: "0", errMsg: "should be in range 2 ~ 32768"},
{dim: "1", errMsg: "should be in range 2 ~ 32768"},
{dim: "", errMsg: "invalid syntax"},
{dim: "中文", errMsg: "invalid syntax"},
{dim: "%$#", errMsg: "invalid syntax"},
{dim: fmt.Sprintf("%d", common.MaxDim+1), errMsg: "float vector dimension should be in range 2 ~ 32768"},
}
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
for _, vecType := range []entity.FieldType{entity.FieldTypeFloatVector, entity.FieldTypeFloat16Vector, entity.FieldTypeBFloat16Vector} {
for _, invalidDim := range invalidDims {
log.Debug("TestCreateBinaryCollectionInvalidDim", zap.String("dim", invalidDim.dim))
collName := common.GenRandomString(prefix, 6)
schema := entity.NewSchema().WithField(
entity.NewField().WithName(common.DefaultInt64FieldName).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)).WithField(
entity.NewField().WithName("pk").WithDataType(vecType).WithTypeParams(entity.TypeParamDim, invalidDim.dim),
).WithName(collName)
// create collection
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, false, invalidDim.errMsg)
}
}
}
func TestCreateVectorWithoutDim(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
collName := common.GenRandomString(prefix, 6)
schema := entity.NewSchema().WithField(
entity.NewField().WithName(common.DefaultInt64FieldName).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)).WithField(
entity.NewField().WithName("vec").WithDataType(entity.FieldTypeFloatVector),
).WithName(collName)
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, false, "dimension is not defined in field type params, check type param `dim` for vector field")
}
// specify dim for sparse vector -> error
func TestCreateCollectionSparseVectorWithDim(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
collName := common.GenRandomString(prefix, 6)
schema := entity.NewSchema().WithField(
entity.NewField().WithName(common.DefaultInt64FieldName).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)).WithField(
entity.NewField().WithName("sparse").WithDataType(entity.FieldTypeSparseVector).WithDim(common.DefaultDim),
).WithName(collName)
// create collection
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, false, "dim should not be specified for sparse vector field sparse")
}
func TestCreateArrayFieldInvalidCapacity(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
collName := common.GenRandomString(prefix, 6)
pkField := entity.NewField().WithName(common.DefaultInt64FieldName).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)
vecField := entity.NewField().WithName(common.DefaultFloatVecFieldName).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim)
arrayField := entity.NewField().WithName(common.DefaultArrayFieldName).WithDataType(entity.FieldTypeArray).WithElementType(entity.FieldTypeFloat)
schema := entity.NewSchema().WithName(collName).WithField(pkField).WithField(vecField).WithField(arrayField)
// create collection
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, false, "type param(max_capacity) should be specified for array field")
// invalid Capacity
for _, invalidCapacity := range []int64{-1, 0, common.MaxCapacity + 1} {
arrayField.WithMaxCapacity(invalidCapacity)
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, false, "the maximum capacity specified for a Array should be in (0, 4096]")
}
}
// test create collection varchar array with invalid max length
func TestCreateVarcharArrayInvalidLength(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
collName := common.GenRandomString(prefix, 6)
pkField := entity.NewField().WithName(common.DefaultInt64FieldName).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)
vecField := entity.NewField().WithName(common.DefaultFloatVecFieldName).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim)
arrayVarcharField := entity.NewField().WithName(common.DefaultArrayFieldName).WithDataType(entity.FieldTypeArray).WithElementType(entity.FieldTypeVarChar).WithMaxCapacity(100)
schema := entity.NewSchema().WithName(collName).WithField(pkField).WithField(vecField).WithField(arrayVarcharField)
// create collection
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, false, "type param(max_length) should be specified for varChar field")
// invalid Capacity
for _, invalidLength := range []int64{-1, 0, common.MaxLength + 1} {
arrayVarcharField.WithMaxLength(invalidLength)
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, false, "the maximum length specified for a VarChar should be in (0, 65535]")
}
}
// test create collection varchar array with invalid max length
func TestCreateVarcharInvalidLength(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
collName := common.GenRandomString(prefix, 6)
varcharField := entity.NewField().WithName(common.DefaultVarcharFieldName).WithDataType(entity.FieldTypeVarChar).WithIsPrimaryKey(true)
vecField := entity.NewField().WithName(common.DefaultFloatVecFieldName).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim)
schema := entity.NewSchema().WithName(collName).WithField(varcharField).WithField(vecField)
// create collection
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, false, "type param(max_length) should be specified for varChar field")
// invalid Capacity
for _, invalidLength := range []int64{-1, 0, common.MaxLength + 1} {
varcharField.WithMaxLength(invalidLength)
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, false, "the maximum length specified for a VarChar should be in (0, 65535]")
}
}
func TestCreateArrayNotSupportedFieldType(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
collName := common.GenRandomString(prefix, 6)
// not supported ElementType: Array, Json, FloatVector, BinaryVector
pkField := entity.NewField().WithName(common.DefaultInt64FieldName).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)
vecField := entity.NewField().WithName(common.DefaultFloatVecFieldName).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim)
for _, fieldType := range []entity.FieldType{entity.FieldTypeArray, entity.FieldTypeJSON, entity.FieldTypeBinaryVector, entity.FieldTypeFloatVector} {
field := entity.NewField().WithName("array").WithDataType(entity.FieldTypeArray).WithElementType(fieldType)
schema := entity.NewSchema().WithName(collName).WithField(pkField).WithField(vecField).WithField(field)
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, false, fmt.Sprintf("element type %s is not supported", fieldType.Name()))
}
}
// the num of vector fields > default limit=4
func TestCreateMultiVectorExceed(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
collName := common.GenRandomString(prefix, 6)
pkField := entity.NewField().WithName(common.DefaultInt64FieldName).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)
schema := entity.NewSchema().WithName(collName).WithField(pkField)
for i := 0; i < common.MaxVectorFieldNum+1; i++ {
vecField := entity.NewField().WithName(fmt.Sprintf("vec_%d", i)).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim)
schema.WithField(vecField)
}
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, false, fmt.Sprintf("maximum vector field's number should be limited to %d", common.MaxVectorFieldNum))
}
//func TestCreateCollection(t *testing.T) {}
func TestCreateCollectionInvalidShards(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
vecField := entity.NewField().WithName(common.DefaultFloatVecFieldName).WithDataType(entity.FieldTypeFloatVector).WithDim(common.DefaultDim)
int64Field := entity.NewField().WithName(common.DefaultInt64FieldName).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true).WithIsAutoID(true)
for _, shard := range []int32{common.MaxShardNum + 1} {
// pk field with name
collName := common.GenRandomString(prefix, 6)
schema := entity.NewSchema().WithName(collName).WithField(int64Field).WithField(vecField)
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, schema).WithShardNum(shard))
common.CheckErr(t, err, false, fmt.Sprintf("maximum shards's number should be limited to %d", common.MaxShardNum))
}
}
func TestCreateCollectionInvalid(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
collName := common.GenRandomString(prefix, 6)
type mSchemaErr struct {
schema *entity.Schema
errMsg string
}
mSchemaErrs := []mSchemaErr{
{schema: nil, errMsg: "duplicated field name"},
{schema: entity.NewSchema(), errMsg: "collection name should not be empty"},
{schema: entity.NewSchema().WithName("aaa"), errMsg: "primary key is not specified"},
{schema: entity.NewSchema().WithName("aaa").WithField(entity.NewField()), errMsg: "primary key is not specified"},
{schema: entity.NewSchema().WithName("aaa").WithField(entity.NewField().WithIsPrimaryKey(true)), errMsg: "the data type of primary key should be Int64 or VarChar"},
{schema: entity.NewSchema().WithName("aaa").WithField(entity.NewField().WithIsPrimaryKey(true).WithDataType(entity.FieldTypeVarChar)), errMsg: "field name should not be empty"},
}
for _, mSchema := range mSchemaErrs {
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(collName, mSchema.schema))
common.CheckErr(t, err, false, mSchema.errMsg)
}
}

11
tests/go_client/testcases/helper/collection_helper.go Normal file
View File

@ -0,0 +1,11 @@
package helper
type CreateCollectionParams struct {
CollectionFieldsType CollectionFieldsType // collection fields type
}
func NewCreateCollectionParams(collectionFieldsType CollectionFieldsType) *CreateCollectionParams {
return &CreateCollectionParams{
CollectionFieldsType: collectionFieldsType,
}
}

324
tests/go_client/testcases/helper/data_helper.go Normal file
View File

@ -0,0 +1,324 @@
package helper
import (
"bytes"
"strconv"
"github.com/milvus-io/milvus/client/v2/column"
"github.com/milvus-io/milvus/client/v2/entity"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/tests/go_client/common"
"go.uber.org/zap"
)
// insert params
type InsertParams struct {
Schema *entity.Schema
PartitionName string
Start int
Nb int
IsRows bool
}
func NewInsertParams(schema *entity.Schema, nb int) *InsertParams {
return &InsertParams{
Schema: schema,
Nb: nb,
}
}
func (opt *InsertParams) TWithPartitionName(partitionName string) *InsertParams {
opt.PartitionName = partitionName
return opt
}
func (opt *InsertParams) TWithStart(start int) *InsertParams {
opt.Start = start
return opt
}
func (opt *InsertParams) TWithIsRows(isRows bool) *InsertParams {
opt.IsRows = isRows
return opt
}
// GenColumnDataOption -- create column data --
type GenColumnOption struct {
dim int64
maxLen int64
start int
fieldName string
elementType entity.FieldType
}
func (opt *GenColumnOption) TWithDim(dim int64) *GenColumnOption {
opt.dim = dim
return opt
}
func (opt *GenColumnOption) TWithMaxLen(maxLen int64) *GenColumnOption {
opt.maxLen = maxLen
return opt
}
func (opt *GenColumnOption) TWithStart(start int) *GenColumnOption {
opt.start = start
return opt
}
func (opt *GenColumnOption) TWithFieldName(fieldName string) *GenColumnOption {
opt.fieldName = fieldName
return opt
}
func (opt *GenColumnOption) TWithElementType(eleType entity.FieldType) *GenColumnOption {
opt.elementType = eleType
return opt
}
func TNewColumnOption() *GenColumnOption {
return &GenColumnOption{
dim: common.DefaultDim,
maxLen: common.TestMaxLen,
start: 0,
}
}
func GenArrayColumnData(nb int, eleType entity.FieldType, option GenColumnOption) column.Column {
start := option.start
fieldName := option.fieldName
if option.fieldName == "" {
fieldName = GetFieldNameByElementType(eleType)
}
capacity := int(option.maxLen)
switch eleType {
case entity.FieldTypeBool:
boolValues := make([][]bool, 0, nb)
for i := start; i < start+nb; i++ {
boolArray := make([]bool, 0, capacity)
for j := 0; j < capacity; j++ {
boolArray = append(boolArray, i%2 == 0)
}
boolValues = append(boolValues, boolArray)
}
return column.NewColumnBoolArray(fieldName, boolValues)
case entity.FieldTypeInt8:
int8Values := make([][]int8, 0, nb)
for i := start; i < start+nb; i++ {
int8Array := make([]int8, 0, capacity)
for j := 0; j < capacity; j++ {
int8Array = append(int8Array, int8(i+j))
}
int8Values = append(int8Values, int8Array)
}
return column.NewColumnInt8Array(fieldName, int8Values)
case entity.FieldTypeInt16:
int16Values := make([][]int16, 0, nb)
for i := start; i < start+nb; i++ {
int16Array := make([]int16, 0, capacity)
for j := 0; j < capacity; j++ {
int16Array = append(int16Array, int16(i+j))
}
int16Values = append(int16Values, int16Array)
}
return column.NewColumnInt16Array(fieldName, int16Values)
case entity.FieldTypeInt32:
int32Values := make([][]int32, 0, nb)
for i := start; i < start+nb; i++ {
int32Array := make([]int32, 0, capacity)
for j := 0; j < capacity; j++ {
int32Array = append(int32Array, int32(i+j))
}
int32Values = append(int32Values, int32Array)
}
return column.NewColumnInt32Array(fieldName, int32Values)
case entity.FieldTypeInt64:
int64Values := make([][]int64, 0, nb)
for i := start; i < start+nb; i++ {
int64Array := make([]int64, 0, capacity)
for j := 0; j < capacity; j++ {
int64Array = append(int64Array, int64(i+j))
}
int64Values = append(int64Values, int64Array)
}
return column.NewColumnInt64Array(fieldName, int64Values)
case entity.FieldTypeFloat:
floatValues := make([][]float32, 0, nb)
for i := start; i < start+nb; i++ {
floatArray := make([]float32, 0, capacity)
for j := 0; j < capacity; j++ {
floatArray = append(floatArray, float32(i+j))
}
floatValues = append(floatValues, floatArray)
}
return column.NewColumnFloatArray(fieldName, floatValues)
case entity.FieldTypeDouble:
doubleValues := make([][]float64, 0, nb)
for i := start; i < start+nb; i++ {
doubleArray := make([]float64, 0, capacity)
for j := 0; j < capacity; j++ {
doubleArray = append(doubleArray, float64(i+j))
}
doubleValues = append(doubleValues, doubleArray)
}
return column.NewColumnDoubleArray(fieldName, doubleValues)
case entity.FieldTypeVarChar:
varcharValues := make([][][]byte, 0, nb)
for i := start; i < start+nb; i++ {
varcharArray := make([][]byte, 0, capacity)
for j := 0; j < capacity; j++ {
var buf bytes.Buffer
buf.WriteString(strconv.Itoa(i + j))
varcharArray = append(varcharArray, buf.Bytes())
}
varcharValues = append(varcharValues, varcharArray)
}
return column.NewColumnVarCharArray(fieldName, varcharValues)
default:
log.Fatal("GenArrayColumnData failed", zap.Any("ElementType", eleType))
return nil
}
}
// GenColumnData GenColumnDataOption
func GenColumnData(nb int, fieldType entity.FieldType, option GenColumnOption) column.Column {
dim := int(option.dim)
maxLen := int(option.maxLen)
start := option.start
fieldName := option.fieldName
if option.fieldName == "" {
fieldName = GetFieldNameByFieldType(fieldType, option.elementType)
}
switch fieldType {
case entity.FieldTypeInt64:
int64Values := make([]int64, 0, nb)
for i := start; i < start+nb; i++ {
int64Values = append(int64Values, int64(i))
}
return column.NewColumnInt64(fieldName, int64Values)
case entity.FieldTypeInt8:
int8Values := make([]int8, 0, nb)
for i := start; i < start+nb; i++ {
int8Values = append(int8Values, int8(i))
}
return column.NewColumnInt8(fieldName, int8Values)
case entity.FieldTypeInt16:
int16Values := make([]int16, 0, nb)
for i := start; i < start+nb; i++ {
int16Values = append(int16Values, int16(i))
}
return column.NewColumnInt16(fieldName, int16Values)
case entity.FieldTypeInt32:
int32Values := make([]int32, 0, nb)
for i := start; i < start+nb; i++ {
int32Values = append(int32Values, int32(i))
}
return column.NewColumnInt32(fieldName, int32Values)
case entity.FieldTypeBool:
boolValues := make([]bool, 0, nb)
for i := start; i < start+nb; i++ {
boolValues = append(boolValues, i/2 == 0)
}
return column.NewColumnBool(fieldName, boolValues)
case entity.FieldTypeFloat:
floatValues := make([]float32, 0, nb)
for i := start; i < start+nb; i++ {
floatValues = append(floatValues, float32(i))
}
return column.NewColumnFloat(fieldName, floatValues)
case entity.FieldTypeDouble:
floatValues := make([]float64, 0, nb)
for i := start; i < start+nb; i++ {
floatValues = append(floatValues, float64(i))
}
return column.NewColumnDouble(fieldName, floatValues)
case entity.FieldTypeVarChar:
varcharValues := make([]string, 0, nb)
for i := start; i < start+nb; i++ {
varcharValues = append(varcharValues, strconv.Itoa(i))
}
return column.NewColumnVarChar(fieldName, varcharValues)
case entity.FieldTypeArray:
return GenArrayColumnData(nb, option.elementType, option)
case entity.FieldTypeFloatVector:
vecFloatValues := make([][]float32, 0, nb)
for i := start; i < start+nb; i++ {
vec := common.GenFloatVector(dim)
vecFloatValues = append(vecFloatValues, vec)
}
return column.NewColumnFloatVector(fieldName, int(option.dim), vecFloatValues)
case entity.FieldTypeBinaryVector:
binaryVectors := make([][]byte, 0, nb)
for i := 0; i < nb; i++ {
vec := common.GenBinaryVector(dim)
binaryVectors = append(binaryVectors, vec)
}
return column.NewColumnBinaryVector(fieldName, dim, binaryVectors)
case entity.FieldTypeFloat16Vector:
fp16Vectors := make([][]byte, 0, nb)
for i := start; i < start+nb; i++ {
vec := common.GenFloat16Vector(dim)
fp16Vectors = append(fp16Vectors, vec)
}
return column.NewColumnFloat16Vector(fieldName, dim, fp16Vectors)
case entity.FieldTypeBFloat16Vector:
bf16Vectors := make([][]byte, 0, nb)
for i := start; i < start+nb; i++ {
vec := common.GenBFloat16Vector(dim)
bf16Vectors = append(bf16Vectors, vec)
}
return column.NewColumnBFloat16Vector(fieldName, dim, bf16Vectors)
case entity.FieldTypeSparseVector:
vectors := make([]entity.SparseEmbedding, 0, nb)
for i := start; i < start+nb; i++ {
vec := common.GenSparseVector(maxLen)
vectors = append(vectors, vec)
}
return column.NewColumnSparseVectors(fieldName, vectors)
default:
log.Fatal("GenColumnData failed", zap.Any("FieldType", fieldType))
return nil
}
}
func GenDynamicFieldData(start int, nb int) []column.Column {
type ListStruct struct {
List []int64 `json:"list" milvus:"name:list"`
}
// gen number, string bool list data column
numberValues := make([]int32, 0, nb)
stringValues := make([]string, 0, nb)
boolValues := make([]bool, 0, nb)
//listValues := make([][]byte, 0, Nb)
//m := make(map[string]interface{})
for i := start; i < start+nb; i++ {
numberValues = append(numberValues, int32(i))
stringValues = append(stringValues, strconv.Itoa(i))
boolValues = append(boolValues, i%3 == 0)
//m["list"] = ListStruct{
// List: []int64{int64(i), int64(i + 1)},
//}
//bs, err := json.Marshal(m)
//if err != nil {
// log.Fatalf("Marshal json field failed: %s", err)
//}
//listValues = append(listValues, bs)
}
data := []column.Column{
column.NewColumnInt32(common.DefaultDynamicNumberField, numberValues),
column.NewColumnString(common.DefaultDynamicStringField, stringValues),
column.NewColumnBool(common.DefaultDynamicBoolField, boolValues),
//entity.NewColumnJSONBytes(DefaultDynamicListField, listValues),
}
return data
}

299
tests/go_client/testcases/helper/field_helper.go Normal file
View File

@ -0,0 +1,299 @@
package helper
import (
"github.com/milvus-io/milvus/client/v2/entity"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/tests/go_client/common"
"go.uber.org/zap"
)
func GetFieldNameByElementType(t entity.FieldType) string {
switch t {
case entity.FieldTypeBool:
return common.DefaultBoolArrayField
case entity.FieldTypeInt8:
return common.DefaultInt8ArrayField
case entity.FieldTypeInt16:
return common.DefaultInt16ArrayField
case entity.FieldTypeInt32:
return common.DefaultInt32ArrayField
case entity.FieldTypeInt64:
return common.DefaultInt64ArrayField
case entity.FieldTypeFloat:
return common.DefaultFloatArrayField
case entity.FieldTypeDouble:
return common.DefaultDoubleArrayField
case entity.FieldTypeVarChar:
return common.DefaultVarcharArrayField
default:
return common.DefaultArrayFieldName
}
}
func GetFieldNameByFieldType(t entity.FieldType, eleType ...entity.FieldType) string {
switch t {
case entity.FieldTypeBool:
return common.DefaultBoolFieldName
case entity.FieldTypeInt8:
return common.DefaultInt8FieldName
case entity.FieldTypeInt16:
return common.DefaultInt16FieldName
case entity.FieldTypeInt32:
return common.DefaultInt32FieldName
case entity.FieldTypeInt64:
return common.DefaultInt64FieldName
case entity.FieldTypeFloat:
return common.DefaultFloatFieldName
case entity.FieldTypeDouble:
return common.DefaultDoubleFieldName
case entity.FieldTypeVarChar:
return common.DefaultVarcharFieldName
case entity.FieldTypeJSON:
return common.DefaultJSONFieldName
case entity.FieldTypeArray:
return GetFieldNameByElementType(eleType[0])
case entity.FieldTypeBinaryVector:
return common.DefaultBinaryVecFieldName
case entity.FieldTypeFloatVector:
return common.DefaultFloatVecFieldName
case entity.FieldTypeFloat16Vector:
return common.DefaultFloat16VecFieldName
case entity.FieldTypeBFloat16Vector:
return common.DefaultBFloat16VecFieldName
case entity.FieldTypeSparseVector:
return common.DefaultSparseVecFieldName
default:
return ""
}
}
type CollectionFieldsType int32
const (
// FieldTypeNone zero value place holder
Int64Vec CollectionFieldsType = 1 // int64 + floatVec
VarcharBinary CollectionFieldsType = 2 // varchar + binaryVec
Int64VecJSON CollectionFieldsType = 3 // int64 + floatVec + json
Int64VecArray CollectionFieldsType = 4 // int64 + floatVec + array
Int64VarcharSparseVec CollectionFieldsType = 5 // int64 + varchar + sparse vector
Int64MultiVec CollectionFieldsType = 6 // int64 + floatVec + binaryVec + fp16Vec + bf16vec
AllFields CollectionFieldsType = 7 // all fields excepted sparse
)
type GenFieldsOption struct {
AutoID bool // is auto id
Dim int64
IsDynamic bool
MaxLength int64 // varchar len or array capacity
MaxCapacity int64
IsPartitionKey bool
ElementType entity.FieldType
}
func TNewFieldsOption() *GenFieldsOption {
return &GenFieldsOption{
AutoID: false,
Dim: common.DefaultDim,
MaxLength: common.TestMaxLen,
MaxCapacity: common.TestCapacity,
IsDynamic: false,
IsPartitionKey: false,
ElementType: entity.FieldTypeNone,
}
}
func (opt *GenFieldsOption) TWithAutoID(autoID bool) *GenFieldsOption {
opt.AutoID = autoID
return opt
}
func (opt *GenFieldsOption) TWithDim(dim int64) *GenFieldsOption {
opt.Dim = dim
return opt
}
func (opt *GenFieldsOption) TWithIsDynamic(isDynamic bool) *GenFieldsOption {
opt.IsDynamic = isDynamic
return opt
}
func (opt *GenFieldsOption) TWithIsPartitionKey(isPartitionKey bool) *GenFieldsOption {
opt.IsPartitionKey = isPartitionKey
return opt
}
func (opt *GenFieldsOption) TWithElementType(elementType entity.FieldType) *GenFieldsOption {
opt.ElementType = elementType
return opt
}
func (opt *GenFieldsOption) TWithMaxLen(maxLen int64) *GenFieldsOption {
opt.MaxLength = maxLen
return opt
}
func (opt *GenFieldsOption) TWithMaxCapacity(maxCapacity int64) *GenFieldsOption {
opt.MaxCapacity = maxCapacity
return opt
}
// factory
type FieldsFactory struct{}
// product
type CollectionFields interface {
GenFields(opts GenFieldsOption) []*entity.Field
}
type FieldsInt64Vec struct{}
func (cf FieldsInt64Vec) GenFields(option GenFieldsOption) []*entity.Field {
pkField := entity.NewField().WithName(GetFieldNameByFieldType(entity.FieldTypeInt64)).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)
vecField := entity.NewField().WithName(GetFieldNameByFieldType(entity.FieldTypeFloatVector)).WithDataType(entity.FieldTypeFloatVector).WithDim(option.Dim)
if option.AutoID {
pkField.WithIsAutoID(option.AutoID)
}
return []*entity.Field{pkField, vecField}
}
type FieldsVarcharBinary struct{}
func (cf FieldsVarcharBinary) GenFields(option GenFieldsOption) []*entity.Field {
pkField := entity.NewField().WithName(GetFieldNameByFieldType(entity.FieldTypeVarChar)).WithDataType(entity.FieldTypeVarChar).
WithIsPrimaryKey(true).WithMaxLength(option.MaxLength)
vecField := entity.NewField().WithName(GetFieldNameByFieldType(entity.FieldTypeBinaryVector)).WithDataType(entity.FieldTypeBinaryVector).WithDim(option.Dim)
if option.AutoID {
pkField.WithIsAutoID(option.AutoID)
}
return []*entity.Field{pkField, vecField}
}
type FieldsInt64VecJSON struct{}
func (cf FieldsInt64VecJSON) GenFields(option GenFieldsOption) []*entity.Field {
pkField := entity.NewField().WithName(GetFieldNameByFieldType(entity.FieldTypeInt64)).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)
vecField := entity.NewField().WithName(GetFieldNameByFieldType(entity.FieldTypeFloatVector)).WithDataType(entity.FieldTypeFloatVector).WithDim(option.Dim)
jsonField := entity.NewField().WithName(GetFieldNameByFieldType(entity.FieldTypeJSON)).WithDataType(entity.FieldTypeJSON)
if option.AutoID {
pkField.WithIsAutoID(option.AutoID)
}
return []*entity.Field{pkField, vecField, jsonField}
}
type FieldsInt64VecArray struct{}
func (cf FieldsInt64VecArray) GenFields(option GenFieldsOption) []*entity.Field {
pkField := entity.NewField().WithName(GetFieldNameByFieldType(entity.FieldTypeInt64)).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)
vecField := entity.NewField().WithName(GetFieldNameByFieldType(entity.FieldTypeFloatVector)).WithDataType(entity.FieldTypeFloatVector).WithDim(option.Dim)
fields := []*entity.Field{
pkField, vecField,
}
for _, eleType := range GetAllArrayElementType() {
arrayField := entity.NewField().WithName(GetFieldNameByElementType(eleType)).WithDataType(entity.FieldTypeArray).WithElementType(eleType).WithMaxCapacity(option.MaxCapacity)
if eleType == entity.FieldTypeVarChar {
arrayField.WithMaxLength(option.MaxLength)
}
fields = append(fields, arrayField)
}
if option.AutoID {
pkField.WithIsAutoID(option.AutoID)
}
return fields
}
type FieldsInt64VarcharSparseVec struct{}
func (cf FieldsInt64VarcharSparseVec) GenFields(option GenFieldsOption) []*entity.Field {
pkField := entity.NewField().WithName(GetFieldNameByFieldType(entity.FieldTypeInt64)).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)
varcharField := entity.NewField().WithName(GetFieldNameByFieldType(entity.FieldTypeVarChar)).WithDataType(entity.FieldTypeVarChar).WithMaxLength(option.MaxLength)
sparseVecField := entity.NewField().WithName(GetFieldNameByFieldType(entity.FieldTypeSparseVector)).WithDataType(entity.FieldTypeSparseVector)
if option.AutoID {
pkField.WithIsAutoID(option.AutoID)
}
return []*entity.Field{pkField, varcharField, sparseVecField}
}
type FieldsInt64MultiVec struct{}
func (cf FieldsInt64MultiVec) GenFields(option GenFieldsOption) []*entity.Field {
pkField := entity.NewField().WithName(GetFieldNameByFieldType(entity.FieldTypeInt64)).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)
fields := []*entity.Field{
pkField,
}
for _, fieldType := range GetAllVectorFieldType() {
if fieldType == entity.FieldTypeSparseVector {
continue
}
vecField := entity.NewField().WithName(GetFieldNameByFieldType(fieldType)).WithDataType(fieldType).WithDim(option.Dim)
fields = append(fields, vecField)
}
if option.AutoID {
pkField.WithIsAutoID(option.AutoID)
}
return fields
}
type FieldsAllFields struct{} // except sparse vector field
func (cf FieldsAllFields) GenFields(option GenFieldsOption) []*entity.Field {
pkField := entity.NewField().WithName(GetFieldNameByFieldType(entity.FieldTypeInt64)).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)
fields := []*entity.Field{
pkField,
}
// scalar fields and array fields
for _, fieldType := range GetAllScaleFieldType() {
if fieldType == entity.FieldTypeInt64 {
continue
} else if fieldType == entity.FieldTypeArray {
for _, eleType := range GetAllArrayElementType() {
arrayField := entity.NewField().WithName(GetFieldNameByElementType(eleType)).WithDataType(entity.FieldTypeArray).WithElementType(eleType).WithMaxCapacity(option.MaxCapacity)
if eleType == entity.FieldTypeVarChar {
arrayField.WithMaxLength(option.MaxLength)
}
fields = append(fields, arrayField)
}
} else if fieldType == entity.FieldTypeVarChar {
varcharField := entity.NewField().WithName(GetFieldNameByFieldType(fieldType)).WithDataType(fieldType).WithMaxLength(option.MaxLength)
fields = append(fields, varcharField)
} else {
scalarField := entity.NewField().WithName(GetFieldNameByFieldType(fieldType)).WithDataType(fieldType)
fields = append(fields, scalarField)
}
}
for _, fieldType := range GetAllVectorFieldType() {
if fieldType == entity.FieldTypeSparseVector {
continue
}
vecField := entity.NewField().WithName(GetFieldNameByFieldType(fieldType)).WithDataType(fieldType).WithDim(option.Dim)
fields = append(fields, vecField)
}
if option.AutoID {
pkField.WithIsAutoID(option.AutoID)
}
return fields
}
func (ff FieldsFactory) GenFieldsForCollection(collectionFieldsType CollectionFieldsType, option *GenFieldsOption) []*entity.Field {
log.Info("GenFieldsForCollection", zap.Any("GenFieldsOption", option))
switch collectionFieldsType {
case Int64Vec:
return FieldsInt64Vec{}.GenFields(*option)
case VarcharBinary:
return FieldsVarcharBinary{}.GenFields(*option)
case Int64VecJSON:
return FieldsInt64VecJSON{}.GenFields(*option)
case Int64VecArray:
return FieldsInt64VecArray{}.GenFields(*option)
case Int64VarcharSparseVec:
return FieldsInt64VarcharSparseVec{}.GenFields(*option)
case Int64MultiVec:
return FieldsInt64MultiVec{}.GenFields(*option)
case AllFields:
return FieldsAllFields{}.GenFields(*option)
default:
return FieldsInt64Vec{}.GenFields(*option)
}
}

192
tests/go_client/testcases/helper/helper.go Normal file
View File

@ -0,0 +1,192 @@
package helper
import (
"context"
"testing"
"time"
clientv2 "github.com/milvus-io/milvus/client/v2"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/tests/go_client/base"
"github.com/milvus-io/milvus/tests/go_client/common"
"go.uber.org/zap"
"github.com/milvus-io/milvus/client/v2/entity"
)
func CreateContext(t *testing.T, timeout time.Duration) context.Context {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
t.Cleanup(func() {
cancel()
})
return ctx
}
//var ArrayFieldType =
func GetAllArrayElementType() []entity.FieldType {
return []entity.FieldType{
entity.FieldTypeBool,
entity.FieldTypeInt8,
entity.FieldTypeInt16,
entity.FieldTypeInt32,
entity.FieldTypeInt64,
entity.FieldTypeFloat,
entity.FieldTypeDouble,
entity.FieldTypeVarChar,
}
}
func GetAllVectorFieldType() []entity.FieldType {
return []entity.FieldType{
entity.FieldTypeBinaryVector,
entity.FieldTypeFloatVector,
entity.FieldTypeFloat16Vector,
entity.FieldTypeBFloat16Vector,
entity.FieldTypeSparseVector,
}
}
func GetAllScaleFieldType() []entity.FieldType {
return []entity.FieldType{
entity.FieldTypeBool,
entity.FieldTypeInt8,
entity.FieldTypeInt16,
entity.FieldTypeInt32,
entity.FieldTypeInt64,
entity.FieldTypeFloat,
entity.FieldTypeDouble,
entity.FieldTypeVarChar,
entity.FieldTypeArray,
entity.FieldTypeJSON,
}
}
func GetAllFieldsType() []entity.FieldType {
allFieldType := GetAllScaleFieldType()
allFieldType = append(allFieldType, entity.FieldTypeBinaryVector,
entity.FieldTypeFloatVector,
entity.FieldTypeFloat16Vector,
entity.FieldTypeBFloat16Vector,
//entity.FieldTypeSparseVector, max vector fields num is 4
)
return allFieldType
}
func GetInvalidPkFieldType() []entity.FieldType {
nonPkFieldTypes := []entity.FieldType{
entity.FieldTypeNone,
entity.FieldTypeBool,
entity.FieldTypeInt8,
entity.FieldTypeInt16,
entity.FieldTypeInt32,
entity.FieldTypeFloat,
entity.FieldTypeDouble,
entity.FieldTypeString,
entity.FieldTypeJSON,
entity.FieldTypeArray,
}
return nonPkFieldTypes
}
func GetInvalidPartitionKeyFieldType() []entity.FieldType {
nonPkFieldTypes := []entity.FieldType{
entity.FieldTypeBool,
entity.FieldTypeInt8,
entity.FieldTypeInt16,
entity.FieldTypeInt32,
entity.FieldTypeFloat,
entity.FieldTypeDouble,
entity.FieldTypeJSON,
entity.FieldTypeArray,
entity.FieldTypeFloatVector,
}
return nonPkFieldTypes
}
// ----------------- prepare data --------------------------
type CollectionPrepare struct{}
var CollPrepare CollectionPrepare
var FieldsFact FieldsFactory
func (chainTask *CollectionPrepare) CreateCollection(ctx context.Context, t *testing.T, mc *base.MilvusClient,
cp *CreateCollectionParams, fieldOpt *GenFieldsOption, schemaOpt *GenSchemaOption) (*CollectionPrepare, *entity.Schema) {
fields := FieldsFact.GenFieldsForCollection(cp.CollectionFieldsType, fieldOpt)
schemaOpt.Fields = fields
schema := GenSchema(schemaOpt)
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(schema.CollectionName, schema))
common.CheckErr(t, err, true)
t.Cleanup(func() {
err := mc.DropCollection(ctx, clientv2.NewDropCollectionOption(schema.CollectionName))
common.CheckErr(t, err, true)
})
return chainTask, schema
}
func (chainTask *CollectionPrepare) InsertData(ctx context.Context, t *testing.T, mc *base.MilvusClient,
ip *InsertParams, option *GenColumnOption) (*CollectionPrepare, clientv2.InsertResult) {
if nil == ip.Schema || ip.Schema.CollectionName == "" {
log.Fatal("[InsertData] Nil Schema is not expected")
}
fields := ip.Schema.Fields
insertOpt := clientv2.NewColumnBasedInsertOption(ip.Schema.CollectionName)
for _, field := range fields {
column := GenColumnData(ip.Nb, field.DataType, *option)
insertOpt.WithColumns(column)
}
insertRes, err := mc.Insert(ctx, insertOpt)
common.CheckErr(t, err, true)
return chainTask, insertRes
}
func (chainTask *CollectionPrepare) FlushData(ctx context.Context, t *testing.T, mc *base.MilvusClient, collName string) *CollectionPrepare {
flushTask, err := mc.Flush(ctx, clientv2.NewFlushOption(collName))
common.CheckErr(t, err, true)
err = flushTask.Await(ctx)
common.CheckErr(t, err, true)
return chainTask
}
func (chainTask *CollectionPrepare) CreateIndex(ctx context.Context, t *testing.T, mc *base.MilvusClient, ip *IndexParams) *CollectionPrepare {
if nil == ip.Schema || ip.Schema.CollectionName == "" {
log.Fatal("[CreateIndex] Empty collection name is not expected")
}
collName := ip.Schema.CollectionName
mFieldIndex := ip.FieldIndexMap
for _, field := range ip.Schema.Fields {
if field.DataType >= 100 {
if idx, ok := mFieldIndex[field.Name]; ok {
log.Info("CreateIndex", zap.String("indexName", idx.Name()), zap.Any("indexType", idx.IndexType()), zap.Any("indexParams", idx.Params()))
createIndexTask, err := mc.CreateIndex(ctx, clientv2.NewCreateIndexOption(collName, field.Name, idx))
common.CheckErr(t, err, true)
err = createIndexTask.Await(ctx)
common.CheckErr(t, err, true)
} else {
idx := GetDefaultVectorIndex(field.DataType)
log.Info("CreateIndex", zap.String("indexName", idx.Name()), zap.Any("indexType", idx.IndexType()), zap.Any("indexParams", idx.Params()))
createIndexTask, err := mc.CreateIndex(ctx, clientv2.NewCreateIndexOption(collName, field.Name, idx))
common.CheckErr(t, err, true)
err = createIndexTask.Await(ctx)
common.CheckErr(t, err, true)
}
}
}
return chainTask
}
func (chainTask *CollectionPrepare) Load(ctx context.Context, t *testing.T, mc *base.MilvusClient, lp *LoadParams) *CollectionPrepare {
if lp.CollectionName == "" {
log.Fatal("[Load] Empty collection name is not expected")
}
loadTask, err := mc.LoadCollection(ctx, clientv2.NewLoadCollectionOption(lp.CollectionName).WithReplica(lp.Replica))
common.CheckErr(t, err, true)
err = loadTask.Await(ctx)
common.CheckErr(t, err, true)
return chainTask
}

35
tests/go_client/testcases/helper/index_helper.go Normal file
View File

@ -0,0 +1,35 @@
package helper
import (
"github.com/milvus-io/milvus/client/v2/entity"
"github.com/milvus-io/milvus/client/v2/index"
)
func GetDefaultVectorIndex(fieldType entity.FieldType) index.Index {
switch fieldType {
case entity.FieldTypeFloatVector, entity.FieldTypeFloat16Vector, entity.FieldTypeBFloat16Vector, entity.FieldTypeSparseVector:
return index.NewHNSWIndex(entity.COSINE, 8, 200)
case entity.FieldTypeBinaryVector:
return nil
// return binary index
default:
return nil
// return auto index
}
}
type IndexParams struct {
Schema *entity.Schema
FieldIndexMap map[string]index.Index
}
func NewIndexParams(schema *entity.Schema) *IndexParams {
return &IndexParams{
Schema: schema,
}
}
func (opt *IndexParams) TWithFieldIndex(mFieldIndex map[string]index.Index) *IndexParams {
opt.FieldIndexMap = mFieldIndex
return opt
}

55
tests/go_client/testcases/helper/read_helper.go Normal file
View File

@ -0,0 +1,55 @@
package helper
import (
"github.com/milvus-io/milvus/client/v2/entity"
"github.com/milvus-io/milvus/tests/go_client/common"
)
type LoadParams struct {
CollectionName string
Replica int
}
func NewLoadParams(name string) *LoadParams {
return &LoadParams{
CollectionName: name,
}
}
func (opt *LoadParams) TWithReplica(replica int) *LoadParams {
opt.Replica = replica
return opt
}
// GenSearchVectors gen search vectors
func GenSearchVectors(nq int, dim int, dataType entity.FieldType) []entity.Vector {
vectors := make([]entity.Vector, 0, nq)
switch dataType {
case entity.FieldTypeFloatVector:
for i := 0; i < nq; i++ {
vector := common.GenFloatVector(dim)
vectors = append(vectors, entity.FloatVector(vector))
}
case entity.FieldTypeBinaryVector:
for i := 0; i < nq; i++ {
vector := common.GenBinaryVector(dim)
vectors = append(vectors, entity.BinaryVector(vector))
}
case entity.FieldTypeFloat16Vector:
for i := 0; i < nq; i++ {
vector := common.GenFloat16Vector(dim)
vectors = append(vectors, entity.Float16Vector(vector))
}
case entity.FieldTypeBFloat16Vector:
for i := 0; i < nq; i++ {
vector := common.GenBFloat16Vector(dim)
vectors = append(vectors, entity.BFloat16Vector(vector))
}
case entity.FieldTypeSparseVector:
for i := 0; i < nq; i++ {
vec := common.GenSparseVector(dim)
vectors = append(vectors, vec)
}
}
return vectors
}

68
tests/go_client/testcases/helper/schema_helper.go Normal file
View File

@ -0,0 +1,68 @@
package helper
import (
"github.com/milvus-io/milvus/client/v2/entity"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/tests/go_client/common"
)
type GenSchemaOption struct {
CollectionName string
Description string
AutoID bool
Fields []*entity.Field
EnableDynamicField bool
}
func TNewSchemaOption() *GenSchemaOption {
return &GenSchemaOption{}
}
func (opt *GenSchemaOption) TWithName(collectionName string) *GenSchemaOption {
opt.CollectionName = collectionName
return opt
}
func (opt *GenSchemaOption) TWithDescription(description string) *GenSchemaOption {
opt.Description = description
return opt
}
func (opt *GenSchemaOption) TWithAutoID(autoID bool) *GenSchemaOption {
opt.AutoID = autoID
return opt
}
func (opt *GenSchemaOption) TWithEnableDynamicField(enableDynamicField bool) *GenSchemaOption {
opt.EnableDynamicField = enableDynamicField
return opt
}
func (opt *GenSchemaOption) TWithFields(fields []*entity.Field) *GenSchemaOption {
opt.Fields = fields
return opt
}
func GenSchema(option *GenSchemaOption) *entity.Schema {
if len(option.Fields) == 0 {
log.Fatal("Require at least a primary field and a vector field")
}
if option.CollectionName == "" {
option.CollectionName = common.GenRandomString("pre", 6)
}
schema := entity.NewSchema().WithName(option.CollectionName)
for _, field := range option.Fields {
schema.WithField(field)
}
if option.Description != "" {
schema.WithDescription(option.Description)
}
if option.AutoID {
schema.WithAutoID(option.AutoID)
}
if option.EnableDynamicField {
schema.WithDynamicFieldEnabled(option.EnableDynamicField)
}
return schema
}

74
tests/go_client/testcases/main_test.go Normal file
View File

@ -0,0 +1,74 @@
package testcases
import (
"context"
"flag"
"os"
"testing"
"time"
"go.uber.org/zap"
clientv2 "github.com/milvus-io/milvus/client/v2"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/tests/go_client/base"
"github.com/milvus-io/milvus/tests/go_client/common"
)
var addr = flag.String("addr", "localhost:19530", "server host and port")
var defaultCfg = clientv2.ClientConfig{Address: *addr}
// teardown
func teardown() {
log.Info("Start to tear down all.....")
ctx, cancel := context.WithTimeout(context.Background(), time.Second*common.DefaultTimeout)
defer cancel()
mc, err := base.NewMilvusClient(ctx, &defaultCfg)
if err != nil {
log.Fatal("teardown failed to connect milvus with error", zap.Error(err))
}
defer mc.Close(ctx)
// clear dbs
dbs, _ := mc.ListDatabases(ctx, clientv2.NewListDatabaseOption())
for _, db := range dbs {
if db != common.DefaultDb {
_ = mc.UsingDatabase(ctx, clientv2.NewUsingDatabaseOption(db))
collections, _ := mc.ListCollections(ctx, clientv2.NewListCollectionOption())
for _, coll := range collections {
_ = mc.DropCollection(ctx, clientv2.NewDropCollectionOption(coll))
}
_ = mc.DropDatabase(ctx, clientv2.NewDropDatabaseOption(db))
}
}
}
// create connect
func createDefaultMilvusClient(ctx context.Context, t *testing.T) *base.MilvusClient {
t.Helper()
var (
mc *base.MilvusClient
err error
)
mc, err = base.NewMilvusClient(ctx, &defaultCfg)
common.CheckErr(t, err, true)
t.Cleanup(func() {
mc.Close(ctx)
})
return mc
}
func TestMain(m *testing.M) {
flag.Parse()
log.Info("Parser Milvus address", zap.String("address", *addr))
code := m.Run()
if code != 0 {
log.Error("Tests failed and exited", zap.Int("code", code))
}
teardown()
os.Exit(code)
}

42
tests/go_client/testcases/search_test.go Normal file
View File

@ -0,0 +1,42 @@
package testcases
import (
"testing"
"time"
clientv2 "github.com/milvus-io/milvus/client/v2"
"github.com/milvus-io/milvus/client/v2/entity"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/tests/go_client/common"
hp "github.com/milvus-io/milvus/tests/go_client/testcases/helper"
"go.uber.org/zap"
)
func TestSearch(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
cp := hp.NewCreateCollectionParams(hp.Int64Vec)
_, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, cp, hp.TNewFieldsOption(), hp.TNewSchemaOption())
log.Info("schema", zap.Any("schema", schema))
insertParams := hp.NewInsertParams(schema, common.DefaultNb)
hp.CollPrepare.InsertData(ctx, t, mc, insertParams, hp.TNewColumnOption())
// flush -> index -> load
hp.CollPrepare.FlushData(ctx, t, mc, schema.CollectionName)
hp.CollPrepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
hp.CollPrepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
// search
vectors := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector)
resSearch, err := mc.Search(ctx, clientv2.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors).WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
common.CheckSearchResult(t, resSearch, common.DefaultNq, common.DefaultLimit)
log.Info("search", zap.Any("resSearch", resSearch))
log.Info("search", zap.Any("ids", resSearch[0].IDs))
log.Info("search", zap.Any("scores", resSearch[0].Scores))
id, _ := resSearch[0].IDs.GetAsInt64(0)
log.Info("search", zap.Int64("ids", id))
}