test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-11-29 18:38:44 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

enhance: support null and default value in restful api (#35825)

Browse Source 
#31728

Signed-off-by: lixinguo <xinguo.li@zilliz.com>
Co-authored-by: lixinguo <xinguo.li@zilliz.com>
This commit is contained in:
smellthemoon 2024-10-11 21:23:20 +08:00 committed by GitHub
parent 116d7c18cc
commit 5e1c392243
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 617 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
internal/distributed/proxy/httpserver/constant.go
View File

@ -77,6 +77,7 @@ const (
HTTPIndexField = "fieldName"
HTTPAliasName = "aliasName"
HTTPRequestData = "data"
HTTPRequestDefaultValue = "defaultValue"
DefaultDbName = "default"
DefaultIndexName = "vector_idx"
DefaultAliasName = "the_alias"

8
internal/distributed/proxy/httpserver/handler_v1.go
View File

@ -718,7 +718,7 @@ func (h *HandlersV1) insert(c *gin.Context) {
return nil, RestRequestInterceptorErr
}
body, _ := c.Get(gin.BodyBytesKey)
err, httpReq.Data = checkAndSetData(string(body.([]byte)), collSchema)
err, httpReq.Data, _ = checkAndSetData(string(body.([]byte)), collSchema)
if err != nil {
log.Warn("high level restful api, fail to deal with insert data", zap.Any("body", body), zap.Error(err))
HTTPAbortReturn(c, http.StatusOK, gin.H{
@ -728,7 +728,7 @@ func (h *HandlersV1) insert(c *gin.Context) {
return nil, RestRequestInterceptorErr
}
insertReq := req.(*milvuspb.InsertRequest)
insertReq.FieldsData, err = anyToColumns(httpReq.Data, collSchema)
insertReq.FieldsData, err = anyToColumns(httpReq.Data, nil, collSchema)
if err != nil {
log.Warn("high level restful api, fail to deal with insert data", zap.Any("data", httpReq.Data), zap.Error(err))
HTTPAbortReturn(c, http.StatusOK, gin.H{
@ -817,7 +817,7 @@ func (h *HandlersV1) upsert(c *gin.Context) {
}
}
body, _ := c.Get(gin.BodyBytesKey)
err, httpReq.Data = checkAndSetData(string(body.([]byte)), collSchema)
err, httpReq.Data, _ = checkAndSetData(string(body.([]byte)), collSchema)
if err != nil {
log.Warn("high level restful api, fail to deal with upsert data", zap.Any("body", body), zap.Error(err))
HTTPAbortReturn(c, http.StatusOK, gin.H{
@ -827,7 +827,7 @@ func (h *HandlersV1) upsert(c *gin.Context) {
return nil, RestRequestInterceptorErr
}
upsertReq := req.(*milvuspb.UpsertRequest)
upsertReq.FieldsData, err = anyToColumns(httpReq.Data, collSchema)
upsertReq.FieldsData, err = anyToColumns(httpReq.Data, nil, collSchema)
if err != nil {
log.Warn("high level restful api, fail to deal with upsert data", zap.Any("data", httpReq.Data), zap.Error(err))
HTTPAbortReturn(c, http.StatusOK, gin.H{

21
internal/distributed/proxy/httpserver/handler_v2.go
View File

@ -730,7 +730,8 @@ func (h *HandlersV2) insert(ctx context.Context, c *gin.Context, anyReq any, dbN
return nil, err
}
body, _ := c.Get(gin.BodyBytesKey)
err, httpReq.Data = checkAndSetData(string(body.([]byte)), collSchema)
var validDataMap map[string][]bool
err, httpReq.Data, validDataMap = checkAndSetData(string(body.([]byte)), collSchema)
if err != nil {
log.Ctx(ctx).Warn("high level restful api, fail to deal with insert data", zap.Error(err), zap.String("body", string(body.([]byte))))
HTTPAbortReturn(c, http.StatusOK, gin.H{
@ -741,7 +742,7 @@ func (h *HandlersV2) insert(ctx context.Context, c *gin.Context, anyReq any, dbN
}
req.NumRows = uint32(len(httpReq.Data))
req.FieldsData, err = anyToColumns(httpReq.Data, collSchema)
req.FieldsData, err = anyToColumns(httpReq.Data, validDataMap, collSchema)
if err != nil {
log.Ctx(ctx).Warn("high level restful api, fail to deal with insert data", zap.Any("data", httpReq.Data), zap.Error(err))
HTTPAbortReturn(c, http.StatusOK, gin.H{
@ -808,7 +809,8 @@ func (h *HandlersV2) upsert(ctx context.Context, c *gin.Context, anyReq any, dbN
return nil, err
}
body, _ := c.Get(gin.BodyBytesKey)
err, httpReq.Data = checkAndSetData(string(body.([]byte)), collSchema)
var validDataMap map[string][]bool
err, httpReq.Data, validDataMap = checkAndSetData(string(body.([]byte)), collSchema)
if err != nil {
log.Ctx(ctx).Warn("high level restful api, fail to deal with upsert data", zap.Any("body", body), zap.Error(err))
HTTPAbortReturn(c, http.StatusOK, gin.H{
@ -819,7 +821,7 @@ func (h *HandlersV2) upsert(ctx context.Context, c *gin.Context, anyReq any, dbN
}
req.NumRows = uint32(len(httpReq.Data))
req.FieldsData, err = anyToColumns(httpReq.Data, collSchema)
req.FieldsData, err = anyToColumns(httpReq.Data, validDataMap, collSchema)
if err != nil {
log.Ctx(ctx).Warn("high level restful api, fail to deal with upsert data", zap.Any("data", httpReq.Data), zap.Error(err))
HTTPAbortReturn(c, http.StatusOK, gin.H{
@ -1179,6 +1181,17 @@ func (h *HandlersV2) createCollection(ctx context.Context, c *gin.Context, anyRe
IsPartitionKey: field.IsPartitionKey,
DataType: dataType,
TypeParams: []*commonpb.KeyValuePair{},
Nullable: field.Nullable,
}
fieldSchema.DefaultValue, err = convertDefaultValue(field.DefaultValue, dataType)
if err != nil {
log.Ctx(ctx).Warn("convert defaultValue fail", zap.Any("defaultValue", field.DefaultValue))
HTTPAbortReturn(c, http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(err),
HTTPReturnMessage: "convert defaultValue fail, err:" + err.Error(),
})
return nil, err
}
if dataType == schemapb.DataType_Array {
if _, ok := schemapb.DataType_value[field.ElementDataType]; !ok {

93
internal/distributed/proxy/httpserver/handler_v2_test.go
View File

@ -564,6 +564,15 @@ func TestCreateCollection(t *testing.T) {
requestBody: []byte(`{"collectionName": "` + DefaultCollectionName + `", "schema": {
"fields": [
{"fieldName": "book_id", "dataType": "Int64", "isPrimary": true, "elementTypeParams": {}},
{"fieldName": "null_fid", "dataType": "Int64", "nullable": true},
{"fieldName": "default_fid_bool", "dataType": "Bool", "defaultValue": true},
{"fieldName": "default_fid_int8", "dataType": "Int8", "defaultValue": 10},
{"fieldName": "default_fid_int16", "dataType": "Int16", "defaultValue": 10},
{"fieldName": "default_fid_int32", "dataType": "Int32", "defaultValue": 10},
{"fieldName": "default_fid_int64", "dataType": "Int64", "defaultValue": 10},
{"fieldName": "default_fid_float32", "dataType": "Float", "defaultValue": 10},
{"fieldName": "default_fid_double", "dataType": "Double", "defaultValue": 10},
{"fieldName": "default_fid_varchar", "dataType": "VarChar", "defaultValue": "a"},
{"fieldName": "word_count", "dataType": "Array", "elementDataType": "Int64", "elementTypeParams": {"max_capacity": 2}},
{"fieldName": "book_intro", "dataType": "FloatVector", "elementTypeParams": {"dim": 2}}
]
@ -651,6 +660,90 @@ func TestCreateCollection(t *testing.T) {
errCode: 65535,
})
postTestCases = append(postTestCases, requestBodyTestCase{
path: path,
requestBody: []byte(`{"collectionName": "` + DefaultCollectionName + `", "schema": {
"fields": [
{"fieldName": "book_id", "dataType": "Int64", "isPrimary": true, "elementTypeParams": {}},
{"fieldName": "word_count", "dataType": "Int64", "elementTypeParams": {}},
{"fieldName": "default_fid", "dataType": "Bool", "defaultValue":10, "elementTypeParams": {}},
{"fieldName": "book_intro", "dataType": "SparseFloatVector", "elementTypeParams": {"dim": 2}}
]
}, "indexParams": [{"fieldName": "book_intro", "indexName": "book_intro_vector", "metricType": "L2"}]}`),
errMsg: "convert defaultValue fail, err:Wrong defaultValue type: invalid parameter[expected=bool][actual=10]",
errCode: 1100,
})
postTestCases = append(postTestCases, requestBodyTestCase{
path: path,
requestBody: []byte(`{"collectionName": "` + DefaultCollectionName + `", "schema": {
"fields": [
{"fieldName": "book_id", "dataType": "Int64", "isPrimary": true, "elementTypeParams": {}},
{"fieldName": "word_count", "dataType": "Int64", "elementTypeParams": {}},
{"fieldName": "default_fid", "dataType": "VarChar", "defaultValue":true, "elementTypeParams": {}},
{"fieldName": "book_intro", "dataType": "SparseFloatVector", "elementTypeParams": {"dim": 2}}
]
}, "indexParams": [{"fieldName": "book_intro", "indexName": "book_intro_vector", "metricType": "L2"}]}`),
errMsg: "convert defaultValue fail, err:Wrong defaultValue type: invalid parameter[expected=string][actual=true]",
errCode: 1100,
})
postTestCases = append(postTestCases, requestBodyTestCase{
path: path,
requestBody: []byte(`{"collectionName": "` + DefaultCollectionName + `", "schema": {
"fields": [
{"fieldName": "book_id", "dataType": "Int64", "isPrimary": true, "elementTypeParams": {}},
{"fieldName": "word_count", "dataType": "Int64", "elementTypeParams": {}},
{"fieldName": "default_fid", "dataType": "Int8", "defaultValue":"10", "elementTypeParams": {}},
{"fieldName": "book_intro", "dataType": "SparseFloatVector", "elementTypeParams": {"dim": 2}}
]
}, "indexParams": [{"fieldName": "book_intro", "indexName": "book_intro_vector", "metricType": "L2"}]}`),
errMsg: "convert defaultValue fail, err:Wrong defaultValue type: invalid parameter[expected=number][actual=10]",
errCode: 1100,
})
postTestCases = append(postTestCases, requestBodyTestCase{
path: path,
requestBody: []byte(`{"collectionName": "` + DefaultCollectionName + `", "schema": {
"fields": [
{"fieldName": "book_id", "dataType": "Int64", "isPrimary": true, "elementTypeParams": {}},
{"fieldName": "word_count", "dataType": "Int64", "elementTypeParams": {}},
{"fieldName": "default_fid", "dataType": "Int64", "defaultValue":"10", "elementTypeParams": {}},
{"fieldName": "book_intro", "dataType": "SparseFloatVector", "elementTypeParams": {"dim": 2}}
]
}, "indexParams": [{"fieldName": "book_intro", "indexName": "book_intro_vector", "metricType": "L2"}]}`),
errMsg: "convert defaultValue fail, err:Wrong defaultValue type: invalid parameter[expected=number][actual=10]",
errCode: 1100,
})
postTestCases = append(postTestCases, requestBodyTestCase{
path: path,
requestBody: []byte(`{"collectionName": "` + DefaultCollectionName + `", "schema": {
"fields": [
{"fieldName": "book_id", "dataType": "Int64", "isPrimary": true, "elementTypeParams": {}},
{"fieldName": "word_count", "dataType": "Int64", "elementTypeParams": {}},
{"fieldName": "default_fid", "dataType": "Float", "defaultValue":"10", "elementTypeParams": {}},
{"fieldName": "book_intro", "dataType": "SparseFloatVector", "elementTypeParams": {"dim": 2}}
]
}, "indexParams": [{"fieldName": "book_intro", "indexName": "book_intro_vector", "metricType": "L2"}]}`),
errMsg: "convert defaultValue fail, err:Wrong defaultValue type: invalid parameter[expected=number][actual=10]",
errCode: 1100,
})
postTestCases = append(postTestCases, requestBodyTestCase{
path: path,
requestBody: []byte(`{"collectionName": "` + DefaultCollectionName + `", "schema": {
"fields": [
{"fieldName": "book_id", "dataType": "Int64", "isPrimary": true, "elementTypeParams": {}},
{"fieldName": "word_count", "dataType": "Int64", "elementTypeParams": {}},
{"fieldName": "default_fid", "dataType": "Double", "defaultValue":"10", "elementTypeParams": {}},
{"fieldName": "book_intro", "dataType": "SparseFloatVector", "elementTypeParams": {"dim": 2}}
]
}, "indexParams": [{"fieldName": "book_intro", "indexName": "book_intro_vector", "metricType": "L2"}]}`),
errMsg: "convert defaultValue fail, err:Wrong defaultValue type: invalid parameter[expected=number][actual=10]",
errCode: 1100,
})
for _, testcase := range postTestCases {
t.Run("post"+testcase.path, func(t *testing.T) {
req := httptest.NewRequest(http.MethodPost, testcase.path, bytes.NewReader(testcase.requestBody))

2
internal/distributed/proxy/httpserver/request_v2.go
View File

@ -320,6 +320,8 @@ type FieldSchema struct {
IsPrimary bool `json:"isPrimary"`
IsPartitionKey bool `json:"isPartitionKey"`
ElementTypeParams map[string]interface{} `json:"elementTypeParams" binding:"required"`
Nullable bool `json:"nullable" binding:"required"`
DefaultValue interface{} `json:"defaultValue" binding:"required"`
}
type CollectionSchema struct {

215
internal/distributed/proxy/httpserver/utils.go
View File

@ -218,12 +218,13 @@ func printIndexes(indexes []*milvuspb.IndexDescription) []gin.H {
// --------------------- insert param --------------------- //
func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error, []map[string]interface{}) {
func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error, []map[string]interface{}, map[string][]bool) {
var reallyDataArray []map[string]interface{}
validDataMap := make(map[string][]bool)
dataResult := gjson.Get(body, HTTPRequestData)
dataResultArray := dataResult.Array()
if len(dataResultArray) == 0 {
return merr.ErrMissingRequiredParameters, reallyDataArray
return merr.ErrMissingRequiredParameters, reallyDataArray, validDataMap
}
var fieldNames []string
@ -238,11 +239,21 @@ func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error,
fieldType := field.DataType
fieldName := field.Name
if field.Nullable || field.DefaultValue != nil {
value := gjson.Get(data.Raw, fieldName)
if value.Type == gjson.Null {
validDataMap[fieldName] = append(validDataMap[fieldName], false)
continue
} else {
validDataMap[fieldName] = append(validDataMap[fieldName], true)
}
}
dataString := gjson.Get(data.Raw, fieldName).String()
if field.IsPrimaryKey && field.AutoID {
if dataString != "" {
return merr.WrapErrParameterInvalid("", "set primary key but autoID == true"), reallyDataArray
return merr.WrapErrParameterInvalid("", "set primary key but autoID == true"), reallyDataArray, validDataMap
}
continue
}
@ -250,84 +261,84 @@ func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error,
switch fieldType {
case schemapb.DataType_FloatVector:
if dataString == "" {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], "", "missing vector field: "+fieldName), reallyDataArray
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], "", "missing vector field: "+fieldName), reallyDataArray, validDataMap
}
var vectorArray []float32
err := json.Unmarshal([]byte(dataString), &vectorArray)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray, validDataMap
}
reallyData[fieldName] = vectorArray
case schemapb.DataType_BinaryVector:
if dataString == "" {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], "", "missing vector field: "+fieldName), reallyDataArray
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], "", "missing vector field: "+fieldName), reallyDataArray, validDataMap
}
vectorStr := gjson.Get(data.Raw, fieldName).Raw
var vectorArray []byte
err := json.Unmarshal([]byte(vectorStr), &vectorArray)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray, validDataMap
}
reallyData[fieldName] = vectorArray
case schemapb.DataType_SparseFloatVector:
if dataString == "" {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], "", "missing vector field: "+fieldName), reallyDataArray
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], "", "missing vector field: "+fieldName), reallyDataArray, validDataMap
}
sparseVec, err := typeutil.CreateSparseFloatRowFromJSON([]byte(dataString))
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray, validDataMap
}
reallyData[fieldName] = sparseVec
case schemapb.DataType_Float16Vector:
if dataString == "" {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], "", "missing vector field: "+fieldName), reallyDataArray
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], "", "missing vector field: "+fieldName), reallyDataArray, validDataMap
}
vectorStr := gjson.Get(data.Raw, fieldName).Raw
var vectorArray []byte
err := json.Unmarshal([]byte(vectorStr), &vectorArray)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray, validDataMap
}
reallyData[fieldName] = vectorArray
case schemapb.DataType_BFloat16Vector:
if dataString == "" {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], "", "missing vector field: "+fieldName), reallyDataArray
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], "", "missing vector field: "+fieldName), reallyDataArray, validDataMap
}
vectorStr := gjson.Get(data.Raw, fieldName).Raw
var vectorArray []byte
err := json.Unmarshal([]byte(vectorStr), &vectorArray)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray, validDataMap
}
reallyData[fieldName] = vectorArray
case schemapb.DataType_Bool:
result, err := cast.ToBoolE(dataString)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray, validDataMap
}
reallyData[fieldName] = result
case schemapb.DataType_Int8:
result, err := cast.ToInt8E(dataString)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray, validDataMap
}
reallyData[fieldName] = result
case schemapb.DataType_Int16:
result, err := cast.ToInt16E(dataString)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray, validDataMap
}
reallyData[fieldName] = result
case schemapb.DataType_Int32:
result, err := cast.ToInt32E(dataString)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray, validDataMap
}
reallyData[fieldName] = result
case schemapb.DataType_Int64:
result, err := json.Number(dataString).Int64()
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray, validDataMap
}
reallyData[fieldName] = result
case schemapb.DataType_Array:
@ -337,7 +348,7 @@ func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error,
err := json.Unmarshal([]byte(dataString), &arr)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)]+
" of "+schemapb.DataType_name[int32(field.ElementType)], dataString, err.Error()), reallyDataArray
" of "+schemapb.DataType_name[int32(field.ElementType)], dataString, err.Error()), reallyDataArray, validDataMap
}
reallyData[fieldName] = &schemapb.ScalarField{
Data: &schemapb.ScalarField_BoolData{
@ -351,7 +362,7 @@ func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error,
err := json.Unmarshal([]byte(dataString), &arr)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)]+
" of "+schemapb.DataType_name[int32(field.ElementType)], dataString, err.Error()), reallyDataArray
" of "+schemapb.DataType_name[int32(field.ElementType)], dataString, err.Error()), reallyDataArray, validDataMap
}
reallyData[fieldName] = &schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
@ -365,7 +376,7 @@ func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error,
err := json.Unmarshal([]byte(dataString), &arr)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)]+
" of "+schemapb.DataType_name[int32(field.ElementType)], dataString, err.Error()), reallyDataArray
" of "+schemapb.DataType_name[int32(field.ElementType)], dataString, err.Error()), reallyDataArray, validDataMap
}
reallyData[fieldName] = &schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
@ -379,7 +390,7 @@ func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error,
err := json.Unmarshal([]byte(dataString), &arr)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)]+
" of "+schemapb.DataType_name[int32(field.ElementType)], dataString, err.Error()), reallyDataArray
" of "+schemapb.DataType_name[int32(field.ElementType)], dataString, err.Error()), reallyDataArray, validDataMap
}
reallyData[fieldName] = &schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
@ -390,18 +401,10 @@ func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error,
}
case schemapb.DataType_Int64:
arr := make([]int64, 0)
numArr := make([]json.Number, 0)
err := json.Unmarshal([]byte(dataString), &numArr)
err := json.Unmarshal([]byte(dataString), &arr)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)]+
" of "+schemapb.DataType_name[int32(field.ElementType)], dataString, err.Error()), reallyDataArray
}
for _, num := range numArr {
intVal, err := num.Int64()
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray
}
arr = append(arr, intVal)
" of "+schemapb.DataType_name[int32(field.ElementType)], dataString, err.Error()), reallyDataArray, validDataMap
}
reallyData[fieldName] = &schemapb.ScalarField{
Data: &schemapb.ScalarField_LongData{
@ -415,7 +418,7 @@ func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error,
err := json.Unmarshal([]byte(dataString), &arr)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)]+
" of "+schemapb.DataType_name[int32(field.ElementType)], dataString, err.Error()), reallyDataArray
" of "+schemapb.DataType_name[int32(field.ElementType)], dataString, err.Error()), reallyDataArray, validDataMap
}
reallyData[fieldName] = &schemapb.ScalarField{
Data: &schemapb.ScalarField_FloatData{
@ -429,7 +432,7 @@ func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error,
err := json.Unmarshal([]byte(dataString), &arr)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)]+
" of "+schemapb.DataType_name[int32(field.ElementType)], dataString, err.Error()), reallyDataArray
" of "+schemapb.DataType_name[int32(field.ElementType)], dataString, err.Error()), reallyDataArray, validDataMap
}
reallyData[fieldName] = &schemapb.ScalarField{
Data: &schemapb.ScalarField_DoubleData{
@ -443,7 +446,7 @@ func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error,
err := json.Unmarshal([]byte(dataString), &arr)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)]+
" of "+schemapb.DataType_name[int32(field.ElementType)], dataString, err.Error()), reallyDataArray
" of "+schemapb.DataType_name[int32(field.ElementType)], dataString, err.Error()), reallyDataArray, validDataMap
}
reallyData[fieldName] = &schemapb.ScalarField{
Data: &schemapb.ScalarField_StringData{
@ -458,13 +461,13 @@ func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error,
case schemapb.DataType_Float:
result, err := cast.ToFloat32E(dataString)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray, validDataMap
}
reallyData[fieldName] = result
case schemapb.DataType_Double:
result, err := cast.ToFloat64E(dataString)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray, validDataMap
}
reallyData[fieldName] = result
case schemapb.DataType_VarChar:
@ -472,7 +475,7 @@ func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error,
case schemapb.DataType_String:
reallyData[fieldName] = dataString
default:
return merr.WrapErrParameterInvalid("", schemapb.DataType_name[int32(fieldType)], "fieldName: "+fieldName), reallyDataArray
return merr.WrapErrParameterInvalid("", schemapb.DataType_name[int32(fieldType)], "fieldName: "+fieldName), reallyDataArray, validDataMap
}
}
@ -505,10 +508,10 @@ func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error,
reallyDataArray = append(reallyDataArray, reallyData)
} else {
return merr.WrapErrParameterInvalid(gjson.JSON, data.Type, "NULL:0, FALSE:1, NUMBER:2, STRING:3, TRUE:4, JSON:5"), reallyDataArray
return merr.WrapErrParameterInvalid(gjson.JSON, data.Type, "NULL:0, FALSE:1, NUMBER:2, STRING:3, TRUE:4, JSON:5"), reallyDataArray, validDataMap
}
}
return nil, reallyDataArray
return nil, reallyDataArray, validDataMap
}
func containsString(arr []string, s string) bool {
@ -612,7 +615,7 @@ func convertToIntArray(dataType schemapb.DataType, arr interface{}) []int32 {
return res
}
func anyToColumns(rows []map[string]interface{}, sch *schemapb.CollectionSchema) ([]*schemapb.FieldData, error) {
func anyToColumns(rows []map[string]interface{}, validDataMap map[string][]bool, sch *schemapb.CollectionSchema) ([]*schemapb.FieldData, error) {
rowsLen := len(rows)
if rowsLen == 0 {
return []*schemapb.FieldData{}, fmt.Errorf("no row need to be convert to columns")
@ -703,6 +706,9 @@ func anyToColumns(rows []map[string]interface{}, sch *schemapb.CollectionSchema)
continue
}
candi, ok := set[field.Name]
if (field.Nullable || field.DefaultValue != nil) && !ok {
continue
}
if !ok {
return nil, fmt.Errorf("row %d does not has field %s", idx, field.Name)
}
@ -950,6 +956,7 @@ func anyToColumns(rows []map[string]interface{}, sch *schemapb.CollectionSchema)
default:
return nil, fmt.Errorf("the type(%v) of field(%v) is not supported, use other sdk please", colData.Type, name)
}
colData.ValidData = validDataMap[name]
columns = append(columns, colData)
}
if isDynamic {
@ -1166,26 +1173,62 @@ func buildQueryResp(rowsNum int64, needFields []string, fieldDataList []*schemap
for j := 0; j < columnNum; j++ {
switch fieldDataList[j].Type {
case schemapb.DataType_Bool:
if len(fieldDataList[j].ValidData) != 0 && !fieldDataList[j].ValidData[i] {
row[fieldDataList[j].FieldName] = nil
continue
}
row[fieldDataList[j].FieldName] = fieldDataList[j].GetScalars().GetBoolData().Data[i]
case schemapb.DataType_Int8:
if len(fieldDataList[j].ValidData) != 0 && !fieldDataList[j].ValidData[i] {
row[fieldDataList[j].FieldName] = nil
continue
}
row[fieldDataList[j].FieldName] = int8(fieldDataList[j].GetScalars().GetIntData().Data[i])
case schemapb.DataType_Int16:
if len(fieldDataList[j].ValidData) != 0 && !fieldDataList[j].ValidData[i] {
row[fieldDataList[j].FieldName] = nil
continue
}
row[fieldDataList[j].FieldName] = int16(fieldDataList[j].GetScalars().GetIntData().Data[i])
case schemapb.DataType_Int32:
if len(fieldDataList[j].ValidData) != 0 && !fieldDataList[j].ValidData[i] {
row[fieldDataList[j].FieldName] = nil
continue
}
row[fieldDataList[j].FieldName] = fieldDataList[j].GetScalars().GetIntData().Data[i]
case schemapb.DataType_Int64:
if len(fieldDataList[j].ValidData) != 0 && !fieldDataList[j].ValidData[i] {
row[fieldDataList[j].FieldName] = nil
continue
}
if enableInt64 {
row[fieldDataList[j].FieldName] = fieldDataList[j].GetScalars().GetLongData().Data[i]
} else {
row[fieldDataList[j].FieldName] = strconv.FormatInt(fieldDataList[j].GetScalars().GetLongData().Data[i], 10)
}
case schemapb.DataType_Float:
if len(fieldDataList[j].ValidData) != 0 && !fieldDataList[j].ValidData[i] {
row[fieldDataList[j].FieldName] = nil
continue
}
row[fieldDataList[j].FieldName] = fieldDataList[j].GetScalars().GetFloatData().Data[i]
case schemapb.DataType_Double:
if len(fieldDataList[j].ValidData) != 0 && !fieldDataList[j].ValidData[i] {
row[fieldDataList[j].FieldName] = nil
continue
}
row[fieldDataList[j].FieldName] = fieldDataList[j].GetScalars().GetDoubleData().Data[i]
case schemapb.DataType_String:
if len(fieldDataList[j].ValidData) != 0 && !fieldDataList[j].ValidData[i] {
row[fieldDataList[j].FieldName] = nil
continue
}
row[fieldDataList[j].FieldName] = fieldDataList[j].GetScalars().GetStringData().Data[i]
case schemapb.DataType_VarChar:
if len(fieldDataList[j].ValidData) != 0 && !fieldDataList[j].ValidData[i] {
row[fieldDataList[j].FieldName] = nil
continue
}
row[fieldDataList[j].FieldName] = fieldDataList[j].GetScalars().GetStringData().Data[i]
case schemapb.DataType_BinaryVector:
row[fieldDataList[j].FieldName] = fieldDataList[j].GetVectors().GetBinaryVector()[i*(fieldDataList[j].GetVectors().GetDim()/8) : (i+1)*(fieldDataList[j].GetVectors().GetDim()/8)]
@ -1198,8 +1241,16 @@ func buildQueryResp(rowsNum int64, needFields []string, fieldDataList []*schemap
case schemapb.DataType_SparseFloatVector:
row[fieldDataList[j].FieldName] = typeutil.SparseFloatBytesToMap(fieldDataList[j].GetVectors().GetSparseFloatVector().Contents[i])
case schemapb.DataType_Array:
if len(fieldDataList[j].ValidData) != 0 && !fieldDataList[j].ValidData[i] {
row[fieldDataList[j].FieldName] = nil
continue
}
row[fieldDataList[j].FieldName] = fieldDataList[j].GetScalars().GetArrayData().Data[i]
case schemapb.DataType_JSON:
if len(fieldDataList[j].ValidData) != 0 && !fieldDataList[j].ValidData[i] {
row[fieldDataList[j].FieldName] = nil
continue
}
data, ok := fieldDataList[j].GetScalars().Data.(*schemapb.ScalarField_JsonData)
if ok && !fieldDataList[j].IsDynamic {
row[fieldDataList[j].FieldName] = data.JsonData.Data[i]
@ -1299,3 +1350,85 @@ func convertConsistencyLevel(reqConsistencyLevel string) (commonpb.ConsistencyLe
// ConsistencyLevel_Session default in PyMilvus
return commonpb.ConsistencyLevel_Session, true, nil
}
func convertDefaultValue(value interface{}, dataType schemapb.DataType) (*schemapb.ValueField, error) {
if value == nil {
return nil, nil
}
switch dataType {
case schemapb.DataType_Bool:
v, ok := value.(bool)
if !ok {
return nil, merr.WrapErrParameterInvalid("bool", value, "Wrong defaultValue type")
}
data := &schemapb.ValueField{
Data: &schemapb.ValueField_BoolData{
BoolData: v,
},
}
return data, nil
case schemapb.DataType_Int8, schemapb.DataType_Int16, schemapb.DataType_Int32:
// all passed number is float64 type
v, ok := value.(float64)
if !ok {
return nil, merr.WrapErrParameterInvalid("number", value, "Wrong defaultValue type")
}
data := &schemapb.ValueField{
Data: &schemapb.ValueField_IntData{
IntData: int32(v),
},
}
return data, nil
case schemapb.DataType_Int64:
v, ok := value.(float64)
if !ok {
return nil, merr.WrapErrParameterInvalid("number", value, "Wrong defaultValue type")
}
data := &schemapb.ValueField{
Data: &schemapb.ValueField_LongData{
LongData: int64(v),
},
}
return data, nil
case schemapb.DataType_Float:
v, ok := value.(float64)
if !ok {
return nil, merr.WrapErrParameterInvalid("number", value, "Wrong defaultValue type")
}
data := &schemapb.ValueField{
Data: &schemapb.ValueField_FloatData{
FloatData: float32(v),
},
}
return data, nil
case schemapb.DataType_Double:
v, ok := value.(float64)
if !ok {
return nil, merr.WrapErrParameterInvalid("number", value, "Wrong defaultValue type")
}
data := &schemapb.ValueField{
Data: &schemapb.ValueField_DoubleData{
DoubleData: v,
},
}
return data, nil
case schemapb.DataType_String, schemapb.DataType_VarChar:
v, ok := value.(string)
if !ok {
return nil, merr.WrapErrParameterInvalid("string", value, "Wrong defaultValue type")
}
data := &schemapb.ValueField{
Data: &schemapb.ValueField_StringData{
StringData: v,
},
}
return data, nil
default:
return nil, merr.WrapErrParameterInvalidMsg(fmt.Sprintf("Unexpected default value type: %d", dataType))
}
}

343
internal/distributed/proxy/httpserver/utils_test.go
View File

@ -496,12 +496,12 @@ func TestInsertWithDynamicFields(t *testing.T) {
req := InsertReq{}
coll := generateCollectionSchema(schemapb.DataType_Int64)
var err error
err, req.Data = checkAndSetData(body, coll)
err, req.Data, _ = checkAndSetData(body, coll)
assert.Equal(t, nil, err)
assert.Equal(t, int64(0), req.Data[0]["id"])
assert.Equal(t, int64(1), req.Data[0]["book_id"])
assert.Equal(t, int64(2), req.Data[0]["word_count"])
fieldsData, err := anyToColumns(req.Data, coll)
fieldsData, err := anyToColumns(req.Data, nil, coll)
assert.Equal(t, nil, err)
assert.Equal(t, true, fieldsData[len(fieldsData)-1].IsDynamic)
assert.Equal(t, schemapb.DataType_JSON, fieldsData[len(fieldsData)-1].Type)
@ -521,7 +521,7 @@ func TestInsertWithoutVector(t *testing.T) {
float16VectorField.Name = "float16Vector"
bfloat16VectorField := generateVectorFieldSchema(schemapb.DataType_BFloat16Vector)
bfloat16VectorField.Name = "bfloat16Vector"
err, _ = checkAndSetData(body, &schemapb.CollectionSchema{
err, _, _ = checkAndSetData(body, &schemapb.CollectionSchema{
Name: DefaultCollectionName,
Fields: []*schemapb.FieldSchema{
primaryField, floatVectorField,
@ -530,7 +530,7 @@ func TestInsertWithoutVector(t *testing.T) {
})
assert.Error(t, err)
assert.Equal(t, true, strings.HasPrefix(err.Error(), "missing vector field"))
err, _ = checkAndSetData(body, &schemapb.CollectionSchema{
err, _, _ = checkAndSetData(body, &schemapb.CollectionSchema{
Name: DefaultCollectionName,
Fields: []*schemapb.FieldSchema{
primaryField, binaryVectorField,
@ -539,7 +539,7 @@ func TestInsertWithoutVector(t *testing.T) {
})
assert.Error(t, err)
assert.Equal(t, true, strings.HasPrefix(err.Error(), "missing vector field"))
err, _ = checkAndSetData(body, &schemapb.CollectionSchema{
err, _, _ = checkAndSetData(body, &schemapb.CollectionSchema{
Name: DefaultCollectionName,
Fields: []*schemapb.FieldSchema{
primaryField, float16VectorField,
@ -548,7 +548,7 @@ func TestInsertWithoutVector(t *testing.T) {
})
assert.Error(t, err)
assert.Equal(t, true, strings.HasPrefix(err.Error(), "missing vector field"))
err, _ = checkAndSetData(body, &schemapb.CollectionSchema{
err, _, _ = checkAndSetData(body, &schemapb.CollectionSchema{
Name: DefaultCollectionName,
Fields: []*schemapb.FieldSchema{
primaryField, bfloat16VectorField,
@ -568,16 +568,81 @@ func TestInsertWithInt64(t *testing.T) {
DataType: schemapb.DataType_Array,
ElementType: schemapb.DataType_Int64,
})
err, data := checkAndSetData(body, coll)
err, data, validData := checkAndSetData(body, coll)
assert.Equal(t, nil, err)
assert.Equal(t, 1, len(data))
assert.Equal(t, 0, len(validData))
assert.Equal(t, int64(9999999999999999), data[0][FieldBookID])
arr, _ := data[0][arrayFieldName].(*schemapb.ScalarField)
assert.Equal(t, int64(9999999999999999), arr.GetLongData().GetData()[0])
}
body = "{\"data\": {\"book_id\": 9999999999999999, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999.0]}}"
err, _ = checkAndSetData(body, coll)
assert.Error(t, err)
func TestInsertWithNullableField(t *testing.T) {
arrayFieldName := "array-int64"
coll := generateCollectionSchema(schemapb.DataType_Int64)
coll.Fields = append(coll.Fields, &schemapb.FieldSchema{
Name: arrayFieldName,
DataType: schemapb.DataType_Array,
ElementType: schemapb.DataType_Int64,
})
coll.Fields = append(coll.Fields, &schemapb.FieldSchema{
Name: "nullable",
DataType: schemapb.DataType_Int64,
Nullable: true,
})
body := "{\"data\": [{\"book_id\": 9999999999999999, \"\nullable\": null,\"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]},{\"book_id\": 1, \"nullable\": 1,\"book_intro\": [0.3, 0.4], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}]"
err, data, validData := checkAndSetData(body, coll)
assert.Equal(t, nil, err)
assert.Equal(t, 2, len(data))
assert.Equal(t, 1, len(validData))
assert.Equal(t, 2, len(validData["nullable"]))
assert.False(t, validData["nullable"][0])
assert.True(t, validData["nullable"][1])
assert.Equal(t, int64(9999999999999999), data[0][FieldBookID])
arr, _ := data[0][arrayFieldName].(*schemapb.ScalarField)
assert.Equal(t, int64(9999999999999999), arr.GetLongData().GetData()[0])
assert.Equal(t, 4, len(data[0]))
assert.Equal(t, 5, len(data[1]))
fieldData, err := anyToColumns(data, validData, coll)
assert.Equal(t, nil, err)
assert.Equal(t, len(coll.Fields)+1, len(fieldData))
}
func TestInsertWithDefaultValueField(t *testing.T) {
arrayFieldName := "array-int64"
coll := generateCollectionSchema(schemapb.DataType_Int64)
coll.Fields = append(coll.Fields, &schemapb.FieldSchema{
Name: arrayFieldName,
DataType: schemapb.DataType_Array,
ElementType: schemapb.DataType_Int64,
})
coll.Fields = append(coll.Fields, &schemapb.FieldSchema{
Name: "fid",
DataType: schemapb.DataType_Int64,
DefaultValue: &schemapb.ValueField{
Data: &schemapb.ValueField_LongData{
LongData: 10,
},
},
})
body := "{\"data\": [{\"book_id\": 9999999999999999, \"\fid\": null,\"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]},{\"book_id\": 1, \"fid\": 1,\"book_intro\": [0.3, 0.4], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}]"
err, data, validData := checkAndSetData(body, coll)
assert.Equal(t, nil, err)
assert.Equal(t, 2, len(data))
assert.Equal(t, 1, len(validData))
assert.Equal(t, 2, len(validData["fid"]))
assert.False(t, validData["fid"][0])
assert.True(t, validData["fid"][1])
assert.Equal(t, int64(9999999999999999), data[0][FieldBookID])
arr, _ := data[0][arrayFieldName].(*schemapb.ScalarField)
assert.Equal(t, int64(9999999999999999), arr.GetLongData().GetData()[0])
assert.Equal(t, 4, len(data[0]))
assert.Equal(t, 5, len(data[1]))
fieldData, err := anyToColumns(data, validData, coll)
assert.Equal(t, nil, err)
assert.Equal(t, len(coll.Fields)+1, len(fieldData))
}
func TestSerialize(t *testing.T) {
@ -1076,6 +1141,247 @@ func newFieldData(fieldDatas []*schemapb.FieldData, firstFieldType schemapb.Data
}
}
func newNullableFieldData(fieldDatas []*schemapb.FieldData, firstFieldType schemapb.DataType) []*schemapb.FieldData {
fieldData1 := schemapb.FieldData{
Type: schemapb.DataType_Bool,
FieldName: "field-bool",
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_BoolData{
BoolData: &schemapb.BoolArray{
Data: []bool{true, true, true},
},
},
},
},
ValidData: []bool{true, false, true},
IsDynamic: false,
}
fieldDatas = append(fieldDatas, &fieldData1)
fieldData2 := schemapb.FieldData{
Type: schemapb.DataType_Int8,
FieldName: "field-int8",
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: []int32{0, 1, 2},
},
},
},
},
ValidData: []bool{true, false, true},
IsDynamic: false,
}
fieldDatas = append(fieldDatas, &fieldData2)
fieldData3 := schemapb.FieldData{
Type: schemapb.DataType_Int16,
FieldName: "field-int16",
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: []int32{0, 1, 2},
},
},
},
},
ValidData: []bool{true, false, true},
IsDynamic: false,
}
fieldDatas = append(fieldDatas, &fieldData3)
fieldData4 := schemapb.FieldData{
Type: schemapb.DataType_Int32,
FieldName: "field-int32",
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: []int32{0, 1, 2},
},
},
},
},
ValidData: []bool{true, false, true},
IsDynamic: false,
}
fieldDatas = append(fieldDatas, &fieldData4)
fieldData5 := schemapb.FieldData{
Type: schemapb.DataType_Float,
FieldName: "field-float",
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_FloatData{
FloatData: &schemapb.FloatArray{
Data: []float32{0, 1, 2},
},
},
},
},
ValidData: []bool{true, false, true},
IsDynamic: false,
}
fieldDatas = append(fieldDatas, &fieldData5)
fieldData6 := schemapb.FieldData{
Type: schemapb.DataType_Double,
FieldName: "field-double",
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_DoubleData{
DoubleData: &schemapb.DoubleArray{
Data: []float64{0, 1, 2},
},
},
},
},
ValidData: []bool{true, false, true},
IsDynamic: false,
}
fieldDatas = append(fieldDatas, &fieldData6)
fieldData7 := schemapb.FieldData{
Type: schemapb.DataType_String,
FieldName: "field-string",
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_StringData{
StringData: &schemapb.StringArray{
Data: []string{"0", "1", "2"},
},
},
},
},
ValidData: []bool{true, false, true},
IsDynamic: false,
}
fieldDatas = append(fieldDatas, &fieldData7)
fieldData8 := schemapb.FieldData{
Type: schemapb.DataType_VarChar,
FieldName: "field-varchar",
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_StringData{
StringData: &schemapb.StringArray{
Data: []string{"0", "1", "2"},
},
},
},
},
ValidData: []bool{true, false, true},
IsDynamic: false,
}
fieldDatas = append(fieldDatas, &fieldData8)
fieldData9 := schemapb.FieldData{
Type: schemapb.DataType_JSON,
FieldName: "field-json",
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_JsonData{
JsonData: &schemapb.JSONArray{
Data: [][]byte{[]byte(`{"XXX": 0}`), []byte(`{"XXX": 0}`), []byte(`{"XXX": 0}`)},
},
},
},
},
ValidData: []bool{true, false, true},
IsDynamic: false,
}
fieldDatas = append(fieldDatas, &fieldData9)
fieldData10 := schemapb.FieldData{
Type: schemapb.DataType_Array,
FieldName: "field-array",
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_ArrayData{
ArrayData: &schemapb.ArrayArray{
Data: []*schemapb.ScalarField{
{Data: &schemapb.ScalarField_BoolData{BoolData: &schemapb.BoolArray{Data: []bool{true}}}},
{Data: &schemapb.ScalarField_BoolData{BoolData: &schemapb.BoolArray{Data: []bool{true}}}},
{Data: &schemapb.ScalarField_BoolData{BoolData: &schemapb.BoolArray{Data: []bool{true}}}},
},
},
},
},
},
ValidData: []bool{true, false, true},
IsDynamic: false,
}
fieldData11 := schemapb.FieldData{
Type: schemapb.DataType_Int64,
FieldName: "field-int64",
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_LongData{
LongData: &schemapb.LongArray{
Data: []int64{0, 1, 2},
},
},
},
},
ValidData: []bool{true, false, true},
IsDynamic: false,
}
fieldDatas = append(fieldDatas, &fieldData11)
switch firstFieldType {
case schemapb.DataType_None:
return fieldDatas
case schemapb.DataType_Bool:
return []*schemapb.FieldData{&fieldData1}
case schemapb.DataType_Int8:
return []*schemapb.FieldData{&fieldData2}
case schemapb.DataType_Int16:
return []*schemapb.FieldData{&fieldData3}
case schemapb.DataType_Int32:
return []*schemapb.FieldData{&fieldData4}
case schemapb.DataType_Float:
return []*schemapb.FieldData{&fieldData5}
case schemapb.DataType_Double:
return []*schemapb.FieldData{&fieldData6}
case schemapb.DataType_String:
return []*schemapb.FieldData{&fieldData7}
case schemapb.DataType_VarChar:
return []*schemapb.FieldData{&fieldData8}
case schemapb.DataType_BinaryVector:
vectorField := generateVectorFieldData(firstFieldType)
return []*schemapb.FieldData{&vectorField}
case schemapb.DataType_FloatVector:
vectorField := generateVectorFieldData(firstFieldType)
return []*schemapb.FieldData{&vectorField}
case schemapb.DataType_Float16Vector:
vectorField := generateVectorFieldData(firstFieldType)
return []*schemapb.FieldData{&vectorField}
case schemapb.DataType_BFloat16Vector:
vectorField := generateVectorFieldData(firstFieldType)
return []*schemapb.FieldData{&vectorField}
case schemapb.DataType_Array:
return []*schemapb.FieldData{&fieldData10}
case schemapb.DataType_JSON:
return []*schemapb.FieldData{&fieldData9}
case schemapb.DataType_SparseFloatVector:
vectorField := generateVectorFieldData(firstFieldType)
return []*schemapb.FieldData{&vectorField}
case schemapb.DataType_Int64:
return []*schemapb.FieldData{&fieldData11}
default:
return []*schemapb.FieldData{
{
FieldName: "wrong-field-type",
Type: firstFieldType,
},
}
}
}
func newSearchResult(results []map[string]interface{}) []map[string]interface{} {
for i, result := range results {
result["field-bool"] = true
@ -1229,19 +1535,21 @@ func newRowsWithArray(results []map[string]interface{}) []map[string]interface{}
func TestArray(t *testing.T) {
body, _ := generateRequestBody(schemapb.DataType_Int64)
collectionSchema := generateCollectionSchema(schemapb.DataType_Int64)
err, rows := checkAndSetData(string(body), collectionSchema)
err, rows, validRows := checkAndSetData(string(body), collectionSchema)
assert.Equal(t, nil, err)
assert.Equal(t, 0, len(validRows))
assert.Equal(t, true, compareRows(rows, generateRawRows(schemapb.DataType_Int64), compareRow))
data, err := anyToColumns(rows, collectionSchema)
data, err := anyToColumns(rows, validRows, collectionSchema)
assert.Equal(t, nil, err)
assert.Equal(t, len(collectionSchema.Fields)+1, len(data))
body, _ = generateRequestBodyWithArray(schemapb.DataType_Int64)
collectionSchema = newCollectionSchemaWithArray(generateCollectionSchema(schemapb.DataType_Int64))
err, rows = checkAndSetData(string(body), collectionSchema)
err, rows, validRows = checkAndSetData(string(body), collectionSchema)
assert.Equal(t, nil, err)
assert.Equal(t, 0, len(validRows))
assert.Equal(t, true, compareRows(rows, newRowsWithArray(generateRawRows(schemapb.DataType_Int64)), compareRow))
data, err = anyToColumns(rows, collectionSchema)
data, err = anyToColumns(rows, validRows, collectionSchema)
assert.Equal(t, nil, err)
assert.Equal(t, len(collectionSchema.Fields)+1, len(data))
}
@ -1297,7 +1605,7 @@ func TestVector(t *testing.T) {
},
EnableDynamicField: true,
}
err, rows := checkAndSetData(string(body), collectionSchema)
err, rows, validRows := checkAndSetData(string(body), collectionSchema)
assert.Equal(t, nil, err)
for _, row := range rows {
assert.Equal(t, 1, len(row[binaryVector].([]byte)))
@ -1306,7 +1614,8 @@ func TestVector(t *testing.T) {
// all test sparse rows have 2 elements, each should be of 8 bytes
assert.Equal(t, 16, len(row[sparseFloatVector].([]byte)))
}
data, err := anyToColumns(rows, collectionSchema)
assert.Equal(t, 0, len(validRows))
data, err := anyToColumns(rows, validRows, collectionSchema)
assert.Equal(t, nil, err)
assert.Equal(t, len(collectionSchema.Fields)+1, len(data))
@ -1317,7 +1626,7 @@ func TestVector(t *testing.T) {
}
row[field] = value
body, _ = wrapRequestBody([]map[string]interface{}{row})
err, _ = checkAndSetData(string(body), collectionSchema)
err, _, _ = checkAndSetData(string(body), collectionSchema)
assert.Error(t, err)
}