test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-02 11:59:00 +08:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

Return status and add try catch exception in index builder

Browse Source 
Signed-off-by: bigsheeper <yihao.dai@zilliz.com>
This commit is contained in:
bigsheeper 2020-12-29 17:53:27 +08:00 committed by yefu.chen
parent b1b8747477
commit bb8da15691
10 changed files with 156 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
internal/core/CMakeLists.txt
View File

@ -214,3 +214,9 @@ install(
install(FILES ${CMAKE_BINARY_DIR}/src/indexbuilder/libmilvus_indexbuilder.so
DESTINATION lib)
install(
DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/common/
DESTINATION include/common/
FILES_MATCHING PATTERN "*_c.h"
)

30
internal/core/src/common/status_c.h Normal file
View File

@ -0,0 +1,30 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#pragma once
#ifdef __cplusplus
extern "C" {
#endif
enum ErrorCode {
Success = 0,
UnexpectedException = 1,
};
typedef struct CStatus {
int error_code;
const char* error_msg;
} CStatus;
#ifdef __cplusplus
}
#endif

89
internal/core/src/indexbuilder/index_c.cpp
View File

@ -26,11 +26,27 @@ class CGODebugUtils {
}
};
CIndex
CreateIndex(const char* serialized_type_params, const char* serialized_index_params) {
auto index = std::make_unique<milvus::indexbuilder::IndexWrapper>(serialized_type_params, serialized_index_params);
return index.release();
CStatus
CreateIndex(const char* serialized_type_params, const char* serialized_index_params, CIndex* res_index) {
auto status = CStatus();
try {
// std::cout << "strlen(serialized_type_params): " << CGODebugUtils::Strlen(serialized_type_params,
// type_params_size)
// << std::endl;
// std::cout << "type_params_size: " << type_params_size << std::endl;
// std::cout << "strlen(serialized_index_params): "
// << CGODebugUtils::Strlen(serialized_index_params, index_params_size) << std::endl;
// std::cout << "index_params_size: " << index_params_size << std::endl;
auto index =
std::make_unique<milvus::indexbuilder::IndexWrapper>(serialized_type_params, serialized_index_params);
*res_index = index.release();
status.error_code = Success;
status.error_msg = "";
} catch (std::runtime_error& e) {
status.error_code = UnexpectedException;
status.error_msg = strdup(e.what());
}
return status;
}
void
@ -39,30 +55,57 @@ DeleteIndex(CIndex index) {
delete cIndex;
}
void
CStatus
BuildFloatVecIndexWithoutIds(CIndex index, int64_t float_value_num, const float* vectors) {
auto cIndex = (milvus::indexbuilder::IndexWrapper*)index;
auto dim = cIndex->dim();
auto row_nums = float_value_num / dim;
auto ds = milvus::knowhere::GenDataset(row_nums, dim, vectors);
cIndex->BuildWithoutIds(ds);
auto status = CStatus();
try {
auto cIndex = (milvus::indexbuilder::IndexWrapper*)index;
auto dim = cIndex->dim();
auto row_nums = float_value_num / dim;
auto ds = milvus::knowhere::GenDataset(row_nums, dim, vectors);
cIndex->BuildWithoutIds(ds);
status.error_code = Success;
status.error_msg = "";
} catch (std::runtime_error& e) {
status.error_code = UnexpectedException;
status.error_msg = strdup(e.what());
}
return status;
}
void
CStatus
BuildBinaryVecIndexWithoutIds(CIndex index, int64_t data_size, const uint8_t* vectors) {
auto cIndex = (milvus::indexbuilder::IndexWrapper*)index;
auto dim = cIndex->dim();
auto row_nums = (data_size * 8) / dim;
auto ds = milvus::knowhere::GenDataset(row_nums, dim, vectors);
cIndex->BuildWithoutIds(ds);
auto status = CStatus();
try {
auto cIndex = (milvus::indexbuilder::IndexWrapper*)index;
auto dim = cIndex->dim();
auto row_nums = (data_size * 8) / dim;
auto ds = milvus::knowhere::GenDataset(row_nums, dim, vectors);
cIndex->BuildWithoutIds(ds);
status.error_code = Success;
status.error_msg = "";
} catch (std::runtime_error& e) {
status.error_code = UnexpectedException;
status.error_msg = strdup(e.what());
}
return status;
}
char*
SerializeToSlicedBuffer(CIndex index, int32_t* buffer_size) {
auto cIndex = (milvus::indexbuilder::IndexWrapper*)index;
auto binary = cIndex->Serialize();
*buffer_size = binary.size;
return binary.data;
CStatus
SerializeToSlicedBuffer(CIndex index, int32_t* buffer_size, char** res_buffer) {
auto status = CStatus();
try {
auto cIndex = (milvus::indexbuilder::IndexWrapper*)index;
auto binary = cIndex->Serialize();
*buffer_size = binary.size;
*res_buffer = binary.data;
status.error_code = Success;
status.error_msg = "";
} catch (std::runtime_error& e) {
status.error_code = UnexpectedException;
status.error_msg = strdup(e.what());
}
return status;
}
void

13
internal/core/src/indexbuilder/index_c.h
View File

@ -28,26 +28,27 @@ extern "C" {
#include <stdint.h>
#include "segcore/collection_c.h"
#include "common/status_c.h"
typedef void* CIndex;
// TODO: how could we pass map between go and c++ more efficiently?
// Solution: using protobuf instead of json, this way significantly increase programming efficiency
CIndex
CreateIndex(const char* serialized_type_params, const char* serialized_index_params);
CStatus
CreateIndex(const char* serialized_type_params, const char* serialized_index_params, CIndex* res_index);
void
DeleteIndex(CIndex index);
void
CStatus
BuildFloatVecIndexWithoutIds(CIndex index, int64_t float_value_num, const float* vectors);
void
CStatus
BuildBinaryVecIndexWithoutIds(CIndex index, int64_t data_size, const uint8_t* vectors);
char*
SerializeToSlicedBuffer(CIndex index, int32_t* buffer_size);
CStatus
SerializeToSlicedBuffer(CIndex index, int32_t* buffer_size, char** res_buffer);
void
LoadFromSlicedBuffer(CIndex index, const char* serialized_sliced_blob_buffer, int32_t size);

10
internal/core/src/segcore/collection_c.h
View File

@ -15,16 +15,6 @@
extern "C" {
#endif
enum ErrorCode {
Success = 0,
UnexpectedException = 1,
};
typedef struct CStatus {
int error_code;
const char* error_msg;
} CStatus;
typedef void* CCollection;
CCollection

1
internal/core/src/segcore/load_index_c.h
View File

@ -18,6 +18,7 @@ extern "C" {
#include <stdint.h>
#include "segcore/collection_c.h"
#include "common/status_c.h"
typedef void* CLoadIndexInfo;
typedef void* CBinarySet;

1
internal/core/src/segcore/plan_c.h
View File

@ -16,6 +16,7 @@ extern "C" {
#include <stdbool.h>
#include <stdint.h>
#include "segcore/collection_c.h"
#include "common/status_c.h"
typedef void* CPlan;
typedef void* CPlaceholderGroup;

1
internal/core/src/segcore/reduce_c.h
View File

@ -16,6 +16,7 @@ extern "C" {
#include <stdbool.h>
#include <stdint.h>
#include "segcore/segment_c.h"
#include "common/status_c.h"
typedef void* CMarshaledHits;

1
internal/core/src/segcore/segment_c.h
View File

@ -19,6 +19,7 @@ extern "C" {
#include "segcore/plan_c.h"
#include "segcore/load_index_c.h"
#include "common/status_c.h"
typedef void* CSegmentBase;
typedef void* CQueryResult;

53
internal/indexbuilder/index.go
View File

@ -13,6 +13,8 @@ package indexbuilder
*/
import "C"
import (
"errors"
"strconv"
"unsafe"
"github.com/golang/protobuf/proto"
@ -38,11 +40,20 @@ type CIndex struct {
func (index *CIndex) Serialize() ([]*Blob, error) {
/*
char*
SerializeToSlicedBuffer(CIndex index, int32_t* buffer_size);
CStatus
SerializeToSlicedBuffer(CIndex index, int32_t* buffer_size, char** res_buffer);
*/
var cDumpedSlicedBuffer *C.char
var bufferSize int32
var cDumpedSlicedBuffer *C.char = C.SerializeToSlicedBuffer(index.indexPtr, (*C.int32_t)(unsafe.Pointer(&bufferSize)))
status := C.SerializeToSlicedBuffer(index.indexPtr, (*C.int32_t)(unsafe.Pointer(&bufferSize)), &cDumpedSlicedBuffer)
errorCode := status.error_code
if errorCode != 0 {
errorMsg := C.GoString(status.error_msg)
defer C.free(unsafe.Pointer(status.error_msg))
return nil, errors.New("SerializeToSlicedBuffer failed, C runtime error detected, error code = " + strconv.Itoa(int(errorCode)) + ", error msg = " + errorMsg)
}
defer C.free(unsafe.Pointer(cDumpedSlicedBuffer))
dumpedSlicedBuffer := C.GoBytes(unsafe.Pointer(cDumpedSlicedBuffer), (C.int32_t)(bufferSize))
@ -81,19 +92,31 @@ func (index *CIndex) Load(blobs []*Blob) error {
func (index *CIndex) BuildFloatVecIndexWithoutIds(vectors []float32) error {
/*
void
CStatus
BuildFloatVecIndexWithoutIds(CIndex index, int64_t float_value_num, const float* vectors);
*/
C.BuildFloatVecIndexWithoutIds(index.indexPtr, (C.int64_t)(len(vectors)), (*C.float)(&vectors[0]))
status := C.BuildFloatVecIndexWithoutIds(index.indexPtr, (C.int64_t)(len(vectors)), (*C.float)(&vectors[0]))
errorCode := status.error_code
if errorCode != 0 {
errorMsg := C.GoString(status.error_msg)
defer C.free(unsafe.Pointer(status.error_msg))
return errors.New("BuildFloatVecIndexWithoutIds failed, C runtime error detected, error code = " + strconv.Itoa(int(errorCode)) + ", error msg = " + errorMsg)
}
return nil
}
func (index *CIndex) BuildBinaryVecIndexWithoutIds(vectors []byte) error {
/*
void
CStatus
BuildBinaryVecIndexWithoutIds(CIndex index, int64_t data_size, const uint8_t* vectors);
*/
C.BuildBinaryVecIndexWithoutIds(index.indexPtr, (C.int64_t)(len(vectors)), (*C.uint8_t)(&vectors[0]))
status := C.BuildBinaryVecIndexWithoutIds(index.indexPtr, (C.int64_t)(len(vectors)), (*C.uint8_t)(&vectors[0]))
errorCode := status.error_code
if errorCode != 0 {
errorMsg := C.GoString(status.error_msg)
defer C.free(unsafe.Pointer(status.error_msg))
return errors.New(" failed, C runtime error detected, error code = " + strconv.Itoa(int(errorCode)) + ", error msg = " + errorMsg)
}
return nil
}
@ -127,11 +150,21 @@ func NewCIndex(typeParams, indexParams map[string]string) (Index, error) {
indexParamsPointer := C.CString(indexParamsStr)
/*
CIndex
CStatus
CreateIndex(const char* serialized_type_params,
const char* serialized_index_params);
const char* serialized_index_params,
CIndex* res_index);
*/
var indexPtr C.CIndex
status := C.CreateIndex(typeParamsPointer, indexParamsPointer, &indexPtr)
errorCode := status.error_code
if errorCode != 0 {
errorMsg := C.GoString(status.error_msg)
defer C.free(unsafe.Pointer(status.error_msg))
return nil, errors.New(" failed, C runtime error detected, error code = " + strconv.Itoa(int(errorCode)) + ", error msg = " + errorMsg)
}
return &CIndex{
indexPtr: C.CreateIndex(typeParamsPointer, indexParamsPointer),
indexPtr: indexPtr,
}, nil
}