test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-01 11:29:48 +08:00
Code Issues Actions 11 Packages Projects Releases Wiki Activity

Http update endpoints (#2187)

Browse Source 
* replace table with collection in http module

Signed-off-by: Yhz <yinghao.zou@zilliz.com>

* add search_by_ids in http module

Signed-off-by: Yhz <yinghao.zou@zilliz.com>

* add API

Signed-off-by: Yhz <yinghao.zou@zilliz.com>

* add new API and unittest

Signed-off-by: Yhz <yinghao.zou@zilliz.com>

* Update endpoint in http module(#2186); change vector id to string format(2185)

Signed-off-by: Yhz <yinghao.zou@zilliz.com>

* allow filter when show partition

Signed-off-by: Yhz <yinghao.zou@zilliz.com>

* move constant variables defination to .cpp

Signed-off-by: Yhz <yinghao.zou@zilliz.com>
This commit is contained in:
BossZou 2020-05-06 14:07:35 +08:00 committed by GitHub
parent fc1e6a74be
commit 82ab21aec3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 488 additions and 759 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -39,6 +39,8 @@ Please mark all change in change log and use the issue from GitHub
- \#2167 Merge log_config.conf with server_config.yaml
- \#2173 Check storage permission
- \#2178 Using elkan K-Means to improve IVF
- \#2185 Change id to string format in http module
- \#2186 Update endpoints in http module
- \#2190 Fix memory usage is twice of index size when using GPU searching
## Task

91
core/src/server/web_impl/Constants.cpp Normal file
View File

@ -0,0 +1,91 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "server/web_impl/Constants.h"
namespace milvus {
namespace server {
namespace web {
const char* NAME_ENGINE_TYPE_FLAT = "FLAT";
const char* NAME_ENGINE_TYPE_IVFFLAT = "IVFFLAT";
const char* NAME_ENGINE_TYPE_IVFSQ8 = "IVFSQ8";
const char* NAME_ENGINE_TYPE_IVFSQ8H = "IVFSQ8H";
const char* NAME_ENGINE_TYPE_RNSG = "RNSG";
const char* NAME_ENGINE_TYPE_IVFPQ = "IVFPQ";
const char* NAME_ENGINE_TYPE_HNSW = "HNSW";
const char* NAME_ENGINE_TYPE_ANNOY = "ANNOY";
const char* NAME_METRIC_TYPE_L2 = "L2";
const char* NAME_METRIC_TYPE_IP = "IP";
const char* NAME_METRIC_TYPE_HAMMING = "HAMMING";
const char* NAME_METRIC_TYPE_JACCARD = "JACCARD";
const char* NAME_METRIC_TYPE_TANIMOTO = "TANIMOTO";
const char* NAME_METRIC_TYPE_SUBSTRUCTURE = "SUBSTRUCTURE";
const char* NAME_METRIC_TYPE_SUPERSTRUCTURE = "SUPERSTRUCTURE";
////////////////////////////////////////////////////
const int64_t VALUE_COLLECTION_INDEX_FILE_SIZE_DEFAULT = 1024;
const char* VALUE_COLLECTION_METRIC_TYPE_DEFAULT = "L2";
const char* VALUE_PARTITION_TAG_DEFAULT = "";
const char* VALUE_INDEX_INDEX_TYPE_DEFAULT = NAME_ENGINE_TYPE_FLAT;
const int64_t VALUE_INDEX_NLIST_DEFAULT = 16384;
const int64_t VALUE_CONFIG_CPU_CACHE_CAPACITY_DEFAULT = 4;
const bool VALUE_CONFIG_CACHE_INSERT_DATA_DEFAULT = false;
/////////////////////////////////////////////////////
const std::unordered_map<engine::EngineType, std::string> IndexMap = {
{engine::EngineType::FAISS_IDMAP, NAME_ENGINE_TYPE_FLAT},
{engine::EngineType::FAISS_IVFFLAT, NAME_ENGINE_TYPE_IVFFLAT},
{engine::EngineType::FAISS_IVFSQ8, NAME_ENGINE_TYPE_IVFSQ8},
{engine::EngineType::FAISS_IVFSQ8H, NAME_ENGINE_TYPE_IVFSQ8H},
{engine::EngineType::NSG_MIX, NAME_ENGINE_TYPE_RNSG},
{engine::EngineType::FAISS_PQ, NAME_ENGINE_TYPE_IVFPQ},
{engine::EngineType::HNSW, NAME_ENGINE_TYPE_HNSW},
{engine::EngineType::ANNOY, NAME_ENGINE_TYPE_ANNOY},
};
const std::unordered_map<std::string, engine::EngineType> IndexNameMap = {
{NAME_ENGINE_TYPE_FLAT, engine::EngineType::FAISS_IDMAP},
{NAME_ENGINE_TYPE_IVFFLAT, engine::EngineType::FAISS_IVFFLAT},
{NAME_ENGINE_TYPE_IVFSQ8, engine::EngineType::FAISS_IVFSQ8},
{NAME_ENGINE_TYPE_IVFSQ8H, engine::EngineType::FAISS_IVFSQ8H},
{NAME_ENGINE_TYPE_RNSG, engine::EngineType::NSG_MIX},
{NAME_ENGINE_TYPE_IVFPQ, engine::EngineType::FAISS_PQ},
{NAME_ENGINE_TYPE_HNSW, engine::EngineType::HNSW},
{NAME_ENGINE_TYPE_ANNOY, engine::EngineType::ANNOY},
};
const std::unordered_map<engine::MetricType, std::string> MetricMap = {
{engine::MetricType::L2, NAME_METRIC_TYPE_L2},
{engine::MetricType::IP, NAME_METRIC_TYPE_IP},
{engine::MetricType::HAMMING, NAME_METRIC_TYPE_HAMMING},
{engine::MetricType::JACCARD, NAME_METRIC_TYPE_JACCARD},
{engine::MetricType::TANIMOTO, NAME_METRIC_TYPE_TANIMOTO},
{engine::MetricType::SUBSTRUCTURE, NAME_METRIC_TYPE_SUBSTRUCTURE},
{engine::MetricType::SUPERSTRUCTURE, NAME_METRIC_TYPE_SUPERSTRUCTURE},
};
const std::unordered_map<std::string, engine::MetricType> MetricNameMap = {
{NAME_METRIC_TYPE_L2, engine::MetricType::L2},
{NAME_METRIC_TYPE_IP, engine::MetricType::IP},
{NAME_METRIC_TYPE_HAMMING, engine::MetricType::HAMMING},
{NAME_METRIC_TYPE_JACCARD, engine::MetricType::JACCARD},
{NAME_METRIC_TYPE_TANIMOTO, engine::MetricType::TANIMOTO},
{NAME_METRIC_TYPE_SUBSTRUCTURE, engine::MetricType::SUBSTRUCTURE},
{NAME_METRIC_TYPE_SUPERSTRUCTURE, engine::MetricType::SUPERSTRUCTURE},
};
} // namespace web
} // namespace server
} // namespace milvus

91
core/src/server/web_impl/Constants.h
View File

@ -11,71 +11,46 @@
#pragma once
#include "db/engine/ExecutionEngine.h"
#include <string>
#include <unordered_map>
namespace milvus {
namespace server {
namespace web {
////////////////////////////////////////////////////
static const char* WEB_LOG_PREFIX = "[Web] ";
extern const char* NAME_ENGINE_TYPE_FLAT;
extern const char* NAME_ENGINE_TYPE_IVFFLAT;
extern const char* NAME_ENGINE_TYPE_IVFSQ8;
extern const char* NAME_ENGINE_TYPE_IVFSQ8H;
extern const char* NAME_ENGINE_TYPE_RNSG;
extern const char* NAME_ENGINE_TYPE_IVFPQ;
extern const char* NAME_ENGINE_TYPE_HNSW;
extern const char* NAME_ENGINE_TYPE_ANNOY;
extern const char* NAME_METRIC_TYPE_L2;
extern const char* NAME_METRIC_TYPE_IP;
extern const char* NAME_METRIC_TYPE_HAMMING;
extern const char* NAME_METRIC_TYPE_JACCARD;
extern const char* NAME_METRIC_TYPE_TANIMOTO;
extern const char* NAME_METRIC_TYPE_SUBSTRUCTURE;
extern const char* NAME_METRIC_TYPE_SUPERSTRUCTURE;
////////////////////////////////////////////////////
extern const int64_t VALUE_COLLECTION_INDEX_FILE_SIZE_DEFAULT;
extern const char* VALUE_COLLECTION_METRIC_TYPE_DEFAULT;
extern const char* VALUE_PARTITION_TAG_DEFAULT;
extern const char* VALUE_INDEX_INDEX_TYPE_DEFAULT;
extern const int64_t VALUE_INDEX_NLIST_DEFAULT;
extern const int64_t VALUE_CONFIG_CPU_CACHE_CAPACITY_DEFAULT;
extern const bool VALUE_CONFIG_CACHE_INSERT_DATA_DEFAULT;
static const char* CORS_KEY_METHODS = "Access-Control-Allow-Methods";
static const char* CORS_KEY_ORIGIN = "Access-Control-Allow-Origin";
static const char* CORS_KEY_HEADERS = "Access-Control-Allow-Headers";
static const char* CORS_KEY_AGE = "Access-Control-Max-Age";
static const char* CORS_VALUE_METHODS = "GET, POST, PUT, OPTIONS, DELETE";
static const char* CORS_VALUE_ORIGIN = "*";
static const char* CORS_VALUE_HEADERS =
"DNT, User-Agent, X-Requested-With, If-Modified-Since, Cache-Control, Content-Type, Range, Authorization";
static const char* CORS_VALUE_AGE = "1728000";
////////////////////////////////////////////////////
static const char* NAME_ENGINE_TYPE_FLAT = "FLAT";
static const char* NAME_ENGINE_TYPE_IVFFLAT = "IVFFLAT";
static const char* NAME_ENGINE_TYPE_IVFSQ8 = "IVFSQ8";
static const char* NAME_ENGINE_TYPE_IVFSQ8H = "IVFSQ8H";
static const char* NAME_ENGINE_TYPE_RNSG = "RNSG";
static const char* NAME_ENGINE_TYPE_IVFPQ = "IVFPQ";
static const char* NAME_ENGINE_TYPE_HNSW = "HNSW";
static const char* NAME_ENGINE_TYPE_ANNOY = "ANNOY";
static const char* NAME_METRIC_TYPE_L2 = "L2";
static const char* NAME_METRIC_TYPE_IP = "IP";
static const char* NAME_METRIC_TYPE_HAMMING = "HAMMING";
static const char* NAME_METRIC_TYPE_JACCARD = "JACCARD";
static const char* NAME_METRIC_TYPE_TANIMOTO = "TANIMOTO";
static const char* NAME_METRIC_TYPE_SUBSTRUCTURE = "SUBSTRUCTURE";
static const char* NAME_METRIC_TYPE_SUPERSTRUCTURE = "SUPERSTRUCTURE";
////////////////////////////////////////////////////
static const char* KEY_TABLE_TABLE_NAME = "table_name";
static const char* KEY_TABLE_DIMENSION = "dimension";
static const char* KEY_TABLE_INDEX_FILE_SIZE = "index_file_size";
static const char* KEY_TABLE_INDEX_METRIC_TYPE = "metric_type";
static const char* KEY_TABLE_COUNT = "count";
static const char* KEY_INDEX_INDEX_TYPE = "index_type";
static const char* KEY_INDEX_NLIST = "nlist";
static const char* KEY_PARTITION_NAME = "partition_name";
static const char* KEY_PARTITION_TAG = "partition_tag";
////////////////////////////////////////////////////
static const int64_t VALUE_TABLE_INDEX_FILE_SIZE_DEFAULT = 1024;
static const char* VALUE_TABLE_METRIC_TYPE_DEFAULT = "L2";
static const char* VALUE_PARTITION_TAG_DEFAULT = "";
static const char* VALUE_INDEX_INDEX_TYPE_DEFAULT = NAME_ENGINE_TYPE_FLAT;
static const int64_t VALUE_INDEX_NLIST_DEFAULT = 16384;
static const int64_t VALUE_CONFIG_CPU_CACHE_CAPACITY_DEFAULT = 4;
static const bool VALUE_CONFIG_CACHE_INSERT_DATA_DEFAULT = false;
/////////////////////////////////////////////////////
extern const std::unordered_map<engine::EngineType, std::string> IndexMap;
extern const std::unordered_map<std::string, engine::EngineType> IndexNameMap;
extern const std::unordered_map<engine::MetricType, std::string> MetricMap;
extern const std::unordered_map<std::string, engine::MetricType> MetricNameMap;
} // namespace web
} // namespace server

43
core/src/server/web_impl/Types.h
View File

@ -17,7 +17,6 @@
#include <oatpp/core/data/mapping/type/Object.hpp>
#include <oatpp/web/protocol/http/Http.hpp>
#include "db/engine/ExecutionEngine.h"
#include "server/web_impl/Constants.h"
namespace milvus {
@ -73,48 +72,6 @@ enum StatusCode : int {
MAX = ILLEGAL_QUERY_PARAM
};
static const std::unordered_map<engine::EngineType, std::string> IndexMap = {
{engine::EngineType::FAISS_IDMAP, NAME_ENGINE_TYPE_FLAT},
{engine::EngineType::FAISS_IVFFLAT, NAME_ENGINE_TYPE_IVFFLAT},
{engine::EngineType::FAISS_IVFSQ8, NAME_ENGINE_TYPE_IVFSQ8},
{engine::EngineType::FAISS_IVFSQ8H, NAME_ENGINE_TYPE_IVFSQ8H},
{engine::EngineType::NSG_MIX, NAME_ENGINE_TYPE_RNSG},
{engine::EngineType::FAISS_PQ, NAME_ENGINE_TYPE_IVFPQ},
{engine::EngineType::HNSW, NAME_ENGINE_TYPE_HNSW},
{engine::EngineType::ANNOY, NAME_ENGINE_TYPE_ANNOY},
};
static const std::unordered_map<std::string, engine::EngineType> IndexNameMap = {
{NAME_ENGINE_TYPE_FLAT, engine::EngineType::FAISS_IDMAP},
{NAME_ENGINE_TYPE_IVFFLAT, engine::EngineType::FAISS_IVFFLAT},
{NAME_ENGINE_TYPE_IVFSQ8, engine::EngineType::FAISS_IVFSQ8},
{NAME_ENGINE_TYPE_IVFSQ8H, engine::EngineType::FAISS_IVFSQ8H},
{NAME_ENGINE_TYPE_RNSG, engine::EngineType::NSG_MIX},
{NAME_ENGINE_TYPE_IVFPQ, engine::EngineType::FAISS_PQ},
{NAME_ENGINE_TYPE_HNSW, engine::EngineType::HNSW},
{NAME_ENGINE_TYPE_ANNOY, engine::EngineType::ANNOY},
};
static const std::unordered_map<engine::MetricType, std::string> MetricMap = {
{engine::MetricType::L2, NAME_METRIC_TYPE_L2},
{engine::MetricType::IP, NAME_METRIC_TYPE_IP},
{engine::MetricType::HAMMING, NAME_METRIC_TYPE_HAMMING},
{engine::MetricType::JACCARD, NAME_METRIC_TYPE_JACCARD},
{engine::MetricType::TANIMOTO, NAME_METRIC_TYPE_TANIMOTO},
{engine::MetricType::SUBSTRUCTURE, NAME_METRIC_TYPE_SUBSTRUCTURE},
{engine::MetricType::SUPERSTRUCTURE, NAME_METRIC_TYPE_SUPERSTRUCTURE},
};
static const std::unordered_map<std::string, engine::MetricType> MetricNameMap = {
{NAME_METRIC_TYPE_L2, engine::MetricType::L2},
{NAME_METRIC_TYPE_IP, engine::MetricType::IP},
{NAME_METRIC_TYPE_HAMMING, engine::MetricType::HAMMING},
{NAME_METRIC_TYPE_JACCARD, engine::MetricType::JACCARD},
{NAME_METRIC_TYPE_TANIMOTO, engine::MetricType::TANIMOTO},
{NAME_METRIC_TYPE_SUBSTRUCTURE, engine::MetricType::SUBSTRUCTURE},
{NAME_METRIC_TYPE_SUPERSTRUCTURE, engine::MetricType::SUPERSTRUCTURE},
};
} // namespace web
} // namespace server
} // namespace milvus

49
core/src/server/web_impl/controller/WebController.hpp
View File

@ -19,16 +19,17 @@
#include <oatpp/parser/json/mapping/ObjectMapper.hpp>
#include <oatpp/web/server/api/ApiController.hpp>
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "server/web_impl/Constants.h"
#include "server/web_impl/dto/CollectionDto.hpp"
#include "server/web_impl/dto/ConfigDto.hpp"
#include "server/web_impl/dto/IndexDto.hpp"
#include "server/web_impl/dto/PartitionDto.hpp"
#include "server/web_impl/dto/TableDto.hpp"
#include "server/web_impl/dto/VectorDto.hpp"
#include "server/web_impl/handler/WebRequestHandler.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#define WEB_LOG_PREFIX "[Web] "
namespace milvus {
namespace server {
@ -206,22 +207,22 @@ class WebController : public oatpp::web::server::api::ApiController {
#endif
ADD_CORS(TablesOptions)
ADD_CORS(CollectionsOptions)
ENDPOINT("OPTIONS", "/collections", TablesOptions) {
ENDPOINT("OPTIONS", "/collections", CollectionsOptions) {
return createResponse(Status::CODE_204, "No Content");
}
ADD_CORS(CreateTable)
ADD_CORS(CreateCollection)
ENDPOINT("POST", "/collections", CreateTable, BODY_DTO(TableRequestDto::ObjectWrapper, body)) {
ENDPOINT("POST", "/collections", CreateCollection, BODY_DTO(CollectionRequestDto::ObjectWrapper, body)) {
TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "POST \'/collections\'");
tr.RecordSection("Received request.");
WebRequestHandler handler = WebRequestHandler();
std::shared_ptr<OutgoingResponse> response;
auto status_dto = handler.CreateTable(body);
auto status_dto = handler.CreateCollection(body);
switch (status_dto->code->getValue()) {
case StatusCode::SUCCESS:
response = createDtoResponse(Status::CODE_201, status_dto);
@ -236,16 +237,16 @@ class WebController : public oatpp::web::server::api::ApiController {
return response;
}
ADD_CORS(ShowTables)
ADD_CORS(ShowCollections)
ENDPOINT("GET", "/collections", ShowTables, QUERIES(const QueryParams&, query_params)) {
ENDPOINT("GET", "/collections", ShowCollections, QUERIES(const QueryParams&, query_params)) {
TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "GET \'/collections\'");
tr.RecordSection("Received request.");
WebRequestHandler handler = WebRequestHandler();
String result;
auto status_dto = handler.ShowTables(query_params, result);
auto status_dto = handler.ShowCollections(query_params, result);
std::shared_ptr<OutgoingResponse> response;
switch (status_dto->code->getValue()) {
case StatusCode::SUCCESS:
@ -262,15 +263,15 @@ class WebController : public oatpp::web::server::api::ApiController {
return response;
}
ADD_CORS(TableOptions)
ADD_CORS(CollectionOptions)
ENDPOINT("OPTIONS", "/collections/{collection_name}", TableOptions) {
ENDPOINT("OPTIONS", "/collections/{collection_name}", CollectionOptions) {
return createResponse(Status::CODE_204, "No Content");
}
ADD_CORS(GetTable)
ADD_CORS(GetCollection)
ENDPOINT("GET", "/collections/{collection_name}", GetTable, PATH(String, collection_name),
ENDPOINT("GET", "/collections/{collection_name}", GetCollection, PATH(String, collection_name),
QUERIES(const QueryParams&, query_params)) {
TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "GET \'/collections/" + collection_name->std_str() + "\'");
tr.RecordSection("Received request.");
@ -278,7 +279,7 @@ class WebController : public oatpp::web::server::api::ApiController {
WebRequestHandler handler = WebRequestHandler();
String response_str;
auto status_dto = handler.GetTable(collection_name, query_params, response_str);
auto status_dto = handler.GetCollection(collection_name, query_params, response_str);
std::shared_ptr<OutgoingResponse> response;
switch (status_dto->code->getValue()) {
@ -299,16 +300,16 @@ class WebController : public oatpp::web::server::api::ApiController {
return response;
}
ADD_CORS(DropTable)
ADD_CORS(DropCollection)
ENDPOINT("DELETE", "/collections/{collection_name}", DropTable, PATH(String, collection_name)) {
ENDPOINT("DELETE", "/collections/{collection_name}", DropCollection, PATH(String, collection_name)) {
TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "DELETE \'/collections/" + collection_name->std_str() + "\'");
tr.RecordSection("Received request.");
WebRequestHandler handler = WebRequestHandler();
std::shared_ptr<OutgoingResponse> response;
auto status_dto = handler.DropTable(collection_name);
auto status_dto = handler.DropCollection(collection_name);
switch (status_dto->code->getValue()) {
case StatusCode::SUCCESS:
response = createDtoResponse(Status::CODE_204, status_dto);
@ -460,7 +461,7 @@ class WebController : public oatpp::web::server::api::ApiController {
ADD_CORS(ShowPartitions)
ENDPOINT("GET", "/collections/{collection_name}/partitions", ShowPartitions, PATH(String, collection_name),
QUERIES(const QueryParams&, query_params)) {
QUERIES(const QueryParams&, query_params), BODY_STRING(String, body)) {
TimeRecorder tr(std::string(WEB_LOG_PREFIX) + "GET \'/collections/" + collection_name->std_str() +
"/partitions\'");
tr.RecordSection("Received request.");
@ -472,7 +473,7 @@ class WebController : public oatpp::web::server::api::ApiController {
auto handler = WebRequestHandler();
std::shared_ptr<OutgoingResponse> response;
auto status_dto = handler.ShowPartitions(collection_name, query_params, partition_list_dto);
auto status_dto = handler.ShowPartitions(collection_name, query_params, body, partition_list_dto);
switch (status_dto->code->getValue()) {
case StatusCode::SUCCESS:
response = createDtoResponse(Status::CODE_200, partition_list_dto);
@ -577,10 +578,10 @@ class WebController : public oatpp::web::server::api::ApiController {
* GetVectorByID ?id=
*/
ENDPOINT("GET", "/collections/{collection_name}/vectors", GetVectors, PATH(String, collection_name),
QUERIES(const QueryParams&, query_params)) {
BODY_STRING(String, body), QUERIES(const QueryParams&, query_params)) {
auto handler = WebRequestHandler();
String response;
auto status_dto = handler.GetVector(collection_name, query_params, response);
auto status_dto = handler.GetVector(collection_name, body, query_params, response);
switch (status_dto->code->getValue()) {
case StatusCode::SUCCESS:

23
core/src/server/web_impl/dto/TableDto.hpp → core/src/server/web_impl/dto/CollectionDto.hpp
View File

@ -21,18 +21,17 @@ namespace web {
#include OATPP_CODEGEN_BEGIN(DTO)
class TableRequestDto : public oatpp::data::mapping::type::Object {
DTO_INIT(TableRequestDto, Object)
class CollectionRequestDto : public oatpp::data::mapping::type::Object {
DTO_INIT(CollectionRequestDto, Object)
DTO_FIELD(String, collection_name, "collection_name");
DTO_FIELD(Int64, dimension, "dimension");
DTO_FIELD(Int64, index_file_size, "index_file_size") = VALUE_TABLE_INDEX_FILE_SIZE_DEFAULT;
DTO_FIELD(String, metric_type, "metric_type") = VALUE_TABLE_METRIC_TYPE_DEFAULT;
DTO_FIELD(Int64, index_file_size, "index_file_size") = VALUE_COLLECTION_INDEX_FILE_SIZE_DEFAULT;
DTO_FIELD(String, metric_type, "metric_type") = VALUE_COLLECTION_METRIC_TYPE_DEFAULT;
};
class TableFieldsDto : public oatpp::data::mapping::type::Object {
DTO_INIT(TableFieldsDto, Object)
class CollectionFieldsDto : public oatpp::data::mapping::type::Object {
DTO_INIT(CollectionFieldsDto, Object)
DTO_FIELD(String, collection_name);
DTO_FIELD(Int64, dimension);
@ -43,16 +42,16 @@ class TableFieldsDto : public oatpp::data::mapping::type::Object {
DTO_FIELD(String, index_params);
};
class TableListDto : public OObject {
DTO_INIT(TableListDto, Object)
class CollectionListDto : public OObject {
DTO_INIT(CollectionListDto, Object)
DTO_FIELD(List<String>::ObjectWrapper, collection_names);
};
class TableListFieldsDto : public OObject {
DTO_INIT(TableListFieldsDto, Object)
class CollectionListFieldsDto : public OObject {
DTO_INIT(CollectionListFieldsDto, Object)
DTO_FIELD(List<TableFieldsDto::ObjectWrapper>::ObjectWrapper, collections);
DTO_FIELD(List<CollectionFieldsDto::ObjectWrapper>::ObjectWrapper, collections);
DTO_FIELD(Int64, count) = 0L;
};

130
core/src/server/web_impl/handler/WebRequestHandler.cpp
View File

@ -85,7 +85,7 @@ WebRequestHandler::AddStatusToJson(nlohmann::json& json, int64_t code, const std
}
Status
WebRequestHandler::IsBinaryTable(const std::string& collection_name, bool& bin) {
WebRequestHandler::IsBinaryCollection(const std::string& collection_name, bool& bin) {
CollectionSchema schema;
auto status = request_handler_.DescribeCollection(context_ptr_, collection_name, schema);
if (status.ok()) {
@ -131,7 +131,7 @@ WebRequestHandler::CopyRecordsFromJson(const nlohmann::json& json, engine::Vecto
///////////////////////// WebRequestHandler methods ///////////////////////////////////////
Status
WebRequestHandler::GetTableMetaInfo(const std::string& collection_name, nlohmann::json& json_out) {
WebRequestHandler::GetCollectionMetaInfo(const std::string& collection_name, nlohmann::json& json_out) {
CollectionSchema schema;
auto status = request_handler_.DescribeCollection(context_ptr_, collection_name, schema);
if (!status.ok()) {
@ -162,7 +162,7 @@ WebRequestHandler::GetTableMetaInfo(const std::string& collection_name, nlohmann
}
Status
WebRequestHandler::GetTableStat(const std::string& collection_name, nlohmann::json& json_out) {
WebRequestHandler::GetCollectionStat(const std::string& collection_name, nlohmann::json& json_out) {
std::string collection_info;
auto status = request_handler_.ShowCollectionInfo(context_ptr_, collection_name, collection_info);
@ -170,6 +170,8 @@ WebRequestHandler::GetTableStat(const std::string& collection_name, nlohmann::js
try {
json_out = nlohmann::json::parse(collection_info);
} catch (std::exception& e) {
return Status(SERVER_UNEXPECTED_ERROR,
"Error occurred when parsing collection stat information: " + std::string(e.what()));
}
}
@ -248,7 +250,7 @@ WebRequestHandler::Cmd(const std::string& cmd, std::string& result_str) {
}
Status
WebRequestHandler::PreLoadTable(const nlohmann::json& json, std::string& result_str) {
WebRequestHandler::PreLoadCollection(const nlohmann::json& json, std::string& result_str) {
if (!json.contains("collection_name")) {
return Status(BODY_FIELD_LOSS, "Field \"load\" must contains collection_name");
}
@ -408,6 +410,10 @@ WebRequestHandler::Search(const std::string& collection_name, const nlohmann::js
}
int64_t topk = json["topk"];
if (!json.contains("params")) {
return Status(BODY_FIELD_LOSS, "Field \'params\' is required");
}
std::vector<std::string> partition_tags;
if (json.contains("partition_tags")) {
auto tags = json["partition_tags"];
@ -420,6 +426,22 @@ WebRequestHandler::Search(const std::string& collection_name, const nlohmann::js
}
}
TopKQueryResult result;
Status status;
if (json.contains("ids")) {
auto vec_ids = json["ids"];
if (!vec_ids.is_array()) {
return Status(BODY_PARSE_FAIL, "Field \"ids\" must be ad array");
}
std::vector<int64_t> id_array;
for (auto& id_str : vec_ids) {
id_array.emplace_back(std::stol(id_str.get<std::string>()));
}
// std::vector<int64_t> id_array(vec_ids.begin(), vec_ids.end());
status = request_handler_.SearchByID(context_ptr_, collection_name, id_array, topk, json["params"],
partition_tags, result);
} else {
std::vector<std::string> file_id_vec;
if (json.contains("file_ids")) {
auto ids = json["file_ids"];
@ -431,12 +453,8 @@ WebRequestHandler::Search(const std::string& collection_name, const nlohmann::js
}
}
if (!json.contains("params")) {
return Status(BODY_FIELD_LOSS, "Field \'params\' is required");
}
bool bin_flag = false;
auto status = IsBinaryTable(collection_name, bin_flag);
status = IsBinaryCollection(collection_name, bin_flag);
if (!status.ok()) {
return status;
}
@ -451,10 +469,9 @@ WebRequestHandler::Search(const std::string& collection_name, const nlohmann::js
return status;
}
TopKQueryResult result;
status = request_handler_.Search(context_ptr_, collection_name, vectors_data, topk, json["params"], partition_tags,
file_id_vec, result);
status = request_handler_.Search(context_ptr_, collection_name, vectors_data, topk, json["params"],
partition_tags, file_id_vec, result);
}
if (!status.ok()) {
return status;
}
@ -769,7 +786,7 @@ WebRequestHandler::GetVectorsByIDs(const std::string& collection_name, const std
}
bool bin;
status = IsBinaryTable(collection_name, bin);
status = IsBinaryCollection(collection_name, bin);
if (!status.ok()) {
return status;
}
@ -1059,7 +1076,7 @@ WebRequestHandler::SetGpuConfig(const GPUConfigDto::ObjectWrapper& gpu_config_dt
* Collection {
*/
StatusDto::ObjectWrapper
WebRequestHandler::CreateTable(const TableRequestDto::ObjectWrapper& collection_schema) {
WebRequestHandler::CreateCollection(const CollectionRequestDto::ObjectWrapper& collection_schema) {
if (nullptr == collection_schema->collection_name.get()) {
RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Field \'collection_name\' is missing")
}
@ -1133,7 +1150,7 @@ WebRequestHandler::CreateHybridCollection(const milvus::server::web::OString& bo
}
StatusDto::ObjectWrapper
WebRequestHandler::ShowTables(const OQueryParams& query_params, OString& result) {
WebRequestHandler::ShowCollections(const OQueryParams& query_params, OString& result) {
int64_t offset = 0;
auto status = ParseQueryInteger(query_params, "offset", offset);
if (!status.ok()) {
@ -1173,7 +1190,7 @@ WebRequestHandler::ShowTables(const OQueryParams& query_params, OString& result)
nlohmann::json collections_json;
for (int64_t i = offset; i < page_size + offset; i++) {
nlohmann::json collection_json;
status = GetTableMetaInfo(collections.at(i), collection_json);
status = GetCollectionMetaInfo(collections.at(i), collection_json);
if (!status.ok()) {
ASSIGN_RETURN_STATUS_DTO(status)
}
@ -1194,7 +1211,7 @@ WebRequestHandler::ShowTables(const OQueryParams& query_params, OString& result)
}
StatusDto::ObjectWrapper
WebRequestHandler::GetTable(const OString& collection_name, const OQueryParams& query_params, OString& result) {
WebRequestHandler::GetCollection(const OString& collection_name, const OQueryParams& query_params, OString& result) {
if (nullptr == collection_name.get()) {
RETURN_STATUS_DTO(PATH_PARAM_LOSS, "Path param \'collection_name\' is required!");
}
@ -1207,11 +1224,11 @@ WebRequestHandler::GetTable(const OString& collection_name, const OQueryParams&
if (!stat.empty() && stat == "stat") {
nlohmann::json json;
status = GetTableStat(collection_name->std_str(), json);
status = GetCollectionStat(collection_name->std_str(), json);
result = status.ok() ? json.dump().c_str() : "NULL";
} else {
nlohmann::json json;
status = GetTableMetaInfo(collection_name->std_str(), json);
status = GetCollectionMetaInfo(collection_name->std_str(), json);
result = status.ok() ? json.dump().c_str() : "NULL";
}
@ -1219,7 +1236,7 @@ WebRequestHandler::GetTable(const OString& collection_name, const OQueryParams&
}
StatusDto::ObjectWrapper
WebRequestHandler::DropTable(const OString& collection_name) {
WebRequestHandler::DropCollection(const OString& collection_name) {
auto status = request_handler_.DropCollection(context_ptr_, collection_name->std_str());
ASSIGN_RETURN_STATUS_DTO(status)
@ -1294,7 +1311,7 @@ WebRequestHandler::CreatePartition(const OString& collection_name, const Partiti
}
StatusDto::ObjectWrapper
WebRequestHandler::ShowPartitions(const OString& collection_name, const OQueryParams& query_params,
WebRequestHandler::ShowPartitions(const OString& collection_name, const OQueryParams& query_params, const OString& body,
PartitionListDto::ObjectWrapper& partition_list_dto) {
int64_t offset = 0;
auto status = ParseQueryInteger(query_params, "offset", offset);
@ -1313,6 +1330,35 @@ WebRequestHandler::ShowPartitions(const OString& collection_name, const OQueryPa
Status(SERVER_UNEXPECTED_ERROR, "Query param 'offset' or 'page_size' should equal or bigger than 0"));
}
if (nullptr != body.get() && body->getSize() > 0) {
auto body_json = nlohmann::json::parse(body->c_str());
if (!body_json.contains("filter")) {
RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Field \'filter\' is required.")
}
auto filter_json = body_json["filter"];
if (filter_json.contains("partition_tag")) {
std::string tag = filter_json["partition_tag"];
bool exists = false;
status = request_handler_.HasPartition(context_ptr_, collection_name->std_str(), tag, exists);
if (!status.ok()) {
ASSIGN_RETURN_STATUS_DTO(status)
}
auto partition_dto = PartitionFieldsDto::createShared();
if (exists) {
partition_list_dto->count = 1;
partition_dto->partition_tag = tag.c_str();
} else {
partition_list_dto->count = 0;
}
partition_list_dto->partitions = partition_list_dto->partitions->createShared();
partition_list_dto->partitions->pushBack(partition_dto);
ASSIGN_RETURN_STATUS_DTO(status)
} else {
RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Unknown field.")
}
}
bool all_required = false;
auto required = query_params.get("all_required");
if (nullptr != required.get()) {
@ -1465,7 +1511,7 @@ WebRequestHandler::Insert(const OString& collection_name, const OString& body, V
// step 1: copy vectors
bool bin_flag;
auto status = IsBinaryTable(collection_name->std_str(), bin_flag);
auto status = IsBinaryCollection(collection_name->std_str(), bin_flag);
if (!status.ok()) {
ASSIGN_RETURN_STATUS_DTO(status)
}
@ -1488,8 +1534,9 @@ WebRequestHandler::Insert(const OString& collection_name, const OString& body, V
}
auto& id_array = vectors.id_array_;
id_array.clear();
for (auto& id : ids_json) {
id_array.emplace_back(id.get<int64_t>());
for (auto& id_str : ids_json) {
int64_t id = std::stol(id_str.get<std::string>());
id_array.emplace_back(id);
}
}
@ -1578,7 +1625,7 @@ WebRequestHandler::InsertEntity(const OString& collection_name, const milvus::se
}
case engine::meta::hybrid::DataType::VECTOR: {
bool bin_flag;
status = IsBinaryTable(collection_name->c_str(), bin_flag);
status = IsBinaryCollection(collection_name->c_str(), bin_flag);
if (!status.ok()) {
ASSIGN_RETURN_STATUS_DTO(status)
}
@ -1612,17 +1659,26 @@ WebRequestHandler::InsertEntity(const OString& collection_name, const milvus::se
}
StatusDto::ObjectWrapper
WebRequestHandler::GetVector(const OString& collection_name, const OQueryParams& query_params, OString& response) {
int64_t id = 0;
auto status = ParseQueryInteger(query_params, "id", id, false);
if (!status.ok()) {
RETURN_STATUS_DTO(status.code(), status.message().c_str());
WebRequestHandler::GetVector(const OString& collection_name, const OString& body, const OQueryParams& query_params,
OString& response) {
auto status = Status::OK();
try {
auto body_json = nlohmann::json::parse(body->c_str());
if (!body_json.contains("ids")) {
RETURN_STATUS_DTO(BODY_FIELD_LOSS, "Field \'ids\' is required.")
}
auto ids = body_json["ids"];
if (!ids.is_array()) {
RETURN_STATUS_DTO(BODY_PARSE_FAIL, "Field \'ids\' must be a array.")
}
std::vector<int64_t> ids = {id};
std::vector<int64_t> vector_ids;
for (auto& id : ids) {
vector_ids.push_back(std::stol(id.get<std::string>()));
}
engine::VectorsData vectors;
nlohmann::json vectors_json;
status = GetVectorsByIDs(collection_name->std_str(), ids, vectors_json);
status = GetVectorsByIDs(collection_name->std_str(), vector_ids, vectors_json);
if (!status.ok()) {
response = "NULL";
ASSIGN_RETURN_STATUS_DTO(status)
@ -1635,10 +1691,12 @@ WebRequestHandler::GetVector(const OString& collection_name, const OQueryParams&
} else {
json["vectors"] = vectors_json;
}
response = json.dump().c_str();
} catch (std::exception& e) {
RETURN_STATUS_DTO(SERVER_UNEXPECTED_ERROR, e.what());
}
ASSIGN_RETURN_STATUS_DTO(status)
ASSIGN_RETURN_STATUS_DTO(status);
}
StatusDto::ObjectWrapper
@ -1718,7 +1776,7 @@ WebRequestHandler::SystemOp(const OString& op, const OString& body_str, OString&
nlohmann::json j = nlohmann::json::parse(body_str->c_str());
if (op->equals("task")) {
if (j.contains("load")) {
status = PreLoadTable(j["load"], result_str);
status = PreLoadCollection(j["load"], result_str);
} else if (j.contains("flush")) {
status = Flush(j["flush"], result_str);
}

22
core/src/server/web_impl/handler/WebRequestHandler.h
View File

@ -27,11 +27,11 @@
#include "server/context/Context.h"
#include "server/delivery/RequestHandler.h"
#include "server/web_impl/Types.h"
#include "server/web_impl/dto/CollectionDto.hpp"
#include "server/web_impl/dto/ConfigDto.hpp"
#include "server/web_impl/dto/DevicesDto.hpp"
#include "server/web_impl/dto/IndexDto.hpp"
#include "server/web_impl/dto/PartitionDto.hpp"
#include "server/web_impl/dto/TableDto.hpp"
#include "server/web_impl/dto/VectorDto.hpp"
#include "thirdparty/nlohmann/json.hpp"
#include "utils/Status.h"
@ -82,17 +82,17 @@ class WebRequestHandler {
AddStatusToJson(nlohmann::json& json, int64_t code, const std::string& msg);
Status
IsBinaryTable(const std::string& collection_name, bool& bin);
IsBinaryCollection(const std::string& collection_name, bool& bin);
Status
CopyRecordsFromJson(const nlohmann::json& json, engine::VectorsData& vectors, bool bin);
protected:
Status
GetTableMetaInfo(const std::string& collection_name, nlohmann::json& json_out);
GetCollectionMetaInfo(const std::string& collection_name, nlohmann::json& json_out);
Status
GetTableStat(const std::string& collection_name, nlohmann::json& json_out);
GetCollectionStat(const std::string& collection_name, nlohmann::json& json_out);
Status
GetSegmentVectors(const std::string& collection_name, const std::string& segment_name, int64_t page_size,
@ -109,7 +109,7 @@ class WebRequestHandler {
Cmd(const std::string& cmd, std::string& result_str);
Status
PreLoadTable(const nlohmann::json& json, std::string& result_str);
PreLoadCollection(const nlohmann::json& json, std::string& result_str);
Status
Flush(const nlohmann::json& json, std::string& result_str);
@ -166,18 +166,18 @@ class WebRequestHandler {
#endif
StatusDto::ObjectWrapper
CreateTable(const TableRequestDto::ObjectWrapper& table_schema);
CreateCollection(const CollectionRequestDto::ObjectWrapper& table_schema);
StatusDto::ObjectWrapper
ShowTables(const OQueryParams& query_params, OString& result);
ShowCollections(const OQueryParams& query_params, OString& result);
StatusDto::ObjectWrapper
CreateHybridCollection(const OString& body);
StatusDto::ObjectWrapper
GetTable(const OString& collection_name, const OQueryParams& query_params, OString& result);
GetCollection(const OString& collection_name, const OQueryParams& query_params, OString& result);
StatusDto::ObjectWrapper
DropTable(const OString& collection_name);
DropCollection(const OString& collection_name);
StatusDto::ObjectWrapper
CreateIndex(const OString& collection_name, const OString& body);
@ -192,7 +192,7 @@ class WebRequestHandler {
CreatePartition(const OString& collection_name, const PartitionRequestDto::ObjectWrapper& param);
StatusDto::ObjectWrapper
ShowPartitions(const OString& collection_name, const OQueryParams& query_params,
ShowPartitions(const OString& collection_name, const OQueryParams& query_params, const OString& body,
PartitionListDto::ObjectWrapper& partition_list_dto);
StatusDto::ObjectWrapper
@ -220,7 +220,7 @@ class WebRequestHandler {
InsertEntity(const OString& collection_name, const OString& body, VectorIdsDto::ObjectWrapper& ids_dto);
StatusDto::ObjectWrapper
GetVector(const OString& collection_name, const OQueryParams& query_params, OString& response);
GetVector(const OString& collection_name, const OString& body, const OQueryParams& query_params, OString& response);
StatusDto::ObjectWrapper
VectorsOp(const OString& collection_name, const OString& payload, OString& response);

696
core/unittest/server/test_web.cpp
View File

File diff suppressed because it is too large Load Diff