test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-05 05:18:52 +08:00
Code Issues Actions 14 Packages Projects Releases Wiki Activity

temp update

Browse Source 
Former-commit-id: 42adb2a912930d35c53d7fefca1f35d556b45c05
This commit is contained in:
zhiru 2019-06-14 15:32:28 +08:00
parent 0f8ac6dc65
commit 48fa9efd44
1 changed files with 120 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

254
cpp/unittest/faiss_wrapper/wrapper_test.cpp
View File

@ -9,144 +9,130 @@
#include "wrapper/Operand.h" #include "wrapper/Operand.h"
#include "wrapper/Index.h" #include "wrapper/Index.h"
#include "wrapper/IndexBuilder.h" #include "wrapper/IndexBuilder.h"
#include <regex>
using namespace zilliz::milvus::engine; using namespace zilliz::milvus::engine;
TEST(xxx, Wrapper_Test){
// std::string xxx = "dialect+driver://username:password@host:port/database";
//mysql://scott:tiger@localhost/mydatabase TEST(operand_test, Wrapper_Test) {
using std::cout;
using std::endl;
std::string littel_xx = "dixx://"; auto opd = std::make_shared<Operand>();
std::regex xx_regex("([a-zA-Z0-9_-\\.]*):\\/\\/([a-zA-Z0-9_-\\.]*)\\:([a-zA-Z0-9_-\\.]*)\\@([a-zA-Z0-9_-\\.]*)\\:([0-9]*)\\/([a-zA-Z0-9_-\\.]*)"); opd->index_type = "IVF";
std::smatch pieces_match; opd->preproc = "OPQ";
std::regex_match(littel_xx, pieces_match, xx_regex); opd->postproc = "PQ";
opd->metric_type = "L2";
opd->d = 64;
for(auto &x : pieces_match) { auto opd_str = operand_to_str(opd);
std::cout << "hehhe: " << x.str() << std::endl; auto new_opd = str_to_operand(opd_str);
}
// TODO: fix all place where using opd to build index.
assert(new_opd->get_index_type(10000) == opd->get_index_type(10000));
} }
//TEST(operand_test, Wrapper_Test) { TEST(build_test, Wrapper_Test) {
// using std::cout; // dimension of the vectors to index
// using std::endl; int d = 3;
//
// auto opd = std::make_shared<Operand>(); // make a set of nt training vectors in the unit cube
// opd->index_type = "IVF"; size_t nt = 10000;
// opd->preproc = "OPQ";
// opd->postproc = "PQ"; // a reasonable number of cetroids to index nb vectors
// opd->metric_type = "L2"; int ncentroids = 16;
// opd->d = 64;
// std::random_device rd;
// auto opd_str = operand_to_str(opd); std::mt19937 gen(rd());
// auto new_opd = str_to_operand(opd_str);
// std::vector<float> xb;
// // TODO: fix all place where using opd to build index. std::vector<long> ids;
// assert(new_opd->get_index_type(10000) == opd->get_index_type(10000));
//} //prepare train data
// std::uniform_real_distribution<> dis_xt(-1.0, 1.0);
//TEST(build_test, Wrapper_Test) { std::vector<float> xt(nt * d);
// // dimension of the vectors to index for (size_t i = 0; i < nt * d; i++) {
// int d = 3; xt[i] = dis_xt(gen);
// }
// // make a set of nt training vectors in the unit cube
// size_t nt = 10000; //train the index
// auto opd = std::make_shared<Operand>();
// // a reasonable number of cetroids to index nb vectors opd->index_type = "IVF";
// int ncentroids = 16; opd->d = d;
// opd->ncent = ncentroids;
// std::random_device rd; IndexBuilderPtr index_builder_1 = GetIndexBuilder(opd);
// std::mt19937 gen(rd()); auto index_1 = index_builder_1->build_all(0, xb, ids, nt, xt);
// ASSERT_TRUE(index_1 != nullptr);
// std::vector<float> xb;
// std::vector<long> ids; // size of the database we plan to index
// size_t nb = 100000;
// //prepare train data
// std::uniform_real_distribution<> dis_xt(-1.0, 1.0); //prepare raw data
// std::vector<float> xt(nt * d); xb.resize(nb);
// for (size_t i = 0; i < nt * d; i++) { ids.resize(nb);
// xt[i] = dis_xt(gen); for (size_t i = 0; i < nb; i++) {
// } xb[i] = dis_xt(gen);
// ids[i] = i;
// //train the index }
// auto opd = std::make_shared<Operand>(); index_1->add_with_ids(nb, xb.data(), ids.data());
// opd->index_type = "IVF";
// opd->d = d; //search in first quadrant
// opd->ncent = ncentroids; int nq = 1, k = 10;
// IndexBuilderPtr index_builder_1 = GetIndexBuilder(opd); std::vector<float> xq = {0.5, 0.5, 0.5};
// auto index_1 = index_builder_1->build_all(0, xb, ids, nt, xt); float *result_dists = new float[k];
// ASSERT_TRUE(index_1 != nullptr); long *result_ids = new long[k];
// index_1->search(nq, xq.data(), k, result_dists, result_ids);
// // size of the database we plan to index
// size_t nb = 100000; for (int i = 0; i < k; i++) {
// if (result_ids[i] < 0) {
// //prepare raw data ASSERT_TRUE(false);
// xb.resize(nb); break;
// ids.resize(nb); }
// for (size_t i = 0; i < nb; i++) {
// xb[i] = dis_xt(gen); long id = result_ids[i];
// ids[i] = i; std::cout << "No." << id << " [" << xb[id * 3] << ", " << xb[id * 3 + 1] << ", "
// } << xb[id * 3 + 2] << "] distance = " << result_dists[i] << std::endl;
// index_1->add_with_ids(nb, xb.data(), ids.data());
// //makesure result vector is in first quadrant
// //search in first quadrant ASSERT_TRUE(xb[id * 3] > 0.0);
// int nq = 1, k = 10; ASSERT_TRUE(xb[id * 3 + 1] > 0.0);
// std::vector<float> xq = {0.5, 0.5, 0.5}; ASSERT_TRUE(xb[id * 3 + 2] > 0.0);
// float *result_dists = new float[k]; }
// long *result_ids = new long[k];
// index_1->search(nq, xq.data(), k, result_dists, result_ids); delete[] result_dists;
// delete[] result_ids;
// for (int i = 0; i < k; i++) { }
// if (result_ids[i] < 0) {
// ASSERT_TRUE(false); TEST(gpu_build_test, Wrapper_Test) {
// break; using std::vector;
// }
// int d = 256;
// long id = result_ids[i]; int nb = 3 * 1000 * 100;
// std::cout << "No." << id << " [" << xb[id * 3] << ", " << xb[id * 3 + 1] << ", " int nq = 100;
// << xb[id * 3 + 2] << "] distance = " << result_dists[i] << std::endl; vector<float> xb(d * nb);
// vector<float> xq(d * nq);
// //makesure result vector is in first quadrant vector<long> ids(nb);
// ASSERT_TRUE(xb[id * 3] > 0.0);
// ASSERT_TRUE(xb[id * 3 + 1] > 0.0); std::random_device rd;
// ASSERT_TRUE(xb[id * 3 + 2] > 0.0); std::mt19937 gen(rd());
// } std::uniform_real_distribution<> dis_xt(-1.0, 1.0);
// for (auto &e : xb) { e = float(dis_xt(gen)); }
// delete[] result_dists; for (auto &e : xq) { e = float(dis_xt(gen)); }
// delete[] result_ids; for (int i = 0; i < nb; ++i) { ids[i] = i; }
//}
// auto opd = std::make_shared<Operand>();
//TEST(gpu_build_test, Wrapper_Test) { opd->index_type = "IVF";
// using std::vector; opd->d = d;
// opd->ncent = 256;
// int d = 256;
// int nb = 3 * 1000 * 100; IndexBuilderPtr index_builder_1 = GetIndexBuilder(opd);
// int nq = 100; auto index_1 = index_builder_1->build_all(nb, xb.data(), ids.data());
// vector<float> xb(d * nb); assert(index_1->ntotal == nb);
// vector<float> xq(d * nq); assert(index_1->dim == d);
// vector<long> ids(nb);
// // sanity check: search 5 first vectors of xb
// std::random_device rd; int k = 1;
// std::mt19937 gen(rd()); vector<long> I(5 * k);
// std::uniform_real_distribution<> dis_xt(-1.0, 1.0); vector<float> D(5 * k);
// for (auto &e : xb) { e = float(dis_xt(gen)); } index_1->search(5, xb.data(), k, D.data(), I.data());
// for (auto &e : xq) { e = float(dis_xt(gen)); } for (int i = 0; i < 5; ++i) { assert(i == I[i]); }
// for (int i = 0; i < nb; ++i) { ids[i] = i; } }
//
// auto opd = std::make_shared<Operand>();
// opd->index_type = "IVF";
// opd->d = d;
// opd->ncent = 256;
//
// IndexBuilderPtr index_builder_1 = GetIndexBuilder(opd);
// auto index_1 = index_builder_1->build_all(nb, xb.data(), ids.data());
// assert(index_1->ntotal == nb);
// assert(index_1->dim == d);
//
// // sanity check: search 5 first vectors of xb
// int k = 1;
// vector<long> I(5 * k);
// vector<float> D(5 * k);
// index_1->search(5, xb.data(), k, D.data(), I.data());
// for (int i = 0; i < 5; ++i) { assert(i == I[i]); }
//}