test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-02 03:48:37 +08:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

Update group process

Browse Source 
Former-commit-id: 6c4cae438a66556cfbb0c34d7ec3fce85d89c844
This commit is contained in:
jinhai 2019-04-09 14:20:58 +08:00
parent 095fccd4a5
commit 3864c217ac
3 changed files with 37 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
pyengine/engine/controller/meta_manager.py
View File

@ -15,7 +15,6 @@ class MetaManager(object):
# add into database
db.session.add(new_group)
db.session.commit()
return ErrorCode.SUCCESS_CODE, group_name
@ -27,6 +26,11 @@ class MetaManager(object):
else:
return ErrorCode.FAULT_CODE, None
@staticmethod
def GetAllGroup():
groups = GroupTable.query.all()
return groups
@staticmethod
def DeleteGroup(group):
db.session.delete(group)
@ -35,6 +39,6 @@ class MetaManager(object):
def DeleteGroupFiles(group_name):
records = FileTable.query.filter(FileTable.group_name == group_name).all()
for record in records:
print("record.group_name: ", record.group_name)
# print("record.group_name: ", record.group_name)
db.session.delete(record)

22
pyengine/engine/controller/tests/test_vector_engine.py
View File

@ -1,5 +1,6 @@
from engine.controller.vector_engine import VectorEngine
from engine.settings import DATABASE_DIRECTORY
from engine.controller.error_code import ErrorCode
from flask import jsonify
import pytest
import os
@ -22,23 +23,22 @@ class TestVectorEngine:
def test_group(self):
# Make sure there is no group
code, group_id = VectorEngine.DeleteGroup('test_group')
assert code == VectorEngine.SUCCESS_CODE
assert code == ErrorCode.SUCCESS_CODE
assert group_id == 'test_group'
# Add a group
code, group_id = VectorEngine.AddGroup('test_group', 8)
assert code == VectorEngine.SUCCESS_CODE
assert code == ErrorCode.SUCCESS_CODE
assert group_id == 'test_group'
# Check the group existing
code, group_id = VectorEngine.GetGroup('test_group')
assert code == VectorEngine.SUCCESS_CODE
assert code == ErrorCode.SUCCESS_CODE
assert group_id == 'test_group'
# Check the group list
code, group_list = VectorEngine.GetGroupList()
assert code == VectorEngine.SUCCESS_CODE
print("group_list: ", group_list)
assert code == ErrorCode.SUCCESS_CODE
assert group_list == [{'group_name': 'test_group', 'file_number': 0}]
# Add Vector for not exist group
@ -48,21 +48,21 @@ class TestVectorEngine:
# Add vector for exist group
code, vector_id = VectorEngine.AddVector('test_group', self.__vectors)
assert code == VectorEngine.SUCCESS_CODE
assert code == ErrorCode.SUCCESS_CODE
assert vector_id == ['test_group.0', 'test_group.1', 'test_group.2', 'test_group.3', 'test_group.4', 'test_group.5', 'test_group.6', 'test_group.7', 'test_group.8', 'test_group.9']
# Check search vector interface
code, vector_id = VectorEngine.SearchVector('test_group', self.__vector, self.__limit)
assert code == VectorEngine.SUCCESS_CODE
assert code == ErrorCode.SUCCESS_CODE
assert vector_id == ['test_group.0']
# Check create index interface
code = VectorEngine.CreateIndex('test_group')
assert code == VectorEngine.SUCCESS_CODE
assert code == ErrorCode.SUCCESS_CODE
# Remove the group
code, group_id = VectorEngine.DeleteGroup('test_group')
assert code == VectorEngine.SUCCESS_CODE
assert code == ErrorCode.SUCCESS_CODE
assert group_id == 'test_group'
# Check the group is disppeared
@ -81,7 +81,7 @@ class TestVectorEngine:
# Clear raw file
code = VectorEngine.ClearRawFile('test_group')
assert code == VectorEngine.SUCCESS_CODE
assert code == ErrorCode.SUCCESS_CODE
def test_raw_file(self):
filename = VectorEngine.InsertVectorIntoRawFile('test_group', 'test_group.raw', self.__vector, 0)
@ -99,7 +99,7 @@ class TestVectorEngine:
assert np.all(vector_list == expected_list)
code = VectorEngine.ClearRawFile('test_group')
assert code == VectorEngine.SUCCESS_CODE
assert code == ErrorCode.SUCCESS_CODE

45
pyengine/engine/controller/vector_engine.py
View File

@ -29,6 +29,7 @@ class VectorEngine(object):
else:
StorageManager.AddGroup(group_name)
MetaManager.AddGroup(group_name, dimension)
MetaManager.Sync()
return ErrorCode.SUCCESS_CODE, group_name
@ -43,13 +44,8 @@ class VectorEngine(object):
if(group):
MetaManager.DeleteGroup(group)
StorageManager.DeleteGroup(group_name)
records = FileTable.query.filter(FileTable.group_name == group_name).all()
for record in records:
print("record.group_name: ", record.group_name)
db.session.delete(record)
db.session.commit()
MetaManager.DeleteGroupFiles(group_name)
MetaManager.Sync()
return VectorEngine.SUCCESS_CODE, group_name
else:
return VectorEngine.SUCCESS_CODE, group_name
@ -57,40 +53,39 @@ class VectorEngine(object):
@staticmethod
def GetGroupList():
group = GroupTable.query.all()
groups = MetaManager.GetAllGroup()
group_list = []
for group_tuple in group:
for group_tuple in groups:
group_item = {}
group_item['group_name'] = group_tuple.group_name
group_item['file_number'] = group_tuple.file_number
group_item['file_number'] = 0
group_list.append(group_item)
print(group_list)
return VectorEngine.SUCCESS_CODE, group_list
@staticmethod
def AddVector(group_id, vectors):
print(group_id, vectors)
code, _, = VectorEngine.GetGroup(group_id)
if code == VectorEngine.FAULT_CODE:
def AddVector(group_name, vectors):
print(group_name, vectors)
error, _ = MetaManager.GetGroup(group_name)
if error == VectorEngine.FAULT_CODE:
return VectorEngine.GROUP_NOT_EXIST, 'invalid'
vector_str_list = []
for vector in vectors:
file = FileTable.query.filter(FileTable.group_name == group_id).filter(FileTable.type == 'raw').first()
group = GroupTable.query.filter(GroupTable.group_name == group_id).first()
file = FileTable.query.filter(FileTable.group_name == group_name).filter(FileTable.type == 'raw').first()
group = GroupTable.query.filter(GroupTable.group_name == group_name).first()
if file:
print('insert into exist file')
# create vector id
vector_id = file.seq_no + 1
# insert into raw file
VectorEngine.InsertVectorIntoRawFile(group_id, file.filename, vector, vector_id)
VectorEngine.InsertVectorIntoRawFile(group_name, file.filename, vector, vector_id)
# check if the file can be indexed
if file.row_number + 1 >= ROW_LIMIT:
raw_vector_array, raw_vector_id_array = VectorEngine.GetVectorListFromRawFile(group_id)
raw_vector_array, raw_vector_id_array = VectorEngine.GetVectorListFromRawFile(group_name)
d = group.dimension
# create index
@ -101,7 +96,7 @@ class VectorEngine(object):
index_filename = file.filename + '_index'
serialize.write_index(file_name=index_filename, index=index)
FileTable.query.filter(FileTable.group_name == group_id).filter(FileTable.type == 'raw').update({'row_number':file.row_number + 1,
FileTable.query.filter(FileTable.group_name == group_name).filter(FileTable.type == 'raw').update({'row_number':file.row_number + 1,
'type': 'index',
'filename': index_filename,
'seq_no': file.seq_no + 1})
@ -109,7 +104,7 @@ class VectorEngine(object):
VectorEngine.group_dict = None
else:
# we still can insert into exist raw file, update database
FileTable.query.filter(FileTable.group_name == group_id).filter(FileTable.type == 'raw').update({'row_number':file.row_number + 1,
FileTable.query.filter(FileTable.group_name == group_name).filter(FileTable.type == 'raw').update({'row_number':file.row_number + 1,
'seq_no': file.seq_no + 1})
db.session.commit()
print('Update db for raw file insertion')
@ -117,16 +112,16 @@ class VectorEngine(object):
else:
print('add a new raw file')
# first raw file
raw_filename = group_id + '.raw'
raw_filename = group_name + '.raw'
# create vector id
vector_id = 0
# create and insert vector into raw file
VectorEngine.InsertVectorIntoRawFile(group_id, raw_filename, vector, vector_id)
VectorEngine.InsertVectorIntoRawFile(group_name, raw_filename, vector, vector_id)
# insert a record into database
db.session.add(FileTable(group_id, raw_filename, 'raw', 1))
db.session.add(FileTable(group_name, raw_filename, 'raw', 1))
db.session.commit()
vector_str_list.append(group_id + '.' + str(vector_id))
vector_str_list.append(group_name + '.' + str(vector_id))
return VectorEngine.SUCCESS_CODE, vector_str_list