test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-02 11:59:00 +08:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

Support ScaNN index (#26099)

Browse Source 
Signed-off-by: chasingegg <chao.gao@zilliz.com>
This commit is contained in:
Gao 2023-08-11 14:21:29 +08:00 committed by GitHub
parent 54d6f0ea23
commit b6fcbb0998
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 38 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
internal/core/thirdparty/knowhere/CMakeLists.txt vendored
View File

@ -11,7 +11,7 @@
# or implied. See the License for the specific language governing permissions and limitations under the License.
#-------------------------------------------------------------------------------
set( KNOWHERE_VERSION b7d0b0a )
set( KNOWHERE_VERSION 4f99dc0)
message(STATUS "Building knowhere-${KNOWHERE_SOURCE_VER} from source")
message(STATUS ${CMAKE_BUILD_TYPE})
@ -30,7 +30,7 @@ endif ()
set( CMAKE_PREFIX_PATH ${CONAN_BOOST_ROOT} )
FetchContent_Declare(
knowhere
GIT_REPOSITORY "https://github.com/milvus-io/knowhere.git"
GIT_REPOSITORY "https://github.com/zilliztech/knowhere.git"
GIT_TAG ${KNOWHERE_VERSION}
SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/knowhere-src
BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/knowhere-build

1
internal/querynodev2/mock_data.go
View File

@ -50,6 +50,7 @@ const (
IndexFaissIDMap = "FLAT"
IndexFaissIVFFlat = "IVF_FLAT"
IndexFaissIVFPQ = "IVF_PQ"
IndexScaNN = "SCANN"
IndexFaissIVFSQ8 = "IVF_SQ8"
IndexFaissBinIDMap = "BIN_FLAT"
IndexFaissBinIVFFlat = "BIN_IVF_FLAT"

1
internal/querynodev2/segments/mock_data.go
View File

@ -56,6 +56,7 @@ const (
IndexFaissIVFFlat = "IVF_FLAT"
IndexFaissIVFPQ = "IVF_PQ"
IndexFaissIVFSQ8 = "IVF_SQ8"
IndexScaNN = "SCANN"
IndexFaissBinIDMap = "BIN_FLAT"
IndexFaissBinIVFFlat = "BIN_IVF_FLAT"
IndexHNSW = "HNSW"

6
internal/util/indexcgowrapper/index_test.go
View File

@ -18,6 +18,7 @@ const (
IndexFaissIVFFlat = "IVF_FLAT"
IndexFaissIVFPQ = "IVF_PQ"
IndexFaissIVFSQ8 = "IVF_SQ8"
IndexScaNN = "SCANN"
IndexFaissBinIDMap = "BIN_FLAT"
IndexFaissBinIVFFlat = "BIN_IVF_FLAT"
@ -52,6 +53,8 @@ func generateFloatVectorTestCases() []vecTestCase {
{IndexFaissIVFPQ, metric.IP, false, schemapb.DataType_FloatVector},
{IndexFaissIVFSQ8, metric.L2, false, schemapb.DataType_FloatVector},
{IndexFaissIVFSQ8, metric.IP, false, schemapb.DataType_FloatVector},
{IndexScaNN, metric.L2, false, schemapb.DataType_FloatVector},
{IndexScaNN, metric.IP, false, schemapb.DataType_FloatVector},
{IndexHNSW, metric.L2, false, schemapb.DataType_FloatVector},
{IndexHNSW, metric.IP, false, schemapb.DataType_FloatVector},
}
@ -89,6 +92,9 @@ func generateParams(indexType, metricType string) (map[string]string, map[string
indexParams[common.DimKey] = strconv.Itoa(dim)
indexParams["nlist"] = strconv.Itoa(nlist)
indexParams["nbits"] = strconv.Itoa(nbits)
} else if indexType == IndexScaNN {
indexParams[common.DimKey] = strconv.Itoa(dim)
indexParams["nlist"] = strconv.Itoa(nlist)
} else if indexType == IndexHNSW {
indexParams[common.DimKey] = strconv.Itoa(dim)
indexParams["M"] = strconv.Itoa(16)

1
pkg/util/indexparamcheck/conf_adapter_mgr.go
View File

@ -48,6 +48,7 @@ func (mgr *indexCheckerMgrImpl) registerIndexChecker() {
mgr.checkers[IndexFaissIDMap] = newFlatChecker()
mgr.checkers[IndexFaissIvfFlat] = newIVFBaseChecker()
mgr.checkers[IndexFaissIvfPQ] = newIVFPQChecker()
mgr.checkers[IndexScaNN] = newIVFBaseChecker()
mgr.checkers[IndexFaissIvfSQ8] = newIVFSQChecker()
mgr.checkers[IndexFaissBinIDMap] = newBinFlatChecker()
mgr.checkers[IndexFaissBinIvfFlat] = newBinIVFFlatChecker()

12
pkg/util/indexparamcheck/conf_adapter_mgr_test.go
View File

@ -41,6 +41,12 @@ func Test_GetConfAdapterMgrInstance(t *testing.T) {
_, ok = adapter.(*ivfBaseChecker)
assert.Equal(t, true, ok)
adapter, err = adapterMgr.GetChecker(IndexScaNN)
assert.Equal(t, nil, err)
assert.NotEqual(t, nil, adapter)
_, ok = adapter.(*ivfBaseChecker)
assert.Equal(t, true, ok)
adapter, err = adapterMgr.GetChecker(IndexFaissIvfPQ)
assert.Equal(t, nil, err)
assert.NotEqual(t, nil, adapter)
@ -95,6 +101,12 @@ func TestConfAdapterMgrImpl_GetAdapter(t *testing.T) {
_, ok = adapter.(*ivfBaseChecker)
assert.Equal(t, true, ok)
adapter, err = adapterMgr.GetChecker(IndexScaNN)
assert.Equal(t, nil, err)
assert.NotEqual(t, nil, adapter)
_, ok = adapter.(*ivfBaseChecker)
assert.Equal(t, true, ok)
adapter, err = adapterMgr.GetChecker(IndexFaissIvfPQ)
assert.Equal(t, nil, err)
assert.NotEqual(t, nil, adapter)

1
pkg/util/indexparamcheck/index_type.go
View File

@ -21,6 +21,7 @@ const (
IndexFaissIDMap IndexType = "FLAT" // no index is built.
IndexFaissIvfFlat IndexType = "IVF_FLAT"
IndexFaissIvfPQ IndexType = "IVF_PQ"
IndexScaNN IndexType = "SCANN"
IndexFaissIvfSQ8 IndexType = "IVF_SQ8"
IndexFaissBinIDMap IndexType = "BIN_FLAT"
IndexFaissBinIvfFlat IndexType = "BIN_IVF_FLAT"

11
tests/integration/getvector/get_vector_test.go
View File

@ -287,6 +287,17 @@ func (s *TestGetVectorSuite) TestGetVector_IVF_PQ() {
s.run()
}
func (s *TestGetVectorSuite) TestGetVector_SCANN() {
s.nq = 10
s.topK = 10
s.indexType = integration.IndexScaNN
s.metricType = metric.L2
s.pkType = schemapb.DataType_Int64
s.vecType = schemapb.DataType_FloatVector
s.searchFailed = false
s.run()
}
func (s *TestGetVectorSuite) TestGetVector_IVF_SQ8() {
s.nq = 10
s.topK = 10

5
tests/integration/util_index.go
View File

@ -35,6 +35,7 @@ const (
IndexFaissIDMap = indexparamcheck.IndexFaissIDMap
IndexFaissIvfFlat = indexparamcheck.IndexFaissIvfFlat
IndexFaissIvfPQ = indexparamcheck.IndexFaissIvfPQ
IndexScaNN = indexparamcheck.IndexScaNN
IndexFaissIvfSQ8 = indexparamcheck.IndexFaissIvfSQ8
IndexFaissBinIDMap = indexparamcheck.IndexFaissBinIDMap
IndexFaissBinIvfFlat = indexparamcheck.IndexFaissBinIvfFlat
@ -124,7 +125,7 @@ func ConstructIndexParam(dim int, indexType string, metricType string) []*common
switch indexType {
case IndexFaissIDMap, IndexFaissBinIDMap:
// no index param is required
case IndexFaissIvfFlat, IndexFaissBinIvfFlat, IndexFaissIvfSQ8:
case IndexFaissIvfFlat, IndexFaissBinIvfFlat, IndexFaissIvfSQ8, IndexScaNN:
params = append(params, &commonpb.KeyValuePair{
Key: "nlist",
Value: "100",
@ -163,7 +164,7 @@ func GetSearchParams(indexType string, metricType string) map[string]any {
switch indexType {
case IndexFaissIDMap, IndexFaissBinIDMap:
params[common.MetricTypeKey] = metricType
case IndexFaissIvfFlat, IndexFaissBinIvfFlat, IndexFaissIvfSQ8, IndexFaissIvfPQ:
case IndexFaissIvfFlat, IndexFaissBinIvfFlat, IndexFaissIvfSQ8, IndexFaissIvfPQ, IndexScaNN:
params["nprobe"] = 8
case IndexHNSW:
params["ef"] = 200