milvus/pkg/util/paramtable/autoindex_param.go

// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package paramtable

import (
	"fmt"

	"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
	"github.com/milvus-io/milvus/pkg/common"
	"github.com/milvus-io/milvus/pkg/config"
	"github.com/milvus-io/milvus/pkg/util/funcutil"
	"github.com/milvus-io/milvus/pkg/util/indexparamcheck"
)

// /////////////////////////////////////////////////////////////////////////////
// --- common ---
type autoIndexConfig struct {
	Enable         ParamItem `refreshable:"true"`
	EnableOptimize ParamItem `refreshable:"true"`

	IndexParams           ParamItem  `refreshable:"true"`
	SparseIndexParams     ParamItem  `refreshable:"true"`
	BinaryIndexParams     ParamItem  `refreshable:"true"`
	PrepareParams         ParamItem  `refreshable:"true"`
	LoadAdaptParams       ParamItem  `refreshable:"true"`
	ExtraParams           ParamItem  `refreshable:"true"`
	IndexType             ParamItem  `refreshable:"true"`
	AutoIndexTypeName     ParamItem  `refreshable:"true"`
	AutoIndexSearchConfig ParamItem  `refreshable:"true"`
	AutoIndexTuningConfig ParamGroup `refreshable:"true"`

	ScalarAutoIndexEnable  ParamItem `refreshable:"true"`
	ScalarAutoIndexParams  ParamItem `refreshable:"true"`
	ScalarNumericIndexType ParamItem `refreshable:"true"`
	ScalarIntIndexType     ParamItem `refreshable:"true"`
	ScalarVarcharIndexType ParamItem `refreshable:"true"`
	ScalarBoolIndexType    ParamItem `refreshable:"true"`
	ScalarFloatIndexType   ParamItem `refreshable:"true"`
}

func (p *autoIndexConfig) init(base *BaseTable) {
	p.Enable = ParamItem{
		Key:          "autoIndex.enable",
		Version:      "2.2.0",
		DefaultValue: "false",
		PanicIfEmpty: true,
	}
	p.Enable.Init(base.mgr)

	p.EnableOptimize = ParamItem{
		Key:          "autoIndex.optimize",
		Version:      "2.4.0",
		DefaultValue: "true",
		PanicIfEmpty: true,
	}
	p.EnableOptimize.Init(base.mgr)

	p.IndexParams = ParamItem{
		Key:          "autoIndex.params.build",
		Version:      "2.2.0",
		DefaultValue: `{"M": 18,"efConstruction": 240,"index_type": "HNSW", "metric_type": "COSINE"}`,
		Export:       true,
	}
	p.IndexParams.Init(base.mgr)

	p.SparseIndexParams = ParamItem{
		Key:          "autoIndex.params.sparse.build",
		Version:      "2.4.5",
		DefaultValue: `{"index_type": "SPARSE_INVERTED_INDEX", "metric_type": "IP"}`,
		Export:       true,
	}
	p.SparseIndexParams.Init(base.mgr)

	p.BinaryIndexParams = ParamItem{
		Key:          "autoIndex.params.binary.build",
		Version:      "2.4.5",
		DefaultValue: `{"nlist": 1024, "index_type": "BIN_IVF_FLAT", "metric_type": "HAMMING"}`,
		Export:       true,
	}
	p.BinaryIndexParams.Init(base.mgr)

	p.PrepareParams = ParamItem{
		Key:     "autoIndex.params.prepare",
		Version: "2.3.2",
	}
	p.PrepareParams.Init(base.mgr)

	p.LoadAdaptParams = ParamItem{
		Key:     "autoIndex.params.load",
		Version: "2.4.5",
	}
	p.LoadAdaptParams.Init(base.mgr)

	p.ExtraParams = ParamItem{
		Key:     "autoIndex.params.extra",
		Version: "2.2.0",
	}
	p.ExtraParams.Init(base.mgr)

	p.IndexType = ParamItem{
		Version: "2.2.0",
		Formatter: func(v string) string {
			m := p.IndexParams.GetAsJSONMap()
			if m == nil {
				return ""
			}
			return m[common.IndexTypeKey]
		},
	}
	p.IndexType.Init(base.mgr)

	p.AutoIndexTypeName = ParamItem{
		Key:     "autoIndex.type",
		Version: "2.2.0",
	}
	p.AutoIndexTypeName.Init(base.mgr)

	p.AutoIndexSearchConfig = ParamItem{
		Key:     "autoindex.params.search",
		Version: "2.2.0",
	}
	p.AutoIndexSearchConfig.Init(base.mgr)

	p.AutoIndexTuningConfig = ParamGroup{
		KeyPrefix: "autoindex.params.tuning.",
		Version:   "2.3.0",
	}
	p.AutoIndexTuningConfig.Init(base.mgr)

	p.panicIfNotValidAndSetDefaultMetricType(base.mgr)

	p.ScalarAutoIndexEnable = ParamItem{
		Key:          "scalarAutoIndex.enable",
		Version:      "2.4.0",
		DefaultValue: "false",
		PanicIfEmpty: true,
	}
	p.ScalarAutoIndexEnable.Init(base.mgr)

	p.ScalarAutoIndexParams = ParamItem{
		Key:          "scalarAutoIndex.params.build",
		Version:      "2.4.0",
		DefaultValue: `{"int": "HYBRID","varchar": "HYBRID","bool": "BITMAP", "float": "INVERTED"}`,
	}
	p.ScalarAutoIndexParams.Init(base.mgr)

	// Deprecated param
	p.ScalarNumericIndexType = ParamItem{
		Version: "2.4.0",
		Formatter: func(v string) string {
			m := p.ScalarAutoIndexParams.GetAsJSONMap()
			if m == nil {
				return ""
			}
			return m["numeric"]
		},
	}
	p.ScalarNumericIndexType.Init(base.mgr)

	p.ScalarIntIndexType = ParamItem{
		Version: "2.5.0",
		Formatter: func(v string) string {
			m := p.ScalarAutoIndexParams.GetAsJSONMap()
			if m == nil {
				return ""
			}
			return m["int"]
		},
	}
	p.ScalarIntIndexType.Init(base.mgr)

	p.ScalarFloatIndexType = ParamItem{
		Version: "2.5.0",
		Formatter: func(v string) string {
			m := p.ScalarAutoIndexParams.GetAsJSONMap()
			if m == nil {
				return ""
			}
			return m["float"]
		},
	}
	p.ScalarFloatIndexType.Init(base.mgr)

	p.ScalarVarcharIndexType = ParamItem{
		Version: "2.4.0",
		Formatter: func(v string) string {
			m := p.ScalarAutoIndexParams.GetAsJSONMap()
			if m == nil {
				return ""
			}
			return m["varchar"]
		},
	}
	p.ScalarVarcharIndexType.Init(base.mgr)

	p.ScalarBoolIndexType = ParamItem{
		Version: "2.4.0",
		Formatter: func(v string) string {
			m := p.ScalarAutoIndexParams.GetAsJSONMap()
			if m == nil {
				return ""
			}
			return m["bool"]
		},
	}
	p.ScalarBoolIndexType.Init(base.mgr)
}

func (p *autoIndexConfig) panicIfNotValidAndSetDefaultMetricType(mgr *config.Manager) {
	p.panicIfNotValidAndSetDefaultMetricTypeHelper(p.IndexParams.Key, p.IndexParams.GetAsJSONMap(), schemapb.DataType_FloatVector, mgr)
	p.panicIfNotValidAndSetDefaultMetricTypeHelper(p.BinaryIndexParams.Key, p.BinaryIndexParams.GetAsJSONMap(), schemapb.DataType_BinaryVector, mgr)
	p.panicIfNotValidAndSetDefaultMetricTypeHelper(p.SparseIndexParams.Key, p.SparseIndexParams.GetAsJSONMap(), schemapb.DataType_SparseFloatVector, mgr)
}

func (p *autoIndexConfig) panicIfNotValidAndSetDefaultMetricTypeHelper(key string, m map[string]string, dtype schemapb.DataType, mgr *config.Manager) {
	if m == nil {
		panic(fmt.Sprintf("%s invalid, should be json format", key))
	}

	indexType, ok := m[common.IndexTypeKey]
	if !ok {
		panic(fmt.Sprintf("%s invalid, index type not found", key))
	}

	checker, err := indexparamcheck.GetIndexCheckerMgrInstance().GetChecker(indexType)
	if err != nil {
		panic(fmt.Sprintf("%s invalid, unsupported index type: %s", key, indexType))
	}

	checker.SetDefaultMetricTypeIfNotExist(m, dtype)

	if err := checker.StaticCheck(m); err != nil {
		panic(fmt.Sprintf("%s invalid, parameters invalid, error: %s", key, err.Error()))
	}

	p.reset(key, m, mgr)
}

func (p *autoIndexConfig) reset(key string, m map[string]string, mgr *config.Manager) {
	j := funcutil.MapToJSON(m)
	mgr.SetConfig(key, string(j))
}