Move segcore chunk_size configuration to querynode.yaml (#7913)

Signed-off-by: yudong.cai <yudong.cai@zilliz.com>

configs/advanced/query_node.yaml

@@ -25,3 +25,6 @@ queryNode:
 
     searchResult:
       recvBufSize: 64
+
+  segcore:
+    chunkSize: 32768  # 32M

configs/advanced/segcore.yaml

@@ -1,14 +0,0 @@
-# Copyright (C) 2019-2020 Zilliz. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software distributed under the License
-# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-# or implied. See the License for the specific language governing permissions and limitations under the License.
-
-segcore:
-  chunk_size: 32768 # 32M
-

internal/core/src/index/knowhere/knowhere/archive/KnowhereConfig.h

@@ -24,7 +24,7 @@ class KnowhereConfig {
      * set SIMD type
      */
     enum SimdType {
-        AUTO = 1,  // enable all and depend on the system
+        AUTO = 0,  // enable all and depend on the system
         SSE,       // only enable SSE
         AVX2,      // only enable AVX2
         AVX512,    // only enable AVX512

internal/core/src/segcore/SegcoreConfig.h

@@ -70,7 +70,7 @@ class SegcoreConfig {
     }
 
  private:
-    int64_t size_per_chunk_ = 32768;
+    int64_t size_per_chunk_ = 32 * 1024;
     std::map<MetricType, SmallIndexConf> table_;
 };
 

internal/core/src/segcore/segcore_init_c.cpp

@@ -19,7 +19,7 @@
 
 namespace milvus::segcore {
 static void
-SegcoreInitImpl(const char* config_dir) {
+SegcoreInitImpl() {
     namespace eg = milvus::engine;
     eg::KnowhereConfig::SetSimdType(eg::KnowhereConfig::SimdType::AUTO);
     eg::KnowhereConfig::SetBlasThreshold(16384);
@@ -28,23 +28,17 @@ SegcoreInitImpl(const char* config_dir) {
     eg::KnowhereConfig::SetStatisticsLevel(0);
     el::Configurations el_conf;
     el_conf.setGlobally(el::ConfigurationType::Enabled, std::to_string(false));
-
-    // initializing segcore config
-    try {
-        SegcoreConfig& config = SegcoreConfig::default_config();
-        if (config_dir != NULL) {
-            std::string config_file = std::string(config_dir) + "advanced/segcore.yaml";
-            config.parse_from(config_file);
-            std::cout << "Parse config file: " << config_file << ", chunk_size: " << config.get_size_per_chunk()
-                      << std::endl;
-        }
-    } catch (std::exception& e) {
-        PanicInfo("parse config fail: " + std::string(e.what()));
-    }
 }
 }  // namespace milvus::segcore
 
 extern "C" void
-SegcoreInit(const char* config_dir) {
-    milvus::segcore::SegcoreInitImpl(config_dir);
+SegcoreInit() {
+    milvus::segcore::SegcoreInitImpl();
+}
+
+extern "C" void
+SegcoreSetChunkSize(const int64_t value) {
+    milvus::segcore::SegcoreConfig& config = milvus::segcore::SegcoreConfig::default_config();
+    config.set_size_per_chunk(value);
+    std::cout << "set config chunk_size: " << config.get_size_per_chunk() << std::endl;
 }

internal/core/src/segcore/segcore_init_c.h

@@ -16,7 +16,10 @@ extern "C" {
 #endif
 
 void
-SegcoreInit(const char* config_dir);
+SegcoreInit();
+
+void
+SegcoreSetChunkSize(const int64_t);
 
 #ifdef __cplusplus
 }

internal/core/unittest/test_init.cpp

@@ -19,5 +19,6 @@
 TEST(Init, Naive) {
     using namespace milvus;
     using namespace milvus::segcore;
-    SegcoreInit(NULL);
+    SegcoreInit();
+    SegcoreSetChunkSize(32768);
 }

internal/querynode/param_table.go

@@ -68,6 +68,9 @@ type ParamTable struct {
 	MsgChannelSubName string
 	SliceIndex        int
 
+	// segcore
+	ChunkSize int64
+
 	Log log.Config
 }
 
@@ -111,6 +114,8 @@ func (p *ParamTable) Init() {
 		p.initStatsPublishInterval()
 		p.initStatsChannelName()
 
+		p.initSegcoreChunkSize()
+
 		p.initLogCfg()
 	})
 }
@@ -254,6 +259,10 @@ func (p *ParamTable) initStatsChannelName() {
 	p.StatsChannelName = channels
 }
 
+func (p *ParamTable) initSegcoreChunkSize() {
+	p.ChunkSize = p.ParseInt64("queryNode.segcore.chunkSize")
+}
+
 func (p *ParamTable) initLogCfg() {
 	p.Log = log.Config{}
 	format, err := p.Load("log.format")

internal/querynode/query_node.go

@@ -29,7 +29,6 @@ import (
 	"errors"
 	"strconv"
 	"sync/atomic"
-	"unsafe"
 
 	"go.uber.org/zap"
 
@@ -99,6 +98,14 @@ func (node *QueryNode) Register() error {
 	return nil
 }
 
+func (node *QueryNode) InitSegcore() {
+	C.SegcoreInit()
+
+	// override segcore chunk size
+	cChunkSize := C.int64_t(Params.ChunkSize)
+	C.SegcoreSetChunkSize(cChunkSize)
+}
+
 func (node *QueryNode) Init() error {
 	//ctx := context.Background()
 	connectEtcdFn := func() error {
@@ -124,9 +131,7 @@ func (node *QueryNode) Init() error {
 		node.etcdKV)
 	node.streaming = newStreaming(node.queryNodeLoopCtx, node.msFactory, node.etcdKV)
 
-	cConfigDir := C.CString(Params.BaseTable.GetConfigDir())
-	C.SegcoreInit(cConfigDir)
-	C.free(unsafe.Pointer(cConfigDir))
+	node.InitSegcore()
 
 	if node.rootCoord == nil {
 		log.Error("null root coordinator detected")