test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-03 12:29:36 +08:00
Code Issues Actions 13 Packages Projects Releases Wiki Activity

Fix spawn too many threads (#26293)

Browse Source 
- Low the thread pool cap
- Limit CGO calls concurrency

Signed-off-by: yah01 <yah2er0ne@outlook.com>
This commit is contained in:
yah01 2023-08-11 18:29:29 +08:00 committed by GitHub
parent a0198ce8ae
commit 48422dd4c5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 7 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.clang-tidy
View File

@ -18,7 +18,7 @@
# The checks defined here will be run and will display by default as warnings. # The checks defined here will be run and will display by default as warnings.
Checks: > Checks: >
-*, clang-diagnostic-*, -clang-diagnostic-error, -*, clang-diagnostic-*, -clang-diagnostic-error,
clang-analyzer-*, -clang-analyzer-alpha*, clang-analyzer-*,
google-*, -google-runtime-references, -google-readability-todo, google-*, -google-runtime-references, -google-readability-todo,
modernize-*, -modernize-use-trailing-return-type, -modernize-use-nodiscard, modernize-*, -modernize-use-trailing-return-type, -modernize-use-nodiscard,
performance-*, performance-*,

6
configs/milvus.yaml
View File

@ -453,9 +453,9 @@ common:
entityExpiration: -1 # Entity expiration in seconds, CAUTION make sure entityExpiration >= retentionDuration and -1 means never expire entityExpiration: -1 # Entity expiration in seconds, CAUTION make sure entityExpiration >= retentionDuration and -1 means never expire
indexSliceSize: 16 # MB indexSliceSize: 16 # MB
threadCoreCoefficient: threadCoreCoefficient:
highPriority: 100 # This parameter specify how many times the number of threads is the number of cores in high priority thread pool highPriority: 10 # This parameter specify how many times the number of threads is the number of cores in high priority thread pool
middlePriority: 50 # This parameter specify how many times the number of threads is the number of cores in middle priority thread pool middlePriority: 5 # This parameter specify how many times the number of threads is the number of cores in middle priority thread pool
lowPriority: 10 # This parameter specify how many times the number of threads is the number of cores in low priority thread pool lowPriority: 1 # This parameter specify how many times the number of threads is the number of cores in low priority thread pool
DiskIndex: DiskIndex:
MaxDegree: 56 MaxDegree: 56
SearchListSize: 100 SearchListSize: 100

6
internal/querynodev2/segments/segment_loader.go
View File

@ -301,6 +301,9 @@ func (loader *segmentLoader) notifyLoadFinish(segments ...*querypb.SegmentLoadIn
func (loader *segmentLoader) requestResource(ctx context.Context, infos ...*querypb.SegmentLoadInfo) (LoadResource, int, error) { func (loader *segmentLoader) requestResource(ctx context.Context, infos ...*querypb.SegmentLoadInfo) (LoadResource, int, error) {
resource := LoadResource{} resource := LoadResource{}
loader.mut.Lock()
defer loader.mut.Unlock()
memoryUsage := hardware.GetUsedMemoryCount() memoryUsage := hardware.GetUsedMemoryCount()
totalMemory := hardware.GetMemoryCount() totalMemory := hardware.GetMemoryCount()
@ -310,9 +313,6 @@ func (loader *segmentLoader) requestResource(ctx context.Context, infos ...*quer
} }
diskCap := paramtable.Get().QueryNodeCfg.DiskCapacityLimit.GetAsUint64() diskCap := paramtable.Get().QueryNodeCfg.DiskCapacityLimit.GetAsUint64()
loader.mut.Lock()
defer loader.mut.Unlock()
poolCap := runtime.NumCPU() * paramtable.Get().CommonCfg.HighPriorityThreadCoreCoefficient.GetAsInt() poolCap := runtime.NumCPU() * paramtable.Get().CommonCfg.HighPriorityThreadCoreCoefficient.GetAsInt()
if loader.committedResource.WorkNum >= poolCap { if loader.committedResource.WorkNum >= poolCap {
return resource, 0, merr.WrapErrServiceRequestLimitExceeded(int32(poolCap)) return resource, 0, merr.WrapErrServiceRequestLimitExceeded(int32(poolCap))