test/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2024-12-02 03:48:37 +08:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

enhance: Add metrics for segment index files size (#32979)

Browse Source 
issue:#32980

Signed-off-by: jaime <yun.zhang@zilliz.com>
This commit is contained in:
jaime 2024-05-13 17:59:32 +08:00 committed by GitHub
parent 293f14a8b9
commit ba625835bc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 73 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
internal/datacoord/index_builder_test.go
View File

@ -97,7 +97,7 @@ func createMetaTable(catalog metastore.DataCoordCatalog) *meta {
IsDeleted: false,
CreateTime: 0,
IndexFileKeys: nil,
IndexSize: 0,
IndexSize: 1,
},
},
segID + 1: {
@ -115,7 +115,7 @@ func createMetaTable(catalog metastore.DataCoordCatalog) *meta {
IsDeleted: false,
CreateTime: 1111,
IndexFileKeys: nil,
IndexSize: 0,
IndexSize: 1,
},
},
segID + 2: {
@ -133,7 +133,7 @@ func createMetaTable(catalog metastore.DataCoordCatalog) *meta {
IsDeleted: true,
CreateTime: 1111,
IndexFileKeys: nil,
IndexSize: 0,
IndexSize: 1,
},
},
segID + 3: {
@ -151,7 +151,7 @@ func createMetaTable(catalog metastore.DataCoordCatalog) *meta {
IsDeleted: false,
CreateTime: 1111,
IndexFileKeys: nil,
IndexSize: 0,
IndexSize: 1,
},
},
segID + 4: {
@ -169,7 +169,7 @@ func createMetaTable(catalog metastore.DataCoordCatalog) *meta {
IsDeleted: false,
CreateTime: 1111,
IndexFileKeys: nil,
IndexSize: 0,
IndexSize: 1,
},
},
segID + 5: {
@ -187,7 +187,7 @@ func createMetaTable(catalog metastore.DataCoordCatalog) *meta {
IsDeleted: false,
CreateTime: 1111,
IndexFileKeys: nil,
IndexSize: 0,
IndexSize: 1,
},
},
segID + 6: {
@ -205,7 +205,7 @@ func createMetaTable(catalog metastore.DataCoordCatalog) *meta {
IsDeleted: false,
CreateTime: 1111,
IndexFileKeys: nil,
IndexSize: 0,
IndexSize: 1,
},
},
segID + 7: {
@ -223,7 +223,7 @@ func createMetaTable(catalog metastore.DataCoordCatalog) *meta {
IsDeleted: false,
CreateTime: 1111,
IndexFileKeys: nil,
IndexSize: 0,
IndexSize: 1,
},
},
segID + 8: {
@ -241,7 +241,7 @@ func createMetaTable(catalog metastore.DataCoordCatalog) *meta {
IsDeleted: false,
CreateTime: 1111,
IndexFileKeys: nil,
IndexSize: 0,
IndexSize: 1,
},
},
segID + 9: {
@ -259,7 +259,7 @@ func createMetaTable(catalog metastore.DataCoordCatalog) *meta {
IsDeleted: false,
CreateTime: 1111,
IndexFileKeys: nil,
IndexSize: 0,
IndexSize: 1,
},
},
segID + 10: {
@ -277,7 +277,7 @@ func createMetaTable(catalog metastore.DataCoordCatalog) *meta {
IsDeleted: false,
CreateTime: 1111,
IndexFileKeys: nil,
IndexSize: 0,
IndexSize: 1,
},
},
},
@ -296,7 +296,7 @@ func createMetaTable(catalog metastore.DataCoordCatalog) *meta {
IsDeleted: false,
CreateTime: 0,
IndexFileKeys: nil,
IndexSize: 0,
IndexSize: 1,
},
buildID + 1: {
SegmentID: segID + 1,
@ -312,7 +312,7 @@ func createMetaTable(catalog metastore.DataCoordCatalog) *meta {
IsDeleted: false,
CreateTime: 1111,
IndexFileKeys: nil,
IndexSize: 0,
IndexSize: 1,
},
buildID + 2: {
SegmentID: segID + 2,
@ -328,7 +328,7 @@ func createMetaTable(catalog metastore.DataCoordCatalog) *meta {
IsDeleted: true,
CreateTime: 1111,
IndexFileKeys: nil,
IndexSize: 0,
IndexSize: 1,
},
buildID + 3: {
SegmentID: segID + 3,
@ -344,7 +344,7 @@ func createMetaTable(catalog metastore.DataCoordCatalog) *meta {
IsDeleted: false,
CreateTime: 1111,
IndexFileKeys: nil,
IndexSize: 0,
IndexSize: 1,
},
buildID + 4: {
SegmentID: segID + 4,
@ -360,7 +360,7 @@ func createMetaTable(catalog metastore.DataCoordCatalog) *meta {
IsDeleted: false,
CreateTime: 1111,
IndexFileKeys: nil,
IndexSize: 0,
IndexSize: 1,
},
buildID + 5: {
SegmentID: segID + 5,
@ -376,7 +376,7 @@ func createMetaTable(catalog metastore.DataCoordCatalog) *meta {
IsDeleted: false,
CreateTime: 1111,
IndexFileKeys: nil,
IndexSize: 0,
IndexSize: 1,
},
buildID + 6: {
SegmentID: segID + 6,
@ -392,7 +392,7 @@ func createMetaTable(catalog metastore.DataCoordCatalog) *meta {
IsDeleted: false,
CreateTime: 1111,
IndexFileKeys: nil,
IndexSize: 0,
IndexSize: 1,
},
buildID + 7: {
SegmentID: segID + 7,
@ -408,7 +408,7 @@ func createMetaTable(catalog metastore.DataCoordCatalog) *meta {
IsDeleted: false,
CreateTime: 1111,
IndexFileKeys: nil,
IndexSize: 0,
IndexSize: 1,
},
buildID + 8: {
SegmentID: segID + 8,
@ -424,7 +424,7 @@ func createMetaTable(catalog metastore.DataCoordCatalog) *meta {
IsDeleted: false,
CreateTime: 1111,
IndexFileKeys: nil,
IndexSize: 0,
IndexSize: 1,
},
buildID + 9: {
SegmentID: segID + 9,
@ -440,7 +440,7 @@ func createMetaTable(catalog metastore.DataCoordCatalog) *meta {
IsDeleted: false,
CreateTime: 1111,
IndexFileKeys: nil,
IndexSize: 0,
IndexSize: 1,
},
buildID + 10: {
SegmentID: segID + 10,
@ -456,7 +456,7 @@ func createMetaTable(catalog metastore.DataCoordCatalog) *meta {
IsDeleted: false,
CreateTime: 1111,
IndexFileKeys: nil,
IndexSize: 0,
IndexSize: 1,
},
},
}

16
internal/datacoord/meta.go
View File

@ -319,6 +319,22 @@ func (m *meta) GetCollectionBinlogSize() (int64, map[UniqueID]int64, map[UniqueI
return total, collectionBinlogSize, partitionBinlogSize
}
// GetCollectionIndexFilesSize returns the total index files size of all segment for each collection.
func (m *meta) GetCollectionIndexFilesSize() uint64 {
var total uint64
for _, segmentIdx := range m.indexMeta.GetAllSegIndexes() {
coll, ok := m.collections[segmentIdx.CollectionID]
if ok {
metrics.DataCoordStoredIndexFilesSize.WithLabelValues(coll.DatabaseName,
fmt.Sprint(segmentIdx.CollectionID), fmt.Sprint(segmentIdx.SegmentID)).Set(float64(segmentIdx.IndexSize))
total += segmentIdx.IndexSize
} else {
log.Warn("not found database name", zap.Int64("collectionID", segmentIdx.CollectionID))
}
}
return total
}
func (m *meta) GetAllCollectionNumRows() map[int64]int64 {
m.RLock()
defer m.RUnlock()

15
internal/datacoord/meta_test.go
View File

@ -615,6 +615,21 @@ func TestMeta_Basic(t *testing.T) {
assert.Equal(t, int64(size0+size1), total)
})
t.Run("Test GetCollectionBinlogSize", func(t *testing.T) {
meta := createMetaTable(&datacoord.Catalog{})
ret := meta.GetCollectionIndexFilesSize()
assert.Equal(t, uint64(0), ret)
meta.collections = map[UniqueID]*collectionInfo{
100: {
ID: 100,
DatabaseName: "db",
},
}
ret = meta.GetCollectionIndexFilesSize()
assert.Equal(t, uint64(11), ret)
})
t.Run("Test AddAllocation", func(t *testing.T) {
meta, _ := newMemoryMeta()
err := meta.AddAllocation(1, &Allocation{

2
internal/datacoord/metrics_info.go
View File

@ -38,6 +38,8 @@ import (
// getQuotaMetrics returns DataCoordQuotaMetrics.
func (s *Server) getQuotaMetrics() *metricsinfo.DataCoordQuotaMetrics {
total, colSizes, partSizes := s.meta.GetCollectionBinlogSize()
// Just generate the metrics data regularly
_ = s.meta.GetCollectionIndexFilesSize()
return &metricsinfo.DataCoordQuotaMetrics{
TotalBinlogSize: total,
CollectionBinlogSize: colSizes,

20
pkg/metrics/datacoord_metrics.go
View File

@ -140,7 +140,6 @@ var (
collectionIDLabelName,
segmentIDLabelName,
})
DataCoordSegmentBinLogFileCount = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: milvusNamespace,
@ -152,6 +151,18 @@ var (
segmentIDLabelName,
})
DataCoordStoredIndexFilesSize = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: milvusNamespace,
Subsystem: typeutil.DataCoordRole,
Name: "stored_index_files_size",
Help: "index files size of the segments",
}, []string{
databaseLabelName,
collectionIDLabelName,
segmentIDLabelName,
})
DataCoordDmlChannelNum = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: milvusNamespace,
@ -295,6 +306,7 @@ func RegisterDataCoord(registry *prometheus.Registry) {
registry.MustRegister(DataCoordConsumeDataNodeTimeTickLag)
registry.MustRegister(DataCoordCheckpointUnixSeconds)
registry.MustRegister(DataCoordStoredBinlogSize)
registry.MustRegister(DataCoordStoredIndexFilesSize)
registry.MustRegister(DataCoordSegmentBinLogFileCount)
registry.MustRegister(DataCoordDmlChannelNum)
registry.MustRegister(DataCoordCompactedSegmentSize)
@ -317,7 +329,11 @@ func CleanupDataCoordSegmentMetrics(collectionID int64, segmentID int64) {
collectionIDLabelName: fmt.Sprint(collectionID),
segmentIDLabelName: fmt.Sprint(segmentID),
})
DataCoordStoredBinlogSize.Delete(prometheus.Labels{
DataCoordStoredBinlogSize.DeletePartialMatch(prometheus.Labels{
collectionIDLabelName: fmt.Sprint(collectionID),
segmentIDLabelName: fmt.Sprint(segmentID),
})
DataCoordStoredIndexFilesSize.DeletePartialMatch(prometheus.Labels{
collectionIDLabelName: fmt.Sprint(collectionID),
segmentIDLabelName: fmt.Sprint(segmentID),
})