test/gf
1
0
Fork 0
mirror of https://gitee.com/johng/gf.git synced 2024-11-30 11:18:02 +08:00
Code Issues Actions 15 Packages Projects Releases Wiki Activity

improve rotation feature for package glog

Browse Source
This commit is contained in:
John 2020-03-26 20:58:57 +08:00
parent 4258a3bbc9
commit ba56eb87b1
6 changed files with 91 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
.example/other/test.go
View File

@ -1,20 +1,14 @@
package main package main
import ( import (
"os" "fmt"
"time" "github.com/gogf/gf/os/gfile"
"github.com/gogf/gf/text/gregex"
) )
func main() { func main() {
file, err := os.Create("/tmp/testfile") fmt.Println(gfile.Basename("/tmp/1585227151172826000/access.20200326205231173924.log"))
if err != nil { fmt.Println(
panic(err) gregex.IsMatchString(`.+\.\d{20}\.log`,
} gfile.Basename("/tmp/1585227151172826000/access.20200326205231173924.log")))
for {
_, err = file.Write([]byte("test\n"))
if err != nil {
panic(err)
}
time.Sleep(5 * time.Second)
}
} }

3
os/gfile/gfile.go
View File

@ -177,11 +177,12 @@ func Stat(path string) (os.FileInfo, error) {
} }
// Move renames (moves) <src> to <dst> path. // Move renames (moves) <src> to <dst> path.
// If <dst> already exists and is not a directory, it'll be replaced.
func Move(src string, dst string) error { func Move(src string, dst string) error {
return os.Rename(src, dst) return os.Rename(src, dst)
} }
// Alias of Move. // Rename is alias of Move.
// See Move. // See Move.
func Rename(src string, dst string) error { func Rename(src string, dst string) error {
return Move(src, dst) return Move(src, dst)

6
os/glog/glog_logger.go
View File

@ -57,9 +57,9 @@ func New() *Logger {
logger := &Logger{ logger := &Logger{
config: DefaultConfig(), config: DefaultConfig(),
} }
// Initialize the internal handler after one second. // Initialize the internal handler after some delay.
gtimer.AddOnce(time.Second, func() { gtimer.AddOnce(500*time.Millisecond, func() {
gtimer.AddOnce(logger.config.RotateInterval, logger.rotateChecksTimely) gtimer.AddOnce(logger.config.RotateCheckInterval, logger.rotateChecksTimely)
}) })
return logger return logger
} }

52
os/glog/glog_logger_config.go
View File

@ -20,37 +20,37 @@ import (
// Config is the configuration object for logger. // Config is the configuration object for logger.
type Config struct { type Config struct {
Writer io.Writer // Customized io.Writer. Writer io.Writer // Customized io.Writer.
Flags int // Extra flags for logging output features. Flags int // Extra flags for logging output features.
Path string // Logging directory path. Path string // Logging directory path.
File string // Format for logging file. File string // Format for logging file.
Level int // Output level. Level int // Output level.
Prefix string // Prefix string for every logging content. Prefix string // Prefix string for every logging content.
StSkip int // Skip count for stack. StSkip int // Skip count for stack.
StStatus int // Stack status(1: enabled - default; 0: disabled) StStatus int // Stack status(1: enabled - default; 0: disabled)
StFilter string // Stack string filter. StFilter string // Stack string filter.
HeaderPrint bool `c:"header"` // Print header or not(true in default). HeaderPrint bool `c:"header"` // Print header or not(true in default).
StdoutPrint bool `c:"stdout"` // Output to stdout or not(true in default). StdoutPrint bool `c:"stdout"` // Output to stdout or not(true in default).
LevelPrefixes map[int]string // Logging level to its prefix string mapping. LevelPrefixes map[int]string // Logging level to its prefix string mapping.
RotateSize int64 // Rotate the logging file if its size > 0 in bytes. RotateSize int64 // Rotate the logging file if its size > 0 in bytes.
RotateExpire time.Duration // Rotate the logging file if its mtime exceeds this duration. RotateExpire time.Duration // Rotate the logging file if its mtime exceeds this duration.
RotateBackLimit int // Max backup for rotated files, default is 0, means no backups. RotateBackupLimit int // Max backup for rotated files, default is 0, means no backups.
RotateBackExpire time.Duration // Max expire for rotated files, which is 0 in default, means no expiration. RotateBackupExpire time.Duration // Max expire for rotated files, which is 0 in default, means no expiration.
RotateBackCompress int // Compress level for rotated files using gzip algorithm. It's 0 in default, means no compression. RotateBackupCompress int // Compress level for rotated files using gzip algorithm. It's 0 in default, means no compression.
RotateInterval time.Duration // Asynchronizely checks the backups and expiration at intervals. It's 1 hour in default. RotateCheckInterval time.Duration // Asynchronizely checks the backups and expiration at intervals. It's 1 hour in default.
} }
// DefaultConfig returns the default configuration for logger. // DefaultConfig returns the default configuration for logger.
func DefaultConfig() Config { func DefaultConfig() Config {
c := Config{ c := Config{
File: gDEFAULT_FILE_FORMAT, File: gDEFAULT_FILE_FORMAT,
Flags: F_TIME_STD, Flags: F_TIME_STD,
Level: LEVEL_ALL, Level: LEVEL_ALL,
StStatus: 1, StStatus: 1,
HeaderPrint: true, HeaderPrint: true,
StdoutPrint: true, StdoutPrint: true,
LevelPrefixes: make(map[int]string, len(defaultLevelPrefixes)), LevelPrefixes: make(map[int]string, len(defaultLevelPrefixes)),
RotateInterval: time.Hour, RotateCheckInterval: time.Hour,
} }
for k, v := range defaultLevelPrefixes { for k, v := range defaultLevelPrefixes {
c.LevelPrefixes[k] = v c.LevelPrefixes[k] = v

60
os/glog/glog_logger_rotate.go
View File

@ -34,7 +34,7 @@ func (l *Logger) rotateFileBySize(now time.Time) {
// doRotateFile rotates the given logging file. // doRotateFile rotates the given logging file.
func (l *Logger) doRotateFile(filePath string) error { func (l *Logger) doRotateFile(filePath string) error {
// No backups, it then just removes the current logging file. // No backups, it then just removes the current logging file.
if l.config.RotateBackLimit == 0 { if l.config.RotateBackupLimit == 0 {
if err := gfile.Remove(filePath); err != nil { if err := gfile.Remove(filePath); err != nil {
return err return err
} }
@ -45,16 +45,31 @@ func (l *Logger) doRotateFile(filePath string) error {
var ( var (
dirPath = gfile.Dir(filePath) dirPath = gfile.Dir(filePath)
fileName = gfile.Name(filePath) fileName = gfile.Name(filePath)
fileExt = gfile.Ext(filePath) fileExtName = gfile.ExtName(filePath)
newFilePath = "" newFilePath = ""
) )
// Rename the logging file by adding extra time information to milliseconds, like: // Rename the logging file by adding extra datetime information to microseconds, like:
// access.log -> access.20200102190000899.log // access.log -> access.20200326101301899002.log
// access.20200102.log -> access.20200102.20200102190000899.log // access.20200102.log -> access.20200102.20200326101301899002.log
newFilePath = gfile.Join( for {
dirPath, var (
fmt.Sprintf(`%s.%s%s`, fileName, gtime.Now().Format("YmdHisu"), fileExt), now = gtime.Now()
) micro = now.Microsecond() % 1000
)
for micro < 100 {
micro *= 10
}
newFilePath = gfile.Join(
dirPath,
fmt.Sprintf(
`%s.%s%d.%s`,
fileName, now.Format("YmdHisu"), micro, fileExtName,
),
)
if !gfile.Exists(newFilePath) {
break
}
}
if err := gfile.Rename(filePath, newFilePath); err != nil { if err := gfile.Rename(filePath, newFilePath); err != nil {
return err return err
} }
@ -63,7 +78,7 @@ func (l *Logger) doRotateFile(filePath string) error {
// rotateChecksTimely timely checks the backups expiration and the compression. // rotateChecksTimely timely checks the backups expiration and the compression.
func (l *Logger) rotateChecksTimely() { func (l *Logger) rotateChecksTimely() {
defer gtimer.AddOnce(l.config.RotateInterval, l.rotateChecksTimely) defer gtimer.AddOnce(l.config.RotateCheckInterval, l.rotateChecksTimely)
// Checks whether file rotation not enabled. // Checks whether file rotation not enabled.
if l.config.RotateSize <= 0 && l.config.RotateExpire == 0 { if l.config.RotateSize <= 0 && l.config.RotateExpire == 0 {
return return
@ -110,16 +125,15 @@ func (l *Logger) rotateChecksTimely() {
// Rotated file compression. // Rotated file compression.
// ============================================================= // =============================================================
needCompressFileArray := garray.NewStrArray() needCompressFileArray := garray.NewStrArray()
if l.config.RotateBackCompress > 0 { if l.config.RotateBackupCompress > 0 {
for _, file := range files { for _, file := range files {
// Eg: access.20200102190000899.log.gz // Eg: access.20200326101301899002.log.gz
if gfile.ExtName(file) == "gz" { if gfile.ExtName(file) == "gz" {
continue continue
} }
// Eg: // Eg:
// access.20200102190000899 // access.20200326101301899002.log
// access.20200102190000899.log if gregex.IsMatchString(`.+\.\d{20}\.log`, gfile.Basename(file)) {
if gregex.IsMatchString(`.+\.\d{14,}`, gfile.Name(file)) {
needCompressFileArray.Append(file) needCompressFileArray.Append(file)
} }
} }
@ -148,9 +162,9 @@ func (l *Logger) rotateChecksTimely() {
backupFilesMap = make(map[string]*garray.SortedArray) backupFilesMap = make(map[string]*garray.SortedArray)
originalLoggingFilePath = "" originalLoggingFilePath = ""
) )
if l.config.RotateBackLimit > 0 || l.config.RotateBackExpire > 0 { if l.config.RotateBackupLimit > 0 || l.config.RotateBackupExpire > 0 {
for _, file := range files { for _, file := range files {
originalLoggingFilePath, _ = gregex.ReplaceString(`\.\d{14,}`, "", file) originalLoggingFilePath, _ = gregex.ReplaceString(`\.\d{20}`, "", file)
if backupFilesMap[originalLoggingFilePath] == nil { if backupFilesMap[originalLoggingFilePath] == nil {
backupFilesMap[originalLoggingFilePath] = garray.NewSortedArray(func(a, b interface{}) int { backupFilesMap[originalLoggingFilePath] = garray.NewSortedArray(func(a, b interface{}) int {
// Sorted by rotated/backup file mtime. // Sorted by rotated/backup file mtime.
@ -165,23 +179,23 @@ func (l *Logger) rotateChecksTimely() {
}) })
} }
// Check if this file a rotated/backup file. // Check if this file a rotated/backup file.
if gregex.IsMatchString(`.+\.\d{14,}`, gfile.Name(file)) { if gregex.IsMatchString(`.+\.\d{20}\.log`, gfile.Basename(file)) {
backupFilesMap[originalLoggingFilePath].Add(file) backupFilesMap[originalLoggingFilePath].Add(file)
} }
} }
intlog.Printf(`calculated backup files map: %+v`, backupFilesMap) intlog.Printf(`calculated backup files map: %+v`, backupFilesMap)
for _, array := range backupFilesMap { for _, array := range backupFilesMap {
diff := array.Len() - l.config.RotateBackLimit diff := array.Len() - l.config.RotateBackupLimit
for i := 0; i < diff; i++ { for i := 0; i < diff; i++ {
path := array.PopLeft().(string) path := array.PopLeft().(string)
intlog.Printf(`remove exceeded backup file: %s`, path) intlog.Printf(`remove exceeded backup limit file: %s`, path)
if err := gfile.Remove(path); err != nil { if err := gfile.Remove(path); err != nil {
intlog.Print(err) intlog.Print(err)
} }
} }
} }
// Backup expiration checks. // Backup expiration checks.
if l.config.RotateBackExpire > 0 { if l.config.RotateBackupExpire > 0 {
var ( var (
mtime time.Time mtime time.Time
subDuration time.Duration subDuration time.Duration
@ -191,10 +205,10 @@ func (l *Logger) rotateChecksTimely() {
path := v.(string) path := v.(string)
mtime = gfile.MTime(path) mtime = gfile.MTime(path)
subDuration = now.Sub(mtime) subDuration = now.Sub(mtime)
if subDuration > l.config.RotateBackExpire { if subDuration > l.config.RotateBackupExpire {
intlog.Printf( intlog.Printf(
`%v - %v = %v > %v, remove expired backup file: %s`, `%v - %v = %v > %v, remove expired backup file: %s`,
now, mtime, subDuration, l.config.RotateBackExpire, path, now, mtime, subDuration, l.config.RotateBackupExpire, path,
) )
if err := gfile.Remove(path); err != nil { if err := gfile.Remove(path); err != nil {
intlog.Print(err) intlog.Print(err)

32
os/glog/glog_z_unit_rotate_test.go
View File

@ -22,14 +22,14 @@ func Test_Rotate_Size(t *testing.T) {
l := glog.New() l := glog.New()
p := gfile.Join(gfile.TempDir(), gtime.TimestampNanoStr()) p := gfile.Join(gfile.TempDir(), gtime.TimestampNanoStr())
err := l.SetConfigWithMap(g.Map{ err := l.SetConfigWithMap(g.Map{
"Path": p, "Path": p,
"File": "access.log", "File": "access.log",
"StdoutPrint": false, "StdoutPrint": false,
"RotateSize": 10, "RotateSize": 10,
"RotateBackLimit": 2, "RotateBackupLimit": 2,
"RotateBackExpire": 5 * time.Second, "RotateBackupExpire": 5 * time.Second,
"RotateBackCompress": 9, "RotateBackupCompress": 9,
"RotateInterval": time.Second, // For unit testing only. "RotateCheckInterval": time.Second, // For unit testing only.
}) })
t.Assert(err, nil) t.Assert(err, nil)
defer gfile.Remove(p) defer gfile.Remove(p)
@ -60,14 +60,14 @@ func Test_Rotate_Expire(t *testing.T) {
l := glog.New() l := glog.New()
p := gfile.Join(gfile.TempDir(), gtime.TimestampNanoStr()) p := gfile.Join(gfile.TempDir(), gtime.TimestampNanoStr())
err := l.SetConfigWithMap(g.Map{ err := l.SetConfigWithMap(g.Map{
"Path": p, "Path": p,
"File": "access.log", "File": "access.log",
"StdoutPrint": false, "StdoutPrint": false,
"RotateExpire": time.Second, "RotateExpire": time.Second,
"RotateBackLimit": 2, "RotateBackupLimit": 2,
"RotateBackExpire": 5 * time.Second, "RotateBackupExpire": 5 * time.Second,
"RotateBackCompress": 9, "RotateBackupCompress": 9,
"RotateInterval": time.Second, // For unit testing only. "RotateCheckInterval": time.Second, // For unit testing only.
}) })
t.Assert(err, nil) t.Assert(err, nil)
defer gfile.Remove(p) defer gfile.Remove(p)