test/gf
1
0
Fork 0
mirror of https://gitee.com/johng/gf.git synced 2024-11-30 03:07:45 +08:00
Code Issues Actions 15 Packages Projects Releases Wiki Activity

improve rotation feature for package glog

Browse Source
This commit is contained in:
John 2020-03-26 20:58:57 +08:00
parent 4258a3bbc9
commit ba56eb87b1
6 changed files with 91 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
.example/other/test.go
View File

@ -1,20 +1,14 @@
package main
import (
"os"
"time"
"fmt"
"github.com/gogf/gf/os/gfile"
"github.com/gogf/gf/text/gregex"
)
func main() {
file, err := os.Create("/tmp/testfile")
if err != nil {
panic(err)
}
for {
_, err = file.Write([]byte("test\n"))
if err != nil {
panic(err)
}
time.Sleep(5 * time.Second)
}
fmt.Println(gfile.Basename("/tmp/1585227151172826000/access.20200326205231173924.log"))
fmt.Println(
gregex.IsMatchString(`.+\.\d{20}\.log`,
gfile.Basename("/tmp/1585227151172826000/access.20200326205231173924.log")))
}

3
os/gfile/gfile.go
View File

@ -177,11 +177,12 @@ func Stat(path string) (os.FileInfo, error) {
}
// Move renames (moves) <src> to <dst> path.
// If <dst> already exists and is not a directory, it'll be replaced.
func Move(src string, dst string) error {
return os.Rename(src, dst)
}
// Alias of Move.
// Rename is alias of Move.
// See Move.
func Rename(src string, dst string) error {
return Move(src, dst)

6
os/glog/glog_logger.go
View File

@ -57,9 +57,9 @@ func New() *Logger {
logger := &Logger{
config: DefaultConfig(),
}
// Initialize the internal handler after one second.
gtimer.AddOnce(time.Second, func() {
gtimer.AddOnce(logger.config.RotateInterval, logger.rotateChecksTimely)
// Initialize the internal handler after some delay.
gtimer.AddOnce(500*time.Millisecond, func() {
gtimer.AddOnce(logger.config.RotateCheckInterval, logger.rotateChecksTimely)
})
return logger
}

52
os/glog/glog_logger_config.go
View File

@ -20,37 +20,37 @@ import (
// Config is the configuration object for logger.
type Config struct {
Writer io.Writer // Customized io.Writer.
Flags int // Extra flags for logging output features.
Path string // Logging directory path.
File string // Format for logging file.
Level int // Output level.
Prefix string // Prefix string for every logging content.
StSkip int // Skip count for stack.
StStatus int // Stack status(1: enabled - default; 0: disabled)
StFilter string // Stack string filter.
HeaderPrint bool `c:"header"` // Print header or not(true in default).
StdoutPrint bool `c:"stdout"` // Output to stdout or not(true in default).
LevelPrefixes map[int]string // Logging level to its prefix string mapping.
RotateSize int64 // Rotate the logging file if its size > 0 in bytes.
RotateExpire time.Duration // Rotate the logging file if its mtime exceeds this duration.
RotateBackLimit int // Max backup for rotated files, default is 0, means no backups.
RotateBackExpire time.Duration // Max expire for rotated files, which is 0 in default, means no expiration.
RotateBackCompress int // Compress level for rotated files using gzip algorithm. It's 0 in default, means no compression.
RotateInterval time.Duration // Asynchronizely checks the backups and expiration at intervals. It's 1 hour in default.
Writer io.Writer // Customized io.Writer.
Flags int // Extra flags for logging output features.
Path string // Logging directory path.
File string // Format for logging file.
Level int // Output level.
Prefix string // Prefix string for every logging content.
StSkip int // Skip count for stack.
StStatus int // Stack status(1: enabled - default; 0: disabled)
StFilter string // Stack string filter.
HeaderPrint bool `c:"header"` // Print header or not(true in default).
StdoutPrint bool `c:"stdout"` // Output to stdout or not(true in default).
LevelPrefixes map[int]string // Logging level to its prefix string mapping.
RotateSize int64 // Rotate the logging file if its size > 0 in bytes.
RotateExpire time.Duration // Rotate the logging file if its mtime exceeds this duration.
RotateBackupLimit int // Max backup for rotated files, default is 0, means no backups.
RotateBackupExpire time.Duration // Max expire for rotated files, which is 0 in default, means no expiration.
RotateBackupCompress int // Compress level for rotated files using gzip algorithm. It's 0 in default, means no compression.
RotateCheckInterval time.Duration // Asynchronizely checks the backups and expiration at intervals. It's 1 hour in default.
}
// DefaultConfig returns the default configuration for logger.
func DefaultConfig() Config {
c := Config{
File: gDEFAULT_FILE_FORMAT,
Flags: F_TIME_STD,
Level: LEVEL_ALL,
StStatus: 1,
HeaderPrint: true,
StdoutPrint: true,
LevelPrefixes: make(map[int]string, len(defaultLevelPrefixes)),
RotateInterval: time.Hour,
File: gDEFAULT_FILE_FORMAT,
Flags: F_TIME_STD,
Level: LEVEL_ALL,
StStatus: 1,
HeaderPrint: true,
StdoutPrint: true,
LevelPrefixes: make(map[int]string, len(defaultLevelPrefixes)),
RotateCheckInterval: time.Hour,
}
for k, v := range defaultLevelPrefixes {
c.LevelPrefixes[k] = v

60
os/glog/glog_logger_rotate.go
View File

@ -34,7 +34,7 @@ func (l *Logger) rotateFileBySize(now time.Time) {
// doRotateFile rotates the given logging file.
func (l *Logger) doRotateFile(filePath string) error {
// No backups, it then just removes the current logging file.
if l.config.RotateBackLimit == 0 {
if l.config.RotateBackupLimit == 0 {
if err := gfile.Remove(filePath); err != nil {
return err
}
@ -45,16 +45,31 @@ func (l *Logger) doRotateFile(filePath string) error {
var (
dirPath = gfile.Dir(filePath)
fileName = gfile.Name(filePath)
fileExt = gfile.Ext(filePath)
fileExtName = gfile.ExtName(filePath)
newFilePath = ""
)
// Rename the logging file by adding extra time information to milliseconds, like:
// access.log -> access.20200102190000899.log
// access.20200102.log -> access.20200102.20200102190000899.log
newFilePath = gfile.Join(
dirPath,
fmt.Sprintf(`%s.%s%s`, fileName, gtime.Now().Format("YmdHisu"), fileExt),
)
// Rename the logging file by adding extra datetime information to microseconds, like:
// access.log -> access.20200326101301899002.log
// access.20200102.log -> access.20200102.20200326101301899002.log
for {
var (
now = gtime.Now()
micro = now.Microsecond() % 1000
)
for micro < 100 {
micro *= 10
}
newFilePath = gfile.Join(
dirPath,
fmt.Sprintf(
`%s.%s%d.%s`,
fileName, now.Format("YmdHisu"), micro, fileExtName,
),
)
if !gfile.Exists(newFilePath) {
break
}
}
if err := gfile.Rename(filePath, newFilePath); err != nil {
return err
}
@ -63,7 +78,7 @@ func (l *Logger) doRotateFile(filePath string) error {
// rotateChecksTimely timely checks the backups expiration and the compression.
func (l *Logger) rotateChecksTimely() {
defer gtimer.AddOnce(l.config.RotateInterval, l.rotateChecksTimely)
defer gtimer.AddOnce(l.config.RotateCheckInterval, l.rotateChecksTimely)
// Checks whether file rotation not enabled.
if l.config.RotateSize <= 0 && l.config.RotateExpire == 0 {
return
@ -110,16 +125,15 @@ func (l *Logger) rotateChecksTimely() {
// Rotated file compression.
// =============================================================
needCompressFileArray := garray.NewStrArray()
if l.config.RotateBackCompress > 0 {
if l.config.RotateBackupCompress > 0 {
for _, file := range files {
// Eg: access.20200102190000899.log.gz
// Eg: access.20200326101301899002.log.gz
if gfile.ExtName(file) == "gz" {
continue
}
// Eg:
// access.20200102190000899
// access.20200102190000899.log
if gregex.IsMatchString(`.+\.\d{14,}`, gfile.Name(file)) {
// access.20200326101301899002.log
if gregex.IsMatchString(`.+\.\d{20}\.log`, gfile.Basename(file)) {
needCompressFileArray.Append(file)
}
}
@ -148,9 +162,9 @@ func (l *Logger) rotateChecksTimely() {
backupFilesMap = make(map[string]*garray.SortedArray)
originalLoggingFilePath = ""
)
if l.config.RotateBackLimit > 0 || l.config.RotateBackExpire > 0 {
if l.config.RotateBackupLimit > 0 || l.config.RotateBackupExpire > 0 {
for _, file := range files {
originalLoggingFilePath, _ = gregex.ReplaceString(`\.\d{14,}`, "", file)
originalLoggingFilePath, _ = gregex.ReplaceString(`\.\d{20}`, "", file)
if backupFilesMap[originalLoggingFilePath] == nil {
backupFilesMap[originalLoggingFilePath] = garray.NewSortedArray(func(a, b interface{}) int {
// Sorted by rotated/backup file mtime.
@ -165,23 +179,23 @@ func (l *Logger) rotateChecksTimely() {
})
}
// Check if this file a rotated/backup file.
if gregex.IsMatchString(`.+\.\d{14,}`, gfile.Name(file)) {
if gregex.IsMatchString(`.+\.\d{20}\.log`, gfile.Basename(file)) {
backupFilesMap[originalLoggingFilePath].Add(file)
}
}
intlog.Printf(`calculated backup files map: %+v`, backupFilesMap)
for _, array := range backupFilesMap {
diff := array.Len() - l.config.RotateBackLimit
diff := array.Len() - l.config.RotateBackupLimit
for i := 0; i < diff; i++ {
path := array.PopLeft().(string)
intlog.Printf(`remove exceeded backup file: %s`, path)
intlog.Printf(`remove exceeded backup limit file: %s`, path)
if err := gfile.Remove(path); err != nil {
intlog.Print(err)
}
}
}
// Backup expiration checks.
if l.config.RotateBackExpire > 0 {
if l.config.RotateBackupExpire > 0 {
var (
mtime time.Time
subDuration time.Duration
@ -191,10 +205,10 @@ func (l *Logger) rotateChecksTimely() {
path := v.(string)
mtime = gfile.MTime(path)
subDuration = now.Sub(mtime)
if subDuration > l.config.RotateBackExpire {
if subDuration > l.config.RotateBackupExpire {
intlog.Printf(
`%v - %v = %v > %v, remove expired backup file: %s`,
now, mtime, subDuration, l.config.RotateBackExpire, path,
now, mtime, subDuration, l.config.RotateBackupExpire, path,
)
if err := gfile.Remove(path); err != nil {
intlog.Print(err)

32
os/glog/glog_z_unit_rotate_test.go
View File

@ -22,14 +22,14 @@ func Test_Rotate_Size(t *testing.T) {
l := glog.New()
p := gfile.Join(gfile.TempDir(), gtime.TimestampNanoStr())
err := l.SetConfigWithMap(g.Map{
"Path": p,
"File": "access.log",
"StdoutPrint": false,
"RotateSize": 10,
"RotateBackLimit": 2,
"RotateBackExpire": 5 * time.Second,
"RotateBackCompress": 9,
"RotateInterval": time.Second, // For unit testing only.
"Path": p,
"File": "access.log",
"StdoutPrint": false,
"RotateSize": 10,
"RotateBackupLimit": 2,
"RotateBackupExpire": 5 * time.Second,
"RotateBackupCompress": 9,
"RotateCheckInterval": time.Second, // For unit testing only.
})
t.Assert(err, nil)
defer gfile.Remove(p)
@ -60,14 +60,14 @@ func Test_Rotate_Expire(t *testing.T) {
l := glog.New()
p := gfile.Join(gfile.TempDir(), gtime.TimestampNanoStr())
err := l.SetConfigWithMap(g.Map{
"Path": p,
"File": "access.log",
"StdoutPrint": false,
"RotateExpire": time.Second,
"RotateBackLimit": 2,
"RotateBackExpire": 5 * time.Second,
"RotateBackCompress": 9,
"RotateInterval": time.Second, // For unit testing only.
"Path": p,
"File": "access.log",
"StdoutPrint": false,
"RotateExpire": time.Second,
"RotateBackupLimit": 2,
"RotateBackupExpire": 5 * time.Second,
"RotateBackupCompress": 9,
"RotateCheckInterval": time.Second, // For unit testing only.
})
t.Assert(err, nil)
defer gfile.Remove(p)