milvus/internal/log/zap_text_encoder.go
Zach 84d73bd618
add package name in log contents (#18251)
Signed-off-by: Zach41 <zongmei.zhang@zilliz.com>
2022-07-12 23:02:26 +08:00

635 lines
17 KiB
Go

// Copyright 2019 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
package log
import (
"encoding/base64"
"encoding/json"
"fmt"
"math"
"sync"
"time"
"unicode/utf8"
"go.uber.org/zap/buffer"
"go.uber.org/zap/zapcore"
)
// DefaultTimeEncoder serializes time.Time to a human-readable formatted string
func DefaultTimeEncoder(t time.Time, enc zapcore.PrimitiveArrayEncoder) {
s := t.Format("2006/01/02 15:04:05.000 -07:00")
if e, ok := enc.(*textEncoder); ok {
for _, c := range []byte(s) {
e.buf.AppendByte(c)
}
return
}
enc.AppendString(s)
}
// ShortCallerEncoder serializes a caller in file:line format.
func ShortCallerEncoder(caller zapcore.EntryCaller, enc zapcore.PrimitiveArrayEncoder) {
enc.AppendString(caller.TrimmedPath())
}
// For JSON-escaping; see textEncoder.safeAddString below.
const _hex = "0123456789abcdef"
var _textPool = sync.Pool{New: func() interface{} {
return &textEncoder{}
}}
var (
_pool = buffer.NewPool()
// Get retrieves a buffer from the pool, creating one if necessary.
Get = _pool.Get
)
func getTextEncoder() *textEncoder {
return _textPool.Get().(*textEncoder)
}
func putTextEncoder(enc *textEncoder) {
if enc.reflectBuf != nil {
enc.reflectBuf.Free()
}
enc.EncoderConfig = nil
enc.buf = nil
enc.spaced = false
enc.openNamespaces = 0
enc.reflectBuf = nil
enc.reflectEnc = nil
_textPool.Put(enc)
}
type textEncoder struct {
*zapcore.EncoderConfig
buf *buffer.Buffer
spaced bool // include spaces after colons and commas
openNamespaces int
disableErrorVerbose bool
// for encoding generic values by reflection
reflectBuf *buffer.Buffer
reflectEnc *json.Encoder
}
// NewTextEncoder creates a fast, low-allocation Text encoder. The encoder
// appropriately escapes all field keys and values.
func NewTextEncoder(cfg *Config) zapcore.Encoder {
cc := zapcore.EncoderConfig{
// Keys can be anything except the empty string.
TimeKey: "time",
LevelKey: "level",
NameKey: "name",
CallerKey: "caller",
MessageKey: "message",
StacktraceKey: "stack",
LineEnding: zapcore.DefaultLineEnding,
EncodeLevel: zapcore.CapitalLevelEncoder,
EncodeTime: DefaultTimeEncoder,
EncodeDuration: zapcore.StringDurationEncoder,
EncodeCaller: ShortCallerEncoder,
}
if cfg.DisableTimestamp {
cc.TimeKey = ""
}
switch cfg.Format {
case "text", "":
return &textEncoder{
EncoderConfig: &cc,
buf: _pool.Get(),
spaced: false,
disableErrorVerbose: cfg.DisableErrorVerbose,
}
case "json":
return zapcore.NewJSONEncoder(cc)
default:
panic(fmt.Sprintf("unsupport log format: %s", cfg.Format))
}
}
func (enc *textEncoder) AddArray(key string, arr zapcore.ArrayMarshaler) error {
enc.addKey(key)
return enc.AppendArray(arr)
}
func (enc *textEncoder) AddObject(key string, obj zapcore.ObjectMarshaler) error {
enc.addKey(key)
return enc.AppendObject(obj)
}
func (enc *textEncoder) AddBinary(key string, val []byte) {
enc.AddString(key, base64.StdEncoding.EncodeToString(val))
}
func (enc *textEncoder) AddByteString(key string, val []byte) {
enc.addKey(key)
enc.AppendByteString(val)
}
func (enc *textEncoder) AddBool(key string, val bool) {
enc.addKey(key)
enc.AppendBool(val)
}
func (enc *textEncoder) AddComplex128(key string, val complex128) {
enc.addKey(key)
enc.AppendComplex128(val)
}
func (enc *textEncoder) AddDuration(key string, val time.Duration) {
enc.addKey(key)
enc.AppendDuration(val)
}
func (enc *textEncoder) AddFloat64(key string, val float64) {
enc.addKey(key)
enc.AppendFloat64(val)
}
func (enc *textEncoder) AddInt64(key string, val int64) {
enc.addKey(key)
enc.AppendInt64(val)
}
func (enc *textEncoder) resetReflectBuf() {
if enc.reflectBuf == nil {
enc.reflectBuf = _pool.Get()
enc.reflectEnc = json.NewEncoder(enc.reflectBuf)
} else {
enc.reflectBuf.Reset()
}
}
func (enc *textEncoder) AddReflected(key string, obj interface{}) error {
enc.resetReflectBuf()
err := enc.reflectEnc.Encode(obj)
if err != nil {
return err
}
enc.reflectBuf.TrimNewline()
enc.addKey(key)
enc.AppendByteString(enc.reflectBuf.Bytes())
return nil
}
func (enc *textEncoder) OpenNamespace(key string) {
enc.addKey(key)
enc.buf.AppendByte('{')
enc.openNamespaces++
}
func (enc *textEncoder) AddString(key, val string) {
enc.addKey(key)
enc.AppendString(val)
}
func (enc *textEncoder) AddTime(key string, val time.Time) {
enc.addKey(key)
enc.AppendTime(val)
}
func (enc *textEncoder) AddUint64(key string, val uint64) {
enc.addKey(key)
enc.AppendUint64(val)
}
func (enc *textEncoder) AppendArray(arr zapcore.ArrayMarshaler) error {
enc.addElementSeparator()
ne := enc.cloned()
ne.buf.AppendByte('[')
err := arr.MarshalLogArray(ne)
ne.buf.AppendByte(']')
enc.AppendByteString(ne.buf.Bytes())
ne.buf.Free()
putTextEncoder(ne)
return err
}
func (enc *textEncoder) AppendObject(obj zapcore.ObjectMarshaler) error {
enc.addElementSeparator()
ne := enc.cloned()
ne.buf.AppendByte('{')
err := obj.MarshalLogObject(ne)
ne.buf.AppendByte('}')
enc.AppendByteString(ne.buf.Bytes())
ne.buf.Free()
putTextEncoder(ne)
return err
}
func (enc *textEncoder) AppendBool(val bool) {
enc.addElementSeparator()
enc.buf.AppendBool(val)
}
func (enc *textEncoder) AppendByteString(val []byte) {
enc.addElementSeparator()
if !enc.needDoubleQuotes(string(val)) {
enc.safeAddByteString(val)
return
}
enc.buf.AppendByte('"')
enc.safeAddByteString(val)
enc.buf.AppendByte('"')
}
func (enc *textEncoder) AppendComplex128(val complex128) {
enc.addElementSeparator()
// Cast to a platform-independent, fixed-size type.
r, i := real(val), imag(val)
enc.buf.AppendFloat(r, 64)
enc.buf.AppendByte('+')
enc.buf.AppendFloat(i, 64)
enc.buf.AppendByte('i')
}
func (enc *textEncoder) AppendDuration(val time.Duration) {
cur := enc.buf.Len()
enc.EncodeDuration(val, enc)
if cur == enc.buf.Len() {
// User-supplied EncodeDuration is a no-op. Fall back to nanoseconds to keep
// JSON valid.
enc.AppendInt64(int64(val))
}
}
func (enc *textEncoder) AppendInt64(val int64) {
enc.addElementSeparator()
enc.buf.AppendInt(val)
}
func (enc *textEncoder) AppendReflected(val interface{}) error {
enc.resetReflectBuf()
err := enc.reflectEnc.Encode(val)
if err != nil {
return err
}
enc.reflectBuf.TrimNewline()
enc.AppendByteString(enc.reflectBuf.Bytes())
return nil
}
func (enc *textEncoder) AppendString(val string) {
enc.addElementSeparator()
enc.safeAddStringWithQuote(val)
}
func (enc *textEncoder) AppendTime(val time.Time) {
cur := enc.buf.Len()
enc.EncodeTime(val, enc)
if cur == enc.buf.Len() {
// User-supplied EncodeTime is a no-op. Fall back to nanos since epoch to keep
// output JSON valid.
enc.AppendInt64(val.UnixNano())
}
}
func (enc *textEncoder) beginQuoteFiled() {
if enc.buf.Len() > 0 {
enc.buf.AppendByte(' ')
}
enc.buf.AppendByte('[')
}
func (enc *textEncoder) endQuoteFiled() {
enc.buf.AppendByte(']')
}
func (enc *textEncoder) AppendUint64(val uint64) {
enc.addElementSeparator()
enc.buf.AppendUint(val)
}
func (enc *textEncoder) AddComplex64(k string, v complex64) { enc.AddComplex128(k, complex128(v)) }
func (enc *textEncoder) AddFloat32(k string, v float32) { enc.AddFloat64(k, float64(v)) }
func (enc *textEncoder) AddInt(k string, v int) { enc.AddInt64(k, int64(v)) }
func (enc *textEncoder) AddInt32(k string, v int32) { enc.AddInt64(k, int64(v)) }
func (enc *textEncoder) AddInt16(k string, v int16) { enc.AddInt64(k, int64(v)) }
func (enc *textEncoder) AddInt8(k string, v int8) { enc.AddInt64(k, int64(v)) }
func (enc *textEncoder) AddUint(k string, v uint) { enc.AddUint64(k, uint64(v)) }
func (enc *textEncoder) AddUint32(k string, v uint32) { enc.AddUint64(k, uint64(v)) }
func (enc *textEncoder) AddUint16(k string, v uint16) { enc.AddUint64(k, uint64(v)) }
func (enc *textEncoder) AddUint8(k string, v uint8) { enc.AddUint64(k, uint64(v)) }
func (enc *textEncoder) AddUintptr(k string, v uintptr) { enc.AddUint64(k, uint64(v)) }
func (enc *textEncoder) AppendComplex64(v complex64) { enc.AppendComplex128(complex128(v)) }
func (enc *textEncoder) AppendFloat64(v float64) { enc.appendFloat(v, 64) }
func (enc *textEncoder) AppendFloat32(v float32) { enc.appendFloat(float64(v), 32) }
func (enc *textEncoder) AppendInt(v int) { enc.AppendInt64(int64(v)) }
func (enc *textEncoder) AppendInt32(v int32) { enc.AppendInt64(int64(v)) }
func (enc *textEncoder) AppendInt16(v int16) { enc.AppendInt64(int64(v)) }
func (enc *textEncoder) AppendInt8(v int8) { enc.AppendInt64(int64(v)) }
func (enc *textEncoder) AppendUint(v uint) { enc.AppendUint64(uint64(v)) }
func (enc *textEncoder) AppendUint32(v uint32) { enc.AppendUint64(uint64(v)) }
func (enc *textEncoder) AppendUint16(v uint16) { enc.AppendUint64(uint64(v)) }
func (enc *textEncoder) AppendUint8(v uint8) { enc.AppendUint64(uint64(v)) }
func (enc *textEncoder) AppendUintptr(v uintptr) { enc.AppendUint64(uint64(v)) }
func (enc *textEncoder) Clone() zapcore.Encoder {
clone := enc.cloned()
clone.buf.Write(enc.buf.Bytes())
return clone
}
func (enc *textEncoder) cloned() *textEncoder {
clone := getTextEncoder()
clone.EncoderConfig = enc.EncoderConfig
clone.spaced = enc.spaced
clone.openNamespaces = enc.openNamespaces
clone.disableErrorVerbose = enc.disableErrorVerbose
clone.buf = _pool.Get()
return clone
}
func (enc *textEncoder) EncodeEntry(ent zapcore.Entry, fields []zapcore.Field) (*buffer.Buffer, error) {
final := enc.cloned()
if final.TimeKey != "" {
final.beginQuoteFiled()
final.AppendTime(ent.Time)
final.endQuoteFiled()
}
if final.LevelKey != "" {
final.beginQuoteFiled()
cur := final.buf.Len()
final.EncodeLevel(ent.Level, final)
if cur == final.buf.Len() {
// User-supplied EncodeLevel was a no-op. Fall back to strings to keep
// output JSON valid.
final.AppendString(ent.Level.String())
}
final.endQuoteFiled()
}
if ent.LoggerName != "" && final.NameKey != "" {
final.beginQuoteFiled()
cur := final.buf.Len()
nameEncoder := final.EncodeName
// if no name encoder provided, fall back to FullNameEncoder for backwards
// compatibility
if nameEncoder == nil {
nameEncoder = zapcore.FullNameEncoder
}
nameEncoder(ent.LoggerName, final)
if cur == final.buf.Len() {
// User-supplied EncodeName was a no-op. Fall back to strings to
// keep output JSON valid.
final.AppendString(ent.LoggerName)
}
final.endQuoteFiled()
}
if ent.Caller.Defined && final.CallerKey != "" {
final.beginQuoteFiled()
cur := final.buf.Len()
final.EncodeCaller(ent.Caller, final)
if cur == final.buf.Len() {
// User-supplied EncodeCaller was a no-op. Fall back to strings to
// keep output JSON valid.
final.AppendString(ent.Caller.String())
}
final.endQuoteFiled()
}
// add Message
if len(ent.Message) > 0 {
final.beginQuoteFiled()
final.AppendString(ent.Message)
final.endQuoteFiled()
}
if enc.buf.Len() > 0 {
final.buf.AppendByte(' ')
final.buf.Write(enc.buf.Bytes())
}
final.addFields(fields)
final.closeOpenNamespaces()
if ent.Stack != "" && final.StacktraceKey != "" {
final.beginQuoteFiled()
final.AddString(final.StacktraceKey, ent.Stack)
final.endQuoteFiled()
}
if final.LineEnding != "" {
final.buf.AppendString(final.LineEnding)
} else {
final.buf.AppendString(zapcore.DefaultLineEnding)
}
ret := final.buf
putTextEncoder(final)
return ret, nil
}
func (enc *textEncoder) truncate() {
enc.buf.Reset()
}
func (enc *textEncoder) closeOpenNamespaces() {
for i := 0; i < enc.openNamespaces; i++ {
enc.buf.AppendByte('}')
}
}
func (enc *textEncoder) addKey(key string) {
enc.addElementSeparator()
enc.safeAddStringWithQuote(key)
enc.buf.AppendByte('=')
}
func (enc *textEncoder) addElementSeparator() {
last := enc.buf.Len() - 1
if last < 0 {
return
}
switch enc.buf.Bytes()[last] {
case '{', '[', ':', ',', ' ', '=':
return
default:
enc.buf.AppendByte(',')
}
}
func (enc *textEncoder) appendFloat(val float64, bitSize int) {
enc.addElementSeparator()
switch {
case math.IsNaN(val):
enc.buf.AppendString("NaN")
case math.IsInf(val, 1):
enc.buf.AppendString("+Inf")
case math.IsInf(val, -1):
enc.buf.AppendString("-Inf")
default:
enc.buf.AppendFloat(val, bitSize)
}
}
// safeAddString JSON-escapes a string and appends it to the internal buffer.
// Unlike the standard library's encoder, it doesn't attempt to protect the
// user from browser vulnerabilities or JSONP-related problems.
func (enc *textEncoder) safeAddString(s string) {
for i := 0; i < len(s); {
if enc.tryAddRuneSelf(s[i]) {
i++
continue
}
r, size := utf8.DecodeRuneInString(s[i:])
if enc.tryAddRuneError(r, size) {
i++
continue
}
enc.buf.AppendString(s[i : i+size])
i += size
}
}
// safeAddStringWithQuote will automatically add quotoes.
func (enc *textEncoder) safeAddStringWithQuote(s string) {
if !enc.needDoubleQuotes(s) {
enc.safeAddString(s)
return
}
enc.buf.AppendByte('"')
enc.safeAddString(s)
enc.buf.AppendByte('"')
}
// safeAddByteString is no-alloc equivalent of safeAddString(string(s)) for s []byte.
func (enc *textEncoder) safeAddByteString(s []byte) {
for i := 0; i < len(s); {
if enc.tryAddRuneSelf(s[i]) {
i++
continue
}
r, size := utf8.DecodeRune(s[i:])
if enc.tryAddRuneError(r, size) {
i++
continue
}
enc.buf.Write(s[i : i+size])
i += size
}
}
// See [log-fileds](https://github.com/tikv/rfcs/blob/master/text/2018-12-19-unified-log-format.md#log-fields-section).
func (enc *textEncoder) needDoubleQuotes(s string) bool {
for i := 0; i < len(s); {
b := s[i]
if b <= 0x20 {
return true
}
switch b {
case '\\', '"', '[', ']', '=':
return true
}
i++
}
return false
}
// tryAddRuneSelf appends b if it is valid UTF-8 character represented in a single byte.
func (enc *textEncoder) tryAddRuneSelf(b byte) bool {
if b >= utf8.RuneSelf {
return false
}
if 0x20 <= b && b != '\\' && b != '"' {
enc.buf.AppendByte(b)
return true
}
switch b {
case '\\', '"':
enc.buf.AppendByte('\\')
enc.buf.AppendByte(b)
case '\n':
enc.buf.AppendByte('\\')
enc.buf.AppendByte('n')
case '\r':
enc.buf.AppendByte('\\')
enc.buf.AppendByte('r')
case '\t':
enc.buf.AppendByte('\\')
enc.buf.AppendByte('t')
default:
// Encode bytes < 0x20, except for the escape sequences above.
enc.buf.AppendString(`\u00`)
enc.buf.AppendByte(_hex[b>>4])
enc.buf.AppendByte(_hex[b&0xF])
}
return true
}
func (enc *textEncoder) tryAddRuneError(r rune, size int) bool {
if r == utf8.RuneError && size == 1 {
enc.buf.AppendString(`\ufffd`)
return true
}
return false
}
func (enc *textEncoder) addFields(fields []zapcore.Field) {
for _, f := range fields {
if f.Type == zapcore.ErrorType {
// handle ErrorType in pingcap/log to fix "[key=?,keyVerbose=?]" problem.
// see more detail at https://github.com/pingcap/log/pull/5
enc.encodeError(f)
continue
}
enc.beginQuoteFiled()
f.AddTo(enc)
enc.endQuoteFiled()
}
}
func (enc *textEncoder) encodeError(f zapcore.Field) {
err := f.Interface.(error)
basic := err.Error()
enc.beginQuoteFiled()
enc.AddString(f.Key, basic)
enc.endQuoteFiled()
if enc.disableErrorVerbose {
return
}
if e, isFormatter := err.(fmt.Formatter); isFormatter {
verbose := fmt.Sprintf("%+v", e)
if verbose != basic {
// This is a rich error type, like those produced by
// errors.
enc.beginQuoteFiled()
enc.AddString(f.Key+"Verbose", verbose)
enc.endQuoteFiled()
}
}
}