Rainbond/event/manager.go

519 lines
12 KiB
Go
Raw Normal View History

2018-03-14 14:12:26 +08:00
// Copyright (C) 2014-2018 Goodrain Co., Ltd.
2017-11-07 11:40:44 +08:00
// RAINBOND, Application Management Platform
2018-03-14 14:33:31 +08:00
2017-11-07 11:40:44 +08:00
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version. For any non-GPL usage of Rainbond,
// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
// must be obtained first.
2018-03-14 14:33:31 +08:00
2017-11-07 11:40:44 +08:00
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
2018-03-14 14:33:31 +08:00
2017-11-07 11:40:44 +08:00
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package event
import (
"fmt"
2018-09-02 16:36:28 +08:00
"io"
"os"
2020-08-26 18:41:46 +08:00
"strings"
2017-11-07 11:40:44 +08:00
"sync"
"sync/atomic"
"time"
2020-09-06 11:09:48 +08:00
"github.com/sirupsen/logrus"
"github.com/goodrain/rainbond/discover"
"github.com/goodrain/rainbond/discover/config"
eventclient "github.com/goodrain/rainbond/eventlog/entry/grpc/client"
eventpb "github.com/goodrain/rainbond/eventlog/entry/grpc/pb"
2019-08-25 22:19:27 +08:00
"github.com/goodrain/rainbond/util"
2019-12-24 22:41:19 +08:00
etcdutil "github.com/goodrain/rainbond/util/etcd"
2019-08-25 22:19:27 +08:00
"github.com/pquerna/ffjson/ffjson"
2017-11-07 11:40:44 +08:00
"golang.org/x/net/context"
)
//Manager 操作日志,客户端服务
//客户端负载均衡
type Manager interface {
GetLogger(eventID string) Logger
Start() error
Close() error
ReleaseLogger(Logger)
}
2019-08-16 19:44:18 +08:00
// EventConfig event config struct
2017-11-07 11:40:44 +08:00
type EventConfig struct {
EventLogServers []string
2019-12-24 22:41:19 +08:00
DiscoverArgs *etcdutil.ClientArgs
2017-11-07 11:40:44 +08:00
}
type manager struct {
ctx context.Context
cancel context.CancelFunc
config EventConfig
qos int32
loggers map[string]Logger
handles map[string]handle
lock sync.Mutex
eventServer []string
abnormalServer map[string]string
dis discover.Discover
}
var defaultManager Manager
const (
//REQUESTTIMEOUT time out
REQUESTTIMEOUT = 1000 * time.Millisecond
//MAXRETRIES 重试
MAXRETRIES = 3 // Before we abandon
2018-08-31 16:55:10 +08:00
buffersize = 1000
2017-11-07 11:40:44 +08:00
)
//NewManager 创建manager
func NewManager(conf EventConfig) error {
2019-12-24 22:41:19 +08:00
dis, err := discover.GetDiscover(config.DiscoverConfig{EtcdClientArgs: conf.DiscoverArgs})
2017-11-07 11:40:44 +08:00
if err != nil {
logrus.Error("create discover manager error.", err.Error())
if len(conf.EventLogServers) < 1 {
return err
}
}
2017-12-20 10:23:55 +08:00
ctx, cancel := context.WithCancel(context.Background())
2017-11-07 11:40:44 +08:00
defaultManager = &manager{
ctx: ctx,
cancel: cancel,
config: conf,
loggers: make(map[string]Logger, 1024),
handles: make(map[string]handle),
eventServer: conf.EventLogServers,
dis: dis,
abnormalServer: make(map[string]string),
}
return defaultManager.Start()
}
//GetManager 获取日志服务
func GetManager() Manager {
return defaultManager
}
2019-08-29 14:15:46 +08:00
// NewTestManager -
func NewTestManager(m Manager) {
2019-08-25 22:19:27 +08:00
defaultManager = m
}
2017-11-07 11:40:44 +08:00
//CloseManager 关闭日志服务
func CloseManager() {
if defaultManager != nil {
defaultManager.Close()
}
2017-11-07 11:40:44 +08:00
}
func (m *manager) Start() error {
m.lock.Lock()
defer m.lock.Unlock()
for i := 0; i < len(m.eventServer); i++ {
h := handle{
2018-08-31 16:55:10 +08:00
cacheChan: make(chan []byte, buffersize),
2017-11-07 11:40:44 +08:00
stop: make(chan struct{}),
server: m.eventServer[i],
manager: m,
ctx: m.ctx,
}
m.handles[m.eventServer[i]] = h
go h.HandleLog()
}
2017-12-20 10:23:55 +08:00
if m.dis != nil {
m.dis.AddProject("event_log_event_grpc", m)
}
2017-11-07 11:40:44 +08:00
go m.GC()
return nil
}
func (m *manager) UpdateEndpoints(endpoints ...*config.Endpoint) {
m.lock.Lock()
defer m.lock.Unlock()
if endpoints == nil || len(endpoints) < 1 {
return
}
//清空不可用节点信息,以服务发现为主
m.abnormalServer = make(map[string]string)
//增加新节点
var new = make(map[string]string)
for _, end := range endpoints {
new[end.URL] = end.URL
if _, ok := m.handles[end.URL]; !ok {
h := handle{
2018-08-31 16:55:10 +08:00
cacheChan: make(chan []byte, buffersize),
2017-11-07 11:40:44 +08:00
stop: make(chan struct{}),
server: end.URL,
manager: m,
ctx: m.ctx,
}
m.handles[end.URL] = h
2020-07-18 15:50:18 +08:00
logrus.Infof("Add event server endpoint,%s", end.URL)
2017-11-07 11:40:44 +08:00
go h.HandleLog()
}
}
//删除旧节点
for k := range m.handles {
if _, ok := new[k]; !ok {
delete(m.handles, k)
2020-07-18 15:50:18 +08:00
logrus.Infof("Remove event server endpoint,%s", k)
2017-11-07 11:40:44 +08:00
}
}
var eventServer []string
for k := range new {
eventServer = append(eventServer, k)
}
m.eventServer = eventServer
logrus.Debugf("update event handle core success,handle core count:%d, event server count:%d", len(m.handles), len(m.eventServer))
2017-11-07 11:40:44 +08:00
}
func (m *manager) Error(err error) {
}
func (m *manager) Close() error {
m.cancel()
if m.dis != nil {
m.dis.Stop()
}
2017-11-07 11:40:44 +08:00
return nil
}
func (m *manager) GC() {
util.IntermittentExec(m.ctx, func() {
m.lock.Lock()
defer m.lock.Unlock()
var needRelease []string
for k, l := range m.loggers {
//1min 未release ,自动gc
if l.CreateTime().Add(time.Minute).Before(time.Now()) {
needRelease = append(needRelease, k)
}
}
if len(needRelease) > 0 {
for _, event := range needRelease {
logrus.Infof("start auto release event logger. %s", event)
delete(m.loggers, event)
}
}
}, time.Second*20)
}
//GetLogger
//使用完成后必须调用ReleaseLogger方法
func (m *manager) GetLogger(eventID string) Logger {
m.lock.Lock()
defer m.lock.Unlock()
if eventID == " " || len(eventID) == 0 {
eventID = "system"
}
if l, ok := m.loggers[eventID]; ok {
return l
}
2019-08-25 22:19:27 +08:00
l := NewLogger(eventID, m.getLBChan())
2017-11-07 11:40:44 +08:00
m.loggers[eventID] = l
return l
}
func (m *manager) ReleaseLogger(l Logger) {
m.lock.Lock()
defer m.lock.Unlock()
if l, ok := m.loggers[l.Event()]; ok {
delete(m.loggers, l.Event())
}
}
type handle struct {
server string
stop chan struct{}
cacheChan chan []byte
ctx context.Context
manager *manager
}
func (m *manager) DiscardedLoggerChan(cacheChan chan []byte) {
m.lock.Lock()
defer m.lock.Unlock()
for k, v := range m.handles {
if v.cacheChan == cacheChan {
logrus.Warnf("event server %s can not link, will ignore it.", k)
m.abnormalServer[k] = k
}
}
for _, v := range m.loggers {
if v.GetChan() == cacheChan {
v.SetChan(m.getLBChan())
}
}
}
func (m *manager) getLBChan() chan []byte {
for i := 0; i < len(m.eventServer); i++ {
index := m.qos % int32(len(m.eventServer))
m.qos = atomic.AddInt32(&(m.qos), 1)
server := m.eventServer[index]
if _, ok := m.abnormalServer[server]; ok {
2019-08-16 19:44:18 +08:00
logrus.Warnf("server[%s] is abnormal, skip it", server)
2017-11-07 11:40:44 +08:00
continue
}
if h, ok := m.handles[server]; ok {
return h.cacheChan
}
h := handle{
2018-08-31 16:55:10 +08:00
cacheChan: make(chan []byte, buffersize),
2017-11-07 11:40:44 +08:00
stop: make(chan struct{}),
server: server,
manager: m,
ctx: m.ctx,
}
m.handles[server] = h
go h.HandleLog()
return h.cacheChan
}
2018-08-31 16:55:10 +08:00
//not select, return first handle chan
2017-11-07 11:40:44 +08:00
for _, v := range m.handles {
return v.cacheChan
}
return nil
}
func (m *manager) RemoveHandle(server string) {
m.lock.Lock()
defer m.lock.Unlock()
if _, ok := m.handles[server]; ok {
delete(m.handles, server)
}
}
func (m *handle) HandleLog() error {
defer m.manager.RemoveHandle(m.server)
return util.Exec(m.ctx, func() error {
2018-01-31 17:42:34 +08:00
ctx, cancel := context.WithCancel(m.ctx)
defer cancel()
client, err := eventclient.NewEventClient(ctx, m.server)
2017-11-07 11:40:44 +08:00
if err != nil {
logrus.Error("create event client error.", err.Error())
return err
}
logrus.Infof("start a event log handle core. connect server %s", m.server)
2018-01-31 17:42:34 +08:00
logClient, err := client.Log(ctx)
2017-11-07 11:40:44 +08:00
if err != nil {
logrus.Error("create event log client error.", err.Error())
//切换使用此chan的logger到其他chan
m.manager.DiscardedLoggerChan(m.cacheChan)
return err
}
for {
select {
case <-m.ctx.Done():
logClient.CloseSend()
return nil
case <-m.stop:
logClient.CloseSend()
return nil
case me := <-m.cacheChan:
err := logClient.Send(&eventpb.LogMessage{Log: me})
if err != nil {
logrus.Error("send event log error.", err.Error())
logClient.CloseSend()
//切换使用此chan的logger到其他chan
m.manager.DiscardedLoggerChan(m.cacheChan)
return nil
}
}
}
}, time.Second*3)
}
func (m *handle) Stop() {
close(m.stop)
}
//Logger 日志发送器
type Logger interface {
Info(string, map[string]string)
Error(string, map[string]string)
Debug(string, map[string]string)
Event() string
CreateTime() time.Time
GetChan() chan []byte
SetChan(chan []byte)
2018-09-02 16:58:07 +08:00
GetWriter(step, level string) LoggerWriter
2017-11-07 11:40:44 +08:00
}
2019-08-25 22:19:27 +08:00
// NewLogger creates a new Logger.
func NewLogger(eventID string, sendCh chan []byte) Logger {
return &logger{
event: eventID,
sendChan: sendCh,
createTime: time.Now(),
}
}
2017-11-07 11:40:44 +08:00
type logger struct {
event string
sendChan chan []byte
createTime time.Time
}
func (l *logger) GetChan() chan []byte {
return l.sendChan
}
func (l *logger) SetChan(ch chan []byte) {
l.sendChan = ch
}
func (l *logger) Event() string {
return l.event
}
func (l *logger) CreateTime() time.Time {
return l.createTime
}
func (l *logger) Info(message string, info map[string]string) {
if info == nil {
info = make(map[string]string)
}
info["level"] = "info"
l.send(message, info)
}
func (l *logger) Error(message string, info map[string]string) {
if info == nil {
info = make(map[string]string)
}
info["level"] = "error"
l.send(message, info)
}
func (l *logger) Debug(message string, info map[string]string) {
if info == nil {
info = make(map[string]string)
}
info["level"] = "debug"
l.send(message, info)
}
func (l *logger) send(message string, info map[string]string) {
info["event_id"] = l.event
info["message"] = message
info["time"] = time.Now().Format(time.RFC3339)
log, err := ffjson.Marshal(info)
if err == nil && l.sendChan != nil {
util.SendNoBlocking(log, l.sendChan)
}
}
2018-09-02 16:58:07 +08:00
//LoggerWriter logger writer
type LoggerWriter interface {
io.Writer
2019-08-29 21:18:12 +08:00
SetFormat(map[string]interface{})
2018-09-02 16:58:07 +08:00
}
func (l *logger) GetWriter(step, level string) LoggerWriter {
2018-09-02 16:36:28 +08:00
return &loggerWriter{
l: l,
step: step,
level: level,
}
}
type loggerWriter struct {
2020-08-26 18:41:46 +08:00
l *logger
step string
level string
fmt map[string]interface{}
tmp []byte
lastMessage string
2018-09-02 16:36:28 +08:00
}
2019-08-29 21:18:12 +08:00
func (l *loggerWriter) SetFormat(f map[string]interface{}) {
2018-09-02 16:58:07 +08:00
l.fmt = f
}
2018-09-02 16:36:28 +08:00
func (l *loggerWriter) Write(b []byte) (n int, err error) {
if b != nil && len(b) > 0 {
2020-08-27 10:20:07 +08:00
if !strings.HasSuffix(string(b), "\n") {
2020-08-26 18:41:46 +08:00
l.tmp = append(l.tmp, b...)
return len(b), nil
}
2020-08-27 10:20:07 +08:00
var message string
if len(l.tmp) > 0 {
message = string(append(l.tmp, b...))
l.tmp = l.tmp[:0]
} else {
message = string(b)
}
2019-08-29 21:18:12 +08:00
// if loggerWriter has format, and then use it format message
if len(l.fmt) > 0 {
newLineMap := make(map[string]interface{}, len(l.fmt))
for k, v := range l.fmt {
if v == "%s" {
newLineMap[k] = fmt.Sprintf(v.(string), message)
} else {
newLineMap[k] = v
}
}
messageb, _ := ffjson.Marshal(newLineMap)
message = string(messageb)
2018-09-02 16:58:07 +08:00
}
2020-08-26 18:41:46 +08:00
if l.step == "build-progress" {
if strings.HasPrefix(message, "Progress ") && strings.HasPrefix(l.lastMessage, "Progress ") {
l.lastMessage = message
return len(b), nil
}
// send last message
if !strings.HasPrefix(message, "Progress ") && strings.HasPrefix(l.lastMessage, "Progress ") {
l.l.send(message, map[string]string{"step": l.lastMessage, "level": l.level})
}
}
2018-09-02 16:58:07 +08:00
l.l.send(message, map[string]string{"step": l.step, "level": l.level})
2020-08-26 18:41:46 +08:00
l.lastMessage = message
2018-09-02 16:36:28 +08:00
}
return len(b), nil
}
//GetTestLogger GetTestLogger
func GetTestLogger() Logger {
return &testLogger{}
}
type testLogger struct {
}
func (l *testLogger) GetChan() chan []byte {
return nil
}
func (l *testLogger) SetChan(ch chan []byte) {
}
func (l *testLogger) Event() string {
return "test"
}
func (l *testLogger) CreateTime() time.Time {
return time.Now()
}
func (l *testLogger) Info(message string, info map[string]string) {
fmt.Println("info:", message)
}
func (l *testLogger) Error(message string, info map[string]string) {
fmt.Println("error:", message)
}
func (l *testLogger) Debug(message string, info map[string]string) {
fmt.Println("debug:", message)
}
2018-09-02 16:58:07 +08:00
type testLoggerWriter struct {
}
2019-08-29 21:18:12 +08:00
func (l *testLoggerWriter) SetFormat(f map[string]interface{}) {
2018-09-02 16:58:07 +08:00
}
func (l *testLoggerWriter) Write(b []byte) (n int, err error) {
return os.Stdout.Write(b)
}
func (l *testLogger) GetWriter(step, level string) LoggerWriter {
return &testLoggerWriter{}
2018-09-02 16:36:28 +08:00
}