From f5d7f54a55ec60bcf5ad2c951316cfcdde5f1616 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E8=8C=B6=E6=B8=85=E6=AC=A2?= Date: Fri, 25 Apr 2025 18:34:24 +0800 Subject: [PATCH 1/6] =?UTF-8?q?=E6=94=AF=E6=8C=81ws=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E5=AE=9E=E7=8E=B0,=20=E5=BE=85=E7=B2=BE=E7=BB=86=E5=8C=96?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E6=8E=A7=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- instance/ws_log.go | 120 +++++++++++++++++++++++++++++++++++++++++++++ logger.go | 31 ++++-------- 2 files changed, 130 insertions(+), 21 deletions(-) create mode 100644 instance/ws_log.go diff --git a/instance/ws_log.go b/instance/ws_log.go new file mode 100644 index 0000000..9404e2d --- /dev/null +++ b/instance/ws_log.go @@ -0,0 +1,120 @@ +// Package instance ... +// +// Description : instance ... +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 2025-04-25 18:16 +package instance + +import ( + "git.zhangdeman.cn/zhangdeman/websocket" + "git.zhangdeman.cn/zhangdeman/websocket/abstract" + "git.zhangdeman.cn/zhangdeman/websocket/config" + "git.zhangdeman.cn/zhangdeman/websocket/context" + "git.zhangdeman.cn/zhangdeman/websocket/storage" + "github.com/gin-gonic/gin" + "io" +) + +// NewWebsocketLog 初始化长连接日志 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 18:12 2024/7/22 +func NewWebsocketLog(serverPort int, route *gin.Engine) (*WsLog, error) { + wsLog := &WsLog{} + if err := websocket.NewWebsocketServeWithGinRouter(serverPort, route, wsLog); nil != err { + return nil, err + } + wsServer, _ := websocket.GetWsServer(serverPort, wsLog.GetModuleFlag()) + // logger.SetWsLoggerConnect(wsServer.GetConfig().ConnectionManager) + return &WsLog{ + port: serverPort, + connectManager: wsServer.GetConfig().ConnectionManager, + }, nil +} + +// CloseWebsocketLog ... +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 13:27 2024/7/25 +func CloseWebsocketLog(wsLog *WsLog) { + if nil == wsLog { + return + } + wsServer, _ := websocket.GetWsServer(wsLog.port, wsLog.GetModuleFlag()) + allConnect := wsServer.GetConfig().ConnectionManager.GetCtxList() + for _, conn := range allConnect { + _ = conn.Session.Close() + } +} + +type WsLog struct { + port int + connectManager storage.IConnection +} + +func (e WsLog) Connect(ctx *context.WSContext) error { + return nil +} + +func (e WsLog) Disconnect(ctx *context.WSContext) { +} + +func (e WsLog) Close(ctx *context.WSContext, code int, message string) error { + return nil +} + +func (e WsLog) HandshakeURL() []string { + return []string{ + "/system/logger/trace", + } +} + +func (e WsLog) GetCommandList() []abstract.ICommand { + return []abstract.ICommand{} +} + +func (e WsLog) GetModuleFlag() string { + return "ws-log" +} + +func (e WsLog) GetServerPort() int { + return e.port +} + +func (e WsLog) GetWSServerConfig() []config.SetWSServerConfig { + return []config.SetWSServerConfig{ + config.SetPingPeriod(5), + config.SetPongWait(60), + config.SetWriteWait(60), + config.SetMaxMessageBufferSize(81920), + config.SetMaxMessageSize(81920), + } +} + +func (e WsLog) Writer() io.Writer { + return &wsWriter{ + connectionManager: e.connectManager, + } +} + +type wsWriter struct { + connectionManager storage.IConnection +} + +// Write ws的writer +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 22:24 2024/7/22 +func (w *wsWriter) Write(p []byte) (n int, err error) { + p = []byte(" " + string(p)) + allConnList := w.connectionManager.GetCtxList() + for _, conn := range allConnList { + _ = conn.Session.Write(p) + } + return 0, nil +} diff --git a/logger.go b/logger.go index 1de6473..cbefa13 100644 --- a/logger.go +++ b/logger.go @@ -9,6 +9,7 @@ package logger import ( "git.zhangdeman.cn/zhangdeman/consts" + "git.zhangdeman.cn/zhangdeman/logger/instance" "io" "os" @@ -60,7 +61,15 @@ func NewLogger(inputLoggerLevel consts.LogLevel, splitConfig *RotateLogConfig, o fileHandlerList := []zapcore.Core{ zapcore.NewCore(o.Encoder, zapcore.AddSync(loggerWriter), loggerLevelDeal), - zapcore.NewCore(o.Encoder, zapcore.AddSync(&wsWriter{}), loggerLevelDeal), // 设置ws日志输出 + // zapcore.NewCore(o.Encoder, zapcore.AddSync(&wsWriter{}), loggerLevelDeal), // 设置ws日志输出 + } + + // TODO: 通过配置控制ws logger + if wsLoggerInstance, err := instance.NewWebsocketLog(80, nil); nil != err { + return nil, err + } else { + // 设置ws日志输出 + fileHandlerList = append(fileHandlerList, zapcore.NewCore(o.Encoder, zapcore.AddSync(wsLoggerInstance.Writer()), loggerLevelDeal)) } // 设置控制台输出 @@ -138,23 +147,3 @@ func (l *Logger) getWriter() (io.Writer, error) { return hook, nil } - -type wsWriter struct { -} - -// Write ws的writer -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 22:24 2024/7/22 -func (w *wsWriter) Write(p []byte) (n int, err error) { - if nil == wsLoggerConnect { - return 0, nil - } - p = []byte(" " + string(p)) - allConnList := wsLoggerConnect.GetCtxList() - for _, conn := range allConnList { - _ = conn.Session.Write(p) - } - return 0, nil -} From 77bf0b2a0073115804d4d4f37db7d7571c46cdf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E8=8C=B6=E6=B8=85=E6=AC=A2?= Date: Fri, 25 Apr 2025 18:46:36 +0800 Subject: [PATCH 2/6] =?UTF-8?q?ws=20logger=E6=94=AF=E6=8C=81=E9=80=9A?= =?UTF-8?q?=E8=BF=87=E9=85=8D=E7=BD=AE=E6=8E=A7=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.go | 45 ++++++++++++++++++++++++++++++--------------- logger.go | 14 ++++++++------ 2 files changed, 38 insertions(+), 21 deletions(-) diff --git a/config.go b/config.go index f2e096b..198add3 100644 --- a/config.go +++ b/config.go @@ -9,14 +9,13 @@ package logger import ( "fmt" + "github.com/gin-gonic/gin" "os" "path/filepath" "strings" "time" "git.zhangdeman.cn/zhangdeman/consts" - "git.zhangdeman.cn/zhangdeman/websocket/storage" - "go.uber.org/zap/zapcore" ) @@ -224,19 +223,22 @@ func defaultEncodeDuration(d time.Duration, enc zapcore.PrimitiveArrayEncoder) { // // Date : 11:41 下午 2021/1/2 type OptionLogger struct { - UseJsonFormat bool // 日志使用json格式 - MessageKey string // message 字段 - LevelKey string // level 字段 - TimeKey string // 时间字段 - CallerKey string // 记录日志的文件的代码行数 - UseShortCaller bool // 使用短的调用文件格式 - TimeEncoder zapcore.TimeEncoder // 格式化时间的函数 - EncodeDuration zapcore.DurationEncoder // 原始时间信息 - WithCaller bool // 是否打印文件行号 - WithCallerSkip int // 跳过的调用数 - ConsoleOutput bool // 控制台输出 - Encoder zapcore.Encoder // 编码函数 - WsLoggerConnect storage.IConnection `json:"-"` // ws 日志连接管理实例 + UseJsonFormat bool // 日志使用json格式 + MessageKey string // message 字段 + LevelKey string // level 字段 + TimeKey string // 时间字段 + CallerKey string // 记录日志的文件的代码行数 + UseShortCaller bool // 使用短的调用文件格式 + TimeEncoder zapcore.TimeEncoder // 格式化时间的函数 + EncodeDuration zapcore.DurationEncoder // 原始时间信息 + WithCaller bool // 是否打印文件行号 + WithCallerSkip int // 跳过的调用数 + ConsoleOutput bool // 控制台输出 + Encoder zapcore.Encoder // 编码函数 + WsServerProt int // ws端口 + WsGroup string // ws分组 + GinRouter *gin.Engine // gin路由实例 + EnableWsLog bool // 启用wsLog } // SetLoggerOptionFunc 设置日志配置 @@ -417,3 +419,16 @@ func GetEncoder(option ...SetLoggerOptionFunc) zapcore.Encoder { } return zapcore.NewJSONEncoder(ec) } + +// EnableWsLogger 启用wsLogger +func EnableWsLogger(serverPort int, wsGroup string, router *gin.Engine) SetLoggerOptionFunc { + return func(o *OptionLogger) { + o.GinRouter = router + o.WsServerProt = serverPort + if len(wsGroup) == 0 { + wsGroup = "ws-log" + } + o.WsGroup = wsGroup + o.EnableWsLog = true + } +} diff --git a/logger.go b/logger.go index cbefa13..b2806b8 100644 --- a/logger.go +++ b/logger.go @@ -64,12 +64,14 @@ func NewLogger(inputLoggerLevel consts.LogLevel, splitConfig *RotateLogConfig, o // zapcore.NewCore(o.Encoder, zapcore.AddSync(&wsWriter{}), loggerLevelDeal), // 设置ws日志输出 } - // TODO: 通过配置控制ws logger - if wsLoggerInstance, err := instance.NewWebsocketLog(80, nil); nil != err { - return nil, err - } else { - // 设置ws日志输出 - fileHandlerList = append(fileHandlerList, zapcore.NewCore(o.Encoder, zapcore.AddSync(wsLoggerInstance.Writer()), loggerLevelDeal)) + // 通过配置控制ws logger + if o.EnableWsLog { + if wsLoggerInstance, err := instance.NewWebsocketLog(o.WsServerProt, o.GinRouter); nil != err { + return nil, err + } else { + // 设置ws日志输出 + fileHandlerList = append(fileHandlerList, zapcore.NewCore(o.Encoder, zapcore.AddSync(wsLoggerInstance.Writer()), loggerLevelDeal)) + } } // 设置控制台输出 From c472c9c0783efb1e44d3a60cbba7ab9dbed5fb9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E8=8C=B6=E6=B8=85=E6=AC=A2?= Date: Sat, 26 Apr 2025 20:13:45 +0800 Subject: [PATCH 3/6] remove ws log --- instance/ws_log.go | 120 --------------------------------------------- 1 file changed, 120 deletions(-) delete mode 100644 instance/ws_log.go diff --git a/instance/ws_log.go b/instance/ws_log.go deleted file mode 100644 index 9404e2d..0000000 --- a/instance/ws_log.go +++ /dev/null @@ -1,120 +0,0 @@ -// Package instance ... -// -// Description : instance ... -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 2025-04-25 18:16 -package instance - -import ( - "git.zhangdeman.cn/zhangdeman/websocket" - "git.zhangdeman.cn/zhangdeman/websocket/abstract" - "git.zhangdeman.cn/zhangdeman/websocket/config" - "git.zhangdeman.cn/zhangdeman/websocket/context" - "git.zhangdeman.cn/zhangdeman/websocket/storage" - "github.com/gin-gonic/gin" - "io" -) - -// NewWebsocketLog 初始化长连接日志 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 18:12 2024/7/22 -func NewWebsocketLog(serverPort int, route *gin.Engine) (*WsLog, error) { - wsLog := &WsLog{} - if err := websocket.NewWebsocketServeWithGinRouter(serverPort, route, wsLog); nil != err { - return nil, err - } - wsServer, _ := websocket.GetWsServer(serverPort, wsLog.GetModuleFlag()) - // logger.SetWsLoggerConnect(wsServer.GetConfig().ConnectionManager) - return &WsLog{ - port: serverPort, - connectManager: wsServer.GetConfig().ConnectionManager, - }, nil -} - -// CloseWebsocketLog ... -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 13:27 2024/7/25 -func CloseWebsocketLog(wsLog *WsLog) { - if nil == wsLog { - return - } - wsServer, _ := websocket.GetWsServer(wsLog.port, wsLog.GetModuleFlag()) - allConnect := wsServer.GetConfig().ConnectionManager.GetCtxList() - for _, conn := range allConnect { - _ = conn.Session.Close() - } -} - -type WsLog struct { - port int - connectManager storage.IConnection -} - -func (e WsLog) Connect(ctx *context.WSContext) error { - return nil -} - -func (e WsLog) Disconnect(ctx *context.WSContext) { -} - -func (e WsLog) Close(ctx *context.WSContext, code int, message string) error { - return nil -} - -func (e WsLog) HandshakeURL() []string { - return []string{ - "/system/logger/trace", - } -} - -func (e WsLog) GetCommandList() []abstract.ICommand { - return []abstract.ICommand{} -} - -func (e WsLog) GetModuleFlag() string { - return "ws-log" -} - -func (e WsLog) GetServerPort() int { - return e.port -} - -func (e WsLog) GetWSServerConfig() []config.SetWSServerConfig { - return []config.SetWSServerConfig{ - config.SetPingPeriod(5), - config.SetPongWait(60), - config.SetWriteWait(60), - config.SetMaxMessageBufferSize(81920), - config.SetMaxMessageSize(81920), - } -} - -func (e WsLog) Writer() io.Writer { - return &wsWriter{ - connectionManager: e.connectManager, - } -} - -type wsWriter struct { - connectionManager storage.IConnection -} - -// Write ws的writer -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 22:24 2024/7/22 -func (w *wsWriter) Write(p []byte) (n int, err error) { - p = []byte(" " + string(p)) - allConnList := w.connectionManager.GetCtxList() - for _, conn := range allConnList { - _ = conn.Session.Write(p) - } - return 0, nil -} From 21efb0bb85c84006a9d0f13f9694b9064eb84d52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E8=8C=B6=E6=B8=85=E6=AC=A2?= Date: Sat, 26 Apr 2025 22:30:37 +0800 Subject: [PATCH 4/6] =?UTF-8?q?=E5=AE=8C=E6=88=90=E5=9F=BA=E4=BA=8Ezinc?= =?UTF-8?q?=E7=9A=84=E6=97=A5=E5=BF=97=E6=94=B6=E9=9B=86=E7=9A=84=E9=AA=A8?= =?UTF-8?q?=E6=9E=B6=E6=90=AD=E5=BB=BA,=20=E7=BB=86=E8=8A=82=E5=BE=85?= =?UTF-8?q?=E4=B8=B0=E5=AF=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.go | 4 -- go.mod | 2 +- go.sum | 2 + logger.go | 36 +++++++++----- wrapper/zinc_search.go | 107 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 133 insertions(+), 18 deletions(-) create mode 100644 wrapper/zinc_search.go diff --git a/config.go b/config.go index 198add3..60e4866 100644 --- a/config.go +++ b/config.go @@ -235,10 +235,6 @@ type OptionLogger struct { WithCallerSkip int // 跳过的调用数 ConsoleOutput bool // 控制台输出 Encoder zapcore.Encoder // 编码函数 - WsServerProt int // ws端口 - WsGroup string // ws分组 - GinRouter *gin.Engine // gin路由实例 - EnableWsLog bool // 启用wsLog } // SetLoggerOptionFunc 设置日志配置 diff --git a/go.mod b/go.mod index 3c6d645..ab7b865 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( git.zhangdeman.cn/zhangdeman/consts v0.0.0-20250425024726-cc17224cb995 git.zhangdeman.cn/zhangdeman/gin v0.0.0-20250413074621-24f33309b9d8 git.zhangdeman.cn/zhangdeman/network v0.0.0-20250425082854-19992ab41c78 - git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20241223084948-de2e49144fcd + git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20250426132259-73cf1be49c7f git.zhangdeman.cn/zhangdeman/websocket v0.0.0-20241125101541-c5ea194c9c1e github.com/gin-gonic/gin v1.10.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible diff --git a/go.sum b/go.sum index e2b1c1c..b310d49 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ git.zhangdeman.cn/zhangdeman/op_type v0.0.0-20240122104027-4928421213c0 h1:gUDlQ git.zhangdeman.cn/zhangdeman/op_type v0.0.0-20240122104027-4928421213c0/go.mod h1:VHb9qmhaPDAQDcS6vUiDCamYjZ4R5lD1XtVsh55KsMI= git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20241223084948-de2e49144fcd h1:q7GG14qgXKB4MEXQFOe7/UYebsqMfPaSX80TcPdOosI= git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20241223084948-de2e49144fcd/go.mod h1:+D6uPSljwHywjVY5WSBY4TRVMj26TN5f5cFGEYMldjs= +git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20250426132259-73cf1be49c7f h1:7QgAcGnmVEVyIPeWH0ZkQN/jpzklYXsKCenTR2GpxbE= +git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20250426132259-73cf1be49c7f/go.mod h1:Ig3GZC2hJDkQp7F8Tm53GvMWLh9bdbbauow/vxGO4YA= git.zhangdeman.cn/zhangdeman/trace v0.0.0-20250412104923-c1ecb1bfe8d5 h1:dD1Q/MIrRmIhKqfYPH+y167ca9CKwTPuQt3c1hXWGJ8= git.zhangdeman.cn/zhangdeman/trace v0.0.0-20250412104923-c1ecb1bfe8d5/go.mod h1:PB486NC82nuvn5yi+U2i48ogX/9EAETWAHd8O9TwY9k= git.zhangdeman.cn/zhangdeman/util v0.0.0-20240618042405-6ee2c904644e h1:Q973S6CcWr1ICZhFI1STFOJ+KUImCl2BaIXm6YppBqI= diff --git a/logger.go b/logger.go index b2806b8..48dfc1c 100644 --- a/logger.go +++ b/logger.go @@ -9,7 +9,6 @@ package logger import ( "git.zhangdeman.cn/zhangdeman/consts" - "git.zhangdeman.cn/zhangdeman/logger/instance" "io" "os" @@ -60,18 +59,9 @@ func NewLogger(inputLoggerLevel consts.LogLevel, splitConfig *RotateLogConfig, o } fileHandlerList := []zapcore.Core{ - zapcore.NewCore(o.Encoder, zapcore.AddSync(loggerWriter), loggerLevelDeal), - // zapcore.NewCore(o.Encoder, zapcore.AddSync(&wsWriter{}), loggerLevelDeal), // 设置ws日志输出 - } - - // 通过配置控制ws logger - if o.EnableWsLog { - if wsLoggerInstance, err := instance.NewWebsocketLog(o.WsServerProt, o.GinRouter); nil != err { - return nil, err - } else { - // 设置ws日志输出 - fileHandlerList = append(fileHandlerList, zapcore.NewCore(o.Encoder, zapcore.AddSync(wsLoggerInstance.Writer()), loggerLevelDeal)) - } + zapcore.NewCore(o.Encoder, zapcore.AddSync(loggerWriter), loggerLevelDeal), // 指定的日志输出位置 + zapcore.NewCore(o.Encoder, zapcore.AddSync(&wsWriter{}), loggerLevelDeal), // 设置ws日志输出 + // TODO: zinc日志同步 } // 设置控制台输出 @@ -149,3 +139,23 @@ func (l *Logger) getWriter() (io.Writer, error) { return hook, nil } + +type wsWriter struct { +} + +// Write ws的writer +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 22:24 2024/7/22 +func (w *wsWriter) Write(p []byte) (n int, err error) { + if nil == wsLoggerConnect { + return 0, nil + } + p = []byte(" " + string(p)) + allConnList := wsLoggerConnect.GetCtxList() + for _, conn := range allConnList { + _ = conn.Session.Write(p) + } + return 0, nil +} diff --git a/wrapper/zinc_search.go b/wrapper/zinc_search.go new file mode 100644 index 0000000..5d7c4fc --- /dev/null +++ b/wrapper/zinc_search.go @@ -0,0 +1,107 @@ +// Package wrapper ... +// +// Description : 基于zinc实现日志收集 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 2025-04-26 21:24 +package wrapper + +import ( + "fmt" + "git.zhangdeman.cn/zhangdeman/consts" + "git.zhangdeman.cn/zhangdeman/serialize" + "go.uber.org/zap/zapcore" + "io" + "strings" + "sync" + "time" +) + +// ZincConfig zinc服务配置 +type ZincConfig struct { + Authorization string `json:"authorization" dc:"授权secret,生成方式base64(user:password)"` + Domain string `json:"domain" dc:"zinc服务域名"` + Timeout int `json:"timeout" dc:"超时时间,单位毫秒,默认5000"` + Async bool `json:"async" dc:"数据异步写入"` + Index string `json:"index" dc:"日志使用的索引"` + LogLevel consts.LogLevel `json:"log_level" dc:"记录的日志等级"` + CreateType string `json:"create_type" dc:"日志同步的类型: single - 单个同步 batch - 批量创建"` + BufferSize int `json:"buffer_size" dc:"批量创建时, 数据缓存buffer大小, 默认1000"` + ForceSyncTime int `json:"force_sync_time" dc:"批量同步日志时,强制同步的时间间隔,buffer没满也会强制同步, 单位: 秒"` +} + +const ( + DefaultTimeout = 5000 // 默认超时时间 + DefaultBufferSize = 1000 // 默认buffer大小 +) + +const ( + CreateTypeSingle = "single" // 逐条日志同步 + CreateTypeBatch = "batch" // 批量日志同步 +) + +func NewZincLogConnect(cfg *ZincConfig) io.Writer { + return &zincLogConnect{ + config: cfg, + lock: &sync.RWMutex{}, + buffer: nil, + } +} + +// zincLogConnect zinc日志收集器 +type zincLogConnect struct { + config *ZincConfig // zinc配置 + lock *sync.RWMutex // 操作锁 + buffer []map[string]any // 数据缓冲buffer +} + +// getPutLogFullUrl 获取请求地址,使用批量创建 +func (zlc *zincLogConnect) getPutLogFullUrl() string { + if zlc.config.CreateType == CreateTypeBatch { + // 批量创建 + return fmt.Sprintf("%v/api/_bulkv2", strings.TrimRight(zlc.config.Domain, "/")) + } + // 单条创建 + return fmt.Sprintf("%v/api/%v/_doc", strings.TrimRight(zlc.config.Domain, "/"), zlc.config.Index) +} + +// Write 日志写入, 要求字节数据必须是个合法的json序列化之后的结果 +func (zlc *zincLogConnect) Write(logData []byte) (int, error) { + var ( + res map[string]any + err error + ) + if err = serialize.JSON.UnmarshalWithNumber(logData, &res); nil != err { + return 0, err + } + if zlc.config.CreateType == CreateTypeBatch { + // 批量 + zlc.lock.Lock() + // 数据蟹醋buffer + zlc.buffer = append(zlc.buffer, res) + if len(zlc.buffer) >= zlc.config.BufferSize { + // buffer已满, 数据写入zinc + zlc.flush() + } + } + // 单个 + return 0, nil +} + +// flush 日志刷入zinc服务 +func (zlc *zincLogConnect) flush() { + // TODO: 数据同步 + // 日志刷入完成后, 清空buffer + zlc.buffer = []map[string]any{} +} + +// flushTask 批量同步日志, 强制同步的定时任务 +func (zlc *zincLogConnect) flushTask() { + for { + time.Sleep(time.Second * time.Duration(zlc.config.ForceSyncTime)) + zlc.lock.Lock() + zlc.flush() + zlc.lock.Lock() + } +} From ea78e582bb9a79d7f6cf680c1a1ea376798c93c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E8=8C=B6=E6=B8=85=E6=AC=A2?= Date: Sun, 27 Apr 2025 13:18:30 +0800 Subject: [PATCH 5/6] =?UTF-8?q?=E5=AE=8C=E6=88=90=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E5=86=99=E5=85=A5zinc=E7=9A=84=E8=83=BD=E5=8A=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.go | 46 ++++++-------- define.go | 25 ++++++++ go.sum | 57 +++-------------- logger.go | 6 +- wrapper/zinc_search.go | 107 ------------------------------- zinc_search.go | 140 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 198 insertions(+), 183 deletions(-) delete mode 100644 wrapper/zinc_search.go create mode 100644 zinc_search.go diff --git a/config.go b/config.go index 60e4866..caebfa6 100644 --- a/config.go +++ b/config.go @@ -9,7 +9,6 @@ package logger import ( "fmt" - "github.com/gin-gonic/gin" "os" "path/filepath" "strings" @@ -223,18 +222,19 @@ func defaultEncodeDuration(d time.Duration, enc zapcore.PrimitiveArrayEncoder) { // // Date : 11:41 下午 2021/1/2 type OptionLogger struct { - UseJsonFormat bool // 日志使用json格式 - MessageKey string // message 字段 - LevelKey string // level 字段 - TimeKey string // 时间字段 - CallerKey string // 记录日志的文件的代码行数 - UseShortCaller bool // 使用短的调用文件格式 - TimeEncoder zapcore.TimeEncoder // 格式化时间的函数 - EncodeDuration zapcore.DurationEncoder // 原始时间信息 - WithCaller bool // 是否打印文件行号 - WithCallerSkip int // 跳过的调用数 - ConsoleOutput bool // 控制台输出 - Encoder zapcore.Encoder // 编码函数 + UseJsonFormat bool // 日志使用json格式 + MessageKey string // message 字段 + LevelKey string // level 字段 + TimeKey string // 时间字段 + CallerKey string // 记录日志的文件的代码行数 + UseShortCaller bool // 使用短的调用文件格式 + TimeEncoder zapcore.TimeEncoder // 格式化时间的函数 + EncodeDuration zapcore.DurationEncoder // 原始时间信息 + WithCaller bool // 是否打印文件行号 + WithCallerSkip int // 跳过的调用数 + ConsoleOutput bool // 控制台输出 + Encoder zapcore.Encoder // 编码函数 + ZincCollectConfig *ZincConfig // zinc采集配置 } // SetLoggerOptionFunc 设置日志配置 @@ -378,6 +378,13 @@ func WithEncodeDuration(encoder zapcore.DurationEncoder) SetLoggerOptionFunc { } } +// WithZincLogCollect zinc日志采集 +func WithZincLogCollect(zincCfg *ZincConfig) SetLoggerOptionFunc { + return func(o *OptionLogger) { + o.ZincCollectConfig = zincCfg + } +} + // GetEncoder 获取空中台输出的encoder // // Author : go_developer@163.com<白茶清欢> @@ -415,16 +422,3 @@ func GetEncoder(option ...SetLoggerOptionFunc) zapcore.Encoder { } return zapcore.NewJSONEncoder(ec) } - -// EnableWsLogger 启用wsLogger -func EnableWsLogger(serverPort int, wsGroup string, router *gin.Engine) SetLoggerOptionFunc { - return func(o *OptionLogger) { - o.GinRouter = router - o.WsServerProt = serverPort - if len(wsGroup) == 0 { - wsGroup = "ws-log" - } - o.WsGroup = wsGroup - o.EnableWsLog = true - } -} diff --git a/define.go b/define.go index 49999da..d07f2d9 100644 --- a/define.go +++ b/define.go @@ -83,6 +83,7 @@ type InputLogConfig struct { UseShortFile bool `json:"use_short_file" yaml:"use_short_file"` // 是否使用短文件格式 CallerSkip int `json:"caller_skip" yaml:"caller_skip"` // 日志记录的文件跳过多少层 MaxAge int `json:"max_age" yaml:"max_age"` // 日志最长保存时间, 单位 : 秒 + ZincSyncConfig *ZincConfig `json:"zinc_sync_config" yaml:"zinc_sync_config"` // 日志同步至zinc的配置 } // GetLogInstanceFromInputConfig 从输入配置获取日志实例 @@ -103,6 +104,8 @@ func GetLogInstanceFromInputConfig(logConf *InputLogConfig) (*zap.Logger, error) if logConf.Console { logConfList = append(logConfList, WithConsoleOutput()) } + // 配置zinc日志同步 + logConfList = append(logConfList, WithZincLogCollect(logConf.ZincSyncConfig)) var ( err error @@ -149,3 +152,25 @@ func inputLevel2ZapLevel(inputLoggerLevel consts.LogLevel) zapcore.Level { } return loggerLevel } + +// ZincConfig zinc服务配置 +type ZincConfig struct { + Authorization string `json:"authorization" dc:"授权secret,生成方式base64(user:password)"` + Domain string `json:"domain" dc:"zinc服务域名"` + Timeout int `json:"timeout" dc:"超时时间,单位毫秒,默认5000"` + Async bool `json:"async" dc:"数据异步写入"` + Index string `json:"index" dc:"日志使用的索引"` + CreateType string `json:"create_type" dc:"日志同步的类型: single - 单个同步 batch - 批量创建"` + BufferSize int `json:"buffer_size" dc:"批量创建时, 数据缓存buffer大小, 默认1000"` + ForceSyncTime int `json:"force_sync_time" dc:"批量同步日志时,强制同步的时间间隔,buffer没满也会强制同步, 单位: 秒"` +} + +const ( + DefaultTimeout = 5000 // 默认超时时间 + DefaultBufferSize = 1000 // 默认buffer大小 +) + +const ( + CreateTypeSingle = "single" // 逐条日志同步 + CreateTypeBatch = "batch" // 批量日志同步 +) diff --git a/go.sum b/go.sum index b310d49..a20fd68 100644 --- a/go.sum +++ b/go.sum @@ -1,25 +1,15 @@ -git.zhangdeman.cn/zhangdeman/consts v0.0.0-20250227040546-863c03f34bb8 h1:VEifPc+vkpEQoX9rj7zxmT1m+IA81XjOxe7+Z1aqWNM= -git.zhangdeman.cn/zhangdeman/consts v0.0.0-20250227040546-863c03f34bb8/go.mod h1:IXXaZkb7vGzGnGM5RRWrASAuwrVSNxuoe0DmeXx5g6k= git.zhangdeman.cn/zhangdeman/consts v0.0.0-20250425024726-cc17224cb995 h1:LmPRAf0AsxRVFPibdpZR89ajlsz8hof2IvMMyTqiEq4= git.zhangdeman.cn/zhangdeman/consts v0.0.0-20250425024726-cc17224cb995/go.mod h1:5p8CEKGBxi7qPtTXDI3HDmqKAfIm5i/aBWdrbkbdNjc= git.zhangdeman.cn/zhangdeman/easylock v0.0.0-20230731062340-983985c12eda h1:bMD6r9gjRy7cO+T4zRQVYAesgIblBdTnhzT1vN5wjvI= git.zhangdeman.cn/zhangdeman/easylock v0.0.0-20230731062340-983985c12eda/go.mod h1:dT0rmHcJ9Z9IqWeMIt7YzR88nKkNV2V3dfG0j9Q6lK0= git.zhangdeman.cn/zhangdeman/easymap v0.0.0-20241101082529-28a6c68e38a4 h1:s6d4b6yY+NaK1AzoBD1pxqsuygEHQz0Oie86c45geDw= git.zhangdeman.cn/zhangdeman/easymap v0.0.0-20241101082529-28a6c68e38a4/go.mod h1:V4Dfg1v/JVIZGEKCm6/aehs8hK+Xow1dkL1yiQymXlQ= -git.zhangdeman.cn/zhangdeman/gin v0.0.0-20250225092214-3432087fbd07 h1:SadUKF3SZhuRTXaCKyQWEavN9fVLNbHL/GBz/KoiL6o= -git.zhangdeman.cn/zhangdeman/gin v0.0.0-20250225092214-3432087fbd07/go.mod h1:T2Q8Wcq98yTuSSaEPZVAZfs0DMSxeXMN10GOQCha5g4= -git.zhangdeman.cn/zhangdeman/gin v0.0.0-20250228104311-2fd9195b77e7 h1:koL8c0do1mOLFY+wLMqSpojgHSwVRbV6sSsJVKo9WfA= -git.zhangdeman.cn/zhangdeman/gin v0.0.0-20250228104311-2fd9195b77e7/go.mod h1:T2Q8Wcq98yTuSSaEPZVAZfs0DMSxeXMN10GOQCha5g4= git.zhangdeman.cn/zhangdeman/gin v0.0.0-20250413074621-24f33309b9d8 h1:8Wt/SSVJSBR8/nddY+YYERTExc0DHrmShkk6GFUdWzw= git.zhangdeman.cn/zhangdeman/gin v0.0.0-20250413074621-24f33309b9d8/go.mod h1:HRY0KP893Oo1TZLPKv3XhAIhnMdtIipiylNsmJfZzD4= -git.zhangdeman.cn/zhangdeman/network v0.0.0-20250224022106-1c57dcf5afd9 h1:MphPBVuufQt4O2Nm+A2ldG/dMmb0LXZwAGcE/OkTRoU= -git.zhangdeman.cn/zhangdeman/network v0.0.0-20250224022106-1c57dcf5afd9/go.mod h1:vSHUJdlbSVDheL+e7KtdG3n/fgb26J/JOMVuEiXG+A8= git.zhangdeman.cn/zhangdeman/network v0.0.0-20250425082854-19992ab41c78 h1:IFFHrl01z//K0aoNGzLkb46SQfFDaiSONwivWZpa3uE= git.zhangdeman.cn/zhangdeman/network v0.0.0-20250425082854-19992ab41c78/go.mod h1:LKktkiO4at2XvPh/i9VPuT17DxLbBcDHUGMdrRSHNO4= git.zhangdeman.cn/zhangdeman/op_type v0.0.0-20240122104027-4928421213c0 h1:gUDlQMuJ4xNfP2Abl1Msmpa3fASLWYkNlqDFF/6GN0Y= git.zhangdeman.cn/zhangdeman/op_type v0.0.0-20240122104027-4928421213c0/go.mod h1:VHb9qmhaPDAQDcS6vUiDCamYjZ4R5lD1XtVsh55KsMI= -git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20241223084948-de2e49144fcd h1:q7GG14qgXKB4MEXQFOe7/UYebsqMfPaSX80TcPdOosI= -git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20241223084948-de2e49144fcd/go.mod h1:+D6uPSljwHywjVY5WSBY4TRVMj26TN5f5cFGEYMldjs= git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20250426132259-73cf1be49c7f h1:7QgAcGnmVEVyIPeWH0ZkQN/jpzklYXsKCenTR2GpxbE= git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20250426132259-73cf1be49c7f/go.mod h1:Ig3GZC2hJDkQp7F8Tm53GvMWLh9bdbbauow/vxGO4YA= git.zhangdeman.cn/zhangdeman/trace v0.0.0-20250412104923-c1ecb1bfe8d5 h1:dD1Q/MIrRmIhKqfYPH+y167ca9CKwTPuQt3c1hXWGJ8= @@ -28,27 +18,15 @@ git.zhangdeman.cn/zhangdeman/util v0.0.0-20240618042405-6ee2c904644e h1:Q973S6Cc git.zhangdeman.cn/zhangdeman/util v0.0.0-20240618042405-6ee2c904644e/go.mod h1:VpPjBlwz8U+OxZuxzHQBv1aEEZ3pStH6bZvT21ADEbI= git.zhangdeman.cn/zhangdeman/websocket v0.0.0-20241125101541-c5ea194c9c1e h1:YE2Gi+M03UDImIpWa3I7jzSesyfu2RL8x/4ONs5v0oE= git.zhangdeman.cn/zhangdeman/websocket v0.0.0-20241125101541-c5ea194c9c1e/go.mod h1:L/7JugxKZL3JP9JP/XDvPAPz0FQXG1u181Su1+u/d1c= -git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20250124091620-c757e551a8c9 h1:yF770WIDNwyiKL0nwmBGmjZvNCLXtHQL4xJyffPjTMU= -git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20250124091620-c757e551a8c9/go.mod h1:I76wxEsWq7KnMQ84elpwTjEqq4I49QFw60tp5h7iGBs= -git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20250302133417-c1588abcb436 h1:SM4zc54W2wmM72+4pMNQ8iS371H6lj4J8rj8KJKf7pw= -git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20250302133417-c1588abcb436/go.mod h1:YJ1FlvFgkfAHkxkt3l5rKKUqEpQkNMbCFDzDmgteEU8= git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20250321102712-1cbfbe959740 h1:zPUoylfJTbc0EcxW+NEzOTBmoeFZ2I/rLFBnEzxb4Wk= git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20250321102712-1cbfbe959740/go.mod h1:1ct92dbVc49pmXusA/iGfcQUJzcYmJ+cjAhgc3sDv1I= -github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= -github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 h1:OYA+5W64v3OgClL+IrOD63t4i/RW7RqrAVl9LTZ9UqQ= github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394/go.mod h1:Q8n74mJTIgjX4RBBcHnJ05h//6/k6foqmgE45jTQtxg= -github.com/bytedance/sonic v1.12.9 h1:Od1BvK55NnewtGaJsTDeAOSnLVO2BTSLOe0+ooKokmQ= -github.com/bytedance/sonic v1.12.9/go.mod h1:uVvFidNmlt9+wa31S1urfwwthTWteBgG0hWuoKAXTx8= -github.com/bytedance/sonic v1.12.10 h1:uVCQr6oS5669E9ZVW0HyksTLfNS7Q/9hV6IVS4nEMsI= -github.com/bytedance/sonic v1.12.10/go.mod h1:uVvFidNmlt9+wa31S1urfwwthTWteBgG0hWuoKAXTx8= github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ= github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/bytedance/sonic/loader v0.2.3 h1:yctD0Q3v2NOGfSWPLPvG2ggA2kV6TS6s4wioyEqssH0= -github.com/bytedance/sonic/loader v0.2.3/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= @@ -57,15 +35,10 @@ github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1 h1:CaO/zOnF8VvUfEbhRatPcwKVWamvbYd8tQGRWacE9kU= github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1/go.mod h1:+hnT3ywWDTAFrW5aE+u2Sa/wT555ZqwoCS+pk3p6ry4= -github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= -github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY= github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok= -github.com/gin-contrib/sse v1.0.0 h1:y3bT1mUWUxDpW4JLQg/HnTqV4rozuW4tC9eFKTxYI9E= -github.com/gin-contrib/sse v1.0.0/go.mod h1:zNuFdwarAygJBht0NTKiSi3jRf6RbqeILZ9Sp6Slhe0= github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= @@ -78,8 +51,6 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.25.0 h1:5Dh7cjvzR7BRZadnsVOzPhWsrwUr0nmsZJxEAnFLNO8= -github.com/go-playground/validator/v10 v10.25.0/go.mod h1:GGzBIJMuE98Ic/kJsBXbz1x/7cByt++cQ+YOuDM5wus= github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= @@ -89,6 +60,8 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= +github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= @@ -99,6 +72,8 @@ github.com/jonboulle/clockwork v0.3.0 h1:9BSCMi8C+0qdApAp4auwX0RkLGUjs956h0EkuQy github.com/jonboulle/clockwork v0.3.0/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= @@ -121,8 +96,6 @@ github.com/mcuadros/go-defaults v1.2.0 h1:FODb8WSf0uGaY8elWJAkoLL0Ri6AlZ1bFlenk5 github.com/mcuadros/go-defaults v1.2.0/go.mod h1:WEZtHEVIGYVDqkKSWBdWKUVdRyKlMfulPaGDWIVeCWY= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -134,8 +107,6 @@ github.com/mssola/user_agent v0.6.0 h1:uwPR4rtWlCHRFyyP9u2KOV0u8iQXmS7Z7feTrstQw github.com/mssola/user_agent v0.6.0/go.mod h1:TTPno8LPY3wAIEKRpAtkdMT0f8SE24pLRGPahjCH4uw= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= -github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -144,18 +115,20 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sbabiv/xml2map v1.2.1 h1:1lT7t0hhUvXZCkdxqtq4n8/ZCnwLWGq4rDuDv5XOoFE= github.com/sbabiv/xml2map v1.2.1/go.mod h1:2TPoAfcaM7+Sd4iriPvzyntb2mx7GY+kkQpB/GQa/eo= +github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= +github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= +github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= +github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= @@ -175,33 +148,19 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/arch v0.14.0 h1:z9JUEZWr8x4rR0OU6c4/4t6E6jOZ8/QBS2bBYBm4tx4= -golang.org/x/arch v0.14.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/arch v0.16.0 h1:foMtLTdyOmIniqWCHjY6+JxuC54XP1fDwx4N0ASyW+U= golang.org/x/arch v0.16.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE= -golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= -golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= -golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= -golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= -golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= -golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= -golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= -google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/logger.go b/logger.go index 48dfc1c..76d0363 100644 --- a/logger.go +++ b/logger.go @@ -61,7 +61,6 @@ func NewLogger(inputLoggerLevel consts.LogLevel, splitConfig *RotateLogConfig, o fileHandlerList := []zapcore.Core{ zapcore.NewCore(o.Encoder, zapcore.AddSync(loggerWriter), loggerLevelDeal), // 指定的日志输出位置 zapcore.NewCore(o.Encoder, zapcore.AddSync(&wsWriter{}), loggerLevelDeal), // 设置ws日志输出 - // TODO: zinc日志同步 } // 设置控制台输出 @@ -69,6 +68,11 @@ func NewLogger(inputLoggerLevel consts.LogLevel, splitConfig *RotateLogConfig, o fileHandlerList = append(fileHandlerList, zapcore.NewCore(o.Encoder, zapcore.AddSync(os.Stdout), loggerLevelDeal)) } + // 设置zinc日志输出 + if nil != o.ZincCollectConfig { + fileHandlerList = append(fileHandlerList, zapcore.NewCore(o.Encoder, zapcore.AddSync(NewZincLogConnect(o.ZincCollectConfig)), loggerLevelDeal)) + } + // 最后创建具体的Logger core := zapcore.NewTee(fileHandlerList...) diff --git a/wrapper/zinc_search.go b/wrapper/zinc_search.go deleted file mode 100644 index 5d7c4fc..0000000 --- a/wrapper/zinc_search.go +++ /dev/null @@ -1,107 +0,0 @@ -// Package wrapper ... -// -// Description : 基于zinc实现日志收集 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 2025-04-26 21:24 -package wrapper - -import ( - "fmt" - "git.zhangdeman.cn/zhangdeman/consts" - "git.zhangdeman.cn/zhangdeman/serialize" - "go.uber.org/zap/zapcore" - "io" - "strings" - "sync" - "time" -) - -// ZincConfig zinc服务配置 -type ZincConfig struct { - Authorization string `json:"authorization" dc:"授权secret,生成方式base64(user:password)"` - Domain string `json:"domain" dc:"zinc服务域名"` - Timeout int `json:"timeout" dc:"超时时间,单位毫秒,默认5000"` - Async bool `json:"async" dc:"数据异步写入"` - Index string `json:"index" dc:"日志使用的索引"` - LogLevel consts.LogLevel `json:"log_level" dc:"记录的日志等级"` - CreateType string `json:"create_type" dc:"日志同步的类型: single - 单个同步 batch - 批量创建"` - BufferSize int `json:"buffer_size" dc:"批量创建时, 数据缓存buffer大小, 默认1000"` - ForceSyncTime int `json:"force_sync_time" dc:"批量同步日志时,强制同步的时间间隔,buffer没满也会强制同步, 单位: 秒"` -} - -const ( - DefaultTimeout = 5000 // 默认超时时间 - DefaultBufferSize = 1000 // 默认buffer大小 -) - -const ( - CreateTypeSingle = "single" // 逐条日志同步 - CreateTypeBatch = "batch" // 批量日志同步 -) - -func NewZincLogConnect(cfg *ZincConfig) io.Writer { - return &zincLogConnect{ - config: cfg, - lock: &sync.RWMutex{}, - buffer: nil, - } -} - -// zincLogConnect zinc日志收集器 -type zincLogConnect struct { - config *ZincConfig // zinc配置 - lock *sync.RWMutex // 操作锁 - buffer []map[string]any // 数据缓冲buffer -} - -// getPutLogFullUrl 获取请求地址,使用批量创建 -func (zlc *zincLogConnect) getPutLogFullUrl() string { - if zlc.config.CreateType == CreateTypeBatch { - // 批量创建 - return fmt.Sprintf("%v/api/_bulkv2", strings.TrimRight(zlc.config.Domain, "/")) - } - // 单条创建 - return fmt.Sprintf("%v/api/%v/_doc", strings.TrimRight(zlc.config.Domain, "/"), zlc.config.Index) -} - -// Write 日志写入, 要求字节数据必须是个合法的json序列化之后的结果 -func (zlc *zincLogConnect) Write(logData []byte) (int, error) { - var ( - res map[string]any - err error - ) - if err = serialize.JSON.UnmarshalWithNumber(logData, &res); nil != err { - return 0, err - } - if zlc.config.CreateType == CreateTypeBatch { - // 批量 - zlc.lock.Lock() - // 数据蟹醋buffer - zlc.buffer = append(zlc.buffer, res) - if len(zlc.buffer) >= zlc.config.BufferSize { - // buffer已满, 数据写入zinc - zlc.flush() - } - } - // 单个 - return 0, nil -} - -// flush 日志刷入zinc服务 -func (zlc *zincLogConnect) flush() { - // TODO: 数据同步 - // 日志刷入完成后, 清空buffer - zlc.buffer = []map[string]any{} -} - -// flushTask 批量同步日志, 强制同步的定时任务 -func (zlc *zincLogConnect) flushTask() { - for { - time.Sleep(time.Second * time.Duration(zlc.config.ForceSyncTime)) - zlc.lock.Lock() - zlc.flush() - zlc.lock.Lock() - } -} diff --git a/zinc_search.go b/zinc_search.go new file mode 100644 index 0000000..4334da3 --- /dev/null +++ b/zinc_search.go @@ -0,0 +1,140 @@ +// Package logger ... +// +// Description : 基于zinc实现日志收集 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 2025-04-26 21:24 +package logger + +import ( + "bytes" + "fmt" + "git.zhangdeman.cn/zhangdeman/consts" + "git.zhangdeman.cn/zhangdeman/serialize" + "io" + "log" + "net/http" + "strings" + "sync" + "time" +) + +func NewZincLogConnect(cfg *ZincConfig) io.Writer { + zlc := &zincLogConnect{ + config: cfg, + lock: &sync.RWMutex{}, + buffer: nil, + } + // 批量写入强制同步任务 + go zlc.flushTask() + return zlc +} + +// zincLogConnect zinc日志收集器 +type zincLogConnect struct { + config *ZincConfig // zinc配置 + lock *sync.RWMutex // 操作锁 + buffer []map[string]any // 数据缓冲buffer +} + +// getPutLogFullUrl 获取请求地址,使用批量创建 +func (zlc *zincLogConnect) getPutLogFullUrl() string { + if zlc.config.CreateType == CreateTypeBatch { + // 批量创建 + return fmt.Sprintf("%v/api/_bulkv2", strings.TrimRight(zlc.config.Domain, "/")) + } + // 单条创建 + return fmt.Sprintf("%v/api/%v/_doc", strings.TrimRight(zlc.config.Domain, "/"), zlc.config.Index) +} + +// Write 日志写入, 要求字节数据必须是个合法的json序列化之后的结果 +func (zlc *zincLogConnect) Write(logData []byte) (int, error) { + if zlc.config.CreateType == CreateTypeBatch { + var ( + res map[string]any + err error + ) + if err = serialize.JSON.UnmarshalWithNumber(logData, &res); nil != err { + return 0, err + } + // 批量 + zlc.lock.Lock() + // 数据写入buffer + zlc.buffer = append(zlc.buffer, res) + if len(zlc.buffer) >= zlc.config.BufferSize { + // buffer已满, 数据写入zinc + zlc.flush() + } + zlc.lock.Unlock() + return 0, nil + } + // 单个直接写入 + zlc.writeData(logData) + return 0, nil +} + +// flush 日志刷入zinc服务(针对批量数据写入的) +func (zlc *zincLogConnect) flush() { + if zlc.config.CreateType != CreateTypeBatch { + return + } + if len(zlc.buffer) == 0 { + return + } + zlc.writeData(serialize.JSON.MarshalForByteIgnoreError(map[string]any{ + "index": zlc.config.Index, + "records": zlc.buffer, + })) + // 清空buffer + zlc.buffer = []map[string]any{} +} + +// flushTask 批量同步日志, 强制同步的定时任务(针对批量数据写入) +func (zlc *zincLogConnect) flushTask() { + if zlc.config.CreateType != CreateTypeBatch { + return + } + for { + time.Sleep(time.Millisecond * time.Duration(zlc.config.ForceSyncTime)) + zlc.lock.Lock() + zlc.flush() + zlc.lock.Unlock() + } +} + +// writeData 数据写入zinc +func (zlc *zincLogConnect) writeData(paramData []byte) { + req, _ := http.NewRequest(http.MethodPost, zlc.getPutLogFullUrl(), bytes.NewReader(paramData)) + req.Header.Set(consts.HeaderKeyAuthorization.String(), "Basic "+zlc.config.Authorization) // 设置authorization + req.Header.Set(consts.HeaderKeyContentType.String(), consts.MimeTypeJson) // json请求 + client := &http.Client{} + + if zlc.config.Async { + // 异步请求 + go func() { + resp, err := client.Do(req) + if err != nil { + log.Print("异步日志写入zinc失败(请求发送失败): " + err.Error()) + return + } + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + // 状态码非2xx + log.Print("异步日志写入zinc失败(zinc服务响应状态码异常): " + resp.Status) + return + } + }() + } else { + // 同步请求 + resp, err := client.Do(req) + if err != nil { + log.Print("异步日志写入zinc失败(请求发送失败): " + err.Error()) + return + } + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + // 状态码非2xx + log.Print("异步日志写入zinc失败(zinc服务响应状态码异常): " + resp.Status) + return + } + } +} From f6a779fad7f976e58adb6d9b74f61c5e8fc1db73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E8=8C=B6=E6=B8=85=E6=AC=A2?= Date: Sun, 27 Apr 2025 14:51:42 +0800 Subject: [PATCH 6/6] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=80=BC=E6=A3=80=E6=B5=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- define.go | 5 +++-- zinc_search.go | 9 +++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/define.go b/define.go index d07f2d9..9b2899f 100644 --- a/define.go +++ b/define.go @@ -166,8 +166,9 @@ type ZincConfig struct { } const ( - DefaultTimeout = 5000 // 默认超时时间 - DefaultBufferSize = 1000 // 默认buffer大小 + DefaultTimeout = 5000 // 默认超时时间 + DefaultBufferSize = 1000 // 默认buffer大小 + DefaultForceFlushLogTime = 1000 // 强制刷新日志的时间间隔, 单位毫秒 ) const ( diff --git a/zinc_search.go b/zinc_search.go index 4334da3..28cd9b7 100644 --- a/zinc_search.go +++ b/zinc_search.go @@ -21,6 +21,15 @@ import ( ) func NewZincLogConnect(cfg *ZincConfig) io.Writer { + if cfg.Timeout <= 0 { + cfg.Timeout = DefaultTimeout + } + if cfg.BufferSize <= 0 { + cfg.BufferSize = DefaultBufferSize + } + if cfg.ForceSyncTime <= 0 { + cfg.ForceSyncTime = DefaultForceFlushLogTime + } zlc := &zincLogConnect{ config: cfg, lock: &sync.RWMutex{},