zhangdeman/logger
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

完成基于zinc的日志收集的骨架搭建, 细节待丰富

Browse Source
This commit is contained in:
白茶清欢 2025-04-26 22:30:37 +08:00
parent c472c9c078
commit 21efb0bb85
5 changed files with 133 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
config.go
View File

@ -235,10 +235,6 @@ type OptionLogger struct {
WithCallerSkip int // 跳过的调用数
ConsoleOutput bool // 控制台输出
Encoder zapcore.Encoder // 编码函数
WsServerProt int // ws端口
WsGroup string // ws分组
GinRouter *gin.Engine // gin路由实例
EnableWsLog bool // 启用wsLog
}
// SetLoggerOptionFunc 设置日志配置

2
go.mod
View File

@ -8,7 +8,7 @@ require (
git.zhangdeman.cn/zhangdeman/consts v0.0.0-20250425024726-cc17224cb995
git.zhangdeman.cn/zhangdeman/gin v0.0.0-20250413074621-24f33309b9d8
git.zhangdeman.cn/zhangdeman/network v0.0.0-20250425082854-19992ab41c78
git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20241223084948-de2e49144fcd
git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20250426132259-73cf1be49c7f
git.zhangdeman.cn/zhangdeman/websocket v0.0.0-20241125101541-c5ea194c9c1e
github.com/gin-gonic/gin v1.10.0
github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible

2
go.sum
View File

@ -20,6 +20,8 @@ git.zhangdeman.cn/zhangdeman/op_type v0.0.0-20240122104027-4928421213c0 h1:gUDlQ
git.zhangdeman.cn/zhangdeman/op_type v0.0.0-20240122104027-4928421213c0/go.mod h1:VHb9qmhaPDAQDcS6vUiDCamYjZ4R5lD1XtVsh55KsMI=
git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20241223084948-de2e49144fcd h1:q7GG14qgXKB4MEXQFOe7/UYebsqMfPaSX80TcPdOosI=
git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20241223084948-de2e49144fcd/go.mod h1:+D6uPSljwHywjVY5WSBY4TRVMj26TN5f5cFGEYMldjs=
git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20250426132259-73cf1be49c7f h1:7QgAcGnmVEVyIPeWH0ZkQN/jpzklYXsKCenTR2GpxbE=
git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20250426132259-73cf1be49c7f/go.mod h1:Ig3GZC2hJDkQp7F8Tm53GvMWLh9bdbbauow/vxGO4YA=
git.zhangdeman.cn/zhangdeman/trace v0.0.0-20250412104923-c1ecb1bfe8d5 h1:dD1Q/MIrRmIhKqfYPH+y167ca9CKwTPuQt3c1hXWGJ8=
git.zhangdeman.cn/zhangdeman/trace v0.0.0-20250412104923-c1ecb1bfe8d5/go.mod h1:PB486NC82nuvn5yi+U2i48ogX/9EAETWAHd8O9TwY9k=
git.zhangdeman.cn/zhangdeman/util v0.0.0-20240618042405-6ee2c904644e h1:Q973S6CcWr1ICZhFI1STFOJ+KUImCl2BaIXm6YppBqI=

36
logger.go
View File

@ -9,7 +9,6 @@ package logger
import (
"git.zhangdeman.cn/zhangdeman/consts"
"git.zhangdeman.cn/zhangdeman/logger/instance"
"io"
"os"
@ -60,18 +59,9 @@ func NewLogger(inputLoggerLevel consts.LogLevel, splitConfig *RotateLogConfig, o
}
fileHandlerList := []zapcore.Core{
zapcore.NewCore(o.Encoder, zapcore.AddSync(loggerWriter), loggerLevelDeal),
// zapcore.NewCore(o.Encoder, zapcore.AddSync(&wsWriter{}), loggerLevelDeal), // 设置ws日志输出
}
// 通过配置控制ws logger
if o.EnableWsLog {
if wsLoggerInstance, err := instance.NewWebsocketLog(o.WsServerProt, o.GinRouter); nil != err {
return nil, err
} else {
// 设置ws日志输出
fileHandlerList = append(fileHandlerList, zapcore.NewCore(o.Encoder, zapcore.AddSync(wsLoggerInstance.Writer()), loggerLevelDeal))
}
zapcore.NewCore(o.Encoder, zapcore.AddSync(loggerWriter), loggerLevelDeal), // 指定的日志输出位置
zapcore.NewCore(o.Encoder, zapcore.AddSync(&wsWriter{}), loggerLevelDeal), // 设置ws日志输出
// TODO: zinc日志同步
}
// 设置控制台输出
@ -149,3 +139,23 @@ func (l *Logger) getWriter() (io.Writer, error) {
return hook, nil
}
type wsWriter struct {
}
// Write ws的writer
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 22:24 2024/7/22
func (w *wsWriter) Write(p []byte) (n int, err error) {
if nil == wsLoggerConnect {
return 0, nil
}
p = []byte(" " + string(p))
allConnList := wsLoggerConnect.GetCtxList()
for _, conn := range allConnList {
_ = conn.Session.Write(p)
}
return 0, nil
}

107
wrapper/zinc_search.go Normal file
View File

@ -0,0 +1,107 @@
// Package wrapper ...
//
// Description : 基于zinc实现日志收集
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2025-04-26 21:24
package wrapper
import (
"fmt"
"git.zhangdeman.cn/zhangdeman/consts"
"git.zhangdeman.cn/zhangdeman/serialize"
"go.uber.org/zap/zapcore"
"io"
"strings"
"sync"
"time"
)
// ZincConfig zinc服务配置
type ZincConfig struct {
Authorization string `json:"authorization" dc:"授权secret,生成方式base64(user:password)"`
Domain string `json:"domain" dc:"zinc服务域名"`
Timeout int `json:"timeout" dc:"超时时间,单位毫秒,默认5000"`
Async bool `json:"async" dc:"数据异步写入"`
Index string `json:"index" dc:"日志使用的索引"`
LogLevel consts.LogLevel `json:"log_level" dc:"记录的日志等级"`
CreateType string `json:"create_type" dc:"日志同步的类型: single - 单个同步 batch - 批量创建"`
BufferSize int `json:"buffer_size" dc:"批量创建时, 数据缓存buffer大小, 默认1000"`
ForceSyncTime int `json:"force_sync_time" dc:"批量同步日志时,强制同步的时间间隔,buffer没满也会强制同步, 单位: 秒"`
}
const (
DefaultTimeout = 5000 // 默认超时时间
DefaultBufferSize = 1000 // 默认buffer大小
)
const (
CreateTypeSingle = "single" // 逐条日志同步
CreateTypeBatch = "batch" // 批量日志同步
)
func NewZincLogConnect(cfg *ZincConfig) io.Writer {
return &zincLogConnect{
config: cfg,
lock: &sync.RWMutex{},
buffer: nil,
}
}
// zincLogConnect zinc日志收集器
type zincLogConnect struct {
config *ZincConfig // zinc配置
lock *sync.RWMutex // 操作锁
buffer []map[string]any // 数据缓冲buffer
}
// getPutLogFullUrl 获取请求地址,使用批量创建
func (zlc *zincLogConnect) getPutLogFullUrl() string {
if zlc.config.CreateType == CreateTypeBatch {
// 批量创建
return fmt.Sprintf("%v/api/_bulkv2", strings.TrimRight(zlc.config.Domain, "/"))
}
// 单条创建
return fmt.Sprintf("%v/api/%v/_doc", strings.TrimRight(zlc.config.Domain, "/"), zlc.config.Index)
}
// Write 日志写入, 要求字节数据必须是个合法的json序列化之后的结果
func (zlc *zincLogConnect) Write(logData []byte) (int, error) {
var (
res map[string]any
err error
)
if err = serialize.JSON.UnmarshalWithNumber(logData, &res); nil != err {
return 0, err
}
if zlc.config.CreateType == CreateTypeBatch {
// 批量
zlc.lock.Lock()
// 数据蟹醋buffer
zlc.buffer = append(zlc.buffer, res)
if len(zlc.buffer) >= zlc.config.BufferSize {
// buffer已满, 数据写入zinc
zlc.flush()
}
}
// 单个
return 0, nil
}
// flush 日志刷入zinc服务
func (zlc *zincLogConnect) flush() {
// TODO: 数据同步
// 日志刷入完成后, 清空buffer
zlc.buffer = []map[string]any{}
}
// flushTask 批量同步日志, 强制同步的定时任务
func (zlc *zincLogConnect) flushTask() {
for {
time.Sleep(time.Second * time.Duration(zlc.config.ForceSyncTime))
zlc.lock.Lock()
zlc.flush()
zlc.lock.Lock()
}
}