完成日志写入zinc的能力

This commit is contained in:
白茶清欢 2025-04-27 13:18:30 +08:00
parent 21efb0bb85
commit ea78e582bb
6 changed files with 198 additions and 183 deletions

View File

@ -9,7 +9,6 @@ package logger
import (
"fmt"
"github.com/gin-gonic/gin"
"os"
"path/filepath"
"strings"
@ -223,18 +222,19 @@ func defaultEncodeDuration(d time.Duration, enc zapcore.PrimitiveArrayEncoder) {
//
// Date : 11:41 下午 2021/1/2
type OptionLogger struct {
UseJsonFormat bool // 日志使用json格式
MessageKey string // message 字段
LevelKey string // level 字段
TimeKey string // 时间字段
CallerKey string // 记录日志的文件的代码行数
UseShortCaller bool // 使用短的调用文件格式
TimeEncoder zapcore.TimeEncoder // 格式化时间的函数
EncodeDuration zapcore.DurationEncoder // 原始时间信息
WithCaller bool // 是否打印文件行号
WithCallerSkip int // 跳过的调用数
ConsoleOutput bool // 控制台输出
Encoder zapcore.Encoder // 编码函数
UseJsonFormat bool // 日志使用json格式
MessageKey string // message 字段
LevelKey string // level 字段
TimeKey string // 时间字段
CallerKey string // 记录日志的文件的代码行数
UseShortCaller bool // 使用短的调用文件格式
TimeEncoder zapcore.TimeEncoder // 格式化时间的函数
EncodeDuration zapcore.DurationEncoder // 原始时间信息
WithCaller bool // 是否打印文件行号
WithCallerSkip int // 跳过的调用数
ConsoleOutput bool // 控制台输出
Encoder zapcore.Encoder // 编码函数
ZincCollectConfig *ZincConfig // zinc采集配置
}
// SetLoggerOptionFunc 设置日志配置
@ -378,6 +378,13 @@ func WithEncodeDuration(encoder zapcore.DurationEncoder) SetLoggerOptionFunc {
}
}
// WithZincLogCollect zinc日志采集
func WithZincLogCollect(zincCfg *ZincConfig) SetLoggerOptionFunc {
return func(o *OptionLogger) {
o.ZincCollectConfig = zincCfg
}
}
// GetEncoder 获取空中台输出的encoder
//
// Author : go_developer@163.com<白茶清欢>
@ -415,16 +422,3 @@ func GetEncoder(option ...SetLoggerOptionFunc) zapcore.Encoder {
}
return zapcore.NewJSONEncoder(ec)
}
// EnableWsLogger 启用wsLogger
func EnableWsLogger(serverPort int, wsGroup string, router *gin.Engine) SetLoggerOptionFunc {
return func(o *OptionLogger) {
o.GinRouter = router
o.WsServerProt = serverPort
if len(wsGroup) == 0 {
wsGroup = "ws-log"
}
o.WsGroup = wsGroup
o.EnableWsLog = true
}
}

View File

@ -83,6 +83,7 @@ type InputLogConfig struct {
UseShortFile bool `json:"use_short_file" yaml:"use_short_file"` // 是否使用短文件格式
CallerSkip int `json:"caller_skip" yaml:"caller_skip"` // 日志记录的文件跳过多少层
MaxAge int `json:"max_age" yaml:"max_age"` // 日志最长保存时间, 单位 : 秒
ZincSyncConfig *ZincConfig `json:"zinc_sync_config" yaml:"zinc_sync_config"` // 日志同步至zinc的配置
}
// GetLogInstanceFromInputConfig 从输入配置获取日志实例
@ -103,6 +104,8 @@ func GetLogInstanceFromInputConfig(logConf *InputLogConfig) (*zap.Logger, error)
if logConf.Console {
logConfList = append(logConfList, WithConsoleOutput())
}
// 配置zinc日志同步
logConfList = append(logConfList, WithZincLogCollect(logConf.ZincSyncConfig))
var (
err error
@ -149,3 +152,25 @@ func inputLevel2ZapLevel(inputLoggerLevel consts.LogLevel) zapcore.Level {
}
return loggerLevel
}
// ZincConfig zinc服务配置
type ZincConfig struct {
Authorization string `json:"authorization" dc:"授权secret,生成方式base64(user:password)"`
Domain string `json:"domain" dc:"zinc服务域名"`
Timeout int `json:"timeout" dc:"超时时间,单位毫秒,默认5000"`
Async bool `json:"async" dc:"数据异步写入"`
Index string `json:"index" dc:"日志使用的索引"`
CreateType string `json:"create_type" dc:"日志同步的类型: single - 单个同步 batch - 批量创建"`
BufferSize int `json:"buffer_size" dc:"批量创建时, 数据缓存buffer大小, 默认1000"`
ForceSyncTime int `json:"force_sync_time" dc:"批量同步日志时,强制同步的时间间隔,buffer没满也会强制同步, 单位: 秒"`
}
const (
DefaultTimeout = 5000 // 默认超时时间
DefaultBufferSize = 1000 // 默认buffer大小
)
const (
CreateTypeSingle = "single" // 逐条日志同步
CreateTypeBatch = "batch" // 批量日志同步
)

57
go.sum
View File

@ -1,25 +1,15 @@
git.zhangdeman.cn/zhangdeman/consts v0.0.0-20250227040546-863c03f34bb8 h1:VEifPc+vkpEQoX9rj7zxmT1m+IA81XjOxe7+Z1aqWNM=
git.zhangdeman.cn/zhangdeman/consts v0.0.0-20250227040546-863c03f34bb8/go.mod h1:IXXaZkb7vGzGnGM5RRWrASAuwrVSNxuoe0DmeXx5g6k=
git.zhangdeman.cn/zhangdeman/consts v0.0.0-20250425024726-cc17224cb995 h1:LmPRAf0AsxRVFPibdpZR89ajlsz8hof2IvMMyTqiEq4=
git.zhangdeman.cn/zhangdeman/consts v0.0.0-20250425024726-cc17224cb995/go.mod h1:5p8CEKGBxi7qPtTXDI3HDmqKAfIm5i/aBWdrbkbdNjc=
git.zhangdeman.cn/zhangdeman/easylock v0.0.0-20230731062340-983985c12eda h1:bMD6r9gjRy7cO+T4zRQVYAesgIblBdTnhzT1vN5wjvI=
git.zhangdeman.cn/zhangdeman/easylock v0.0.0-20230731062340-983985c12eda/go.mod h1:dT0rmHcJ9Z9IqWeMIt7YzR88nKkNV2V3dfG0j9Q6lK0=
git.zhangdeman.cn/zhangdeman/easymap v0.0.0-20241101082529-28a6c68e38a4 h1:s6d4b6yY+NaK1AzoBD1pxqsuygEHQz0Oie86c45geDw=
git.zhangdeman.cn/zhangdeman/easymap v0.0.0-20241101082529-28a6c68e38a4/go.mod h1:V4Dfg1v/JVIZGEKCm6/aehs8hK+Xow1dkL1yiQymXlQ=
git.zhangdeman.cn/zhangdeman/gin v0.0.0-20250225092214-3432087fbd07 h1:SadUKF3SZhuRTXaCKyQWEavN9fVLNbHL/GBz/KoiL6o=
git.zhangdeman.cn/zhangdeman/gin v0.0.0-20250225092214-3432087fbd07/go.mod h1:T2Q8Wcq98yTuSSaEPZVAZfs0DMSxeXMN10GOQCha5g4=
git.zhangdeman.cn/zhangdeman/gin v0.0.0-20250228104311-2fd9195b77e7 h1:koL8c0do1mOLFY+wLMqSpojgHSwVRbV6sSsJVKo9WfA=
git.zhangdeman.cn/zhangdeman/gin v0.0.0-20250228104311-2fd9195b77e7/go.mod h1:T2Q8Wcq98yTuSSaEPZVAZfs0DMSxeXMN10GOQCha5g4=
git.zhangdeman.cn/zhangdeman/gin v0.0.0-20250413074621-24f33309b9d8 h1:8Wt/SSVJSBR8/nddY+YYERTExc0DHrmShkk6GFUdWzw=
git.zhangdeman.cn/zhangdeman/gin v0.0.0-20250413074621-24f33309b9d8/go.mod h1:HRY0KP893Oo1TZLPKv3XhAIhnMdtIipiylNsmJfZzD4=
git.zhangdeman.cn/zhangdeman/network v0.0.0-20250224022106-1c57dcf5afd9 h1:MphPBVuufQt4O2Nm+A2ldG/dMmb0LXZwAGcE/OkTRoU=
git.zhangdeman.cn/zhangdeman/network v0.0.0-20250224022106-1c57dcf5afd9/go.mod h1:vSHUJdlbSVDheL+e7KtdG3n/fgb26J/JOMVuEiXG+A8=
git.zhangdeman.cn/zhangdeman/network v0.0.0-20250425082854-19992ab41c78 h1:IFFHrl01z//K0aoNGzLkb46SQfFDaiSONwivWZpa3uE=
git.zhangdeman.cn/zhangdeman/network v0.0.0-20250425082854-19992ab41c78/go.mod h1:LKktkiO4at2XvPh/i9VPuT17DxLbBcDHUGMdrRSHNO4=
git.zhangdeman.cn/zhangdeman/op_type v0.0.0-20240122104027-4928421213c0 h1:gUDlQMuJ4xNfP2Abl1Msmpa3fASLWYkNlqDFF/6GN0Y=
git.zhangdeman.cn/zhangdeman/op_type v0.0.0-20240122104027-4928421213c0/go.mod h1:VHb9qmhaPDAQDcS6vUiDCamYjZ4R5lD1XtVsh55KsMI=
git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20241223084948-de2e49144fcd h1:q7GG14qgXKB4MEXQFOe7/UYebsqMfPaSX80TcPdOosI=
git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20241223084948-de2e49144fcd/go.mod h1:+D6uPSljwHywjVY5WSBY4TRVMj26TN5f5cFGEYMldjs=
git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20250426132259-73cf1be49c7f h1:7QgAcGnmVEVyIPeWH0ZkQN/jpzklYXsKCenTR2GpxbE=
git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20250426132259-73cf1be49c7f/go.mod h1:Ig3GZC2hJDkQp7F8Tm53GvMWLh9bdbbauow/vxGO4YA=
git.zhangdeman.cn/zhangdeman/trace v0.0.0-20250412104923-c1ecb1bfe8d5 h1:dD1Q/MIrRmIhKqfYPH+y167ca9CKwTPuQt3c1hXWGJ8=
@ -28,27 +18,15 @@ git.zhangdeman.cn/zhangdeman/util v0.0.0-20240618042405-6ee2c904644e h1:Q973S6Cc
git.zhangdeman.cn/zhangdeman/util v0.0.0-20240618042405-6ee2c904644e/go.mod h1:VpPjBlwz8U+OxZuxzHQBv1aEEZ3pStH6bZvT21ADEbI=
git.zhangdeman.cn/zhangdeman/websocket v0.0.0-20241125101541-c5ea194c9c1e h1:YE2Gi+M03UDImIpWa3I7jzSesyfu2RL8x/4ONs5v0oE=
git.zhangdeman.cn/zhangdeman/websocket v0.0.0-20241125101541-c5ea194c9c1e/go.mod h1:L/7JugxKZL3JP9JP/XDvPAPz0FQXG1u181Su1+u/d1c=
git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20250124091620-c757e551a8c9 h1:yF770WIDNwyiKL0nwmBGmjZvNCLXtHQL4xJyffPjTMU=
git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20250124091620-c757e551a8c9/go.mod h1:I76wxEsWq7KnMQ84elpwTjEqq4I49QFw60tp5h7iGBs=
git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20250302133417-c1588abcb436 h1:SM4zc54W2wmM72+4pMNQ8iS371H6lj4J8rj8KJKf7pw=
git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20250302133417-c1588abcb436/go.mod h1:YJ1FlvFgkfAHkxkt3l5rKKUqEpQkNMbCFDzDmgteEU8=
git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20250321102712-1cbfbe959740 h1:zPUoylfJTbc0EcxW+NEzOTBmoeFZ2I/rLFBnEzxb4Wk=
git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20250321102712-1cbfbe959740/go.mod h1:1ct92dbVc49pmXusA/iGfcQUJzcYmJ+cjAhgc3sDv1I=
github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg=
github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 h1:OYA+5W64v3OgClL+IrOD63t4i/RW7RqrAVl9LTZ9UqQ=
github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394/go.mod h1:Q8n74mJTIgjX4RBBcHnJ05h//6/k6foqmgE45jTQtxg=
github.com/bytedance/sonic v1.12.9 h1:Od1BvK55NnewtGaJsTDeAOSnLVO2BTSLOe0+ooKokmQ=
github.com/bytedance/sonic v1.12.9/go.mod h1:uVvFidNmlt9+wa31S1urfwwthTWteBgG0hWuoKAXTx8=
github.com/bytedance/sonic v1.12.10 h1:uVCQr6oS5669E9ZVW0HyksTLfNS7Q/9hV6IVS4nEMsI=
github.com/bytedance/sonic v1.12.10/go.mod h1:uVvFidNmlt9+wa31S1urfwwthTWteBgG0hWuoKAXTx8=
github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ=
github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4=
github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
github.com/bytedance/sonic/loader v0.2.3 h1:yctD0Q3v2NOGfSWPLPvG2ggA2kV6TS6s4wioyEqssH0=
github.com/bytedance/sonic/loader v0.2.3/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI=
github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY=
github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI=
github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4=
@ -57,15 +35,10 @@ github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQ
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1 h1:CaO/zOnF8VvUfEbhRatPcwKVWamvbYd8tQGRWacE9kU=
github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1/go.mod h1:+hnT3ywWDTAFrW5aE+u2Sa/wT555ZqwoCS+pk3p6ry4=
github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM=
github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8=
github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY=
github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok=
github.com/gin-contrib/sse v1.0.0 h1:y3bT1mUWUxDpW4JLQg/HnTqV4rozuW4tC9eFKTxYI9E=
github.com/gin-contrib/sse v1.0.0/go.mod h1:zNuFdwarAygJBht0NTKiSi3jRf6RbqeILZ9Sp6Slhe0=
github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w=
github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM=
github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU=
@ -78,8 +51,6 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.25.0 h1:5Dh7cjvzR7BRZadnsVOzPhWsrwUr0nmsZJxEAnFLNO8=
github.com/go-playground/validator/v10 v10.25.0/go.mod h1:GGzBIJMuE98Ic/kJsBXbz1x/7cByt++cQ+YOuDM5wus=
github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k=
github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo=
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
@ -89,6 +60,8 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g=
github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
@ -99,6 +72,8 @@ github.com/jonboulle/clockwork v0.3.0 h1:9BSCMi8C+0qdApAp4auwX0RkLGUjs956h0EkuQy
github.com/jonboulle/clockwork v0.3.0/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
@ -121,8 +96,6 @@ github.com/mcuadros/go-defaults v1.2.0 h1:FODb8WSf0uGaY8elWJAkoLL0Ri6AlZ1bFlenk5
github.com/mcuadros/go-defaults v1.2.0/go.mod h1:WEZtHEVIGYVDqkKSWBdWKUVdRyKlMfulPaGDWIVeCWY=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@ -134,8 +107,6 @@ github.com/mssola/user_agent v0.6.0 h1:uwPR4rtWlCHRFyyP9u2KOV0u8iQXmS7Z7feTrstQw
github.com/mssola/user_agent v0.6.0/go.mod h1:TTPno8LPY3wAIEKRpAtkdMT0f8SE24pLRGPahjCH4uw=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@ -144,18 +115,20 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sbabiv/xml2map v1.2.1 h1:1lT7t0hhUvXZCkdxqtq4n8/ZCnwLWGq4rDuDv5XOoFE=
github.com/sbabiv/xml2map v1.2.1/go.mod h1:2TPoAfcaM7+Sd4iriPvzyntb2mx7GY+kkQpB/GQa/eo=
github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY=
github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec=
github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY=
github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60=
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
@ -175,33 +148,19 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
golang.org/x/arch v0.14.0 h1:z9JUEZWr8x4rR0OU6c4/4t6E6jOZ8/QBS2bBYBm4tx4=
golang.org/x/arch v0.14.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
golang.org/x/arch v0.16.0 h1:foMtLTdyOmIniqWCHjY6+JxuC54XP1fDwx4N0ASyW+U=
golang.org/x/arch v0.16.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE=
golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs=
golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ=
golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8=
golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk=
golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA=
golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I=
golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY=
golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM=
google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

View File

@ -61,7 +61,6 @@ func NewLogger(inputLoggerLevel consts.LogLevel, splitConfig *RotateLogConfig, o
fileHandlerList := []zapcore.Core{
zapcore.NewCore(o.Encoder, zapcore.AddSync(loggerWriter), loggerLevelDeal), // 指定的日志输出位置
zapcore.NewCore(o.Encoder, zapcore.AddSync(&wsWriter{}), loggerLevelDeal), // 设置ws日志输出
// TODO: zinc日志同步
}
// 设置控制台输出
@ -69,6 +68,11 @@ func NewLogger(inputLoggerLevel consts.LogLevel, splitConfig *RotateLogConfig, o
fileHandlerList = append(fileHandlerList, zapcore.NewCore(o.Encoder, zapcore.AddSync(os.Stdout), loggerLevelDeal))
}
// 设置zinc日志输出
if nil != o.ZincCollectConfig {
fileHandlerList = append(fileHandlerList, zapcore.NewCore(o.Encoder, zapcore.AddSync(NewZincLogConnect(o.ZincCollectConfig)), loggerLevelDeal))
}
// 最后创建具体的Logger
core := zapcore.NewTee(fileHandlerList...)

View File

@ -1,107 +0,0 @@
// Package wrapper ...
//
// Description : 基于zinc实现日志收集
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2025-04-26 21:24
package wrapper
import (
"fmt"
"git.zhangdeman.cn/zhangdeman/consts"
"git.zhangdeman.cn/zhangdeman/serialize"
"go.uber.org/zap/zapcore"
"io"
"strings"
"sync"
"time"
)
// ZincConfig zinc服务配置
type ZincConfig struct {
Authorization string `json:"authorization" dc:"授权secret,生成方式base64(user:password)"`
Domain string `json:"domain" dc:"zinc服务域名"`
Timeout int `json:"timeout" dc:"超时时间,单位毫秒,默认5000"`
Async bool `json:"async" dc:"数据异步写入"`
Index string `json:"index" dc:"日志使用的索引"`
LogLevel consts.LogLevel `json:"log_level" dc:"记录的日志等级"`
CreateType string `json:"create_type" dc:"日志同步的类型: single - 单个同步 batch - 批量创建"`
BufferSize int `json:"buffer_size" dc:"批量创建时, 数据缓存buffer大小, 默认1000"`
ForceSyncTime int `json:"force_sync_time" dc:"批量同步日志时,强制同步的时间间隔,buffer没满也会强制同步, 单位: 秒"`
}
const (
DefaultTimeout = 5000 // 默认超时时间
DefaultBufferSize = 1000 // 默认buffer大小
)
const (
CreateTypeSingle = "single" // 逐条日志同步
CreateTypeBatch = "batch" // 批量日志同步
)
func NewZincLogConnect(cfg *ZincConfig) io.Writer {
return &zincLogConnect{
config: cfg,
lock: &sync.RWMutex{},
buffer: nil,
}
}
// zincLogConnect zinc日志收集器
type zincLogConnect struct {
config *ZincConfig // zinc配置
lock *sync.RWMutex // 操作锁
buffer []map[string]any // 数据缓冲buffer
}
// getPutLogFullUrl 获取请求地址,使用批量创建
func (zlc *zincLogConnect) getPutLogFullUrl() string {
if zlc.config.CreateType == CreateTypeBatch {
// 批量创建
return fmt.Sprintf("%v/api/_bulkv2", strings.TrimRight(zlc.config.Domain, "/"))
}
// 单条创建
return fmt.Sprintf("%v/api/%v/_doc", strings.TrimRight(zlc.config.Domain, "/"), zlc.config.Index)
}
// Write 日志写入, 要求字节数据必须是个合法的json序列化之后的结果
func (zlc *zincLogConnect) Write(logData []byte) (int, error) {
var (
res map[string]any
err error
)
if err = serialize.JSON.UnmarshalWithNumber(logData, &res); nil != err {
return 0, err
}
if zlc.config.CreateType == CreateTypeBatch {
// 批量
zlc.lock.Lock()
// 数据蟹醋buffer
zlc.buffer = append(zlc.buffer, res)
if len(zlc.buffer) >= zlc.config.BufferSize {
// buffer已满, 数据写入zinc
zlc.flush()
}
}
// 单个
return 0, nil
}
// flush 日志刷入zinc服务
func (zlc *zincLogConnect) flush() {
// TODO: 数据同步
// 日志刷入完成后, 清空buffer
zlc.buffer = []map[string]any{}
}
// flushTask 批量同步日志, 强制同步的定时任务
func (zlc *zincLogConnect) flushTask() {
for {
time.Sleep(time.Second * time.Duration(zlc.config.ForceSyncTime))
zlc.lock.Lock()
zlc.flush()
zlc.lock.Lock()
}
}

140
zinc_search.go Normal file
View File

@ -0,0 +1,140 @@
// Package logger ...
//
// Description : 基于zinc实现日志收集
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2025-04-26 21:24
package logger
import (
"bytes"
"fmt"
"git.zhangdeman.cn/zhangdeman/consts"
"git.zhangdeman.cn/zhangdeman/serialize"
"io"
"log"
"net/http"
"strings"
"sync"
"time"
)
func NewZincLogConnect(cfg *ZincConfig) io.Writer {
zlc := &zincLogConnect{
config: cfg,
lock: &sync.RWMutex{},
buffer: nil,
}
// 批量写入强制同步任务
go zlc.flushTask()
return zlc
}
// zincLogConnect zinc日志收集器
type zincLogConnect struct {
config *ZincConfig // zinc配置
lock *sync.RWMutex // 操作锁
buffer []map[string]any // 数据缓冲buffer
}
// getPutLogFullUrl 获取请求地址,使用批量创建
func (zlc *zincLogConnect) getPutLogFullUrl() string {
if zlc.config.CreateType == CreateTypeBatch {
// 批量创建
return fmt.Sprintf("%v/api/_bulkv2", strings.TrimRight(zlc.config.Domain, "/"))
}
// 单条创建
return fmt.Sprintf("%v/api/%v/_doc", strings.TrimRight(zlc.config.Domain, "/"), zlc.config.Index)
}
// Write 日志写入, 要求字节数据必须是个合法的json序列化之后的结果
func (zlc *zincLogConnect) Write(logData []byte) (int, error) {
if zlc.config.CreateType == CreateTypeBatch {
var (
res map[string]any
err error
)
if err = serialize.JSON.UnmarshalWithNumber(logData, &res); nil != err {
return 0, err
}
// 批量
zlc.lock.Lock()
// 数据写入buffer
zlc.buffer = append(zlc.buffer, res)
if len(zlc.buffer) >= zlc.config.BufferSize {
// buffer已满, 数据写入zinc
zlc.flush()
}
zlc.lock.Unlock()
return 0, nil
}
// 单个直接写入
zlc.writeData(logData)
return 0, nil
}
// flush 日志刷入zinc服务(针对批量数据写入的)
func (zlc *zincLogConnect) flush() {
if zlc.config.CreateType != CreateTypeBatch {
return
}
if len(zlc.buffer) == 0 {
return
}
zlc.writeData(serialize.JSON.MarshalForByteIgnoreError(map[string]any{
"index": zlc.config.Index,
"records": zlc.buffer,
}))
// 清空buffer
zlc.buffer = []map[string]any{}
}
// flushTask 批量同步日志, 强制同步的定时任务(针对批量数据写入)
func (zlc *zincLogConnect) flushTask() {
if zlc.config.CreateType != CreateTypeBatch {
return
}
for {
time.Sleep(time.Millisecond * time.Duration(zlc.config.ForceSyncTime))
zlc.lock.Lock()
zlc.flush()
zlc.lock.Unlock()
}
}
// writeData 数据写入zinc
func (zlc *zincLogConnect) writeData(paramData []byte) {
req, _ := http.NewRequest(http.MethodPost, zlc.getPutLogFullUrl(), bytes.NewReader(paramData))
req.Header.Set(consts.HeaderKeyAuthorization.String(), "Basic "+zlc.config.Authorization) // 设置authorization
req.Header.Set(consts.HeaderKeyContentType.String(), consts.MimeTypeJson) // json请求
client := &http.Client{}
if zlc.config.Async {
// 异步请求
go func() {
resp, err := client.Do(req)
if err != nil {
log.Print("异步日志写入zinc失败(请求发送失败): " + err.Error())
return
}
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
// 状态码非2xx
log.Print("异步日志写入zinc失败(zinc服务响应状态码异常): " + resp.Status)
return
}
}()
} else {
// 同步请求
resp, err := client.Do(req)
if err != nil {
log.Print("异步日志写入zinc失败(请求发送失败): " + err.Error())
return
}
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
// 状态码非2xx
log.Print("异步日志写入zinc失败(zinc服务响应状态码异常): " + resp.Status)
return
}
}
}