diff --git a/config.go b/config.go index f2e096b..caebfa6 100644 --- a/config.go +++ b/config.go @@ -15,8 +15,6 @@ import ( "time" "git.zhangdeman.cn/zhangdeman/consts" - "git.zhangdeman.cn/zhangdeman/websocket/storage" - "go.uber.org/zap/zapcore" ) @@ -224,19 +222,19 @@ func defaultEncodeDuration(d time.Duration, enc zapcore.PrimitiveArrayEncoder) { // // Date : 11:41 下午 2021/1/2 type OptionLogger struct { - UseJsonFormat bool // 日志使用json格式 - MessageKey string // message 字段 - LevelKey string // level 字段 - TimeKey string // 时间字段 - CallerKey string // 记录日志的文件的代码行数 - UseShortCaller bool // 使用短的调用文件格式 - TimeEncoder zapcore.TimeEncoder // 格式化时间的函数 - EncodeDuration zapcore.DurationEncoder // 原始时间信息 - WithCaller bool // 是否打印文件行号 - WithCallerSkip int // 跳过的调用数 - ConsoleOutput bool // 控制台输出 - Encoder zapcore.Encoder // 编码函数 - WsLoggerConnect storage.IConnection `json:"-"` // ws 日志连接管理实例 + UseJsonFormat bool // 日志使用json格式 + MessageKey string // message 字段 + LevelKey string // level 字段 + TimeKey string // 时间字段 + CallerKey string // 记录日志的文件的代码行数 + UseShortCaller bool // 使用短的调用文件格式 + TimeEncoder zapcore.TimeEncoder // 格式化时间的函数 + EncodeDuration zapcore.DurationEncoder // 原始时间信息 + WithCaller bool // 是否打印文件行号 + WithCallerSkip int // 跳过的调用数 + ConsoleOutput bool // 控制台输出 + Encoder zapcore.Encoder // 编码函数 + ZincCollectConfig *ZincConfig // zinc采集配置 } // SetLoggerOptionFunc 设置日志配置 @@ -380,6 +378,13 @@ func WithEncodeDuration(encoder zapcore.DurationEncoder) SetLoggerOptionFunc { } } +// WithZincLogCollect zinc日志采集 +func WithZincLogCollect(zincCfg *ZincConfig) SetLoggerOptionFunc { + return func(o *OptionLogger) { + o.ZincCollectConfig = zincCfg + } +} + // GetEncoder 获取空中台输出的encoder // // Author : go_developer@163.com<白茶清欢> diff --git a/define.go b/define.go index 49999da..9b2899f 100644 --- a/define.go +++ b/define.go @@ -83,6 +83,7 @@ type InputLogConfig struct { UseShortFile bool `json:"use_short_file" yaml:"use_short_file"` // 是否使用短文件格式 CallerSkip int `json:"caller_skip" yaml:"caller_skip"` // 日志记录的文件跳过多少层 MaxAge int `json:"max_age" yaml:"max_age"` // 日志最长保存时间, 单位 : 秒 + ZincSyncConfig *ZincConfig `json:"zinc_sync_config" yaml:"zinc_sync_config"` // 日志同步至zinc的配置 } // GetLogInstanceFromInputConfig 从输入配置获取日志实例 @@ -103,6 +104,8 @@ func GetLogInstanceFromInputConfig(logConf *InputLogConfig) (*zap.Logger, error) if logConf.Console { logConfList = append(logConfList, WithConsoleOutput()) } + // 配置zinc日志同步 + logConfList = append(logConfList, WithZincLogCollect(logConf.ZincSyncConfig)) var ( err error @@ -149,3 +152,26 @@ func inputLevel2ZapLevel(inputLoggerLevel consts.LogLevel) zapcore.Level { } return loggerLevel } + +// ZincConfig zinc服务配置 +type ZincConfig struct { + Authorization string `json:"authorization" dc:"授权secret,生成方式base64(user:password)"` + Domain string `json:"domain" dc:"zinc服务域名"` + Timeout int `json:"timeout" dc:"超时时间,单位毫秒,默认5000"` + Async bool `json:"async" dc:"数据异步写入"` + Index string `json:"index" dc:"日志使用的索引"` + CreateType string `json:"create_type" dc:"日志同步的类型: single - 单个同步 batch - 批量创建"` + BufferSize int `json:"buffer_size" dc:"批量创建时, 数据缓存buffer大小, 默认1000"` + ForceSyncTime int `json:"force_sync_time" dc:"批量同步日志时,强制同步的时间间隔,buffer没满也会强制同步, 单位: 秒"` +} + +const ( + DefaultTimeout = 5000 // 默认超时时间 + DefaultBufferSize = 1000 // 默认buffer大小 + DefaultForceFlushLogTime = 1000 // 强制刷新日志的时间间隔, 单位毫秒 +) + +const ( + CreateTypeSingle = "single" // 逐条日志同步 + CreateTypeBatch = "batch" // 批量日志同步 +) diff --git a/go.mod b/go.mod index 3c6d645..ab7b865 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( git.zhangdeman.cn/zhangdeman/consts v0.0.0-20250425024726-cc17224cb995 git.zhangdeman.cn/zhangdeman/gin v0.0.0-20250413074621-24f33309b9d8 git.zhangdeman.cn/zhangdeman/network v0.0.0-20250425082854-19992ab41c78 - git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20241223084948-de2e49144fcd + git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20250426132259-73cf1be49c7f git.zhangdeman.cn/zhangdeman/websocket v0.0.0-20241125101541-c5ea194c9c1e github.com/gin-gonic/gin v1.10.0 github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible diff --git a/go.sum b/go.sum index e2b1c1c..a20fd68 100644 --- a/go.sum +++ b/go.sum @@ -1,52 +1,32 @@ -git.zhangdeman.cn/zhangdeman/consts v0.0.0-20250227040546-863c03f34bb8 h1:VEifPc+vkpEQoX9rj7zxmT1m+IA81XjOxe7+Z1aqWNM= -git.zhangdeman.cn/zhangdeman/consts v0.0.0-20250227040546-863c03f34bb8/go.mod h1:IXXaZkb7vGzGnGM5RRWrASAuwrVSNxuoe0DmeXx5g6k= git.zhangdeman.cn/zhangdeman/consts v0.0.0-20250425024726-cc17224cb995 h1:LmPRAf0AsxRVFPibdpZR89ajlsz8hof2IvMMyTqiEq4= git.zhangdeman.cn/zhangdeman/consts v0.0.0-20250425024726-cc17224cb995/go.mod h1:5p8CEKGBxi7qPtTXDI3HDmqKAfIm5i/aBWdrbkbdNjc= git.zhangdeman.cn/zhangdeman/easylock v0.0.0-20230731062340-983985c12eda h1:bMD6r9gjRy7cO+T4zRQVYAesgIblBdTnhzT1vN5wjvI= git.zhangdeman.cn/zhangdeman/easylock v0.0.0-20230731062340-983985c12eda/go.mod h1:dT0rmHcJ9Z9IqWeMIt7YzR88nKkNV2V3dfG0j9Q6lK0= git.zhangdeman.cn/zhangdeman/easymap v0.0.0-20241101082529-28a6c68e38a4 h1:s6d4b6yY+NaK1AzoBD1pxqsuygEHQz0Oie86c45geDw= git.zhangdeman.cn/zhangdeman/easymap v0.0.0-20241101082529-28a6c68e38a4/go.mod h1:V4Dfg1v/JVIZGEKCm6/aehs8hK+Xow1dkL1yiQymXlQ= -git.zhangdeman.cn/zhangdeman/gin v0.0.0-20250225092214-3432087fbd07 h1:SadUKF3SZhuRTXaCKyQWEavN9fVLNbHL/GBz/KoiL6o= -git.zhangdeman.cn/zhangdeman/gin v0.0.0-20250225092214-3432087fbd07/go.mod h1:T2Q8Wcq98yTuSSaEPZVAZfs0DMSxeXMN10GOQCha5g4= -git.zhangdeman.cn/zhangdeman/gin v0.0.0-20250228104311-2fd9195b77e7 h1:koL8c0do1mOLFY+wLMqSpojgHSwVRbV6sSsJVKo9WfA= -git.zhangdeman.cn/zhangdeman/gin v0.0.0-20250228104311-2fd9195b77e7/go.mod h1:T2Q8Wcq98yTuSSaEPZVAZfs0DMSxeXMN10GOQCha5g4= git.zhangdeman.cn/zhangdeman/gin v0.0.0-20250413074621-24f33309b9d8 h1:8Wt/SSVJSBR8/nddY+YYERTExc0DHrmShkk6GFUdWzw= git.zhangdeman.cn/zhangdeman/gin v0.0.0-20250413074621-24f33309b9d8/go.mod h1:HRY0KP893Oo1TZLPKv3XhAIhnMdtIipiylNsmJfZzD4= -git.zhangdeman.cn/zhangdeman/network v0.0.0-20250224022106-1c57dcf5afd9 h1:MphPBVuufQt4O2Nm+A2ldG/dMmb0LXZwAGcE/OkTRoU= -git.zhangdeman.cn/zhangdeman/network v0.0.0-20250224022106-1c57dcf5afd9/go.mod h1:vSHUJdlbSVDheL+e7KtdG3n/fgb26J/JOMVuEiXG+A8= git.zhangdeman.cn/zhangdeman/network v0.0.0-20250425082854-19992ab41c78 h1:IFFHrl01z//K0aoNGzLkb46SQfFDaiSONwivWZpa3uE= git.zhangdeman.cn/zhangdeman/network v0.0.0-20250425082854-19992ab41c78/go.mod h1:LKktkiO4at2XvPh/i9VPuT17DxLbBcDHUGMdrRSHNO4= git.zhangdeman.cn/zhangdeman/op_type v0.0.0-20240122104027-4928421213c0 h1:gUDlQMuJ4xNfP2Abl1Msmpa3fASLWYkNlqDFF/6GN0Y= git.zhangdeman.cn/zhangdeman/op_type v0.0.0-20240122104027-4928421213c0/go.mod h1:VHb9qmhaPDAQDcS6vUiDCamYjZ4R5lD1XtVsh55KsMI= -git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20241223084948-de2e49144fcd h1:q7GG14qgXKB4MEXQFOe7/UYebsqMfPaSX80TcPdOosI= -git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20241223084948-de2e49144fcd/go.mod h1:+D6uPSljwHywjVY5WSBY4TRVMj26TN5f5cFGEYMldjs= +git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20250426132259-73cf1be49c7f h1:7QgAcGnmVEVyIPeWH0ZkQN/jpzklYXsKCenTR2GpxbE= +git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20250426132259-73cf1be49c7f/go.mod h1:Ig3GZC2hJDkQp7F8Tm53GvMWLh9bdbbauow/vxGO4YA= git.zhangdeman.cn/zhangdeman/trace v0.0.0-20250412104923-c1ecb1bfe8d5 h1:dD1Q/MIrRmIhKqfYPH+y167ca9CKwTPuQt3c1hXWGJ8= git.zhangdeman.cn/zhangdeman/trace v0.0.0-20250412104923-c1ecb1bfe8d5/go.mod h1:PB486NC82nuvn5yi+U2i48ogX/9EAETWAHd8O9TwY9k= git.zhangdeman.cn/zhangdeman/util v0.0.0-20240618042405-6ee2c904644e h1:Q973S6CcWr1ICZhFI1STFOJ+KUImCl2BaIXm6YppBqI= git.zhangdeman.cn/zhangdeman/util v0.0.0-20240618042405-6ee2c904644e/go.mod h1:VpPjBlwz8U+OxZuxzHQBv1aEEZ3pStH6bZvT21ADEbI= git.zhangdeman.cn/zhangdeman/websocket v0.0.0-20241125101541-c5ea194c9c1e h1:YE2Gi+M03UDImIpWa3I7jzSesyfu2RL8x/4ONs5v0oE= git.zhangdeman.cn/zhangdeman/websocket v0.0.0-20241125101541-c5ea194c9c1e/go.mod h1:L/7JugxKZL3JP9JP/XDvPAPz0FQXG1u181Su1+u/d1c= -git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20250124091620-c757e551a8c9 h1:yF770WIDNwyiKL0nwmBGmjZvNCLXtHQL4xJyffPjTMU= -git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20250124091620-c757e551a8c9/go.mod h1:I76wxEsWq7KnMQ84elpwTjEqq4I49QFw60tp5h7iGBs= -git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20250302133417-c1588abcb436 h1:SM4zc54W2wmM72+4pMNQ8iS371H6lj4J8rj8KJKf7pw= -git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20250302133417-c1588abcb436/go.mod h1:YJ1FlvFgkfAHkxkt3l5rKKUqEpQkNMbCFDzDmgteEU8= git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20250321102712-1cbfbe959740 h1:zPUoylfJTbc0EcxW+NEzOTBmoeFZ2I/rLFBnEzxb4Wk= git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20250321102712-1cbfbe959740/go.mod h1:1ct92dbVc49pmXusA/iGfcQUJzcYmJ+cjAhgc3sDv1I= -github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= -github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 h1:OYA+5W64v3OgClL+IrOD63t4i/RW7RqrAVl9LTZ9UqQ= github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394/go.mod h1:Q8n74mJTIgjX4RBBcHnJ05h//6/k6foqmgE45jTQtxg= -github.com/bytedance/sonic v1.12.9 h1:Od1BvK55NnewtGaJsTDeAOSnLVO2BTSLOe0+ooKokmQ= -github.com/bytedance/sonic v1.12.9/go.mod h1:uVvFidNmlt9+wa31S1urfwwthTWteBgG0hWuoKAXTx8= -github.com/bytedance/sonic v1.12.10 h1:uVCQr6oS5669E9ZVW0HyksTLfNS7Q/9hV6IVS4nEMsI= -github.com/bytedance/sonic v1.12.10/go.mod h1:uVvFidNmlt9+wa31S1urfwwthTWteBgG0hWuoKAXTx8= github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ= github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/bytedance/sonic/loader v0.2.3 h1:yctD0Q3v2NOGfSWPLPvG2ggA2kV6TS6s4wioyEqssH0= -github.com/bytedance/sonic/loader v0.2.3/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= @@ -55,15 +35,10 @@ github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1 h1:CaO/zOnF8VvUfEbhRatPcwKVWamvbYd8tQGRWacE9kU= github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1/go.mod h1:+hnT3ywWDTAFrW5aE+u2Sa/wT555ZqwoCS+pk3p6ry4= -github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= -github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY= github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok= -github.com/gin-contrib/sse v1.0.0 h1:y3bT1mUWUxDpW4JLQg/HnTqV4rozuW4tC9eFKTxYI9E= -github.com/gin-contrib/sse v1.0.0/go.mod h1:zNuFdwarAygJBht0NTKiSi3jRf6RbqeILZ9Sp6Slhe0= github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= @@ -76,8 +51,6 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.25.0 h1:5Dh7cjvzR7BRZadnsVOzPhWsrwUr0nmsZJxEAnFLNO8= -github.com/go-playground/validator/v10 v10.25.0/go.mod h1:GGzBIJMuE98Ic/kJsBXbz1x/7cByt++cQ+YOuDM5wus= github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= @@ -87,6 +60,8 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= +github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= @@ -97,6 +72,8 @@ github.com/jonboulle/clockwork v0.3.0 h1:9BSCMi8C+0qdApAp4auwX0RkLGUjs956h0EkuQy github.com/jonboulle/clockwork v0.3.0/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= @@ -119,8 +96,6 @@ github.com/mcuadros/go-defaults v1.2.0 h1:FODb8WSf0uGaY8elWJAkoLL0Ri6AlZ1bFlenk5 github.com/mcuadros/go-defaults v1.2.0/go.mod h1:WEZtHEVIGYVDqkKSWBdWKUVdRyKlMfulPaGDWIVeCWY= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -132,8 +107,6 @@ github.com/mssola/user_agent v0.6.0 h1:uwPR4rtWlCHRFyyP9u2KOV0u8iQXmS7Z7feTrstQw github.com/mssola/user_agent v0.6.0/go.mod h1:TTPno8LPY3wAIEKRpAtkdMT0f8SE24pLRGPahjCH4uw= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= -github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -142,18 +115,20 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sbabiv/xml2map v1.2.1 h1:1lT7t0hhUvXZCkdxqtq4n8/ZCnwLWGq4rDuDv5XOoFE= github.com/sbabiv/xml2map v1.2.1/go.mod h1:2TPoAfcaM7+Sd4iriPvzyntb2mx7GY+kkQpB/GQa/eo= +github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= +github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= +github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= +github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= @@ -173,33 +148,19 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/arch v0.14.0 h1:z9JUEZWr8x4rR0OU6c4/4t6E6jOZ8/QBS2bBYBm4tx4= -golang.org/x/arch v0.14.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/arch v0.16.0 h1:foMtLTdyOmIniqWCHjY6+JxuC54XP1fDwx4N0ASyW+U= golang.org/x/arch v0.16.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE= -golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= -golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= -golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= -golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= -golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= -golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= -golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= -google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/logger.go b/logger.go index 1de6473..76d0363 100644 --- a/logger.go +++ b/logger.go @@ -59,8 +59,8 @@ func NewLogger(inputLoggerLevel consts.LogLevel, splitConfig *RotateLogConfig, o } fileHandlerList := []zapcore.Core{ - zapcore.NewCore(o.Encoder, zapcore.AddSync(loggerWriter), loggerLevelDeal), - zapcore.NewCore(o.Encoder, zapcore.AddSync(&wsWriter{}), loggerLevelDeal), // 设置ws日志输出 + zapcore.NewCore(o.Encoder, zapcore.AddSync(loggerWriter), loggerLevelDeal), // 指定的日志输出位置 + zapcore.NewCore(o.Encoder, zapcore.AddSync(&wsWriter{}), loggerLevelDeal), // 设置ws日志输出 } // 设置控制台输出 @@ -68,6 +68,11 @@ func NewLogger(inputLoggerLevel consts.LogLevel, splitConfig *RotateLogConfig, o fileHandlerList = append(fileHandlerList, zapcore.NewCore(o.Encoder, zapcore.AddSync(os.Stdout), loggerLevelDeal)) } + // 设置zinc日志输出 + if nil != o.ZincCollectConfig { + fileHandlerList = append(fileHandlerList, zapcore.NewCore(o.Encoder, zapcore.AddSync(NewZincLogConnect(o.ZincCollectConfig)), loggerLevelDeal)) + } + // 最后创建具体的Logger core := zapcore.NewTee(fileHandlerList...) diff --git a/zinc_search.go b/zinc_search.go new file mode 100644 index 0000000..28cd9b7 --- /dev/null +++ b/zinc_search.go @@ -0,0 +1,149 @@ +// Package logger ... +// +// Description : 基于zinc实现日志收集 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 2025-04-26 21:24 +package logger + +import ( + "bytes" + "fmt" + "git.zhangdeman.cn/zhangdeman/consts" + "git.zhangdeman.cn/zhangdeman/serialize" + "io" + "log" + "net/http" + "strings" + "sync" + "time" +) + +func NewZincLogConnect(cfg *ZincConfig) io.Writer { + if cfg.Timeout <= 0 { + cfg.Timeout = DefaultTimeout + } + if cfg.BufferSize <= 0 { + cfg.BufferSize = DefaultBufferSize + } + if cfg.ForceSyncTime <= 0 { + cfg.ForceSyncTime = DefaultForceFlushLogTime + } + zlc := &zincLogConnect{ + config: cfg, + lock: &sync.RWMutex{}, + buffer: nil, + } + // 批量写入强制同步任务 + go zlc.flushTask() + return zlc +} + +// zincLogConnect zinc日志收集器 +type zincLogConnect struct { + config *ZincConfig // zinc配置 + lock *sync.RWMutex // 操作锁 + buffer []map[string]any // 数据缓冲buffer +} + +// getPutLogFullUrl 获取请求地址,使用批量创建 +func (zlc *zincLogConnect) getPutLogFullUrl() string { + if zlc.config.CreateType == CreateTypeBatch { + // 批量创建 + return fmt.Sprintf("%v/api/_bulkv2", strings.TrimRight(zlc.config.Domain, "/")) + } + // 单条创建 + return fmt.Sprintf("%v/api/%v/_doc", strings.TrimRight(zlc.config.Domain, "/"), zlc.config.Index) +} + +// Write 日志写入, 要求字节数据必须是个合法的json序列化之后的结果 +func (zlc *zincLogConnect) Write(logData []byte) (int, error) { + if zlc.config.CreateType == CreateTypeBatch { + var ( + res map[string]any + err error + ) + if err = serialize.JSON.UnmarshalWithNumber(logData, &res); nil != err { + return 0, err + } + // 批量 + zlc.lock.Lock() + // 数据写入buffer + zlc.buffer = append(zlc.buffer, res) + if len(zlc.buffer) >= zlc.config.BufferSize { + // buffer已满, 数据写入zinc + zlc.flush() + } + zlc.lock.Unlock() + return 0, nil + } + // 单个直接写入 + zlc.writeData(logData) + return 0, nil +} + +// flush 日志刷入zinc服务(针对批量数据写入的) +func (zlc *zincLogConnect) flush() { + if zlc.config.CreateType != CreateTypeBatch { + return + } + if len(zlc.buffer) == 0 { + return + } + zlc.writeData(serialize.JSON.MarshalForByteIgnoreError(map[string]any{ + "index": zlc.config.Index, + "records": zlc.buffer, + })) + // 清空buffer + zlc.buffer = []map[string]any{} +} + +// flushTask 批量同步日志, 强制同步的定时任务(针对批量数据写入) +func (zlc *zincLogConnect) flushTask() { + if zlc.config.CreateType != CreateTypeBatch { + return + } + for { + time.Sleep(time.Millisecond * time.Duration(zlc.config.ForceSyncTime)) + zlc.lock.Lock() + zlc.flush() + zlc.lock.Unlock() + } +} + +// writeData 数据写入zinc +func (zlc *zincLogConnect) writeData(paramData []byte) { + req, _ := http.NewRequest(http.MethodPost, zlc.getPutLogFullUrl(), bytes.NewReader(paramData)) + req.Header.Set(consts.HeaderKeyAuthorization.String(), "Basic "+zlc.config.Authorization) // 设置authorization + req.Header.Set(consts.HeaderKeyContentType.String(), consts.MimeTypeJson) // json请求 + client := &http.Client{} + + if zlc.config.Async { + // 异步请求 + go func() { + resp, err := client.Do(req) + if err != nil { + log.Print("异步日志写入zinc失败(请求发送失败): " + err.Error()) + return + } + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + // 状态码非2xx + log.Print("异步日志写入zinc失败(zinc服务响应状态码异常): " + resp.Status) + return + } + }() + } else { + // 同步请求 + resp, err := client.Do(req) + if err != nil { + log.Print("异步日志写入zinc失败(请求发送失败): " + err.Error()) + return + } + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + // 状态码非2xx + log.Print("异步日志写入zinc失败(zinc服务响应状态码异常): " + resp.Status) + return + } + } +}