Compare commits
6 Commits
feature/cr
...
feature/my
| Author | SHA1 | Date | |
|---|---|---|---|
| ce1321b5eb | |||
| 0faf50ceba | |||
| 9daa1ba251 | |||
| af174ba501 | |||
| e152823b21 | |||
| ad39fe5b68 |
@ -1,28 +0,0 @@
|
||||
// Package crawler ...
|
||||
//
|
||||
// Description : crawler ...
|
||||
//
|
||||
// Author : go_developer@163.com<白茶清欢>
|
||||
//
|
||||
// Date : 2021-12-20 4:46 PM
|
||||
package crawler
|
||||
|
||||
import "github.com/gocolly/colly"
|
||||
|
||||
// StartHTMLCollector 获取页面爬虫实例
|
||||
//
|
||||
// Author : go_developer@163.com<白茶清欢>
|
||||
//
|
||||
// Date : 4:47 PM 2021/12/20
|
||||
func StartHTMLCollector(domainList []string, visitURL string, requestHandler IRequestHandler) error {
|
||||
c := colly.NewCollector()
|
||||
// 设置域名白名单, 不设置, 默认所有均可访问
|
||||
c.AllowedDomains = domainList
|
||||
c.OnRequest(requestHandler.OnRequest())
|
||||
c.OnError(requestHandler.OnError())
|
||||
// html处理
|
||||
c.OnHTML(requestHandler.OnHTML())
|
||||
c.OnResponse(requestHandler.OnResponse())
|
||||
c.OnScraped(requestHandler.OnScraped())
|
||||
return c.Visit(visitURL)
|
||||
}
|
||||
@ -1,64 +0,0 @@
|
||||
// Package crawler ...
|
||||
//
|
||||
// Description : crawler ...
|
||||
//
|
||||
// Author : go_developer@163.com<白茶清欢>
|
||||
//
|
||||
// Date : 2021-12-20 5:58 PM
|
||||
package crawler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/gocolly/colly"
|
||||
)
|
||||
|
||||
// TestStartCollector ...
|
||||
//
|
||||
// Author : go_developer@163.com<白茶清欢>
|
||||
//
|
||||
// Date : 5:59 PM 2021/12/20
|
||||
func TestStartCollector(t *testing.T) {
|
||||
if err := StartHTMLCollector([]string{}, "https://go.zhangdeman.cn", &testHandler{}); nil != err {
|
||||
panic("出现异常 : " + err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
type testHandler struct {
|
||||
}
|
||||
|
||||
func (t *testHandler) OnRequest() colly.RequestCallback {
|
||||
return func(r *colly.Request) {
|
||||
fmt.Println("开始请求 : ", r.URL)
|
||||
}
|
||||
}
|
||||
|
||||
func (t *testHandler) OnError() colly.ErrorCallback {
|
||||
return func(response *colly.Response, err error) {
|
||||
fmt.Println("请求异常 : " + err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func (t *testHandler) OnResponse() colly.ResponseCallback {
|
||||
return func(response *colly.Response) {
|
||||
fmt.Println("响应数据 : ", response.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func (t *testHandler) OnHTML() (string, colly.HTMLCallback) {
|
||||
return "a[href]", func(e *colly.HTMLElement) {
|
||||
link := e.Attr("href")
|
||||
|
||||
// Print link
|
||||
|
||||
fmt.Printf("Link found: %q -> %s\n", e.Text, link)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
func (t *testHandler) OnScraped() colly.ScrapedCallback {
|
||||
return func(response *colly.Response) {
|
||||
|
||||
}
|
||||
}
|
||||
@ -1,8 +0,0 @@
|
||||
// Package crawler ...
|
||||
//
|
||||
// Description : crawler ...
|
||||
//
|
||||
// Author : go_developer@163.com<白茶清欢>
|
||||
//
|
||||
// Date : 2021-12-20 4:41 PM
|
||||
package crawler
|
||||
@ -1,30 +0,0 @@
|
||||
// Package crawler ...
|
||||
//
|
||||
// Description : crawler ...
|
||||
//
|
||||
// Author : go_developer@163.com<白茶清欢>
|
||||
//
|
||||
// Date : 2021-12-20 4:50 PM
|
||||
package crawler
|
||||
|
||||
import (
|
||||
"github.com/gocolly/colly"
|
||||
)
|
||||
|
||||
// IRequestHandler 请求结果的处理
|
||||
//
|
||||
// Author : go_developer@163.com<白茶清欢>
|
||||
//
|
||||
// Date : 4:50 PM 2021/12/20
|
||||
type IRequestHandler interface {
|
||||
// OnRequest 在发起请求前被调用
|
||||
OnRequest() colly.RequestCallback
|
||||
// OnError 请求过程中如果发生错误被调用
|
||||
OnError() colly.ErrorCallback
|
||||
// OnResponse 收到回复后被调用
|
||||
OnResponse() colly.ResponseCallback
|
||||
// OnHTML 在OnResponse之后被调用，如果收到的内容是HTML
|
||||
OnHTML() (string, colly.HTMLCallback)
|
||||
// OnScraped 在OnHTML之后被调用
|
||||
OnScraped() colly.ScrapedCallback
|
||||
}
|
||||
18
gin/static/define.go
Normal file
18
gin/static/define.go
Normal file
@ -0,0 +1,18 @@
|
||||
// Package static ...
|
||||
//
|
||||
// Description : static
|
||||
//
|
||||
// Author : go_developer@163.com<白茶清欢>
|
||||
//
|
||||
// Date : 2021/12/24 2:18 PM
|
||||
package static
|
||||
|
||||
// MapRule 定义映射规则
|
||||
//
|
||||
// Author : go_developer@163.com<白茶清欢>
|
||||
//
|
||||
// Date : 2:18 PM 2021/12/24
|
||||
type MapRule struct {
|
||||
URIPrefix string `json:"uri_prefix"` // 路由前缀
|
||||
StaticDirPath string `json:"static_dir_path"` // 静态资源路由
|
||||
}
|
||||
25
gin/static/static.go
Normal file
25
gin/static/static.go
Normal file
@ -0,0 +1,25 @@
|
||||
// Package static ...
|
||||
//
|
||||
// Description : 启动静态资源服务器
|
||||
//
|
||||
// Author : go_developer@163.com<白茶清欢>
|
||||
//
|
||||
// Date : 2021/12/24 2:14 PM
|
||||
package static
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// Register 静态资源服务器
|
||||
//
|
||||
// Author : go_developer@163.com<白茶清欢>
|
||||
//
|
||||
// Date : 2:17 PM 2021/12/24
|
||||
func Register(router *gin.Engine, ruleList []*MapRule) {
|
||||
for _, rule := range ruleList {
|
||||
router.StaticFS(rule.URIPrefix, http.Dir(rule.StaticDirPath))
|
||||
}
|
||||
}
|
||||
17
go.mod
17
go.mod
@ -14,7 +14,6 @@ require (
|
||||
github.com/gin-gonic/gin v1.7.6
|
||||
github.com/go-redis/redis/v8 v8.11.4
|
||||
github.com/go-redis/redis_rate/v9 v9.1.2
|
||||
github.com/gocolly/colly v1.2.0
|
||||
github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/shirou/gopsutil v3.21.10+incompatible
|
||||
@ -28,16 +27,11 @@ require (
|
||||
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
|
||||
gopkg.in/yaml.v2 v2.4.0
|
||||
gorm.io/driver/mysql v1.2.0
|
||||
gorm.io/gorm v1.22.3
|
||||
gorm.io/gorm v1.22.4
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.8.0 // indirect
|
||||
github.com/StackExchange/wmi v1.2.1 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.1 // indirect
|
||||
github.com/antchfx/htmlquery v1.2.4 // indirect
|
||||
github.com/antchfx/xmlquery v1.3.9 // indirect
|
||||
github.com/antchfx/xpath v1.2.0 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
||||
github.com/coreos/bbolt v1.3.4 // indirect
|
||||
github.com/coreos/go-semver v0.3.0 // indirect
|
||||
@ -57,7 +51,6 @@ require (
|
||||
github.com/go-playground/universal-translator v0.17.0 // indirect
|
||||
github.com/go-playground/validator/v10 v10.4.1 // indirect
|
||||
github.com/go-sql-driver/mysql v1.6.0 // indirect
|
||||
github.com/gobwas/glob v0.2.3 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/golang/protobuf v1.5.2 // indirect
|
||||
@ -75,10 +68,9 @@ require (
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.2 // indirect
|
||||
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/jinzhu/now v1.1.2 // indirect
|
||||
github.com/jinzhu/now v1.1.4 // indirect
|
||||
github.com/jonboulle/clockwork v0.2.2 // indirect
|
||||
github.com/json-iterator/go v1.1.11 // indirect
|
||||
github.com/kennygrant/sanitize v1.2.4 // indirect
|
||||
github.com/klauspost/compress v1.13.6 // indirect
|
||||
github.com/leodido/go-urn v1.2.0 // indirect
|
||||
github.com/lestrrat-go/strftime v1.0.5 // indirect
|
||||
@ -95,7 +87,6 @@ require (
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
|
||||
github.com/richardlehane/mscfb v1.0.3 // indirect
|
||||
github.com/richardlehane/msoleps v1.0.1 // indirect
|
||||
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
|
||||
github.com/soheilhy/cmux v0.1.5 // indirect
|
||||
github.com/spf13/afero v1.6.0 // indirect
|
||||
github.com/spf13/cast v1.4.1 // indirect
|
||||
@ -103,7 +94,6 @@ require (
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/spf13/viper v1.9.0 // indirect
|
||||
github.com/subosito/gotenv v1.2.0 // indirect
|
||||
github.com/temoto/robotstxt v1.1.2 // indirect
|
||||
github.com/tidwall/match v1.1.1 // indirect
|
||||
github.com/tidwall/pretty v1.2.0 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.9 // indirect
|
||||
@ -115,11 +105,10 @@ require (
|
||||
go.uber.org/atomic v1.7.0 // indirect
|
||||
go.uber.org/multierr v1.6.0 // indirect
|
||||
golang.org/x/crypto v0.0.0-20210920023735-84f357641f63 // indirect
|
||||
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f // indirect
|
||||
golang.org/x/net v0.0.0-20210917221730-978cfadd31cf // indirect
|
||||
golang.org/x/sys v0.0.0-20211123173158-ef496fb156ab // indirect
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71 // indirect
|
||||
google.golang.org/grpc v1.40.0 // indirect
|
||||
google.golang.org/protobuf v1.27.1 // indirect
|
||||
|
||||
32
go.sum
32
go.sum
@ -44,8 +44,6 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
|
||||
github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
|
||||
github.com/Shopify/sarama v1.30.0 h1:TOZL6r37xJBDEMLx4yjB77jxbZYXPaDow08TSK6vIL0=
|
||||
github.com/Shopify/sarama v1.30.0/go.mod h1:zujlQQx1kzHsh4jfV1USnptCQrHAEZ2Hk8fTKCulPVs=
|
||||
github.com/Shopify/toxiproxy/v2 v2.1.6-0.20210914104332-15ea381dcdae h1:ePgznFqEG1v3AjMklnK8H7BSc++FDSo7xfK9K7Af+0Y=
|
||||
@ -57,14 +55,6 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
|
||||
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
|
||||
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
|
||||
github.com/antchfx/htmlquery v1.2.4 h1:qLteofCMe/KGovBI6SQgmou2QNyedFUW+pE+BpeZ494=
|
||||
github.com/antchfx/htmlquery v1.2.4/go.mod h1:2xO6iu3EVWs7R2JYqBbp8YzG50gj/ofqs5/0VZoDZLc=
|
||||
github.com/antchfx/xmlquery v1.3.9 h1:Y+zyMdiUZ4fasTQTkDb3DflOXP7+obcYEh80SISBmnQ=
|
||||
github.com/antchfx/xmlquery v1.3.9/go.mod h1:wojC/BxjEkjJt6dPiAqUzoXO5nIMWtxHS8PD8TmN4ks=
|
||||
github.com/antchfx/xpath v1.2.0 h1:mbwv7co+x0RwgeGAOHdrKy89GvHaGvxxBtPK0uF9Zr8=
|
||||
github.com/antchfx/xpath v1.2.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
|
||||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
github.com/apolloconfig/agollo/v4 v4.0.9 h1:4YzFSOVTZjGU2ag0XxL0y311TAkEsqh9a5pBS8FOgaI=
|
||||
github.com/apolloconfig/agollo/v4 v4.0.9/go.mod h1:n/7qxpKOTbegygLmO5OKmFWCdy3T+S/zioBGlo457Dk=
|
||||
@ -159,10 +149,6 @@ github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfC
|
||||
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
|
||||
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
|
||||
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
|
||||
github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI=
|
||||
github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA=
|
||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||
@ -308,6 +294,9 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI=
|
||||
github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/jinzhu/now v1.1.3/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/jinzhu/now v1.1.4 h1:tHnRBy1i5F2Dh8BAFxqFzxKqqvezXrL2OW1TnX+Mlas=
|
||||
github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ=
|
||||
github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
|
||||
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
|
||||
@ -321,8 +310,6 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
|
||||
github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
|
||||
github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
|
||||
@ -441,8 +428,6 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
|
||||
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/sagikazarmark/crypt v0.1.0/go.mod h1:B/mN0msZuINBtQ1zZLEQcegFJJf9vnYIR88KRMEuODE=
|
||||
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
|
||||
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
github.com/shirou/gopsutil v3.21.10+incompatible h1:AL2kpVykjkqeN+MFe1WcwSBVUjGjvdU8/ubvCuXAjrU=
|
||||
github.com/shirou/gopsutil v3.21.10+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
|
||||
@ -481,8 +466,6 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
|
||||
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
|
||||
github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg=
|
||||
github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
|
||||
github.com/tevid/gohamcrest v1.1.1 h1:ou+xSqlIw1xfGTg1uq1nif/htZ2S3EzRqLm2BP+tYU0=
|
||||
github.com/tevid/gohamcrest v1.1.1/go.mod h1:3UvtWlqm8j5JbwYZh80D/PVBt0mJ1eJiYgZMibh0H/k=
|
||||
github.com/tidwall/gjson v1.11.0 h1:C16pk7tQNiH6VlCrtIXL1w8GaOsi1X3W8KDkE1BuYd4=
|
||||
@ -618,7 +601,6 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL
|
||||
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
@ -626,7 +608,6 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/
|
||||
golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
@ -640,10 +621,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
|
||||
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
|
||||
golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210917221730-978cfadd31cf h1:R150MpwJIv1MpS0N/pc+NhTM8ajzvlmxlY5OYsrevXQ=
|
||||
golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM=
|
||||
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@ -852,7 +831,6 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
|
||||
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
||||
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
|
||||
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
@ -958,6 +936,8 @@ gorm.io/driver/mysql v1.2.0 h1:l8+9VwjjyzEkw0PNPBOr2JHhLOGVk7XEnl5hk42bcvs=
|
||||
gorm.io/driver/mysql v1.2.0/go.mod h1:4RQmTg4okPghdt+kbe6e1bTXIQp7Ny1NnBn/3Z6ghjk=
|
||||
gorm.io/gorm v1.22.3 h1:/JS6z+GStEQvJNW3t1FTwJwG/gZ+A7crFdRqtvG5ehA=
|
||||
gorm.io/gorm v1.22.3/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0=
|
||||
gorm.io/gorm v1.22.4 h1:8aPcyEJhY0MAt8aY6Dc524Pn+pO29K+ydu+e/cXSpQM=
|
||||
gorm.io/gorm v1.22.4/go.mod h1:1aeVC+pe9ZmvKZban/gW4QPra7PRoTEssyc922qCAkk=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
|
||||
@ -10,8 +10,11 @@ package wrapper
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"go.uber.org/zap/zapcore"
|
||||
@ -47,15 +50,36 @@ func NewGormV2(loggerLevel zapcore.Level, consoleOutput bool, encoder zapcore.En
|
||||
}, nil
|
||||
}
|
||||
|
||||
// NewGormLoggerWithInstance 获取gorm日志实现
|
||||
//
|
||||
// Author : go_developer@163.com<白茶清欢>
|
||||
//
|
||||
// Date : 3:36 PM 2021/12/24
|
||||
func NewGormLoggerWithInstance(ctx *gin.Context, dbClient *gorm.DB, instance *zap.Logger, node string, extraCtxFieldList []string) logger.Interface {
|
||||
return &Gorm{
|
||||
dbClient: dbClient,
|
||||
instance: instance,
|
||||
traceIDField: "",
|
||||
extraCtxFieldList: extraCtxFieldList,
|
||||
flag: "",
|
||||
node: node,
|
||||
ctx: ctx,
|
||||
}
|
||||
}
|
||||
|
||||
// Gorm v2 版本库日志实现
|
||||
//
|
||||
// Author : go_developer@163.com<白茶清欢>
|
||||
//
|
||||
// Date : 9:55 下午 2021/3/1
|
||||
type Gorm struct {
|
||||
instance *zap.Logger // 日志实例
|
||||
traceIDField string // 串联请求上下文的的ID
|
||||
flag string // 数据库标识
|
||||
dbClient *gorm.DB
|
||||
instance *zap.Logger // 日志实例
|
||||
traceIDField string // 串联请求上下文的的ID
|
||||
extraCtxFieldList []string // 从请求上线问提取的字段
|
||||
flag string // 数据库标识
|
||||
node string // 数据库节点 master / slave
|
||||
ctx *gin.Context // gin上下文
|
||||
}
|
||||
|
||||
// LogMode ...
|
||||
@ -73,12 +97,7 @@ func (g *Gorm) LogMode(level logger.LogLevel) logger.Interface {
|
||||
//
|
||||
// Date : 10:18 下午 2021/3/1
|
||||
func (g *Gorm) Info(ctx context.Context, s string, i ...interface{}) {
|
||||
g.instance.Info(
|
||||
"Info日志",
|
||||
zap.String(g.traceIDField, g.getTraceID(ctx)),
|
||||
zap.String("db_flag", g.flag),
|
||||
zap.String("message", fmt.Sprintf(s, i...)),
|
||||
)
|
||||
g.write(nil, "info")
|
||||
}
|
||||
|
||||
// Warn ...
|
||||
@ -87,12 +106,7 @@ func (g *Gorm) Info(ctx context.Context, s string, i ...interface{}) {
|
||||
//
|
||||
// Date : 10:16 下午 2021/3/1
|
||||
func (g *Gorm) Warn(ctx context.Context, s string, i ...interface{}) {
|
||||
g.instance.Warn(
|
||||
"SQL执行产生Warning",
|
||||
zap.String(g.traceIDField, g.getTraceID(ctx)),
|
||||
zap.String("db_flag", g.flag),
|
||||
zap.String("message", fmt.Sprintf(s, i...)),
|
||||
)
|
||||
g.write(nil, "warn")
|
||||
}
|
||||
|
||||
// Error 日志
|
||||
@ -101,12 +115,7 @@ func (g *Gorm) Warn(ctx context.Context, s string, i ...interface{}) {
|
||||
//
|
||||
// Date : 10:18 下午 2021/3/1
|
||||
func (g *Gorm) Error(ctx context.Context, s string, i ...interface{}) {
|
||||
g.instance.Warn(
|
||||
"SQL执行产生Error",
|
||||
zap.String(g.traceIDField, g.getTraceID(ctx)),
|
||||
zap.String("db_flag", g.flag),
|
||||
zap.String("message", fmt.Sprintf(s, i...)),
|
||||
)
|
||||
g.write(nil, "error")
|
||||
}
|
||||
|
||||
// Trace Trace 记录
|
||||
@ -123,8 +132,7 @@ func (g *Gorm) Trace(ctx context.Context, begin time.Time, fc func() (string, in
|
||||
sql, affectRows = fc()
|
||||
}
|
||||
|
||||
g.instance.Info(
|
||||
"SQL执行记录",
|
||||
dataList := []zap.Field{
|
||||
zap.String(g.traceIDField, g.getTraceID(ctx)),
|
||||
zap.String("db_flag", g.flag),
|
||||
zap.Int64("begin_time", start),
|
||||
@ -133,7 +141,45 @@ func (g *Gorm) Trace(ctx context.Context, begin time.Time, fc func() (string, in
|
||||
zap.String("sql", sql),
|
||||
zap.Int64("affect_rows", affectRows),
|
||||
zap.Error(err),
|
||||
)
|
||||
}
|
||||
g.write(dataList, "info")
|
||||
|
||||
}
|
||||
|
||||
// write ...
|
||||
//
|
||||
// Author : go_developer@163.com<白茶清欢>
|
||||
//
|
||||
// Date : 4:11 PM 2021/12/24
|
||||
func (g *Gorm) write(dataList []zap.Field, level string) {
|
||||
if nil == g.instance {
|
||||
// 未设置日志实例
|
||||
return
|
||||
}
|
||||
if nil == dataList {
|
||||
dataList = make([]zap.Field, 0)
|
||||
}
|
||||
if nil != g.ctx {
|
||||
for _, extraField := range g.extraCtxFieldList {
|
||||
dataList = append(dataList, zap.Any(extraField, g.ctx.Value(extraField)))
|
||||
}
|
||||
}
|
||||
|
||||
// 补齐 flag、node
|
||||
sql := g.dbClient.Dialector.Explain(g.dbClient.Statement.SQL.String(), g.dbClient.Statement.Vars...)
|
||||
affectRows := g.dbClient.RowsAffected
|
||||
dataList = append(dataList, zap.String("db_node", g.node), zap.String("db_flag", g.flag), zap.String("execute_sql", sql), zap.Int64("affext_rows", affectRows))
|
||||
message := "SQL执行记录"
|
||||
switch strings.ToLower(level) {
|
||||
case "info":
|
||||
g.instance.Info(message, dataList...)
|
||||
case "warn":
|
||||
g.instance.Warn(message, dataList...)
|
||||
case "error":
|
||||
g.instance.Error(message, dataList...)
|
||||
default:
|
||||
g.instance.Info(message, dataList...)
|
||||
}
|
||||
}
|
||||
|
||||
// getTraceID 获取traceID
|
||||
@ -145,7 +191,7 @@ func (g *Gorm) getTraceID(ctx context.Context) string {
|
||||
return fmt.Sprintf("%v", ctx.Value(g.traceIDField))
|
||||
}
|
||||
|
||||
// GetGormSQL 获取tracefn
|
||||
// GetGormSQL 获取trace fn
|
||||
//
|
||||
// Author : go_developer@163.com<白茶清欢>
|
||||
//
|
||||
|
||||
@ -10,14 +10,107 @@ package mysql
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"git.zhangdeman.cn/zhangdeman/gopkg/logger"
|
||||
|
||||
"git.zhangdeman.cn/zhangdeman/gopkg/logger/wrapper"
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"git.zhangdeman.cn/zhangdeman/gopkg/logger"
|
||||
gormLogger "gorm.io/gorm/logger"
|
||||
|
||||
"gorm.io/driver/mysql"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// NewDBClient ...
|
||||
//
|
||||
// Author : go_developer@163.com<白茶清欢>
|
||||
//
|
||||
// Date : 3:09 PM 2021/12/24
|
||||
func NewDBClient(masterConf *DBConfig, slaveConf *DBConfig, logConf *LogConfig, loggerInstance *zap.Logger, extraRequestFieldList []string) (*DBClient, error) {
|
||||
client := &DBClient{
|
||||
extraFieldList: extraRequestFieldList,
|
||||
}
|
||||
var err error
|
||||
// 日志初始化失败
|
||||
if client.loggerInstance, err = getLogInstance(logConf, loggerInstance); nil != err {
|
||||
return nil, err
|
||||
}
|
||||
if client.master, err = GetDatabaseClient(masterConf, nil); nil != err {
|
||||
return nil, err
|
||||
}
|
||||
if client.slave, err = GetDatabaseClient(slaveConf, nil); nil != err {
|
||||
return nil, err
|
||||
}
|
||||
return client, nil
|
||||
}
|
||||
|
||||
// DBClient 包装日志实例
|
||||
//
|
||||
// Author : go_developer@163.com<白茶清欢>
|
||||
//
|
||||
// Date : 3:09 PM 2021/12/24
|
||||
type DBClient struct {
|
||||
loggerInstance *zap.Logger
|
||||
master *gorm.DB
|
||||
slave *gorm.DB
|
||||
extraFieldList []string
|
||||
}
|
||||
|
||||
// GetMaster 获取主库连接
|
||||
//
|
||||
// Author : go_developer@163.com<白茶清欢>
|
||||
//
|
||||
// Date : 3:28 PM 2021/12/24
|
||||
func (dc *DBClient) GetMaster(ctx *gin.Context) *gorm.DB {
|
||||
session := dc.master.Session(&gorm.Session{})
|
||||
session.Logger = dc.getLogger(ctx, session, "slave")
|
||||
return session
|
||||
}
|
||||
|
||||
// GetSlave 获取从库链接
|
||||
//
|
||||
// Author : go_developer@163.com<白茶清欢>
|
||||
//
|
||||
// Date : 3:29 PM 2021/12/24
|
||||
func (dc *DBClient) GetSlave(ctx *gin.Context) *gorm.DB {
|
||||
session := dc.slave.Session(&gorm.Session{})
|
||||
session.Logger = dc.getLogger(ctx, session, "slave")
|
||||
return session
|
||||
}
|
||||
|
||||
// getLogger 获取日志实例
|
||||
//
|
||||
// Author : go_developer@163.com<白茶清欢>
|
||||
//
|
||||
// Date : 3:45 PM 2021/12/24
|
||||
func (dc *DBClient) getLogger(ctx *gin.Context, dbClient *gorm.DB, node string) gormLogger.Interface {
|
||||
return wrapper.NewGormLoggerWithInstance(ctx, dbClient, dc.loggerInstance, node, dc.extraFieldList)
|
||||
}
|
||||
|
||||
// getLogInstance 获取日志实例
|
||||
//
|
||||
// Author : go_developer@163.com<白茶清欢>
|
||||
//
|
||||
// Date : 3:20 PM 2021/12/24
|
||||
func getLogInstance(logConf *LogConfig, loggerInstance *zap.Logger) (*zap.Logger, error) {
|
||||
if nil != loggerInstance {
|
||||
return loggerInstance, nil
|
||||
}
|
||||
logConfList := []logger.SetLoggerOptionFunc{logger.WithEncoder(logConf.Encoder), logger.WithCallerSkip(logConf.Skip), logger.WithCaller()}
|
||||
if logConf.ConsoleOutput {
|
||||
logConfList = append(logConfList, logger.WithConsoleOutput())
|
||||
}
|
||||
|
||||
var (
|
||||
err error
|
||||
)
|
||||
if loggerInstance, err = logger.NewLogger(logConf.Level, logConf.SplitConfig, logConfList...); nil != err {
|
||||
return nil, err
|
||||
}
|
||||
return loggerInstance, nil
|
||||
}
|
||||
|
||||
// GetDatabaseClient 获取日志实例
|
||||
//
|
||||
// Author : go_developer@163.com<白茶清欢>
|
||||
@ -25,35 +118,22 @@ import (
|
||||
// Date : 10:49 下午 2021/3/1
|
||||
func GetDatabaseClient(conf *DBConfig, logConf *LogConfig) (*gorm.DB, error) {
|
||||
var (
|
||||
instance *gorm.DB
|
||||
err error
|
||||
instance *gorm.DB
|
||||
err error
|
||||
loggerInstance *zap.Logger
|
||||
)
|
||||
|
||||
if instance, err = gorm.Open(mysql.Open(buildConnectionDSN(conf)), &gorm.Config{}); nil != err {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(logConf.TraceFieldName) == 0 {
|
||||
logConf.TraceFieldName = defaultTraceFieldName
|
||||
if nil != logConf {
|
||||
if loggerInstance, err = getLogInstance(logConf, nil); nil != err {
|
||||
return nil, err
|
||||
}
|
||||
instance.Logger = wrapper.NewGormLoggerWithInstance(nil, instance, loggerInstance, "", nil)
|
||||
}
|
||||
|
||||
splitConfigFuncList := []logger.SetRotateLogConfigFunc{
|
||||
logger.WithTimeIntervalType(logConf.SplitConfig.TimeIntervalType),
|
||||
logger.WithDivisionChar(logConf.SplitConfig.DivisionChar),
|
||||
logger.WithMaxAge(logConf.SplitConfig.MaxAge),
|
||||
}
|
||||
|
||||
splitConfig, _ := logger.NewRotateLogConfig(logConf.SplitConfig.LogPath, logConf.SplitConfig.LogFileName, splitConfigFuncList...)
|
||||
|
||||
if instance.Logger, err = wrapper.NewGormV2(
|
||||
logConf.Level,
|
||||
logConf.ConsoleOutput,
|
||||
logConf.Encoder,
|
||||
splitConfig,
|
||||
logConf.TraceFieldName,
|
||||
logConf.Skip); nil != err {
|
||||
return nil, CreateDBLogError(err)
|
||||
}
|
||||
return instance, nil
|
||||
}
|
||||
|
||||
|
||||
@ -42,8 +42,3 @@ type LogConfig struct {
|
||||
TraceFieldName string
|
||||
Skip int
|
||||
}
|
||||
|
||||
const (
|
||||
// defaultTraceFieldName 默认trace_id字段
|
||||
defaultTraceFieldName = "trace_id"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user