From e79ac95858e24f2d1dfa3cb29bb6d0347b0abb49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E8=8C=B6=E6=B8=85=E6=AC=A2?= Date: Fri, 1 Sep 2023 23:17:04 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8D=87=E7=BA=A7=E6=8F=90=E5=8F=96=E6=8C=87?= =?UTF-8?q?=E5=AE=9A=E6=B7=B1=E5=BA=A6=E7=9A=84=E6=95=B0=E6=8D=AE=E6=9C=80?= =?UTF-8?q?=E7=BB=88=E5=AD=97=E6=AE=B5=E4=BD=9C=E4=B8=BAlist?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 5 +-- go.sum | 13 +----- tool/gabs.go | 104 ++++++++++++++++++++++++++++++++++++---------- tool/gabs_test.go | 46 +++++++++++++++++++- 4 files changed, 132 insertions(+), 36 deletions(-) diff --git a/go.mod b/go.mod index 54b62f4..4deb18f 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,9 @@ go 1.20 require ( git.zhangdeman.cn/zhangdeman/easylock v0.0.0-20230731062340-983985c12eda + git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20230811032817-e6ad534a9a10 git.zhangdeman.cn/zhangdeman/util v0.0.0-20230811070456-d6a489d5860b git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20230811071513-cfc46e8d82e1 - github.com/Jeffail/gabs v1.4.0 github.com/pkg/errors v0.9.1 github.com/smartystreets/goconvey v1.8.1 github.com/tidwall/gjson v1.16.0 @@ -17,10 +17,8 @@ require ( require ( git.zhangdeman.cn/zhangdeman/consts v0.0.0-20230811030300-6f850372c88c // indirect git.zhangdeman.cn/zhangdeman/easymap v0.0.0-20230307094841-e437ba87af10 // indirect - git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20230811032817-e6ad534a9a10 // indirect github.com/BurntSushi/toml v1.3.2 // indirect github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 // indirect - github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1 // indirect github.com/go-ini/ini v1.67.0 // indirect github.com/gopherjs/gopherjs v1.17.2 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect @@ -28,6 +26,7 @@ require ( github.com/mozillazg/go-pinyin v0.20.0 // indirect github.com/smarty/assertions v1.15.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/stretchr/testify v1.8.4 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect ) diff --git a/go.sum b/go.sum index 7aed6b6..f4169f1 100644 --- a/go.sum +++ b/go.sum @@ -6,23 +6,15 @@ git.zhangdeman.cn/zhangdeman/easymap v0.0.0-20230307094841-e437ba87af10 h1:+Lg4v git.zhangdeman.cn/zhangdeman/easymap v0.0.0-20230307094841-e437ba87af10/go.mod h1:+Lc0zYF8sylRi75A7NGmObrLxugwAZa8WVpWh2eh5X0= git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20230811032817-e6ad534a9a10 h1:orhcMAKrcOajsBJCgssnb9O8YcLsPJvWuXF511gs5dc= git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20230811032817-e6ad534a9a10/go.mod h1:CzX5/WwGDTnKmewarnjkK5XcSRbgszTQTdTL3OUc/s4= -git.zhangdeman.cn/zhangdeman/util v0.0.0-20230810085948-024a0e5e963a h1:PED+zfKJdSbewYQNpcjj3uXoGDSA5+8LBGScObfuHAA= -git.zhangdeman.cn/zhangdeman/util v0.0.0-20230810085948-024a0e5e963a/go.mod h1:trYFOShINaQBvinQrH4A0G2kfL22Y2lygEcAiGDt/sc= git.zhangdeman.cn/zhangdeman/util v0.0.0-20230811070456-d6a489d5860b h1:vnmxYrNdX6f5sEVjjkM1fIR+i32kHJ4g9DJqug9KKek= git.zhangdeman.cn/zhangdeman/util v0.0.0-20230811070456-d6a489d5860b/go.mod h1:Yum5+tgP+Wf1GWUAyQz1Qh8Ab9m5+90GYkYdzqVs0lA= -git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20230810085842-1e901000e998 h1:+qefkhBULVZbTGxJO4JqT1C1JQLaccJA6GLKP1qImqE= -git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20230810085842-1e901000e998/go.mod h1:WcNcIIxTSpvjpEhfE64Ktis3q+Fw+Onw7NC2/I1U2Qs= git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20230811071513-cfc46e8d82e1 h1:k2iu9KgRxeroytB+N+/XapAxt1di7o2pNTISjFlYDJ8= git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20230811071513-cfc46e8d82e1/go.mod h1:kvjAbtGTo14gKCS0X4rxnb2sPkskHOUy2NXcx34t6Mw= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/Jeffail/gabs v1.4.0 h1://5fYRRTq1edjfIrQGvdkcd22pkYUrHZ5YC/H2GJVAo= -github.com/Jeffail/gabs v1.4.0/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc= github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 h1:OYA+5W64v3OgClL+IrOD63t4i/RW7RqrAVl9LTZ9UqQ= github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394/go.mod h1:Q8n74mJTIgjX4RBBcHnJ05h//6/k6foqmgE45jTQtxg= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1 h1:CaO/zOnF8VvUfEbhRatPcwKVWamvbYd8tQGRWacE9kU= -github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1/go.mod h1:+hnT3ywWDTAFrW5aE+u2Sa/wT555ZqwoCS+pk3p6ry4= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= @@ -42,7 +34,8 @@ github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sS github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.16.0 h1:SyXa+dsSPpUlcwEDuKuEBJEz5vzTvOea+9rjyYodQFg= github.com/tidwall/gjson v1.16.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -53,8 +46,6 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/tool/gabs.go b/tool/gabs.go index 28557ce..3d06ba1 100644 --- a/tool/gabs.go +++ b/tool/gabs.go @@ -8,12 +8,15 @@ package json_tool import ( + "fmt" + "io" + "log" + "os" + "strings" + "github.com/pkg/errors" "github.com/tidwall/gjson" "github.com/tidwall/sjson" - "log/slog" - "os" - "strings" ) const ( @@ -26,8 +29,8 @@ const ( // // Date : 17:59 2023/9/1 type FilterOption struct { - DebugModel bool // 调试模式 - LogInstance *slog.Logger // 日志实例 + DebugModel bool // 调试模式 + LogInstance io.Writer // 日志实例 } // FilterDataRule 参数过滤规则 @@ -52,10 +55,8 @@ func NewDataFilter(source string, filterRule []*FilterDataRule, filterOption *Fi filterOption = &FilterOption{} } if filterOption.DebugModel && nil == filterOption.LogInstance { - filterOption.LogInstance = slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{ - Level: slog.LevelDebug, - })) - slog.SetDefault(filterOption.LogInstance) + filterOption.LogInstance = os.Stdout + log.SetOutput(filterOption.LogInstance) } return &DataFilter{ source: source, @@ -83,18 +84,29 @@ type DataFilter struct { // // Date : 2022/1/22 9:36 PM func (df *DataFilter) Filter() (string, error) { - df.logPrint(logLevelInfo, "输入的原始数据", slog.String("source_data", df.source)) + df.logPrint(logLevelDebug, "source_data => ", df.source) var ( err error ) for _, itemRule := range df.filterRule { - if !df.isArrPath(itemRule.SourceKey) && !df.isArrPath(itemRule.MapKey) { + sourceIsArr := df.isArrPath(itemRule.SourceKey) + mapIsArr := df.isArrPath(itemRule.MapKey) + if !sourceIsArr && !mapIsArr { // 输入输出均不是数组, 最简单的场景 if err = df.setKV(itemRule); nil != err { return "", err } } + sourcePathArr := df.getArrPathList(itemRule.SourceKey) + mapPathArr := df.getArrPathList(itemRule.MapKey) + if len(mapPathArr) > len(sourcePathArr) { + df.logPrint(logLevelFatal, "映射的层级深度大于数据源深度", "source_path => "+itemRule.SourceKey, "map_path => "+itemRule.MapKey) + return "", fmt.Errorf("映射的层级深度大于数据源深度, source_path => %v map_path => %v", itemRule.SourceKey, itemRule.MapKey) + } + if !mapIsArr { + } + } return df.rewriteResult, nil } @@ -108,6 +120,49 @@ func (df *DataFilter) isArrPath(path string) bool { return strings.Contains(path, "[]") } +// a.[].b.[].c.[].d +// e.[].f.[].g +func (df *DataFilter) getDataAsSlice(sourceData string, pathList []string) []interface{} { + //fmt.Println(sourceData, pathList) + result := make([]interface{}, 0) + if len(pathList) == 0 { + return result + } + if len(pathList) == 1 { + return []interface{}{gjson.Get(sourceData, pathList[0]).Value()} + } + for idx, itemPath := range pathList { + if len(pathList)-1 == idx { + val := gjson.Get(sourceData, itemPath).Value() + if nil == val { + return result + } + result = append(result, val) + return result + } + + currentPathVal := gjson.Get(sourceData, itemPath).Array() + for _, sonItem := range currentPathVal { + result = append(result, df.getDataAsSlice(sonItem.String(), pathList[idx:])...) + } + } + return result +} + +// setValue 设置值 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 22:04 2023/9/1 +func (df *DataFilter) setValue(path string, value interface{}) error { + var ( + err error + ) + + df.rewriteResult, err = sjson.Set(df.rewriteResult, path, value) + return err +} + // setKV 设置相关值 // // Author : go_developer@163.com<白茶清欢> @@ -132,6 +187,20 @@ func (df *DataFilter) setKV(rule *FilterDataRule) error { return err } +// getArrPathList 获取路径列表 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 21:32 2023/9/1 +func (df *DataFilter) getArrPathList(inputPath string) []string { + pathArr := strings.Split(inputPath, "[]") + arr := make([]string, 0) + for _, item := range pathArr { + arr = append(arr, strings.Trim(item, ".")) + } + return arr +} + // logPrint 打印日志 // // Author : go_developer@163.com<白茶清欢> @@ -142,20 +211,13 @@ func (df *DataFilter) logPrint(level string, msg string, logAttr ...interface{}) // 未开启调试模式 return } - switch level { - case logLevelFatal: - slog.Error(msg, logAttr...) - case logLevelWarn: - slog.Warn(msg, logAttr...) - case logLevelInfo: - slog.Info(msg, logAttr...) - case logLevelDebug: - slog.Debug(msg, logAttr...) - } + logData := append([]interface{}{level, msg}, logAttr...) + log.Println(logData...) } // 日志等级定义 const ( + logLevelPanic = "PANIC" logLevelFatal = "FATAL" logLevelWarn = "WARN" logLevelInfo = "INFO" diff --git a/tool/gabs_test.go b/tool/gabs_test.go index 0a78692..494c169 100644 --- a/tool/gabs_test.go +++ b/tool/gabs_test.go @@ -8,9 +8,11 @@ package json_tool import ( + "encoding/json" "fmt" - "git.zhangdeman.cn/zhangdeman/serialize" "testing" + + "git.zhangdeman.cn/zhangdeman/serialize" ) // TestDataFilter_FilterNormalData 最基础对象 @@ -41,3 +43,45 @@ func TestDataFilter_FilterNormalData(t *testing.T) { } fmt.Println(df.Filter()) } + +// TestDataFilter_getDataAsSlice 测试循环读取数据 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 22:21 2023/9/1 +func TestDataFilter_getDataAsSlice(t *testing.T) { + data := map[string]interface{}{ + "list": []interface{}{ + map[string]interface{}{ + "name": "1", + "age": "2", + "list_test": []interface{}{ + map[string]interface{}{ + "a": "a", + "b": "b", + }, + }, + }, + map[string]interface{}{ + "name": "3", + "age": "4", + "list_test": []interface{}{ + map[string]interface{}{ + "a": "a1", + "b": "b1", + }, + }, + }, + map[string]interface{}{ + "name": "5", + "age": "6", + }, + }, + } + byteData, _ := json.Marshal(data) + df := &DataFilter{} + fmt.Println(df.getDataAsSlice(string(byteData), []string{"list", "name"})) + fmt.Println(df.getDataAsSlice(string(byteData), []string{"list", "age"})) + fmt.Println(df.getDataAsSlice(string(byteData), []string{"list", "age", "list_test", "a"})) + fmt.Println(df.getDataAsSlice(string(byteData), []string{"list", "age", "list_test", "b"})) +}