完成数据过主干逻辑,细节健壮性有待继续完善 #4
5
go.mod
5
go.mod
@ -4,9 +4,9 @@ go 1.20
|
||||
|
||||
require (
|
||||
git.zhangdeman.cn/zhangdeman/easylock v0.0.0-20230731062340-983985c12eda
|
||||
git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20230811032817-e6ad534a9a10
|
||||
git.zhangdeman.cn/zhangdeman/util v0.0.0-20230811070456-d6a489d5860b
|
||||
git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20230811071513-cfc46e8d82e1
|
||||
github.com/Jeffail/gabs v1.4.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/smartystreets/goconvey v1.8.1
|
||||
github.com/tidwall/gjson v1.16.0
|
||||
@ -17,10 +17,8 @@ require (
|
||||
require (
|
||||
git.zhangdeman.cn/zhangdeman/consts v0.0.0-20230811030300-6f850372c88c // indirect
|
||||
git.zhangdeman.cn/zhangdeman/easymap v0.0.0-20230307094841-e437ba87af10 // indirect
|
||||
git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20230811032817-e6ad534a9a10 // indirect
|
||||
github.com/BurntSushi/toml v1.3.2 // indirect
|
||||
github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 // indirect
|
||||
github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
github.com/gopherjs/gopherjs v1.17.2 // indirect
|
||||
github.com/jtolds/gls v4.20.0+incompatible // indirect
|
||||
@ -28,6 +26,7 @@ require (
|
||||
github.com/mozillazg/go-pinyin v0.20.0 // indirect
|
||||
github.com/smarty/assertions v1.15.0 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
github.com/stretchr/testify v1.8.4 // indirect
|
||||
github.com/tidwall/match v1.1.1 // indirect
|
||||
github.com/tidwall/pretty v1.2.1 // indirect
|
||||
)
|
||||
|
||||
13
go.sum
13
go.sum
@ -6,23 +6,15 @@ git.zhangdeman.cn/zhangdeman/easymap v0.0.0-20230307094841-e437ba87af10 h1:+Lg4v
|
||||
git.zhangdeman.cn/zhangdeman/easymap v0.0.0-20230307094841-e437ba87af10/go.mod h1:+Lc0zYF8sylRi75A7NGmObrLxugwAZa8WVpWh2eh5X0=
|
||||
git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20230811032817-e6ad534a9a10 h1:orhcMAKrcOajsBJCgssnb9O8YcLsPJvWuXF511gs5dc=
|
||||
git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20230811032817-e6ad534a9a10/go.mod h1:CzX5/WwGDTnKmewarnjkK5XcSRbgszTQTdTL3OUc/s4=
|
||||
git.zhangdeman.cn/zhangdeman/util v0.0.0-20230810085948-024a0e5e963a h1:PED+zfKJdSbewYQNpcjj3uXoGDSA5+8LBGScObfuHAA=
|
||||
git.zhangdeman.cn/zhangdeman/util v0.0.0-20230810085948-024a0e5e963a/go.mod h1:trYFOShINaQBvinQrH4A0G2kfL22Y2lygEcAiGDt/sc=
|
||||
git.zhangdeman.cn/zhangdeman/util v0.0.0-20230811070456-d6a489d5860b h1:vnmxYrNdX6f5sEVjjkM1fIR+i32kHJ4g9DJqug9KKek=
|
||||
git.zhangdeman.cn/zhangdeman/util v0.0.0-20230811070456-d6a489d5860b/go.mod h1:Yum5+tgP+Wf1GWUAyQz1Qh8Ab9m5+90GYkYdzqVs0lA=
|
||||
git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20230810085842-1e901000e998 h1:+qefkhBULVZbTGxJO4JqT1C1JQLaccJA6GLKP1qImqE=
|
||||
git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20230810085842-1e901000e998/go.mod h1:WcNcIIxTSpvjpEhfE64Ktis3q+Fw+Onw7NC2/I1U2Qs=
|
||||
git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20230811071513-cfc46e8d82e1 h1:k2iu9KgRxeroytB+N+/XapAxt1di7o2pNTISjFlYDJ8=
|
||||
git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20230811071513-cfc46e8d82e1/go.mod h1:kvjAbtGTo14gKCS0X4rxnb2sPkskHOUy2NXcx34t6Mw=
|
||||
github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
|
||||
github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
|
||||
github.com/Jeffail/gabs v1.4.0 h1://5fYRRTq1edjfIrQGvdkcd22pkYUrHZ5YC/H2GJVAo=
|
||||
github.com/Jeffail/gabs v1.4.0/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc=
|
||||
github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 h1:OYA+5W64v3OgClL+IrOD63t4i/RW7RqrAVl9LTZ9UqQ=
|
||||
github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394/go.mod h1:Q8n74mJTIgjX4RBBcHnJ05h//6/k6foqmgE45jTQtxg=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1 h1:CaO/zOnF8VvUfEbhRatPcwKVWamvbYd8tQGRWacE9kU=
|
||||
github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1/go.mod h1:+hnT3ywWDTAFrW5aE+u2Sa/wT555ZqwoCS+pk3p6ry4=
|
||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g=
|
||||
@ -42,7 +34,8 @@ github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sS
|
||||
github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60=
|
||||
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
|
||||
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
|
||||
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
github.com/tidwall/gjson v1.16.0 h1:SyXa+dsSPpUlcwEDuKuEBJEz5vzTvOea+9rjyYodQFg=
|
||||
github.com/tidwall/gjson v1.16.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
@ -53,8 +46,6 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
|
||||
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
|
||||
github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
|
||||
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
|
||||
104
tool/gabs.go
104
tool/gabs.go
@ -8,12 +8,15 @@
|
||||
package json_tool
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/tidwall/gjson"
|
||||
"github.com/tidwall/sjson"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -26,8 +29,8 @@ const (
|
||||
//
|
||||
// Date : 17:59 2023/9/1
|
||||
type FilterOption struct {
|
||||
DebugModel bool // 调试模式
|
||||
LogInstance *slog.Logger // 日志实例
|
||||
DebugModel bool // 调试模式
|
||||
LogInstance io.Writer // 日志实例
|
||||
}
|
||||
|
||||
// FilterDataRule 参数过滤规则
|
||||
@ -52,10 +55,8 @@ func NewDataFilter(source string, filterRule []*FilterDataRule, filterOption *Fi
|
||||
filterOption = &FilterOption{}
|
||||
}
|
||||
if filterOption.DebugModel && nil == filterOption.LogInstance {
|
||||
filterOption.LogInstance = slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{
|
||||
Level: slog.LevelDebug,
|
||||
}))
|
||||
slog.SetDefault(filterOption.LogInstance)
|
||||
filterOption.LogInstance = os.Stdout
|
||||
log.SetOutput(filterOption.LogInstance)
|
||||
}
|
||||
return &DataFilter{
|
||||
source: source,
|
||||
@ -83,18 +84,29 @@ type DataFilter struct {
|
||||
//
|
||||
// Date : 2022/1/22 9:36 PM
|
||||
func (df *DataFilter) Filter() (string, error) {
|
||||
df.logPrint(logLevelInfo, "输入的原始数据", slog.String("source_data", df.source))
|
||||
df.logPrint(logLevelDebug, "source_data => ", df.source)
|
||||
var (
|
||||
err error
|
||||
)
|
||||
|
||||
for _, itemRule := range df.filterRule {
|
||||
if !df.isArrPath(itemRule.SourceKey) && !df.isArrPath(itemRule.MapKey) {
|
||||
sourceIsArr := df.isArrPath(itemRule.SourceKey)
|
||||
mapIsArr := df.isArrPath(itemRule.MapKey)
|
||||
if !sourceIsArr && !mapIsArr {
|
||||
// 输入输出均不是数组, 最简单的场景
|
||||
if err = df.setKV(itemRule); nil != err {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
sourcePathArr := df.getArrPathList(itemRule.SourceKey)
|
||||
mapPathArr := df.getArrPathList(itemRule.MapKey)
|
||||
if len(mapPathArr) > len(sourcePathArr) {
|
||||
df.logPrint(logLevelFatal, "映射的层级深度大于数据源深度", "source_path => "+itemRule.SourceKey, "map_path => "+itemRule.MapKey)
|
||||
return "", fmt.Errorf("映射的层级深度大于数据源深度, source_path => %v map_path => %v", itemRule.SourceKey, itemRule.MapKey)
|
||||
}
|
||||
if !mapIsArr {
|
||||
}
|
||||
|
||||
}
|
||||
return df.rewriteResult, nil
|
||||
}
|
||||
@ -108,6 +120,49 @@ func (df *DataFilter) isArrPath(path string) bool {
|
||||
return strings.Contains(path, "[]")
|
||||
}
|
||||
|
||||
// a.[].b.[].c.[].d
|
||||
// e.[].f.[].g
|
||||
func (df *DataFilter) getDataAsSlice(sourceData string, pathList []string) []interface{} {
|
||||
//fmt.Println(sourceData, pathList)
|
||||
result := make([]interface{}, 0)
|
||||
if len(pathList) == 0 {
|
||||
return result
|
||||
}
|
||||
if len(pathList) == 1 {
|
||||
return []interface{}{gjson.Get(sourceData, pathList[0]).Value()}
|
||||
}
|
||||
for idx, itemPath := range pathList {
|
||||
if len(pathList)-1 == idx {
|
||||
val := gjson.Get(sourceData, itemPath).Value()
|
||||
if nil == val {
|
||||
return result
|
||||
}
|
||||
result = append(result, val)
|
||||
return result
|
||||
}
|
||||
|
||||
currentPathVal := gjson.Get(sourceData, itemPath).Array()
|
||||
for _, sonItem := range currentPathVal {
|
||||
result = append(result, df.getDataAsSlice(sonItem.String(), pathList[idx:])...)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// setValue 设置值
|
||||
//
|
||||
// Author : go_developer@163.com<白茶清欢>
|
||||
//
|
||||
// Date : 22:04 2023/9/1
|
||||
func (df *DataFilter) setValue(path string, value interface{}) error {
|
||||
var (
|
||||
err error
|
||||
)
|
||||
|
||||
df.rewriteResult, err = sjson.Set(df.rewriteResult, path, value)
|
||||
return err
|
||||
}
|
||||
|
||||
// setKV 设置相关值
|
||||
//
|
||||
// Author : go_developer@163.com<白茶清欢>
|
||||
@ -132,6 +187,20 @@ func (df *DataFilter) setKV(rule *FilterDataRule) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// getArrPathList 获取路径列表
|
||||
//
|
||||
// Author : go_developer@163.com<白茶清欢>
|
||||
//
|
||||
// Date : 21:32 2023/9/1
|
||||
func (df *DataFilter) getArrPathList(inputPath string) []string {
|
||||
pathArr := strings.Split(inputPath, "[]")
|
||||
arr := make([]string, 0)
|
||||
for _, item := range pathArr {
|
||||
arr = append(arr, strings.Trim(item, "."))
|
||||
}
|
||||
return arr
|
||||
}
|
||||
|
||||
// logPrint 打印日志
|
||||
//
|
||||
// Author : go_developer@163.com<白茶清欢>
|
||||
@ -142,20 +211,13 @@ func (df *DataFilter) logPrint(level string, msg string, logAttr ...interface{})
|
||||
// 未开启调试模式
|
||||
return
|
||||
}
|
||||
switch level {
|
||||
case logLevelFatal:
|
||||
slog.Error(msg, logAttr...)
|
||||
case logLevelWarn:
|
||||
slog.Warn(msg, logAttr...)
|
||||
case logLevelInfo:
|
||||
slog.Info(msg, logAttr...)
|
||||
case logLevelDebug:
|
||||
slog.Debug(msg, logAttr...)
|
||||
}
|
||||
logData := append([]interface{}{level, msg}, logAttr...)
|
||||
log.Println(logData...)
|
||||
}
|
||||
|
||||
// 日志等级定义
|
||||
const (
|
||||
logLevelPanic = "PANIC"
|
||||
logLevelFatal = "FATAL"
|
||||
logLevelWarn = "WARN"
|
||||
logLevelInfo = "INFO"
|
||||
|
||||
@ -8,9 +8,11 @@
|
||||
package json_tool
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"git.zhangdeman.cn/zhangdeman/serialize"
|
||||
"testing"
|
||||
|
||||
"git.zhangdeman.cn/zhangdeman/serialize"
|
||||
)
|
||||
|
||||
// TestDataFilter_FilterNormalData 最基础对象
|
||||
@ -41,3 +43,45 @@ func TestDataFilter_FilterNormalData(t *testing.T) {
|
||||
}
|
||||
fmt.Println(df.Filter())
|
||||
}
|
||||
|
||||
// TestDataFilter_getDataAsSlice 测试循环读取数据
|
||||
//
|
||||
// Author : go_developer@163.com<白茶清欢>
|
||||
//
|
||||
// Date : 22:21 2023/9/1
|
||||
func TestDataFilter_getDataAsSlice(t *testing.T) {
|
||||
data := map[string]interface{}{
|
||||
"list": []interface{}{
|
||||
map[string]interface{}{
|
||||
"name": "1",
|
||||
"age": "2",
|
||||
"list_test": []interface{}{
|
||||
map[string]interface{}{
|
||||
"a": "a",
|
||||
"b": "b",
|
||||
},
|
||||
},
|
||||
},
|
||||
map[string]interface{}{
|
||||
"name": "3",
|
||||
"age": "4",
|
||||
"list_test": []interface{}{
|
||||
map[string]interface{}{
|
||||
"a": "a1",
|
||||
"b": "b1",
|
||||
},
|
||||
},
|
||||
},
|
||||
map[string]interface{}{
|
||||
"name": "5",
|
||||
"age": "6",
|
||||
},
|
||||
},
|
||||
}
|
||||
byteData, _ := json.Marshal(data)
|
||||
df := &DataFilter{}
|
||||
fmt.Println(df.getDataAsSlice(string(byteData), []string{"list", "name"}))
|
||||
fmt.Println(df.getDataAsSlice(string(byteData), []string{"list", "age"}))
|
||||
fmt.Println(df.getDataAsSlice(string(byteData), []string{"list", "age", "list_test", "a"}))
|
||||
fmt.Println(df.getDataAsSlice(string(byteData), []string{"list", "age", "list_test", "b"}))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user