zhangdeman/json_filter
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request '完成数据过主干逻辑,细节健壮性有待继续完善' (#4) from feature/data_filter into master

Browse Source 
Reviewed-on: #4
This commit is contained in:
白茶清欢
2023-09-02 22:17:13 +08:00
parent 78b3ae43e5 2230411f76
commit 024f35ac67
5 changed files with 539 additions and 382 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
go.mod
View File

@ -4,9 +4,9 @@ go 1.20
require (
git.zhangdeman.cn/zhangdeman/easylock v0.0.0-20230731062340-983985c12eda
git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20230811032817-e6ad534a9a10
git.zhangdeman.cn/zhangdeman/util v0.0.0-20230811070456-d6a489d5860b
git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20230811071513-cfc46e8d82e1
github.com/Jeffail/gabs v1.4.0
github.com/pkg/errors v0.9.1
github.com/smartystreets/goconvey v1.8.1
github.com/tidwall/gjson v1.16.0
@ -17,10 +17,8 @@ require (
require (
git.zhangdeman.cn/zhangdeman/consts v0.0.0-20230811030300-6f850372c88c // indirect
git.zhangdeman.cn/zhangdeman/easymap v0.0.0-20230307094841-e437ba87af10 // indirect
git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20230811032817-e6ad534a9a10 // indirect
github.com/BurntSushi/toml v1.3.2 // indirect
github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 // indirect
github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/gopherjs/gopherjs v1.17.2 // indirect
github.com/jtolds/gls v4.20.0+incompatible // indirect
@ -28,6 +26,7 @@ require (
github.com/mozillazg/go-pinyin v0.20.0 // indirect
github.com/smarty/assertions v1.15.0 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/stretchr/testify v1.8.4 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
)

13
go.sum
View File

@ -6,23 +6,15 @@ git.zhangdeman.cn/zhangdeman/easymap v0.0.0-20230307094841-e437ba87af10 h1:+Lg4v
git.zhangdeman.cn/zhangdeman/easymap v0.0.0-20230307094841-e437ba87af10/go.mod h1:+Lc0zYF8sylRi75A7NGmObrLxugwAZa8WVpWh2eh5X0=
git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20230811032817-e6ad534a9a10 h1:orhcMAKrcOajsBJCgssnb9O8YcLsPJvWuXF511gs5dc=
git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20230811032817-e6ad534a9a10/go.mod h1:CzX5/WwGDTnKmewarnjkK5XcSRbgszTQTdTL3OUc/s4=
git.zhangdeman.cn/zhangdeman/util v0.0.0-20230810085948-024a0e5e963a h1:PED+zfKJdSbewYQNpcjj3uXoGDSA5+8LBGScObfuHAA=
git.zhangdeman.cn/zhangdeman/util v0.0.0-20230810085948-024a0e5e963a/go.mod h1:trYFOShINaQBvinQrH4A0G2kfL22Y2lygEcAiGDt/sc=
git.zhangdeman.cn/zhangdeman/util v0.0.0-20230811070456-d6a489d5860b h1:vnmxYrNdX6f5sEVjjkM1fIR+i32kHJ4g9DJqug9KKek=
git.zhangdeman.cn/zhangdeman/util v0.0.0-20230811070456-d6a489d5860b/go.mod h1:Yum5+tgP+Wf1GWUAyQz1Qh8Ab9m5+90GYkYdzqVs0lA=
git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20230810085842-1e901000e998 h1:+qefkhBULVZbTGxJO4JqT1C1JQLaccJA6GLKP1qImqE=
git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20230810085842-1e901000e998/go.mod h1:WcNcIIxTSpvjpEhfE64Ktis3q+Fw+Onw7NC2/I1U2Qs=
git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20230811071513-cfc46e8d82e1 h1:k2iu9KgRxeroytB+N+/XapAxt1di7o2pNTISjFlYDJ8=
git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20230811071513-cfc46e8d82e1/go.mod h1:kvjAbtGTo14gKCS0X4rxnb2sPkskHOUy2NXcx34t6Mw=
github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/Jeffail/gabs v1.4.0 h1://5fYRRTq1edjfIrQGvdkcd22pkYUrHZ5YC/H2GJVAo=
github.com/Jeffail/gabs v1.4.0/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc=
github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 h1:OYA+5W64v3OgClL+IrOD63t4i/RW7RqrAVl9LTZ9UqQ=
github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394/go.mod h1:Q8n74mJTIgjX4RBBcHnJ05h//6/k6foqmgE45jTQtxg=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1 h1:CaO/zOnF8VvUfEbhRatPcwKVWamvbYd8tQGRWacE9kU=
github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1/go.mod h1:+hnT3ywWDTAFrW5aE+u2Sa/wT555ZqwoCS+pk3p6ry4=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g=
@ -42,7 +34,8 @@ github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sS
github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60=
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/gjson v1.16.0 h1:SyXa+dsSPpUlcwEDuKuEBJEz5vzTvOea+9rjyYodQFg=
github.com/tidwall/gjson v1.16.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
@ -53,8 +46,6 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

461
tool/gabs.go
View File

@ -8,19 +8,32 @@
package json_tool
import (
"encoding/json"
"fmt"
"git.zhangdeman.cn/zhangdeman/wrapper"
"reflect"
"io"
"log"
"os"
"strings"
"github.com/pkg/errors"
"github.com/tidwall/gjson"
"github.com/Jeffail/gabs"
"github.com/tidwall/sjson"
)
const (
// virtualRoot 虚拟根节点
virtualRoot = "__VIRTUAL_ROOT__"
)
// FilterOption 过滤选项
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 17:59 2023/9/1
type FilterOption struct {
DebugModel bool // 调试模式
LogInstance io.Writer // 日志实例
}
// FilterDataRule 参数过滤规则
//
// Author : go_developer@163.com<白茶清欢>
@ -38,12 +51,45 @@ type FilterDataRule struct {
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2022/1/22 9:50 PM
func NewDataFilter(source string, filterRule []*FilterDataRule) *DataFilter {
return &DataFilter{
source: source,
filterRule: filterRule,
hasDealDiffPath: make(map[string]string),
func NewDataFilter(source string, filterRule []*FilterDataRule, filterOption *FilterOption) *DataFilter {
source = fmt.Sprintf(`{"%v":%v}`, virtualRoot, source)
if nil == filterOption {
filterOption = &FilterOption{}
}
if filterOption.DebugModel && nil == filterOption.LogInstance {
filterOption.LogInstance = os.Stdout
log.SetOutput(filterOption.LogInstance)
}
df := &DataFilter{
source: source,
filterRule: make([]*FilterDataRule, 0),
rewriteResult: "{}",
filterOption: filterOption,
}
// 去除末尾的 .[]
for _, item := range filterRule {
item.MapKey = virtualRoot + "." + strings.TrimRight(item.MapKey, ".[]")
item.SourceKey = virtualRoot + "." + strings.TrimRight(item.SourceKey, ".[]")
mapIsArr := df.isArrPath(item.MapKey)
if !mapIsArr {
df.filterRule = append(df.filterRule, item)
continue
}
if len(df.getArrPathList(item.SourceKey)) < len(df.getArrPathList(item.MapKey)) {
panic("map result deep more than source data deep")
}
formatRes := make([]*FilterDataRule, 0)
r, _ := df.unfoldSameDeepArr(item.SourceKey, item.MapKey, "", "")
for _, itemUnfoldResult := range r {
itemMapArr := strings.Split(itemUnfoldResult.MapKey, ".")
if len(itemMapArr) != len(strings.Split(item.MapKey, ".")) {
continue
}
formatRes = append(formatRes, itemUnfoldResult)
}
df.filterRule = append(df.filterRule, formatRes...)
}
return df
}
// DataFilter 数据过滤
@ -52,10 +98,10 @@ func NewDataFilter(source string, filterRule []*FilterDataRule) *DataFilter {
//
// Date : 2022/1/22 9:20 PM
type DataFilter struct {
source string
filterRule []*FilterDataRule
itemKeyToSlice bool
hasDealDiffPath map[string]string
source string
filterRule []*FilterDataRule
rewriteResult string // json数据重写结果
filterOption *FilterOption // 过滤选项
}
// Filter 数据过滤
@ -64,231 +110,224 @@ type DataFilter struct {
//
// Date : 2022/1/22 9:36 PM
func (df *DataFilter) Filter() (string, error) {
df.logPrint(logLevelDebug, "source_data => ", df.source)
var (
jsonObject *gabs.Container
err error
err error
)
// 格式化映射规则
df.formatRule()
// 记录 obj => slice 的数据类型
obg2slice := make(map[string]string)
// 创建数据的根结点
jsonObject = gabs.New()
for _, item := range df.filterRule {
// 数据源路径不识数组, 多个key写入到同一个map key, 并且map key 不是以[]结尾, 自动格式化
// 目标位置, 是一个数组
if df.pathIsArrayValue(item.MapKey) {
realMapKey := strings.Trim(item.MapKey, ".[]")
if exist := jsonObject.Exists(realMapKey); !exist {
if _, err = jsonObject.ArrayP(realMapKey); nil != err {
return "", err
}
}
valueResult := gjson.Get(df.source, item.SourceKey)
dataType := df.getValueType(valueResult)
if _, exist := obg2slice[realMapKey]; !exist {
obg2slice[realMapKey] = dataType
}
if dataType != obg2slice[realMapKey] {
return "", errors.New(realMapKey + " 预期写入的字段数据类型不一致")
}
sourcePathArr := strings.Split(item.SourceKey, ".[].")
mapPathArr := strings.Split(realMapKey, ".[].")
result := gabs.New()
_, _ = result.ArrayP(mapPathArr[0])
df.SetArrayData("{\""+sourcePathArr[0]+"\":"+gjson.Get(df.source, sourcePathArr[0]).String()+"}", result, sourcePathArr, mapPathArr)
if err = jsonObject.ArrayAppend(valueResult.Value(), realMapKey); nil != err {
for _, itemRule := range df.filterRule {
sourceIsArr := df.isArrPath(itemRule.SourceKey)
mapIsArr := df.isArrPath(itemRule.MapKey)
if !sourceIsArr && !mapIsArr {
// 输入输出均不是数组, 最简单的场景
if err = df.setKV(itemRule); nil != err {
return "", err
}
continue
}
sourceSearchResult := gjson.Get(df.source, item.SourceKey)
if !sourceSearchResult.Exists() {
if item.WithDefault {
if _, err = jsonObject.SetP(item.DefaultValue, item.MapKey); nil != err {
return "", err
}
sourcePathArr := df.getArrPathList(itemRule.SourceKey)
mapPathArr := df.getArrPathList(itemRule.MapKey)
if len(mapPathArr) > len(sourcePathArr) {
df.logPrint(logLevelFatal, "映射的层级深度大于数据源深度", "source_path => "+itemRule.SourceKey, "map_path => "+itemRule.MapKey)
return "", fmt.Errorf("映射的层级深度大于数据源深度, source_path => %v map_path => %v", itemRule.SourceKey, itemRule.MapKey)
}
// 映射至非数组字段
if !mapIsArr {
if err = df.setValue(itemRule.MapKey, df.getDataAsSlice(df.source, df.getArrPathList(itemRule.SourceKey))); nil != err {
df.logPrint(logLevelFatal, "映射非数组, 数据源为数组, 设置失败", "source_path => "+itemRule.SourceKey, "map_path => "+itemRule.MapKey, " err => "+err.Error())
return "", fmt.Errorf("映射的层级深度大于数据源深度, source_path => %v map_path => %v", itemRule.SourceKey, itemRule.MapKey)
}
continue
}
if _, err = jsonObject.SetP(sourceSearchResult.Value(), item.MapKey); nil != err {
return "", err
if len(mapPathArr) == len(sourcePathArr) {
// 数组深度一致
continue
}
}
return jsonObject.String(), nil
df.logPrint(logLevelDebug, "过滤结果", df.rewriteResult)
return gjson.Get(df.rewriteResult, virtualRoot).String(), nil
}
// UserItemToSlice 支持多个独立的字段合并到slice中
// isArrPath 是否为数组路径
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 3:27 PM 2022/1/24
func (df *DataFilter) UserItemToSlice() {
df.itemKeyToSlice = true
}
// getValueType 获取数据类型
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2022/1/23 12:45 AM
func (df *DataFilter) getValueType(valueResult gjson.Result) string {
dataTypeVal := reflect.TypeOf(valueResult.Value())
if nil == dataTypeVal {
return "NIL"
}
dataType := dataTypeVal.String()
if strings.Contains(dataType, "int") {
return "int64"
}
if strings.Contains(dataType, "float") {
return "float64"
}
return dataType
}
// pathIsArrayValue 判断路径是否为数组值
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2022/1/23 12:56 AM
func (df *DataFilter) pathIsArrayValue(path string) bool {
// Date : 16:00 2023/9/1
func (df *DataFilter) isArrPath(path string) bool {
return strings.Contains(path, "[]")
}
// formatRule 格式化映射规则
// setSameDeepArr 展开同深度数组的
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2:43 PM 2022/1/24
func (df *DataFilter) formatRule() {
mapKeyCnt := make(map[string]int)
for _, item := range df.filterRule {
// source 为数组, map 不是
if df.pathIsArrayValue(item.SourceKey) {
if !df.pathIsArrayValue(item.MapKey) {
item.MapKey = item.MapKey + ".[]"
} else {
// source 是数组, map也是数组, 检测数组层级匹配
sourcePathArr := strings.Split(item.SourceKey, ".[].")
mapPathArr := strings.Split(item.MapKey, ".[].")
if len(sourcePathArr) == len(mapPathArr) {
// 数组层级深度相同,无需特殊处理
continue
}
// 数组层级深度不同,重新对对齐数据
diffArr := sourcePathArr[0 : len(sourcePathArr)-len(mapPathArr)+1]
if newPath := df.dealDiffArr(diffArr); len(newPath) > 0 {
sourcePathArr[len(sourcePathArr)-len(mapPathArr)] = newPath
item.SourceKey = strings.Join(sourcePathArr[len(sourcePathArr)-len(mapPathArr):], ".[].")
}
}
// Date : 12:19 2023/9/2
func (df *DataFilter) unfoldSameDeepArr(sourcePath string, mapPath string, sourceRootPath string, mapRootPath string) ([]*FilterDataRule, error) {
df.logPrint(logLevelDebug, " 同深数组展开", sourceRootPath, mapRootPath)
result := make([]*FilterDataRule, 0)
if len(sourcePath) == 0 {
return result, nil
}
sourcePathArr := df.getArrPathList(sourcePath)
mapPathArr := df.getArrPathList(mapPath)
if 1 == len(mapPathArr) {
sourceKey := sourceRootPath + "." + sourcePathArr[0]
if len(sourcePathArr[0:]) > 0 {
sourceKey = sourceRootPath + "." + strings.Join(sourcePathArr[0:], ".[].")
}
result = append(result, &FilterDataRule{
SourceKey: sourceKey,
MapKey: mapRootPath + "." + mapPathArr[0],
DefaultValue: nil,
WithDefault: false,
})
return result, nil
}
// 数组 展开
for idx := 0; idx < len(mapPathArr); idx++ {
if len(sourceRootPath) > 0 {
sourceRootPath = fmt.Sprintf("%v.%v", sourceRootPath, sourcePathArr[idx])
} else {
if df.pathIsArrayValue(item.MapKey) {
continue
}
// source 不是数组, map 也不是
if !df.itemKeyToSlice {
continue
}
mapKeyCnt[item.MapKey]++
sourceRootPath = sourcePathArr[idx]
}
}
// 多个source指向一个map,自动转化为list
for _, item := range df.filterRule {
if mapKeyCnt[item.MapKey] > 1 {
item.MapKey = item.MapKey + ".[]"
}
}
}
// dealDiffArr 提取数据映射关系
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 5:04 下午 2022/1/25
func (df *DataFilter) dealDiffArr(diffArr []string) string {
if len(diffArr) == 0 {
return ""
}
diffArrStr := strings.Join(diffArr, ".[].")
if _, exist := df.hasDealDiffPath[diffArrStr]; exist {
// 已经处理过, 不再重复处理
return df.hasDealDiffPath[diffArrStr]
}
// 没处理过, 开始处理
jsonResultList := df.getArrayData(df.source, diffArr)
if len(jsonResultList) == 0 {
return ""
}
newPath := wrapper.StringFromRandom(8, "").Value()
var result map[string]interface{}
_ = json.Unmarshal([]byte(df.source), &result)
JSONObject, _ := gabs.Consume(result)
_, _ = JSONObject.ArrayP(newPath)
for _, item := range jsonResultList {
if err := JSONObject.ArrayAppendP(item.Value(), newPath); nil != err {
fmt.Println(err.Error())
}
}
df.source = JSONObject.String()
df.hasDealDiffPath[diffArrStr] = newPath
return newPath
}
// getArrayData 获取数据
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2:22 下午 2022/1/26
func (df *DataFilter) getArrayData(source string, pathArr []string) []gjson.Result {
if len(pathArr) == 1 {
return gjson.Get(source, pathArr[0]).Array()
}
resultList := make([]gjson.Result, 0)
dataList := gjson.Get(source, pathArr[0]).Array()
for idx := 0; idx < len(dataList); idx++ {
resultList = append(resultList, df.getArrayData(dataList[idx].String(), pathArr[1:])...)
}
return resultList
}
// SetArrayData 设置数组数据
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 5:05 下午 2022/2/2
func (df *DataFilter) SetArrayData(sourceData string, jsonObject *gabs.Container, sourcePathArr []string, mapPathArr []string) *gabs.Container {
jsonObject = gabs.New()
for idx, sourcePath := range sourcePathArr {
if idx < len(sourcePathArr)-1 {
if !jsonObject.Exists(sourcePath) {
_, _ = jsonObject.ArrayP(sourcePath)
}
}
instance, _ := gabs.ParseJSON([]byte(sourceData))
if !instance.Exists() {
fmt.Println(sourcePathArr[len(sourcePathArr)-1] + " 不存在")
if len(mapRootPath) > 0 {
mapRootPath = fmt.Sprintf("%v.%v", mapRootPath, mapPathArr[idx])
} else {
dataList, _ := instance.Children()
for _, item := range dataList {
cItem := gabs.New()
cItem.SetP(gjson.Get(item.String(), sourcePath).String(), mapPathArr[idx])
jsonObject.ArrayAppendP(cItem.Data(), mapPathArr[idx])
mapRootPath = mapPathArr[idx]
}
valList := gjson.Get(df.source, sourceRootPath)
if valList.Value() == nil {
continue
}
for i := 0; i < len(valList.Array()); i++ {
result = append(result, &FilterDataRule{
SourceKey: sourceRootPath,
MapKey: mapRootPath,
DefaultValue: nil,
WithDefault: false,
})
r, e := df.unfoldSameDeepArr(
strings.Join(sourcePathArr[idx+1:], "[]"),
strings.Join(mapPathArr[idx+1:], "[]"),
fmt.Sprintf("%v.%v", sourceRootPath, i),
fmt.Sprintf("%v.%v", mapRootPath, i),
)
if nil != e {
return nil, e
}
//jsonObject.ArrayAppend(jsonObject.Data())
// fmt.Println("数据 : ", jsonObject.String())
// jsonObject.ArrayAppendP(result.Data(), mapPathArr[idx])
result = append(result, r...)
}
}
return result, nil
}
// a.[].b.[].c.[].d
// g
// getDataAsSlice 抽取制定深度，生成list
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 12:17 2023/9/2
func (df *DataFilter) getDataAsSlice(sourceData string, pathList []string) []interface{} {
//fmt.Println(sourceData, pathList)
result := make([]interface{}, 0)
if len(pathList) == 0 {
return result
}
if len(pathList) == 1 {
return []interface{}{gjson.Get(sourceData, pathList[0]).Value()}
}
for idx, itemPath := range pathList {
if len(pathList)-1 == idx {
val := gjson.Get(sourceData, itemPath).Value()
if nil == val {
return result
}
result = append(result, val)
return result
}
df.SetArrayData(gjson.Get(sourceData, sourcePathArr[idx]).String(), jsonObject, sourcePathArr[idx+1:], mapPathArr[idx+1:])
// jsonObject.ArrayAppendP(v.Data(), mapPathArr[idx])
currentPathVal := gjson.Get(sourceData, itemPath).Array()
for _, sonItem := range currentPathVal {
result = append(result, df.getDataAsSlice(sonItem.String(), pathList[idx:])...)
}
}
fmt.Println("最终 : ", jsonObject.String())
return jsonObject
return result
}
// setValue 设置值
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 22:04 2023/9/1
func (df *DataFilter) setValue(path string, value interface{}) error {
var (
err error
)
df.rewriteResult, err = sjson.Set(df.rewriteResult, path, value)
return err
}
// setKV 设置相关值
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 16:03 2023/9/1
func (df *DataFilter) setKV(rule *FilterDataRule) error {
var (
err error
)
sourceValue := gjson.Get(df.source, rule.SourceKey)
if sourceValue.Exists() {
// 原始数据存在对应路径
df.rewriteResult, err = sjson.Set(df.rewriteResult, rule.MapKey, sourceValue.Value())
return err
}
if !rule.WithDefault {
// 路径不存在, 且禁用默认值
return errors.New(rule.SourceKey + " : source path not found, and default value is forbidden")
}
// 使用默认值填充
df.rewriteResult, err = sjson.Set(df.rewriteResult, rule.MapKey, rule.DefaultValue)
return err
}
// getArrPathList 获取路径列表
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 21:32 2023/9/1
func (df *DataFilter) getArrPathList(inputPath string) []string {
pathArr := strings.Split(inputPath, "[]")
arr := make([]string, 0)
for _, item := range pathArr {
arr = append(arr, strings.Trim(item, "."))
}
return arr
}
// logPrint 打印日志
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 18:00 2023/9/1
func (df *DataFilter) logPrint(level string, msg string, logAttr ...interface{}) {
if !df.filterOption.DebugModel {
// 未开启调试模式
return
}
logData := append([]interface{}{level, msg}, logAttr...)
log.Println(logData...)
}
// 日志等级定义
const (
logLevelPanic = "PANIC"
logLevelFatal = "FATAL"
logLevelWarn = "WARN"
logLevelInfo = "INFO"
logLevelDebug = "DEBUG"
)

285
tool/gabs_test.go Normal file
View File

@ -0,0 +1,285 @@
// Package json_tool ...
//
// Description : json_tool ...
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2023-09-01 16:07
package json_tool
import (
"encoding/json"
"fmt"
"strings"
"testing"
"git.zhangdeman.cn/zhangdeman/serialize"
)
// TestDataFilter_FilterNormalData 最基础对象
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 16:13 2023/9/1
func TestDataFilter_FilterNormalData(t *testing.T) {
source := map[string]interface{}{
"name": "zhangdeman",
"extra": map[string]interface{}{
"age": 18,
"height": 180,
"slice": []int{1, 2, 3},
},
"slice": []int{1, 2, 3},
}
df := &DataFilter{
source: serialize.JSON.MarshalForString(source),
filterRule: []*FilterDataRule{
{SourceKey: "name", MapKey: "user_name", DefaultValue: "油猴", WithDefault: true},
{SourceKey: "extra.age", MapKey: "user_age", DefaultValue: "18", WithDefault: true},
{SourceKey: "slice", MapKey: "user_index", DefaultValue: "[4,5,6]", WithDefault: true},
{SourceKey: "none", MapKey: "none_default", DefaultValue: map[string]interface{}{"a": "a"}, WithDefault: true},
{SourceKey: "extra", MapKey: "extra_object", DefaultValue: map[string]interface{}{"a": "a"}, WithDefault: true},
},
filterOption: &FilterOption{DebugModel: true},
}
fmt.Println(df.Filter())
}
// TestDataFilter_getDataAsSlice 测试循环读取数据
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 22:21 2023/9/1
func TestDataFilter_getDataAsSlice(t *testing.T) {
data := map[string]interface{}{
"list": []interface{}{
map[string]interface{}{
"name": "1",
"age": "2",
"list_test": []interface{}{
map[string]interface{}{
"a": "a",
"b": "b",
},
},
},
map[string]interface{}{
"name": "3",
"age": "4",
"list_test": []interface{}{
map[string]interface{}{
"a": "a1",
"b": "b1",
},
},
},
map[string]interface{}{
"name": "5",
"age": "6",
},
},
}
byteData, _ := json.Marshal(data)
df := &DataFilter{}
fmt.Println(df.getDataAsSlice(string(byteData), []string{"list", "name"}))
fmt.Println(df.getDataAsSlice(string(byteData), []string{"list", "age"}))
fmt.Println(df.getDataAsSlice(string(byteData), []string{"list", "age", "list_test", "a"}))
fmt.Println(df.getDataAsSlice(string(byteData), []string{"list", "age", "list_test", "b"}))
}
// TestDataFilter_sourceArrAndMapSingle map 非数组，数据源数组
//
// Author : zhangdeman001@ke.com<张德满>
//
// Date : 11:24 2023/9/2
func TestDataFilter_sourceArrAndMapSingle(t *testing.T) {
source := map[string]interface{}{
"list": []interface{}{
map[string]interface{}{
"name": "1",
"age": "2",
"list_test": []interface{}{
map[string]interface{}{
"a": "a",
"b": "b",
"c": map[string]interface{}{
"a": "1",
},
"d": []int{1, 2, 3},
},
},
},
map[string]interface{}{
"name": "3",
"age": "4",
"list_test": []interface{}{
map[string]interface{}{
"a": "a1",
"b": "b1",
"c": map[string]interface{}{
"a": "a",
},
"d": []int{1, 2, 3},
},
},
},
map[string]interface{}{
"name": "5",
"age": "6",
},
},
}
df := &DataFilter{
source: serialize.JSON.MarshalForString(source),
filterRule: []*FilterDataRule{
{SourceKey: "list.[].list_test.[].a", MapKey: "a_list", DefaultValue: "[]", WithDefault: true},
{SourceKey: "list.[].list_test.[].b", MapKey: "b_list", DefaultValue: "[]", WithDefault: true},
{SourceKey: "list.[].list_test.[].c", MapKey: "c_list", DefaultValue: "[]", WithDefault: true},
{SourceKey: "list.[].list_test.[].d", MapKey: "d_list", DefaultValue: "[]", WithDefault: true},
},
filterOption: &FilterOption{DebugModel: true},
}
fmt.Println(df.Filter())
}
// TestDataFilter_unfoldSameDeepArr 测试展开同深度数组
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 15:42 2023/9/2
func TestDataFilter_unfoldSameDeepArr(t *testing.T) {
source := map[string]interface{}{
"list": []interface{}{
map[string]interface{}{
"name": "1",
"age": "2",
"list_test": []interface{}{
map[string]interface{}{
"a": "a",
"b": "b",
"c": map[string]interface{}{
"a": "1",
},
"d": []int{1, 2, 3},
},
},
},
map[string]interface{}{
"name": "3",
"age": "4",
"list_test": []interface{}{
map[string]interface{}{
"a": "a1",
"b": "b1",
"c": map[string]interface{}{
"a": "a",
},
"d": []int{1, 2, 3},
},
map[string]interface{}{
"a": "a2",
"b": "b1",
"c": map[string]interface{}{
"a": "a",
},
"d": []int{1, 2, 3},
},
},
},
map[string]interface{}{
"name": "5",
"age": "6",
},
},
}
df := &DataFilter{
source: serialize.JSON.MarshalForString(source),
filterRule: []*FilterDataRule{
{SourceKey: "list.[].list_test.[].a", MapKey: "a_list.[].a.[].val", DefaultValue: "[]", WithDefault: true},
{SourceKey: "list.[].list_test.[].b", MapKey: "b_list.[].b.[].val", DefaultValue: "[]", WithDefault: true},
{SourceKey: "list.[].list_test.[].c", MapKey: "c_list.[].c.[].val", DefaultValue: "[]", WithDefault: true},
{SourceKey: "list.[].list_test.[].d", MapKey: "d_list.[].d.[].val", DefaultValue: "[]", WithDefault: true},
},
filterOption: &FilterOption{DebugModel: true},
}
r, _ := df.unfoldSameDeepArr("list.[].list_test.[].a", "a_list.[].a_not_equal_list", "", "")
formatRes := make([]*FilterDataRule, 0)
for _, item := range r {
itemMapArr := strings.Split(item.MapKey, ".")
if len(itemMapArr) != 3 && len(itemMapArr) != 5 {
continue
}
formatRes = append(formatRes, item)
}
fmt.Println(serialize.JSON.MarshalForString(formatRes))
}
// TestDataFilter_filterSameDeepArr ...
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 20:41 2023/9/2
func TestDataFilter_filterSameDeepArr(t *testing.T) {
source := map[string]interface{}{
"list": []interface{}{
map[string]interface{}{
"name": "1",
"age": "2",
"list_test": []interface{}{
map[string]interface{}{
"a": "a",
"b": "b",
"c": map[string]interface{}{
"a": "1",
},
"d": []int{1, 2, 3},
},
},
},
map[string]interface{}{
"name": "3",
"age": "4",
"list_test": []interface{}{
map[string]interface{}{
"a": "a1",
"b": "b1",
"c": map[string]interface{}{
"a": "a",
},
"d": []int{1, 2, 3},
},
map[string]interface{}{
"a": "a2",
"b": "b1",
"c": map[string]interface{}{
"a": "a",
},
"d": []int{1, 2, 3},
},
},
},
map[string]interface{}{
"name": "5",
"age": "6",
},
},
}
filterRuleList := []*FilterDataRule{
{SourceKey: "name", MapKey: "user_name", DefaultValue: "油猴", WithDefault: true},
{SourceKey: "extra.age", MapKey: "user_age", DefaultValue: "18", WithDefault: true},
{SourceKey: "slice", MapKey: "user_index", DefaultValue: "[4,5,6]", WithDefault: true},
{SourceKey: "none", MapKey: "none_default", DefaultValue: map[string]interface{}{"a": "a"}, WithDefault: true},
{SourceKey: "extra", MapKey: "extra_object", DefaultValue: map[string]interface{}{"a": "a"}, WithDefault: true},
{SourceKey: "list.[].list_test.[].a", MapKey: "a_list.[].a.[].val", DefaultValue: "[]", WithDefault: true},
{SourceKey: "list.[].list_test.[].a", MapKey: "a_list.[].a_not_equal_list", DefaultValue: "[]", WithDefault: true},
{SourceKey: "list.[].list_test.[].a", MapKey: "a_list.[].a.[].val1", DefaultValue: "[]", WithDefault: true},
{SourceKey: "list.[].list_test.[].b", MapKey: "b_list.[].b.[].val", DefaultValue: "[]", WithDefault: true},
{SourceKey: "list.[].list_test.[].c", MapKey: "c_list.[].c.[].val", DefaultValue: "[]", WithDefault: true},
{SourceKey: "list.[].list_test.[].d", MapKey: "d_list.[].d.[].val", DefaultValue: "[]", WithDefault: true},
}
filterOption := &FilterOption{DebugModel: true}
df := NewDataFilter(serialize.JSON.MarshalForString(source), filterRuleList, filterOption)
_, _ = df.Filter()
}

157
tool/json_test.go
View File

@ -42,15 +42,6 @@ func TestJSON(t *testing.T) {
fmt.Println(tree.String())
}
// TestType 判断数据类型断言
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 10:59 下午 2021/3/14
func TestType(t *testing.T) {
}
// TestSelect 测试动态选择字段
//
// Author : go_developer@163.com<白茶清欢>
@ -127,151 +118,3 @@ func TestParse(t *testing.T) {
byteData, _ := json.Marshal(source)
fmt.Println(GetJSONDataStruct(string(byteData)))
}
// TestParseWithType 测试获取JSON数据结构
//
// Author : go_developer@163.com<张德满>
//
// Date : 10:59 PM 2022/1/9
func TestParseWithType(t *testing.T) {
source := map[string]interface{}{
"name": "zhangdeman",
"extra": map[string]interface{}{
"age": 18,
"height": 180,
"slice": []int{1, 2, 3},
"obj": map[string]interface{}{
"la": "aaaa",
},
},
"slice": []int{1, 2, 3},
"map": map[string]interface{}{"a": 1, "d": 5.5, "e": "qqq"},
"empty_obj": map[string]interface{}{},
"empty_list": make([]interface{}, 0),
"table": []map[string]interface{}{
{"name": "alex", "age": 18, "number": 1, "obj": map[string]interface{}{"enen": "en"}},
{"name": "bob", "age": 28, "number": 2},
},
"two_slice": []map[string]interface{}{
{
"students": []map[string]interface{}{
{
"name": "enen",
"age": 18,
"score": []float64{1, 2, 3, 45},
},
},
"other": []interface{}{"others"},
"read_only": 1,
},
},
}
byteData, _ := json.Marshal(source)
fmt.Println(GetJSONDataStructWithType(string(byteData)))
}
// TestDataFilter 测试数据过滤
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2022/1/22 10:19 PM
func TestDataFilter(t *testing.T) {
source := map[string]interface{}{
"name": "zhangdeman",
"extra": map[string]interface{}{
"age": 18,
"height": 180,
"slice": []int{1, 2, 3},
},
"slice_data": []int{1, 2, 3},
"map": map[string]interface{}{"a": 1, "b": 2, "c": 4},
"table": []map[string]interface{}{
{"name": "alex", "age": 18, "number": 1},
{"name": "bob", "age": 28, "number": 2},
{"name": "bob", "age": 28, "number": 2, "list": []int{1, 2, 3}},
},
}
rule := []*FilterDataRule{
{SourceKey: "name", MapKey: "user_name", DefaultValue: "用户姓名默认值"},
{SourceKey: "name", MapKey: "username", DefaultValue: "用户姓名默认值"},
{SourceKey: "name", MapKey: "user.name", DefaultValue: "用户姓名默认值"},
{SourceKey: "extra.age", MapKey: "user.age", DefaultValue: "用户年龄默认值"},
{SourceKey: "extra.age", MapKey: "user_age", DefaultValue: "用户年龄默认值"},
{SourceKey: "extra.height", MapKey: "user.height", DefaultValue: "扩展高度默认值"},
{SourceKey: "extra.height", MapKey: "user_height", DefaultValue: "扩展高度默认值"},
{SourceKey: "table.[].name", MapKey: "slice.[].name_modify", DefaultValue: "列表姓名默认值"},
{SourceKey: "table.[].list", MapKey: "slice.[].data_list", DefaultValue: "[\"567\",\"678\",\"789\"]"},
}
byteData, _ := json.Marshal(source)
filter := NewDataFilter(string(byteData), rule)
fmt.Println(filter.Filter())
}
// TestDataFilterForObiToSlice ...
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2022/1/23 12:06 AM
func TestDataFilterForObiToSlice(t *testing.T) {
source := map[string]interface{}{
"name": "zhangdeman",
"age": 18,
"height": 180,
"extra": map[string]interface{}{
"age": 18,
"height": 180,
"slice": []int{1, 2, 3},
},
"slice_data": []int{1, 2, 3},
"map": map[string]interface{}{"a": 1, "b": 2, "c": 4},
"table": []map[string]interface{}{
{"name": "alex", "age": 18, "number": 1},
{"name": "bob", "age": 28, "number": 2},
{"name": "bob", "age": 28, "number": 2, "list": []int{1, 2, 3}},
},
}
rule := []*FilterDataRule{
// {SourceKey: "name", MapKey: "slice.[]", DefaultValue: "用户姓名默认值"},
{SourceKey: "age", MapKey: "slice", DefaultValue: "用户姓名默认值"},
{SourceKey: "height", MapKey: "slice", DefaultValue: "用户姓名默认值"},
}
byteData, _ := json.Marshal(source)
filter := NewDataFilter(string(byteData), rule)
filter.UserItemToSlice()
fmt.Println(filter.Filter())
}
// TestDataFilterDiffArr ...
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 12:27 下午 2022/1/26
func TestDataFilterDiffArr(t *testing.T) {
source := map[string]interface{}{
"name": "zhangdeman",
"age": 18,
"height": 180,
"extra": map[string]interface{}{
"age": 18,
"height": 180,
"slice": []int{1, 2, 3},
},
"slice_data": []int{1, 2, 3},
"map": map[string]interface{}{"a": 1, "b": 2, "c": 4},
"table": []map[string]interface{}{
{"user_list": []interface{}{map[string]interface{}{"name": "alex", "age": 18, "number": 1}}},
{"user_list": []interface{}{map[string]interface{}{"name": "bob", "age": 28, "number": 2}}},
{"user_list": []interface{}{map[string]interface{}{"name": "andy", "age": 28, "number": 2}}},
},
}
rule := []*FilterDataRule{
// {SourceKey: "name", MapKey: "slice.[]", DefaultValue: "用户姓名默认值"},
{SourceKey: "table.[].user_list.[].name", MapKey: "user_list.[].detail.name", DefaultValue: "用户姓名默认值"},
{SourceKey: "table.[].user_list.[].age", MapKey: "user_list.[].detail.age", DefaultValue: "用户姓名默认值"},
}
byteData, _ := json.Marshal(source)
filter := NewDataFilter(string(byteData), rule)
filter.UserItemToSlice()
filter.Filter()
//fmt.Println()
}