zhangdeman/json_filter
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

完成数据过主干逻辑,细节健壮性有待继续完善 #4

Merged
zhangdeman merged 13 commits from feature/data_filter into master 2023-09-02 22:17:14 +08:00
Conversation 0 Commits 13 Files Changed 5 +230 -13
2 changed files with 230 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit f3baa17bd5 - Show all commits

100
tool/gabs.go
View File

@ -19,6 +19,11 @@ import (
"github.com/tidwall/sjson"
)
const (
// virtualRoot 虚拟根节点
virtualRoot = "__VIRTUAL_ROOT__"
)
// FilterOption 过滤选项
//
// Author : go_developer@163.com<白茶清欢>
@ -54,17 +59,36 @@ func NewDataFilter(source string, filterRule []*FilterDataRule, filterOption *Fi
filterOption.LogInstance = os.Stdout
log.SetOutput(filterOption.LogInstance)
}
df := &DataFilter{
source: source,
filterRule: make([]*FilterDataRule, 0),
rewriteResult: "{}",
filterOption: filterOption,
}
// 去除末尾的 .[]
for _, item := range filterRule {
item.MapKey = strings.TrimRight(item.MapKey, ".[]")
item.SourceKey = strings.TrimRight(item.SourceKey, ".[]")
mapIsArr := df.isArrPath(item.MapKey)
if !mapIsArr {
df.filterRule = append(df.filterRule, item)
continue
}
if len(df.getArrPathList(item.SourceKey)) < len(df.getArrPathList(item.MapKey)) {
panic("map result deep more than source data deep")
}
formatRes := make([]*FilterDataRule, 0)
r, _ := df.unfoldSameDeepArr(item.SourceKey, item.MapKey, "", "")
for _, itemUnfoldResult := range r {
itemMapArr := strings.Split(itemUnfoldResult.MapKey, ".")
if len(itemMapArr) != len(strings.Split(item.MapKey, ".")) {
continue
}
formatRes = append(formatRes, itemUnfoldResult)
}
df.filterRule = append(df.filterRule, formatRes...)
}
return &DataFilter{
source: source,
filterRule: filterRule,
rewriteResult: "{}",
filterOption: filterOption,
}
return df
}
// DataFilter 数据过滤
@ -120,8 +144,8 @@ func (df *DataFilter) Filter() (string, error) {
// 数组深度一致
continue
}
}
df.logPrint(logLevelDebug, "过滤结果", df.rewriteResult)
return df.rewriteResult, nil
}
@ -134,22 +158,72 @@ func (df *DataFilter) isArrPath(path string) bool {
return strings.Contains(path, "[]")
}
// setSameDeepArr 设置同深度的数组
// setSameDeepArr 展开同深度数组的
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 12:19 2023/9/2
func (df *DataFilter) setSameDeepArr(sourceVal string, sourcePath string, mapPath string) error {
func (df *DataFilter) unfoldSameDeepArr(sourcePath string, mapPath string, sourceRootPath string, mapRootPath string) ([]*FilterDataRule, error) {
df.logPrint(logLevelDebug, " 同深数组展开", sourceRootPath, mapRootPath)
result := make([]*FilterDataRule, 0)
if len(sourcePath) == 0 {
return result, nil
}
sourcePathArr := df.getArrPathList(sourcePath)
mapPathArr := df.getArrPathList(mapPath)
for idx, itemSourcePath := range sourcePathArr {
sourceValueArr := gjson.Get(sourceVal, sourcePath).Array()
if 1 == len(mapPathArr) {
sourceKey := sourceRootPath + "." + sourcePathArr[0]
if len(sourcePathArr[0:]) > 0 {
sourceKey = sourceRootPath + "." + strings.Join(sourcePathArr[0:], ".[].")
}
result = append(result, &FilterDataRule{
SourceKey: sourceKey,
MapKey: mapRootPath + "." + mapPathArr[0],
DefaultValue: nil,
WithDefault: false,
})
return result, nil
}
return nil
// 数组 展开
for idx := 0; idx < len(mapPathArr); idx++ {
if len(sourceRootPath) > 0 {
sourceRootPath = fmt.Sprintf("%v.%v", sourceRootPath, sourcePathArr[idx])
} else {
sourceRootPath = sourcePathArr[idx]
}
if len(mapRootPath) > 0 {
mapRootPath = fmt.Sprintf("%v.%v", mapRootPath, mapPathArr[idx])
} else {
mapRootPath = mapPathArr[idx]
}
valList := gjson.Get(df.source, sourceRootPath)
if valList.Value() == nil {
continue
}
for i := 0; i < len(valList.Array()); i++ {
result = append(result, &FilterDataRule{
SourceKey: sourceRootPath,
MapKey: mapRootPath,
DefaultValue: nil,
WithDefault: false,
})
r, e := df.unfoldSameDeepArr(
strings.Join(sourcePathArr[idx+1:], "[]"),
strings.Join(mapPathArr[idx+1:], "[]"),
fmt.Sprintf("%v.%v", sourceRootPath, i),
fmt.Sprintf("%v.%v", mapRootPath, i),
)
if nil != e {
return nil, e
}
result = append(result, r...)
}
}
return result, nil
}
// a.[].b.[].c.[].d
// e.[].f.[].g
// g
// getDataAsSlice 抽取制定深度，生成list
//
// Author : go_developer@163.com<白茶清欢>

143
tool/gabs_test.go
View File

@ -10,6 +10,7 @@ package json_tool
import (
"encoding/json"
"fmt"
"strings"
"testing"
"git.zhangdeman.cn/zhangdeman/serialize"
@ -140,3 +141,145 @@ func TestDataFilter_sourceArrAndMapSingle(t *testing.T) {
}
fmt.Println(df.Filter())
}
// TestDataFilter_unfoldSameDeepArr 测试展开同深度数组
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 15:42 2023/9/2
func TestDataFilter_unfoldSameDeepArr(t *testing.T) {
source := map[string]interface{}{
"list": []interface{}{
map[string]interface{}{
"name": "1",
"age": "2",
"list_test": []interface{}{
map[string]interface{}{
"a": "a",
"b": "b",
"c": map[string]interface{}{
"a": "1",
},
"d": []int{1, 2, 3},
},
},
},
map[string]interface{}{
"name": "3",
"age": "4",
"list_test": []interface{}{
map[string]interface{}{
"a": "a1",
"b": "b1",
"c": map[string]interface{}{
"a": "a",
},
"d": []int{1, 2, 3},
},
map[string]interface{}{
"a": "a2",
"b": "b1",
"c": map[string]interface{}{
"a": "a",
},
"d": []int{1, 2, 3},
},
},
},
map[string]interface{}{
"name": "5",
"age": "6",
},
},
}
df := &DataFilter{
source: serialize.JSON.MarshalForString(source),
filterRule: []*FilterDataRule{
{SourceKey: "list.[].list_test.[].a", MapKey: "a_list.[].a.[].val", DefaultValue: "[]", WithDefault: true},
{SourceKey: "list.[].list_test.[].b", MapKey: "b_list.[].b.[].val", DefaultValue: "[]", WithDefault: true},
{SourceKey: "list.[].list_test.[].c", MapKey: "c_list.[].c.[].val", DefaultValue: "[]", WithDefault: true},
{SourceKey: "list.[].list_test.[].d", MapKey: "d_list.[].d.[].val", DefaultValue: "[]", WithDefault: true},
},
filterOption: &FilterOption{DebugModel: true},
}
r, _ := df.unfoldSameDeepArr("list.[].list_test.[].a", "a_list.[].a_not_equal_list", "", "")
formatRes := make([]*FilterDataRule, 0)
for _, item := range r {
itemMapArr := strings.Split(item.MapKey, ".")
if len(itemMapArr) != 3 && len(itemMapArr) != 5 {
continue
}
formatRes = append(formatRes, item)
}
fmt.Println(serialize.JSON.MarshalForString(formatRes))
}
// TestDataFilter_filterSameDeepArr ...
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 20:41 2023/9/2
func TestDataFilter_filterSameDeepArr(t *testing.T) {
source := map[string]interface{}{
"list": []interface{}{
map[string]interface{}{
"name": "1",
"age": "2",
"list_test": []interface{}{
map[string]interface{}{
"a": "a",
"b": "b",
"c": map[string]interface{}{
"a": "1",
},
"d": []int{1, 2, 3},
},
},
},
map[string]interface{}{
"name": "3",
"age": "4",
"list_test": []interface{}{
map[string]interface{}{
"a": "a1",
"b": "b1",
"c": map[string]interface{}{
"a": "a",
},
"d": []int{1, 2, 3},
},
map[string]interface{}{
"a": "a2",
"b": "b1",
"c": map[string]interface{}{
"a": "a",
},
"d": []int{1, 2, 3},
},
},
},
map[string]interface{}{
"name": "5",
"age": "6",
},
},
}
filterRuleList := []*FilterDataRule{
{SourceKey: "name", MapKey: "user_name", DefaultValue: "油猴", WithDefault: true},
{SourceKey: "extra.age", MapKey: "user_age", DefaultValue: "18", WithDefault: true},
{SourceKey: "slice", MapKey: "user_index", DefaultValue: "[4,5,6]", WithDefault: true},
{SourceKey: "none", MapKey: "none_default", DefaultValue: map[string]interface{}{"a": "a"}, WithDefault: true},
{SourceKey: "extra", MapKey: "extra_object", DefaultValue: map[string]interface{}{"a": "a"}, WithDefault: true},
{SourceKey: "list.[].list_test.[].a", MapKey: "a_list.[].a.[].val", DefaultValue: "[]", WithDefault: true},
{SourceKey: "list.[].list_test.[].a", MapKey: "a_list.[].a_not_equal_list", DefaultValue: "[]", WithDefault: true},
{SourceKey: "list.[].list_test.[].a", MapKey: "a_list.[].a.[].val1", DefaultValue: "[]", WithDefault: true},
{SourceKey: "list.[].list_test.[].b", MapKey: "b_list.[].b.[].val", DefaultValue: "[]", WithDefault: true},
{SourceKey: "list.[].list_test.[].c", MapKey: "c_list.[].c.[].val", DefaultValue: "[]", WithDefault: true},
{SourceKey: "list.[].list_test.[].d", MapKey: "d_list.[].d.[].val", DefaultValue: "[]", WithDefault: true},
}
filterOption := &FilterOption{DebugModel: true}
df := NewDataFilter(serialize.JSON.MarshalForString(source), filterRuleList, filterOption)
_, _ = df.Filter()
}