优化数据过滤处理
This commit is contained in:
parent
ce1e7fe9da
commit
68d3d76062
120
filter.go
120
filter.go
@ -10,12 +10,13 @@ package filter
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
|
||||
"git.zhangdeman.cn/zhangdeman/consts"
|
||||
"git.zhangdeman.cn/zhangdeman/json_filter/gjson_hack"
|
||||
"git.zhangdeman.cn/zhangdeman/serialize"
|
||||
"git.zhangdeman.cn/zhangdeman/wrapper"
|
||||
"reflect"
|
||||
"strings"
|
||||
|
||||
"github.com/tidwall/gjson"
|
||||
"github.com/tidwall/sjson"
|
||||
@ -54,8 +55,7 @@ type filter struct {
|
||||
// Date : 11:59 2022/7/4
|
||||
func (f *filter) Deal() error {
|
||||
var (
|
||||
err error
|
||||
formatVal any
|
||||
err error
|
||||
)
|
||||
|
||||
for _, rule := range f.filterRuleList {
|
||||
@ -63,14 +63,6 @@ func (f *filter) Deal() error {
|
||||
// 未配置目标路径则, 目标路径和源路径保持一致
|
||||
rule.TargetPath = rule.SourcePath
|
||||
}
|
||||
if strings.Contains(rule.SourcePath, gjson_hack.ArrayIdxTpl) {
|
||||
// 数组，验证数组层级是否一致
|
||||
sourceArr := strings.Split(rule.SourcePath, gjson_hack.ArrayIdxTpl)
|
||||
TargetArr := strings.Split(rule.TargetPath, gjson_hack.ArrayIdxTpl)
|
||||
if len(sourceArr) != len(TargetArr) {
|
||||
return errors.New(rule.SourcePath + " and " + rule.TargetPath + " array deep not match")
|
||||
}
|
||||
}
|
||||
if f.IsArray(rule) {
|
||||
// 对于list的处理, 展开层级, 并自动追加到f.filterRuleList 后面
|
||||
if err = f.handleArray(rule); nil != err {
|
||||
@ -78,32 +70,7 @@ func (f *filter) Deal() error {
|
||||
}
|
||||
continue
|
||||
}
|
||||
sourceResult := gjson.Get(f.sourceData, rule.SourcePath)
|
||||
if formatVal, err = f.getValue(rule.DataType, sourceResult, rule.DefaultValue); nil != err {
|
||||
return fmt.Errorf("%s = %v can not convert to %s : %s", rule.SourcePath, sourceResult.Value(), rule.DataType, err.Error())
|
||||
}
|
||||
if reflect.TypeOf(formatVal).Kind() == reflect.Map {
|
||||
// 获取的数据是map类型, 处理数据覆盖
|
||||
// eg : 配置如下两个规则 process.id(string) 、process(map[string]any)
|
||||
// 若输入数据的process.id为int类型, 则格式化后的process.id必为 string, 应为 process.id 规则的控制更精细
|
||||
gjsonVal := gjson.Get(f.formatResult, rule.TargetPath)
|
||||
if gjsonVal.Exists() && gjsonVal.IsObject() {
|
||||
var (
|
||||
existRes = map[string]any{}
|
||||
formatRes = map[string]any{}
|
||||
)
|
||||
// 已存在, 且是对象
|
||||
_ = serialize.JSON.UnmarshalWithNumber([]byte(gjsonVal.String()), &existRes)
|
||||
if err = serialize.JSON.Transition(formatVal, &formatRes); nil != err {
|
||||
return errors.New("conflict data path config deal fail : " + err.Error())
|
||||
}
|
||||
for k, v := range existRes {
|
||||
formatRes[k] = v
|
||||
}
|
||||
formatVal = formatRes // 重新赋值 formatVal
|
||||
}
|
||||
}
|
||||
if f.formatResult, err = sjson.Set(f.formatResult, rule.TargetPath, formatVal); nil != err {
|
||||
if err = f.setResult(rule); nil != err {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@ -125,34 +92,71 @@ func (f *filter) IsArray(rule MapRule) bool {
|
||||
//
|
||||
// Date : 17:41 2023/1/1
|
||||
func (f *filter) handleArray(rule MapRule) error {
|
||||
// TODO : 对于list的处理, 展开层级, 并自动追加到f.filterRuleList 后面
|
||||
// 对于list的处理, 展开层级, 并自动追加到f.filterRuleList 后面
|
||||
var (
|
||||
err error
|
||||
)
|
||||
|
||||
sourcePathArray := strings.Split(rule.SourcePath, "[]")
|
||||
for idx, item := range sourcePathArray {
|
||||
sourcePathArray[idx] = strings.Trim(item, ".")
|
||||
expendRes := &gjson_hack.ExpendArrayResult{
|
||||
PathList: make([]string, 0),
|
||||
PathMap: map[string]string{},
|
||||
}
|
||||
mapPathArray := strings.Split(strings.TrimRight(rule.TargetPath, ".[]"), "[]")
|
||||
for idx, item := range mapPathArray {
|
||||
mapPathArray[idx] = strings.Trim(item, ".")
|
||||
if err = gjson_hack.ExpandArrayPath(f.sourceData, rule.SourcePath, rule.TargetPath, expendRes); nil != err {
|
||||
return err
|
||||
}
|
||||
if len(sourcePathArray) != len(mapPathArray) {
|
||||
if len(mapPathArray) != 1 {
|
||||
return errors.New("map rule is invalid")
|
||||
}
|
||||
// 提取某一个list下的字段, 组成一个list
|
||||
res := make([]string, 0)
|
||||
if len(sourcePathArray[0]) == 0 {
|
||||
f.getAllFinalData(&res, gjson.Parse(f.sourceData).Array(), sourcePathArray[1:])
|
||||
} else {
|
||||
f.getAllFinalData(&res, gjson.Get(f.sourceData, sourcePathArray[0]).Array(), sourcePathArray[1:])
|
||||
}
|
||||
if f.formatResult, err = sjson.Set(f.formatResult, mapPathArray[0], res); nil != err {
|
||||
for _, itemPath := range expendRes.PathList {
|
||||
if err = f.setResult(MapRule{
|
||||
SourcePath: itemPath,
|
||||
TargetPath: expendRes.PathMap[itemPath],
|
||||
Required: rule.Required,
|
||||
DataType: rule.DataType,
|
||||
DefaultValue: rule.DefaultValue,
|
||||
}); nil != err {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// setResult 设置结果
|
||||
//
|
||||
// Author : go_developer@163.com<白茶清欢>
|
||||
//
|
||||
// Date : 19:03 2024/11/30
|
||||
func (f *filter) setResult(rule MapRule) error {
|
||||
var (
|
||||
err error
|
||||
formatVal any
|
||||
)
|
||||
|
||||
sourceResult := gjson.Get(f.sourceData, rule.SourcePath)
|
||||
if formatVal, err = f.getValue(rule.DataType, sourceResult, rule.DefaultValue); nil != err {
|
||||
return fmt.Errorf("%s = %v can not convert to %s : %s", rule.SourcePath, sourceResult.Value(), rule.DataType, err.Error())
|
||||
}
|
||||
if reflect.TypeOf(formatVal).Kind() == reflect.Map {
|
||||
// 获取的数据是map类型, 处理数据覆盖
|
||||
// eg : 配置如下两个规则 process.id(string) 、process(map[string]any)
|
||||
// 若输入数据的process.id为int类型, 则格式化后的process.id必为 string, 应为 process.id 规则的控制更精细
|
||||
gjsonVal := gjson.Get(f.formatResult, rule.TargetPath)
|
||||
if gjsonVal.Exists() && gjsonVal.IsObject() {
|
||||
var (
|
||||
existRes = map[string]any{}
|
||||
formatRes = map[string]any{}
|
||||
)
|
||||
// 已存在, 且是对象
|
||||
_ = serialize.JSON.UnmarshalWithNumber([]byte(gjsonVal.String()), &existRes)
|
||||
if err = serialize.JSON.Transition(formatVal, &formatRes); nil != err {
|
||||
return errors.New("conflict data path config deal fail : " + err.Error())
|
||||
}
|
||||
for k, v := range existRes {
|
||||
formatRes[k] = v
|
||||
}
|
||||
formatVal = formatRes // 重新赋值 formatVal
|
||||
}
|
||||
}
|
||||
if f.formatResult, err = sjson.Set(f.formatResult, rule.TargetPath, formatVal); nil != err {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user