zhangdeman/json_filter
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

普通对象数据过滤

Browse Source
This commit is contained in:
白茶清欢
2023-09-01 16:26:17 +08:00
parent 435b339604
commit 48f3d80275
3 changed files with 75 additions and 421 deletions
Download Patch File Download Diff File Expand all files Collapse all files

275
tool/gabs.go
View File

@ -8,17 +8,10 @@
package json_tool
import (
"encoding/json"
"fmt"
"git.zhangdeman.cn/zhangdeman/wrapper"
"reflect"
"strings"
"github.com/pkg/errors"
"github.com/tidwall/gjson"
"github.com/Jeffail/gabs"
"github.com/tidwall/sjson"
"strings"
)
const (
@ -43,23 +36,10 @@ type FilterDataRule struct {
//
// Date : 2022/1/22 9:50 PM
func NewDataFilter(source string, filterRule []*FilterDataRule) *DataFilter {
jsonTree := gjson.Parse(source)
isVirtual := false
if jsonTree.IsArray() {
source = fmt.Sprintf(`{"%v":%v}`, virtualRoot, source)
isVirtual = true
}
if isVirtual {
for _, item := range filterRule {
item.MapKey = virtualRoot + "." + item.MapKey
item.SourceKey = virtualRoot + "." + item.SourceKey
}
}
return &DataFilter{
source: source,
filterRule: filterRule,
hasDealDiffPath: make(map[string]string),
source: source,
filterRule: filterRule,
rewriteResult: "{}",
}
}
@ -69,11 +49,9 @@ func NewDataFilter(source string, filterRule []*FilterDataRule) *DataFilter {
//
// Date : 2022/1/22 9:20 PM
type DataFilter struct {
source string
filterRule []*FilterDataRule
itemKeyToSlice bool
hasDealDiffPath map[string]string
isVirtualRoot bool
source string
filterRule []*FilterDataRule
rewriteResult string // json数据重写结果
}
// Filter 数据过滤
@ -83,234 +61,49 @@ type DataFilter struct {
// Date : 2022/1/22 9:36 PM
func (df *DataFilter) Filter() (string, error) {
var (
jsonObject *gabs.Container
err error
err error
)
// 格式化映射规则
df.formatRule()
// 记录 obj => slice 的数据类型
obg2slice := make(map[string]string)
// 创建数据的根结点
jsonObject = gabs.New()
for _, item := range df.filterRule {
// 数据源路径不识数组, 多个key写入到同一个map key, 并且map key 不是以[]结尾, 自动格式化
// 目标位置, 是一个数组
if df.pathIsArrayValue(item.MapKey) {
realMapKey := strings.TrimRight(item.MapKey, ".[]")
if exist := jsonObject.Exists(realMapKey); !exist {
if _, err = jsonObject.ArrayP(realMapKey); nil != err {
return "", err
}
}
valueResult := gjson.Get(df.source, item.SourceKey)
dataType := df.getValueType(valueResult)
if _, exist := obg2slice[realMapKey]; !exist {
obg2slice[realMapKey] = dataType
}
if dataType != obg2slice[realMapKey] {
return "", errors.New(realMapKey + " 预期写入的字段数据类型不一致")
}
sourcePathArr := strings.Split(item.SourceKey, ".[].")
mapPathArr := strings.Split(realMapKey, ".[].")
result := gabs.New()
_, _ = result.ArrayP(mapPathArr[0])
df.SetArrayData("{\""+sourcePathArr[0]+"\":"+gjson.Get(df.source, sourcePathArr[0]).String()+"}", result, sourcePathArr, mapPathArr)
if err = jsonObject.ArrayAppend(valueResult.Value(), realMapKey); nil != err {
for _, itemRule := range df.filterRule {
if !df.isArrPath(itemRule.SourceKey) && !df.isArrPath(itemRule.MapKey) {
// 输入输出均不是数组, 最简单的场景
if err = df.setKV(itemRule); nil != err {
return "", err
}
continue
}
sourceSearchResult := gjson.Get(df.source, item.SourceKey)
if !sourceSearchResult.Exists() {
if item.WithDefault {
if _, err = jsonObject.SetP(item.DefaultValue, item.MapKey); nil != err {
return "", err
}
}
continue
}
if _, err = jsonObject.SetP(sourceSearchResult.Value(), item.MapKey); nil != err {
return "", err
}
}
final := jsonObject.String()
if df.isVirtualRoot {
return gjson.Get(final, virtualRoot).String(), nil
}
return final, nil
return df.rewriteResult, nil
}
// UserItemToSlice 支持多个独立的字段合并到slice中
// isArrPath 是否为数组路径
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 3:27 PM 2022/1/24
func (df *DataFilter) UserItemToSlice() {
df.itemKeyToSlice = true
}
// getValueType 获取数据类型
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2022/1/23 12:45 AM
func (df *DataFilter) getValueType(valueResult gjson.Result) string {
dataTypeVal := reflect.TypeOf(valueResult.Value())
if nil == dataTypeVal {
return "NIL"
}
dataType := dataTypeVal.String()
if strings.Contains(dataType, "int") {
return "int64"
}
if strings.Contains(dataType, "float") {
return "float64"
}
return dataType
}
// pathIsArrayValue 判断路径是否为数组值
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2022/1/23 12:56 AM
func (df *DataFilter) pathIsArrayValue(path string) bool {
// Date : 16:00 2023/9/1
func (df *DataFilter) isArrPath(path string) bool {
return strings.Contains(path, "[]")
}
// formatRule 格式化映射规则
// setKV 设置相关值
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2:43 PM 2022/1/24
func (df *DataFilter) formatRule() {
mapKeyCnt := make(map[string]int)
for _, item := range df.filterRule {
// source 为数组, map 不是
if df.pathIsArrayValue(item.SourceKey) {
if !df.pathIsArrayValue(item.MapKey) {
item.MapKey = item.MapKey + ".[]"
} else {
// source 是数组, map也是数组, 检测数组层级匹配
sourcePathArr := strings.Split(item.SourceKey, ".[].")
mapPathArr := strings.Split(item.MapKey, ".[].")
if len(sourcePathArr) == len(mapPathArr) {
// 数组层级深度相同,无需特殊处理
continue
}
// 数组层级深度不同,重新对对齐数据
diffArr := sourcePathArr[0 : len(sourcePathArr)-len(mapPathArr)+1]
if newPath := df.dealDiffArr(diffArr); len(newPath) > 0 {
sourcePathArr[len(sourcePathArr)-len(mapPathArr)] = newPath
item.SourceKey = strings.Join(sourcePathArr[len(sourcePathArr)-len(mapPathArr):], ".[].")
}
}
} else {
if df.pathIsArrayValue(item.MapKey) {
continue
}
// source 不是数组, map 也不是
if !df.itemKeyToSlice {
continue
}
mapKeyCnt[item.MapKey]++
}
// Date : 16:03 2023/9/1
func (df *DataFilter) setKV(rule *FilterDataRule) error {
var (
err error
)
sourceValue := gjson.Get(df.source, rule.SourceKey)
if sourceValue.Exists() {
// 原始数据存在对应路径
df.rewriteResult, err = sjson.Set(df.rewriteResult, rule.MapKey, sourceValue.Value())
return err
}
// 多个source指向一个map,自动转化为list
for _, item := range df.filterRule {
if mapKeyCnt[item.MapKey] > 1 {
item.MapKey = item.MapKey + ".[]"
}
if !rule.WithDefault {
// 路径不存在, 且禁用默认值
return errors.New(rule.SourceKey + " : source path not found, and default value is forbidden")
}
}
// dealDiffArr 提取数据映射关系
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 5:04 下午 2022/1/25
func (df *DataFilter) dealDiffArr(diffArr []string) string {
if len(diffArr) == 0 {
return ""
}
diffArrStr := strings.Join(diffArr, ".[].")
if _, exist := df.hasDealDiffPath[diffArrStr]; exist {
// 已经处理过, 不再重复处理
return df.hasDealDiffPath[diffArrStr]
}
// 没处理过, 开始处理
jsonResultList := df.getArrayData(df.source, diffArr)
if len(jsonResultList) == 0 {
return ""
}
newPath := wrapper.StringFromRandom(8, "").Value()
var result map[string]interface{}
_ = json.Unmarshal([]byte(df.source), &result)
JSONObject, _ := gabs.Consume(result)
_, _ = JSONObject.ArrayP(newPath)
for _, item := range jsonResultList {
if err := JSONObject.ArrayAppendP(item.Value(), newPath); nil != err {
fmt.Println(err.Error())
}
}
df.source = JSONObject.String()
df.hasDealDiffPath[diffArrStr] = newPath
return newPath
}
// getArrayData 获取数据
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2:22 下午 2022/1/26
func (df *DataFilter) getArrayData(source string, pathArr []string) []gjson.Result {
if len(pathArr) == 1 {
return gjson.Get(source, pathArr[0]).Array()
}
resultList := make([]gjson.Result, 0)
dataList := gjson.Get(source, pathArr[0]).Array()
for idx := 0; idx < len(dataList); idx++ {
resultList = append(resultList, df.getArrayData(dataList[idx].String(), pathArr[1:])...)
}
return resultList
}
// SetArrayData 设置数组数据
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 5:05 下午 2022/2/2
func (df *DataFilter) SetArrayData(sourceData string, jsonObject *gabs.Container, sourcePathArr []string, mapPathArr []string) *gabs.Container {
jsonObject = gabs.New()
for idx, sourcePath := range sourcePathArr {
if idx < len(sourcePathArr)-1 {
if !jsonObject.Exists(sourcePath) {
_, _ = jsonObject.ArrayP(sourcePath)
}
}
instance, _ := gabs.ParseJSON([]byte(sourceData))
if !instance.Exists() {
fmt.Println(sourcePathArr[len(sourcePathArr)-1] + " 不存在")
} else {
dataList, _ := instance.Children()
for _, item := range dataList {
cItem := gabs.New()
cItem.SetP(gjson.Get(item.String(), sourcePath).String(), mapPathArr[idx])
jsonObject.ArrayAppendP(cItem.Data(), mapPathArr[idx])
}
//jsonObject.ArrayAppend(jsonObject.Data())
// fmt.Println("数据 : ", jsonObject.String())
// jsonObject.ArrayAppendP(result.Data(), mapPathArr[idx])
}
df.SetArrayData(gjson.Get(sourceData, sourcePathArr[idx]).String(), jsonObject, sourcePathArr[idx+1:], mapPathArr[idx+1:])
// jsonObject.ArrayAppendP(v.Data(), mapPathArr[idx])
}
fmt.Println("最终 : ", jsonObject.String())
return jsonObject
// 使用默认值填充
df.rewriteResult, err = sjson.Set(df.rewriteResult, rule.MapKey, rule.DefaultValue)
return err
}