zhangdeman/json_filter
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

完成数据过主干逻辑,细节健壮性有待继续完善 #4

Merged
zhangdeman merged 13 commits from feature/data_filter into master 2023-09-02 22:17:14 +08:00
Conversation 0 Commits 13 Files Changed 5 +75 -421
3 changed files with 75 additions and 421 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 48f3d80275 - Show all commits

275
tool/gabs.go
View File

@ -8,17 +8,10 @@
package json_tool
import (
"encoding/json"
"fmt"
"git.zhangdeman.cn/zhangdeman/wrapper"
"reflect"
"strings"
"github.com/pkg/errors"
"github.com/tidwall/gjson"
"github.com/Jeffail/gabs"
"github.com/tidwall/sjson"
"strings"
)
const (
@ -43,23 +36,10 @@ type FilterDataRule struct {
//
// Date : 2022/1/22 9:50 PM
func NewDataFilter(source string, filterRule []*FilterDataRule) *DataFilter {
jsonTree := gjson.Parse(source)
isVirtual := false
if jsonTree.IsArray() {
source = fmt.Sprintf(`{"%v":%v}`, virtualRoot, source)
isVirtual = true
}
if isVirtual {
for _, item := range filterRule {
item.MapKey = virtualRoot + "." + item.MapKey
item.SourceKey = virtualRoot + "." + item.SourceKey
}
}
return &DataFilter{
source: source,
filterRule: filterRule,
hasDealDiffPath: make(map[string]string),
source: source,
filterRule: filterRule,
rewriteResult: "{}",
}
}
@ -69,11 +49,9 @@ func NewDataFilter(source string, filterRule []*FilterDataRule) *DataFilter {
//
// Date : 2022/1/22 9:20 PM
type DataFilter struct {
source string
filterRule []*FilterDataRule
itemKeyToSlice bool
hasDealDiffPath map[string]string
isVirtualRoot bool
source string
filterRule []*FilterDataRule
rewriteResult string // json数据重写结果
}
// Filter 数据过滤
@ -83,234 +61,49 @@ type DataFilter struct {
// Date : 2022/1/22 9:36 PM
func (df *DataFilter) Filter() (string, error) {
var (
jsonObject *gabs.Container
err error
err error
)
// 格式化映射规则
df.formatRule()
// 记录 obj => slice 的数据类型
obg2slice := make(map[string]string)
// 创建数据的根结点
jsonObject = gabs.New()
for _, item := range df.filterRule {
// 数据源路径不识数组, 多个key写入到同一个map key, 并且map key 不是以[]结尾, 自动格式化
// 目标位置, 是一个数组
if df.pathIsArrayValue(item.MapKey) {
realMapKey := strings.TrimRight(item.MapKey, ".[]")
if exist := jsonObject.Exists(realMapKey); !exist {
if _, err = jsonObject.ArrayP(realMapKey); nil != err {
return "", err
}
}
valueResult := gjson.Get(df.source, item.SourceKey)
dataType := df.getValueType(valueResult)
if _, exist := obg2slice[realMapKey]; !exist {
obg2slice[realMapKey] = dataType
}
if dataType != obg2slice[realMapKey] {
return "", errors.New(realMapKey + " 预期写入的字段数据类型不一致")
}
sourcePathArr := strings.Split(item.SourceKey, ".[].")
mapPathArr := strings.Split(realMapKey, ".[].")
result := gabs.New()
_, _ = result.ArrayP(mapPathArr[0])
df.SetArrayData("{\""+sourcePathArr[0]+"\":"+gjson.Get(df.source, sourcePathArr[0]).String()+"}", result, sourcePathArr, mapPathArr)
if err = jsonObject.ArrayAppend(valueResult.Value(), realMapKey); nil != err {
for _, itemRule := range df.filterRule {
if !df.isArrPath(itemRule.SourceKey) && !df.isArrPath(itemRule.MapKey) {
// 输入输出均不是数组, 最简单的场景
if err = df.setKV(itemRule); nil != err {
return "", err
}
continue
}
sourceSearchResult := gjson.Get(df.source, item.SourceKey)
if !sourceSearchResult.Exists() {
if item.WithDefault {
if _, err = jsonObject.SetP(item.DefaultValue, item.MapKey); nil != err {
return "", err
}
}
continue
}
if _, err = jsonObject.SetP(sourceSearchResult.Value(), item.MapKey); nil != err {
return "", err
}
}
final := jsonObject.String()
if df.isVirtualRoot {
return gjson.Get(final, virtualRoot).String(), nil
}
return final, nil
return df.rewriteResult, nil
}
// UserItemToSlice 支持多个独立的字段合并到slice中
// isArrPath 是否为数组路径
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 3:27 PM 2022/1/24
func (df *DataFilter) UserItemToSlice() {
df.itemKeyToSlice = true
}
// getValueType 获取数据类型
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2022/1/23 12:45 AM
func (df *DataFilter) getValueType(valueResult gjson.Result) string {
dataTypeVal := reflect.TypeOf(valueResult.Value())
if nil == dataTypeVal {
return "NIL"
}
dataType := dataTypeVal.String()
if strings.Contains(dataType, "int") {
return "int64"
}
if strings.Contains(dataType, "float") {
return "float64"
}
return dataType
}
// pathIsArrayValue 判断路径是否为数组值
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2022/1/23 12:56 AM
func (df *DataFilter) pathIsArrayValue(path string) bool {
// Date : 16:00 2023/9/1
func (df *DataFilter) isArrPath(path string) bool {
return strings.Contains(path, "[]")
}
// formatRule 格式化映射规则
// setKV 设置相关值
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2:43 PM 2022/1/24
func (df *DataFilter) formatRule() {
mapKeyCnt := make(map[string]int)
for _, item := range df.filterRule {
// source 为数组, map 不是
if df.pathIsArrayValue(item.SourceKey) {
if !df.pathIsArrayValue(item.MapKey) {
item.MapKey = item.MapKey + ".[]"
} else {
// source 是数组, map也是数组, 检测数组层级匹配
sourcePathArr := strings.Split(item.SourceKey, ".[].")
mapPathArr := strings.Split(item.MapKey, ".[].")
if len(sourcePathArr) == len(mapPathArr) {
// 数组层级深度相同,无需特殊处理
continue
}
// 数组层级深度不同,重新对对齐数据
diffArr := sourcePathArr[0 : len(sourcePathArr)-len(mapPathArr)+1]
if newPath := df.dealDiffArr(diffArr); len(newPath) > 0 {
sourcePathArr[len(sourcePathArr)-len(mapPathArr)] = newPath
item.SourceKey = strings.Join(sourcePathArr[len(sourcePathArr)-len(mapPathArr):], ".[].")
}
}
} else {
if df.pathIsArrayValue(item.MapKey) {
continue
}
// source 不是数组, map 也不是
if !df.itemKeyToSlice {
continue
}
mapKeyCnt[item.MapKey]++
}
// Date : 16:03 2023/9/1
func (df *DataFilter) setKV(rule *FilterDataRule) error {
var (
err error
)
sourceValue := gjson.Get(df.source, rule.SourceKey)
if sourceValue.Exists() {
// 原始数据存在对应路径
df.rewriteResult, err = sjson.Set(df.rewriteResult, rule.MapKey, sourceValue.Value())
return err
}
// 多个source指向一个map,自动转化为list
for _, item := range df.filterRule {
if mapKeyCnt[item.MapKey] > 1 {
item.MapKey = item.MapKey + ".[]"
}
if !rule.WithDefault {
// 路径不存在, 且禁用默认值
return errors.New(rule.SourceKey + " : source path not found, and default value is forbidden")
}
}
// dealDiffArr 提取数据映射关系
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 5:04 下午 2022/1/25
func (df *DataFilter) dealDiffArr(diffArr []string) string {
if len(diffArr) == 0 {
return ""
}
diffArrStr := strings.Join(diffArr, ".[].")
if _, exist := df.hasDealDiffPath[diffArrStr]; exist {
// 已经处理过, 不再重复处理
return df.hasDealDiffPath[diffArrStr]
}
// 没处理过, 开始处理
jsonResultList := df.getArrayData(df.source, diffArr)
if len(jsonResultList) == 0 {
return ""
}
newPath := wrapper.StringFromRandom(8, "").Value()
var result map[string]interface{}
_ = json.Unmarshal([]byte(df.source), &result)
JSONObject, _ := gabs.Consume(result)
_, _ = JSONObject.ArrayP(newPath)
for _, item := range jsonResultList {
if err := JSONObject.ArrayAppendP(item.Value(), newPath); nil != err {
fmt.Println(err.Error())
}
}
df.source = JSONObject.String()
df.hasDealDiffPath[diffArrStr] = newPath
return newPath
}
// getArrayData 获取数据
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2:22 下午 2022/1/26
func (df *DataFilter) getArrayData(source string, pathArr []string) []gjson.Result {
if len(pathArr) == 1 {
return gjson.Get(source, pathArr[0]).Array()
}
resultList := make([]gjson.Result, 0)
dataList := gjson.Get(source, pathArr[0]).Array()
for idx := 0; idx < len(dataList); idx++ {
resultList = append(resultList, df.getArrayData(dataList[idx].String(), pathArr[1:])...)
}
return resultList
}
// SetArrayData 设置数组数据
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 5:05 下午 2022/2/2
func (df *DataFilter) SetArrayData(sourceData string, jsonObject *gabs.Container, sourcePathArr []string, mapPathArr []string) *gabs.Container {
jsonObject = gabs.New()
for idx, sourcePath := range sourcePathArr {
if idx < len(sourcePathArr)-1 {
if !jsonObject.Exists(sourcePath) {
_, _ = jsonObject.ArrayP(sourcePath)
}
}
instance, _ := gabs.ParseJSON([]byte(sourceData))
if !instance.Exists() {
fmt.Println(sourcePathArr[len(sourcePathArr)-1] + " 不存在")
} else {
dataList, _ := instance.Children()
for _, item := range dataList {
cItem := gabs.New()
cItem.SetP(gjson.Get(item.String(), sourcePath).String(), mapPathArr[idx])
jsonObject.ArrayAppendP(cItem.Data(), mapPathArr[idx])
}
//jsonObject.ArrayAppend(jsonObject.Data())
// fmt.Println("数据 : ", jsonObject.String())
// jsonObject.ArrayAppendP(result.Data(), mapPathArr[idx])
}
df.SetArrayData(gjson.Get(sourceData, sourcePathArr[idx]).String(), jsonObject, sourcePathArr[idx+1:], mapPathArr[idx+1:])
// jsonObject.ArrayAppendP(v.Data(), mapPathArr[idx])
}
fmt.Println("最终 : ", jsonObject.String())
return jsonObject
// 使用默认值填充
df.rewriteResult, err = sjson.Set(df.rewriteResult, rule.MapKey, rule.DefaultValue)
return err
}

41
tool/gabs_test.go Normal file
View File

@ -0,0 +1,41 @@
// Package json_tool ...
//
// Description : json_tool ...
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2023-09-01 16:07
package json_tool
import (
"fmt"
"git.zhangdeman.cn/zhangdeman/serialize"
"testing"
)
// TestDataFilter_FilterNormalData 最基础对象
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 16:13 2023/9/1
func TestDataFilter_FilterNormalData(t *testing.T) {
source := map[string]interface{}{
"name": "zhangdeman",
"extra": map[string]interface{}{
"age": 18,
"height": 180,
"slice": []int{1, 2, 3},
},
"slice": []int{1, 2, 3},
}
df := &DataFilter{
source: serialize.JSON.MarshalForString(source),
filterRule: []*FilterDataRule{
{SourceKey: "name", MapKey: "user_name", DefaultValue: "油猴", WithDefault: true},
{SourceKey: "extra.age", MapKey: "user_age", DefaultValue: "18", WithDefault: true},
{SourceKey: "slice", MapKey: "user_index", DefaultValue: "[4,5,6]", WithDefault: true},
{SourceKey: "none", MapKey: "none_default", DefaultValue: map[string]interface{}{"a": "a"}, WithDefault: true},
},
}
fmt.Println(df.Filter())
}

180
tool/json_test.go
View File

@ -42,15 +42,6 @@ func TestJSON(t *testing.T) {
fmt.Println(tree.String())
}
// TestType 判断数据类型断言
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 10:59 下午 2021/3/14
func TestType(t *testing.T) {
}
// TestSelect 测试动态选择字段
//
// Author : go_developer@163.com<白茶清欢>
@ -127,174 +118,3 @@ func TestParse(t *testing.T) {
byteData, _ := json.Marshal(source)
fmt.Println(GetJSONDataStruct(string(byteData)))
}
// TestParseWithType 测试获取JSON数据结构
//
// Author : go_developer@163.com<张德满>
//
// Date : 10:59 PM 2022/1/9
func TestParseWithType(t *testing.T) {
source := map[string]interface{}{
"name": "zhangdeman",
"extra": map[string]interface{}{
"age": 18,
"height": 180,
"slice": []int{1, 2, 3},
"obj": map[string]interface{}{
"la": "aaaa",
},
},
"slice": []int{1, 2, 3},
"map": map[string]interface{}{"a": 1, "d": 5.5, "e": "qqq"},
"empty_obj": map[string]interface{}{},
"empty_list": make([]interface{}, 0),
"table": []map[string]interface{}{
{"name": "alex", "age": 18, "number": 1, "obj": map[string]interface{}{"enen": "en"}},
{"name": "bob", "age": 28, "number": 2},
},
"two_slice": []map[string]interface{}{
{
"students": []map[string]interface{}{
{
"name": "enen",
"age": 18,
"score": []float64{1, 2, 3, 45},
},
},
"other": []interface{}{"others"},
"read_only": 1,
},
},
}
byteData, _ := json.Marshal(source)
fmt.Println(GetJSONDataStructWithType(string(byteData)))
}
// TestDataFilter 测试数据过滤
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2022/1/22 10:19 PM
func TestDataFilter(t *testing.T) {
source := map[string]interface{}{
"name": "zhangdeman",
"extra": map[string]interface{}{
"age": 18,
"height": 180,
"slice": []int{1, 2, 3},
},
"slice_data": []int{1, 2, 3},
"map": map[string]interface{}{"a": 1, "b": 2, "c": 4},
"table": []map[string]interface{}{
{"name": "alex", "age": 18, "number": 1},
{"name": "bob", "age": 28, "number": 2},
{"name": "bob", "age": 28, "number": 2, "list": []int{1, 2, 3}},
},
}
rule := []*FilterDataRule{
{SourceKey: "name", MapKey: "user_name", DefaultValue: "用户姓名默认值"},
{SourceKey: "name", MapKey: "username", DefaultValue: "用户姓名默认值"},
{SourceKey: "name", MapKey: "user.name", DefaultValue: "用户姓名默认值"},
{SourceKey: "extra.age", MapKey: "user.age", DefaultValue: "用户年龄默认值"},
{SourceKey: "extra.age", MapKey: "user_age", DefaultValue: "用户年龄默认值"},
{SourceKey: "extra.height", MapKey: "user.height", DefaultValue: "扩展高度默认值"},
{SourceKey: "extra.height", MapKey: "user_height", DefaultValue: "扩展高度默认值"},
{SourceKey: "table.[].name", MapKey: "slice.[].name_modify", DefaultValue: "列表姓名默认值"},
{SourceKey: "table.[].list", MapKey: "slice.[].data_list", DefaultValue: "[\"567\",\"678\",\"789\"]"},
}
byteData, _ := json.Marshal(source)
filter := NewDataFilter(string(byteData), rule)
fmt.Println(filter.Filter())
}
// TestDataFilterForObiToSlice ...
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2022/1/23 12:06 AM
func TestDataFilterForObiToSlice(t *testing.T) {
source := map[string]interface{}{
"name": "zhangdeman",
"age": 18,
"height": 180,
"extra": map[string]interface{}{
"age": 18,
"height": 180,
"slice": []int{1, 2, 3},
},
"slice_data": []int{1, 2, 3},
"map": map[string]interface{}{"a": 1, "b": 2, "c": 4},
"table": []map[string]interface{}{
{"name": "alex", "age": 18, "number": 1},
{"name": "bob", "age": 28, "number": 2},
{"name": "bob", "age": 28, "number": 2, "list": []int{1, 2, 3}},
},
}
rule := []*FilterDataRule{
// {SourceKey: "name", MapKey: "slice.[]", DefaultValue: "用户姓名默认值"},
{SourceKey: "age", MapKey: "slice", DefaultValue: "用户姓名默认值"},
{SourceKey: "height", MapKey: "slice", DefaultValue: "用户姓名默认值"},
}
byteData, _ := json.Marshal(source)
filter := NewDataFilter(string(byteData), rule)
filter.UserItemToSlice()
fmt.Println(filter.Filter())
}
// TestDataFilterDiffArr ...
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 12:27 下午 2022/1/26
func TestDataFilterDiffArr(t *testing.T) {
source := map[string]interface{}{
"name": "zhangdeman",
"age": 18,
"height": 180,
"extra": map[string]interface{}{
"age": 18,
"height": 180,
"slice": []int{1, 2, 3},
},
"slice_data": []int{1, 2, 3},
"map": map[string]interface{}{"a": 1, "b": 2, "c": 4},
"table": []map[string]interface{}{
{"user_list": []interface{}{map[string]interface{}{"name": "alex", "age": 18, "number": 1}}},
{"user_list": []interface{}{map[string]interface{}{"name": "bob", "age": 28, "number": 2}}},
{"user_list": []interface{}{map[string]interface{}{"name": "andy", "age": 28, "number": 2}}},
},
}
rule := []*FilterDataRule{
// {SourceKey: "name", MapKey: "slice.[]", DefaultValue: "用户姓名默认值"},
{SourceKey: "table.[].user_list.[].name", MapKey: "user_list.[].detail.name", DefaultValue: "用户姓名默认值"},
{SourceKey: "table.[].user_list.[].age", MapKey: "user_list.[].detail.age", DefaultValue: "用户姓名默认值"},
}
byteData, _ := json.Marshal(source)
filter := NewDataFilter(string(byteData), rule)
filter.UserItemToSlice()
filter.Filter()
//fmt.Println()
}
// TestDataFilterRootArr ...
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 20:45 2023/8/31
func TestDataFilterRootArr(t *testing.T) {
source := []map[string]interface{}{
{"user_list": []interface{}{map[string]interface{}{"name": "alex", "age": 18, "number": 1}}},
{"user_list": []interface{}{map[string]interface{}{"name": "bob", "age": 28, "number": 2}}},
{"user_list": []interface{}{map[string]interface{}{"name": "andy", "age": 28, "number": 2}}},
}
rule := []*FilterDataRule{
// {SourceKey: "name", MapKey: "slice.[]", DefaultValue: "用户姓名默认值"},
{SourceKey: "[].user_list.[].name", MapKey: "user_list.[].detail.name", DefaultValue: "用户姓名默认值"},
{SourceKey: "[].user_list.[].age", MapKey: "user_list.[].detail.age", DefaultValue: "用户姓名默认值"},
}
byteData, _ := json.Marshal(source)
filter := NewDataFilter(string(byteData), rule)
filter.UserItemToSlice()
filter.Filter()
//fmt.Println()
}