diff --git a/tool/gabs.go b/tool/gabs.go index fae0428..b70b261 100644 --- a/tool/gabs.go +++ b/tool/gabs.go @@ -8,17 +8,10 @@ package json_tool import ( - "encoding/json" - "fmt" - "git.zhangdeman.cn/zhangdeman/wrapper" - "reflect" - "strings" - "github.com/pkg/errors" - "github.com/tidwall/gjson" - - "github.com/Jeffail/gabs" + "github.com/tidwall/sjson" + "strings" ) const ( @@ -43,23 +36,10 @@ type FilterDataRule struct { // // Date : 2022/1/22 9:50 PM func NewDataFilter(source string, filterRule []*FilterDataRule) *DataFilter { - - jsonTree := gjson.Parse(source) - isVirtual := false - if jsonTree.IsArray() { - source = fmt.Sprintf(`{"%v":%v}`, virtualRoot, source) - isVirtual = true - } - if isVirtual { - for _, item := range filterRule { - item.MapKey = virtualRoot + "." + item.MapKey - item.SourceKey = virtualRoot + "." + item.SourceKey - } - } return &DataFilter{ - source: source, - filterRule: filterRule, - hasDealDiffPath: make(map[string]string), + source: source, + filterRule: filterRule, + rewriteResult: "{}", } } @@ -69,11 +49,9 @@ func NewDataFilter(source string, filterRule []*FilterDataRule) *DataFilter { // // Date : 2022/1/22 9:20 PM type DataFilter struct { - source string - filterRule []*FilterDataRule - itemKeyToSlice bool - hasDealDiffPath map[string]string - isVirtualRoot bool + source string + filterRule []*FilterDataRule + rewriteResult string // json数据重写结果 } // Filter 数据过滤 @@ -83,234 +61,49 @@ type DataFilter struct { // Date : 2022/1/22 9:36 PM func (df *DataFilter) Filter() (string, error) { var ( - jsonObject *gabs.Container - err error + err error ) - // 格式化映射规则 - df.formatRule() - // 记录 obj => slice 的数据类型 - obg2slice := make(map[string]string) - // 创建数据的根结点 - jsonObject = gabs.New() - for _, item := range df.filterRule { - // 数据源路径不识数组, 多个key写入到同一个map key, 并且map key 不是以[]结尾, 自动格式化 - // 目标位置, 是一个数组 - if df.pathIsArrayValue(item.MapKey) { - realMapKey := strings.TrimRight(item.MapKey, ".[]") - if exist := jsonObject.Exists(realMapKey); !exist { - if _, err = jsonObject.ArrayP(realMapKey); nil != err { - return "", err - } - } - valueResult := gjson.Get(df.source, item.SourceKey) - dataType := df.getValueType(valueResult) - if _, exist := obg2slice[realMapKey]; !exist { - obg2slice[realMapKey] = dataType - } - if dataType != obg2slice[realMapKey] { - return "", errors.New(realMapKey + " 预期写入的字段数据类型不一致") - } - sourcePathArr := strings.Split(item.SourceKey, ".[].") - mapPathArr := strings.Split(realMapKey, ".[].") - result := gabs.New() - _, _ = result.ArrayP(mapPathArr[0]) - df.SetArrayData("{\""+sourcePathArr[0]+"\":"+gjson.Get(df.source, sourcePathArr[0]).String()+"}", result, sourcePathArr, mapPathArr) - if err = jsonObject.ArrayAppend(valueResult.Value(), realMapKey); nil != err { + for _, itemRule := range df.filterRule { + if !df.isArrPath(itemRule.SourceKey) && !df.isArrPath(itemRule.MapKey) { + // 输入输出均不是数组, 最简单的场景 + if err = df.setKV(itemRule); nil != err { return "", err } - continue - } - sourceSearchResult := gjson.Get(df.source, item.SourceKey) - if !sourceSearchResult.Exists() { - if item.WithDefault { - if _, err = jsonObject.SetP(item.DefaultValue, item.MapKey); nil != err { - return "", err - } - } - continue - } - if _, err = jsonObject.SetP(sourceSearchResult.Value(), item.MapKey); nil != err { - return "", err } } - final := jsonObject.String() - if df.isVirtualRoot { - return gjson.Get(final, virtualRoot).String(), nil - } - return final, nil + return df.rewriteResult, nil } -// UserItemToSlice 支持多个独立的字段合并到slice中 +// isArrPath 是否为数组路径 // // Author : go_developer@163.com<白茶清欢> // -// Date : 3:27 PM 2022/1/24 -func (df *DataFilter) UserItemToSlice() { - df.itemKeyToSlice = true -} - -// getValueType 获取数据类型 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 2022/1/23 12:45 AM -func (df *DataFilter) getValueType(valueResult gjson.Result) string { - dataTypeVal := reflect.TypeOf(valueResult.Value()) - if nil == dataTypeVal { - return "NIL" - } - dataType := dataTypeVal.String() - if strings.Contains(dataType, "int") { - return "int64" - } - if strings.Contains(dataType, "float") { - return "float64" - } - return dataType -} - -// pathIsArrayValue 判断路径是否为数组值 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 2022/1/23 12:56 AM -func (df *DataFilter) pathIsArrayValue(path string) bool { +// Date : 16:00 2023/9/1 +func (df *DataFilter) isArrPath(path string) bool { return strings.Contains(path, "[]") } -// formatRule 格式化映射规则 +// setKV 设置相关值 // // Author : go_developer@163.com<白茶清欢> // -// Date : 2:43 PM 2022/1/24 -func (df *DataFilter) formatRule() { - mapKeyCnt := make(map[string]int) - for _, item := range df.filterRule { - // source 为数组, map 不是 - if df.pathIsArrayValue(item.SourceKey) { - if !df.pathIsArrayValue(item.MapKey) { - item.MapKey = item.MapKey + ".[]" - } else { - // source 是数组, map也是数组, 检测数组层级匹配 - sourcePathArr := strings.Split(item.SourceKey, ".[].") - mapPathArr := strings.Split(item.MapKey, ".[].") - if len(sourcePathArr) == len(mapPathArr) { - // 数组层级深度相同,无需特殊处理 - continue - } - // 数组层级深度不同,重新对对齐数据 - diffArr := sourcePathArr[0 : len(sourcePathArr)-len(mapPathArr)+1] - if newPath := df.dealDiffArr(diffArr); len(newPath) > 0 { - sourcePathArr[len(sourcePathArr)-len(mapPathArr)] = newPath - item.SourceKey = strings.Join(sourcePathArr[len(sourcePathArr)-len(mapPathArr):], ".[].") - } - } - } else { - if df.pathIsArrayValue(item.MapKey) { - continue - } - // source 不是数组, map 也不是 - if !df.itemKeyToSlice { - continue - } - mapKeyCnt[item.MapKey]++ - } +// Date : 16:03 2023/9/1 +func (df *DataFilter) setKV(rule *FilterDataRule) error { + var ( + err error + ) + sourceValue := gjson.Get(df.source, rule.SourceKey) + if sourceValue.Exists() { + // 原始数据存在对应路径 + df.rewriteResult, err = sjson.Set(df.rewriteResult, rule.MapKey, sourceValue.Value()) + return err } - // 多个source指向一个map,自动转化为list - for _, item := range df.filterRule { - if mapKeyCnt[item.MapKey] > 1 { - item.MapKey = item.MapKey + ".[]" - } + if !rule.WithDefault { + // 路径不存在, 且禁用默认值 + return errors.New(rule.SourceKey + " : source path not found, and default value is forbidden") } -} - -// dealDiffArr 提取数据映射关系 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 5:04 下午 2022/1/25 -func (df *DataFilter) dealDiffArr(diffArr []string) string { - if len(diffArr) == 0 { - return "" - } - diffArrStr := strings.Join(diffArr, ".[].") - if _, exist := df.hasDealDiffPath[diffArrStr]; exist { - // 已经处理过, 不再重复处理 - return df.hasDealDiffPath[diffArrStr] - } - - // 没处理过, 开始处理 - jsonResultList := df.getArrayData(df.source, diffArr) - if len(jsonResultList) == 0 { - return "" - } - newPath := wrapper.StringFromRandom(8, "").Value() - var result map[string]interface{} - _ = json.Unmarshal([]byte(df.source), &result) - JSONObject, _ := gabs.Consume(result) - _, _ = JSONObject.ArrayP(newPath) - for _, item := range jsonResultList { - if err := JSONObject.ArrayAppendP(item.Value(), newPath); nil != err { - fmt.Println(err.Error()) - } - } - - df.source = JSONObject.String() - df.hasDealDiffPath[diffArrStr] = newPath - return newPath -} - -// getArrayData 获取数据 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 2:22 下午 2022/1/26 -func (df *DataFilter) getArrayData(source string, pathArr []string) []gjson.Result { - if len(pathArr) == 1 { - return gjson.Get(source, pathArr[0]).Array() - } - resultList := make([]gjson.Result, 0) - dataList := gjson.Get(source, pathArr[0]).Array() - for idx := 0; idx < len(dataList); idx++ { - resultList = append(resultList, df.getArrayData(dataList[idx].String(), pathArr[1:])...) - } - return resultList -} - -// SetArrayData 设置数组数据 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 5:05 下午 2022/2/2 -func (df *DataFilter) SetArrayData(sourceData string, jsonObject *gabs.Container, sourcePathArr []string, mapPathArr []string) *gabs.Container { - jsonObject = gabs.New() - - for idx, sourcePath := range sourcePathArr { - if idx < len(sourcePathArr)-1 { - if !jsonObject.Exists(sourcePath) { - _, _ = jsonObject.ArrayP(sourcePath) - } - } - instance, _ := gabs.ParseJSON([]byte(sourceData)) - if !instance.Exists() { - fmt.Println(sourcePathArr[len(sourcePathArr)-1] + " 不存在") - } else { - dataList, _ := instance.Children() - for _, item := range dataList { - cItem := gabs.New() - cItem.SetP(gjson.Get(item.String(), sourcePath).String(), mapPathArr[idx]) - jsonObject.ArrayAppendP(cItem.Data(), mapPathArr[idx]) - } - //jsonObject.ArrayAppend(jsonObject.Data()) - // fmt.Println("数据 : ", jsonObject.String()) - // jsonObject.ArrayAppendP(result.Data(), mapPathArr[idx]) - - } - - df.SetArrayData(gjson.Get(sourceData, sourcePathArr[idx]).String(), jsonObject, sourcePathArr[idx+1:], mapPathArr[idx+1:]) - // jsonObject.ArrayAppendP(v.Data(), mapPathArr[idx]) - } - fmt.Println("最终 : ", jsonObject.String()) - return jsonObject + // 使用默认值填充 + df.rewriteResult, err = sjson.Set(df.rewriteResult, rule.MapKey, rule.DefaultValue) + return err } diff --git a/tool/gabs_test.go b/tool/gabs_test.go new file mode 100644 index 0000000..b297837 --- /dev/null +++ b/tool/gabs_test.go @@ -0,0 +1,41 @@ +// Package json_tool ... +// +// Description : json_tool ... +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 2023-09-01 16:07 +package json_tool + +import ( + "fmt" + "git.zhangdeman.cn/zhangdeman/serialize" + "testing" +) + +// TestDataFilter_FilterNormalData 最基础对象 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 16:13 2023/9/1 +func TestDataFilter_FilterNormalData(t *testing.T) { + source := map[string]interface{}{ + "name": "zhangdeman", + "extra": map[string]interface{}{ + "age": 18, + "height": 180, + "slice": []int{1, 2, 3}, + }, + "slice": []int{1, 2, 3}, + } + df := &DataFilter{ + source: serialize.JSON.MarshalForString(source), + filterRule: []*FilterDataRule{ + {SourceKey: "name", MapKey: "user_name", DefaultValue: "油猴", WithDefault: true}, + {SourceKey: "extra.age", MapKey: "user_age", DefaultValue: "18", WithDefault: true}, + {SourceKey: "slice", MapKey: "user_index", DefaultValue: "[4,5,6]", WithDefault: true}, + {SourceKey: "none", MapKey: "none_default", DefaultValue: map[string]interface{}{"a": "a"}, WithDefault: true}, + }, + } + fmt.Println(df.Filter()) +} diff --git a/tool/json_test.go b/tool/json_test.go index 3ae4f41..5fc6af1 100644 --- a/tool/json_test.go +++ b/tool/json_test.go @@ -42,15 +42,6 @@ func TestJSON(t *testing.T) { fmt.Println(tree.String()) } -// TestType 判断数据类型断言 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 10:59 下午 2021/3/14 -func TestType(t *testing.T) { - -} - // TestSelect 测试动态选择字段 // // Author : go_developer@163.com<白茶清欢> @@ -127,174 +118,3 @@ func TestParse(t *testing.T) { byteData, _ := json.Marshal(source) fmt.Println(GetJSONDataStruct(string(byteData))) } - -// TestParseWithType 测试获取JSON数据结构 -// -// Author : go_developer@163.com<张德满> -// -// Date : 10:59 PM 2022/1/9 -func TestParseWithType(t *testing.T) { - source := map[string]interface{}{ - "name": "zhangdeman", - "extra": map[string]interface{}{ - "age": 18, - "height": 180, - "slice": []int{1, 2, 3}, - "obj": map[string]interface{}{ - "la": "aaaa", - }, - }, - "slice": []int{1, 2, 3}, - "map": map[string]interface{}{"a": 1, "d": 5.5, "e": "qqq"}, - "empty_obj": map[string]interface{}{}, - "empty_list": make([]interface{}, 0), - "table": []map[string]interface{}{ - {"name": "alex", "age": 18, "number": 1, "obj": map[string]interface{}{"enen": "en"}}, - {"name": "bob", "age": 28, "number": 2}, - }, - "two_slice": []map[string]interface{}{ - { - "students": []map[string]interface{}{ - { - "name": "enen", - "age": 18, - "score": []float64{1, 2, 3, 45}, - }, - }, - "other": []interface{}{"others"}, - "read_only": 1, - }, - }, - } - byteData, _ := json.Marshal(source) - fmt.Println(GetJSONDataStructWithType(string(byteData))) -} - -// TestDataFilter 测试数据过滤 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 2022/1/22 10:19 PM -func TestDataFilter(t *testing.T) { - source := map[string]interface{}{ - "name": "zhangdeman", - "extra": map[string]interface{}{ - "age": 18, - "height": 180, - "slice": []int{1, 2, 3}, - }, - "slice_data": []int{1, 2, 3}, - "map": map[string]interface{}{"a": 1, "b": 2, "c": 4}, - "table": []map[string]interface{}{ - {"name": "alex", "age": 18, "number": 1}, - {"name": "bob", "age": 28, "number": 2}, - {"name": "bob", "age": 28, "number": 2, "list": []int{1, 2, 3}}, - }, - } - rule := []*FilterDataRule{ - {SourceKey: "name", MapKey: "user_name", DefaultValue: "用户姓名默认值"}, - {SourceKey: "name", MapKey: "username", DefaultValue: "用户姓名默认值"}, - {SourceKey: "name", MapKey: "user.name", DefaultValue: "用户姓名默认值"}, - {SourceKey: "extra.age", MapKey: "user.age", DefaultValue: "用户年龄默认值"}, - {SourceKey: "extra.age", MapKey: "user_age", DefaultValue: "用户年龄默认值"}, - {SourceKey: "extra.height", MapKey: "user.height", DefaultValue: "扩展高度默认值"}, - {SourceKey: "extra.height", MapKey: "user_height", DefaultValue: "扩展高度默认值"}, - {SourceKey: "table.[].name", MapKey: "slice.[].name_modify", DefaultValue: "列表姓名默认值"}, - {SourceKey: "table.[].list", MapKey: "slice.[].data_list", DefaultValue: "[\"567\",\"678\",\"789\"]"}, - } - byteData, _ := json.Marshal(source) - filter := NewDataFilter(string(byteData), rule) - fmt.Println(filter.Filter()) -} - -// TestDataFilterForObiToSlice ... -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 2022/1/23 12:06 AM -func TestDataFilterForObiToSlice(t *testing.T) { - source := map[string]interface{}{ - "name": "zhangdeman", - "age": 18, - "height": 180, - "extra": map[string]interface{}{ - "age": 18, - "height": 180, - "slice": []int{1, 2, 3}, - }, - "slice_data": []int{1, 2, 3}, - "map": map[string]interface{}{"a": 1, "b": 2, "c": 4}, - "table": []map[string]interface{}{ - {"name": "alex", "age": 18, "number": 1}, - {"name": "bob", "age": 28, "number": 2}, - {"name": "bob", "age": 28, "number": 2, "list": []int{1, 2, 3}}, - }, - } - rule := []*FilterDataRule{ - // {SourceKey: "name", MapKey: "slice.[]", DefaultValue: "用户姓名默认值"}, - {SourceKey: "age", MapKey: "slice", DefaultValue: "用户姓名默认值"}, - {SourceKey: "height", MapKey: "slice", DefaultValue: "用户姓名默认值"}, - } - byteData, _ := json.Marshal(source) - filter := NewDataFilter(string(byteData), rule) - filter.UserItemToSlice() - fmt.Println(filter.Filter()) -} - -// TestDataFilterDiffArr ... -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 12:27 下午 2022/1/26 -func TestDataFilterDiffArr(t *testing.T) { - source := map[string]interface{}{ - "name": "zhangdeman", - "age": 18, - "height": 180, - "extra": map[string]interface{}{ - "age": 18, - "height": 180, - "slice": []int{1, 2, 3}, - }, - "slice_data": []int{1, 2, 3}, - "map": map[string]interface{}{"a": 1, "b": 2, "c": 4}, - "table": []map[string]interface{}{ - {"user_list": []interface{}{map[string]interface{}{"name": "alex", "age": 18, "number": 1}}}, - {"user_list": []interface{}{map[string]interface{}{"name": "bob", "age": 28, "number": 2}}}, - {"user_list": []interface{}{map[string]interface{}{"name": "andy", "age": 28, "number": 2}}}, - }, - } - rule := []*FilterDataRule{ - // {SourceKey: "name", MapKey: "slice.[]", DefaultValue: "用户姓名默认值"}, - {SourceKey: "table.[].user_list.[].name", MapKey: "user_list.[].detail.name", DefaultValue: "用户姓名默认值"}, - {SourceKey: "table.[].user_list.[].age", MapKey: "user_list.[].detail.age", DefaultValue: "用户姓名默认值"}, - } - byteData, _ := json.Marshal(source) - filter := NewDataFilter(string(byteData), rule) - filter.UserItemToSlice() - filter.Filter() - //fmt.Println() -} - -// TestDataFilterRootArr ... -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 20:45 2023/8/31 -func TestDataFilterRootArr(t *testing.T) { - source := []map[string]interface{}{ - {"user_list": []interface{}{map[string]interface{}{"name": "alex", "age": 18, "number": 1}}}, - {"user_list": []interface{}{map[string]interface{}{"name": "bob", "age": 28, "number": 2}}}, - {"user_list": []interface{}{map[string]interface{}{"name": "andy", "age": 28, "number": 2}}}, - } - rule := []*FilterDataRule{ - // {SourceKey: "name", MapKey: "slice.[]", DefaultValue: "用户姓名默认值"}, - {SourceKey: "[].user_list.[].name", MapKey: "user_list.[].detail.name", DefaultValue: "用户姓名默认值"}, - {SourceKey: "[].user_list.[].age", MapKey: "user_list.[].detail.age", DefaultValue: "用户姓名默认值"}, - } - byteData, _ := json.Marshal(source) - filter := NewDataFilter(string(byteData), rule) - filter.UserItemToSlice() - filter.Filter() - //fmt.Println() -}