diff --git a/go.mod b/go.mod index 54b62f4..4deb18f 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,9 @@ go 1.20 require ( git.zhangdeman.cn/zhangdeman/easylock v0.0.0-20230731062340-983985c12eda + git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20230811032817-e6ad534a9a10 git.zhangdeman.cn/zhangdeman/util v0.0.0-20230811070456-d6a489d5860b git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20230811071513-cfc46e8d82e1 - github.com/Jeffail/gabs v1.4.0 github.com/pkg/errors v0.9.1 github.com/smartystreets/goconvey v1.8.1 github.com/tidwall/gjson v1.16.0 @@ -17,10 +17,8 @@ require ( require ( git.zhangdeman.cn/zhangdeman/consts v0.0.0-20230811030300-6f850372c88c // indirect git.zhangdeman.cn/zhangdeman/easymap v0.0.0-20230307094841-e437ba87af10 // indirect - git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20230811032817-e6ad534a9a10 // indirect github.com/BurntSushi/toml v1.3.2 // indirect github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 // indirect - github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1 // indirect github.com/go-ini/ini v1.67.0 // indirect github.com/gopherjs/gopherjs v1.17.2 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect @@ -28,6 +26,7 @@ require ( github.com/mozillazg/go-pinyin v0.20.0 // indirect github.com/smarty/assertions v1.15.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/stretchr/testify v1.8.4 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect ) diff --git a/go.sum b/go.sum index 7aed6b6..f4169f1 100644 --- a/go.sum +++ b/go.sum @@ -6,23 +6,15 @@ git.zhangdeman.cn/zhangdeman/easymap v0.0.0-20230307094841-e437ba87af10 h1:+Lg4v git.zhangdeman.cn/zhangdeman/easymap v0.0.0-20230307094841-e437ba87af10/go.mod h1:+Lc0zYF8sylRi75A7NGmObrLxugwAZa8WVpWh2eh5X0= git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20230811032817-e6ad534a9a10 h1:orhcMAKrcOajsBJCgssnb9O8YcLsPJvWuXF511gs5dc= git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20230811032817-e6ad534a9a10/go.mod h1:CzX5/WwGDTnKmewarnjkK5XcSRbgszTQTdTL3OUc/s4= -git.zhangdeman.cn/zhangdeman/util v0.0.0-20230810085948-024a0e5e963a h1:PED+zfKJdSbewYQNpcjj3uXoGDSA5+8LBGScObfuHAA= -git.zhangdeman.cn/zhangdeman/util v0.0.0-20230810085948-024a0e5e963a/go.mod h1:trYFOShINaQBvinQrH4A0G2kfL22Y2lygEcAiGDt/sc= git.zhangdeman.cn/zhangdeman/util v0.0.0-20230811070456-d6a489d5860b h1:vnmxYrNdX6f5sEVjjkM1fIR+i32kHJ4g9DJqug9KKek= git.zhangdeman.cn/zhangdeman/util v0.0.0-20230811070456-d6a489d5860b/go.mod h1:Yum5+tgP+Wf1GWUAyQz1Qh8Ab9m5+90GYkYdzqVs0lA= -git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20230810085842-1e901000e998 h1:+qefkhBULVZbTGxJO4JqT1C1JQLaccJA6GLKP1qImqE= -git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20230810085842-1e901000e998/go.mod h1:WcNcIIxTSpvjpEhfE64Ktis3q+Fw+Onw7NC2/I1U2Qs= git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20230811071513-cfc46e8d82e1 h1:k2iu9KgRxeroytB+N+/XapAxt1di7o2pNTISjFlYDJ8= git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20230811071513-cfc46e8d82e1/go.mod h1:kvjAbtGTo14gKCS0X4rxnb2sPkskHOUy2NXcx34t6Mw= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/Jeffail/gabs v1.4.0 h1://5fYRRTq1edjfIrQGvdkcd22pkYUrHZ5YC/H2GJVAo= -github.com/Jeffail/gabs v1.4.0/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc= github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 h1:OYA+5W64v3OgClL+IrOD63t4i/RW7RqrAVl9LTZ9UqQ= github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394/go.mod h1:Q8n74mJTIgjX4RBBcHnJ05h//6/k6foqmgE45jTQtxg= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1 h1:CaO/zOnF8VvUfEbhRatPcwKVWamvbYd8tQGRWacE9kU= -github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1/go.mod h1:+hnT3ywWDTAFrW5aE+u2Sa/wT555ZqwoCS+pk3p6ry4= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= @@ -42,7 +34,8 @@ github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sS github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.16.0 h1:SyXa+dsSPpUlcwEDuKuEBJEz5vzTvOea+9rjyYodQFg= github.com/tidwall/gjson v1.16.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -53,8 +46,6 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/tool/gabs.go b/tool/gabs.go index bf0100d..70b4ef4 100644 --- a/tool/gabs.go +++ b/tool/gabs.go @@ -8,19 +8,32 @@ package json_tool import ( - "encoding/json" "fmt" - "git.zhangdeman.cn/zhangdeman/wrapper" - "reflect" + "io" + "log" + "os" "strings" "github.com/pkg/errors" - "github.com/tidwall/gjson" - - "github.com/Jeffail/gabs" + "github.com/tidwall/sjson" ) +const ( + // virtualRoot 虚拟根节点 + virtualRoot = "__VIRTUAL_ROOT__" +) + +// FilterOption 过滤选项 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 17:59 2023/9/1 +type FilterOption struct { + DebugModel bool // 调试模式 + LogInstance io.Writer // 日志实例 +} + // FilterDataRule 参数过滤规则 // // Author : go_developer@163.com<白茶清欢> @@ -38,12 +51,45 @@ type FilterDataRule struct { // Author : go_developer@163.com<白茶清欢> // // Date : 2022/1/22 9:50 PM -func NewDataFilter(source string, filterRule []*FilterDataRule) *DataFilter { - return &DataFilter{ - source: source, - filterRule: filterRule, - hasDealDiffPath: make(map[string]string), +func NewDataFilter(source string, filterRule []*FilterDataRule, filterOption *FilterOption) *DataFilter { + source = fmt.Sprintf(`{"%v":%v}`, virtualRoot, source) + if nil == filterOption { + filterOption = &FilterOption{} } + if filterOption.DebugModel && nil == filterOption.LogInstance { + filterOption.LogInstance = os.Stdout + log.SetOutput(filterOption.LogInstance) + } + df := &DataFilter{ + source: source, + filterRule: make([]*FilterDataRule, 0), + rewriteResult: "{}", + filterOption: filterOption, + } + // 去除末尾的 .[] + for _, item := range filterRule { + item.MapKey = virtualRoot + "." + strings.TrimRight(item.MapKey, ".[]") + item.SourceKey = virtualRoot + "." + strings.TrimRight(item.SourceKey, ".[]") + mapIsArr := df.isArrPath(item.MapKey) + if !mapIsArr { + df.filterRule = append(df.filterRule, item) + continue + } + if len(df.getArrPathList(item.SourceKey)) < len(df.getArrPathList(item.MapKey)) { + panic("map result deep more than source data deep") + } + formatRes := make([]*FilterDataRule, 0) + r, _ := df.unfoldSameDeepArr(item.SourceKey, item.MapKey, "", "") + for _, itemUnfoldResult := range r { + itemMapArr := strings.Split(itemUnfoldResult.MapKey, ".") + if len(itemMapArr) != len(strings.Split(item.MapKey, ".")) { + continue + } + formatRes = append(formatRes, itemUnfoldResult) + } + df.filterRule = append(df.filterRule, formatRes...) + } + return df } // DataFilter 数据过滤 @@ -52,10 +98,10 @@ func NewDataFilter(source string, filterRule []*FilterDataRule) *DataFilter { // // Date : 2022/1/22 9:20 PM type DataFilter struct { - source string - filterRule []*FilterDataRule - itemKeyToSlice bool - hasDealDiffPath map[string]string + source string + filterRule []*FilterDataRule + rewriteResult string // json数据重写结果 + filterOption *FilterOption // 过滤选项 } // Filter 数据过滤 @@ -64,231 +110,224 @@ type DataFilter struct { // // Date : 2022/1/22 9:36 PM func (df *DataFilter) Filter() (string, error) { + df.logPrint(logLevelDebug, "source_data => ", df.source) var ( - jsonObject *gabs.Container - err error + err error ) - // 格式化映射规则 - df.formatRule() - // 记录 obj => slice 的数据类型 - obg2slice := make(map[string]string) - // 创建数据的根结点 - jsonObject = gabs.New() - for _, item := range df.filterRule { - // 数据源路径不识数组, 多个key写入到同一个map key, 并且map key 不是以[]结尾, 自动格式化 - // 目标位置, 是一个数组 - if df.pathIsArrayValue(item.MapKey) { - realMapKey := strings.Trim(item.MapKey, ".[]") - if exist := jsonObject.Exists(realMapKey); !exist { - if _, err = jsonObject.ArrayP(realMapKey); nil != err { - return "", err - } - } - valueResult := gjson.Get(df.source, item.SourceKey) - dataType := df.getValueType(valueResult) - if _, exist := obg2slice[realMapKey]; !exist { - obg2slice[realMapKey] = dataType - } - if dataType != obg2slice[realMapKey] { - return "", errors.New(realMapKey + " 预期写入的字段数据类型不一致") - } - sourcePathArr := strings.Split(item.SourceKey, ".[].") - mapPathArr := strings.Split(realMapKey, ".[].") - result := gabs.New() - _, _ = result.ArrayP(mapPathArr[0]) - df.SetArrayData("{\""+sourcePathArr[0]+"\":"+gjson.Get(df.source, sourcePathArr[0]).String()+"}", result, sourcePathArr, mapPathArr) - if err = jsonObject.ArrayAppend(valueResult.Value(), realMapKey); nil != err { + for _, itemRule := range df.filterRule { + sourceIsArr := df.isArrPath(itemRule.SourceKey) + mapIsArr := df.isArrPath(itemRule.MapKey) + if !sourceIsArr && !mapIsArr { + // 输入输出均不是数组, 最简单的场景 + if err = df.setKV(itemRule); nil != err { return "", err } continue } - sourceSearchResult := gjson.Get(df.source, item.SourceKey) - if !sourceSearchResult.Exists() { - if item.WithDefault { - if _, err = jsonObject.SetP(item.DefaultValue, item.MapKey); nil != err { - return "", err - } + sourcePathArr := df.getArrPathList(itemRule.SourceKey) + mapPathArr := df.getArrPathList(itemRule.MapKey) + if len(mapPathArr) > len(sourcePathArr) { + df.logPrint(logLevelFatal, "映射的层级深度大于数据源深度", "source_path => "+itemRule.SourceKey, "map_path => "+itemRule.MapKey) + return "", fmt.Errorf("映射的层级深度大于数据源深度, source_path => %v map_path => %v", itemRule.SourceKey, itemRule.MapKey) + } + + // 映射至非数组字段 + if !mapIsArr { + if err = df.setValue(itemRule.MapKey, df.getDataAsSlice(df.source, df.getArrPathList(itemRule.SourceKey))); nil != err { + df.logPrint(logLevelFatal, "映射非数组, 数据源为数组, 设置失败", "source_path => "+itemRule.SourceKey, "map_path => "+itemRule.MapKey, " err => "+err.Error()) + return "", fmt.Errorf("映射的层级深度大于数据源深度, source_path => %v map_path => %v", itemRule.SourceKey, itemRule.MapKey) } continue } - if _, err = jsonObject.SetP(sourceSearchResult.Value(), item.MapKey); nil != err { - return "", err + + if len(mapPathArr) == len(sourcePathArr) { + // 数组深度一致 + continue } } - return jsonObject.String(), nil + df.logPrint(logLevelDebug, "过滤结果", df.rewriteResult) + return gjson.Get(df.rewriteResult, virtualRoot).String(), nil } -// UserItemToSlice 支持多个独立的字段合并到slice中 +// isArrPath 是否为数组路径 // // Author : go_developer@163.com<白茶清欢> // -// Date : 3:27 PM 2022/1/24 -func (df *DataFilter) UserItemToSlice() { - df.itemKeyToSlice = true -} - -// getValueType 获取数据类型 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 2022/1/23 12:45 AM -func (df *DataFilter) getValueType(valueResult gjson.Result) string { - dataTypeVal := reflect.TypeOf(valueResult.Value()) - if nil == dataTypeVal { - return "NIL" - } - dataType := dataTypeVal.String() - if strings.Contains(dataType, "int") { - return "int64" - } - if strings.Contains(dataType, "float") { - return "float64" - } - return dataType -} - -// pathIsArrayValue 判断路径是否为数组值 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 2022/1/23 12:56 AM -func (df *DataFilter) pathIsArrayValue(path string) bool { +// Date : 16:00 2023/9/1 +func (df *DataFilter) isArrPath(path string) bool { return strings.Contains(path, "[]") } -// formatRule 格式化映射规则 +// setSameDeepArr 展开同深度数组的 // // Author : go_developer@163.com<白茶清欢> // -// Date : 2:43 PM 2022/1/24 -func (df *DataFilter) formatRule() { - mapKeyCnt := make(map[string]int) - for _, item := range df.filterRule { - // source 为数组, map 不是 - if df.pathIsArrayValue(item.SourceKey) { - if !df.pathIsArrayValue(item.MapKey) { - item.MapKey = item.MapKey + ".[]" - } else { - // source 是数组, map也是数组, 检测数组层级匹配 - sourcePathArr := strings.Split(item.SourceKey, ".[].") - mapPathArr := strings.Split(item.MapKey, ".[].") - if len(sourcePathArr) == len(mapPathArr) { - // 数组层级深度相同,无需特殊处理 - continue - } - // 数组层级深度不同,重新对对齐数据 - diffArr := sourcePathArr[0 : len(sourcePathArr)-len(mapPathArr)+1] - if newPath := df.dealDiffArr(diffArr); len(newPath) > 0 { - sourcePathArr[len(sourcePathArr)-len(mapPathArr)] = newPath - item.SourceKey = strings.Join(sourcePathArr[len(sourcePathArr)-len(mapPathArr):], ".[].") - } - } +// Date : 12:19 2023/9/2 +func (df *DataFilter) unfoldSameDeepArr(sourcePath string, mapPath string, sourceRootPath string, mapRootPath string) ([]*FilterDataRule, error) { + df.logPrint(logLevelDebug, " 同深数组展开", sourceRootPath, mapRootPath) + result := make([]*FilterDataRule, 0) + if len(sourcePath) == 0 { + return result, nil + } + sourcePathArr := df.getArrPathList(sourcePath) + mapPathArr := df.getArrPathList(mapPath) + if 1 == len(mapPathArr) { + sourceKey := sourceRootPath + "." + sourcePathArr[0] + if len(sourcePathArr[0:]) > 0 { + sourceKey = sourceRootPath + "." + strings.Join(sourcePathArr[0:], ".[].") + } + result = append(result, &FilterDataRule{ + SourceKey: sourceKey, + MapKey: mapRootPath + "." + mapPathArr[0], + DefaultValue: nil, + WithDefault: false, + }) + return result, nil + } + // 数组 展开 + for idx := 0; idx < len(mapPathArr); idx++ { + if len(sourceRootPath) > 0 { + sourceRootPath = fmt.Sprintf("%v.%v", sourceRootPath, sourcePathArr[idx]) } else { - if df.pathIsArrayValue(item.MapKey) { - continue - } - // source 不是数组, map 也不是 - if !df.itemKeyToSlice { - continue - } - mapKeyCnt[item.MapKey]++ + sourceRootPath = sourcePathArr[idx] } - } - // 多个source指向一个map,自动转化为list - for _, item := range df.filterRule { - if mapKeyCnt[item.MapKey] > 1 { - item.MapKey = item.MapKey + ".[]" - } - } -} - -// dealDiffArr 提取数据映射关系 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 5:04 下午 2022/1/25 -func (df *DataFilter) dealDiffArr(diffArr []string) string { - if len(diffArr) == 0 { - return "" - } - diffArrStr := strings.Join(diffArr, ".[].") - if _, exist := df.hasDealDiffPath[diffArrStr]; exist { - // 已经处理过, 不再重复处理 - return df.hasDealDiffPath[diffArrStr] - } - - // 没处理过, 开始处理 - jsonResultList := df.getArrayData(df.source, diffArr) - if len(jsonResultList) == 0 { - return "" - } - newPath := wrapper.StringFromRandom(8, "").Value() - var result map[string]interface{} - _ = json.Unmarshal([]byte(df.source), &result) - JSONObject, _ := gabs.Consume(result) - _, _ = JSONObject.ArrayP(newPath) - for _, item := range jsonResultList { - if err := JSONObject.ArrayAppendP(item.Value(), newPath); nil != err { - fmt.Println(err.Error()) - } - } - - df.source = JSONObject.String() - df.hasDealDiffPath[diffArrStr] = newPath - return newPath -} - -// getArrayData 获取数据 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 2:22 下午 2022/1/26 -func (df *DataFilter) getArrayData(source string, pathArr []string) []gjson.Result { - if len(pathArr) == 1 { - return gjson.Get(source, pathArr[0]).Array() - } - resultList := make([]gjson.Result, 0) - dataList := gjson.Get(source, pathArr[0]).Array() - for idx := 0; idx < len(dataList); idx++ { - resultList = append(resultList, df.getArrayData(dataList[idx].String(), pathArr[1:])...) - } - return resultList -} - -// SetArrayData 设置数组数据 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 5:05 下午 2022/2/2 -func (df *DataFilter) SetArrayData(sourceData string, jsonObject *gabs.Container, sourcePathArr []string, mapPathArr []string) *gabs.Container { - jsonObject = gabs.New() - - for idx, sourcePath := range sourcePathArr { - if idx < len(sourcePathArr)-1 { - if !jsonObject.Exists(sourcePath) { - _, _ = jsonObject.ArrayP(sourcePath) - } - } - instance, _ := gabs.ParseJSON([]byte(sourceData)) - if !instance.Exists() { - fmt.Println(sourcePathArr[len(sourcePathArr)-1] + " 不存在") + if len(mapRootPath) > 0 { + mapRootPath = fmt.Sprintf("%v.%v", mapRootPath, mapPathArr[idx]) } else { - dataList, _ := instance.Children() - for _, item := range dataList { - cItem := gabs.New() - cItem.SetP(gjson.Get(item.String(), sourcePath).String(), mapPathArr[idx]) - jsonObject.ArrayAppendP(cItem.Data(), mapPathArr[idx]) + mapRootPath = mapPathArr[idx] + } + valList := gjson.Get(df.source, sourceRootPath) + if valList.Value() == nil { + continue + } + for i := 0; i < len(valList.Array()); i++ { + result = append(result, &FilterDataRule{ + SourceKey: sourceRootPath, + MapKey: mapRootPath, + DefaultValue: nil, + WithDefault: false, + }) + r, e := df.unfoldSameDeepArr( + strings.Join(sourcePathArr[idx+1:], "[]"), + strings.Join(mapPathArr[idx+1:], "[]"), + fmt.Sprintf("%v.%v", sourceRootPath, i), + fmt.Sprintf("%v.%v", mapRootPath, i), + ) + if nil != e { + return nil, e } - //jsonObject.ArrayAppend(jsonObject.Data()) - // fmt.Println("数据 : ", jsonObject.String()) - // jsonObject.ArrayAppendP(result.Data(), mapPathArr[idx]) + result = append(result, r...) + } + } + return result, nil +} +// a.[].b.[].c.[].d +// g +// getDataAsSlice 抽取制定深度，生成list +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 12:17 2023/9/2 +func (df *DataFilter) getDataAsSlice(sourceData string, pathList []string) []interface{} { + //fmt.Println(sourceData, pathList) + result := make([]interface{}, 0) + if len(pathList) == 0 { + return result + } + if len(pathList) == 1 { + return []interface{}{gjson.Get(sourceData, pathList[0]).Value()} + } + for idx, itemPath := range pathList { + if len(pathList)-1 == idx { + val := gjson.Get(sourceData, itemPath).Value() + if nil == val { + return result + } + result = append(result, val) + return result } - df.SetArrayData(gjson.Get(sourceData, sourcePathArr[idx]).String(), jsonObject, sourcePathArr[idx+1:], mapPathArr[idx+1:]) - // jsonObject.ArrayAppendP(v.Data(), mapPathArr[idx]) + currentPathVal := gjson.Get(sourceData, itemPath).Array() + for _, sonItem := range currentPathVal { + result = append(result, df.getDataAsSlice(sonItem.String(), pathList[idx:])...) + } } - fmt.Println("最终 : ", jsonObject.String()) - return jsonObject + return result } + +// setValue 设置值 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 22:04 2023/9/1 +func (df *DataFilter) setValue(path string, value interface{}) error { + var ( + err error + ) + + df.rewriteResult, err = sjson.Set(df.rewriteResult, path, value) + return err +} + +// setKV 设置相关值 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 16:03 2023/9/1 +func (df *DataFilter) setKV(rule *FilterDataRule) error { + var ( + err error + ) + sourceValue := gjson.Get(df.source, rule.SourceKey) + if sourceValue.Exists() { + // 原始数据存在对应路径 + df.rewriteResult, err = sjson.Set(df.rewriteResult, rule.MapKey, sourceValue.Value()) + return err + } + if !rule.WithDefault { + // 路径不存在, 且禁用默认值 + return errors.New(rule.SourceKey + " : source path not found, and default value is forbidden") + } + // 使用默认值填充 + df.rewriteResult, err = sjson.Set(df.rewriteResult, rule.MapKey, rule.DefaultValue) + return err +} + +// getArrPathList 获取路径列表 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 21:32 2023/9/1 +func (df *DataFilter) getArrPathList(inputPath string) []string { + pathArr := strings.Split(inputPath, "[]") + arr := make([]string, 0) + for _, item := range pathArr { + arr = append(arr, strings.Trim(item, ".")) + } + return arr +} + +// logPrint 打印日志 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 18:00 2023/9/1 +func (df *DataFilter) logPrint(level string, msg string, logAttr ...interface{}) { + if !df.filterOption.DebugModel { + // 未开启调试模式 + return + } + logData := append([]interface{}{level, msg}, logAttr...) + log.Println(logData...) +} + +// 日志等级定义 +const ( + logLevelPanic = "PANIC" + logLevelFatal = "FATAL" + logLevelWarn = "WARN" + logLevelInfo = "INFO" + logLevelDebug = "DEBUG" +) diff --git a/tool/gabs_test.go b/tool/gabs_test.go new file mode 100644 index 0000000..f37f1c6 --- /dev/null +++ b/tool/gabs_test.go @@ -0,0 +1,285 @@ +// Package json_tool ... +// +// Description : json_tool ... +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 2023-09-01 16:07 +package json_tool + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + + "git.zhangdeman.cn/zhangdeman/serialize" +) + +// TestDataFilter_FilterNormalData 最基础对象 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 16:13 2023/9/1 +func TestDataFilter_FilterNormalData(t *testing.T) { + source := map[string]interface{}{ + "name": "zhangdeman", + "extra": map[string]interface{}{ + "age": 18, + "height": 180, + "slice": []int{1, 2, 3}, + }, + "slice": []int{1, 2, 3}, + } + df := &DataFilter{ + source: serialize.JSON.MarshalForString(source), + filterRule: []*FilterDataRule{ + {SourceKey: "name", MapKey: "user_name", DefaultValue: "油猴", WithDefault: true}, + {SourceKey: "extra.age", MapKey: "user_age", DefaultValue: "18", WithDefault: true}, + {SourceKey: "slice", MapKey: "user_index", DefaultValue: "[4,5,6]", WithDefault: true}, + {SourceKey: "none", MapKey: "none_default", DefaultValue: map[string]interface{}{"a": "a"}, WithDefault: true}, + {SourceKey: "extra", MapKey: "extra_object", DefaultValue: map[string]interface{}{"a": "a"}, WithDefault: true}, + }, + filterOption: &FilterOption{DebugModel: true}, + } + fmt.Println(df.Filter()) +} + +// TestDataFilter_getDataAsSlice 测试循环读取数据 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 22:21 2023/9/1 +func TestDataFilter_getDataAsSlice(t *testing.T) { + data := map[string]interface{}{ + "list": []interface{}{ + map[string]interface{}{ + "name": "1", + "age": "2", + "list_test": []interface{}{ + map[string]interface{}{ + "a": "a", + "b": "b", + }, + }, + }, + map[string]interface{}{ + "name": "3", + "age": "4", + "list_test": []interface{}{ + map[string]interface{}{ + "a": "a1", + "b": "b1", + }, + }, + }, + map[string]interface{}{ + "name": "5", + "age": "6", + }, + }, + } + byteData, _ := json.Marshal(data) + df := &DataFilter{} + fmt.Println(df.getDataAsSlice(string(byteData), []string{"list", "name"})) + fmt.Println(df.getDataAsSlice(string(byteData), []string{"list", "age"})) + fmt.Println(df.getDataAsSlice(string(byteData), []string{"list", "age", "list_test", "a"})) + fmt.Println(df.getDataAsSlice(string(byteData), []string{"list", "age", "list_test", "b"})) +} + +// TestDataFilter_sourceArrAndMapSingle map 非数组，数据源数组 +// +// Author : zhangdeman001@ke.com<张德满> +// +// Date : 11:24 2023/9/2 +func TestDataFilter_sourceArrAndMapSingle(t *testing.T) { + source := map[string]interface{}{ + "list": []interface{}{ + map[string]interface{}{ + "name": "1", + "age": "2", + "list_test": []interface{}{ + map[string]interface{}{ + "a": "a", + "b": "b", + "c": map[string]interface{}{ + "a": "1", + }, + "d": []int{1, 2, 3}, + }, + }, + }, + map[string]interface{}{ + "name": "3", + "age": "4", + "list_test": []interface{}{ + map[string]interface{}{ + "a": "a1", + "b": "b1", + "c": map[string]interface{}{ + "a": "a", + }, + "d": []int{1, 2, 3}, + }, + }, + }, + map[string]interface{}{ + "name": "5", + "age": "6", + }, + }, + } + df := &DataFilter{ + source: serialize.JSON.MarshalForString(source), + filterRule: []*FilterDataRule{ + {SourceKey: "list.[].list_test.[].a", MapKey: "a_list", DefaultValue: "[]", WithDefault: true}, + {SourceKey: "list.[].list_test.[].b", MapKey: "b_list", DefaultValue: "[]", WithDefault: true}, + {SourceKey: "list.[].list_test.[].c", MapKey: "c_list", DefaultValue: "[]", WithDefault: true}, + {SourceKey: "list.[].list_test.[].d", MapKey: "d_list", DefaultValue: "[]", WithDefault: true}, + }, + filterOption: &FilterOption{DebugModel: true}, + } + fmt.Println(df.Filter()) +} + +// TestDataFilter_unfoldSameDeepArr 测试展开同深度数组 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 15:42 2023/9/2 +func TestDataFilter_unfoldSameDeepArr(t *testing.T) { + source := map[string]interface{}{ + "list": []interface{}{ + map[string]interface{}{ + "name": "1", + "age": "2", + "list_test": []interface{}{ + map[string]interface{}{ + "a": "a", + "b": "b", + "c": map[string]interface{}{ + "a": "1", + }, + "d": []int{1, 2, 3}, + }, + }, + }, + map[string]interface{}{ + "name": "3", + "age": "4", + "list_test": []interface{}{ + map[string]interface{}{ + "a": "a1", + "b": "b1", + "c": map[string]interface{}{ + "a": "a", + }, + "d": []int{1, 2, 3}, + }, + map[string]interface{}{ + "a": "a2", + "b": "b1", + "c": map[string]interface{}{ + "a": "a", + }, + "d": []int{1, 2, 3}, + }, + }, + }, + map[string]interface{}{ + "name": "5", + "age": "6", + }, + }, + } + df := &DataFilter{ + source: serialize.JSON.MarshalForString(source), + filterRule: []*FilterDataRule{ + {SourceKey: "list.[].list_test.[].a", MapKey: "a_list.[].a.[].val", DefaultValue: "[]", WithDefault: true}, + {SourceKey: "list.[].list_test.[].b", MapKey: "b_list.[].b.[].val", DefaultValue: "[]", WithDefault: true}, + {SourceKey: "list.[].list_test.[].c", MapKey: "c_list.[].c.[].val", DefaultValue: "[]", WithDefault: true}, + {SourceKey: "list.[].list_test.[].d", MapKey: "d_list.[].d.[].val", DefaultValue: "[]", WithDefault: true}, + }, + filterOption: &FilterOption{DebugModel: true}, + } + r, _ := df.unfoldSameDeepArr("list.[].list_test.[].a", "a_list.[].a_not_equal_list", "", "") + + formatRes := make([]*FilterDataRule, 0) + for _, item := range r { + itemMapArr := strings.Split(item.MapKey, ".") + if len(itemMapArr) != 3 && len(itemMapArr) != 5 { + continue + } + formatRes = append(formatRes, item) + } + fmt.Println(serialize.JSON.MarshalForString(formatRes)) +} + +// TestDataFilter_filterSameDeepArr ... +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 20:41 2023/9/2 +func TestDataFilter_filterSameDeepArr(t *testing.T) { + source := map[string]interface{}{ + "list": []interface{}{ + map[string]interface{}{ + "name": "1", + "age": "2", + "list_test": []interface{}{ + map[string]interface{}{ + "a": "a", + "b": "b", + "c": map[string]interface{}{ + "a": "1", + }, + "d": []int{1, 2, 3}, + }, + }, + }, + map[string]interface{}{ + "name": "3", + "age": "4", + "list_test": []interface{}{ + map[string]interface{}{ + "a": "a1", + "b": "b1", + "c": map[string]interface{}{ + "a": "a", + }, + "d": []int{1, 2, 3}, + }, + map[string]interface{}{ + "a": "a2", + "b": "b1", + "c": map[string]interface{}{ + "a": "a", + }, + "d": []int{1, 2, 3}, + }, + }, + }, + map[string]interface{}{ + "name": "5", + "age": "6", + }, + }, + } + filterRuleList := []*FilterDataRule{ + {SourceKey: "name", MapKey: "user_name", DefaultValue: "油猴", WithDefault: true}, + {SourceKey: "extra.age", MapKey: "user_age", DefaultValue: "18", WithDefault: true}, + {SourceKey: "slice", MapKey: "user_index", DefaultValue: "[4,5,6]", WithDefault: true}, + {SourceKey: "none", MapKey: "none_default", DefaultValue: map[string]interface{}{"a": "a"}, WithDefault: true}, + {SourceKey: "extra", MapKey: "extra_object", DefaultValue: map[string]interface{}{"a": "a"}, WithDefault: true}, + {SourceKey: "list.[].list_test.[].a", MapKey: "a_list.[].a.[].val", DefaultValue: "[]", WithDefault: true}, + {SourceKey: "list.[].list_test.[].a", MapKey: "a_list.[].a_not_equal_list", DefaultValue: "[]", WithDefault: true}, + {SourceKey: "list.[].list_test.[].a", MapKey: "a_list.[].a.[].val1", DefaultValue: "[]", WithDefault: true}, + {SourceKey: "list.[].list_test.[].b", MapKey: "b_list.[].b.[].val", DefaultValue: "[]", WithDefault: true}, + {SourceKey: "list.[].list_test.[].c", MapKey: "c_list.[].c.[].val", DefaultValue: "[]", WithDefault: true}, + {SourceKey: "list.[].list_test.[].d", MapKey: "d_list.[].d.[].val", DefaultValue: "[]", WithDefault: true}, + } + filterOption := &FilterOption{DebugModel: true} + + df := NewDataFilter(serialize.JSON.MarshalForString(source), filterRuleList, filterOption) + _, _ = df.Filter() +} diff --git a/tool/json_test.go b/tool/json_test.go index b655dd5..5fc6af1 100644 --- a/tool/json_test.go +++ b/tool/json_test.go @@ -42,15 +42,6 @@ func TestJSON(t *testing.T) { fmt.Println(tree.String()) } -// TestType 判断数据类型断言 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 10:59 下午 2021/3/14 -func TestType(t *testing.T) { - -} - // TestSelect 测试动态选择字段 // // Author : go_developer@163.com<白茶清欢> @@ -127,151 +118,3 @@ func TestParse(t *testing.T) { byteData, _ := json.Marshal(source) fmt.Println(GetJSONDataStruct(string(byteData))) } - -// TestParseWithType 测试获取JSON数据结构 -// -// Author : go_developer@163.com<张德满> -// -// Date : 10:59 PM 2022/1/9 -func TestParseWithType(t *testing.T) { - source := map[string]interface{}{ - "name": "zhangdeman", - "extra": map[string]interface{}{ - "age": 18, - "height": 180, - "slice": []int{1, 2, 3}, - "obj": map[string]interface{}{ - "la": "aaaa", - }, - }, - "slice": []int{1, 2, 3}, - "map": map[string]interface{}{"a": 1, "d": 5.5, "e": "qqq"}, - "empty_obj": map[string]interface{}{}, - "empty_list": make([]interface{}, 0), - "table": []map[string]interface{}{ - {"name": "alex", "age": 18, "number": 1, "obj": map[string]interface{}{"enen": "en"}}, - {"name": "bob", "age": 28, "number": 2}, - }, - "two_slice": []map[string]interface{}{ - { - "students": []map[string]interface{}{ - { - "name": "enen", - "age": 18, - "score": []float64{1, 2, 3, 45}, - }, - }, - "other": []interface{}{"others"}, - "read_only": 1, - }, - }, - } - byteData, _ := json.Marshal(source) - fmt.Println(GetJSONDataStructWithType(string(byteData))) -} - -// TestDataFilter 测试数据过滤 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 2022/1/22 10:19 PM -func TestDataFilter(t *testing.T) { - source := map[string]interface{}{ - "name": "zhangdeman", - "extra": map[string]interface{}{ - "age": 18, - "height": 180, - "slice": []int{1, 2, 3}, - }, - "slice_data": []int{1, 2, 3}, - "map": map[string]interface{}{"a": 1, "b": 2, "c": 4}, - "table": []map[string]interface{}{ - {"name": "alex", "age": 18, "number": 1}, - {"name": "bob", "age": 28, "number": 2}, - {"name": "bob", "age": 28, "number": 2, "list": []int{1, 2, 3}}, - }, - } - rule := []*FilterDataRule{ - {SourceKey: "name", MapKey: "user_name", DefaultValue: "用户姓名默认值"}, - {SourceKey: "name", MapKey: "username", DefaultValue: "用户姓名默认值"}, - {SourceKey: "name", MapKey: "user.name", DefaultValue: "用户姓名默认值"}, - {SourceKey: "extra.age", MapKey: "user.age", DefaultValue: "用户年龄默认值"}, - {SourceKey: "extra.age", MapKey: "user_age", DefaultValue: "用户年龄默认值"}, - {SourceKey: "extra.height", MapKey: "user.height", DefaultValue: "扩展高度默认值"}, - {SourceKey: "extra.height", MapKey: "user_height", DefaultValue: "扩展高度默认值"}, - {SourceKey: "table.[].name", MapKey: "slice.[].name_modify", DefaultValue: "列表姓名默认值"}, - {SourceKey: "table.[].list", MapKey: "slice.[].data_list", DefaultValue: "[\"567\",\"678\",\"789\"]"}, - } - byteData, _ := json.Marshal(source) - filter := NewDataFilter(string(byteData), rule) - fmt.Println(filter.Filter()) -} - -// TestDataFilterForObiToSlice ... -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 2022/1/23 12:06 AM -func TestDataFilterForObiToSlice(t *testing.T) { - source := map[string]interface{}{ - "name": "zhangdeman", - "age": 18, - "height": 180, - "extra": map[string]interface{}{ - "age": 18, - "height": 180, - "slice": []int{1, 2, 3}, - }, - "slice_data": []int{1, 2, 3}, - "map": map[string]interface{}{"a": 1, "b": 2, "c": 4}, - "table": []map[string]interface{}{ - {"name": "alex", "age": 18, "number": 1}, - {"name": "bob", "age": 28, "number": 2}, - {"name": "bob", "age": 28, "number": 2, "list": []int{1, 2, 3}}, - }, - } - rule := []*FilterDataRule{ - // {SourceKey: "name", MapKey: "slice.[]", DefaultValue: "用户姓名默认值"}, - {SourceKey: "age", MapKey: "slice", DefaultValue: "用户姓名默认值"}, - {SourceKey: "height", MapKey: "slice", DefaultValue: "用户姓名默认值"}, - } - byteData, _ := json.Marshal(source) - filter := NewDataFilter(string(byteData), rule) - filter.UserItemToSlice() - fmt.Println(filter.Filter()) -} - -// TestDataFilterDiffArr ... -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 12:27 下午 2022/1/26 -func TestDataFilterDiffArr(t *testing.T) { - source := map[string]interface{}{ - "name": "zhangdeman", - "age": 18, - "height": 180, - "extra": map[string]interface{}{ - "age": 18, - "height": 180, - "slice": []int{1, 2, 3}, - }, - "slice_data": []int{1, 2, 3}, - "map": map[string]interface{}{"a": 1, "b": 2, "c": 4}, - "table": []map[string]interface{}{ - {"user_list": []interface{}{map[string]interface{}{"name": "alex", "age": 18, "number": 1}}}, - {"user_list": []interface{}{map[string]interface{}{"name": "bob", "age": 28, "number": 2}}}, - {"user_list": []interface{}{map[string]interface{}{"name": "andy", "age": 28, "number": 2}}}, - }, - } - rule := []*FilterDataRule{ - // {SourceKey: "name", MapKey: "slice.[]", DefaultValue: "用户姓名默认值"}, - {SourceKey: "table.[].user_list.[].name", MapKey: "user_list.[].detail.name", DefaultValue: "用户姓名默认值"}, - {SourceKey: "table.[].user_list.[].age", MapKey: "user_list.[].detail.age", DefaultValue: "用户姓名默认值"}, - } - byteData, _ := json.Marshal(source) - filter := NewDataFilter(string(byteData), rule) - filter.UserItemToSlice() - filter.Filter() - //fmt.Println() -}