// Package json_tool ... // // Description : json_tool ... // // Author : go_developer@163.com<张德满> // // Date : 2022/01/22 9:19 PM package json_tool import ( "fmt" "io" "log" "os" "strings" "github.com/pkg/errors" "github.com/tidwall/gjson" "github.com/tidwall/sjson" ) const ( // virtualRoot 虚拟根节点 virtualRoot = "__VIRTUAL_ROOT__" ) // FilterOption 过滤选项 // // Author : go_developer@163.com<白茶清欢> // // Date : 17:59 2023/9/1 type FilterOption struct { DebugModel bool // 调试模式 LogInstance io.Writer // 日志实例 } // FilterDataRule 参数过滤规则 // // Author : go_developer@163.com<白茶清欢> // // Date : 2022/1/22 9:44 PM type FilterDataRule struct { SourceKey string // 原始数据路径 MapKey string // 提取后映射到的数据路径 DefaultValue interface{} // 原始数据路径不存在时的默认值 WithDefault bool // 是否使用默认值 } // NewDataFilter 获取数据过滤方法实例 // // Author : go_developer@163.com<白茶清欢> // // Date : 2022/1/22 9:50 PM func NewDataFilter(source string, filterRule []*FilterDataRule, filterOption *FilterOption) *DataFilter { source = fmt.Sprintf(`{"%v":%v}`, virtualRoot, source) if nil == filterOption { filterOption = &FilterOption{} } if filterOption.DebugModel && nil == filterOption.LogInstance { filterOption.LogInstance = os.Stdout log.SetOutput(filterOption.LogInstance) } df := &DataFilter{ source: source, filterRule: make([]*FilterDataRule, 0), rewriteResult: "{}", filterOption: filterOption, } // 去除末尾的 .[] for _, item := range filterRule { item.MapKey = virtualRoot + "." + strings.TrimRight(item.MapKey, ".[]") item.SourceKey = virtualRoot + "." + strings.TrimRight(item.SourceKey, ".[]") mapIsArr := df.isArrPath(item.MapKey) if !mapIsArr { df.filterRule = append(df.filterRule, item) continue } if len(df.getArrPathList(item.SourceKey)) < len(df.getArrPathList(item.MapKey)) { panic("map result deep more than source data deep") } formatRes := make([]*FilterDataRule, 0) r, _ := df.unfoldSameDeepArr(item.SourceKey, item.MapKey, "", "") for _, itemUnfoldResult := range r { itemMapArr := strings.Split(itemUnfoldResult.MapKey, ".") if len(itemMapArr) != len(strings.Split(item.MapKey, ".")) { continue } formatRes = append(formatRes, itemUnfoldResult) } df.filterRule = append(df.filterRule, formatRes...) } return df } // DataFilter 数据过滤 // // Author : go_developer@163.com<白茶清欢> // // Date : 2022/1/22 9:20 PM type DataFilter struct { source string filterRule []*FilterDataRule rewriteResult string // json数据重写结果 filterOption *FilterOption // 过滤选项 } // Filter 数据过滤 // // Author : go_developer@163.com<白茶清欢> // // Date : 2022/1/22 9:36 PM func (df *DataFilter) Filter() (string, error) { df.logPrint(logLevelDebug, "source_data => ", df.source) var ( err error ) for _, itemRule := range df.filterRule { sourceIsArr := df.isArrPath(itemRule.SourceKey) mapIsArr := df.isArrPath(itemRule.MapKey) if !sourceIsArr && !mapIsArr { // 输入输出均不是数组, 最简单的场景 if err = df.setKV(itemRule); nil != err { return "", err } continue } sourcePathArr := df.getArrPathList(itemRule.SourceKey) mapPathArr := df.getArrPathList(itemRule.MapKey) if len(mapPathArr) > len(sourcePathArr) { df.logPrint(logLevelFatal, "映射的层级深度大于数据源深度", "source_path => "+itemRule.SourceKey, "map_path => "+itemRule.MapKey) return "", fmt.Errorf("映射的层级深度大于数据源深度, source_path => %v map_path => %v", itemRule.SourceKey, itemRule.MapKey) } // 映射至非数组字段 if !mapIsArr { if err = df.setValue(itemRule.MapKey, df.getDataAsSlice(df.source, df.getArrPathList(itemRule.SourceKey))); nil != err { df.logPrint(logLevelFatal, "映射非数组, 数据源为数组, 设置失败", "source_path => "+itemRule.SourceKey, "map_path => "+itemRule.MapKey, " err => "+err.Error()) return "", fmt.Errorf("映射的层级深度大于数据源深度, source_path => %v map_path => %v", itemRule.SourceKey, itemRule.MapKey) } continue } if len(mapPathArr) == len(sourcePathArr) { // 数组深度一致 continue } } df.logPrint(logLevelDebug, "过滤结果", df.rewriteResult) return gjson.Get(df.rewriteResult, virtualRoot).String(), nil } // isArrPath 是否为数组路径 // // Author : go_developer@163.com<白茶清欢> // // Date : 16:00 2023/9/1 func (df *DataFilter) isArrPath(path string) bool { return strings.Contains(path, "[]") } // setSameDeepArr 展开同深度数组的 // // Author : go_developer@163.com<白茶清欢> // // Date : 12:19 2023/9/2 func (df *DataFilter) unfoldSameDeepArr(sourcePath string, mapPath string, sourceRootPath string, mapRootPath string) ([]*FilterDataRule, error) { df.logPrint(logLevelDebug, " 同深数组展开", sourceRootPath, mapRootPath) result := make([]*FilterDataRule, 0) if len(sourcePath) == 0 { return result, nil } sourcePathArr := df.getArrPathList(sourcePath) mapPathArr := df.getArrPathList(mapPath) if 1 == len(mapPathArr) { sourceKey := sourceRootPath + "." + sourcePathArr[0] if len(sourcePathArr[0:]) > 0 { sourceKey = sourceRootPath + "." + strings.Join(sourcePathArr[0:], ".[].") } result = append(result, &FilterDataRule{ SourceKey: sourceKey, MapKey: mapRootPath + "." + mapPathArr[0], DefaultValue: nil, WithDefault: false, }) return result, nil } // 数组 展开 for idx := 0; idx < len(mapPathArr); idx++ { if len(sourceRootPath) > 0 { sourceRootPath = fmt.Sprintf("%v.%v", sourceRootPath, sourcePathArr[idx]) } else { sourceRootPath = sourcePathArr[idx] } if len(mapRootPath) > 0 { mapRootPath = fmt.Sprintf("%v.%v", mapRootPath, mapPathArr[idx]) } else { mapRootPath = mapPathArr[idx] } valList := gjson.Get(df.source, sourceRootPath) if valList.Value() == nil { continue } for i := 0; i < len(valList.Array()); i++ { result = append(result, &FilterDataRule{ SourceKey: sourceRootPath, MapKey: mapRootPath, DefaultValue: nil, WithDefault: false, }) r, e := df.unfoldSameDeepArr( strings.Join(sourcePathArr[idx+1:], "[]"), strings.Join(mapPathArr[idx+1:], "[]"), fmt.Sprintf("%v.%v", sourceRootPath, i), fmt.Sprintf("%v.%v", mapRootPath, i), ) if nil != e { return nil, e } result = append(result, r...) } } return result, nil } // a.[].b.[].c.[].d // g // getDataAsSlice 抽取制定深度,生成list // // Author : go_developer@163.com<白茶清欢> // // Date : 12:17 2023/9/2 func (df *DataFilter) getDataAsSlice(sourceData string, pathList []string) []interface{} { //fmt.Println(sourceData, pathList) result := make([]interface{}, 0) if len(pathList) == 0 { return result } if len(pathList) == 1 { return []interface{}{gjson.Get(sourceData, pathList[0]).Value()} } for idx, itemPath := range pathList { if len(pathList)-1 == idx { val := gjson.Get(sourceData, itemPath).Value() if nil == val { return result } result = append(result, val) return result } currentPathVal := gjson.Get(sourceData, itemPath).Array() for _, sonItem := range currentPathVal { result = append(result, df.getDataAsSlice(sonItem.String(), pathList[idx:])...) } } return result } // setValue 设置值 // // Author : go_developer@163.com<白茶清欢> // // Date : 22:04 2023/9/1 func (df *DataFilter) setValue(path string, value interface{}) error { var ( err error ) df.rewriteResult, err = sjson.Set(df.rewriteResult, path, value) return err } // setKV 设置相关值 // // Author : go_developer@163.com<白茶清欢> // // Date : 16:03 2023/9/1 func (df *DataFilter) setKV(rule *FilterDataRule) error { var ( err error ) sourceValue := gjson.Get(df.source, rule.SourceKey) if sourceValue.Exists() { // 原始数据存在对应路径 df.rewriteResult, err = sjson.Set(df.rewriteResult, rule.MapKey, sourceValue.Value()) return err } if !rule.WithDefault { // 路径不存在, 且禁用默认值 return errors.New(rule.SourceKey + " : source path not found, and default value is forbidden") } // 使用默认值填充 df.rewriteResult, err = sjson.Set(df.rewriteResult, rule.MapKey, rule.DefaultValue) return err } // getArrPathList 获取路径列表 // // Author : go_developer@163.com<白茶清欢> // // Date : 21:32 2023/9/1 func (df *DataFilter) getArrPathList(inputPath string) []string { pathArr := strings.Split(inputPath, "[]") arr := make([]string, 0) for _, item := range pathArr { arr = append(arr, strings.Trim(item, ".")) } return arr } // logPrint 打印日志 // // Author : go_developer@163.com<白茶清欢> // // Date : 18:00 2023/9/1 func (df *DataFilter) logPrint(level string, msg string, logAttr ...interface{}) { if !df.filterOption.DebugModel { // 未开启调试模式 return } logData := append([]interface{}{level, msg}, logAttr...) log.Println(logData...) } // 日志等级定义 const ( logLevelPanic = "PANIC" logLevelFatal = "FATAL" logLevelWarn = "WARN" logLevelInfo = "INFO" logLevelDebug = "DEBUG" )