334 lines
9.1 KiB
Go
334 lines
9.1 KiB
Go
// Package json_tool ...
|
||
//
|
||
// Description : json_tool ...
|
||
//
|
||
// Author : go_developer@163.com<张德满>
|
||
//
|
||
// Date : 2022/01/22 9:19 PM
|
||
package json_tool
|
||
|
||
import (
|
||
"fmt"
|
||
"io"
|
||
"log"
|
||
"os"
|
||
"strings"
|
||
|
||
"github.com/pkg/errors"
|
||
"github.com/tidwall/gjson"
|
||
"github.com/tidwall/sjson"
|
||
)
|
||
|
||
const (
|
||
// virtualRoot 虚拟根节点
|
||
virtualRoot = "__VIRTUAL_ROOT__"
|
||
)
|
||
|
||
// FilterOption 过滤选项
|
||
//
|
||
// Author : go_developer@163.com<白茶清欢>
|
||
//
|
||
// Date : 17:59 2023/9/1
|
||
type FilterOption struct {
|
||
DebugModel bool // 调试模式
|
||
LogInstance io.Writer // 日志实例
|
||
}
|
||
|
||
// FilterDataRule 参数过滤规则
|
||
//
|
||
// Author : go_developer@163.com<白茶清欢>
|
||
//
|
||
// Date : 2022/1/22 9:44 PM
|
||
type FilterDataRule struct {
|
||
SourceKey string // 原始数据路径
|
||
MapKey string // 提取后映射到的数据路径
|
||
DefaultValue interface{} // 原始数据路径不存在时的默认值
|
||
WithDefault bool // 是否使用默认值
|
||
}
|
||
|
||
// NewDataFilter 获取数据过滤方法实例
|
||
//
|
||
// Author : go_developer@163.com<白茶清欢>
|
||
//
|
||
// Date : 2022/1/22 9:50 PM
|
||
func NewDataFilter(source string, filterRule []*FilterDataRule, filterOption *FilterOption) *DataFilter {
|
||
source = fmt.Sprintf(`{"%v":%v}`, virtualRoot, source)
|
||
if nil == filterOption {
|
||
filterOption = &FilterOption{}
|
||
}
|
||
if filterOption.DebugModel && nil == filterOption.LogInstance {
|
||
filterOption.LogInstance = os.Stdout
|
||
log.SetOutput(filterOption.LogInstance)
|
||
}
|
||
df := &DataFilter{
|
||
source: source,
|
||
filterRule: make([]*FilterDataRule, 0),
|
||
rewriteResult: "{}",
|
||
filterOption: filterOption,
|
||
}
|
||
// 去除末尾的 .[]
|
||
for _, item := range filterRule {
|
||
item.MapKey = virtualRoot + "." + strings.TrimRight(item.MapKey, ".[]")
|
||
item.SourceKey = virtualRoot + "." + strings.TrimRight(item.SourceKey, ".[]")
|
||
mapIsArr := df.isArrPath(item.MapKey)
|
||
if !mapIsArr {
|
||
df.filterRule = append(df.filterRule, item)
|
||
continue
|
||
}
|
||
if len(df.getArrPathList(item.SourceKey)) < len(df.getArrPathList(item.MapKey)) {
|
||
panic("map result deep more than source data deep")
|
||
}
|
||
formatRes := make([]*FilterDataRule, 0)
|
||
r, _ := df.unfoldSameDeepArr(item.SourceKey, item.MapKey, "", "")
|
||
for _, itemUnfoldResult := range r {
|
||
itemMapArr := strings.Split(itemUnfoldResult.MapKey, ".")
|
||
if len(itemMapArr) != len(strings.Split(item.MapKey, ".")) {
|
||
continue
|
||
}
|
||
formatRes = append(formatRes, itemUnfoldResult)
|
||
}
|
||
df.filterRule = append(df.filterRule, formatRes...)
|
||
}
|
||
return df
|
||
}
|
||
|
||
// DataFilter 数据过滤
|
||
//
|
||
// Author : go_developer@163.com<白茶清欢>
|
||
//
|
||
// Date : 2022/1/22 9:20 PM
|
||
type DataFilter struct {
|
||
source string
|
||
filterRule []*FilterDataRule
|
||
rewriteResult string // json数据重写结果
|
||
filterOption *FilterOption // 过滤选项
|
||
}
|
||
|
||
// Filter 数据过滤
|
||
//
|
||
// Author : go_developer@163.com<白茶清欢>
|
||
//
|
||
// Date : 2022/1/22 9:36 PM
|
||
func (df *DataFilter) Filter() (string, error) {
|
||
df.logPrint(logLevelDebug, "source_data => ", df.source)
|
||
var (
|
||
err error
|
||
)
|
||
|
||
for _, itemRule := range df.filterRule {
|
||
sourceIsArr := df.isArrPath(itemRule.SourceKey)
|
||
mapIsArr := df.isArrPath(itemRule.MapKey)
|
||
if !sourceIsArr && !mapIsArr {
|
||
// 输入输出均不是数组, 最简单的场景
|
||
if err = df.setKV(itemRule); nil != err {
|
||
return "", err
|
||
}
|
||
continue
|
||
}
|
||
sourcePathArr := df.getArrPathList(itemRule.SourceKey)
|
||
mapPathArr := df.getArrPathList(itemRule.MapKey)
|
||
if len(mapPathArr) > len(sourcePathArr) {
|
||
df.logPrint(logLevelFatal, "映射的层级深度大于数据源深度", "source_path => "+itemRule.SourceKey, "map_path => "+itemRule.MapKey)
|
||
return "", fmt.Errorf("映射的层级深度大于数据源深度, source_path => %v map_path => %v", itemRule.SourceKey, itemRule.MapKey)
|
||
}
|
||
|
||
// 映射至非数组字段
|
||
if !mapIsArr {
|
||
if err = df.setValue(itemRule.MapKey, df.getDataAsSlice(df.source, df.getArrPathList(itemRule.SourceKey))); nil != err {
|
||
df.logPrint(logLevelFatal, "映射非数组, 数据源为数组, 设置失败", "source_path => "+itemRule.SourceKey, "map_path => "+itemRule.MapKey, " err => "+err.Error())
|
||
return "", fmt.Errorf("映射的层级深度大于数据源深度, source_path => %v map_path => %v", itemRule.SourceKey, itemRule.MapKey)
|
||
}
|
||
continue
|
||
}
|
||
|
||
if len(mapPathArr) == len(sourcePathArr) {
|
||
// 数组深度一致
|
||
continue
|
||
}
|
||
}
|
||
df.logPrint(logLevelDebug, "过滤结果", df.rewriteResult)
|
||
return gjson.Get(df.rewriteResult, virtualRoot).String(), nil
|
||
}
|
||
|
||
// isArrPath 是否为数组路径
|
||
//
|
||
// Author : go_developer@163.com<白茶清欢>
|
||
//
|
||
// Date : 16:00 2023/9/1
|
||
func (df *DataFilter) isArrPath(path string) bool {
|
||
return strings.Contains(path, "[]")
|
||
}
|
||
|
||
// setSameDeepArr 展开同深度数组的
|
||
//
|
||
// Author : go_developer@163.com<白茶清欢>
|
||
//
|
||
// Date : 12:19 2023/9/2
|
||
func (df *DataFilter) unfoldSameDeepArr(sourcePath string, mapPath string, sourceRootPath string, mapRootPath string) ([]*FilterDataRule, error) {
|
||
df.logPrint(logLevelDebug, " 同深数组展开", sourceRootPath, mapRootPath)
|
||
result := make([]*FilterDataRule, 0)
|
||
if len(sourcePath) == 0 {
|
||
return result, nil
|
||
}
|
||
sourcePathArr := df.getArrPathList(sourcePath)
|
||
mapPathArr := df.getArrPathList(mapPath)
|
||
if 1 == len(mapPathArr) {
|
||
sourceKey := sourceRootPath + "." + sourcePathArr[0]
|
||
if len(sourcePathArr[0:]) > 0 {
|
||
sourceKey = sourceRootPath + "." + strings.Join(sourcePathArr[0:], ".[].")
|
||
}
|
||
result = append(result, &FilterDataRule{
|
||
SourceKey: sourceKey,
|
||
MapKey: mapRootPath + "." + mapPathArr[0],
|
||
DefaultValue: nil,
|
||
WithDefault: false,
|
||
})
|
||
return result, nil
|
||
}
|
||
// 数组 展开
|
||
for idx := 0; idx < len(mapPathArr); idx++ {
|
||
if len(sourceRootPath) > 0 {
|
||
sourceRootPath = fmt.Sprintf("%v.%v", sourceRootPath, sourcePathArr[idx])
|
||
} else {
|
||
sourceRootPath = sourcePathArr[idx]
|
||
}
|
||
if len(mapRootPath) > 0 {
|
||
mapRootPath = fmt.Sprintf("%v.%v", mapRootPath, mapPathArr[idx])
|
||
} else {
|
||
mapRootPath = mapPathArr[idx]
|
||
}
|
||
valList := gjson.Get(df.source, sourceRootPath)
|
||
if valList.Value() == nil {
|
||
continue
|
||
}
|
||
for i := 0; i < len(valList.Array()); i++ {
|
||
result = append(result, &FilterDataRule{
|
||
SourceKey: sourceRootPath,
|
||
MapKey: mapRootPath,
|
||
DefaultValue: nil,
|
||
WithDefault: false,
|
||
})
|
||
r, e := df.unfoldSameDeepArr(
|
||
strings.Join(sourcePathArr[idx+1:], "[]"),
|
||
strings.Join(mapPathArr[idx+1:], "[]"),
|
||
fmt.Sprintf("%v.%v", sourceRootPath, i),
|
||
fmt.Sprintf("%v.%v", mapRootPath, i),
|
||
)
|
||
if nil != e {
|
||
return nil, e
|
||
}
|
||
result = append(result, r...)
|
||
}
|
||
}
|
||
return result, nil
|
||
}
|
||
|
||
// a.[].b.[].c.[].d
|
||
// g
|
||
// getDataAsSlice 抽取制定深度，生成list
|
||
//
|
||
// Author : go_developer@163.com<白茶清欢>
|
||
//
|
||
// Date : 12:17 2023/9/2
|
||
func (df *DataFilter) getDataAsSlice(sourceData string, pathList []string) []interface{} {
|
||
//fmt.Println(sourceData, pathList)
|
||
result := make([]interface{}, 0)
|
||
if len(pathList) == 0 {
|
||
return result
|
||
}
|
||
if len(pathList) == 1 {
|
||
return []interface{}{gjson.Get(sourceData, pathList[0]).Value()}
|
||
}
|
||
for idx, itemPath := range pathList {
|
||
if len(pathList)-1 == idx {
|
||
val := gjson.Get(sourceData, itemPath).Value()
|
||
if nil == val {
|
||
return result
|
||
}
|
||
result = append(result, val)
|
||
return result
|
||
}
|
||
|
||
currentPathVal := gjson.Get(sourceData, itemPath).Array()
|
||
for _, sonItem := range currentPathVal {
|
||
result = append(result, df.getDataAsSlice(sonItem.String(), pathList[idx:])...)
|
||
}
|
||
}
|
||
return result
|
||
}
|
||
|
||
// setValue 设置值
|
||
//
|
||
// Author : go_developer@163.com<白茶清欢>
|
||
//
|
||
// Date : 22:04 2023/9/1
|
||
func (df *DataFilter) setValue(path string, value interface{}) error {
|
||
var (
|
||
err error
|
||
)
|
||
|
||
df.rewriteResult, err = sjson.Set(df.rewriteResult, path, value)
|
||
return err
|
||
}
|
||
|
||
// setKV 设置相关值
|
||
//
|
||
// Author : go_developer@163.com<白茶清欢>
|
||
//
|
||
// Date : 16:03 2023/9/1
|
||
func (df *DataFilter) setKV(rule *FilterDataRule) error {
|
||
var (
|
||
err error
|
||
)
|
||
sourceValue := gjson.Get(df.source, rule.SourceKey)
|
||
if sourceValue.Exists() {
|
||
// 原始数据存在对应路径
|
||
df.rewriteResult, err = sjson.Set(df.rewriteResult, rule.MapKey, sourceValue.Value())
|
||
return err
|
||
}
|
||
if !rule.WithDefault {
|
||
// 路径不存在, 且禁用默认值
|
||
return errors.New(rule.SourceKey + " : source path not found, and default value is forbidden")
|
||
}
|
||
// 使用默认值填充
|
||
df.rewriteResult, err = sjson.Set(df.rewriteResult, rule.MapKey, rule.DefaultValue)
|
||
return err
|
||
}
|
||
|
||
// getArrPathList 获取路径列表
|
||
//
|
||
// Author : go_developer@163.com<白茶清欢>
|
||
//
|
||
// Date : 21:32 2023/9/1
|
||
func (df *DataFilter) getArrPathList(inputPath string) []string {
|
||
pathArr := strings.Split(inputPath, "[]")
|
||
arr := make([]string, 0)
|
||
for _, item := range pathArr {
|
||
arr = append(arr, strings.Trim(item, "."))
|
||
}
|
||
return arr
|
||
}
|
||
|
||
// logPrint 打印日志
|
||
//
|
||
// Author : go_developer@163.com<白茶清欢>
|
||
//
|
||
// Date : 18:00 2023/9/1
|
||
func (df *DataFilter) logPrint(level string, msg string, logAttr ...interface{}) {
|
||
if !df.filterOption.DebugModel {
|
||
// 未开启调试模式
|
||
return
|
||
}
|
||
logData := append([]interface{}{level, msg}, logAttr...)
|
||
log.Println(logData...)
|
||
}
|
||
|
||
// 日志等级定义
|
||
const (
|
||
logLevelPanic = "PANIC"
|
||
logLevelFatal = "FATAL"
|
||
logLevelWarn = "WARN"
|
||
logLevelInfo = "INFO"
|
||
logLevelDebug = "DEBUG"
|
||
)
|