zhangdeman/json_filter
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

升级提取指定深度的数据最终字段作为list

Browse Source
This commit is contained in:
白茶清欢
2023-09-01 23:17:04 +08:00
parent bae9380210
commit e79ac95858
4 changed files with 132 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
tool/gabs.go
View File

@ -8,12 +8,15 @@
package json_tool
import (
"fmt"
"io"
"log"
"os"
"strings"
"github.com/pkg/errors"
"github.com/tidwall/gjson"
"github.com/tidwall/sjson"
"log/slog"
"os"
"strings"
)
const (
@ -26,8 +29,8 @@ const (
//
// Date : 17:59 2023/9/1
type FilterOption struct {
DebugModel bool // 调试模式
LogInstance *slog.Logger // 日志实例
DebugModel bool // 调试模式
LogInstance io.Writer // 日志实例
}
// FilterDataRule 参数过滤规则
@ -52,10 +55,8 @@ func NewDataFilter(source string, filterRule []*FilterDataRule, filterOption *Fi
filterOption = &FilterOption{}
}
if filterOption.DebugModel && nil == filterOption.LogInstance {
filterOption.LogInstance = slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{
Level: slog.LevelDebug,
}))
slog.SetDefault(filterOption.LogInstance)
filterOption.LogInstance = os.Stdout
log.SetOutput(filterOption.LogInstance)
}
return &DataFilter{
source: source,
@ -83,18 +84,29 @@ type DataFilter struct {
//
// Date : 2022/1/22 9:36 PM
func (df *DataFilter) Filter() (string, error) {
df.logPrint(logLevelInfo, "输入的原始数据", slog.String("source_data", df.source))
df.logPrint(logLevelDebug, "source_data => ", df.source)
var (
err error
)
for _, itemRule := range df.filterRule {
if !df.isArrPath(itemRule.SourceKey) && !df.isArrPath(itemRule.MapKey) {
sourceIsArr := df.isArrPath(itemRule.SourceKey)
mapIsArr := df.isArrPath(itemRule.MapKey)
if !sourceIsArr && !mapIsArr {
// 输入输出均不是数组, 最简单的场景
if err = df.setKV(itemRule); nil != err {
return "", err
}
}
sourcePathArr := df.getArrPathList(itemRule.SourceKey)
mapPathArr := df.getArrPathList(itemRule.MapKey)
if len(mapPathArr) > len(sourcePathArr) {
df.logPrint(logLevelFatal, "映射的层级深度大于数据源深度", "source_path => "+itemRule.SourceKey, "map_path => "+itemRule.MapKey)
return "", fmt.Errorf("映射的层级深度大于数据源深度, source_path => %v map_path => %v", itemRule.SourceKey, itemRule.MapKey)
}
if !mapIsArr {
}
}
return df.rewriteResult, nil
}
@ -108,6 +120,49 @@ func (df *DataFilter) isArrPath(path string) bool {
return strings.Contains(path, "[]")
}
// a.[].b.[].c.[].d
// e.[].f.[].g
func (df *DataFilter) getDataAsSlice(sourceData string, pathList []string) []interface{} {
//fmt.Println(sourceData, pathList)
result := make([]interface{}, 0)
if len(pathList) == 0 {
return result
}
if len(pathList) == 1 {
return []interface{}{gjson.Get(sourceData, pathList[0]).Value()}
}
for idx, itemPath := range pathList {
if len(pathList)-1 == idx {
val := gjson.Get(sourceData, itemPath).Value()
if nil == val {
return result
}
result = append(result, val)
return result
}
currentPathVal := gjson.Get(sourceData, itemPath).Array()
for _, sonItem := range currentPathVal {
result = append(result, df.getDataAsSlice(sonItem.String(), pathList[idx:])...)
}
}
return result
}
// setValue 设置值
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 22:04 2023/9/1
func (df *DataFilter) setValue(path string, value interface{}) error {
var (
err error
)
df.rewriteResult, err = sjson.Set(df.rewriteResult, path, value)
return err
}
// setKV 设置相关值
//
// Author : go_developer@163.com<白茶清欢>
@ -132,6 +187,20 @@ func (df *DataFilter) setKV(rule *FilterDataRule) error {
return err
}
// getArrPathList 获取路径列表
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 21:32 2023/9/1
func (df *DataFilter) getArrPathList(inputPath string) []string {
pathArr := strings.Split(inputPath, "[]")
arr := make([]string, 0)
for _, item := range pathArr {
arr = append(arr, strings.Trim(item, "."))
}
return arr
}
// logPrint 打印日志
//
// Author : go_developer@163.com<白茶清欢>
@ -142,20 +211,13 @@ func (df *DataFilter) logPrint(level string, msg string, logAttr ...interface{})
// 未开启调试模式
return
}
switch level {
case logLevelFatal:
slog.Error(msg, logAttr...)
case logLevelWarn:
slog.Warn(msg, logAttr...)
case logLevelInfo:
slog.Info(msg, logAttr...)
case logLevelDebug:
slog.Debug(msg, logAttr...)
}
logData := append([]interface{}{level, msg}, logAttr...)
log.Println(logData...)
}
// 日志等级定义
const (
logLevelPanic = "PANIC"
logLevelFatal = "FATAL"
logLevelWarn = "WARN"
logLevelInfo = "INFO"

46
tool/gabs_test.go
View File

@ -8,9 +8,11 @@
package json_tool
import (
"encoding/json"
"fmt"
"git.zhangdeman.cn/zhangdeman/serialize"
"testing"
"git.zhangdeman.cn/zhangdeman/serialize"
)
// TestDataFilter_FilterNormalData 最基础对象
@ -41,3 +43,45 @@ func TestDataFilter_FilterNormalData(t *testing.T) {
}
fmt.Println(df.Filter())
}
// TestDataFilter_getDataAsSlice 测试循环读取数据
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 22:21 2023/9/1
func TestDataFilter_getDataAsSlice(t *testing.T) {
data := map[string]interface{}{
"list": []interface{}{
map[string]interface{}{
"name": "1",
"age": "2",
"list_test": []interface{}{
map[string]interface{}{
"a": "a",
"b": "b",
},
},
},
map[string]interface{}{
"name": "3",
"age": "4",
"list_test": []interface{}{
map[string]interface{}{
"a": "a1",
"b": "b1",
},
},
},
map[string]interface{}{
"name": "5",
"age": "6",
},
},
}
byteData, _ := json.Marshal(data)
df := &DataFilter{}
fmt.Println(df.getDataAsSlice(string(byteData), []string{"list", "name"}))
fmt.Println(df.getDataAsSlice(string(byteData), []string{"list", "age"}))
fmt.Println(df.getDataAsSlice(string(byteData), []string{"list", "age", "list_test", "a"}))
fmt.Println(df.getDataAsSlice(string(byteData), []string{"list", "age", "list_test", "b"}))
}