zhangdeman/json_filter
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

迁移json_tool

Browse Source
This commit is contained in:
白茶清欢 2023-02-12 18:26:27 +08:00
parent b1f1d1b622
commit 3b6f7dc289
8 changed files with 1323 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
filter.go
View File

@ -12,12 +12,12 @@ import (
"fmt"
"strings"
"github.com/tidwall/gjson"
"github.com/tidwall/sjson"
"errors"
"git.zhangdeman.cn/zhangdeman/util"
"github.com/tidwall/gjson"
)
// NewFilter 过滤器实例

6
go.mod
View File

@ -3,7 +3,9 @@ module git.zhangdeman.cn/zhangdeman/filter
go 1.17
require (
git.zhangdeman.cn/zhangdeman/util v0.0.0-20230113095943-b4b3e261e0c4
git.zhangdeman.cn/zhangdeman/easylock v0.0.0-20220627070212-c590a0a1c216
git.zhangdeman.cn/zhangdeman/util v0.0.0-20230211164227-256094968151
github.com/Jeffail/gabs v1.4.0
github.com/pkg/errors v0.9.1
github.com/smartystreets/goconvey v1.7.2
github.com/tidwall/gjson v1.14.4
@ -11,9 +13,9 @@ require (
)
require (
github.com/Jeffail/gabs v1.4.0 // indirect
github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 // indirect
github.com/jtolds/gls v4.20.0+incompatible // indirect

13
go.sum
View File

@ -1,7 +1,8 @@
git.zhangdeman.cn/zhangdeman/util v0.0.0-20221021061434-e68b22a6e106 h1:xiiN+rLtBbDGRUbipVuwI1j2iRhuL3ejSm+EnDxzVMk=
git.zhangdeman.cn/zhangdeman/util v0.0.0-20221021061434-e68b22a6e106/go.mod h1:zTir/0IWdK3E7n0GiaogyWHADAQnBtTdl2I6Z2/OPqw=
git.zhangdeman.cn/zhangdeman/util v0.0.0-20230113095943-b4b3e261e0c4 h1:1WclY9P8l8o/NZ3ZR/mupm8LtowjQ/Q4UNGXR32f0OQ=
git.zhangdeman.cn/zhangdeman/util v0.0.0-20230113095943-b4b3e261e0c4/go.mod h1:zTir/0IWdK3E7n0GiaogyWHADAQnBtTdl2I6Z2/OPqw=
git.zhangdeman.cn/zhangdeman/easylock v0.0.0-20220627070212-c590a0a1c216 h1:5M6YsHmbTqLDGkqKykeoxrrd5WsAme6I+akn9sgKUZk=
git.zhangdeman.cn/zhangdeman/easylock v0.0.0-20220627070212-c590a0a1c216/go.mod h1:8Jh+q/L0pWAKQy67m8AKIs0WGGrVKypoSZZbBm4nDVc=
git.zhangdeman.cn/zhangdeman/util v0.0.0-20220626081130-cbac0b676fb8/go.mod h1:G2/OKMbEn89d+YUXQtv9Nlh0LGg14pOqDnbOgBTTRXY=
git.zhangdeman.cn/zhangdeman/util v0.0.0-20230211164227-256094968151 h1:j537bRLQL1FlkdXTIaT9Ecjx5eogkPsGiTOWIEFQlc8=
git.zhangdeman.cn/zhangdeman/util v0.0.0-20230211164227-256094968151/go.mod h1:SyRTkOz6gxUVn3S/Qtkf+rhKV0I1ym8lwsT8YjggYFs=
github.com/Jeffail/gabs v1.4.0 h1://5fYRRTq1edjfIrQGvdkcd22pkYUrHZ5YC/H2GJVAo=
github.com/Jeffail/gabs v1.4.0/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc=
github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 h1:OYA+5W64v3OgClL+IrOD63t4i/RW7RqrAVl9LTZ9UqQ=
@ -9,7 +10,8 @@ github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394/go.mod h1:Q8n74mJTIg
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-ini/ini v1.66.6/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
@ -31,7 +33,6 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/tidwall/gjson v1.14.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=

378
tool/build.go Normal file
View File

@ -0,0 +1,378 @@
// Package json_tool...
//
// Description : 动态构建json
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2021-03-10 10:26 下午
package json_tool
import (
"encoding/json"
"fmt"
"regexp"
"strconv"
"strings"
"git.zhangdeman.cn/zhangdeman/easylock"
"github.com/pkg/errors"
)
const (
// PathSplit json_tool 路径的分割符
PathSplit = "."
)
// JSONode JSOM节点
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 10:33 下午 2021/3/10
type JSONode struct {
Key string // json_tool key
Value interface{} // 对应的值
Child []*JSONode // 子节点
IsRoot bool // 是否根节点
IsHasLastBrother bool // 在此之后是否有其他兄弟节点
IsSlice bool // 是否是list
IsIndexNode bool // 是否是slice的索引节点
Sort int // 此属性用于　slice解析,保证最终排序是对的
IsComplex bool // 是否为复杂数据类型
IsString bool // 是否为字符串
}
// NewDynamicJSON 获取JSON实例
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 10:36 下午 2021/3/10
func NewDynamicJSON() *DynamicJSON {
exp, _ := regexp.Compile(`\[(\d*?)]`)
return &DynamicJSON{
root: &JSONode{
Key: "",
Value: nil,
Child: nil,
IsRoot: true,
},
nodeCnt: 0,
lock: easylock.NewLock(),
sliceExp: exp,
}
}
// DynamicJSON 动态json
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 11:03 下午 2021/3/10
type DynamicJSON struct {
root *JSONode // 节点数
nodeCnt int // 节点数量
lock easylock.EasyLock // 锁
sliceExp *regexp.Regexp // 抽取slice索引的正则
}
// SetValue 设置节点值,如果节点不存在,创建;如果已存在,更新, 多级key使用, value 必须是基础数据类型, 如果是结构体, 需要继续添加path,多级path使用.分割
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 10:45 下午 2021/3/10
func (dj *DynamicJSON) SetValue(path string, value interface{}, isComplexType bool) {
pathList := strings.Split(path, PathSplit)
searchRoot := dj.root
parent := dj.root
for keyIndex, key := range pathList {
searchRoot = dj.search(searchRoot, key)
if nil != searchRoot {
// 查询到结果,更新值
searchRoot.Value = value
parent = searchRoot
continue
}
var val interface{}
if keyIndex == len(pathList)-1 {
val = value
}
_ = dj.createNode(parent, key, val, isComplexType)
if len(parent.Child) > 0 {
searchRoot = parent.Child[len(parent.Child)-1]
parent = parent.Child[len(parent.Child)-1]
}
}
}
// String 获取字符串的格式JSON
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2:16 下午 2021/3/11
func (dj *DynamicJSON) String() string {
tplList := make([]string, 0)
valList := make([]interface{}, 0)
tplListResult, valListResult := dj.buildTpl(dj.root, &tplList, &valList)
return fmt.Sprintf(strings.Join(*tplListResult, ""), *valListResult...)
}
// Map 转化为map
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 9:59 上午 2021/9/9
func (dj *DynamicJSON) Map() (map[string]interface{}, error) {
var res map[string]interface{}
err := json.Unmarshal([]byte(dj.String()), &res)
return res, err
}
// Slice 转化为slice
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 9:59 上午 2021/9/9
func (dj *DynamicJSON) Slice() ([]interface{}, error) {
var res []interface{}
err := json.Unmarshal([]byte(dj.String()), &res)
return res, err
}
// ParseWithReceiver 使用指定结构解析
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 10:01 上午 2021/9/9
func (dj *DynamicJSON) ParseWithReceiver(receiver interface{}) error {
if nil == receiver {
return errors.New("receiver is nil")
}
return json.Unmarshal([]byte(dj.String()), receiver)
}
// buildTpl 构建json模版与绑定数据
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 4:38 下午 2021/3/11
func (dj *DynamicJSON) buildTpl(root *JSONode, tplList *[]string, valList *[]interface{}) (*[]string, *[]interface{}) {
if nil == root {
return tplList, valList
}
startSymbol := dj.getStartSymbol(root)
endSymbol := dj.getEndSymbol(root)
valFormat := dj.getValFormat(root)
// key := "\"" + root.Key + "\""
if !root.IsIndexNode {
if len(root.Child) > 0 {
*tplList = append(*tplList, startSymbol)
} else {
*tplList = append(*tplList, valFormat)
*valList = append(*valList, root.Value)
return tplList, valList
}
} else {
if len(root.Child) == 0 {
*tplList = append(*tplList, valFormat)
*valList = append(*valList, root.Value)
} else {
*tplList = append(*tplList, startSymbol)
}
}
for _, node := range root.Child {
dj.buildTpl(node, tplList, valList)
}
*tplList = append(*tplList, endSymbol)
return tplList, valList
}
// getValFormat 构建值得占位符
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 12:49 下午 2021/3/13
func (dj *DynamicJSON) getValFormat(root *JSONode) string {
key := fmt.Sprintf("\"%s\"", root.Key)
if !root.IsIndexNode {
if len(root.Child) > 0 {
// 还有自节点的情况下,不需要占位符
return ""
}
if root.IsHasLastBrother {
if root.IsString {
return key + ":\"%v\","
}
return key + ":%v,"
}
if root.IsString {
return key + ":\"%v\""
}
return key + ":%v"
}
if len(root.Child) > 0 {
// 是list的索引节点,且有子节点
return ""
}
if root.IsHasLastBrother {
if root.IsString {
return "\"%v\","
}
return "%v,"
}
if root.IsString {
return "\"%v\""
}
return "%v"
}
// getStartSymbol 计算起始的符号
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 12:21 下午 2021/3/13
func (dj *DynamicJSON) getStartSymbol(root *JSONode) string {
if nil == root {
return "{"
}
if root.IsRoot {
if root.IsSlice {
return "["
}
return "{"
}
key := fmt.Sprintf("\"%s\"", root.Key)
if !root.IsIndexNode {
if len(root.Child) > 0 {
if root.IsSlice {
return key + ":["
} else {
return key + ":{"
}
}
return ""
}
if len(root.Child) > 0 {
return "{"
}
return ""
}
// getEndSymbol 计算结束的符号
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 12:21 下午 2021/3/13
func (dj *DynamicJSON) getEndSymbol(root *JSONode) string {
if nil == root {
return "}"
}
if !root.IsIndexNode {
if root.IsHasLastBrother {
return "},"
}
if root.IsSlice {
return "]"
} else {
return "}"
}
}
if len(root.Child) > 0 {
if root.IsHasLastBrother {
return "},"
}
return "}"
}
return ""
}
// Search 搜索一个key TODO : 优化
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 11:19 下午 2021/3/10
func (dj *DynamicJSON) search(root *JSONode, key string) *JSONode {
if root == nil {
return nil
}
for _, node := range root.Child {
if node == nil {
continue
}
if node.Key == key {
return node
}
}
return nil
}
// createNode 创建新的节点
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 10:57 下午 2021/3/10
func (dj *DynamicJSON) createNode(parent *JSONode, key string, value interface{}, isComplexType bool) error {
if nil == parent {
return errors.New("create node error : parent id nil")
}
_ = dj.lock.Lock()
if parent.Child == nil {
parent.Child = make([]*JSONode, 0)
}
if len(parent.Child) > 0 {
// 存在子节点，设置当前子节点还有其他兄弟节点
parent.Child[len(parent.Child)-1].IsHasLastBrother = true
}
newNode := &JSONode{
Key: key,
Value: value,
Child: make([]*JSONode, 0),
IsRoot: false,
IsHasLastBrother: false,
IsComplex: isComplexType,
IsString: false,
}
if !isComplexType {
switch value.(type) {
case string:
newNode.IsString = true
}
}
parent.IsSlice, newNode.Sort = dj.extraSliceIndex(key)
if parent.IsSlice {
newNode.IsIndexNode = true
}
parent.Child = append(parent.Child, newNode)
dj.nodeCnt++
_ = dj.lock.Unlock()
return nil
}
// extraSliceIndex 抽取slice索引
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 9:37 下午 2021/3/11
func (dj *DynamicJSON) extraSliceIndex(key string) (bool, int) {
if len(key) < 3 {
// slice 至少是 [1] 格式
return false, 0
}
if !strings.HasPrefix(key, "[") || !strings.HasSuffix(key, "]") {
return false, 0
}
// 不用正则,直接字符串处理
strByte := []byte(key)
index, err := strconv.Atoi(string(strByte[1 : len(strByte)-1]))
if nil != err {
return false, 0
}
return true, index
}

145
tool/filter.go Normal file
View File

@ -0,0 +1,145 @@
// Package json_tool ...
//
// Description : 将复杂数据结构转化为 JSONNode 树
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2021-03-14 10:40 下午
package json_tool
import (
"encoding/json"
"fmt"
"reflect"
"strings"
"github.com/tidwall/gjson"
"github.com/pkg/errors"
"git.zhangdeman.cn/zhangdeman/util"
)
// MapDataRule 数据映射结果
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 5:09 PM 2022/1/17
type MapDataRule struct {
MapKey string
DefaultValue string
IsComplexType bool
}
// NewFilter 获取解析的实例
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 10:43 下午 2021/3/14
func NewFilter(data interface{}, rule map[string]MapDataRule) *Filter {
return &Filter{
data: data,
rule: rule,
}
}
// Filter 解析json树
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 10:41 下午 2021/3/14
type Filter struct {
data interface{}
rule map[string]MapDataRule
}
// Result 数据过滤结果
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 10:44 下午 2021/3/14
func (f *Filter) Result() (*DynamicJSON, error) {
if !f.isLegalData() {
return nil, errors.New("非法的数据,无法转换成json")
}
result := NewDynamicJSON()
byteData, _ := json.Marshal(f.data)
source := string(byteData)
for extraDataPath, newDataRule := range f.rule {
// 为数组的处理
pathArr := strings.Split(extraDataPath, ".[].")
val := gjson.Get(source, pathArr[0])
if len(pathArr) == 1 {
f.SetValue(result, newDataRule.MapKey, val.Value(), false, 0)
continue
}
// 支持list再抽取一层,处于性能考虑,这么限制,不做递归无限深度处理
// 还能继续抽取,就认为是 []map[string]interface{}
keyList := strings.Split(pathArr[1], ".")
for idx, item := range val.Array() {
data := item.Map()
for _, key := range keyList {
if _, exist := data[key]; exist {
result.SetValue(strings.ReplaceAll(newDataRule.MapKey, "[]", fmt.Sprintf("[%d]", idx)), data[key].String(), newDataRule.IsComplexType)
} else {
// 结果集中不存在对应key,设置默认值
result.SetValue(strings.ReplaceAll(newDataRule.MapKey, "[]", fmt.Sprintf("[%d]", idx)), newDataRule.DefaultValue, newDataRule.IsComplexType)
}
}
}
}
return result, nil
}
// SetValue 设置值
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 12:14 下午 2021/9/10
func (f *Filter) SetValue(result *DynamicJSON, path string, val interface{}, isSetSlice bool, sliceIndex int) {
if !isSetSlice {
result.SetValue(path, val, false)
return
}
}
// isLegalData 判断是否能转换成json结构, 只有slice/map/struct/能转换成slice或map的[]byte是合法的
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 10:46 下午 2021/3/14
func (f *Filter) isLegalData() bool {
val := reflect.ValueOf(f.data)
switch val.Kind() {
case reflect.Slice:
// slice 情况下,对字节数组进行特殊判断
var (
byteData []byte
ok bool
err error
)
if byteData, ok = f.data.([]byte); ok {
// 字节数组转map或者slice
if err = json.Unmarshal(byteData, &f.data); nil != err {
return false
}
return true
}
return true
case reflect.Map:
return true
case reflect.Struct:
// 结构体转为字符串处理
fallthrough
case reflect.Ptr:
// 指针
var err error
if f.data, err = util.Struct.ToMap(f.data); nil != err {
return false
}
return true
default:
return false
}
}

294
tool/gabs.go Normal file
View File

@ -0,0 +1,294 @@
// Package json_tool ...
//
// Description : json_tool ...
//
// Author : go_developer@163.com<张德满>
//
// Date : 2022/01/22 9:19 PM
package json_tool
import (
"encoding/json"
"fmt"
"reflect"
"strings"
"git.zhangdeman.cn/zhangdeman/util"
"github.com/pkg/errors"
"github.com/tidwall/gjson"
"github.com/Jeffail/gabs"
)
// FilterDataRule 参数过滤规则
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2022/1/22 9:44 PM
type FilterDataRule struct {
SourceKey string // 原始数据路径
MapKey string // 提取后映射到的数据路径
DefaultValue interface{} // 原始数据路径不存在时的默认值
WithDefault bool // 是否使用默认值
}
// NewDataFilter 获取数据过滤方法实例
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2022/1/22 9:50 PM
func NewDataFilter(source string, filterRule []*FilterDataRule) *DataFilter {
return &DataFilter{
source: source,
filterRule: filterRule,
hasDealDiffPath: make(map[string]string),
}
}
// DataFilter 数据过滤
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2022/1/22 9:20 PM
type DataFilter struct {
source string
filterRule []*FilterDataRule
itemKeyToSlice bool
hasDealDiffPath map[string]string
}
// Filter 数据过滤
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2022/1/22 9:36 PM
func (df *DataFilter) Filter() (string, error) {
var (
jsonObject *gabs.Container
err error
)
// 格式化映射规则
df.formatRule()
// 记录 obj => slice 的数据类型
obg2slice := make(map[string]string)
// 创建数据的根结点
jsonObject = gabs.New()
for _, item := range df.filterRule {
// 数据源路径不识数组, 多个key写入到同一个map key, 并且map key 不是以[]结尾, 自动格式化
// 目标位置, 是一个数组
if df.pathIsArrayValue(item.MapKey) {
realMapKey := strings.Trim(item.MapKey, ".[]")
if exist := jsonObject.Exists(realMapKey); !exist {
if _, err = jsonObject.ArrayP(realMapKey); nil != err {
return "", err
}
}
valueResult := gjson.Get(df.source, item.SourceKey)
dataType := df.getValueType(valueResult)
if _, exist := obg2slice[realMapKey]; !exist {
obg2slice[realMapKey] = dataType
}
if dataType != obg2slice[realMapKey] {
return "", errors.New(realMapKey + " 预期写入的字段数据类型不一致")
}
sourcePathArr := strings.Split(item.SourceKey, ".[].")
mapPathArr := strings.Split(realMapKey, ".[].")
result := gabs.New()
_, _ = result.ArrayP(mapPathArr[0])
df.SetArrayData("{\""+sourcePathArr[0]+"\":"+gjson.Get(df.source, sourcePathArr[0]).String()+"}", result, sourcePathArr, mapPathArr)
if err = jsonObject.ArrayAppend(valueResult.Value(), realMapKey); nil != err {
return "", err
}
continue
}
sourceSearchResult := gjson.Get(df.source, item.SourceKey)
if !sourceSearchResult.Exists() {
if item.WithDefault {
if _, err = jsonObject.SetP(item.DefaultValue, item.MapKey); nil != err {
return "", err
}
}
continue
}
if _, err = jsonObject.SetP(sourceSearchResult.Value(), item.MapKey); nil != err {
return "", err
}
}
return jsonObject.String(), nil
}
// UserItemToSlice 支持多个独立的字段合并到slice中
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 3:27 PM 2022/1/24
func (df *DataFilter) UserItemToSlice() {
df.itemKeyToSlice = true
}
// getValueType 获取数据类型
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2022/1/23 12:45 AM
func (df *DataFilter) getValueType(valueResult gjson.Result) string {
dataTypeVal := reflect.TypeOf(valueResult.Value())
if nil == dataTypeVal {
return "NIL"
}
dataType := dataTypeVal.String()
if strings.Contains(dataType, "int") {
return "int64"
}
if strings.Contains(dataType, "float") {
return "float64"
}
return dataType
}
// pathIsArrayValue 判断路径是否为数组值
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2022/1/23 12:56 AM
func (df *DataFilter) pathIsArrayValue(path string) bool {
return strings.Contains(path, "[]")
}
// formatRule 格式化映射规则
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2:43 PM 2022/1/24
func (df *DataFilter) formatRule() {
mapKeyCnt := make(map[string]int)
for _, item := range df.filterRule {
// source 为数组, map 不是
if df.pathIsArrayValue(item.SourceKey) {
if !df.pathIsArrayValue(item.MapKey) {
item.MapKey = item.MapKey + ".[]"
} else {
// source 是数组, map也是数组, 检测数组层级匹配
sourcePathArr := strings.Split(item.SourceKey, ".[].")
mapPathArr := strings.Split(item.MapKey, ".[].")
if len(sourcePathArr) == len(mapPathArr) {
// 数组层级深度相同,无需特殊处理
continue
}
// 数组层级深度不同,重新对对齐数据
diffArr := sourcePathArr[0 : len(sourcePathArr)-len(mapPathArr)+1]
if newPath := df.dealDiffArr(diffArr); len(newPath) > 0 {
sourcePathArr[len(sourcePathArr)-len(mapPathArr)] = newPath
item.SourceKey = strings.Join(sourcePathArr[len(sourcePathArr)-len(mapPathArr):], ".[].")
}
}
} else {
if df.pathIsArrayValue(item.MapKey) {
continue
}
// source 不是数组, map 也不是
if !df.itemKeyToSlice {
continue
}
mapKeyCnt[item.MapKey]++
}
}
// 多个source指向一个map,自动转化为list
for _, item := range df.filterRule {
if mapKeyCnt[item.MapKey] > 1 {
item.MapKey = item.MapKey + ".[]"
}
}
}
// dealDiffArr 提取数据映射关系
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 5:04 下午 2022/1/25
func (df *DataFilter) dealDiffArr(diffArr []string) string {
if len(diffArr) == 0 {
return ""
}
diffArrStr := strings.Join(diffArr, ".[].")
if _, exist := df.hasDealDiffPath[diffArrStr]; exist {
// 已经处理过, 不再重复处理
return df.hasDealDiffPath[diffArrStr]
}
// 没处理过, 开始处理
jsonResultList := df.getArrayData(df.source, diffArr)
if len(jsonResultList) == 0 {
return ""
}
newPath := util.String.GenRandomString("", 8)
var result map[string]interface{}
_ = json.Unmarshal([]byte(df.source), &result)
JSONObject, _ := gabs.Consume(result)
_, _ = JSONObject.ArrayP(newPath)
for _, item := range jsonResultList {
if err := JSONObject.ArrayAppendP(item.Value(), newPath); nil != err {
fmt.Println(err.Error())
}
}
df.source = JSONObject.String()
df.hasDealDiffPath[diffArrStr] = newPath
return newPath
}
// getArrayData 获取数据
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2:22 下午 2022/1/26
func (df *DataFilter) getArrayData(source string, pathArr []string) []gjson.Result {
if len(pathArr) == 1 {
return gjson.Get(source, pathArr[0]).Array()
}
resultList := make([]gjson.Result, 0)
dataList := gjson.Get(source, pathArr[0]).Array()
for idx := 0; idx < len(dataList); idx++ {
resultList = append(resultList, df.getArrayData(dataList[idx].String(), pathArr[1:])...)
}
return resultList
}
// SetArrayData 设置数组数据
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 5:05 下午 2022/2/2
func (df *DataFilter) SetArrayData(sourceData string, jsonObject *gabs.Container, sourcePathArr []string, mapPathArr []string) *gabs.Container {
jsonObject = gabs.New()
for idx, sourcePath := range sourcePathArr {
if idx < len(sourcePathArr)-1 {
if !jsonObject.Exists(sourcePath) {
_, _ = jsonObject.ArrayP(sourcePath)
}
}
instance, _ := gabs.ParseJSON([]byte(sourceData))
if !instance.Exists() {
fmt.Println(sourcePathArr[len(sourcePathArr)-1] + " 不存在")
} else {
dataList, _ := instance.Children()
for _, item := range dataList {
cItem := gabs.New()
cItem.SetP(gjson.Get(item.String(), sourcePath).String(), mapPathArr[idx])
jsonObject.ArrayAppendP(cItem.Data(), mapPathArr[idx])
}
//jsonObject.ArrayAppend(jsonObject.Data())
// fmt.Println("数据 : ", jsonObject.String())
// jsonObject.ArrayAppendP(result.Data(), mapPathArr[idx])
}
df.SetArrayData(gjson.Get(sourceData, sourcePathArr[idx]).String(), jsonObject, sourcePathArr[idx+1:], mapPathArr[idx+1:])
// jsonObject.ArrayAppendP(v.Data(), mapPathArr[idx])
}
fmt.Println("最终 : ", jsonObject.String())
return jsonObject
}

277
tool/json_test.go Normal file
View File

@ -0,0 +1,277 @@
// Package json_tool...
//
// Description : json_tool...
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2021-03-10 11:44 下午
package json_tool
import (
"encoding/json"
"fmt"
"testing"
)
// TestJSON ...
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 10:58 下午 2021/3/14
func TestJSON(t *testing.T) {
tree := NewDynamicJSON()
fmt.Println(tree.extraSliceIndex("[200]"))
tree.SetValue("extra.height.value", 180, false)
tree.SetValue("extra.height.unit.use", "cm", false)
tree.SetValue("extra.height.unit.open", "1", false)
tree.SetValue("name", "zhangdeman", false)
tree.SetValue("good.name", "good", false)
tree.SetValue("work", "111", false)
tree.SetValue("good.price", "180", false)
tree.SetValue("good.unit", "$", false)
tree.SetValue("slice.[0].name", "zhang", false)
tree.SetValue("slice.[0].age", 30, false)
tree.SetValue("slice.[1].name", "de", false)
tree.SetValue("slice.[2].name", "man", false)
tree.SetValue("slice.[3]", "zhangdeman", false)
fmt.Println(tree.String())
tree = NewDynamicJSON()
tree.SetValue("[0]", "zhang", false)
tree.SetValue("[1]", "de", false)
tree.SetValue("[2]", "man", false)
fmt.Println(tree.String())
}
// TestType 判断数据类型断言
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 10:59 下午 2021/3/14
func TestType(t *testing.T) {
}
// TestSelect 测试动态选择字段
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 9:47 下午 2021/4/13
func TestSelect(t *testing.T) {
source := map[string]interface{}{
"name": "zhangdeman",
"extra": map[string]interface{}{
"age": 18,
"height": 180,
"slice": []int{1, 2, 3},
},
"slice_data": []int{1, 2, 3},
"map": map[string]interface{}{"a": 1, "b": 2, "c": 4},
"table": []map[string]interface{}{
{"name": "alex", "age": 18, "number": 1},
{"name": "bob", "age": 28, "number": 2},
{"name": "bob", "age": 28, "number": 2, "list": []int{1, 2, 3}},
},
}
rule := map[string]MapDataRule{
"name": {MapKey: "user_name", DefaultValue: "用户姓名默认值", IsComplexType: false},
"extra.age": {MapKey: "user_age", DefaultValue: "用户年龄默认值", IsComplexType: false},
"extra.height": {MapKey: "user_height", DefaultValue: "扩展高度默认值", IsComplexType: false},
"table.[].name": {MapKey: "slice.[].name_modify", DefaultValue: "列表姓名默认值", IsComplexType: false},
"table.[].list": {MapKey: "slice.[].data_list", DefaultValue: "[\"567\",\"678\",\"789\"]", IsComplexType: true},
}
filter := NewFilter(source, rule)
d, e := filter.Result()
if nil != e {
fmt.Println(e)
return
}
fmt.Println(d.String())
}
// TestParse 测试获取JSON数据结构
//
// Author : go_developer@163.com<张德满>
//
// Date : 10:59 PM 2022/1/9
func TestParse(t *testing.T) {
source := map[string]interface{}{
"name": "zhangdeman",
"extra": map[string]interface{}{
"age": 18,
"height": 180,
"slice": []int{1, 2, 3},
"obj": map[string]interface{}{
"la": "aaaa",
},
},
"slice": []int{1, 2, 3},
"map": map[string]interface{}{"a": 1, "b": 2, "c": 4},
"table": []map[string]interface{}{
{"name": "alex", "age": 18, "number": 1, "obj": map[string]interface{}{"enen": "en"}},
{"name": "bob", "age": 28, "number": 2},
},
"two_slice": []map[string]interface{}{
{
"students": []map[string]interface{}{
{
"name": "enen",
"age": 18,
"score": []float64{1, 2, 3, 45},
},
},
"other": []interface{}{"others"},
"read_only": 1,
},
},
}
byteData, _ := json.Marshal(source)
fmt.Println(GetJSONDataStruct(string(byteData)))
}
// TestParseWithType 测试获取JSON数据结构
//
// Author : go_developer@163.com<张德满>
//
// Date : 10:59 PM 2022/1/9
func TestParseWithType(t *testing.T) {
source := map[string]interface{}{
"name": "zhangdeman",
"extra": map[string]interface{}{
"age": 18,
"height": 180,
"slice": []int{1, 2, 3},
"obj": map[string]interface{}{
"la": "aaaa",
},
},
"slice": []int{1, 2, 3},
"map": map[string]interface{}{"a": 1, "d": 5.5, "e": "qqq"},
"empty_obj": map[string]interface{}{},
"empty_list": make([]interface{}, 0),
"table": []map[string]interface{}{
{"name": "alex", "age": 18, "number": 1, "obj": map[string]interface{}{"enen": "en"}},
{"name": "bob", "age": 28, "number": 2},
},
"two_slice": []map[string]interface{}{
{
"students": []map[string]interface{}{
{
"name": "enen",
"age": 18,
"score": []float64{1, 2, 3, 45},
},
},
"other": []interface{}{"others"},
"read_only": 1,
},
},
}
byteData, _ := json.Marshal(source)
fmt.Println(GetJSONDataStructWithType(string(byteData)))
}
// TestDataFilter 测试数据过滤
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2022/1/22 10:19 PM
func TestDataFilter(t *testing.T) {
source := map[string]interface{}{
"name": "zhangdeman",
"extra": map[string]interface{}{
"age": 18,
"height": 180,
"slice": []int{1, 2, 3},
},
"slice_data": []int{1, 2, 3},
"map": map[string]interface{}{"a": 1, "b": 2, "c": 4},
"table": []map[string]interface{}{
{"name": "alex", "age": 18, "number": 1},
{"name": "bob", "age": 28, "number": 2},
{"name": "bob", "age": 28, "number": 2, "list": []int{1, 2, 3}},
},
}
rule := []*FilterDataRule{
{SourceKey: "name", MapKey: "user_name", DefaultValue: "用户姓名默认值"},
{SourceKey: "name", MapKey: "username", DefaultValue: "用户姓名默认值"},
{SourceKey: "name", MapKey: "user.name", DefaultValue: "用户姓名默认值"},
{SourceKey: "extra.age", MapKey: "user.age", DefaultValue: "用户年龄默认值"},
{SourceKey: "extra.age", MapKey: "user_age", DefaultValue: "用户年龄默认值"},
{SourceKey: "extra.height", MapKey: "user.height", DefaultValue: "扩展高度默认值"},
{SourceKey: "extra.height", MapKey: "user_height", DefaultValue: "扩展高度默认值"},
{SourceKey: "table.[].name", MapKey: "slice.[].name_modify", DefaultValue: "列表姓名默认值"},
{SourceKey: "table.[].list", MapKey: "slice.[].data_list", DefaultValue: "[\"567\",\"678\",\"789\"]"},
}
byteData, _ := json.Marshal(source)
filter := NewDataFilter(string(byteData), rule)
fmt.Println(filter.Filter())
}
// TestDataFilterForObiToSlice ...
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2022/1/23 12:06 AM
func TestDataFilterForObiToSlice(t *testing.T) {
source := map[string]interface{}{
"name": "zhangdeman",
"age": 18,
"height": 180,
"extra": map[string]interface{}{
"age": 18,
"height": 180,
"slice": []int{1, 2, 3},
},
"slice_data": []int{1, 2, 3},
"map": map[string]interface{}{"a": 1, "b": 2, "c": 4},
"table": []map[string]interface{}{
{"name": "alex", "age": 18, "number": 1},
{"name": "bob", "age": 28, "number": 2},
{"name": "bob", "age": 28, "number": 2, "list": []int{1, 2, 3}},
},
}
rule := []*FilterDataRule{
// {SourceKey: "name", MapKey: "slice.[]", DefaultValue: "用户姓名默认值"},
{SourceKey: "age", MapKey: "slice", DefaultValue: "用户姓名默认值"},
{SourceKey: "height", MapKey: "slice", DefaultValue: "用户姓名默认值"},
}
byteData, _ := json.Marshal(source)
filter := NewDataFilter(string(byteData), rule)
filter.UserItemToSlice()
fmt.Println(filter.Filter())
}
// TestDataFilterDiffArr ...
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 12:27 下午 2022/1/26
func TestDataFilterDiffArr(t *testing.T) {
source := map[string]interface{}{
"name": "zhangdeman",
"age": 18,
"height": 180,
"extra": map[string]interface{}{
"age": 18,
"height": 180,
"slice": []int{1, 2, 3},
},
"slice_data": []int{1, 2, 3},
"map": map[string]interface{}{"a": 1, "b": 2, "c": 4},
"table": []map[string]interface{}{
{"user_list": []interface{}{map[string]interface{}{"name": "alex", "age": 18, "number": 1}}},
{"user_list": []interface{}{map[string]interface{}{"name": "bob", "age": 28, "number": 2}}},
{"user_list": []interface{}{map[string]interface{}{"name": "andy", "age": 28, "number": 2}}},
},
}
rule := []*FilterDataRule{
// {SourceKey: "name", MapKey: "slice.[]", DefaultValue: "用户姓名默认值"},
{SourceKey: "table.[].user_list.[].name", MapKey: "user_list.[].detail.name", DefaultValue: "用户姓名默认值"},
{SourceKey: "table.[].user_list.[].age", MapKey: "user_list.[].detail.age", DefaultValue: "用户姓名默认值"},
}
byteData, _ := json.Marshal(source)
filter := NewDataFilter(string(byteData), rule)
filter.UserItemToSlice()
filter.Filter()
//fmt.Println()
}

217
tool/parse.go Normal file
View File

@ -0,0 +1,217 @@
// Package json_tool ...
//
// Description : json_tool ...
//
// Author : go_developer@163.com<张德满>
//
// Date : 2022-01-09 10:48 PM
package json_tool
import (
"strings"
"github.com/pkg/errors"
"github.com/tidwall/gjson"
)
// Field ...
//
// Author : go_developer@163.com<张德满>
//
// Date : 2022/1/10 10:47 PM
type Field struct {
Path string `json:"path"` // 路径
Type string `json:"type"` // 类型
Example string `json:"example"` // 示例值
}
const (
// FieldTypeInt ...
FieldTypeInt = "int64"
// FieldTypeIntSlice ...
FieldTypeIntSlice = "[]int64"
// FieldTypeFloat ...
FieldTypeFloat = "float64"
// FieldTypeFloatSlice ...
FieldTypeFloatSlice = "[]float64"
// FieldTypeBool ...
FieldTypeBool = "bool"
// FieldTypeBoolSlice ...
FieldTypeBoolSlice = "[]bool"
// FieldTypeString ...
FieldTypeString = "string"
// FieldTypeStringSLice ...
FieldTypeStringSLice = "string"
// FieldTypeAny ...
FieldTypeAny = "interface{}"
// FieldTypeAnySlice ...
FieldTypeAnySlice = "[]interface{}"
// FieldTypeSlice ...
FieldTypeSlice = "[]interface{}"
// FieldTypeMap ...
FieldTypeMap = "map"
// FieldTypeMapSlice ...
FieldTypeMapSlice = "[]map"
)
// GetJSONDataStruct 获取JSON数据的结构
//
// Author : go_developer@163.com<张德满>
//
// Date : 10:53 PM 2022/1/9
func GetJSONDataStruct(data string) ([]string, error) {
if !gjson.Valid(data) {
return make([]string, 0), errors.New("JSON format is invalid")
}
pathList := make([]string, 0)
r := gjson.Parse(data)
r.ForEach(func(key, value gjson.Result) bool {
if value.Value() == nil {
pathList = append(pathList, key.String())
return true
}
if value.IsObject() {
if value.String() == "{}" {
pathList = append(pathList, key.String())
} else {
list, _ := GetJSONDataStruct(value.String())
for _, k := range list {
pathList = append(pathList, key.String()+"."+k)
}
}
}
if value.IsArray() {
dataList := value.Array()
if len(dataList) > 0 {
if !dataList[0].IsObject() && !dataList[0].IsArray() {
pathList = append(pathList, key.String())
} else {
list, _ := GetJSONDataStruct(dataList[0].String())
for _, k := range list {
pathList = append(pathList, key.String()+".[]."+k)
}
}
} else {
pathList = append(pathList, key.String())
}
}
if !value.IsObject() && !value.IsArray() {
pathList = append(pathList, key.String())
}
return true
})
return pathList, nil
}
// GetJSONDataStructWithType 获取数据结构,并获取类型
//
// Author : go_developer@163.com<张德满>
//
// Date : 2022/1/10 10:47 PM
func GetJSONDataStructWithType(data string) ([]Field, error) {
if !gjson.Valid(data) {
return make([]Field, 0), errors.New("JSON format is invalid")
}
pathList := make([]Field, 0)
r := gjson.Parse(data)
r.ForEach(func(key, value gjson.Result) bool {
if value.Value() == nil {
pathList = append(pathList, Field{
Path: key.String(),
Type: FieldTypeAny,
Example: "nil",
})
return true
}
if value.IsObject() {
if value.String() == "{}" {
pathList = append(pathList, Field{
Path: key.String(),
Type: FieldTypeMap,
Example: "{}",
})
} else {
list, _ := GetJSONDataStructWithType(value.String())
for _, field := range list {
pathList = append(pathList, Field{
Path: key.String() + "." + field.Path,
Type: field.Type,
Example: field.Example,
})
}
}
}
if value.IsArray() {
dataList := value.Array()
if len(dataList) > 0 {
if !dataList[0].IsObject() && !dataList[0].IsArray() {
pathList = append(pathList, Field{
Path: key.String(),
Type: "[]" + GetDataType(dataList[0]),
Example: value.String(),
})
} else {
list, _ := GetJSONDataStructWithType(dataList[0].String())
for _, field := range list {
pathList = append(pathList, Field{
Path: key.String() + ".[]." + field.Path,
Type: field.Type,
Example: field.Example,
})
}
}
} else {
pathList = append(pathList, Field{
Path: key.String(),
Type: FieldTypeSlice,
Example: "[]",
})
}
}
if !value.IsObject() && !value.IsArray() {
pathList = append(pathList, Field{
Path: key.String(),
Type: GetDataType(value),
Example: value.String(),
})
}
return true
})
return pathList, nil
}
// GetDataType 获取数据类型
//
// Author : go_developer@163.com<张德满>
//
// Date : 2022/1/10 11:00 PM
func GetDataType(value gjson.Result) string {
switch value.Type.String() {
default:
return FieldTypeAny
case "Null":
return FieldTypeAny
case "False":
return FieldTypeBool
case "True":
return FieldTypeBool
case "Number":
if strings.Contains(value.String(), ".") {
return FieldTypeFloat
}
return FieldTypeInt
case "String":
return FieldTypeString
case "JSON":
if strings.HasPrefix(strings.TrimSpace(value.String()), "[") {
return FieldTypeSlice
}
return FieldTypeMap
}
}