diff --git a/filter.go b/filter.go index 6c39c0f..1558979 100644 --- a/filter.go +++ b/filter.go @@ -12,12 +12,12 @@ import ( "fmt" "strings" + "github.com/tidwall/gjson" "github.com/tidwall/sjson" "errors" "git.zhangdeman.cn/zhangdeman/util" - "github.com/tidwall/gjson" ) // NewFilter 过滤器实例 diff --git a/go.mod b/go.mod index eefbea6..6350181 100644 --- a/go.mod +++ b/go.mod @@ -3,20 +3,22 @@ module git.zhangdeman.cn/zhangdeman/filter go 1.17 require ( - git.zhangdeman.cn/zhangdeman/util v0.0.0-20221021061434-e68b22a6e106 + git.zhangdeman.cn/zhangdeman/easylock v0.0.0-20220627070212-c590a0a1c216 + git.zhangdeman.cn/zhangdeman/util v0.0.0-20230211164227-256094968151 + github.com/Jeffail/gabs v1.4.0 + github.com/pkg/errors v0.9.1 github.com/smartystreets/goconvey v1.7.2 github.com/tidwall/gjson v1.14.4 github.com/tidwall/sjson v1.2.5 ) require ( - github.com/Jeffail/gabs v1.4.0 // indirect github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/go-ini/ini v1.67.0 // indirect github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/smartystreets/assertions v1.2.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect diff --git a/go.sum b/go.sum index 92d1587..b28f983 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,8 @@ -git.zhangdeman.cn/zhangdeman/util v0.0.0-20221021061434-e68b22a6e106 h1:xiiN+rLtBbDGRUbipVuwI1j2iRhuL3ejSm+EnDxzVMk= -git.zhangdeman.cn/zhangdeman/util v0.0.0-20221021061434-e68b22a6e106/go.mod h1:zTir/0IWdK3E7n0GiaogyWHADAQnBtTdl2I6Z2/OPqw= +git.zhangdeman.cn/zhangdeman/easylock v0.0.0-20220627070212-c590a0a1c216 h1:5M6YsHmbTqLDGkqKykeoxrrd5WsAme6I+akn9sgKUZk= +git.zhangdeman.cn/zhangdeman/easylock v0.0.0-20220627070212-c590a0a1c216/go.mod h1:8Jh+q/L0pWAKQy67m8AKIs0WGGrVKypoSZZbBm4nDVc= +git.zhangdeman.cn/zhangdeman/util v0.0.0-20220626081130-cbac0b676fb8/go.mod h1:G2/OKMbEn89d+YUXQtv9Nlh0LGg14pOqDnbOgBTTRXY= +git.zhangdeman.cn/zhangdeman/util v0.0.0-20230211164227-256094968151 h1:j537bRLQL1FlkdXTIaT9Ecjx5eogkPsGiTOWIEFQlc8= +git.zhangdeman.cn/zhangdeman/util v0.0.0-20230211164227-256094968151/go.mod h1:SyRTkOz6gxUVn3S/Qtkf+rhKV0I1ym8lwsT8YjggYFs= github.com/Jeffail/gabs v1.4.0 h1://5fYRRTq1edjfIrQGvdkcd22pkYUrHZ5YC/H2GJVAo= github.com/Jeffail/gabs v1.4.0/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc= github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 h1:OYA+5W64v3OgClL+IrOD63t4i/RW7RqrAVl9LTZ9UqQ= @@ -7,7 +10,8 @@ github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394/go.mod h1:Q8n74mJTIg github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-ini/ini v1.66.6/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= @@ -29,7 +33,6 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/tidwall/gjson v1.14.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= diff --git a/tool/build.go b/tool/build.go new file mode 100644 index 0000000..c6453e1 --- /dev/null +++ b/tool/build.go @@ -0,0 +1,378 @@ +// Package json_tool... +// +// Description : 动态构建json +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 2021-03-10 10:26 下午 +package json_tool + +import ( + "encoding/json" + "fmt" + "regexp" + "strconv" + "strings" + + "git.zhangdeman.cn/zhangdeman/easylock" + "github.com/pkg/errors" +) + +const ( + // PathSplit json_tool 路径的分割符 + PathSplit = "." +) + +// JSONode JSOM节点 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 10:33 下午 2021/3/10 +type JSONode struct { + Key string // json_tool key + Value interface{} // 对应的值 + Child []*JSONode // 子节点 + IsRoot bool // 是否根节点 + IsHasLastBrother bool // 在此之后是否有其他兄弟节点 + IsSlice bool // 是否是list + IsIndexNode bool // 是否是slice的索引节点 + Sort int // 此属性用于　slice解析,保证最终排序是对的 + IsComplex bool // 是否为复杂数据类型 + IsString bool // 是否为字符串 +} + +// NewDynamicJSON 获取JSON实例 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 10:36 下午 2021/3/10 +func NewDynamicJSON() *DynamicJSON { + exp, _ := regexp.Compile(`\[(\d*?)]`) + return &DynamicJSON{ + root: &JSONode{ + Key: "", + Value: nil, + Child: nil, + IsRoot: true, + }, + nodeCnt: 0, + lock: easylock.NewLock(), + sliceExp: exp, + } +} + +// DynamicJSON 动态json +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 11:03 下午 2021/3/10 +type DynamicJSON struct { + root *JSONode // 节点数 + nodeCnt int // 节点数量 + lock easylock.EasyLock // 锁 + sliceExp *regexp.Regexp // 抽取slice索引的正则 +} + +// SetValue 设置节点值,如果节点不存在,创建;如果已存在,更新, 多级key使用, value 必须是基础数据类型, 如果是结构体, 需要继续添加path,多级path使用.分割 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 10:45 下午 2021/3/10 +func (dj *DynamicJSON) SetValue(path string, value interface{}, isComplexType bool) { + pathList := strings.Split(path, PathSplit) + searchRoot := dj.root + parent := dj.root + for keyIndex, key := range pathList { + searchRoot = dj.search(searchRoot, key) + if nil != searchRoot { + // 查询到结果,更新值 + searchRoot.Value = value + parent = searchRoot + continue + } + var val interface{} + if keyIndex == len(pathList)-1 { + val = value + } + _ = dj.createNode(parent, key, val, isComplexType) + if len(parent.Child) > 0 { + searchRoot = parent.Child[len(parent.Child)-1] + parent = parent.Child[len(parent.Child)-1] + } + } +} + +// String 获取字符串的格式JSON +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 2:16 下午 2021/3/11 +func (dj *DynamicJSON) String() string { + tplList := make([]string, 0) + valList := make([]interface{}, 0) + tplListResult, valListResult := dj.buildTpl(dj.root, &tplList, &valList) + return fmt.Sprintf(strings.Join(*tplListResult, ""), *valListResult...) +} + +// Map 转化为map +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 9:59 上午 2021/9/9 +func (dj *DynamicJSON) Map() (map[string]interface{}, error) { + var res map[string]interface{} + err := json.Unmarshal([]byte(dj.String()), &res) + return res, err +} + +// Slice 转化为slice +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 9:59 上午 2021/9/9 +func (dj *DynamicJSON) Slice() ([]interface{}, error) { + var res []interface{} + err := json.Unmarshal([]byte(dj.String()), &res) + return res, err +} + +// ParseWithReceiver 使用指定结构解析 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 10:01 上午 2021/9/9 +func (dj *DynamicJSON) ParseWithReceiver(receiver interface{}) error { + if nil == receiver { + return errors.New("receiver is nil") + } + return json.Unmarshal([]byte(dj.String()), receiver) +} + +// buildTpl 构建json模版与绑定数据 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 4:38 下午 2021/3/11 +func (dj *DynamicJSON) buildTpl(root *JSONode, tplList *[]string, valList *[]interface{}) (*[]string, *[]interface{}) { + if nil == root { + return tplList, valList + } + startSymbol := dj.getStartSymbol(root) + endSymbol := dj.getEndSymbol(root) + valFormat := dj.getValFormat(root) + // key := "\"" + root.Key + "\"" + if !root.IsIndexNode { + if len(root.Child) > 0 { + *tplList = append(*tplList, startSymbol) + } else { + *tplList = append(*tplList, valFormat) + *valList = append(*valList, root.Value) + return tplList, valList + } + } else { + if len(root.Child) == 0 { + *tplList = append(*tplList, valFormat) + *valList = append(*valList, root.Value) + } else { + *tplList = append(*tplList, startSymbol) + } + } + for _, node := range root.Child { + dj.buildTpl(node, tplList, valList) + } + *tplList = append(*tplList, endSymbol) + + return tplList, valList +} + +// getValFormat 构建值得占位符 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 12:49 下午 2021/3/13 +func (dj *DynamicJSON) getValFormat(root *JSONode) string { + key := fmt.Sprintf("\"%s\"", root.Key) + if !root.IsIndexNode { + if len(root.Child) > 0 { + // 还有自节点的情况下,不需要占位符 + return "" + } + + if root.IsHasLastBrother { + if root.IsString { + return key + ":\"%v\"," + } + return key + ":%v," + } + if root.IsString { + return key + ":\"%v\"" + } + return key + ":%v" + } + + if len(root.Child) > 0 { + // 是list的索引节点,且有子节点 + return "" + } + if root.IsHasLastBrother { + if root.IsString { + return "\"%v\"," + } + return "%v," + } + if root.IsString { + return "\"%v\"" + } + return "%v" +} + +// getStartSymbol 计算起始的符号 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 12:21 下午 2021/3/13 +func (dj *DynamicJSON) getStartSymbol(root *JSONode) string { + if nil == root { + return "{" + } + + if root.IsRoot { + if root.IsSlice { + return "[" + } + return "{" + } + key := fmt.Sprintf("\"%s\"", root.Key) + if !root.IsIndexNode { + if len(root.Child) > 0 { + + if root.IsSlice { + return key + ":[" + } else { + return key + ":{" + } + + } + return "" + } + if len(root.Child) > 0 { + return "{" + } + return "" +} + +// getEndSymbol 计算结束的符号 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 12:21 下午 2021/3/13 +func (dj *DynamicJSON) getEndSymbol(root *JSONode) string { + if nil == root { + return "}" + } + if !root.IsIndexNode { + if root.IsHasLastBrother { + return "}," + } + if root.IsSlice { + return "]" + } else { + return "}" + } + + } + if len(root.Child) > 0 { + if root.IsHasLastBrother { + return "}," + } + return "}" + + } + return "" +} + +// Search 搜索一个key TODO : 优化 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 11:19 下午 2021/3/10 +func (dj *DynamicJSON) search(root *JSONode, key string) *JSONode { + if root == nil { + return nil + } + for _, node := range root.Child { + if node == nil { + continue + } + if node.Key == key { + return node + } + } + return nil +} + +// createNode 创建新的节点 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 10:57 下午 2021/3/10 +func (dj *DynamicJSON) createNode(parent *JSONode, key string, value interface{}, isComplexType bool) error { + if nil == parent { + return errors.New("create node error : parent id nil") + } + _ = dj.lock.Lock() + if parent.Child == nil { + parent.Child = make([]*JSONode, 0) + } + if len(parent.Child) > 0 { + // 存在子节点，设置当前子节点还有其他兄弟节点 + parent.Child[len(parent.Child)-1].IsHasLastBrother = true + } + + newNode := &JSONode{ + Key: key, + Value: value, + Child: make([]*JSONode, 0), + IsRoot: false, + IsHasLastBrother: false, + IsComplex: isComplexType, + IsString: false, + } + if !isComplexType { + switch value.(type) { + case string: + newNode.IsString = true + } + } + parent.IsSlice, newNode.Sort = dj.extraSliceIndex(key) + if parent.IsSlice { + newNode.IsIndexNode = true + } + parent.Child = append(parent.Child, newNode) + dj.nodeCnt++ + _ = dj.lock.Unlock() + return nil +} + +// extraSliceIndex 抽取slice索引 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 9:37 下午 2021/3/11 +func (dj *DynamicJSON) extraSliceIndex(key string) (bool, int) { + if len(key) < 3 { + // slice 至少是 [1] 格式 + return false, 0 + } + + if !strings.HasPrefix(key, "[") || !strings.HasSuffix(key, "]") { + return false, 0 + } + // 不用正则,直接字符串处理 + strByte := []byte(key) + index, err := strconv.Atoi(string(strByte[1 : len(strByte)-1])) + if nil != err { + return false, 0 + } + return true, index +} diff --git a/tool/filter.go b/tool/filter.go new file mode 100644 index 0000000..bf717b9 --- /dev/null +++ b/tool/filter.go @@ -0,0 +1,145 @@ +// Package json_tool ... +// +// Description : 将复杂数据结构转化为 JSONNode 树 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 2021-03-14 10:40 下午 +package json_tool + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + + "github.com/tidwall/gjson" + + "github.com/pkg/errors" + + "git.zhangdeman.cn/zhangdeman/util" +) + +// MapDataRule 数据映射结果 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 5:09 PM 2022/1/17 +type MapDataRule struct { + MapKey string + DefaultValue string + IsComplexType bool +} + +// NewFilter 获取解析的实例 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 10:43 下午 2021/3/14 +func NewFilter(data interface{}, rule map[string]MapDataRule) *Filter { + return &Filter{ + data: data, + rule: rule, + } +} + +// Filter 解析json树 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 10:41 下午 2021/3/14 +type Filter struct { + data interface{} + rule map[string]MapDataRule +} + +// Result 数据过滤结果 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 10:44 下午 2021/3/14 +func (f *Filter) Result() (*DynamicJSON, error) { + if !f.isLegalData() { + return nil, errors.New("非法的数据,无法转换成json") + } + result := NewDynamicJSON() + byteData, _ := json.Marshal(f.data) + source := string(byteData) + for extraDataPath, newDataRule := range f.rule { + // 为数组的处理 + pathArr := strings.Split(extraDataPath, ".[].") + val := gjson.Get(source, pathArr[0]) + if len(pathArr) == 1 { + f.SetValue(result, newDataRule.MapKey, val.Value(), false, 0) + continue + } + // 支持list再抽取一层,处于性能考虑,这么限制,不做递归无限深度处理 + // 还能继续抽取,就认为是 []map[string]interface{} + keyList := strings.Split(pathArr[1], ".") + for idx, item := range val.Array() { + data := item.Map() + for _, key := range keyList { + if _, exist := data[key]; exist { + result.SetValue(strings.ReplaceAll(newDataRule.MapKey, "[]", fmt.Sprintf("[%d]", idx)), data[key].String(), newDataRule.IsComplexType) + } else { + // 结果集中不存在对应key,设置默认值 + result.SetValue(strings.ReplaceAll(newDataRule.MapKey, "[]", fmt.Sprintf("[%d]", idx)), newDataRule.DefaultValue, newDataRule.IsComplexType) + } + } + } + } + return result, nil +} + +// SetValue 设置值 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 12:14 下午 2021/9/10 +func (f *Filter) SetValue(result *DynamicJSON, path string, val interface{}, isSetSlice bool, sliceIndex int) { + if !isSetSlice { + result.SetValue(path, val, false) + return + } +} + +// isLegalData 判断是否能转换成json结构, 只有slice/map/struct/能转换成slice或map的[]byte是合法的 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 10:46 下午 2021/3/14 +func (f *Filter) isLegalData() bool { + val := reflect.ValueOf(f.data) + + switch val.Kind() { + case reflect.Slice: + // slice 情况下,对字节数组进行特殊判断 + var ( + byteData []byte + ok bool + err error + ) + if byteData, ok = f.data.([]byte); ok { + // 字节数组转map或者slice + if err = json.Unmarshal(byteData, &f.data); nil != err { + return false + } + return true + } + return true + case reflect.Map: + return true + case reflect.Struct: + // 结构体转为字符串处理 + fallthrough + case reflect.Ptr: + // 指针 + var err error + if f.data, err = util.Struct.ToMap(f.data); nil != err { + return false + } + return true + default: + return false + } +} diff --git a/tool/gabs.go b/tool/gabs.go new file mode 100644 index 0000000..2fb694a --- /dev/null +++ b/tool/gabs.go @@ -0,0 +1,294 @@ +// Package json_tool ... +// +// Description : json_tool ... +// +// Author : go_developer@163.com<张德满> +// +// Date : 2022/01/22 9:19 PM +package json_tool + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + + "git.zhangdeman.cn/zhangdeman/util" + "github.com/pkg/errors" + + "github.com/tidwall/gjson" + + "github.com/Jeffail/gabs" +) + +// FilterDataRule 参数过滤规则 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 2022/1/22 9:44 PM +type FilterDataRule struct { + SourceKey string // 原始数据路径 + MapKey string // 提取后映射到的数据路径 + DefaultValue interface{} // 原始数据路径不存在时的默认值 + WithDefault bool // 是否使用默认值 +} + +// NewDataFilter 获取数据过滤方法实例 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 2022/1/22 9:50 PM +func NewDataFilter(source string, filterRule []*FilterDataRule) *DataFilter { + return &DataFilter{ + source: source, + filterRule: filterRule, + hasDealDiffPath: make(map[string]string), + } +} + +// DataFilter 数据过滤 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 2022/1/22 9:20 PM +type DataFilter struct { + source string + filterRule []*FilterDataRule + itemKeyToSlice bool + hasDealDiffPath map[string]string +} + +// Filter 数据过滤 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 2022/1/22 9:36 PM +func (df *DataFilter) Filter() (string, error) { + var ( + jsonObject *gabs.Container + err error + ) + // 格式化映射规则 + df.formatRule() + // 记录 obj => slice 的数据类型 + obg2slice := make(map[string]string) + // 创建数据的根结点 + jsonObject = gabs.New() + for _, item := range df.filterRule { + // 数据源路径不识数组, 多个key写入到同一个map key, 并且map key 不是以[]结尾, 自动格式化 + // 目标位置, 是一个数组 + if df.pathIsArrayValue(item.MapKey) { + realMapKey := strings.Trim(item.MapKey, ".[]") + if exist := jsonObject.Exists(realMapKey); !exist { + if _, err = jsonObject.ArrayP(realMapKey); nil != err { + return "", err + } + } + valueResult := gjson.Get(df.source, item.SourceKey) + dataType := df.getValueType(valueResult) + if _, exist := obg2slice[realMapKey]; !exist { + obg2slice[realMapKey] = dataType + } + if dataType != obg2slice[realMapKey] { + return "", errors.New(realMapKey + " 预期写入的字段数据类型不一致") + } + sourcePathArr := strings.Split(item.SourceKey, ".[].") + mapPathArr := strings.Split(realMapKey, ".[].") + + result := gabs.New() + _, _ = result.ArrayP(mapPathArr[0]) + df.SetArrayData("{\""+sourcePathArr[0]+"\":"+gjson.Get(df.source, sourcePathArr[0]).String()+"}", result, sourcePathArr, mapPathArr) + if err = jsonObject.ArrayAppend(valueResult.Value(), realMapKey); nil != err { + return "", err + } + continue + } + sourceSearchResult := gjson.Get(df.source, item.SourceKey) + if !sourceSearchResult.Exists() { + if item.WithDefault { + if _, err = jsonObject.SetP(item.DefaultValue, item.MapKey); nil != err { + return "", err + } + } + continue + } + if _, err = jsonObject.SetP(sourceSearchResult.Value(), item.MapKey); nil != err { + return "", err + } + } + return jsonObject.String(), nil +} + +// UserItemToSlice 支持多个独立的字段合并到slice中 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 3:27 PM 2022/1/24 +func (df *DataFilter) UserItemToSlice() { + df.itemKeyToSlice = true +} + +// getValueType 获取数据类型 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 2022/1/23 12:45 AM +func (df *DataFilter) getValueType(valueResult gjson.Result) string { + dataTypeVal := reflect.TypeOf(valueResult.Value()) + if nil == dataTypeVal { + return "NIL" + } + dataType := dataTypeVal.String() + if strings.Contains(dataType, "int") { + return "int64" + } + if strings.Contains(dataType, "float") { + return "float64" + } + return dataType +} + +// pathIsArrayValue 判断路径是否为数组值 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 2022/1/23 12:56 AM +func (df *DataFilter) pathIsArrayValue(path string) bool { + return strings.Contains(path, "[]") +} + +// formatRule 格式化映射规则 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 2:43 PM 2022/1/24 +func (df *DataFilter) formatRule() { + mapKeyCnt := make(map[string]int) + for _, item := range df.filterRule { + // source 为数组, map 不是 + if df.pathIsArrayValue(item.SourceKey) { + if !df.pathIsArrayValue(item.MapKey) { + item.MapKey = item.MapKey + ".[]" + } else { + // source 是数组, map也是数组, 检测数组层级匹配 + sourcePathArr := strings.Split(item.SourceKey, ".[].") + mapPathArr := strings.Split(item.MapKey, ".[].") + if len(sourcePathArr) == len(mapPathArr) { + // 数组层级深度相同,无需特殊处理 + continue + } + // 数组层级深度不同,重新对对齐数据 + diffArr := sourcePathArr[0 : len(sourcePathArr)-len(mapPathArr)+1] + if newPath := df.dealDiffArr(diffArr); len(newPath) > 0 { + sourcePathArr[len(sourcePathArr)-len(mapPathArr)] = newPath + item.SourceKey = strings.Join(sourcePathArr[len(sourcePathArr)-len(mapPathArr):], ".[].") + } + } + } else { + if df.pathIsArrayValue(item.MapKey) { + continue + } + // source 不是数组, map 也不是 + if !df.itemKeyToSlice { + continue + } + mapKeyCnt[item.MapKey]++ + } + } + // 多个source指向一个map,自动转化为list + for _, item := range df.filterRule { + if mapKeyCnt[item.MapKey] > 1 { + item.MapKey = item.MapKey + ".[]" + } + } +} + +// dealDiffArr 提取数据映射关系 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 5:04 下午 2022/1/25 +func (df *DataFilter) dealDiffArr(diffArr []string) string { + if len(diffArr) == 0 { + return "" + } + diffArrStr := strings.Join(diffArr, ".[].") + if _, exist := df.hasDealDiffPath[diffArrStr]; exist { + // 已经处理过, 不再重复处理 + return df.hasDealDiffPath[diffArrStr] + } + + // 没处理过, 开始处理 + jsonResultList := df.getArrayData(df.source, diffArr) + if len(jsonResultList) == 0 { + return "" + } + newPath := util.String.GenRandom("", 8) + var result map[string]interface{} + _ = json.Unmarshal([]byte(df.source), &result) + JSONObject, _ := gabs.Consume(result) + _, _ = JSONObject.ArrayP(newPath) + for _, item := range jsonResultList { + if err := JSONObject.ArrayAppendP(item.Value(), newPath); nil != err { + fmt.Println(err.Error()) + } + } + + df.source = JSONObject.String() + df.hasDealDiffPath[diffArrStr] = newPath + return newPath +} + +// getArrayData 获取数据 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 2:22 下午 2022/1/26 +func (df *DataFilter) getArrayData(source string, pathArr []string) []gjson.Result { + if len(pathArr) == 1 { + return gjson.Get(source, pathArr[0]).Array() + } + resultList := make([]gjson.Result, 0) + dataList := gjson.Get(source, pathArr[0]).Array() + for idx := 0; idx < len(dataList); idx++ { + resultList = append(resultList, df.getArrayData(dataList[idx].String(), pathArr[1:])...) + } + return resultList +} + +// SetArrayData 设置数组数据 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 5:05 下午 2022/2/2 +func (df *DataFilter) SetArrayData(sourceData string, jsonObject *gabs.Container, sourcePathArr []string, mapPathArr []string) *gabs.Container { + jsonObject = gabs.New() + + for idx, sourcePath := range sourcePathArr { + if idx < len(sourcePathArr)-1 { + if !jsonObject.Exists(sourcePath) { + _, _ = jsonObject.ArrayP(sourcePath) + } + } + instance, _ := gabs.ParseJSON([]byte(sourceData)) + if !instance.Exists() { + fmt.Println(sourcePathArr[len(sourcePathArr)-1] + " 不存在") + } else { + dataList, _ := instance.Children() + for _, item := range dataList { + cItem := gabs.New() + cItem.SetP(gjson.Get(item.String(), sourcePath).String(), mapPathArr[idx]) + jsonObject.ArrayAppendP(cItem.Data(), mapPathArr[idx]) + } + //jsonObject.ArrayAppend(jsonObject.Data()) + // fmt.Println("数据 : ", jsonObject.String()) + // jsonObject.ArrayAppendP(result.Data(), mapPathArr[idx]) + + } + + df.SetArrayData(gjson.Get(sourceData, sourcePathArr[idx]).String(), jsonObject, sourcePathArr[idx+1:], mapPathArr[idx+1:]) + // jsonObject.ArrayAppendP(v.Data(), mapPathArr[idx]) + } + fmt.Println("最终 : ", jsonObject.String()) + return jsonObject +} diff --git a/tool/json_test.go b/tool/json_test.go new file mode 100644 index 0000000..b655dd5 --- /dev/null +++ b/tool/json_test.go @@ -0,0 +1,277 @@ +// Package json_tool... +// +// Description : json_tool... +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 2021-03-10 11:44 下午 +package json_tool + +import ( + "encoding/json" + "fmt" + "testing" +) + +// TestJSON ... +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 10:58 下午 2021/3/14 +func TestJSON(t *testing.T) { + tree := NewDynamicJSON() + fmt.Println(tree.extraSliceIndex("[200]")) + tree.SetValue("extra.height.value", 180, false) + tree.SetValue("extra.height.unit.use", "cm", false) + tree.SetValue("extra.height.unit.open", "1", false) + tree.SetValue("name", "zhangdeman", false) + tree.SetValue("good.name", "good", false) + tree.SetValue("work", "111", false) + tree.SetValue("good.price", "180", false) + tree.SetValue("good.unit", "$", false) + tree.SetValue("slice.[0].name", "zhang", false) + tree.SetValue("slice.[0].age", 30, false) + tree.SetValue("slice.[1].name", "de", false) + tree.SetValue("slice.[2].name", "man", false) + tree.SetValue("slice.[3]", "zhangdeman", false) + fmt.Println(tree.String()) + tree = NewDynamicJSON() + tree.SetValue("[0]", "zhang", false) + tree.SetValue("[1]", "de", false) + tree.SetValue("[2]", "man", false) + fmt.Println(tree.String()) +} + +// TestType 判断数据类型断言 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 10:59 下午 2021/3/14 +func TestType(t *testing.T) { + +} + +// TestSelect 测试动态选择字段 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 9:47 下午 2021/4/13 +func TestSelect(t *testing.T) { + source := map[string]interface{}{ + "name": "zhangdeman", + "extra": map[string]interface{}{ + "age": 18, + "height": 180, + "slice": []int{1, 2, 3}, + }, + "slice_data": []int{1, 2, 3}, + "map": map[string]interface{}{"a": 1, "b": 2, "c": 4}, + "table": []map[string]interface{}{ + {"name": "alex", "age": 18, "number": 1}, + {"name": "bob", "age": 28, "number": 2}, + {"name": "bob", "age": 28, "number": 2, "list": []int{1, 2, 3}}, + }, + } + rule := map[string]MapDataRule{ + "name": {MapKey: "user_name", DefaultValue: "用户姓名默认值", IsComplexType: false}, + "extra.age": {MapKey: "user_age", DefaultValue: "用户年龄默认值", IsComplexType: false}, + "extra.height": {MapKey: "user_height", DefaultValue: "扩展高度默认值", IsComplexType: false}, + "table.[].name": {MapKey: "slice.[].name_modify", DefaultValue: "列表姓名默认值", IsComplexType: false}, + "table.[].list": {MapKey: "slice.[].data_list", DefaultValue: "[\"567\",\"678\",\"789\"]", IsComplexType: true}, + } + filter := NewFilter(source, rule) + d, e := filter.Result() + if nil != e { + fmt.Println(e) + return + } + fmt.Println(d.String()) +} + +// TestParse 测试获取JSON数据结构 +// +// Author : go_developer@163.com<张德满> +// +// Date : 10:59 PM 2022/1/9 +func TestParse(t *testing.T) { + source := map[string]interface{}{ + "name": "zhangdeman", + "extra": map[string]interface{}{ + "age": 18, + "height": 180, + "slice": []int{1, 2, 3}, + "obj": map[string]interface{}{ + "la": "aaaa", + }, + }, + "slice": []int{1, 2, 3}, + "map": map[string]interface{}{"a": 1, "b": 2, "c": 4}, + "table": []map[string]interface{}{ + {"name": "alex", "age": 18, "number": 1, "obj": map[string]interface{}{"enen": "en"}}, + {"name": "bob", "age": 28, "number": 2}, + }, + "two_slice": []map[string]interface{}{ + { + "students": []map[string]interface{}{ + { + "name": "enen", + "age": 18, + "score": []float64{1, 2, 3, 45}, + }, + }, + "other": []interface{}{"others"}, + "read_only": 1, + }, + }, + } + byteData, _ := json.Marshal(source) + fmt.Println(GetJSONDataStruct(string(byteData))) +} + +// TestParseWithType 测试获取JSON数据结构 +// +// Author : go_developer@163.com<张德满> +// +// Date : 10:59 PM 2022/1/9 +func TestParseWithType(t *testing.T) { + source := map[string]interface{}{ + "name": "zhangdeman", + "extra": map[string]interface{}{ + "age": 18, + "height": 180, + "slice": []int{1, 2, 3}, + "obj": map[string]interface{}{ + "la": "aaaa", + }, + }, + "slice": []int{1, 2, 3}, + "map": map[string]interface{}{"a": 1, "d": 5.5, "e": "qqq"}, + "empty_obj": map[string]interface{}{}, + "empty_list": make([]interface{}, 0), + "table": []map[string]interface{}{ + {"name": "alex", "age": 18, "number": 1, "obj": map[string]interface{}{"enen": "en"}}, + {"name": "bob", "age": 28, "number": 2}, + }, + "two_slice": []map[string]interface{}{ + { + "students": []map[string]interface{}{ + { + "name": "enen", + "age": 18, + "score": []float64{1, 2, 3, 45}, + }, + }, + "other": []interface{}{"others"}, + "read_only": 1, + }, + }, + } + byteData, _ := json.Marshal(source) + fmt.Println(GetJSONDataStructWithType(string(byteData))) +} + +// TestDataFilter 测试数据过滤 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 2022/1/22 10:19 PM +func TestDataFilter(t *testing.T) { + source := map[string]interface{}{ + "name": "zhangdeman", + "extra": map[string]interface{}{ + "age": 18, + "height": 180, + "slice": []int{1, 2, 3}, + }, + "slice_data": []int{1, 2, 3}, + "map": map[string]interface{}{"a": 1, "b": 2, "c": 4}, + "table": []map[string]interface{}{ + {"name": "alex", "age": 18, "number": 1}, + {"name": "bob", "age": 28, "number": 2}, + {"name": "bob", "age": 28, "number": 2, "list": []int{1, 2, 3}}, + }, + } + rule := []*FilterDataRule{ + {SourceKey: "name", MapKey: "user_name", DefaultValue: "用户姓名默认值"}, + {SourceKey: "name", MapKey: "username", DefaultValue: "用户姓名默认值"}, + {SourceKey: "name", MapKey: "user.name", DefaultValue: "用户姓名默认值"}, + {SourceKey: "extra.age", MapKey: "user.age", DefaultValue: "用户年龄默认值"}, + {SourceKey: "extra.age", MapKey: "user_age", DefaultValue: "用户年龄默认值"}, + {SourceKey: "extra.height", MapKey: "user.height", DefaultValue: "扩展高度默认值"}, + {SourceKey: "extra.height", MapKey: "user_height", DefaultValue: "扩展高度默认值"}, + {SourceKey: "table.[].name", MapKey: "slice.[].name_modify", DefaultValue: "列表姓名默认值"}, + {SourceKey: "table.[].list", MapKey: "slice.[].data_list", DefaultValue: "[\"567\",\"678\",\"789\"]"}, + } + byteData, _ := json.Marshal(source) + filter := NewDataFilter(string(byteData), rule) + fmt.Println(filter.Filter()) +} + +// TestDataFilterForObiToSlice ... +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 2022/1/23 12:06 AM +func TestDataFilterForObiToSlice(t *testing.T) { + source := map[string]interface{}{ + "name": "zhangdeman", + "age": 18, + "height": 180, + "extra": map[string]interface{}{ + "age": 18, + "height": 180, + "slice": []int{1, 2, 3}, + }, + "slice_data": []int{1, 2, 3}, + "map": map[string]interface{}{"a": 1, "b": 2, "c": 4}, + "table": []map[string]interface{}{ + {"name": "alex", "age": 18, "number": 1}, + {"name": "bob", "age": 28, "number": 2}, + {"name": "bob", "age": 28, "number": 2, "list": []int{1, 2, 3}}, + }, + } + rule := []*FilterDataRule{ + // {SourceKey: "name", MapKey: "slice.[]", DefaultValue: "用户姓名默认值"}, + {SourceKey: "age", MapKey: "slice", DefaultValue: "用户姓名默认值"}, + {SourceKey: "height", MapKey: "slice", DefaultValue: "用户姓名默认值"}, + } + byteData, _ := json.Marshal(source) + filter := NewDataFilter(string(byteData), rule) + filter.UserItemToSlice() + fmt.Println(filter.Filter()) +} + +// TestDataFilterDiffArr ... +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 12:27 下午 2022/1/26 +func TestDataFilterDiffArr(t *testing.T) { + source := map[string]interface{}{ + "name": "zhangdeman", + "age": 18, + "height": 180, + "extra": map[string]interface{}{ + "age": 18, + "height": 180, + "slice": []int{1, 2, 3}, + }, + "slice_data": []int{1, 2, 3}, + "map": map[string]interface{}{"a": 1, "b": 2, "c": 4}, + "table": []map[string]interface{}{ + {"user_list": []interface{}{map[string]interface{}{"name": "alex", "age": 18, "number": 1}}}, + {"user_list": []interface{}{map[string]interface{}{"name": "bob", "age": 28, "number": 2}}}, + {"user_list": []interface{}{map[string]interface{}{"name": "andy", "age": 28, "number": 2}}}, + }, + } + rule := []*FilterDataRule{ + // {SourceKey: "name", MapKey: "slice.[]", DefaultValue: "用户姓名默认值"}, + {SourceKey: "table.[].user_list.[].name", MapKey: "user_list.[].detail.name", DefaultValue: "用户姓名默认值"}, + {SourceKey: "table.[].user_list.[].age", MapKey: "user_list.[].detail.age", DefaultValue: "用户姓名默认值"}, + } + byteData, _ := json.Marshal(source) + filter := NewDataFilter(string(byteData), rule) + filter.UserItemToSlice() + filter.Filter() + //fmt.Println() +} diff --git a/tool/parse.go b/tool/parse.go new file mode 100644 index 0000000..79f231a --- /dev/null +++ b/tool/parse.go @@ -0,0 +1,217 @@ +// Package json_tool ... +// +// Description : json_tool ... +// +// Author : go_developer@163.com<张德满> +// +// Date : 2022-01-09 10:48 PM +package json_tool + +import ( + "strings" + + "github.com/pkg/errors" + "github.com/tidwall/gjson" +) + +// Field ... +// +// Author : go_developer@163.com<张德满> +// +// Date : 2022/1/10 10:47 PM +type Field struct { + Path string `json:"path"` // 路径 + Type string `json:"type"` // 类型 + Example string `json:"example"` // 示例值 +} + +const ( + // FieldTypeInt ... + FieldTypeInt = "int64" + // FieldTypeIntSlice ... + FieldTypeIntSlice = "[]int64" + // FieldTypeFloat ... + FieldTypeFloat = "float64" + // FieldTypeFloatSlice ... + FieldTypeFloatSlice = "[]float64" + // FieldTypeBool ... + FieldTypeBool = "bool" + // FieldTypeBoolSlice ... + FieldTypeBoolSlice = "[]bool" + // FieldTypeString ... + FieldTypeString = "string" + // FieldTypeStringSLice ... + FieldTypeStringSLice = "string" + // FieldTypeAny ... + FieldTypeAny = "interface{}" + // FieldTypeAnySlice ... + FieldTypeAnySlice = "[]interface{}" + // FieldTypeSlice ... + FieldTypeSlice = "[]interface{}" + // FieldTypeMap ... + FieldTypeMap = "map" + // FieldTypeMapSlice ... + FieldTypeMapSlice = "[]map" +) + +// GetJSONDataStruct 获取JSON数据的结构 +// +// Author : go_developer@163.com<张德满> +// +// Date : 10:53 PM 2022/1/9 +func GetJSONDataStruct(data string) ([]string, error) { + if !gjson.Valid(data) { + return make([]string, 0), errors.New("JSON format is invalid") + } + pathList := make([]string, 0) + r := gjson.Parse(data) + r.ForEach(func(key, value gjson.Result) bool { + if value.Value() == nil { + pathList = append(pathList, key.String()) + return true + } + if value.IsObject() { + if value.String() == "{}" { + pathList = append(pathList, key.String()) + } else { + list, _ := GetJSONDataStruct(value.String()) + for _, k := range list { + pathList = append(pathList, key.String()+"."+k) + } + } + } + + if value.IsArray() { + dataList := value.Array() + if len(dataList) > 0 { + if !dataList[0].IsObject() && !dataList[0].IsArray() { + pathList = append(pathList, key.String()) + } else { + list, _ := GetJSONDataStruct(dataList[0].String()) + for _, k := range list { + pathList = append(pathList, key.String()+".[]."+k) + } + } + } else { + pathList = append(pathList, key.String()) + } + } + + if !value.IsObject() && !value.IsArray() { + pathList = append(pathList, key.String()) + } + + return true + }) + return pathList, nil +} + +// GetJSONDataStructWithType 获取数据结构,并获取类型 +// +// Author : go_developer@163.com<张德满> +// +// Date : 2022/1/10 10:47 PM +func GetJSONDataStructWithType(data string) ([]Field, error) { + if !gjson.Valid(data) { + return make([]Field, 0), errors.New("JSON format is invalid") + } + pathList := make([]Field, 0) + r := gjson.Parse(data) + r.ForEach(func(key, value gjson.Result) bool { + if value.Value() == nil { + pathList = append(pathList, Field{ + Path: key.String(), + Type: FieldTypeAny, + Example: "nil", + }) + return true + } + if value.IsObject() { + if value.String() == "{}" { + pathList = append(pathList, Field{ + Path: key.String(), + Type: FieldTypeMap, + Example: "{}", + }) + } else { + list, _ := GetJSONDataStructWithType(value.String()) + for _, field := range list { + pathList = append(pathList, Field{ + Path: key.String() + "." + field.Path, + Type: field.Type, + Example: field.Example, + }) + } + } + } + + if value.IsArray() { + dataList := value.Array() + if len(dataList) > 0 { + if !dataList[0].IsObject() && !dataList[0].IsArray() { + pathList = append(pathList, Field{ + Path: key.String(), + Type: "[]" + GetDataType(dataList[0]), + Example: value.String(), + }) + } else { + list, _ := GetJSONDataStructWithType(dataList[0].String()) + for _, field := range list { + pathList = append(pathList, Field{ + Path: key.String() + ".[]." + field.Path, + Type: field.Type, + Example: field.Example, + }) + } + } + } else { + pathList = append(pathList, Field{ + Path: key.String(), + Type: FieldTypeSlice, + Example: "[]", + }) + } + } + + if !value.IsObject() && !value.IsArray() { + pathList = append(pathList, Field{ + Path: key.String(), + Type: GetDataType(value), + Example: value.String(), + }) + } + + return true + }) + return pathList, nil +} + +// GetDataType 获取数据类型 +// +// Author : go_developer@163.com<张德满> +// +// Date : 2022/1/10 11:00 PM +func GetDataType(value gjson.Result) string { + switch value.Type.String() { + default: + return FieldTypeAny + case "Null": + return FieldTypeAny + case "False": + return FieldTypeBool + case "True": + return FieldTypeBool + case "Number": + if strings.Contains(value.String(), ".") { + return FieldTypeFloat + } + return FieldTypeInt + case "String": + return FieldTypeString + case "JSON": + if strings.HasPrefix(strings.TrimSpace(value.String()), "[") { + return FieldTypeSlice + } + return FieldTypeMap + } +}