Merge pull request 'gjson的二次处理' (#6) from feature/json into master

Reviewed-on: #6
This commit is contained in:
白茶清欢 2024-11-29 17:30:35 +08:00
commit be69c71d4a
7 changed files with 464 additions and 3 deletions

3
gjson_hack/README.md Normal file
View File

@ -0,0 +1,3 @@
# 对gjson库一些已知问题的pack
- 大数字 res.Int() 缺失精度

44
gjson_hack/define.go Normal file
View File

@ -0,0 +1,44 @@
// Package gjson_hack ...
//
// Description : gjson_hack ...
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2024-11-27 11:59
package gjson_hack
import "github.com/tidwall/gjson"
// PathOption 路径操作的选项
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 12:01 2024/11/27
type PathOption struct {
UnfoldArray bool `json:"unfold_array"` // 展开数组
MaxDeep int `json:"max_deep"` // 迭代最大深度, 默认 0 不限制
OnlyFinalPath bool `json:"only_final_path"` // 仅展示最终路径
}
// PathResult ...
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 12:00 2024/11/27
type PathResult struct {
List []string // 全部路径列表
ValueTable map[string]gjson.Result // 路径对应的值
}
const (
ArrayIdxTpl = "{{idx}}"
)
// ExpendArrayResult 展开数组的结果
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 15:46 2024/11/29
type ExpendArrayResult struct {
PathList []string `json:"path_list"` // 路径列表
}

168
gjson_hack/path.go Normal file
View File

@ -0,0 +1,168 @@
// Package gjson_hack ...
//
// Description : gjson_hack ...
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2024-11-27 11:58
package gjson_hack
import (
"errors"
"fmt"
"github.com/tidwall/gjson"
"strings"
)
// newDefaultPathOption 默认路径展开选项
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 12:06 2024/11/27
func newDefaultPathOption() *PathOption {
return &PathOption{
UnfoldArray: true,
MaxDeep: 0,
OnlyFinalPath: false,
}
}
// newPathResult ...
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 12:17 2024/11/27
func newPathResult() *PathResult {
return &PathResult{
List: make([]string, 0),
ValueTable: make(map[string]gjson.Result),
}
}
// Path 查看全部路径
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 12:03 2024/11/27
func Path(jsonStr string, pathOption *PathOption) (*PathResult, error) {
gjsonResult := gjson.Parse(jsonStr)
pathResult := newPathResult()
if nil == pathOption {
pathOption = newDefaultPathOption()
}
doExpandPath(gjsonResult, "", true, pathOption, pathResult)
return pathResult, nil
}
// doExpandPath 展开路径
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 14:57 2024/11/27
func doExpandPath(gjsonResult gjson.Result, rootPath string, hasChildren bool, pathOption *PathOption, pathResult *PathResult) {
if gjsonResult.IsObject() {
// 处理object
gjsonResult.ForEach(func(key, value gjson.Result) bool {
newRootPath := ""
if len(rootPath) == 0 {
newRootPath = key.String()
} else {
newRootPath = rootPath + "." + key.String()
}
if value.IsArray() || value.IsObject() {
if !pathOption.OnlyFinalPath {
// 中间key路径也存储下来
pathResult.List = append(pathResult.List, newRootPath)
pathResult.ValueTable[newRootPath] = value
}
doExpandPath(value, newRootPath, true, pathOption, pathResult)
} else {
pathResult.List = append(pathResult.List, newRootPath)
pathResult.ValueTable[newRootPath] = value
}
return true
})
}
if gjsonResult.IsArray() {
arrayList := gjsonResult.Array()
if len(arrayList) == 0 {
pathResult.List = append(pathResult.List, rootPath)
pathResult.ValueTable[rootPath] = gjsonResult
return
}
if arrayList[0].IsObject() || arrayList[0].IsArray() {
// 每一项是对象或者数组
if pathOption.UnfoldArray {
// 展开数组
for _, itemRes := range arrayList {
// doExpandPath(itemRes, rootPath+"."+fmt.Sprintf("%v", idx), true, pathOption, pathResult)
doExpandPath(itemRes, rootPath+"."+ArrayIdxTpl, true, pathOption, pathResult)
}
} else {
// 不展开数组
doExpandPath(arrayList[0], rootPath+".#", true, pathOption, pathResult)
}
} else {
// 每一项是基础类型
pathResult.List = append(pathResult.List, rootPath)
pathResult.ValueTable[rootPath] = gjsonResult
return
}
}
if strings.HasSuffix(rootPath, ".#") || strings.HasSuffix(rootPath, ArrayIdxTpl) {
// 处理不展开类型数组
return
}
if pathOption.OnlyFinalPath && hasChildren {
// 仅记录最终完整路径, 中间路径不记录
return
}
if _, exist := pathResult.ValueTable[rootPath]; !exist {
pathResult.List = append(pathResult.List, rootPath)
pathResult.ValueTable[rootPath] = gjsonResult
}
return
}
// ExpandArrayPath 根据真实数据展开数组路径
//
// 路径中若是包含 {{idx}} 占位符, 说明是需要展开的数组, 根据数组的时机数据, 进行数组的逐级展开
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 11:13 2024/11/29
func ExpandArrayPath(jsonStr string, pathConfig string, expendArrayResult *ExpendArrayResult) error {
if nil == expendArrayResult {
return errors.New("expendArrayResult can not be nil")
}
if nil == expendArrayResult.PathList {
expendArrayResult.PathList = make([]string, 0)
}
if !strings.Contains(pathConfig, ArrayIdxTpl) {
// 不是数组模板配置, 无需展开
expendArrayResult.PathList = append(expendArrayResult.PathList, pathConfig)
return nil
}
pathConfigArr := strings.Split(pathConfig, ArrayIdxTpl)
pathConfigArr[0] = strings.TrimSuffix(pathConfigArr[0], ".")
suffixPathTpl := strings.TrimPrefix(strings.Join(pathConfigArr[1:], ArrayIdxTpl), ".")
valueResult := gjson.Parse(jsonStr).Get(pathConfigArr[0])
if !valueResult.Exists() {
// 路径不存在, 无需设置具体值
return nil
}
if !valueResult.IsArray() {
// 不是数组,不要继续再向后展开了
expendArrayResult.PathList = append(expendArrayResult.PathList, pathConfigArr[0])
return nil
}
// 继续展开子项
for idx, _ := range valueResult.Array() {
idxStr := fmt.Sprintf("%v", idx)
if err := ExpandArrayPath(jsonStr, pathConfigArr[0]+"."+idxStr+"."+suffixPathTpl, expendArrayResult); nil != err {
return err
}
}
return nil
}

169
gjson_hack/path_test.go Normal file
View File

@ -0,0 +1,169 @@
// Package gjson_hack ...
//
// Description : gjson_hack ...
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2024-11-27 15:52
package gjson_hack
import (
"encoding/json"
"fmt"
"github.com/tidwall/gjson"
"testing"
)
func TestPath(t *testing.T) {
mapData := map[string]any{
"person_list": []map[string]any{
{"name": "zhang", "age": 10},
{"name": "li", "age": 20},
{"name": "wang", "age": 30},
},
"company_info": map[string]any{
"address": "Beijing",
"email": "xxx@xxx.com",
},
"sex": "man",
}
byteData, _ := json.Marshal(mapData)
pathRes, err := Path(string(byteData), nil)
if nil != err {
fmt.Println(err.Error())
} else {
for _, item := range pathRes.List {
fmt.Println(item)
}
}
}
func TestPathOnlyFinallyPath(t *testing.T) {
mapData := map[string]any{
"person_list": []map[string]any{
{"name": "zhang", "age": 10},
{"name": "li", "age": 20},
{"name": "wang", "age": 30},
},
"company_info": map[string]any{
"address": "Beijing",
"email": "xxx@xxx.com",
},
"sex": "man",
"user_list": [][]map[string]any{
[]map[string]any{
{"name": "zhang", "age": 10},
{"name": "li", "age": 20},
{"name": "wang", "age": 30},
},
[]map[string]any{
{"name": "zhang", "age": 20},
{"name": "li", "age": 20},
{"name": "wang", "age": 30},
},
[]map[string]any{
{"name": "zhang", "age": 30},
{"name": "li", "age": 20},
{"name": "wang", "age": 30},
},
},
}
byteData, _ := json.Marshal(mapData)
fmt.Println(gjson.GetBytes(byteData, "user_list.#.#.age"))
fmt.Println(gjson.GetBytes(byteData, "user_list.0.0.age"))
pathRes, err := Path(string(byteData), &PathOption{
UnfoldArray: false,
MaxDeep: 0,
OnlyFinalPath: true,
})
if nil != err {
fmt.Println(err.Error())
} else {
for _, item := range pathRes.List {
fmt.Println(item)
}
}
}
func TestPathOnlyFinallyPathWithUnfoldArray(t *testing.T) {
mapData := map[string]any{
"person_list": []map[string]any{
{"name": "zhang", "age": 10},
{"name": "li", "age": 20},
{"name": "wang", "age": 30},
},
"company_info": map[string]any{
"address": "Beijing",
"email": "xxx@xxx.com",
},
"sex": "man",
"user_list": [][]map[string]any{
[]map[string]any{
{"name": "zhang", "age": 10},
{"name": "li", "age": 20},
{"name": "wang", "age": 30},
},
[]map[string]any{
{"name": "zhang", "age": 10},
{"name": "li", "age": 20},
{"name": "wang", "age": 30},
},
[]map[string]any{
{"name": "zhang", "age": 10},
{"name": "li", "age": 20},
{"name": "wang", "age": 30},
},
},
}
byteData, _ := json.Marshal(mapData)
pathRes, err := Path(string(byteData), &PathOption{
UnfoldArray: true,
MaxDeep: 0,
OnlyFinalPath: false,
})
if nil != err {
fmt.Println(err.Error())
} else {
for _, item := range pathRes.List {
fmt.Println(item)
}
}
}
func TestExpandArrayPath(t *testing.T) {
mapData := map[string]any{
"person_list": []map[string]any{
{"name": "zhang", "age": 10},
{"name": "li", "age": 20},
{"name": "wang", "age": 30},
},
"company_info": map[string]any{
"address": "Beijing",
"email": "xxx@xxx.com",
},
"sex": "man",
"user_list": [][]map[string]any{
[]map[string]any{
{"name": "zhang", "age": 10},
{"name": "li", "age": 20},
{"name": "wang", "age": 30},
},
[]map[string]any{
{"name": "zhang", "age": 10},
{"name": "li", "age": 20},
{"name": "wang", "age": 30},
},
[]map[string]any{
{"name": "zhang", "age": 10},
{"name": "li", "age": 20},
{"name": "wang", "age": 30},
},
},
}
byteData, _ := json.Marshal(mapData)
jsonStr := string(byteData)
// fmt.Println(jsonStr)
var pathExpendRes = &ExpendArrayResult{PathList: nil}
ExpandArrayPath(jsonStr, "user_list.{{idx}}.{{idx}}.age", pathExpendRes)
fmt.Println(pathExpendRes)
}

71
gjson_hack/precision.go Normal file
View File

@ -0,0 +1,71 @@
// Package gjson_hack 精确地类型转换
//
// Description : gjson_hack ...
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 2024-11-27 10:47
package gjson_hack
import (
"errors"
"git.zhangdeman.cn/zhangdeman/util"
"github.com/tidwall/gjson"
)
// Number 结果转换为数字(int64 / uint64 / float64)
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 10:57 2024/11/27
func Number[T int64 | uint64 | float64](gjsonResult gjson.Result, numberReceiver *T) error {
if !gjsonResult.Exists() {
return errors.New("gjson result not found")
}
if gjsonResult.Type != gjson.Number {
return errors.New(gjsonResult.String() + " : value not a number")
}
if err := util.ConvertAssign(numberReceiver, gjsonResult.String()); nil != err {
return errors.New(gjsonResult.String() + " : convert to num fail -> " + err.Error())
}
return nil
}
// Int 转int
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 11:19 2024/11/27
func Int(gjsonResult gjson.Result) (int64, error) {
var intResult int64
if err := Number(gjsonResult, &intResult); nil != err {
return 0, err
}
return intResult, nil
}
// Uint 转为uint64
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 11:21 2024/11/27
func Uint(gjsonResult gjson.Result) (uint64, error) {
var uintResult uint64
if err := Number(gjsonResult, &uintResult); nil != err {
return 0, err
}
return uintResult, nil
}
// Float64 转为float64
//
// Author : go_developer@163.com<白茶清欢>
//
// Date : 11:24 2024/11/27
func Float64(gjsonResult gjson.Result) (float64, error) {
var float64Result float64
if err := Number(gjsonResult, &float64Result); nil != err {
return 0, err
}
return float64Result, nil
}

6
go.mod
View File

@ -5,11 +5,11 @@ go 1.21
toolchain go1.21.5
require (
git.zhangdeman.cn/zhangdeman/consts v0.0.0-20241125065114-f919222003d9
git.zhangdeman.cn/zhangdeman/consts v0.0.0-20241125100843-b1b286c7a701
git.zhangdeman.cn/zhangdeman/easylock v0.0.0-20230731062340-983985c12eda
git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20241108082010-42ae8fe5ebdc
git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20241125105403-cb92be844edc
git.zhangdeman.cn/zhangdeman/util v0.0.0-20240618042405-6ee2c904644e
git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20241125062526-91423fb146e0
git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20241125065949-2f87fe0cd90e
github.com/pkg/errors v0.9.1
github.com/smartystreets/goconvey v1.8.1
github.com/tidwall/gjson v1.18.0

6
go.sum
View File

@ -6,6 +6,8 @@ git.zhangdeman.cn/zhangdeman/consts v0.0.0-20241125061350-1f5050978fc3 h1:/40XIy
git.zhangdeman.cn/zhangdeman/consts v0.0.0-20241125061350-1f5050978fc3/go.mod h1:IXXaZkb7vGzGnGM5RRWrASAuwrVSNxuoe0DmeXx5g6k=
git.zhangdeman.cn/zhangdeman/consts v0.0.0-20241125065114-f919222003d9 h1:TP/M3WnGsxh0Vr6YuS1i28hw1oV//YbdCoI46PUBIA0=
git.zhangdeman.cn/zhangdeman/consts v0.0.0-20241125065114-f919222003d9/go.mod h1:IXXaZkb7vGzGnGM5RRWrASAuwrVSNxuoe0DmeXx5g6k=
git.zhangdeman.cn/zhangdeman/consts v0.0.0-20241125100843-b1b286c7a701 h1:G+lGQmjMOBWGspZfijZvenGUAKpjBBrkRXLg3+GZp0U=
git.zhangdeman.cn/zhangdeman/consts v0.0.0-20241125100843-b1b286c7a701/go.mod h1:IXXaZkb7vGzGnGM5RRWrASAuwrVSNxuoe0DmeXx5g6k=
git.zhangdeman.cn/zhangdeman/easylock v0.0.0-20230731062340-983985c12eda h1:bMD6r9gjRy7cO+T4zRQVYAesgIblBdTnhzT1vN5wjvI=
git.zhangdeman.cn/zhangdeman/easylock v0.0.0-20230731062340-983985c12eda/go.mod h1:dT0rmHcJ9Z9IqWeMIt7YzR88nKkNV2V3dfG0j9Q6lK0=
git.zhangdeman.cn/zhangdeman/easymap v0.0.0-20240311030808-e2a2e6a3c211 h1:I/wOsRpCSRkU9vo1u703slQsmK0wnNeZzsWQOGtIAG0=
@ -18,6 +20,8 @@ git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20240325080031-1f58204e8687 h1:uQc
git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20240325080031-1f58204e8687/go.mod h1:gf7SW2TXATgux8pfdFedMkXWv2515OtIIM/5c4atkFw=
git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20241108082010-42ae8fe5ebdc h1:jtdEMr/xNchJDEoCnvMr4JXT9+biYQu625Cj+dz025w=
git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20241108082010-42ae8fe5ebdc/go.mod h1:XqgER4jDYwskFgj2riJ9XptIjzgYWubY+Zq8iB2WkY0=
git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20241125105403-cb92be844edc h1:rYjlMH5Yy0G8OQgXA8qrV+fqObnB99v+6s8nbiLhzQs=
git.zhangdeman.cn/zhangdeman/serialize v0.0.0-20241125105403-cb92be844edc/go.mod h1:+D6uPSljwHywjVY5WSBY4TRVMj26TN5f5cFGEYMldjs=
git.zhangdeman.cn/zhangdeman/util v0.0.0-20231227095334-7eb5cdbf9253 h1:GO3oZa5a2sqwAzGcLDJtQzmshSWRmoP7IDS8bwFqvC4=
git.zhangdeman.cn/zhangdeman/util v0.0.0-20231227095334-7eb5cdbf9253/go.mod h1:VpPjBlwz8U+OxZuxzHQBv1aEEZ3pStH6bZvT21ADEbI=
git.zhangdeman.cn/zhangdeman/util v0.0.0-20240618042405-6ee2c904644e h1:Q973S6CcWr1ICZhFI1STFOJ+KUImCl2BaIXm6YppBqI=
@ -28,6 +32,8 @@ git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20241106102517-46cd353dd617 h1:4rwv7
git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20241106102517-46cd353dd617/go.mod h1:uEuMBRzTlUJ9k3H+PymZWPn1b48U9lJPQ+ZBWmP+O/c=
git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20241125062526-91423fb146e0 h1:aGQADmQKTF7c8+s3acnn569sTJwjlLhhhZfQ6f4nAH0=
git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20241125062526-91423fb146e0/go.mod h1:lvVOIrYDvCQHUzBdaOwKSrxT9ubcXQJBsafWMWu+I14=
git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20241125065949-2f87fe0cd90e h1:/N5yXmOEH7N/h4S6nv/6os8sRenh95BXogTZ2RJI950=
git.zhangdeman.cn/zhangdeman/wrapper v0.0.0-20241125065949-2f87fe0cd90e/go.mod h1:17TlhgwKTLQLIzYW/R6G39oN5FFPdsEEFDWniv+ovgA=
github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 h1:OYA+5W64v3OgClL+IrOD63t4i/RW7RqrAVl9LTZ9UqQ=