From 761058f8fe811e2b6829ce41db729cd89f82ec8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E8=8C=B6=E6=B8=85=E6=AC=A2?= Date: Tue, 6 May 2025 10:58:20 +0800 Subject: [PATCH 1/7] =?UTF-8?q?=E5=A2=9E=E5=8A=A0IJsonRead=E4=B8=8EIJsonWr?= =?UTF-8?q?ite=E6=8E=A5=E5=8F=A3=E7=BA=A6=E6=9D=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- abstract/json_read.go | 32 ++++++++++++++++++++++++++++++++ abstract/json_write.go | 20 ++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 abstract/json_read.go create mode 100644 abstract/json_write.go diff --git a/abstract/json_read.go b/abstract/json_read.go new file mode 100644 index 0000000..0e824e8 --- /dev/null +++ b/abstract/json_read.go @@ -0,0 +1,32 @@ +// Package abstract ... +// +// Description : abstract ... +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 2025-05-06 10:42 +package abstract + +// IJsonRead json数据读取接口约束 +type IJsonRead interface { + // Exist 指定路径是否存在 + Exist(dataPath string) bool + // IsNil 指定路径是否为nil + IsNil(dataPath string) bool + // Int 转换为int类型 + Int(dataPath string) (int, error) + // Uint 转换为uint类型 + Uint(dataPath string) (uint, error) + // Float 转换为float类型 + Float(dataPath string) (float64, error) + // String 转换为string类型 + String(dataPath string) (string, error) + // Map 转换为map + Map(dataPath string) (map[string]any, error) + // MapWithReceiver 通过指针接收 + MapWithReceiver(dataPath string, receiver any) error + // Array 转换为数组 + Array(dataPath string) ([]any, error) + // ArrayWithReceiver 通过指针接收 + ArrayWithReceiver(dataPath string, receiver any) error +} diff --git a/abstract/json_write.go b/abstract/json_write.go new file mode 100644 index 0000000..da7fb2d --- /dev/null +++ b/abstract/json_write.go @@ -0,0 +1,20 @@ +// Package abstract ... +// +// Description : abstract ... +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 2025-05-06 10:53 +package abstract + +// IJsonWrite json数据写入 +type IJsonWrite interface { + // Set 设置一个路径的值 + Set(dataPath string, data interface{}) error + // Result 最终结果以字符串形式返回 + Result() string + // Map 最终结果以map返回 + Map() (map[string]any, error) + // Array 最终结果以数组返回 + Array() ([]any, error) +} -- 2.36.6 From 8489acbef8d2cfb3b5a62046098e28cd44bb406d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E8=8C=B6=E6=B8=85=E6=AC=A2?= Date: Tue, 6 May 2025 11:51:50 +0800 Subject: [PATCH 2/7] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=9F=BA=E4=BA=8Egsjon?= =?UTF-8?q?=E5=AE=9E=E7=8E=B0=E7=9A=84JsonRead?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- abstract/json_read.go | 6 +- implement/gjson_read.go | 120 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 implement/gjson_read.go diff --git a/abstract/json_read.go b/abstract/json_read.go index 0e824e8..e2058d2 100644 --- a/abstract/json_read.go +++ b/abstract/json_read.go @@ -13,10 +13,12 @@ type IJsonRead interface { Exist(dataPath string) bool // IsNil 指定路径是否为nil IsNil(dataPath string) bool + // Type 路径数据类型 + Type(dataPath string) string // Int 转换为int类型 - Int(dataPath string) (int, error) + Int(dataPath string) (int64, error) // Uint 转换为uint类型 - Uint(dataPath string) (uint, error) + Uint(dataPath string) (uint64, error) // Float 转换为float类型 Float(dataPath string) (float64, error) // String 转换为string类型 diff --git a/implement/gjson_read.go b/implement/gjson_read.go new file mode 100644 index 0000000..57e0c01 --- /dev/null +++ b/implement/gjson_read.go @@ -0,0 +1,120 @@ +// Package implement ... +// +// Description : implement ... +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 2025-05-06 11:00 +package implement + +import ( + "errors" + "git.zhangdeman.cn/zhangdeman/consts" + "git.zhangdeman.cn/zhangdeman/json_filter/abstract" + "git.zhangdeman.cn/zhangdeman/json_filter/gjson_hack" + "git.zhangdeman.cn/zhangdeman/serialize" + "github.com/tidwall/gjson" +) + +// NewGjsonRead ... +func NewGjsonRead(sourceData string) abstract.IJsonRead { + return &gjsonRead{ + sourceData: sourceData, + gjsonResult: gjson.Parse(sourceData), + } +} + +type gjsonRead struct { + sourceData string + gjsonResult gjson.Result +} + +func (g *gjsonRead) Exist(dataPath string) bool { + return g.gjsonResult.Get(dataPath).Exists() +} + +func (g *gjsonRead) IsNil(dataPath string) bool { + return g.gjsonResult.Get(dataPath).Value() == nil +} + +func (g *gjsonRead) Type(dataPath string) string { + pathRes := g.gjsonResult.Get(dataPath) + if pathRes.IsObject() { // map + return consts.DataTypeMapStrAny.String() + } + if pathRes.IsArray() { // slice + return consts.DataTypeSliceAny.String() + } + if pathRes.IsBool() { // bool + return consts.DataTypeBool.String() + } + dataType := pathRes.Type + switch dataType { + case gjson.String: + return consts.DataTypeString.String() + case gjson.Number: + return consts.DataTypeFloat64.String() + case gjson.Null: + return consts.DataTypeAny.String() + case gjson.True, gjson.False: + return consts.DataTypeBool.String() + } + return consts.DataTypeAny.String() // any类型兜底 +} + +func (g *gjsonRead) Int(dataPath string) (int64, error) { + return gjson_hack.Int(g.gjsonResult.Get(dataPath)) +} + +func (g *gjsonRead) Uint(dataPath string) (uint64, error) { + return gjson_hack.Uint(g.gjsonResult.Get(dataPath)) +} + +func (g *gjsonRead) Float(dataPath string) (float64, error) { + return gjson_hack.Float64(g.gjsonResult.Get(dataPath)) +} + +func (g *gjsonRead) String(dataPath string) (string, error) { + if !g.Exist(dataPath) { + return "", errors.New(dataPath + ": not found") + } + return g.gjsonResult.Get(dataPath).String(), nil +} + +func (g *gjsonRead) Map(dataPath string) (map[string]any, error) { + var ( + err error + dataMap map[string]any + ) + if err = g.MapWithReceiver(dataPath, &dataMap); nil != err { + return map[string]any{}, err + } + return dataMap, nil +} + +func (g *gjsonRead) MapWithReceiver(dataPath string, receiver any) error { + strVal, err := g.String(dataPath) + if nil != err { + return err + } + return serialize.JSON.UnmarshalWithNumber([]byte(strVal), receiver) +} + +func (g *gjsonRead) Array(dataPath string) ([]any, error) { + var ( + err error + dataArr []any + ) + if err = g.ArrayWithReceiver(dataPath, &dataArr); nil != err { + return []any{}, err + } + return dataArr, nil +} + +func (g *gjsonRead) ArrayWithReceiver(dataPath string, receiver any) error { + strVal, err := g.String(dataPath) + if nil != err { + return err + } + return serialize.JSON.UnmarshalWithNumber([]byte(strVal), receiver) +} -- 2.36.6 From 1124ff6a331a80915b41474ba5633cfa430b5fac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E8=8C=B6=E6=B8=85=E6=AC=A2?= Date: Tue, 6 May 2025 12:07:58 +0800 Subject: [PATCH 3/7] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=9F=BA=E4=BA=8Esjson?= =?UTF-8?q?=E5=AE=9E=E7=8E=B0=E7=9A=84IJsonWrite?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- abstract/json_write.go | 6 +++- implement/sjson_write.go | 74 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 implement/sjson_write.go diff --git a/abstract/json_write.go b/abstract/json_write.go index da7fb2d..3706c9e 100644 --- a/abstract/json_write.go +++ b/abstract/json_write.go @@ -10,11 +10,15 @@ package abstract // IJsonWrite json数据写入 type IJsonWrite interface { // Set 设置一个路径的值 - Set(dataPath string, data interface{}) error + Set(dataPath string, data any) error // Result 最终结果以字符串形式返回 Result() string // Map 最终结果以map返回 Map() (map[string]any, error) + // MapWithReceiver 外部指针接收返回值 + MapWithReceiver(receiver any) error // Array 最终结果以数组返回 Array() ([]any, error) + // ArrayWithReceiver 外部指针接收返回值 + ArrayWithReceiver(receiver any) error } diff --git a/implement/sjson_write.go b/implement/sjson_write.go new file mode 100644 index 0000000..c3179ab --- /dev/null +++ b/implement/sjson_write.go @@ -0,0 +1,74 @@ +// Package implement ... +// +// Description : implement ... +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 2025-05-06 11:53 +package implement + +import ( + "git.zhangdeman.cn/zhangdeman/json_filter/abstract" + "git.zhangdeman.cn/zhangdeman/serialize" + "github.com/tidwall/sjson" + "sync" +) + +func NewSjsonWrite() abstract.IJsonWrite { + return &SjsonWrite{ + res: "", + l: &sync.RWMutex{}, + } +} + +type SjsonWrite struct { + res string + l *sync.RWMutex +} + +func (s *SjsonWrite) Set(dataPath string, data any) error { + var ( + err error + ) + + s.l.Lock() + defer s.l.Unlock() + if s.res, err = sjson.Set(s.res, dataPath, data); nil != err { + return err + } + return nil +} + +func (s *SjsonWrite) Result() string { + return s.res +} + +func (s *SjsonWrite) Map() (map[string]any, error) { + var ( + mapRes map[string]any + err error + ) + if err = s.MapWithReceiver(&mapRes); nil != err { + return nil, err + } + return mapRes, nil +} + +func (s *SjsonWrite) MapWithReceiver(receiver any) error { + return serialize.JSON.UnmarshalWithNumberForString(s.res, receiver) +} + +func (s *SjsonWrite) Array() ([]any, error) { + var ( + arrRes []any + err error + ) + if err = s.ArrayWithReceiver(&arrRes); nil != err { + return nil, err + } + return arrRes, nil +} + +func (s *SjsonWrite) ArrayWithReceiver(receiver any) error { + return serialize.JSON.UnmarshalWithNumberForString(s.res, receiver) +} -- 2.36.6 From 042aad254b46b957d1c2f434f8629ea8a7c88362 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E8=8C=B6=E6=B8=85=E6=AC=A2?= Date: Tue, 6 May 2025 12:22:26 +0800 Subject: [PATCH 4/7] =?UTF-8?q?IJsonRead=E6=94=AF=E6=8C=81Value,=20?= =?UTF-8?q?=E6=B3=9B=E8=AF=BB=E5=8F=96=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- abstract/json_read.go | 2 ++ filter.go | 22 ++++++++++++++++++++++ implement/gjson_read.go | 6 ++++++ 3 files changed, 30 insertions(+) diff --git a/abstract/json_read.go b/abstract/json_read.go index e2058d2..e9c07a8 100644 --- a/abstract/json_read.go +++ b/abstract/json_read.go @@ -31,4 +31,6 @@ type IJsonRead interface { Array(dataPath string) ([]any, error) // ArrayWithReceiver 通过指针接收 ArrayWithReceiver(dataPath string, receiver any) error + // Value 自适应类型数据读取 + Value(dataPath string, dataType string, defaultValue any) (any, error) } diff --git a/filter.go b/filter.go index 07233ec..7281ab0 100644 --- a/filter.go +++ b/filter.go @@ -10,6 +10,8 @@ package filter import ( "encoding/json" "fmt" + "git.zhangdeman.cn/zhangdeman/json_filter/abstract" + "git.zhangdeman.cn/zhangdeman/json_filter/implement" "reflect" "strings" @@ -21,6 +23,22 @@ import ( "errors" ) +func NewFilterWithJson(sourceData string, filterRuleList []MapRule, jsonRead abstract.IJsonRead, jsonWrite abstract.IJsonWrite) *filter { + if nil == jsonRead { + jsonRead = implement.NewGjsonRead(sourceData) + } + if nil == jsonWrite { + jsonWrite = implement.NewSjsonWrite() + } + return &filter{ + jsonRaad: jsonRead, + jsonWrite: jsonWrite, + sourceData: sourceData, + formatResult: "", + filterRuleList: filterRuleList, + } +} + // NewFilter 过滤器实例 // // Author : go_developer@163.com<白茶清欢> @@ -30,6 +48,8 @@ func NewFilter(sourceData string, filterRuleList []MapRule) *filter { return &filter{ sourceData: sourceData, formatResult: "{}", + jsonRaad: implement.NewGjsonRead(sourceData), + jsonWrite: implement.NewSjsonWrite(), filterRuleList: filterRuleList, } } @@ -40,6 +60,8 @@ func NewFilter(sourceData string, filterRuleList []MapRule) *filter { // // Date : 11:58 2022/7/4 type filter struct { + jsonRaad abstract.IJsonRead + jsonWrite abstract.IJsonWrite sourceData string formatResult string filterRuleList []MapRule diff --git a/implement/gjson_read.go b/implement/gjson_read.go index 57e0c01..9ba9eb9 100644 --- a/implement/gjson_read.go +++ b/implement/gjson_read.go @@ -118,3 +118,9 @@ func (g *gjsonRead) ArrayWithReceiver(dataPath string, receiver any) error { } return serialize.JSON.UnmarshalWithNumber([]byte(strVal), receiver) } +func (g *gjsonRead) Value(dataPath string, dataType string, defaultValue any) (any, error) { + if len(dataType) == 0 { + dataType = g.Type(dataPath) + } + return gjson_hack.Value(consts.DataType(dataType), g.gjsonResult.Get(dataPath), defaultValue) +} -- 2.36.6 From 47726c20b8287283c0e053f09ba68453ceb42b1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E8=8C=B6=E6=B8=85=E6=AC=A2?= Date: Tue, 6 May 2025 12:39:48 +0800 Subject: [PATCH 5/7] =?UTF-8?q?SJsonWrite=E8=AE=BE=E7=BD=AE=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E6=97=B6,=20=E6=94=AF=E6=8C=81=E5=81=9Amap=E5=90=88?= =?UTF-8?q?=E5=B9=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- implement/sjson_write.go | 42 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/implement/sjson_write.go b/implement/sjson_write.go index c3179ab..01e3ef4 100644 --- a/implement/sjson_write.go +++ b/implement/sjson_write.go @@ -8,9 +8,11 @@ package implement import ( + "git.zhangdeman.cn/zhangdeman/consts" "git.zhangdeman.cn/zhangdeman/json_filter/abstract" "git.zhangdeman.cn/zhangdeman/serialize" "github.com/tidwall/sjson" + "reflect" "sync" ) @@ -33,7 +35,45 @@ func (s *SjsonWrite) Set(dataPath string, data any) error { s.l.Lock() defer s.l.Unlock() - if s.res, err = sjson.Set(s.res, dataPath, data); nil != err { + existResRead := NewGjsonRead(s.res) + if !existResRead.Exist(dataPath) || data == nil { + // 路径不存在 + if s.res, err = sjson.Set(s.res, dataPath, data); nil != err { + return err + } + return nil + } + // 路径已存在, 判断是否为map + if existResRead.Type(dataPath) != consts.DataTypeMapStrAny.String() { + if s.res, err = sjson.Set(s.res, dataPath, data); nil != err { + return err + } + return nil + } + // 判断data是否为map + dataType := reflect.TypeOf(data) + if dataType.Kind() == reflect.Ptr { + dataType = dataType.Elem() + } + if dataType.Kind() != reflect.Struct && dataType.Kind() != reflect.Map { + // 既不是map, 也不是struct + if s.res, err = sjson.Set(s.res, dataPath, data); nil != err { + return err + } + return nil + } + // 路径已存在, 已存在数据为map, 且要设置的数据也为map, 进行数据合并 + mergeDataMap := map[string]any{} + existMapVal := map[string]any{} + + if existMapVal, err = existResRead.Map(dataPath); nil != err { + return err + } + // 合并输入数据 + if err = serialize.JSON.MergeDataForReceiver(&mergeDataMap, existMapVal, data); nil != err { + return err + } + if s.res, err = sjson.Set(s.res, dataPath, mergeDataMap); nil != err { return err } return nil -- 2.36.6 From a28bc364e4125a5552d34ad6624d4d98c8cea0c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E8=8C=B6=E6=B8=85=E6=AC=A2?= Date: Tue, 6 May 2025 14:10:38 +0800 Subject: [PATCH 6/7] =?UTF-8?q?=E4=BC=98=E5=8C=96filter=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- filter.go | 48 +++++++++--------------------------------------- 1 file changed, 9 insertions(+), 39 deletions(-) diff --git a/filter.go b/filter.go index 7281ab0..58ba091 100644 --- a/filter.go +++ b/filter.go @@ -8,19 +8,14 @@ package filter import ( - "encoding/json" "fmt" "git.zhangdeman.cn/zhangdeman/json_filter/abstract" "git.zhangdeman.cn/zhangdeman/json_filter/implement" - "reflect" "strings" - "git.zhangdeman.cn/zhangdeman/json_filter/gjson_hack" - "git.zhangdeman.cn/zhangdeman/serialize" - "github.com/tidwall/gjson" - "github.com/tidwall/sjson" - "errors" + "git.zhangdeman.cn/zhangdeman/json_filter/gjson_hack" + "github.com/tidwall/gjson" ) func NewFilterWithJson(sourceData string, filterRuleList []MapRule, jsonRead abstract.IJsonRead, jsonWrite abstract.IJsonWrite) *filter { @@ -34,7 +29,6 @@ func NewFilterWithJson(sourceData string, filterRuleList []MapRule, jsonRead abs jsonRaad: jsonRead, jsonWrite: jsonWrite, sourceData: sourceData, - formatResult: "", filterRuleList: filterRuleList, } } @@ -47,7 +41,6 @@ func NewFilterWithJson(sourceData string, filterRuleList []MapRule, jsonRead abs func NewFilter(sourceData string, filterRuleList []MapRule) *filter { return &filter{ sourceData: sourceData, - formatResult: "{}", jsonRaad: implement.NewGjsonRead(sourceData), jsonWrite: implement.NewSjsonWrite(), filterRuleList: filterRuleList, @@ -63,7 +56,6 @@ type filter struct { jsonRaad abstract.IJsonRead jsonWrite abstract.IJsonWrite sourceData string - formatResult string filterRuleList []MapRule } @@ -148,33 +140,10 @@ func (f *filter) setResult(rule MapRule) error { err error formatVal any ) - - sourceResult := gjson.Get(f.sourceData, rule.SourcePath) - if formatVal, err = gjson_hack.Value(rule.DataType, sourceResult, rule.DefaultValue); nil != err { - return fmt.Errorf("%s = %v can not convert to %s : %s", rule.SourcePath, sourceResult.Value(), rule.DataType, err.Error()) + if formatVal, err = f.jsonRaad.Value(rule.SourcePath, rule.DataType.String(), rule.DefaultValue); nil != err { + return fmt.Errorf("%s = %v can not convert to %s : %s", rule.SourcePath, gjson.Get(f.sourceData, rule.SourcePath).String(), rule.DataType, err.Error()) } - if reflect.TypeOf(formatVal).Kind() == reflect.Map { - // 获取的数据是map类型, 处理数据覆盖 - // eg : 配置如下两个规则 process.id(string) 、process(map[string]any) - // 若输入数据的process.id为int类型, 则格式化后的process.id必为 string, 应为 process.id 规则的控制更精细 - gjsonVal := gjson.Get(f.formatResult, rule.TargetPath) - if gjsonVal.Exists() && gjsonVal.IsObject() { - var ( - existRes = map[string]any{} - formatRes = map[string]any{} - ) - // 已存在, 且是对象 - _ = serialize.JSON.UnmarshalWithNumber([]byte(gjsonVal.String()), &existRes) - if err = serialize.JSON.Transition(formatVal, &formatRes); nil != err { - return errors.New("conflict data path config deal fail : " + err.Error()) - } - for k, v := range existRes { - formatRes[k] = v - } - formatVal = formatRes // 重新赋值 formatVal - } - } - if f.formatResult, err = sjson.Set(f.formatResult, rule.TargetPath, formatVal); nil != err { + if err = f.jsonWrite.Set(rule.TargetPath, formatVal); nil != err { return err } return nil @@ -204,7 +173,7 @@ func (f *filter) getAllFinalData(res *[]string, resultList []gjson.Result, pathA // // Date : 21:18 2022/12/31 func (f *filter) String() string { - return f.formatResult + return f.jsonWrite.Result() } // Byte 获取格式化之后的字节数组 @@ -221,9 +190,10 @@ func (f *filter) Byte() []byte { // Author : go_developer@163.com<白茶清欢> // // Date : 21:20 2022/12/31 -func (f *filter) Parse(receiver interface{}) error { +func (f *filter) Parse(receiver any) error { if nil == receiver { return errors.New("receiver is nil") } - return json.Unmarshal(f.Byte(), receiver) + return f.jsonWrite.MapWithReceiver(receiver) + // return json.Unmarshal(f.Byte(), receiver) } -- 2.36.6 From 38b2d74f3542cf29ae77f903961edd881ae58f08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E8=8C=B6=E6=B8=85=E6=AC=A2?= Date: Tue, 6 May 2025 14:14:18 +0800 Subject: [PATCH 7/7] remove debug code --- sjson_hack/set.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/sjson_hack/set.go b/sjson_hack/set.go index 809fb11..91693a5 100644 --- a/sjson_hack/set.go +++ b/sjson_hack/set.go @@ -9,7 +9,6 @@ package sjson_hack import ( "errors" - "fmt" "git.zhangdeman.cn/zhangdeman/json_filter/gjson_hack" "git.zhangdeman.cn/zhangdeman/wrapper" "github.com/tidwall/sjson" @@ -27,7 +26,6 @@ var ( // // Date : 11:36 2024/12/3 func Set(jsonRes string, path string, value any) (string, error) { - fmt.Println(jsonRes, value) var ( err error res string = jsonRes -- 2.36.6