From 6e8dff9f9cd12458966fba460b70cb286f685587 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E8=8C=B6=E6=B8=85=E6=AC=A2?= Date: Sat, 31 Dec 2022 22:03:19 +0800 Subject: [PATCH 1/7] update --- define.go | 64 ++-------- filter.go | 76 ++++++++---- go.mod | 3 +- go.sum | 5 - lexical.go | 304 ------------------------------------------------ lexical_test.go | 40 ------- syntax.go | 153 ------------------------ syntax_test.go | 38 ------ token.go | 27 ----- 9 files changed, 61 insertions(+), 649 deletions(-) delete mode 100644 lexical.go delete mode 100644 lexical_test.go delete mode 100644 syntax.go delete mode 100644 syntax_test.go delete mode 100644 token.go diff --git a/define.go b/define.go index 877b057..a80d083 100644 --- a/define.go +++ b/define.go @@ -7,63 +7,15 @@ // Date : 2022-07-04 18:02 package filter -import ( - "fmt" - - "git.zhangdeman.cn/zhangdeman/util" -) - -// jsonNode json语法树节点定义 +// MapRule 映射规则 // // Author : go_developer@163.com<白茶清欢> // -// Date : 18:06 2022/7/4 -type jsonNode struct { - Name string // 节点名称 - Parent *jsonNode // 父节点 - Son *jsonNode // 子节点 - PreBrother *jsonNode // 前一个兄弟节点 - NextBrother *jsonNode // 下一个兄弟节点 - Val interface{} // 节点的值 - Type string // 数据类型 +// Date : 12:21 2022/7/4 +type MapRule struct { + SourcePath string `json:"source_path"` // 原路径 + MapPath string `json:"map_path"` // 映射路径 + Required bool `json:"required"` // 必须存在 + DataType string `json:"data_type"` // 数据类型 + DefaultValue string `json:"default_value"` // 默认值, 以字符串传入, 会转换成 DataType } - -// lexicalNode 词法分析的节点 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 18:02 2022/7/4 -type lexicalNode struct { - Val interface{} // 词法分析出来的词 - Type string // 值得类型 - Show bool // 这个值是否对外呈现 - IsToken bool // 是否为关键字 - OutputFormat string // 对外输出的格式 -} - -// ValStr 值转为字符串 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 14:10 2022/7/5 -func (l *lexicalNode) ValStr() string { - if l.Type == "number" { - var str string - _ = util.ConvertAssign(&str, l.Val) - return str - } - return fmt.Sprintf("%v", l.Val) -} - -const ( - // NodeTypeNumber 数据类型 - NodeTypeNumber = "number" - // NodeTypeBool bool 类型 - NodeTypeBool = "bool" - // NodeTypeString 字符串类型 - NodeTypeString = "string" - // NodeTypeObject kv 对象类型 - NodeTypeObject = "object" - // NodeTypeList list类型 - NodeTypeList = "list" -) diff --git a/filter.go b/filter.go index 3cfcd26..b295b87 100644 --- a/filter.go +++ b/filter.go @@ -8,28 +8,16 @@ package filter import ( + "encoding/json" "fmt" + "github.com/tidwall/sjson" "strings" "git.zhangdeman.cn/zhangdeman/util" - "github.com/Jeffail/gabs" "github.com/pkg/errors" "github.com/tidwall/gjson" ) -// MapRule 映射规则 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 12:21 2022/7/4 -type MapRule struct { - SourcePath string `json:"source_path"` // 原路径 - MapPath string `json:"map_path"` // 映射路径 - Required bool `json:"required"` // 必须存在 - DataType string `json:"data_type"` // 数据类型 - DefaultValue string `json:"default_value"` // 默认值, 以字符串传入, 会转换成 DataType -} - // NewFilter 过滤器实例 // // Author : go_developer@163.com<白茶清欢> @@ -38,8 +26,8 @@ type MapRule struct { func NewFilter(sourceData string, filterRuleList []MapRule) *filter { return &filter{ sourceData: sourceData, + formatResult: "{}", filterRuleList: filterRuleList, - jsonObj: &gabs.Container{}, } } @@ -50,8 +38,8 @@ func NewFilter(sourceData string, filterRuleList []MapRule) *filter { // Date : 11:58 2022/7/4 type filter struct { sourceData string + formatResult string filterRuleList []MapRule - jsonObj *gabs.Container // 生成的json对象实例 } // Deal ... @@ -59,27 +47,66 @@ type filter struct { // Author : go_developer@163.com<白茶清欢> // // Date : 11:59 2022/7/4 -func (f *filter) Deal() ([]byte, error) { +func (f *filter) Deal() error { + var ( + err error + formatVal interface{} + ) + for _, rule := range f.filterRuleList { if strings.Contains(rule.SourcePath, "[]") { // 对于list的处理 continue } sourceResult := gjson.Get(f.sourceData, rule.SourcePath) - sourceVal := sourceResult.String() + var ( + sourceVal string + ) + if !sourceResult.Exists() { // 不存在, 使用默认值 sourceVal = rule.DefaultValue + } else { + sourceVal = sourceResult.String() } - formatVal, err := f.getValue(rule.DataType, sourceVal) - if nil != err { - return nil, fmt.Errorf("%s = %v can not convert to %s : %s", rule.SourcePath, sourceResult.Value(), rule.DataType, err.Error()) + if formatVal, err = f.getValue(rule.DataType, sourceVal); nil != err { + return fmt.Errorf("%s = %v can not convert to %s : %s", rule.SourcePath, sourceResult.Value(), rule.DataType, err.Error()) } - if _, err := f.jsonObj.SetP(formatVal, rule.MapPath); nil != err { - return nil, fmt.Errorf("%s set val = %v fail : %s", rule.MapPath, formatVal, err.Error()) + if _, err = sjson.Set(f.formatResult, rule.MapPath, formatVal); nil != err { + return err } } - return f.jsonObj.EncodeJSON(), nil + return nil +} + +// String 获取格式化之后的字符串 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 21:18 2022/12/31 +func (f *filter) String() string { + return f.formatResult +} + +// Byte 获取格式化之后的字节数组 +// +// Author : zhangdeman001@ke.com<张德满> +// +// Date : 21:18 2022/12/31 +func (f *filter) Byte() []byte { + return []byte(f.String()) +} + +// Parse 解析返回结果 +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 21:20 2022/12/31 +func (f *filter) Parse(receiver interface{}) error { + if nil == receiver { + return errors.New("receiver is nil") + } + return json.Unmarshal(f.Byte(), receiver) } // getValue 获取值 @@ -141,4 +168,3 @@ func (f *filter) getValue(dataType string, defaultValue string) (interface{}, er return nil, errors.New(dataType + " is not support!") } } - diff --git a/go.mod b/go.mod index b94c429..9f43edb 100644 --- a/go.mod +++ b/go.mod @@ -11,9 +11,10 @@ require ( require ( github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-ini/ini v1.67.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/stretchr/testify v1.8.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect diff --git a/go.sum b/go.sum index 0df3337..5bd88c8 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,3 @@ -git.zhangdeman.cn/zhangdeman/util v0.0.0-20220704053716-ec9271c0f350 h1:y54/TJ7SCSG1LlKrX7QebDBdv+JoEl8JVs2kEqFoEXs= -git.zhangdeman.cn/zhangdeman/util v0.0.0-20220704053716-ec9271c0f350/go.mod h1:aAIhnUdQewLipl4bddewAsAeSLST9SRvgTcPN5ITkAQ= git.zhangdeman.cn/zhangdeman/util v0.0.0-20221021061434-e68b22a6e106 h1:xiiN+rLtBbDGRUbipVuwI1j2iRhuL3ejSm+EnDxzVMk= git.zhangdeman.cn/zhangdeman/util v0.0.0-20221021061434-e68b22a6e106/go.mod h1:zTir/0IWdK3E7n0GiaogyWHADAQnBtTdl2I6Z2/OPqw= github.com/Jeffail/gabs v1.4.0 h1://5fYRRTq1edjfIrQGvdkcd22pkYUrHZ5YC/H2GJVAo= @@ -9,7 +7,6 @@ github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394/go.mod h1:Q8n74mJTIg github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-ini/ini v1.66.6 h1:h6k2Bb0HWS/BXXHCXj4QHjxPmlIU4NK+7MuLp9SD+4k= github.com/go-ini/ini v1.66.6/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= @@ -24,14 +21,12 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/tidwall/gjson v1.14.1 h1:iymTbGkQBhveq21bEvAQ81I0LEBork8BFe1CUZXdyuo= github.com/tidwall/gjson v1.14.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= diff --git a/lexical.go b/lexical.go deleted file mode 100644 index d52be1a..0000000 --- a/lexical.go +++ /dev/null @@ -1,304 +0,0 @@ -// Package filter ... -// -// Description : JSON 词法分析 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 2022-07-04 17:52 -package filter - -import ( - "git.zhangdeman.cn/zhangdeman/util" - - "github.com/pkg/errors" -) - -// NewLexical 获取实例 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 20:52 2022/7/4 -func NewLexical(jsonData string) *lexical { - return &lexical{ - jsonData: jsonData, - lexicalResult: make([]*lexicalNode, 0), - } -} - -// lexical 词法解析 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 20:42 2022/7/4 -type lexical struct { - jsonData string - keyLeftRightTokenCnt int - lexicalResult []*lexicalNode -} - -// Parse 解析词法 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 18:11 2022/7/4 -func (l *lexical) Parse() error { - // mt.Println(jsonData) - if len(l.jsonData) < 2 { - return errors.New("input data is not json") - } - tmpStr := "" - for _, itemChar := range l.jsonData { - currentChar := string(itemChar) - tmpRealVal, tmpStrType := l.getTmpStrType(tmpStr) - if l.inputCharIsToken(currentChar, tmpStr, tmpStrType) { - if currentChar == keyLeftRightToken { - // 双引号计数 - l.keyLeftRightTokenCnt++ - } - // 是关键词 - if len(tmpStr) > 0 { - l.lexicalResult = append(l.lexicalResult, &lexicalNode{ - Val: tmpRealVal, - IsToken: false, - Type: tmpStrType, - }) - } - l.lexicalResult = append(l.lexicalResult, &lexicalNode{ - Val: currentChar, - IsToken: true, - Type: "string", - }) - tmpStr = "" - } else { - // 不是关键词, 继续向后走 - if currentChar == " " && l.keyLeftRightTokenCnt%2 == 0 { - // 当前字符是空格, 只有在 "" 之间方才有效 , 关键字之间的空格, 忽略即可 - continue - } - if (currentChar == "\n" || currentChar == "\t") && (l.keyLeftRightTokenCnt%2 == 0 || tmpStrType == "number" || tmpStrType == "bool") { - // 数字或者 bool 之后的 \n \t 无意义 , 不在 "" 之间也无意义 - continue - } - tmpStr = tmpStr + currentChar - } - } - - if len(tmpStr) > 0 { - l.lexicalResult = append(l.lexicalResult, &lexicalNode{ - Val: tmpStr, - IsToken: false, - Type: NodeTypeString, - }) - } - - return nil -} - -// inputCharIsToken 输入字符是否为关键字 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 18:15 2022/7/4 -func (l *lexical) inputCharIsToken(inputChar string, tmpStr string, tmpStrType string) bool { - tokenList := []string{ - // list 类型起始 - listLeftToken, - // listRight list 类型结束 - listRightToken, - // 对象起始 - objectLeftToken, - // 对象结束 - objectRightToken, - // key 值的起始 - keyLeftRightToken, - // 转义符 - // escapeCharacterToken, - // 冒号 - colonToken, - // 逗号 - commaToken, - } - preCheck := false - for _, item := range tokenList { - if item == inputChar { - preCheck = true - break - } - } - - if !preCheck { - // 输入一定不是关键字 - return false - } - - if len(tmpStr) == 0 && inputChar == keyLeftRightToken { - // 双引号关键字 - return true - } - tmpStrByte := []byte(tmpStr) - if len(tmpStrByte) > 0 { - // 字符串拼接过程中, 只有 " 可能是关键字 - // 判断转义符个数 - escapeCharacterTokenCnt := 0 - for i := len(tmpStrByte) - 1; i >= 0; i-- { - if string(tmpStrByte[i]) == escapeCharacterToken { - // 前一个是转义符 - escapeCharacterTokenCnt++ - continue - } - // 非连续的转义符不计数 - break - } - // 0 个 或者 偶数个转义符, " 是关键字 - if inputChar == keyLeftRightToken && escapeCharacterTokenCnt%2 == 0 { - return true - } - } - - var preNode *lexicalNode - if len(l.lexicalResult) > 0 { - preNode = l.lexicalResult[len(l.lexicalResult)-1] - } - - if inputChar == colonToken && nil != preNode && preNode.ValStr() == keyLeftRightToken && l.keyLeftRightTokenCnt > 0 && l.keyLeftRightTokenCnt%2 == 0 { - // : 必须出现在偶数 " 之后才是关键字 - return true - } - - // , 是关键字的场景 - // {"name":"zhangsan", "age":"18"} - // [{"name":"zhangsan", "age":"18"}, {"name":"zhangsan", "age":"18"}] - // [[],[]] - // [1,2,3] - // [true,false,true] - if inputChar == commaToken && ( - // 对应 {"name":"zhangsan", "age":"18"} - (nil != preNode && preNode.ValStr() == keyLeftRightToken) || - // 对应[{"name":"zhangsan", "age":"18"}, {"name":"zhangsan", "age":"18"}] - (nil != preNode && preNode.ValStr() == objectRightToken) || - // 对应[[],[]] - (nil != preNode && preNode.ValStr() == listRightToken) || - // 对应 [true,false,true] / [1,2,3] / [1,true,2,false] - (nil != preNode && (preNode.ValStr() == colonToken || preNode.ValStr() == listLeftToken || preNode.ValStr() == commaToken) && (tmpStrType == "number" || tmpStrType == "bool"))) { // 对应 - return true - } - - // { 可能出现的情况 - // {} - // [{}] [{}, {}] - // {"person": {}} - if inputChar == objectLeftToken && len(tmpStr) == 0 && - (nil == preNode || // 对应 {} - (nil != preNode && preNode.ValStr() == listLeftToken) || // 对应 [{}] - (nil != preNode && preNode.ValStr() == colonToken) || // 对应 {"person": {}} - (nil != preNode && preNode.ValStr() == commaToken)) { // 对应 [{}, {}] - // { 是关键字 - return true - } - - // } 可能出出现的情况 - // {} - // [{}] - // {"name":"zhangsan"} - // {"person": {"name":"zhangsan"}} - // {"person": {"name":"zhangsan", "age": 18}} - // {"person": {"work":true}} - // {"person": {"like":1}} - // {"person_list": [{"like":1}]} - if inputChar == objectRightToken && len(tmpStr) == 0 && ( - // 对应 {}, [{}] - (nil != preNode && preNode.ValStr() == objectLeftToken) || - // 对应 {"name":"zhangsan"} - (nil != preNode && preNode.ValStr() == keyLeftRightToken) || - // 对应 {"person": {"name":"zhangsan"}} - (nil != preNode && preNode.ValStr() == objectRightToken)) || - // 对应 {"person": {"work":true}} / {"person": {"like":1}} - (nil != preNode && preNode.ValStr() == colonToken && (tmpStr == "number" || tmpStr == "bool")) || - // 对应 {"person_list": [{"like":1}]} - (nil != preNode && preNode.ValStr() == listRightToken) { - // } 是关键字 - return true - } - - // [ 可能出现的场景 - // [] - // [[],[]] - // "a": [] - if inputChar == listLeftToken && len(tmpStr) == 0 && (nil == preNode || // 对应 [] - (nil != preNode && preNode.ValStr() == listLeftToken) || // 对应 [[],[]] - (nil != preNode && preNode.ValStr() == colonToken)) { // 对应 "a": [] - // [ 是关键字 - return true - } - - // ] 可能出现的场景 - // [] - // [[],[]] - // [{}, {}] - // [1,2,3] - // [true, false] - // ["", "" ] - if inputChar == listRightToken && len(tmpStr) == 0 && ( - //对应 [] - (nil != preNode && preNode.ValStr() == listLeftToken) || - // 对应 [[],[]] - (nil != preNode && preNode.ValStr() == listRightToken) || - // [true, false] / - (nil != preNode && preNode.ValStr() == objectRightToken) || - // 对应 [{}, {}] / [1,2,3] - (nil != preNode && (tmpStrType == "number" || tmpStrType == "bool")) || - // 对应 ["", "" ] - (nil != preNode && preNode.ValStr() == keyLeftRightToken)) { - return true - } - return false -} - -func (l *lexical) getTmpStrType(tmpStr string) (interface{}, string) { - var preNode *lexicalNode - if len(l.lexicalResult) > 0 { - preNode = l.lexicalResult[len(l.lexicalResult)-1] - } - - // 数字只有一个场景 {"age": 18},多以若是数字, 则 前一次解析必为关键字 : - if nil != preNode && preNode.ValStr() == colonToken { - // 判断是否可转数字 - var floatVal float64 - if err := util.ConvertAssign(&floatVal, tmpStr); nil == err { - return floatVal, "number" - } - } - - // 判断是否为 bool - // bool 只有一个场景 {"work": true/false},多以若是数字, 则 前一次解析必为关键字 : - if nil != preNode && preNode.ValStr() == colonToken { - // 判断是否可转数字 - var boolVal bool - if err := util.ConvertAssign(&boolVal, tmpStr); nil == err { - return boolVal, "bool" - } - } - return tmpStr, "string" -} - -// String 将解析后的词法, 转换为字符串 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 14:34 2022/7/5 -func (l *lexical) String() string { - str := "" - for _, item := range l.lexicalResult { - str = str + item.ValStr() - } - return str -} - -// Result 词法分析的结果 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 14:55 2022/7/5 -func (l *lexical) Result() []*lexicalNode { - return l.lexicalResult -} diff --git a/lexical_test.go b/lexical_test.go deleted file mode 100644 index e475662..0000000 --- a/lexical_test.go +++ /dev/null @@ -1,40 +0,0 @@ -// Package filter ... -// -// Description : filter ... -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 2022-07-04 18:13 -package filter - -import ( - "fmt" - "testing" -) - -func Test_parseLexical(t *testing.T) { - jsonData := `{ - "start" : 123456, - "name" : "zhangsan", - "age":"18", - "extension":{ - "sex":"man", - "height":"180" - }, - "teacher_list":[ - { - "name":"t1", - "age":"11" - }, - { - "name":"t2", - "age":"12" - } - ] -}` - //jsonData = `{"name":"zhangsan","age":"18","extension":{"sex":"man","height":"180"},"teacher_list":[{"name":"t1","age":"11"},{"name":"t2","age":"12"}]}` - - instance := NewLexical(jsonData) - _ = instance.Parse() - fmt.Println(instance.String()) -} diff --git a/syntax.go b/syntax.go deleted file mode 100644 index b238cbc..0000000 --- a/syntax.go +++ /dev/null @@ -1,153 +0,0 @@ -// Package filter ... -// -// Description : 语法分析 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 2022-07-04 17:53 -package filter - -import ( - "github.com/pkg/errors" -) - -// NewSyntax 构建JSON语法树 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 14:33 2022/7/5 -func NewSyntax(jsonData string) *syntax { - return &syntax{ - lexicalInstance: NewLexical(jsonData), - syntaxResult: nil, - } -} - -type syntax struct { - // 词法分析实例 - lexicalInstance *lexical - // 词法分析结果 - lexicalResult []*lexicalNode - // 语法分析结果 - syntaxResult *jsonNode -} - -// Parse 构建语法树 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 16:35 2022/7/5 -func (s *syntax) Parse() error { - if err := s.lexicalInstance.Parse(); nil != err { - // 词法解析失败 - return err - } - s.lexicalResult = s.lexicalInstance.Result() - if len(s.lexicalResult) == 0 { - return errors.New("词法解析无任何结果") - } - // TODO : 循环处理 - generateNode, lastIdx, err := s.generateJSONNode(0) - if nil != err { - return err - } - if lastIdx >= len(s.lexicalResult) { - // 词法处理完成 - return nil - } - if nil == s.syntaxResult { - s.syntaxResult = generateNode - } - return nil -} - -// generateJSONNode 构建 JSONNode, 参数 : 从那个索引开始处理 返回值 : 当前 json 节点, 下次从那个索引开始处理, 异常 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 16:58 2022/7/5 -func (s *syntax) generateJSONNode(currentLexicalIdx int) (*jsonNode, int, error) { - generateJSONNode := &jsonNode{ - Name: "", - Parent: nil, - Son: nil, - PreBrother: nil, - NextBrother: nil, - Val: nil, - Type: "", - } - if s.lexicalResult[currentLexicalIdx].ValStr() == objectLeftToken { - // 对象的起始, 类型一定为对象 - generateJSONNode.Type = NodeTypeObject - - // 下一个必定为 " (原因: 词法分析时, 已经过滤掉无用的空格换行符等, JSON数据已压缩成单行字符串) - currentLexicalIdx++ - if s.lexicalResult[currentLexicalIdx].ValStr() != keyLeftRightToken { - // 不是 " 号, 说明数据格式非法 - return nil, currentLexicalIdx, errors.New("JSON格式非法") - } - - // 在下一个必定是 json 的 key - currentLexicalIdx++ - generateJSONNode.Name = s.lexicalResult[currentLexicalIdx].ValStr() - - // 在下一个必定是 " - currentLexicalIdx++ - if s.lexicalResult[currentLexicalIdx].ValStr() != keyLeftRightToken { - // 不是 " 号, 说明数据格式非法 - return nil, currentLexicalIdx, errors.New("JSON格式非法") - } - - // 在下一个必定是 : - currentLexicalIdx++ - if s.lexicalResult[currentLexicalIdx].ValStr() != colonToken { - // 不是 : 号, 说明数据格式非法 - return nil, currentLexicalIdx, errors.New("JSON格式非法") - } - - // 到了取值部分, 具体分析是简单 k -> v or k -> object or k -> list - currentLexicalIdx++ - if s.lexicalResult[currentLexicalIdx].IsToken && - s.lexicalResult[currentLexicalIdx].ValStr() != keyLeftRightToken { - // TODO : 是关键字, 且不是 " 号 , 说明不是简单KV, 后续具体处理 - return generateJSONNode, currentLexicalIdx, nil - } - // 简单KV, 下一个必定是取值 - if s.lexicalResult[currentLexicalIdx].ValStr() == keyLeftRightToken { - // 双引号 - currentLexicalIdx++ - generateJSONNode.Val = s.lexicalResult[currentLexicalIdx].ValStr() - // 跳过字符串闭合的 " 号 - currentLexicalIdx = currentLexicalIdx + 2 - // 判断 闭合 " 之后 是不是 , / ] / } - if s.lexicalResult[currentLexicalIdx].ValStr() == commaToken { - // 跳过 分割的 , - currentLexicalIdx++ - } else { - if s.lexicalResult[currentLexicalIdx].ValStr() == objectRightToken || - s.lexicalResult[currentLexicalIdx].ValStr() == listRightToken { - // 跳过整体对象的闭合标签 - currentLexicalIdx++ - } - if s.lexicalResult[currentLexicalIdx].ValStr() == commaToken { - // 还是 , - currentLexicalIdx++ - } - } - } else { - generateJSONNode.Val = s.lexicalResult[currentLexicalIdx].Val - generateJSONNode.Type = s.lexicalResult[currentLexicalIdx].Type - currentLexicalIdx++ - if s.lexicalResult[currentLexicalIdx].ValStr() == objectRightToken || - s.lexicalResult[currentLexicalIdx].ValStr() == listRightToken { - // 跳过整体对象的闭合标签 - currentLexicalIdx++ - } - if s.lexicalResult[currentLexicalIdx].ValStr() == commaToken { - // 还是 , - currentLexicalIdx++ - } - } - } - return generateJSONNode, currentLexicalIdx, nil -} diff --git a/syntax_test.go b/syntax_test.go deleted file mode 100644 index 2d25f30..0000000 --- a/syntax_test.go +++ /dev/null @@ -1,38 +0,0 @@ -// Package filter ... -// -// Description : filter ... -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 2022-07-05 17:43 -package filter - -import ( - "testing" -) - -func Test_syntax_Parse(t *testing.T) { - jsonData := `{ - "start" : 123456, - "name" : "zhangsan", - "age":"18", - "extension":{ - "sex":"man", - "height":"180" - }, - "teacher_list":[ - { - "name":"t1", - "age":"11" - }, - { - "name":"t2", - "age":"12" - } - ] -}` - //jsonData = `{"name":"zhangsan","age":"18","extension":{"sex":"man","height":"180"},"teacher_list":[{"name":"t1","age":"11"},{"name":"t2","age":"12"}]}` - - instance := NewSyntax(jsonData) - _ = instance.Parse() -} diff --git a/token.go b/token.go deleted file mode 100644 index 5e08207..0000000 --- a/token.go +++ /dev/null @@ -1,27 +0,0 @@ -// Package filter ... -// -// Description : 关键词定义 -// -// Author : go_developer@163.com<白茶清欢> -// -// Date : 2022-07-04 17:53 -package filter - -const ( - // list 类型起始 - listLeftToken = "[" - // listRight list 类型结束 - listRightToken = "]" - // 对象起始 - objectLeftToken = "{" - // 对象结束 - objectRightToken = "}" - // key 值的起始 - keyLeftRightToken = "\"" - // 转义符 - escapeCharacterToken = "\\" - // 冒号 - colonToken = ":" - // 逗号 - commaToken = "," -) -- 2.36.6 From cbde9c370784e784b0984e79425a49ddd1e81721 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E8=8C=B6=E6=B8=85=E6=AC=A2?= Date: Sun, 1 Jan 2023 17:31:25 +0800 Subject: [PATCH 2/7] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=8A=A8=E6=80=81?= =?UTF-8?q?=E8=AE=BE=E7=BD=AE=E5=80=BC=E5=A4=B1=E8=B4=A5=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- filter.go | 2 +- filter_test.go | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 11 +++++-- go.sum | 44 ++++++++++++++++++++++++++++ 4 files changed, 133 insertions(+), 3 deletions(-) create mode 100644 filter_test.go diff --git a/filter.go b/filter.go index b295b87..d3d8541 100644 --- a/filter.go +++ b/filter.go @@ -72,7 +72,7 @@ func (f *filter) Deal() error { if formatVal, err = f.getValue(rule.DataType, sourceVal); nil != err { return fmt.Errorf("%s = %v can not convert to %s : %s", rule.SourcePath, sourceResult.Value(), rule.DataType, err.Error()) } - if _, err = sjson.Set(f.formatResult, rule.MapPath, formatVal); nil != err { + if f.formatResult, err = sjson.Set(f.formatResult, rule.MapPath, formatVal); nil != err { return err } } diff --git a/filter_test.go b/filter_test.go new file mode 100644 index 0000000..5578437 --- /dev/null +++ b/filter_test.go @@ -0,0 +1,79 @@ +// Package filter ... +// +// Description : filter ... +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 2023-01-01 17:00 +package filter + +import ( + "fmt" + "github.com/smartystreets/goconvey/convey" + "testing" +) + +// TestNewFilter ... +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 17:00 2023/1/1 +func TestNewFilter(t *testing.T) { + var ( + err error + ) + testData := `{ + "base":{ + "name":"go_developer", + "age":18, + "height":180 + }, + "company":{ + "name":"wajijiwa", + "start":"2023-01-01" + } +}` + filterRuleList := []MapRule{ + { + SourcePath: "base.name", + MapPath: "user_name", + Required: true, + DataType: "string", + DefaultValue: "", + }, + { + SourcePath: "base.age", + MapPath: "user_age", + Required: true, + DataType: "int", + DefaultValue: "", + }, + { + SourcePath: "base.height", + MapPath: "user_height", + Required: true, + DataType: "string", + DefaultValue: "", + }, + { + SourcePath: "company.name", + MapPath: "company_name", + Required: true, + DataType: "string", + DefaultValue: "", + }, + { + SourcePath: "company.start", + MapPath: "company_start", + Required: true, + DataType: "string", + DefaultValue: "", + }, + } + f := NewFilter(testData, filterRuleList) + convey.Convey("基础对象重写", t, func() { + err = f.Deal() + convey.So(err, convey.ShouldEqual, nil) + fmt.Println(f.String()) + }) +} diff --git a/go.mod b/go.mod index 9f43edb..1c645ca 100644 --- a/go.mod +++ b/go.mod @@ -4,19 +4,26 @@ go 1.17 require ( git.zhangdeman.cn/zhangdeman/util v0.0.0-20221021061434-e68b22a6e106 - github.com/Jeffail/gabs v1.4.0 github.com/pkg/errors v0.9.1 + github.com/smartystreets/goconvey v1.7.2 github.com/tidwall/gjson v1.14.4 + github.com/tidwall/sjson v1.2.5 ) require ( + github.com/Jeffail/gabs v1.4.0 // indirect github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-ini/ini v1.67.0 // indirect + github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 // indirect + github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/smartystreets/assertions v1.2.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect - github.com/tidwall/sjson v1.2.5 // indirect + golang.org/x/mod v0.7.0 // indirect + golang.org/x/sys v0.3.0 // indirect + golang.org/x/tools v0.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 5bd88c8..4f21a83 100644 --- a/go.sum +++ b/go.sum @@ -10,10 +10,18 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/go-ini/ini v1.66.6/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= +github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= +github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= +github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -32,6 +40,42 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.4.0 h1:7mTAgkunk3fr4GAloyyCasadO6h9zSsQZbwvcaIciV4= +golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -- 2.36.6 From 4ed58910654770c5d838002aff2139e4fa34c42e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E8=8C=B6=E6=B8=85=E6=AC=A2?= Date: Sun, 1 Jan 2023 19:28:20 +0800 Subject: [PATCH 3/7] save code --- filter.go | 44 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/filter.go b/filter.go index d3d8541..790694c 100644 --- a/filter.go +++ b/filter.go @@ -54,8 +54,11 @@ func (f *filter) Deal() error { ) for _, rule := range f.filterRuleList { - if strings.Contains(rule.SourcePath, "[]") { + if f.IsArray(rule) { // 对于list的处理 + if err = f.handleArray(rule); nil != err { + return err + } continue } sourceResult := gjson.Get(f.sourceData, rule.SourcePath) @@ -79,6 +82,45 @@ func (f *filter) Deal() error { return nil } +// IsArray 判断是否为数组 +// +// Author : zhangdeman001@ke.com<张德满> +// +// Date : 17:48 2023/1/1 +func (f *filter) IsArray(rule MapRule) bool { + return strings.Contains(rule.SourcePath, "[]") +} + +// handleArray 处理数组(最复杂的场景) +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 17:41 2023/1/1 +func (f *filter) handleArray(rule MapRule) error { + sourcePathArray := strings.Split(rule.SourcePath, "[]") + mapPathArray := strings.Split(rule.MapPath, "[]") + if len(sourcePathArray) != len(mapPathArray) { + if len(mapPathArray) != 1 { + return errors.New("map rule is invalid") + } + // 提取某一个list下的字段, 组成一个list + + return nil + } + return nil +} + +// getAllFinalData ... +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 18:42 2023/1/1 +func (f *filter) getAllFinalData(pathArr []string) []string { + res := make([]string, 0) + + return res +} + // String 获取格式化之后的字符串 // // Author : go_developer@163.com<白茶清欢> -- 2.36.6 From 123c22e8b2e3d26b0fca866620b2ea253a3d9fbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E8=8C=B6=E6=B8=85=E6=AC=A2?= Date: Mon, 2 Jan 2023 21:17:51 +0800 Subject: [PATCH 4/7] save code --- filter.go | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/filter.go b/filter.go index 790694c..3bca265 100644 --- a/filter.go +++ b/filter.go @@ -97,6 +97,10 @@ func (f *filter) IsArray(rule MapRule) bool { // // Date : 17:41 2023/1/1 func (f *filter) handleArray(rule MapRule) error { + var ( + err error + ) + sourcePathArray := strings.Split(rule.SourcePath, "[]") mapPathArray := strings.Split(rule.MapPath, "[]") if len(sourcePathArray) != len(mapPathArray) { @@ -104,21 +108,32 @@ func (f *filter) handleArray(rule MapRule) error { return errors.New("map rule is invalid") } // 提取某一个list下的字段, 组成一个list - + res := make([]string, 0) + f.getAllFinalData(res, gjson.Get(f.sourceData, sourcePathArray[0]).Array(), sourcePathArray[1:]) + if f.formatResult, err = sjson.Set(f.formatResult, mapPathArray[0], res); nil != err { + return err + } return nil } return nil } -// getAllFinalData ... +// extraFinalResult 提取全部最底层结果 // -// Author : go_developer@163.com<白茶清欢> +// Author : zhangdeman001@ke.com<张德满> // -// Date : 18:42 2023/1/1 -func (f *filter) getAllFinalData(pathArr []string) []string { - res := make([]string, 0) - - return res +// Date : 14:00 2023/1/2 +func (f *filter) getAllFinalData(res []string, resultList []gjson.Result, pathArr []string) { + if len(pathArr) == 0 { + for _, item := range resultList { + res = append(res, item.String()) + } + return + } + for _, item := range resultList { + f.extraFinalResult(res, item.Array(), pathArr[1:]) + } + return } // String 获取格式化之后的字符串 -- 2.36.6 From 3dff92188f3c5a640919b470d9610e76887a3743 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E8=8C=B6=E6=B8=85=E6=AC=A2?= Date: Mon, 2 Jan 2023 22:12:48 +0800 Subject: [PATCH 5/7] =?UTF-8?q?=E6=94=AF=E6=8C=81=E6=8F=90=E5=8F=96?= =?UTF-8?q?=E6=8C=87=E5=AE=9A=E7=9A=84=E5=80=BC=E5=88=B0=E4=B8=80=E4=B8=AA?= =?UTF-8?q?=E5=AD=97=E6=AE=B5=EF=BC=8C=E5=BD=A2=E6=88=90=E5=88=97=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- filter.go | 25 ++++++++++++++++++------- filter_test.go | 43 ++++++++++++++++++++++++++++++++++++++++++- go.mod | 3 --- go.sum | 31 ------------------------------- 4 files changed, 60 insertions(+), 42 deletions(-) diff --git a/filter.go b/filter.go index 3bca265..5466d5d 100644 --- a/filter.go +++ b/filter.go @@ -10,9 +10,10 @@ package filter import ( "encoding/json" "fmt" - "github.com/tidwall/sjson" "strings" + "github.com/tidwall/sjson" + "git.zhangdeman.cn/zhangdeman/util" "github.com/pkg/errors" "github.com/tidwall/gjson" @@ -102,14 +103,24 @@ func (f *filter) handleArray(rule MapRule) error { ) sourcePathArray := strings.Split(rule.SourcePath, "[]") - mapPathArray := strings.Split(rule.MapPath, "[]") + for idx, item := range sourcePathArray { + sourcePathArray[idx] = strings.Trim(item, ".") + } + mapPathArray := strings.Split(strings.TrimRight(rule.MapPath, ".[]"), "[]") + for idx, item := range mapPathArray { + mapPathArray[idx] = strings.Trim(item, ".") + } if len(sourcePathArray) != len(mapPathArray) { if len(mapPathArray) != 1 { return errors.New("map rule is invalid") } // 提取某一个list下的字段, 组成一个list res := make([]string, 0) - f.getAllFinalData(res, gjson.Get(f.sourceData, sourcePathArray[0]).Array(), sourcePathArray[1:]) + if len(sourcePathArray[0]) == 0 { + f.getAllFinalData(&res, gjson.Parse(f.sourceData).Array(), sourcePathArray[1:]) + } else { + f.getAllFinalData(&res, gjson.Get(f.sourceData, sourcePathArray[0]).Array(), sourcePathArray[1:]) + } if f.formatResult, err = sjson.Set(f.formatResult, mapPathArray[0], res); nil != err { return err } @@ -123,15 +134,15 @@ func (f *filter) handleArray(rule MapRule) error { // Author : zhangdeman001@ke.com<张德满> // // Date : 14:00 2023/1/2 -func (f *filter) getAllFinalData(res []string, resultList []gjson.Result, pathArr []string) { - if len(pathArr) == 0 { +func (f *filter) getAllFinalData(res *[]string, resultList []gjson.Result, pathArr []string) { + if len(pathArr) == 1 { for _, item := range resultList { - res = append(res, item.String()) + *res = append(*res, item.Get(pathArr[0]).String()) } return } for _, item := range resultList { - f.extraFinalResult(res, item.Array(), pathArr[1:]) + f.getAllFinalData(res, item.Array(), pathArr[1:]) } return } diff --git a/filter_test.go b/filter_test.go index 5578437..f606582 100644 --- a/filter_test.go +++ b/filter_test.go @@ -9,8 +9,9 @@ package filter import ( "fmt" - "github.com/smartystreets/goconvey/convey" "testing" + + "github.com/smartystreets/goconvey/convey" ) // TestNewFilter ... @@ -77,3 +78,43 @@ func TestNewFilter(t *testing.T) { fmt.Println(f.String()) }) } + +// TestNewFilterForArrayOne ... +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 21:43 2023/1/2 +func TestNewFilterForArrayOne(t *testing.T) { + var ( + err error + ) + testData := `[ + { + "name":"zhangdeman" + }, + { + "name":"zhang" + }, + { + "name":"de" + }, + { + "name":"man" + } +]` + filterRuleList := []MapRule{ + { + SourcePath: "[].name", + MapPath: "user_name.[]", + Required: true, + DataType: "string", + DefaultValue: "", + }, + } + f := NewFilter(testData, filterRuleList) + convey.Convey("提取列表字段，构成新的列表", t, func() { + err = f.Deal() + convey.So(err, convey.ShouldEqual, nil) + fmt.Println(f.String()) + }) +} diff --git a/go.mod b/go.mod index 1c645ca..8acef6b 100644 --- a/go.mod +++ b/go.mod @@ -22,8 +22,5 @@ require ( github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect - golang.org/x/mod v0.7.0 // indirect - golang.org/x/sys v0.3.0 // indirect - golang.org/x/tools v0.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 4f21a83..92d1587 100644 --- a/go.sum +++ b/go.sum @@ -40,42 +40,11 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= -golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.4.0 h1:7mTAgkunk3fr4GAloyyCasadO6h9zSsQZbwvcaIciV4= -golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -- 2.36.6 From 4a5f73bb02cdf27f9f4afbd389ca7359c5e44b37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E8=8C=B6=E6=B8=85=E6=AC=A2?= Date: Sat, 7 Jan 2023 23:24:55 +0800 Subject: [PATCH 6/7] update --- filter.go | 3 ++- filter_test.go | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/filter.go b/filter.go index 5466d5d..6c39c0f 100644 --- a/filter.go +++ b/filter.go @@ -14,8 +14,9 @@ import ( "github.com/tidwall/sjson" + "errors" + "git.zhangdeman.cn/zhangdeman/util" - "github.com/pkg/errors" "github.com/tidwall/gjson" ) diff --git a/filter_test.go b/filter_test.go index f606582..ae15bf8 100644 --- a/filter_test.go +++ b/filter_test.go @@ -118,3 +118,43 @@ func TestNewFilterForArrayOne(t *testing.T) { fmt.Println(f.String()) }) } + +// TestNewFilterForArrayTwo ... +// +// Author : go_developer@163.com<白茶清欢> +// +// Date : 22:22 2023/1/2 +func TestNewFilterForArrayTwo(t *testing.T) { + var ( + err error + ) + testData := `{"user_list":[ + { + "name":"zhangdeman1" + }, + { + "name":"zhang1" + }, + { + "name":"de1" + }, + { + "name":"man1" + } +]}` + filterRuleList := []MapRule{ + { + SourcePath: "user_list.[].name", + MapPath: "user.name_list.[]", + Required: true, + DataType: "string", + DefaultValue: "", + }, + } + f := NewFilter(testData, filterRuleList) + convey.Convey("提取列表字段，构成新的列表", t, func() { + err = f.Deal() + convey.So(err, convey.ShouldEqual, nil) + fmt.Println(f.String()) + }) +} -- 2.36.6 From 4db7e68aa6986e2f1b1559c35bc18e74ae589b56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E8=8C=B6=E6=B8=85=E6=AC=A2?= Date: Mon, 6 Feb 2023 11:40:05 +0800 Subject: [PATCH 7/7] update go mod --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 8acef6b..f4d46cc 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module git.zhangdeman.cn/zhangdeman/filter go 1.17 require ( - git.zhangdeman.cn/zhangdeman/util v0.0.0-20221021061434-e68b22a6e106 + git.zhangdeman.cn/zhangdeman/util v0.0.0-20230113095943-b4b3e261e0c4 github.com/pkg/errors v0.9.1 github.com/smartystreets/goconvey v1.7.2 github.com/tidwall/gjson v1.14.4 diff --git a/go.sum b/go.sum index 92d1587..3ae0db9 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ git.zhangdeman.cn/zhangdeman/util v0.0.0-20221021061434-e68b22a6e106 h1:xiiN+rLtBbDGRUbipVuwI1j2iRhuL3ejSm+EnDxzVMk= git.zhangdeman.cn/zhangdeman/util v0.0.0-20221021061434-e68b22a6e106/go.mod h1:zTir/0IWdK3E7n0GiaogyWHADAQnBtTdl2I6Z2/OPqw= +git.zhangdeman.cn/zhangdeman/util v0.0.0-20230113095943-b4b3e261e0c4 h1:1WclY9P8l8o/NZ3ZR/mupm8LtowjQ/Q4UNGXR32f0OQ= +git.zhangdeman.cn/zhangdeman/util v0.0.0-20230113095943-b4b3e261e0c4/go.mod h1:zTir/0IWdK3E7n0GiaogyWHADAQnBtTdl2I6Z2/OPqw= github.com/Jeffail/gabs v1.4.0 h1://5fYRRTq1edjfIrQGvdkcd22pkYUrHZ5YC/H2GJVAo= github.com/Jeffail/gabs v1.4.0/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc= github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 h1:OYA+5W64v3OgClL+IrOD63t4i/RW7RqrAVl9LTZ9UqQ= -- 2.36.6