save code

go.mod

@@ -7,6 +7,7 @@ replace github.com/coreos/bbolt v1.3.4 => go.etcd.io/bbolt v1.3.4
 replace google.golang.org/grpc => google.golang.org/grpc v1.26.0
 
 require (
+	github.com/Jeffail/gabs v1.4.0
 	github.com/Shopify/sarama v1.21.0
 	github.com/apolloconfig/agollo/v4 v4.0.9
 	github.com/coreos/etcd v3.3.27+incompatible

go.sum

@@ -40,6 +40,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DataDog/zstd v1.3.5 h1:DtpNbljikUepEPD16hD4LvIcmhnhdLTiW/5pHgbmp14=
 github.com/DataDog/zstd v1.3.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
+github.com/Jeffail/gabs v1.4.0 h1://5fYRRTq1edjfIrQGvdkcd22pkYUrHZ5YC/H2GJVAo=
+github.com/Jeffail/gabs v1.4.0/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc=
 github.com/Shopify/sarama v1.21.0 h1:0GKs+e8mn1RRUzfg9oUXv3v7ZieQLmOZF/bfnmmGhM8=
 github.com/Shopify/sarama v1.21.0/go.mod h1:yuqtN/pe8cXRWG5zPaO7hCfNJp5MwmkoJEoLjkm5tCQ=
 github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc=

json_tool/filter.go

@@ -75,12 +75,12 @@ func (f *Filter) Result() (*DynamicJSON, error) {
 		}
 		// 支持list再抽取一层,处于性能考虑,这么限制,不做递归无限深度处理
 		// 还能继续抽取,就认为是 []map[string]interface{}
-		ketList := strings.Split(pathArr[1], ".")
+		keyList := strings.Split(pathArr[1], ".")
 		for idx, item := range val.Array() {
 			data := item.Map()
-			for _, key := range ketList {
-				if v, exist := data[key]; exist {
-					result.SetValue(strings.ReplaceAll(newDataRule.MapKey, "[]", fmt.Sprintf("[%d]", idx)), data[key].Value(), v.IsObject() || v.IsArray())
+			for _, key := range keyList {
+				if _, exist := data[key]; exist {
+					result.SetValue(strings.ReplaceAll(newDataRule.MapKey, "[]", fmt.Sprintf("[%d]", idx)), data[key].String(), newDataRule.IsComplexType)
 				} else {
 					// 结果集中不存在对应key,设置默认值
 					result.SetValue(strings.ReplaceAll(newDataRule.MapKey, "[]", fmt.Sprintf("[%d]", idx)), newDataRule.DefaultValue, newDataRule.IsComplexType)

json_tool/gabs.go

@@ -0,0 +1,294 @@
+// Package json_tool ...
+//
+// Description : json_tool ...
+//
+// Author : go_developer@163.com<张德满>
+//
+// Date : 2022/01/22 9:19 PM
+package json_tool
+
+import (
+	"encoding/json"
+	"fmt"
+	"reflect"
+	"strings"
+
+	"git.zhangdeman.cn/zhangdeman/gopkg/util"
+	"github.com/pkg/errors"
+
+	"github.com/tidwall/gjson"
+
+	"github.com/Jeffail/gabs"
+)
+
+// FilterDataRule 参数过滤规则
+//
+// Author : go_developer@163.com<白茶清欢>
+//
+// Date : 2022/1/22 9:44 PM
+type FilterDataRule struct {
+	SourceKey    string      // 原始数据路径
+	MapKey       string      // 提取后映射到的数据路径
+	DefaultValue interface{} // 原始数据路径不存在时的默认值
+	WithDefault  bool        // 是否使用默认值
+}
+
+// NewDataFilter 获取数据过滤方法实例
+//
+// Author : go_developer@163.com<白茶清欢>
+//
+// Date : 2022/1/22 9:50 PM
+func NewDataFilter(source string, filterRule []*FilterDataRule) *DataFilter {
+	return &DataFilter{
+		source:          source,
+		filterRule:      filterRule,
+		hasDealDiffPath: make(map[string]string),
+	}
+}
+
+// DataFilter 数据过滤
+//
+// Author : go_developer@163.com<白茶清欢>
+//
+// Date : 2022/1/22 9:20 PM
+type DataFilter struct {
+	source          string
+	filterRule      []*FilterDataRule
+	itemKeyToSlice  bool
+	hasDealDiffPath map[string]string
+}
+
+// Filter 数据过滤
+//
+// Author : go_developer@163.com<白茶清欢>
+//
+// Date : 2022/1/22 9:36 PM
+func (df *DataFilter) Filter() (string, error) {
+	var (
+		jsonObject *gabs.Container
+		err        error
+	)
+	// 格式化映射规则
+	df.formatRule()
+	// 记录 obj => slice 的数据类型
+	obg2slice := make(map[string]string)
+	// 创建数据的根结点
+	jsonObject = gabs.New()
+	for _, item := range df.filterRule {
+		// 数据源路径不识数组, 多个key写入到同一个map key, 并且map key 不是以[]结尾, 自动格式化
+		// 目标位置, 是一个数组
+		if df.pathIsArrayValue(item.MapKey) {
+			realMapKey := strings.Trim(item.MapKey, ".[]")
+			if exist := jsonObject.Exists(realMapKey); !exist {
+				if _, err = jsonObject.ArrayP(realMapKey); nil != err {
+					return "", err
+				}
+			}
+			valueResult := gjson.Get(df.source, item.SourceKey)
+			dataType := df.getValueType(valueResult)
+			if _, exist := obg2slice[realMapKey]; !exist {
+				obg2slice[realMapKey] = dataType
+			}
+			if dataType != obg2slice[realMapKey] {
+				return "", errors.New(realMapKey + " 预期写入的字段数据类型不一致")
+			}
+			sourcePathArr := strings.Split(item.SourceKey, ".[].")
+			mapPathArr := strings.Split(realMapKey, ".[].")
+
+			result := gabs.New()
+			_, _ = result.ArrayP(mapPathArr[0])
+			df.SetArrayData("{\""+sourcePathArr[0]+"\":"+gjson.Get(df.source, sourcePathArr[0]).String()+"}", result, sourcePathArr, mapPathArr)
+			if err = jsonObject.ArrayAppend(valueResult.Value(), realMapKey); nil != err {
+				return "", err
+			}
+			continue
+		}
+		sourceSearchResult := gjson.Get(df.source, item.SourceKey)
+		if !sourceSearchResult.Exists() {
+			if item.WithDefault {
+				if _, err = jsonObject.SetP(item.DefaultValue, item.MapKey); nil != err {
+					return "", err
+				}
+			}
+			continue
+		}
+		if _, err = jsonObject.SetP(sourceSearchResult.Value(), item.MapKey); nil != err {
+			return "", err
+		}
+	}
+	return jsonObject.String(), nil
+}
+
+// UserItemToSlice 支持多个独立的字段合并到slice中
+//
+// Author : go_developer@163.com<白茶清欢>
+//
+// Date : 3:27 PM 2022/1/24
+func (df *DataFilter) UserItemToSlice() {
+	df.itemKeyToSlice = true
+}
+
+// getValueType 获取数据类型
+//
+// Author : go_developer@163.com<白茶清欢>
+//
+// Date : 2022/1/23 12:45 AM
+func (df *DataFilter) getValueType(valueResult gjson.Result) string {
+	dataTypeVal := reflect.TypeOf(valueResult.Value())
+	if nil == dataTypeVal {
+		return "NIL"
+	}
+	dataType := dataTypeVal.String()
+	if strings.Contains(dataType, "int") {
+		return "int64"
+	}
+	if strings.Contains(dataType, "float") {
+		return "float64"
+	}
+	return dataType
+}
+
+// pathIsArrayValue 判断路径是否为数组值
+//
+// Author : go_developer@163.com<白茶清欢>
+//
+// Date : 2022/1/23 12:56 AM
+func (df *DataFilter) pathIsArrayValue(path string) bool {
+	return strings.Contains(path, "[]")
+}
+
+// formatRule 格式化映射规则
+//
+// Author : go_developer@163.com<白茶清欢>
+//
+// Date : 2:43 PM 2022/1/24
+func (df *DataFilter) formatRule() {
+	mapKeyCnt := make(map[string]int)
+	for _, item := range df.filterRule {
+		// source 为数组, map 不是
+		if df.pathIsArrayValue(item.SourceKey) {
+			if !df.pathIsArrayValue(item.MapKey) {
+				item.MapKey = item.MapKey + ".[]"
+			} else {
+				// source 是数组, map也是数组, 检测数组层级匹配
+				sourcePathArr := strings.Split(item.SourceKey, ".[].")
+				mapPathArr := strings.Split(item.MapKey, ".[].")
+				if len(sourcePathArr) == len(mapPathArr) {
+					// 数组层级深度相同,无需特殊处理
+					continue
+				}
+				// 数组层级深度不同,重新对对齐数据
+				diffArr := sourcePathArr[0 : len(sourcePathArr)-len(mapPathArr)+1]
+				if newPath := df.dealDiffArr(diffArr); len(newPath) > 0 {
+					sourcePathArr[len(sourcePathArr)-len(mapPathArr)] = newPath
+					item.SourceKey = strings.Join(sourcePathArr[len(sourcePathArr)-len(mapPathArr):], ".[].")
+				}
+			}
+		} else {
+			if df.pathIsArrayValue(item.MapKey) {
+				continue
+			}
+			// source 不是数组, map 也不是
+			if !df.itemKeyToSlice {
+				continue
+			}
+			mapKeyCnt[item.MapKey]++
+		}
+	}
+	// 多个source指向一个map,自动转化为list
+	for _, item := range df.filterRule {
+		if mapKeyCnt[item.MapKey] > 1 {
+			item.MapKey = item.MapKey + ".[]"
+		}
+	}
+}
+
+// dealDiffArr 提取数据映射关系
+//
+// Author : go_developer@163.com<白茶清欢>
+//
+// Date : 5:04 下午 2022/1/25
+func (df *DataFilter) dealDiffArr(diffArr []string) string {
+	if len(diffArr) == 0 {
+		return ""
+	}
+	diffArrStr := strings.Join(diffArr, ".[].")
+	if _, exist := df.hasDealDiffPath[diffArrStr]; exist {
+		// 已经处理过, 不再重复处理
+		return df.hasDealDiffPath[diffArrStr]
+	}
+
+	// 没处理过, 开始处理
+	jsonResultList := df.getArrayData(df.source, diffArr)
+	if len(jsonResultList) == 0 {
+		return ""
+	}
+	newPath := util.GenRandomString("", 8)
+	var result map[string]interface{}
+	_ = json.Unmarshal([]byte(df.source), &result)
+	JSONObject, _ := gabs.Consume(result)
+	_, _ = JSONObject.ArrayP(newPath)
+	for _, item := range jsonResultList {
+		if err := JSONObject.ArrayAppendP(item.Value(), newPath); nil != err {
+			fmt.Println(err.Error())
+		}
+	}
+
+	df.source = JSONObject.String()
+	df.hasDealDiffPath[diffArrStr] = newPath
+	return newPath
+}
+
+// getArrayData 获取数据
+//
+// Author : go_developer@163.com<白茶清欢>
+//
+// Date : 2:22 下午 2022/1/26
+func (df *DataFilter) getArrayData(source string, pathArr []string) []gjson.Result {
+	if len(pathArr) == 1 {
+		return gjson.Get(source, pathArr[0]).Array()
+	}
+	resultList := make([]gjson.Result, 0)
+	dataList := gjson.Get(source, pathArr[0]).Array()
+	for idx := 0; idx < len(dataList); idx++ {
+		resultList = append(resultList, df.getArrayData(dataList[idx].String(), pathArr[1:])...)
+	}
+	return resultList
+}
+
+// SetArrayData 设置数组数据
+//
+// Author : go_developer@163.com<白茶清欢>
+//
+// Date : 5:05 下午 2022/2/2
+func (df *DataFilter) SetArrayData(sourceData string, jsonObject *gabs.Container, sourcePathArr []string, mapPathArr []string) *gabs.Container {
+	jsonObject = gabs.New()
+
+	for idx, sourcePath := range sourcePathArr {
+		if idx < len(sourcePathArr)-1 {
+			if !jsonObject.Exists(sourcePath) {
+				_, _ = jsonObject.ArrayP(sourcePath)
+			}
+		}
+		instance, _ := gabs.ParseJSON([]byte(sourceData))
+		if !instance.Exists() {
+			fmt.Println(sourcePathArr[len(sourcePathArr)-1] + " 不存在")
+		} else {
+			dataList, _ := instance.Children()
+			for _, item := range dataList {
+				cItem := gabs.New()
+				cItem.SetP(gjson.Get(item.String(), sourcePath).String(), mapPathArr[idx])
+				jsonObject.ArrayAppendP(cItem.Data(), mapPathArr[idx])
+			}
+			//jsonObject.ArrayAppend(jsonObject.Data())
+			// fmt.Println("数据 : ", jsonObject.String())
+			// jsonObject.ArrayAppendP(result.Data(), mapPathArr[idx])
+
+		}
+
+		df.SetArrayData(gjson.Get(sourceData, sourcePathArr[idx]).String(), jsonObject, sourcePathArr[idx+1:], mapPathArr[idx+1:])
+		// jsonObject.ArrayAppendP(v.Data(), mapPathArr[idx])
+	}
+	fmt.Println("最终 : ", jsonObject.String())
+	return jsonObject
+}

json_tool/json_test.go

@@ -64,8 +64,8 @@ func TestSelect(t *testing.T) {
 			"height": 180,
 			"slice":  []int{1, 2, 3},
 		},
-		"slice": []int{1, 2, 3},
-		"map":   map[string]interface{}{"a": 1, "b": 2, "c": 4},
+		"slice_data": []int{1, 2, 3},
+		"map":        map[string]interface{}{"a": 1, "b": 2, "c": 4},
 		"table": []map[string]interface{}{
 			{"name": "alex", "age": 18, "number": 1},
 			{"name": "bob", "age": 28, "number": 2},
@@ -73,11 +73,7 @@ func TestSelect(t *testing.T) {
 		},
 	}
 	rule := map[string]MapDataRule{
-		"name": {
-			MapKey:        "user_name",
-			DefaultValue:  "用户姓名默认值",
-			IsComplexType: false,
-		},
+		"name":          {MapKey: "user_name", DefaultValue: "用户姓名默认值", IsComplexType: false},
 		"extra.age":     {MapKey: "user_age", DefaultValue: "用户年龄默认值", IsComplexType: false},
 		"extra.height":  {MapKey: "user_height", DefaultValue: "扩展高度默认值", IsComplexType: false},
 		"table.[].name": {MapKey: "slice.[].name_modify", DefaultValue: "列表姓名默认值", IsComplexType: false},
@@ -173,3 +169,109 @@ func TestParseWithType(t *testing.T) {
 	byteData, _ := json.Marshal(source)
 	fmt.Println(GetJSONDataStructWithType(string(byteData)))
 }
+
+// TestDataFilter 测试数据过滤
+//
+// Author : go_developer@163.com<白茶清欢>
+//
+// Date : 2022/1/22 10:19 PM
+func TestDataFilter(t *testing.T) {
+	source := map[string]interface{}{
+		"name": "zhangdeman",
+		"extra": map[string]interface{}{
+			"age":    18,
+			"height": 180,
+			"slice":  []int{1, 2, 3},
+		},
+		"slice_data": []int{1, 2, 3},
+		"map":        map[string]interface{}{"a": 1, "b": 2, "c": 4},
+		"table": []map[string]interface{}{
+			{"name": "alex", "age": 18, "number": 1},
+			{"name": "bob", "age": 28, "number": 2},
+			{"name": "bob", "age": 28, "number": 2, "list": []int{1, 2, 3}},
+		},
+	}
+	rule := []*FilterDataRule{
+		{SourceKey: "name", MapKey: "user_name", DefaultValue: "用户姓名默认值"},
+		{SourceKey: "name", MapKey: "username", DefaultValue: "用户姓名默认值"},
+		{SourceKey: "name", MapKey: "user.name", DefaultValue: "用户姓名默认值"},
+		{SourceKey: "extra.age", MapKey: "user.age", DefaultValue: "用户年龄默认值"},
+		{SourceKey: "extra.age", MapKey: "user_age", DefaultValue: "用户年龄默认值"},
+		{SourceKey: "extra.height", MapKey: "user.height", DefaultValue: "扩展高度默认值"},
+		{SourceKey: "extra.height", MapKey: "user_height", DefaultValue: "扩展高度默认值"},
+		{SourceKey: "table.[].name", MapKey: "slice.[].name_modify", DefaultValue: "列表姓名默认值"},
+		{SourceKey: "table.[].list", MapKey: "slice.[].data_list", DefaultValue: "[\"567\",\"678\",\"789\"]"},
+	}
+	byteData, _ := json.Marshal(source)
+	filter := NewDataFilter(string(byteData), rule)
+	fmt.Println(filter.Filter())
+}
+
+// TestDataFilterForObiToSlice ...
+//
+// Author : go_developer@163.com<白茶清欢>
+//
+// Date : 2022/1/23 12:06 AM
+func TestDataFilterForObiToSlice(t *testing.T) {
+	source := map[string]interface{}{
+		"name":   "zhangdeman",
+		"age":    18,
+		"height": 180,
+		"extra": map[string]interface{}{
+			"age":    18,
+			"height": 180,
+			"slice":  []int{1, 2, 3},
+		},
+		"slice_data": []int{1, 2, 3},
+		"map":        map[string]interface{}{"a": 1, "b": 2, "c": 4},
+		"table": []map[string]interface{}{
+			{"name": "alex", "age": 18, "number": 1},
+			{"name": "bob", "age": 28, "number": 2},
+			{"name": "bob", "age": 28, "number": 2, "list": []int{1, 2, 3}},
+		},
+	}
+	rule := []*FilterDataRule{
+		// {SourceKey: "name", MapKey: "slice.[]", DefaultValue: "用户姓名默认值"},
+		{SourceKey: "age", MapKey: "slice", DefaultValue: "用户姓名默认值"},
+		{SourceKey: "height", MapKey: "slice", DefaultValue: "用户姓名默认值"},
+	}
+	byteData, _ := json.Marshal(source)
+	filter := NewDataFilter(string(byteData), rule)
+	filter.UserItemToSlice()
+	fmt.Println(filter.Filter())
+}
+
+// TestDataFilterDiffArr ...
+//
+// Author : go_developer@163.com<白茶清欢>
+//
+// Date : 12:27 下午 2022/1/26
+func TestDataFilterDiffArr(t *testing.T) {
+	source := map[string]interface{}{
+		"name":   "zhangdeman",
+		"age":    18,
+		"height": 180,
+		"extra": map[string]interface{}{
+			"age":    18,
+			"height": 180,
+			"slice":  []int{1, 2, 3},
+		},
+		"slice_data": []int{1, 2, 3},
+		"map":        map[string]interface{}{"a": 1, "b": 2, "c": 4},
+		"table": []map[string]interface{}{
+			{"user_list": []interface{}{map[string]interface{}{"name": "alex", "age": 18, "number": 1}}},
+			{"user_list": []interface{}{map[string]interface{}{"name": "bob", "age": 28, "number": 2}}},
+			{"user_list": []interface{}{map[string]interface{}{"name": "andy", "age": 28, "number": 2}}},
+		},
+	}
+	rule := []*FilterDataRule{
+		// {SourceKey: "name", MapKey: "slice.[]", DefaultValue: "用户姓名默认值"},
+		{SourceKey: "table.[].user_list.[].name", MapKey: "user_list.[].detail.name", DefaultValue: "用户姓名默认值"},
+		{SourceKey: "table.[].user_list.[].age", MapKey: "user_list.[].detail.age", DefaultValue: "用户姓名默认值"},
+	}
+	byteData, _ := json.Marshal(source)
+	filter := NewDataFilter(string(byteData), rule)
+	filter.UserItemToSlice()
+	filter.Filter()
+	//fmt.Println()
+}