// Package httpclient ... // // Description : httpclient ... // // Author : go_developer@163.com<白茶清欢> // // Date : 2024-05-31 15:22 package httpclient import ( "fmt" "git.zhangdeman.cn/zhangdeman/network/httpclient/implement" "net" "strings" "time" "git.zhangdeman.cn/zhangdeman/consts" "git.zhangdeman.cn/zhangdeman/network/httpclient/define" "git.zhangdeman.cn/zhangdeman/network/httpclient/log" "git.zhangdeman.cn/zhangdeman/network/httpclient/validate" "git.zhangdeman.cn/zhangdeman/serialize" "github.com/go-resty/resty/v2" ) // NewHttpClient 获取http client // // Author : go_developer@163.com<白茶清欢> // // Date : 15:27 2024/5/31 func NewHttpClient(reqConfig *define.Request, reqOption *RequestOption) (*HttpClient, error) { if nil == reqOption { reqOption = &RequestOption{} } if nil == reqOption.ResponseParser { // 没有自定义响应解析实现, 使用内置实现 reqOption.ResponseParser = &implement.Response{} } if nil == reqConfig.Logger { reqConfig.Logger = log.Get() // 未单独指定日志实例, 则使用全局日志实例 } // 验证配置正确性以及初始化默认值 if err := validate.RequestConfig(reqConfig); nil != err { return nil, err } if reqConfig.RetryRule == nil { reqConfig.RetryRule = &define.RequestRetryRule{ RetryCount: 0, RetryTimeInterval: 0, RetryHttpCodeList: []int64{}, RetryBusinessCodeList: []string{}, } // 未指定重试规则, 则使用默认重试规则 } // 初始化成功的 http code list if len(reqConfig.SuccessHttpCodeList) == 0 { reqConfig.SuccessHttpCodeList = []int{} } if len(reqConfig.Static) > 0 { for loc, valMap := range reqConfig.Static { if len(valMap) == 0 { continue } l := strings.ToUpper(loc) switch l { case consts.RequestDataLocationHeader.String(): if reqConfig.Header == nil { reqConfig.Header = make(map[string]any) } for k, v := range valMap { reqConfig.Header[k] = v } case consts.RequestDataLocationCookie.String(): if reqConfig.Cookie == nil { reqConfig.Cookie = make(map[string]any) } for k, v := range valMap { reqConfig.Cookie[k] = v } case consts.RequestDataLocationBody.String(): if reqConfig.Body == nil { reqConfig.Body = make(map[string]any) } for k, v := range valMap { reqConfig.Body[k] = v } case consts.RequestDataLocationQuery.String(): if reqConfig.Query == nil { reqConfig.Query = make(map[string]any) } for k, v := range valMap { reqConfig.Query[k] = v } } } } if reqConfig.Header == nil { reqConfig.Header = make(map[string]any) } if ua, exist := reqConfig.Header[consts.HeaderKeyUserAgent.String()]; !exist || nil == ua || fmt.Sprintf("%v", ua) == "" { reqConfig.Header[consts.HeaderKeyUserAgent.String()] = "resty-v2@network/httpclient" } restyClient, restyRequest := NewRestyClient(reqConfig) hc := &HttpClient{ Client: restyClient, request: restyRequest, reqOption: reqOption, reqCfg: reqConfig, http4xxHandler: make([]define.Http4xxHandler, 0), http5xxHandler: make([]define.Http5xxHandler, 0), httpBusinessErrorHandler: make([]define.HttpBusinessErrorHandler, 0), requestFinishHandler: make([]define.RequestFinishHandler, 0), } hc.OnRequestFinish(func(req *define.Request, rep *define.Response) { if rep.IsSuccess { // 请求成功 log.Record(consts.LogLevelInfo, "接口请求成功", req, rep) return } // 请求失败 log.Record(consts.LogLevelError, "接口请求失败", req, rep) }) return hc, nil } // HttpClient 请求客户端 // // Author : go_developer@163.com<白茶清欢> // // Date : 15:27 2024/5/31 type HttpClient struct { *resty.Client request *resty.Request reqOption *RequestOption reqCfg *define.Request http4xxHandler []define.Http4xxHandler http5xxHandler []define.Http5xxHandler httpBusinessErrorHandler []define.HttpBusinessErrorHandler requestSendErrorHandler []define.RequestSendErrorHandler requestFinishHandler []define.RequestFinishHandler } // OnResponse4xx 4xx处理逻辑 // // Author : go_developer@163.com<白茶清欢> // // Date : 18:20 2024/6/1 func (hc *HttpClient) OnResponse4xx(handlerList ...define.Http4xxHandler) { hc.http4xxHandler = append(hc.http4xxHandler, handlerList...) } // OnResponse5xx 5xx处理逻辑 // // Author : go_developer@163.com<白茶清欢> // // Date : 18:21 2024/6/1 func (hc *HttpClient) OnResponse5xx(handlerList ...define.Http5xxHandler) { hc.http5xxHandler = append(hc.http5xxHandler, handlerList...) } // OnResponseBusinessError 业务错误出路逻辑 // // Author : go_developer@163.com<白茶清欢> // // Date : 18:22 2024/6/1 func (hc *HttpClient) OnResponseBusinessError(handlerList ...define.HttpBusinessErrorHandler) { hc.httpBusinessErrorHandler = append(hc.httpBusinessErrorHandler, handlerList...) } // OnRequestFinish 请求完成事件 // // Author : go_developer@163.com<白茶清欢> // // Date : 18:36 2024/6/1 func (hc *HttpClient) OnRequestFinish(handlerList ...define.RequestFinishHandler) { hc.requestFinishHandler = append(hc.requestFinishHandler, handlerList...) } // getRequestValidateMiddleware 请求验证的Middleware // // Author : go_developer@163.com<白茶清欢> // // Date : 15:40 2024/5/31 func (hc *HttpClient) getRequestValidateMiddleware() resty.RequestMiddleware { return func(client *resty.Client, request *resty.Request) error { return nil } } // getResponseValidateMiddleware 获取响应数据验证的middleware // // Author : go_developer@163.com<白茶清欢> // // Date : 15:42 2024/5/31 func (hc *HttpClient) getResponseValidateMiddleware() resty.ResponseMiddleware { return func(client *resty.Client, response *resty.Response) error { return nil } } // SetRestyClient 设置client // // Author : go_developer@163.com<白茶清欢> // // Date : 15:54 2024/5/31 func (hc *HttpClient) SetRestyClient(restyClient *resty.Client) { hc.Client = restyClient } // GetRestyClient 获取 resty client // // Author : go_developer@163.com<白茶清欢> // // Date : 15:57 2024/5/31 func (hc *HttpClient) GetRestyClient() *resty.Client { return hc.Client } // Request 发送请求 // // Author : go_developer@163.com<白茶清欢> // // Date : 15:52 2024/5/31 func (hc *HttpClient) Request() *define.Response { var ( cacheResult *define.Response ) if cacheResult = hc.getCacheResult(); nil == cacheResult { // 未命中缓存, 直接请求后端接口, 无需检测预热等逻辑 return hc.requestBackendApi() } // 上面若命中缓存, 则后续缓存实例不可能为nil, 无需判断 // 判断是否开启预热 cachePreHeatConfig := hc.reqOption.CacheInstance.PreHeatConfig() if nil == cachePreHeatConfig { log.RecordDebug("接口请求命中缓存, PreHeatConfig未返回预热配置, 不做预热处理", map[string]any{}, hc.reqCfg) return nil } log.RecordDebug("接口请求命中缓存, 进行预热策略处理", map[string]any{ "cache_info": cacheResult.CacheInfo, }, hc.reqCfg) defer func() { // 命中缓存的情况下, 检测缓存预热策略, 判断是否进行缓存预热 go func() { if !cachePreHeatConfig.Enable || (cachePreHeatConfig.MinTTL <= 0 && cachePreHeatConfig.MinPercent <= 0 && !cachePreHeatConfig.Force) { // 无预热配置或未启用预热或者未设置预热规则 log.RecordDebug("接口请求命中缓存, 未配置缓存预热策略", map[string]any{ "cache_pre_heat_config": cachePreHeatConfig, }, hc.reqCfg) return } // 判断是否触发预热 if cachePreHeatConfig.Force { log.RecordDebug("接口请求命中缓存, 强制执行缓存预热, 忽略其他策略配置", map[string]any{ "cache_pre_heat_config": cachePreHeatConfig, }, hc.reqCfg) _ = hc.requestBackendApi() return } // 将百分比的配置归一化成最小剩余时间的配置 if cachePreHeatConfig.MinPercent > 0 { expectMinTTL := hc.reqOption.CacheInstance.CacheTime() * cachePreHeatConfig.MinPercent / 100 log.RecordDebug("接口请求命中缓存, 配置预热策略:有效时长剩余百分比", map[string]any{ "cache_pre_heat_config": cachePreHeatConfig, "percent_min_ttl": expectMinTTL, "min_ttl": cachePreHeatConfig.MinTTL, }, hc.reqCfg) if cachePreHeatConfig.MinTTL == 0 || cachePreHeatConfig.MinTTL > expectMinTTL { cachePreHeatConfig.MinTTL = expectMinTTL } } if cachePreHeatConfig.MinTTL <= 0 { // 未配置最小剩余时间 log.RecordDebug("接口请求命中缓存, 未配置预热市场策略, 不执行预热", map[string]any{ "min_ttl": cachePreHeatConfig.MinTTL, }, hc.reqCfg) return } ttl := hc.reqOption.CacheInstance.TTL(cacheResult.CacheInfo.CacheKey) if ttl < 0 { // 不存在或者未设置有效期 log.RecordDebug("接口请求命中缓存, 当前缓存结果不存在或未设置有效期, 不执行预热", map[string]any{ "min_ttl": cachePreHeatConfig.MinTTL, "note": "预热时间至少在缓存过期前10s触发预热, 以保证足够时间进行预热, 以及不会因为预热尚未完成, 但是大量流量涌入, 进而导致流量穿透", }, hc.reqCfg) return } if ttl > cachePreHeatConfig.MinTTL { log.RecordDebug("接口请求命中缓存, 缓存结果有效期剩余时长大于配置阈值, 不执行预热", map[string]any{ "min_ttl": cachePreHeatConfig.MinTTL, "remaining_ttl": ttl, }, hc.reqCfg) return } log.RecordDebug("接口请求命中缓存, 缓存结果有效期大于剩余时长小于配置阈值, 触发预热", map[string]any{ "min_ttl": cachePreHeatConfig.MinTTL, "remaining_ttl": ttl, }, hc.reqCfg) // 配置了最小剩余时间,并且key剩余有效期小于最小剩余时间 // 预热加锁, 并发请求触发预热, 仅触发一个即可, 使用接口做key if err := hc.reqOption.CacheInstance.Lock(hc.reqCfg.FullUrl); err != nil { log.RecordWarn("接口请求命中缓存, 缓存结果有效期大于剩余时长小于配置阈值, 触发预热, 加锁失败, 未执行预热", map[string]any{ "min_ttl": cachePreHeatConfig.MinTTL, "remaining_ttl": ttl, "err_msg": err.Error(), }, hc.reqCfg) return } log.RecordDebug("接口请求命中缓存, 缓存结果有效期大于剩余时长小于配置阈值, 触发预热, 加锁成功, 执行预热", map[string]any{ "min_ttl": cachePreHeatConfig.MinTTL, "remaining_ttl": ttl, "lock_key": hc.reqCfg.FullUrl, }, hc.reqCfg) _ = hc.requestBackendApi() if err := hc.reqOption.CacheInstance.Unlock(hc.reqCfg.FullUrl); nil != err { log.RecordError("接口请求命中缓存, 缓存结果有效期大于剩余时长小于配置阈值, 触发预热, 执行预热后, 释放锁失败", map[string]any{ "min_ttl": cachePreHeatConfig.MinTTL, "remaining_ttl": ttl, "lock_key": hc.reqCfg.FullUrl, "err_msg": err.Error(), }, hc.reqCfg) return } log.RecordDebug("接口请求命中缓存, 缓存结果有效期大于剩余时长小于配置阈值, 触发预热, 执行预热后, 释放锁成功", map[string]any{ "min_ttl": cachePreHeatConfig.MinTTL, "remaining_ttl": ttl, "lock_key": hc.reqCfg.FullUrl, }, hc.reqCfg) }() }() // 命中缓存必然请求成功, 直接记录成功日志即可 log.Record(consts.LogLevelInfo, "接口请求成功:命中缓存", hc.reqCfg, cacheResult) return cacheResult } // requestBackendApi 请求后端接口 // // Author : go_developer@163.com<白茶清欢> // // Date : 18:47 2024/10/9 func (hc *HttpClient) requestBackendApi() *define.Response { hc.Client.OnBeforeRequest(hc.getRequestValidateMiddleware()) // 请求参数验证中间件必注册 hc.Client.OnAfterResponse(hc.getResponseValidateMiddleware()) // 响应验证中间件必注册 var ( err error ) var response *define.Response // +1 是因为正常便会请求一次, 正常请求1次 + 重试次数 = 请求总次数 for i := 0; i < hc.reqCfg.RetryRule.RetryCount+1; i++ { if i > 0 && nil != response { // 说明是重试, 记录上一次的请求信息 response.RequestFinishTime = time.Now().UnixMilli() response.UsedTime = response.RequestFinishTime - response.RequestStartTime for _, itemAfterResponse := range hc.requestFinishHandler { itemAfterResponse(hc.reqCfg, response) } // 非首次请求, 说明是重试, 暂停指定时间间隔 time.Sleep(time.Duration(hc.reqCfg.RetryRule.RetryTimeInterval) * time.Millisecond) } response = hc.newResponse() response.Seq = i response.RequestCount = i + 1 if response.RestyResponse, err = hc.request.Send(); nil != err { errType := define.RequestFailTypeSend if err.Error() == resty.ErrRateLimitExceeded.Error() { // 命中限流 errType = define.RequestFailTypeRateLimit } else if netErr, ok := err.(net.Error); ok { if netErr.Timeout() { // 请求超时 errType = define.RequestFailTypeTimeoutError // 重置响应状态码 response.HttpCode = 499 response.HttpCodeStatus = "request timeout" } } response.FailInfo = &define.ResponseFailInfo{ Type: errType, Message: err.Error(), } log.RecordDebug("请求发送出现异常", map[string]any{ "err_type": errType, "err_msg": err.Error(), }, hc.reqCfg) if errType == define.RequestFailTypeSend { // 命中限流就不重试了 log.RecordDebug("请求命中限流, 忽略重试策略, 不进行重试", nil, hc.reqCfg) break } if errType == define.RequestFailTypeTimeoutError && !hc.reqOption.ResponseParser.NeedRetry(hc.reqCfg, response) { // 未配置超时重试 log.RecordDebug("请求超时, 未配置超时重试, 不进行重试", nil, hc.reqCfg) break } log.RecordDebug("请求发送出现异常, 进行重试", map[string]any{ "err_type": errType, "err_msg": err.Error(), "time_interval": time.Duration(hc.reqCfg.RetryRule.RetryTimeInterval) * time.Millisecond, }, hc.reqCfg) continue } if nil == response.RestyResponse { response.FailInfo = &define.ResponseFailInfo{ Type: define.RequestFailTypeSend, Message: "response instance is nil", } log.RecordDebug("RestyResponse为nil, 准备重试", map[string]any{ "err_type": response.FailInfo.Type, "time_interval": time.Duration(hc.reqCfg.RetryRule.RetryTimeInterval) * time.Millisecond, }, hc.reqCfg) continue } // 解析返回信息 hc.reqOption.ResponseParser.Parse(hc.reqCfg, response) // 配置了当前code为成功, 或者未配置任何code, 当前code为2xx, 则认为请求成功 isHttpSuccess := hc.reqOption.ResponseParser.HttpSuccess(hc.reqCfg, response) if !isHttpSuccess { // 非 成功 errType := define.RequestFailTypeServerError if response.HttpCode/100 == 4 { // 客户端错误 errType = define.RequestFailTypeClientRequestInvalidError } response.FailInfo = &define.ResponseFailInfo{ Type: errType, Message: "http code is " + response.HttpCodeStatus + ", not success", } needRetry := hc.reqOption.ResponseParser.NeedRetry(hc.reqCfg, response) log.RecordWarn("请求响应的http状态码非成功", map[string]any{ "err_type": errType, "err_msg": response.RestyResponse.Status(), "response_http_code": response.HttpCode, "success_http_code": hc.reqCfg.SuccessHttpCodeList, "allow_retry": needRetry, "time_interval": time.Duration(hc.reqCfg.RetryRule.RetryTimeInterval) * time.Millisecond, }, hc.reqCfg) if !needRetry { // 未配置http code重试 break } continue } if !hc.reqOption.ResponseParser.BusinessSuccess(hc.reqCfg, response) { response.FailInfo = &define.ResponseFailInfo{ Type: define.RequestFailTypeBusinessError, Message: "business code is " + response.Code + ", not success", } needRetry := hc.reqOption.ResponseParser.NeedRetry(hc.reqCfg, response) log.RecordWarn("请求响应状态码成功, 业务状态码非成功", map[string]any{ "err_type": response.FailInfo.Type, "err_msg": response.Message, "response_code": response.Code, "success_code": hc.reqCfg.SuccessCodeList, "allow_retry": needRetry, "time_interval": time.Duration(hc.reqCfg.RetryRule.RetryTimeInterval) * time.Millisecond, }, hc.reqCfg) if needRetry { // 未配置业务code重试 break } continue } response.IsSuccess = true //设置成功 response.CacheInfo.SetCache, response.CacheInfo.CacheError = hc.setCacheResult(response) // 设置缓存 break } response.RequestFinishTime = time.Now().UnixMilli() response.UsedTime = response.RequestFinishTime - response.RequestStartTime // 请求完成hook for _, itemAfterResponse := range hc.requestFinishHandler { itemAfterResponse(hc.reqCfg, response) } return response } // newResponse 默认返回数据 // // Author : go_developer@163.com<白茶清欢> // // Date : 17:44 2024/6/1 func (hc *HttpClient) newResponse() *define.Response { return &define.Response{ Header: map[string]any{}, Cookie: map[string]any{}, Body: map[string]any{}, Code: "", Message: "", Data: "", HttpCode: 0, HttpCodeStatus: "", ResponseDataRule: nil, Seq: 0, RequestStartTime: time.Now().UnixMilli(), RequestFinishTime: 0, UsedTime: 0, RestyResponse: nil, IsSuccess: false, CacheInfo: &define.ResponseCacheInfo{ IsCache: false, SetCache: false, CacheKey: "", CacheValue: "", CacheEnable: nil != hc.reqOption.CacheInstance && hc.reqOption.CacheInstance.Enable(), CacheError: nil, }, RequestCount: 0, FailInfo: nil, } } // getCacheResult 获取缓存结果 // // Author : go_developer@163.com<白茶清欢> // // Date : 16:04 2024/6/3 func (hc *HttpClient) getCacheResult() *define.Response { if nil == hc.reqOption.CacheInstance { log.RecordDebug("接口请求前缓存检测, 未设置缓存实例", map[string]any{}, hc.reqCfg) return nil } if !hc.reqOption.CacheInstance.Enable() { log.RecordDebug("接口请求前缓存检测, 设置缓存实例, 但未启用缓存功能", map[string]any{}, hc.reqCfg) return nil } startTime := time.Now().UnixMilli() cacheKey := hc.reqOption.CacheInstance.GetKey(hc.reqCfg) cacheValue := strings.TrimSpace(hc.reqOption.CacheInstance.GetValue(cacheKey)) if len(cacheValue) == 0 { log.RecordDebug("接口请求前缓存检测, 未读取到缓存数据", map[string]any{}, hc.reqCfg) return nil } response := hc.newResponse() if err := serialize.JSON.UnmarshalWithNumber([]byte(cacheValue), response); nil != err { log.RecordWarn("接口请求前缓存检测, 读取到缓存数据, 数据解析失败, 将跳过缓存, 请求对应接口", map[string]any{ "err_msg": err.Error(), }, hc.reqCfg) return nil } response.CacheInfo.IsCache = true // 设置缓存标记 response.RequestStartTime = startTime // 开始时间 response.RequestFinishTime = time.Now().UnixMilli() // 结束时间 response.UsedTime = response.RequestFinishTime - response.RequestStartTime // 耗时 response.CacheInfo.CacheKey = cacheKey // 缓存key response.CacheInfo.CacheValue = cacheValue // 缓存值 log.RecordDebug("接口请求前缓存检测, 命中缓存, 直接返回缓存数据", map[string]any{}, hc.reqCfg) return response } // setCacheResult ... // // Author : go_developer@163.com<白茶清欢> // // Date : 16:24 2024/6/3 func (hc *HttpClient) setCacheResult(response *define.Response) (bool, error) { if nil == response || nil == hc.reqOption.CacheInstance { log.RecordDebug("接口请求成功后, 缓存设置失败", map[string]any{ "response_is_nil": response == nil, "cache_instance_is_nil": hc.reqOption.CacheInstance == nil, }, hc.reqCfg) return false, nil } // 全局未开启或者当前请求不支持缓存 globalCacheEnable := hc.reqOption.CacheInstance.Enable() currentRequestAllowCache := hc.reqOption.CacheInstance.IsAllow(hc.reqCfg, response) log.RecordDebug("检测缓存是否允许执行", map[string]any{ "current_cache_enable": currentRequestAllowCache, "global_cache_enable": globalCacheEnable, }, hc.reqCfg) if !globalCacheEnable || !currentRequestAllowCache { return false, nil } cacheKey := hc.reqOption.CacheInstance.GetKey(hc.reqCfg) cacheValue := serialize.JSON.MarshalForStringIgnoreError(response) if err := hc.reqOption.CacheInstance.SetValue(cacheKey, cacheValue); nil != err { log.RecordWarn("开启结果缓存, 缓存设置失败", map[string]any{ "current_cache_enable": currentRequestAllowCache, "global_cache_enable": globalCacheEnable, "err_msg": err.Error(), }, hc.reqCfg) return false, err } log.RecordDebug("开启结果缓存, 缓存设置成功", map[string]any{ "current_cache_enable": currentRequestAllowCache, "global_cache_enable": globalCacheEnable, }, hc.reqCfg) response.CacheInfo.CacheKey = cacheKey response.CacheInfo.CacheValue = cacheValue return true, nil }