danial/kami_spider_monorepo
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor(jd): 优化京东服务异常处理与日志格式

Browse Source 
- 引入 JDServiceException 统一处理京东相关异常
- 针对风控错误增加自动重试及代理池失效代理清理机制
- 调整请求重试次数从3次改为2次，提升效率
- 将服务端异常返回改为抛出异常，简化调用逻辑
- 优化 app_store.py 中异常捕获及日志输出逻辑
- ctrip.py 中订单提交相关接口改用抛出异常替代返回错误码
- 增加 _delete_proxy 方法用于代理失效处理
- 修改日志格式化器，生产环境使用 JSON，开发环境输出可读性更好的格式
- 统一日志时间使用本地时间替代 UTC 时间，提升时间可读性
- 完善 trace_id 上下文传递，日志中自动带入 trace_id 信息
This commit is contained in:
danial
2025-11-03 23:48:09 +08:00
parent 6c768b6e7b
commit ef4390217f
3 changed files with 93 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
apps/jd/router/app_store.py
View File

@@ -17,10 +17,11 @@ from apps.jd.services.app_store import AppStoreSpider
from apps.jd.services.ctrip import XiechengCardSpider
from apps.shared.proxy_pool.proxy_pool import ProxyPoolFactory
from core.config import ProxyPoolType
from core.exceptions import JDServiceException
from core.responses import ApiResponse, BusinessCode, error, success
from observability.logging import LoggerAdapter, get_logger_with_trace
router = APIRouter(prefix="/jd", tags=["苹果权益充值"])
router = APIRouter(tags=["苹果权益充值"])
logger: LoggerAdapter = get_logger_with_trace(__name__)
@@ -89,27 +90,33 @@ async def app_store(
),
message="不支持的充值金额",
)
try:
c_trip_spider = XiechengCardSpider(
cookies=cookies,
order_num=order_num,
sku_id=skus[request_data.face_price],
)
code, res = await run_in_threadpool(c_trip_spider.run)
if code == BusinessCode.SUCCESS:
return success(data=res)
return error(code=code, data=res, message="请求完成")
except Exception as e:
logger.error(f"请求失败：{traceback.format_exc()}")
return error(
code=BusinessCode.INTERNAL_ERROR,
data={
"deeplink": "",
"order_id": "",
"pay_id": "",
"remark": str(e),
},
)
err = None
for _ in range(2):
try:
c_trip_spider = XiechengCardSpider(
cookies=cookies,
order_num=order_num,
sku_id=skus[request_data.face_price],
)
code, res = await run_in_threadpool(c_trip_spider.run)
if code == BusinessCode.SUCCESS:
return success(data=res)
return error(code=code, data=res, message="请求完成")
except JDServiceException as e:
err = e
if e.code == BusinessCode.JD_ORDER_RISK_ERR:
proxy_pool = ProxyPoolFactory.get_proxy_pool(
ProxyPoolType.EXPIRING, expire_time=60
)
proxy = proxy_pool.get_proxy(order_id=order_num)
if proxy:
proxy_pool.remove_invalid_proxy(proxy)
continue
raise
except Exception as e:
logger.error(f"请求失败：{traceback.format_exc()}")
return error(code=BusinessCode.INTERNAL_ERROR, message=str(e))
return error(code=BusinessCode.JD_ORDER_RISK_ERR, message=err.message if err else "风控")
case AppleStoreRequestCategoryEnum.Walmart:
skus = {
100.0: "10140177420168",
@@ -126,7 +133,7 @@ async def app_store(
)
try:
code = BusinessCode.INTERNAL_ERROR
for i in range(3):
for i in range(2):
c_trip_spider = XiechengCardSpider(
cookies=cookies,
order_num=order_num,
@@ -148,24 +155,13 @@ async def app_store(
logger.error(f"请求失败：{traceback.format_exc()}")
return error(
code=BusinessCode.INTERNAL_ERROR,
data={
"deeplink": "",
"order_id": "",
"pay_id": "",
"remark": str(e),
},
)
except JDServiceException:
raise
except Exception as e:
logger.error(f"请求失败：{traceback.format_exc()}")
return error(
code=BusinessCode.INTERNAL_ERROR,
data={
"deeplink": "",
"order_id": "",
"pay_id": "",
"remark": str(e),
},
message="请求完成",
message=str(e),
)

107
apps/jd/services/ctrip.py
View File

@@ -2,7 +2,6 @@ import hashlib
import json
from logging import LoggerAdapter
import platform
import re
import time
import fake_useragent
@@ -53,6 +52,9 @@ class XiechengCardSpider:
proxy = self._expiring_pool.get_proxy(order_id=self._order_num)
return ProxySpec(all=proxy) if proxy else None
def _delete_proxy(self, proxy):
self._expiring_pool.remove_invalid_proxy(proxy)
# def get_ticket_res(self):
# for i in range(1):
# try:
@@ -369,7 +371,11 @@ class XiechengCardSpider:
h5st = requests.get(f"http://127.0.0.1:8887/jd/h5st?{params_str}").text
return h5st
def get_x_token(self):
def get_x_token(self, retry=3):
if retry <= 0:
raise JDServiceException(
BusinessCode.JD_ORDER_RISK_ERR, message="风控重试失败"
)
proxy = None
_proxy = self._get_proxy()
if _proxy:
@@ -384,14 +390,17 @@ class XiechengCardSpider:
data=data,
)
logger.info(f"获取x-api-eid-token返回：{response.text}")
if response.text == "参数异常":
return self.get_x_token(retry-1)
res = response.json()
token = res["data"]["token"]
eid = res["data"]["eid"]
return token, eid
def get_current_order(self, retry_count: int = 3):
def get_current_order(self, retry_count: int = 2):
if not retry_count:
return JDServiceException(BusinessCode.JD_ORDER_RISK_ERR)
raise JDServiceException(BusinessCode.JD_ORDER_RISK_ERR, message="风控重试失败")
body = (
'{"deviceUUID":"6785593751540242498","appId":"wxae3e8056daea8727","appVersion":"2.5.2","tenantCode":"jgm","bizModelCode":"3","bizModeClientType":"M","token":"3852b12f8c4d869b7ed3e2b3c68c9436","externalLoginType":1,"referer":"https://item.m.jd.com/","resetGsd":true,"useBestCoupon":"1","locationId":"1-72-2819-0","packageStyle":true,"sceneval":"2","balanceCommonOrderForm":{"supportTransport":false,"action":1,"overseaMerge":false,"international":false,"netBuySourceType":0,"appVersion":"2.5.2","tradeShort":false},"balanceDeviceInfo":{"resolution":"2048*1152"},"cartParam":{"skuItem":{"skuId":"%s","num":"1","orderCashBack":false,"extFlag":{}}}}'
% self.sku_id
@@ -454,6 +463,7 @@ class XiechengCardSpider:
verify=False,
proxies=proxy,
)
logger.info(f"创建订单请求结果 {response.text}")
if (
response.ok
and response.json()
@@ -696,57 +706,22 @@ class XiechengCardSpider:
# 提交预付款订单
self.get_current_order()
order_res = self.submit_order()
logger.info(
f"订单号：{self._order_num}，app_store提交预付款订单返回：{order_res}"
)
# 火爆
if order_res.get("body", {}).get("errorCode") == "7201":
return BusinessCode.JD_ORDER_RISK_ERR, {
"deeplink": "",
"order_id": "",
"pay_id": "",
"remark": str(order_res),
}
raise JDServiceException(BusinessCode.JD_ORDER_RISK_ERR, message=str(order_res))
# 未登录
if order_res.get("body", {}).get("errorCode") == "302":
return BusinessCode.JD_ORDER_CK_ERR, {
"deeplink": "",
"order_id": "",
"pay_id": "",
"remark": order_res.get("body", {}).get("errorReason"),
}
raise JDServiceException(BusinessCode.JD_ORDER_CK_ERR, message=str(order_res))
# 无货
if order_res.get("body", {}).get("errorCode") == "722":
return BusinessCode.JD_ORDER_STOCK_ERR, {
"deeplink": "",
"order_id": "",
"pay_id": "",
"remark": order_res.get("body", {})
.get("submitOrderPromptVO", {})
.get("title"),
}
raise JDServiceException(BusinessCode.JD_ORDER_STOCK_ERR, message=str(order_res))
# 火爆
if order_res.get("body", {}).get("errorCode") == "601":
return BusinessCode.JD_ORDER_RISK_ERR, {
"deeplink": "",
"order_id": "",
"pay_id": "",
"remark": str(order_res),
}
raise JDServiceException(BusinessCode.JD_ORDER_RISK_ERR, message=str(order_res))
if order_res.get("code") != "0":
return BusinessCode.JD_ORDER_NORMAL_ERR, {
"deeplink": "",
"order_id": "",
"pay_id": "",
"remark": str(order_res),
}
raise JDServiceException(BusinessCode.JD_ORDER_NORMAL_ERR, message=str(order_res))
if order_res.get("body", {}).get("errorCode") == "601":
return BusinessCode.JD_ORDER_NORMAL_ERR, {
"deeplink": "",
"order_id": "",
"pay_id": "",
"remark": str(order_res),
}
raise JDServiceException(BusinessCode.JD_ORDER_NORMAL_ERR, message=str(order_res))
order_id = order_res.get("body", {}).get("order", {}).get("orderId")
# 获取支付信息
pay_res = self.get_pay_res(order_id)
@@ -755,19 +730,9 @@ class XiechengCardSpider:
order_res.get("body", {}).get("errorCode") == "302"
and order_res.get("body", {}).get("errorReason") == "未登录"
):
return BusinessCode.JD_ORDER_CK_ERR, {
"deeplink": "",
"order_id": "",
"pay_id": "",
"remark": str(order_res),
}
raise JDServiceException(BusinessCode.JD_ORDER_CK_ERR, message=str(order_res))
if pay_res.get("code") != "0":
return BusinessCode.JD_ORDER_NORMAL_ERR, {
"deeplink": "",
"order_id": "",
"pay_id": "",
"remark": str(pay_res),
}
raise JDServiceException(BusinessCode.JD_ORDER_NORMAL_ERR, message=str(pay_res))
pay_id = pay_res["body"]["payId"]
# 获取微信支付信息
return self.refresh_payment_url(pay_id, order_id)
@@ -779,37 +744,17 @@ class XiechengCardSpider:
f"订单号：{self._order_num}，app_store请求微信渠道返回：{pay_channel_res}"
)
if pay_channel_res.get("code") == "601":
return BusinessCode.JD_ORDER_RISK_ERR, {
"deeplink": "",
"order_id": "",
"pay_id": "",
"remark": str(pay_channel_res),
}
raise JDServiceException(BusinessCode.JD_ORDER_RISK_ERR, message=str(pay_channel_res))
if pay_channel_res.get("code") != "0":
return BusinessCode.JD_ORDER_NORMAL_ERR, {
"deeplink": "",
"order_id": "",
"pay_id": "",
"remark": str(pay_channel_res),
}
raise JDServiceException(BusinessCode.JD_ORDER_NORMAL_ERR, message=str(pay_channel_res))
wx_pay_res = self.plat_wx_pay_res(pay_id)
logger.info(
f"订单号：{self._order_num}，app_store获取微信支付信息返回：{wx_pay_res}"
)
if wx_pay_res.get("code") != "0":
return BusinessCode.JD_ORDER_NORMAL_ERR, {
"deeplink": "",
"order_id": "",
"pay_id": "",
"remark": str(wx_pay_res),
}
raise JDServiceException(BusinessCode.JD_ORDER_NORMAL_ERR, message=str(wx_pay_res))
if wx_pay_res.get("errorCode") == "-1":
return BusinessCode.JD_ORDER_CK_ERR, {
"deeplink": "",
"order_id": "",
"pay_id": "",
"remark": str(wx_pay_res),
}
raise JDServiceException(BusinessCode.JD_ORDER_CK_ERR, message=str(wx_pay_res))
return BusinessCode.SUCCESS, {
"deeplink": wx_pay_res.get("payInfo", {}).get("mweb_url"),
"order_id": str(order_id),

66
observability/logging.py
View File

@@ -18,37 +18,37 @@ trace_id_var: ContextVar[str] = ContextVar("trace_id", default="")
class TraceContextFormatter(logging.Formatter):
"""
Custom formatter that adds trace context to log records.
Formats logs as JSON in production, human-readable in development.
"""
def __init__(self, use_json: bool = True):
"""
Initialize formatter.
Args:
use_json: Whether to format as JSON (True) or human-readable (False)
"""
super().__init__()
self.use_json = use_json
def format(self, record: logging.LogRecord) -> str:
"""
Format log record with trace context.
Args:
record: Log record
Returns:
str: Formatted log message
"""
# Get trace ID from context
trace_id = trace_id_var.get()
if self.use_json:
# JSON format for production
log_data = {
"timestamp": datetime.utcnow().isoformat() + "Z",
"timestamp": datetime.now(),
"level": record.levelname,
"logger": record.name,
"message": record.getMessage(),
@@ -57,61 +57,61 @@ class TraceContextFormatter(logging.Formatter):
"function": record.funcName,
"line": record.lineno,
}
# Add exception info if present
if record.exc_info:
log_data["exception"] = self.formatException(record.exc_info)
# Add extra fields
if hasattr(record, "extra"):
log_data["extra"] = getattr(record, "extra")
return json.dumps(log_data)
else:
# Human-readable format for development
timestamp = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
trace_part = f" [trace_id={trace_id}]" if trace_id else ""
message = f"{timestamp} - {record.levelname:8s} - {record.name:30s}{trace_part} - {record.getMessage()}"
if record.exc_info:
message += "\n" + self.formatException(record.exc_info)
return message
def setup_logging() -> None:
"""
Configure application logging.
This should be called at application startup.
"""
# Determine if we should use JSON format
use_json = settings.is_production
# Create formatter
formatter = TraceContextFormatter(use_json=use_json)
# Configure root logger
root_logger = logging.getLogger()
root_logger.setLevel(settings.log_level)
# Remove existing handlers
root_logger.handlers.clear()
# Create console handler
console_handler = logging.StreamHandler()
console_handler.setLevel(settings.log_level)
console_handler.setFormatter(formatter)
# Add handler to root logger
root_logger.addHandler(console_handler)
# Set levels for third-party loggers
logging.getLogger("uvicorn").setLevel(logging.INFO)
logging.getLogger("uvicorn.access").setLevel(logging.INFO)
logging.getLogger("uvicorn.error").setLevel(logging.INFO)
logging.getLogger("fastapi").setLevel(logging.INFO)
# Reduce noise from OpenTelemetry
logging.getLogger("opentelemetry").setLevel(logging.WARNING)
@@ -119,7 +119,7 @@ def setup_logging() -> None:
def set_trace_id(trace_id: str) -> None:
"""
Set trace ID in context for current request.
Args:
trace_id: Trace ID to set
"""
@@ -129,7 +129,7 @@ def set_trace_id(trace_id: str) -> None:
def get_trace_id() -> str:
"""
Get trace ID from context.
Returns:
str: Current trace ID or empty string
"""
@@ -139,10 +139,10 @@ def get_trace_id() -> str:
def get_logger(name: str) -> logging.Logger:
"""
Get logger instance.
Args:
name: Logger name (usually __name__)
Returns:
logging.Logger: Logger instance
"""
@@ -153,15 +153,17 @@ class LoggerAdapter(logging.LoggerAdapter):
"""
Logger adapter that automatically includes trace context.
"""
def process(self, msg: str, kwargs: MutableMapping[str, Any]) -> tuple[str, MutableMapping[str, Any]]:
def process(
self, msg: str, kwargs: MutableMapping[str, Any]
) -> tuple[str, MutableMapping[str, Any]]:
"""
Process log message to add trace context.
Args:
msg: Log message
kwargs: Keyword arguments
Returns:
tuple: Processed message and kwargs
"""
@@ -176,10 +178,10 @@ class LoggerAdapter(logging.LoggerAdapter):
def get_logger_with_trace(name: str) -> LoggerAdapter:
"""
Get logger adapter with automatic trace context.
Args:
name: Logger name (usually __name__)
Returns:
LoggerAdapter: Logger adapter instance
"""