feat: implement WeChat cross-batch refund reconciliation and fix misc issues

WeChat cross-batch refund reconciliation:
- Add OriginalAmount field to CleanedBill for accurate cumulative refund math
- DeduplicateRawFile detects WeChat status-update rows (已退款/已全额退款) and
  emits WechatRefundUpdates for Go-side reconciliation (Scenario 1)
- WechatPy cleaner surfaces -退款 income rows with no same-batch expense match
  as unresolved_refunds for Go ReconcileRefund (Scenario 2)
- Add ReconcileWechatRefund to repository interface and MongoDB implementation
- upload.go step 15 iterates WechatRefundUpdates and reconciles against bills_cleaned

Bug fixes:
- ReviewStats: add nil repo check to prevent panic when DB is not connected
- JWT: remove hardcoded fallback secret; return 500/401 if JWTSecret not configured
- Remove unused parsePageParam dead code and its strconv import
- BillDetailDrawer: show 不计收支 amount in muted gray instead of red
- test_jd_cleaner.py: replace hardcoded D:\Projects\BillAI path with dynamic __file__ resolution

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-06-16 21:38:25 +08:00
parent e2e1beb6f7
commit a2de8c5078
13 changed files with 246 additions and 31 deletions

View File

@@ -92,6 +92,13 @@ Browser → SvelteKit proxy → Go (Gin) → handler → service → adapter →
### File Processing Pipeline
Upload: ZIP/XLSX → Extract → Convert UTF-8 CSV → Detect bill type (alipay/wechat/jd) → Deduplicate against MongoDB → Clean via Python → Save cleaned data.
### Cross-Batch Refund Reconciliation
When the original purchase and its refund are in different upload batches, within-batch logic can't match them. Two reconciliation paths handle this:
**Alipay** (`handler/upload.go` step 14): After cleaning, `cleaner.unresolved_refunds` (refund rows that found no matching expense in the same file) is returned by FastAPI and iterated in Go. `ReconcileRefund()` looks up the original expense by `transaction_id` or `merchant_order_no` and either soft-deletes (full refund) or deducts `amount` (partial). Alipay refund rows have their own distinct `transaction_id` (suffixed `_<refund_id>`), so they pass raw dedup and are naturally idempotent on re-upload.
**WeChat** (`handler/upload.go` step 15): WeChat re-exports the *same row* (same `transaction_id`) with an updated `当前状态` field (`已全额退款` or `已退款(¥X)`). `DeduplicateRawFile` detects these duplicate rows, extracts the refund info into `DeduplicateResult.WechatRefundUpdates`, and `ReconcileWechatRefund()` applies the update. WeChat's `¥X` is the **cumulative** total refunded, so `CleanedBill.original_amount` (set at first save) is used to compute `remaining = original_amount - X`.
### Adapter (Go ↔ Python)
`adapter.Cleaner` interface. Two modes:
- `http` (default): calls FastAPI at `ANALYZER_URL`

View File

@@ -7,6 +7,23 @@
## [Unreleased]
### 新增
- **微信跨批次退款核销** - 重复上传微信账单时,自动识别"当前状态"字段中的退款信息并核销原始支出记录
- 支持「已全额退款」:原支出记录软删除
- 支持「已退款(¥X)」:累计退款金额从原始金额(`original_amount`)中扣减,剩余金额更新到记录
- 新增 `CleanedBill.original_amount` 字段,存储入库时的原始金额,保证多次部分退款累计计算正确
- `ReconcileWechatRefund` 接口和 MongoDB 实现
- **支付宝跨批次退款核销** - 上传含退款行的支付宝账单时,若原消费记录来自更早的批次,自动在数据库中核销
- 退款行的「交易订单号」带后缀(`原订单号_退款编号`),天然幂等:重复上传时被原始数据去重拦截,不会重复核销
- 全额退款软删除,部分退款扣减金额并追加备注
- `ReconcileRefund` 接口和 MongoDB 实现
### 技术改进
- `server/service/bill.go``DeduplicateRawFile` 在微信账单去重阶段检测已退款状态行,收集 `WechatRefundUpdates` 供后续核销
- `server/service/bill.go``saveCleanedBillsFromCSV` / `saveCleanedBillsFromJSON` 入库时同步写入 `original_amount`
- `analyzer/cleaners/alipay.py``_process_expenses` 收集同批次内未匹配的退款(`unresolved_refunds`),通过 FastAPI 响应透传至 Go
- `analyzer/cleaners/base.py``BaseCleaner` 基类新增 `unresolved_refunds` 属性
## [1.4.0] - 2026-03-23
### 新增

View File

@@ -54,7 +54,23 @@ class WechatCleaner(BaseCleaner):
# 第三步:处理退款(包括转账退款)
final_expense_rows, income_rows = self._process_refunds(expense_rows, income_rows)
# 收集跨批次未匹配的 -退款 行(当前批次无对应支出记录,需 Go 侧跨批次核销)
expense_merchants = {exp[2].strip() for exp in expense_rows}
for refund_row in refund_rows:
if refund_row[2].strip() not in expense_merchants:
amount = float(parse_amount(refund_row[5]))
if amount > 0:
self.unresolved_refunds.append({
"order_no": "",
"merchant_order_no": refund_row[9].strip() if len(refund_row) > 9 else "",
"refund_order_no": refund_row[8].strip() if len(refund_row) > 8 else "",
"amount": amount,
"time": refund_row[0],
"merchant": refund_row[2],
"description": refund_row[3] if len(refund_row) > 3 else "",
})
print(f"\n处理结果:")
print(f" 全额退款删除: {self.stats['fully_refunded']}")
print(f" 部分退款调整: {self.stats['partially_refunded']}")

View File

@@ -11,7 +11,8 @@ import sys
sys.stdout.reconfigure(encoding='utf-8')
def test_jd_cleaner():
zip_path = r'D:\Projects\BillAI\mock_data\京东交易流水(申请时间2026年01月26日13时29分47秒)(密码683263)_209.zip'
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
zip_path = os.path.join(base_dir, 'mock_data', '京东交易流水(申请时间2026年01月26日13时29分47秒)(密码683263)_209.zip')
with zipfile.ZipFile(zip_path, 'r') as zf:
with tempfile.TemporaryDirectory() as tmpdir:

View File

@@ -98,7 +98,11 @@ func Login(c *gin.Context) {
secret := config.Global.JWTSecret
if secret == "" {
secret = "billai-default-secret"
c.JSON(http.StatusInternalServerError, gin.H{
"success": false,
"error": "服务器 JWT 配置缺失",
})
return
}
token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
@@ -144,7 +148,12 @@ func ValidateToken(c *gin.Context) {
secret := config.Global.JWTSecret
if secret == "" {
secret = "billai-default-secret"
c.JSON(http.StatusUnauthorized, gin.H{
"success": false,
"error": "服务器 JWT 配置缺失",
"code": "TOKEN_INVALID",
})
return
}
token, err := jwt.ParseWithClaims(tokenString, &Claims{}, func(token *jwt.Token) (interface{}, error) {

View File

@@ -2,7 +2,6 @@ package handler
import (
"net/http"
"strconv"
"time"
"github.com/gin-gonic/gin"
@@ -154,18 +153,6 @@ func ListBills(c *gin.Context) {
})
}
// parsePageParam 解析分页参数
func parsePageParam(s string, defaultVal int) int {
if s == "" {
return defaultVal
}
val, err := strconv.Atoi(s)
if err != nil || val < 1 {
return defaultVal
}
return val
}
// MonthlyStatsResponse 月度统计响应
type MonthlyStatsResponse struct {
Result bool `json:"result"`
@@ -202,6 +189,13 @@ func MonthlyStats(c *gin.Context) {
// ReviewStats 获取待复核数据统计
func ReviewStats(c *gin.Context) {
repo := repository.GetRepository()
if repo == nil {
c.JSON(http.StatusServiceUnavailable, model.ReviewResponse{
Result: false,
Message: "数据库未连接",
})
return
}
// 从MongoDB查询所有需要复核的账单
bills, err := repo.GetBillsNeedReview()

View File

@@ -269,7 +269,7 @@ func Upload(c *gin.Context) {
}
}
// 14. 核销跨批次退款(本次清洗中未在同批次内匹配到对应支出的退款)
// 14. 核销跨批次退款(支付宝:本次清洗中未在同批次内匹配到对应支出的退款)
var reconciledCount int
if repo != nil && cleanResult != nil {
for _, ur := range cleanResult.UnresolvedRefunds {
@@ -285,7 +285,26 @@ func Upload(c *gin.Context) {
}
}
// 15. 返回成功响应
// 15. 核销跨批次微信退款(重复上传的行中携带的已退款状态)
if repo != nil && len(dedupResult.WechatRefundUpdates) > 0 {
for _, wu := range dedupResult.WechatRefundUpdates {
matched, rErr := repo.ReconcileWechatRefund(wu.TransactionID, wu.FullRefund, wu.CumulativeRefundAmount)
if rErr != nil {
fmt.Printf("⚠️ 微信退款核销失败: %v\n", rErr)
continue
}
if matched {
reconciledCount++
if wu.FullRefund {
fmt.Printf("💰 已核销微信全额退款: 交易单号%s\n", wu.TransactionID)
} else {
fmt.Printf("💰 已核销微信部分退款: 交易单号%s, 累计退款%.2f元\n", wu.TransactionID, wu.CumulativeRefundAmount)
}
}
}
}
// 16. 返回成功响应
message := fmt.Sprintf("处理成功,新增 %d 条记录", cleanedCount)
if dedupResult.DuplicateCount > 0 {
message = fmt.Sprintf("处理成功,新增 %d 条,跳过 %d 条重复记录", cleanedCount, dedupResult.DuplicateCount)

View File

@@ -38,7 +38,13 @@ func AuthRequired() gin.HandlerFunc {
secret := config.Global.JWTSecret
if secret == "" {
secret = "billai-default-secret"
c.JSON(http.StatusUnauthorized, gin.H{
"success": false,
"error": "服务器 JWT 配置缺失",
"code": "TOKEN_INVALID",
})
c.Abort()
return
}
token, err := jwt.ParseWithClaims(tokenString, &Claims{}, func(token *jwt.Token) (interface{}, error) {

View File

@@ -88,9 +88,10 @@ type CleanedBill struct {
Category string `bson:"category" json:"category"` // 交易分类
Merchant string `bson:"merchant" json:"merchant"` // 交易对方
Description string `bson:"description" json:"description"` // 商品说明
IncomeExpense string `bson:"income_expense" json:"income_expense"` // 收/支
Amount float64 `bson:"amount" json:"amount"` // 金额
PayMethod string `bson:"pay_method" json:"pay_method"` // 支付方式
IncomeExpense string `bson:"income_expense" json:"income_expense"` // 收/支
Amount float64 `bson:"amount" json:"amount"` // 金额
OriginalAmount float64 `bson:"original_amount,omitempty" json:"original_amount,omitempty"` // 原始金额(入库时),用于微信跨批次退款核销
PayMethod string `bson:"pay_method" json:"pay_method"` // 支付方式
Status string `bson:"status" json:"status"` // 交易状态
Remark string `bson:"remark" json:"remark"` // 备注
ReviewLevel string `bson:"review_level" json:"review_level"` // 复核等级: HIGH/LOW/空

View File

@@ -567,8 +567,83 @@ func (r *Repository) ReconcileRefund(billType, orderNo, merchantOrderNo string,
return true, nil
}
// 建议: 为提升 ReconcileRefund 查询性能,可为 bills_cleaned 添加索引
// {transaction_id:1, bill_type:1} 和 {merchant_order_no:1, bill_type:1}(与现有"无索引"问题一并处理)
// ReconcileWechatRefund 将跨批次微信退款核销到已存储的清洗后账单
// 微信"当前状态"字段为累计退款金额,使用 original_amount入库时原始金额计算剩余
func (r *Repository) ReconcileWechatRefund(transactionID string, fullRefund bool, cumulativeRefundAmount float64) (bool, error) {
if r.cleanedCollection == nil {
return false, fmt.Errorf("cleaned collection not initialized")
}
if transactionID == "" {
return false, nil
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
filter := bson.M{
"bill_type": "wechat",
"transaction_id": transactionID,
"is_deleted": bson.M{"$ne": true},
}
var bill model.CleanedBill
if err := r.cleanedCollection.FindOne(ctx, filter).Decode(&bill); err != nil {
if err == mongo.ErrNoDocuments {
return false, nil
}
return false, fmt.Errorf("查询待核销微信账单失败: %w", err)
}
now := time.Now()
if fullRefund {
update := bson.M{"$set": bson.M{
"is_deleted": true,
"updated_at": now,
"remark": fmt.Sprintf("[退款核销]全额退款;%s", bill.Remark),
}}
_, err := r.cleanedCollection.UpdateOne(ctx, bson.M{"_id": bill.ID}, update)
if err != nil {
return false, fmt.Errorf("核销微信全额退款失败: %w", err)
}
return true, nil
}
// 部分退款用原始金额original_amount减去累计退款金额得到剩余
originalAmount := bill.OriginalAmount
if originalAmount <= 0 {
originalAmount = bill.Amount // 兼容旧记录(无 original_amount 字段)
}
remaining := originalAmount - cumulativeRefundAmount
if remaining <= refundEpsilon {
update := bson.M{"$set": bson.M{
"is_deleted": true,
"updated_at": now,
"remark": fmt.Sprintf("[退款核销]全额退款%.2f元(原金额%.2f元);%s", cumulativeRefundAmount, originalAmount, bill.Remark),
}}
_, err := r.cleanedCollection.UpdateOne(ctx, bson.M{"_id": bill.ID}, update)
if err != nil {
return false, fmt.Errorf("核销微信退款失败: %w", err)
}
return true, nil
}
remaining = math.Round(remaining*100) / 100
update := bson.M{"$set": bson.M{
"amount": remaining,
"updated_at": now,
"remark": fmt.Sprintf("原金额%.2f元,已退款%.2f元;%s", originalAmount, cumulativeRefundAmount, bill.Remark),
}}
_, err := r.cleanedCollection.UpdateOne(ctx, bson.M{"_id": bill.ID}, update)
if err != nil {
return false, fmt.Errorf("核销微信退款失败: %w", err)
}
return true, nil
}
// 建议: 为提升查询性能,可为 bills_cleaned 添加索引
// {transaction_id:1, bill_type:1} 和 {merchant_order_no:1, bill_type:1}
// GetClient 获取 MongoDB 客户端(用于兼容旧代码)
func (r *Repository) GetClient() *mongo.Client {

View File

@@ -62,4 +62,10 @@ type BillRepository interface {
// 全额退款(剩余金额 <= 0.005)则软删除;部分退款则扣减 amount 并追加备注
// 返回: 是否找到并核销了匹配记录、错误(未找到匹配记录不算错误,返回 matched=false
ReconcileRefund(billType, orderNo, merchantOrderNo string, refundAmount float64, refundTime, merchant, description, refundOrderNo string) (matched bool, err error)
// ReconcileWechatRefund 将跨批次微信退款核销到已存储的清洗后账单
// 微信退款通过重复上传时"当前状态"字段携带退款信息来触发
// fullRefund=true 时软删除原记录;否则用 original_amount - cumulativeRefundAmount 计算剩余金额
// 返回: 是否找到并核销了匹配记录、错误
ReconcileWechatRefund(transactionID string, fullRefund bool, cumulativeRefundAmount float64) (matched bool, err error)
}

View File

@@ -23,13 +23,21 @@ func getRepo() repository.BillRepository {
return repository.GetRepository()
}
// WechatRefundUpdate 微信重复行中携带的退款信息(用于跨批次退款核销)
type WechatRefundUpdate struct {
TransactionID string // 原消费行的交易单号
FullRefund bool // 是否全额退款(已全额退款)
CumulativeRefundAmount float64 // 累计退款金额(已退款(¥X)中的 X与原始金额相减得剩余
}
// DeduplicateResult 去重结果
type DeduplicateResult struct {
OriginalCount int // 原始记录数
DuplicateCount int // 重复记录数
NewCount int // 新记录数
DedupFilePath string // 去重后的文件路径(如果有去重则生成新文件)
BillType string // 检测到的账单类型
OriginalCount int // 原始记录数
DuplicateCount int // 重复记录数
NewCount int // 新记录数
DedupFilePath string // 去重后的文件路径(如果有去重则生成新文件)
BillType string // 检测到的账单类型
WechatRefundUpdates []WechatRefundUpdate // 微信重复行中检测到的退款状态(用于跨批次核销)
}
// DeduplicateRawFile 对原始文件进行去重检查,返回去重后的文件路径
@@ -75,6 +83,17 @@ func DeduplicateRawFile(filePath, uploadBatch string) (*DeduplicateResult, error
return result, nil
}
// 对于微信账单,找到"当前状态"列的索引,用于检测退款状态
wechatStatusIdx := -1
if billType == "wechat" {
for i, col := range header {
if col == "当前状态" {
wechatStatusIdx = i
break
}
}
}
// 检查每行是否重复
var newRows [][]string
for _, row := range dataRows {
@@ -101,6 +120,24 @@ func DeduplicateRawFile(filePath, uploadBatch string) (*DeduplicateResult, error
newRows = append(newRows, row)
} else {
result.DuplicateCount++
// 微信账单:检查重复行是否携带退款状态(跨批次退款核销)
if wechatStatusIdx >= 0 && len(row) > wechatStatusIdx {
status := strings.TrimSpace(row[wechatStatusIdx])
if strings.Contains(status, "已全额退款") {
result.WechatRefundUpdates = append(result.WechatRefundUpdates, WechatRefundUpdate{
TransactionID: transactionID,
FullRefund: true,
})
} else if strings.Contains(status, "已退款") {
if amount := extractWechatRefundAmount(status); amount > 0 {
result.WechatRefundUpdates = append(result.WechatRefundUpdates, WechatRefundUpdate{
TransactionID: transactionID,
FullRefund: false,
CumulativeRefundAmount: amount,
})
}
}
}
}
}
@@ -337,6 +374,7 @@ func saveCleanedBillsFromCSV(filePath, billType, sourceFile, uploadBatch string)
bill.ReviewLevel = row[idx]
}
bill.OriginalAmount = bill.Amount
bills = append(bills, bill)
}
@@ -431,6 +469,7 @@ func saveCleanedBillsFromJSON(filePath, billType, sourceFile, uploadBatch string
bill.ReviewLevel = v
}
bill.OriginalAmount = bill.Amount
bills = append(bills, bill)
}
@@ -512,3 +551,28 @@ func parseAmount(s string) float64 {
}
return 0
}
// extractWechatRefundAmount 从微信"当前状态"字段中提取累计退款金额
// 支持格式: "已退款(¥3.00)"、"已退款(¥3.00)"、"已退款¥3.00"
func extractWechatRefundAmount(status string) float64 {
start := strings.Index(status, "(")
end := strings.LastIndex(status, ")")
var inner string
if start >= 0 && end > start {
inner = status[start+1 : end]
} else {
idx := strings.Index(status, "已退款")
if idx < 0 {
return 0
}
inner = status[idx+len("已退款"):]
}
inner = strings.TrimPrefix(inner, "¥")
inner = strings.TrimPrefix(inner, "¥")
inner = strings.TrimSpace(inner)
v, err := strconv.ParseFloat(inner, 64)
if err != nil {
return 0
}
return v
}

View File

@@ -263,7 +263,7 @@
{:else}
<div>
<div class="text-center mb-6">
<div class="text-3xl font-bold font-mono {record.incomeExpense === '收入' ? 'text-green-600 dark:text-green-400' : 'text-red-600 dark:text-red-400'}">
<div class="text-3xl font-bold font-mono {record.incomeExpense === '收入' ? 'text-green-600 dark:text-green-400' : record.incomeExpense === '不计收支' ? 'text-muted-foreground' : 'text-red-600 dark:text-red-400'}">
¥{record.amount.toFixed(2)}
</div>
<div class="text-sm text-muted-foreground mt-1">{record.incomeExpense || '支出'}金额</div>