diff --git a/AGENTS.md b/AGENTS.md index e8f91b6..61f9abc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -92,6 +92,13 @@ Browser → SvelteKit proxy → Go (Gin) → handler → service → adapter → ### File Processing Pipeline Upload: ZIP/XLSX → Extract → Convert UTF-8 CSV → Detect bill type (alipay/wechat/jd) → Deduplicate against MongoDB → Clean via Python → Save cleaned data. +### Cross-Batch Refund Reconciliation +When the original purchase and its refund are in different upload batches, within-batch logic can't match them. Two reconciliation paths handle this: + +**Alipay** (`handler/upload.go` step 14): After cleaning, `cleaner.unresolved_refunds` (refund rows that found no matching expense in the same file) is returned by FastAPI and iterated in Go. `ReconcileRefund()` looks up the original expense by `transaction_id` or `merchant_order_no` and either soft-deletes (full refund) or deducts `amount` (partial). Alipay refund rows have their own distinct `transaction_id` (suffixed `_`), so they pass raw dedup and are naturally idempotent on re-upload. + +**WeChat** (`handler/upload.go` step 15): WeChat re-exports the *same row* (same `transaction_id`) with an updated `当前状态` field (`已全额退款` or `已退款(¥X)`). `DeduplicateRawFile` detects these duplicate rows, extracts the refund info into `DeduplicateResult.WechatRefundUpdates`, and `ReconcileWechatRefund()` applies the update. WeChat's `¥X` is the **cumulative** total refunded, so `CleanedBill.original_amount` (set at first save) is used to compute `remaining = original_amount - X`. + ### Adapter (Go ↔ Python) `adapter.Cleaner` interface. Two modes: - `http` (default): calls FastAPI at `ANALYZER_URL` diff --git a/CHANGELOG.md b/CHANGELOG.md index 08cf1e7..590c65e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,23 @@ ## [Unreleased] +### 新增 +- **微信跨批次退款核销** - 重复上传微信账单时，自动识别"当前状态"字段中的退款信息并核销原始支出记录 + - 支持「已全额退款」：原支出记录软删除 + - 支持「已退款(¥X)」：累计退款金额从原始金额（`original_amount`）中扣减，剩余金额更新到记录 + - 新增 `CleanedBill.original_amount` 字段，存储入库时的原始金额，保证多次部分退款累计计算正确 + - `ReconcileWechatRefund` 接口和 MongoDB 实现 +- **支付宝跨批次退款核销** - 上传含退款行的支付宝账单时，若原消费记录来自更早的批次，自动在数据库中核销 + - 退款行的「交易订单号」带后缀（`原订单号_退款编号`），天然幂等：重复上传时被原始数据去重拦截，不会重复核销 + - 全额退款软删除，部分退款扣减金额并追加备注 + - `ReconcileRefund` 接口和 MongoDB 实现 + +### 技术改进 +- `server/service/bill.go`：`DeduplicateRawFile` 在微信账单去重阶段检测已退款状态行，收集 `WechatRefundUpdates` 供后续核销 +- `server/service/bill.go`：`saveCleanedBillsFromCSV` / `saveCleanedBillsFromJSON` 入库时同步写入 `original_amount` +- `analyzer/cleaners/alipay.py`：`_process_expenses` 收集同批次内未匹配的退款（`unresolved_refunds`），通过 FastAPI 响应透传至 Go +- `analyzer/cleaners/base.py`：`BaseCleaner` 基类新增 `unresolved_refunds` 属性 + ## [1.4.0] - 2026-03-23 ### 新增 diff --git a/analyzer/cleaners/wechat.py b/analyzer/cleaners/wechat.py index cef9d83..221de77 100644 --- a/analyzer/cleaners/wechat.py +++ b/analyzer/cleaners/wechat.py @@ -54,7 +54,23 @@ class WechatCleaner(BaseCleaner): # 第三步：处理退款（包括转账退款） final_expense_rows, income_rows = self._process_refunds(expense_rows, income_rows) - + + # 收集跨批次未匹配的 -退款 行（当前批次无对应支出记录，需 Go 侧跨批次核销） + expense_merchants = {exp[2].strip() for exp in expense_rows} + for refund_row in refund_rows: + if refund_row[2].strip() not in expense_merchants: + amount = float(parse_amount(refund_row[5])) + if amount > 0: + self.unresolved_refunds.append({ + "order_no": "", + "merchant_order_no": refund_row[9].strip() if len(refund_row) > 9 else "", + "refund_order_no": refund_row[8].strip() if len(refund_row) > 8 else "", + "amount": amount, + "time": refund_row[0], + "merchant": refund_row[2], + "description": refund_row[3] if len(refund_row) > 3 else "", + }) + print(f"\n处理结果:") print(f" 全额退款删除: {self.stats['fully_refunded']} 条") print(f" 部分退款调整: {self.stats['partially_refunded']} 条") diff --git a/analyzer/test_jd_cleaner.py b/analyzer/test_jd_cleaner.py index e0de792..7d0bc40 100644 --- a/analyzer/test_jd_cleaner.py +++ b/analyzer/test_jd_cleaner.py @@ -11,7 +11,8 @@ import sys sys.stdout.reconfigure(encoding='utf-8') def test_jd_cleaner(): - zip_path = r'D:\Projects\BillAI\mock_data\京东交易流水(申请时间2026年01月26日13时29分47秒)(密码683263)_209.zip' + base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + zip_path = os.path.join(base_dir, 'mock_data', '京东交易流水(申请时间2026年01月26日13时29分47秒)(密码683263)_209.zip') with zipfile.ZipFile(zip_path, 'r') as zf: with tempfile.TemporaryDirectory() as tmpdir: diff --git a/server/handler/auth.go b/server/handler/auth.go index 6db9e21..27040b4 100644 --- a/server/handler/auth.go +++ b/server/handler/auth.go @@ -98,7 +98,11 @@ func Login(c *gin.Context) { secret := config.Global.JWTSecret if secret == "" { - secret = "billai-default-secret" + c.JSON(http.StatusInternalServerError, gin.H{ + "success": false, + "error": "服务器 JWT 配置缺失", + }) + return } token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) @@ -144,7 +148,12 @@ func ValidateToken(c *gin.Context) { secret := config.Global.JWTSecret if secret == "" { - secret = "billai-default-secret" + c.JSON(http.StatusUnauthorized, gin.H{ + "success": false, + "error": "服务器 JWT 配置缺失", + "code": "TOKEN_INVALID", + }) + return } token, err := jwt.ParseWithClaims(tokenString, &Claims{}, func(token *jwt.Token) (interface{}, error) { diff --git a/server/handler/bills.go b/server/handler/bills.go index 6f2bd30..3b097b6 100644 --- a/server/handler/bills.go +++ b/server/handler/bills.go @@ -2,7 +2,6 @@ package handler import ( "net/http" - "strconv" "time" "github.com/gin-gonic/gin" @@ -154,18 +153,6 @@ func ListBills(c *gin.Context) { }) } -// parsePageParam 解析分页参数 -func parsePageParam(s string, defaultVal int) int { - if s == "" { - return defaultVal - } - val, err := strconv.Atoi(s) - if err != nil || val < 1 { - return defaultVal - } - return val -} - // MonthlyStatsResponse 月度统计响应 type MonthlyStatsResponse struct { Result bool `json:"result"` @@ -202,6 +189,13 @@ func MonthlyStats(c *gin.Context) { // ReviewStats 获取待复核数据统计 func ReviewStats(c *gin.Context) { repo := repository.GetRepository() + if repo == nil { + c.JSON(http.StatusServiceUnavailable, model.ReviewResponse{ + Result: false, + Message: "数据库未连接", + }) + return + } // 从MongoDB查询所有需要复核的账单 bills, err := repo.GetBillsNeedReview() diff --git a/server/handler/upload.go b/server/handler/upload.go index a640ac1..1bb58b9 100644 --- a/server/handler/upload.go +++ b/server/handler/upload.go @@ -269,7 +269,7 @@ func Upload(c *gin.Context) { } } - // 14. 核销跨批次退款（本次清洗中未在同批次内匹配到对应支出的退款） + // 14. 核销跨批次退款（支付宝：本次清洗中未在同批次内匹配到对应支出的退款） var reconciledCount int if repo != nil && cleanResult != nil { for _, ur := range cleanResult.UnresolvedRefunds { @@ -285,7 +285,26 @@ func Upload(c *gin.Context) { } } - // 15. 返回成功响应 + // 15. 核销跨批次微信退款（重复上传的行中携带的已退款状态） + if repo != nil && len(dedupResult.WechatRefundUpdates) > 0 { + for _, wu := range dedupResult.WechatRefundUpdates { + matched, rErr := repo.ReconcileWechatRefund(wu.TransactionID, wu.FullRefund, wu.CumulativeRefundAmount) + if rErr != nil { + fmt.Printf("⚠️ 微信退款核销失败: %v\n", rErr) + continue + } + if matched { + reconciledCount++ + if wu.FullRefund { + fmt.Printf("💰 已核销微信全额退款: 交易单号%s\n", wu.TransactionID) + } else { + fmt.Printf("💰 已核销微信部分退款: 交易单号%s, 累计退款%.2f元\n", wu.TransactionID, wu.CumulativeRefundAmount) + } + } + } + } + + // 16. 返回成功响应 message := fmt.Sprintf("处理成功，新增 %d 条记录", cleanedCount) if dedupResult.DuplicateCount > 0 { message = fmt.Sprintf("处理成功，新增 %d 条，跳过 %d 条重复记录", cleanedCount, dedupResult.DuplicateCount) diff --git a/server/middleware/auth.go b/server/middleware/auth.go index 2fe58e3..8a80e49 100644 --- a/server/middleware/auth.go +++ b/server/middleware/auth.go @@ -38,7 +38,13 @@ func AuthRequired() gin.HandlerFunc { secret := config.Global.JWTSecret if secret == "" { - secret = "billai-default-secret" + c.JSON(http.StatusUnauthorized, gin.H{ + "success": false, + "error": "服务器 JWT 配置缺失", + "code": "TOKEN_INVALID", + }) + c.Abort() + return } token, err := jwt.ParseWithClaims(tokenString, &Claims{}, func(token *jwt.Token) (interface{}, error) { diff --git a/server/model/bill.go b/server/model/bill.go index 2118c3c..45da2b7 100644 --- a/server/model/bill.go +++ b/server/model/bill.go @@ -88,9 +88,10 @@ type CleanedBill struct { Category string `bson:"category" json:"category"` // 交易分类 Merchant string `bson:"merchant" json:"merchant"` // 交易对方 Description string `bson:"description" json:"description"` // 商品说明 - IncomeExpense string `bson:"income_expense" json:"income_expense"` // 收/支 - Amount float64 `bson:"amount" json:"amount"` // 金额 - PayMethod string `bson:"pay_method" json:"pay_method"` // 支付方式 + IncomeExpense string `bson:"income_expense" json:"income_expense"` // 收/支 + Amount float64 `bson:"amount" json:"amount"` // 金额 + OriginalAmount float64 `bson:"original_amount,omitempty" json:"original_amount,omitempty"` // 原始金额（入库时），用于微信跨批次退款核销 + PayMethod string `bson:"pay_method" json:"pay_method"` // 支付方式 Status string `bson:"status" json:"status"` // 交易状态 Remark string `bson:"remark" json:"remark"` // 备注 ReviewLevel string `bson:"review_level" json:"review_level"` // 复核等级: HIGH/LOW/空 diff --git a/server/repository/mongo/repository.go b/server/repository/mongo/repository.go index 2eaf958..2610141 100644 --- a/server/repository/mongo/repository.go +++ b/server/repository/mongo/repository.go @@ -567,8 +567,83 @@ func (r *Repository) ReconcileRefund(billType, orderNo, merchantOrderNo string, return true, nil } -// 建议: 为提升 ReconcileRefund 查询性能，可为 bills_cleaned 添加索引 -// {transaction_id:1, bill_type:1} 和 {merchant_order_no:1, bill_type:1}（与现有"无索引"问题一并处理） +// ReconcileWechatRefund 将跨批次微信退款核销到已存储的清洗后账单 +// 微信"当前状态"字段为累计退款金额，使用 original_amount（入库时原始金额）计算剩余 +func (r *Repository) ReconcileWechatRefund(transactionID string, fullRefund bool, cumulativeRefundAmount float64) (bool, error) { + if r.cleanedCollection == nil { + return false, fmt.Errorf("cleaned collection not initialized") + } + if transactionID == "" { + return false, nil + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + filter := bson.M{ + "bill_type": "wechat", + "transaction_id": transactionID, + "is_deleted": bson.M{"$ne": true}, + } + + var bill model.CleanedBill + if err := r.cleanedCollection.FindOne(ctx, filter).Decode(&bill); err != nil { + if err == mongo.ErrNoDocuments { + return false, nil + } + return false, fmt.Errorf("查询待核销微信账单失败: %w", err) + } + + now := time.Now() + + if fullRefund { + update := bson.M{"$set": bson.M{ + "is_deleted": true, + "updated_at": now, + "remark": fmt.Sprintf("[退款核销]全额退款;%s", bill.Remark), + }} + _, err := r.cleanedCollection.UpdateOne(ctx, bson.M{"_id": bill.ID}, update) + if err != nil { + return false, fmt.Errorf("核销微信全额退款失败: %w", err) + } + return true, nil + } + + // 部分退款：用原始金额（original_amount）减去累计退款金额得到剩余 + originalAmount := bill.OriginalAmount + if originalAmount <= 0 { + originalAmount = bill.Amount // 兼容旧记录（无 original_amount 字段） + } + remaining := originalAmount - cumulativeRefundAmount + + if remaining <= refundEpsilon { + update := bson.M{"$set": bson.M{ + "is_deleted": true, + "updated_at": now, + "remark": fmt.Sprintf("[退款核销]全额退款%.2f元(原金额%.2f元);%s", cumulativeRefundAmount, originalAmount, bill.Remark), + }} + _, err := r.cleanedCollection.UpdateOne(ctx, bson.M{"_id": bill.ID}, update) + if err != nil { + return false, fmt.Errorf("核销微信退款失败: %w", err) + } + return true, nil + } + + remaining = math.Round(remaining*100) / 100 + update := bson.M{"$set": bson.M{ + "amount": remaining, + "updated_at": now, + "remark": fmt.Sprintf("原金额%.2f元,已退款%.2f元;%s", originalAmount, cumulativeRefundAmount, bill.Remark), + }} + _, err := r.cleanedCollection.UpdateOne(ctx, bson.M{"_id": bill.ID}, update) + if err != nil { + return false, fmt.Errorf("核销微信退款失败: %w", err) + } + return true, nil +} + +// 建议: 为提升查询性能，可为 bills_cleaned 添加索引 +// {transaction_id:1, bill_type:1} 和 {merchant_order_no:1, bill_type:1} // GetClient 获取 MongoDB 客户端（用于兼容旧代码） func (r *Repository) GetClient() *mongo.Client { diff --git a/server/repository/repository.go b/server/repository/repository.go index e4f3907..7a4a5c0 100644 --- a/server/repository/repository.go +++ b/server/repository/repository.go @@ -62,4 +62,10 @@ type BillRepository interface { // 全额退款（剩余金额 <= 0.005）则软删除；部分退款则扣减 amount 并追加备注 // 返回: 是否找到并核销了匹配记录、错误（未找到匹配记录不算错误，返回 matched=false） ReconcileRefund(billType, orderNo, merchantOrderNo string, refundAmount float64, refundTime, merchant, description, refundOrderNo string) (matched bool, err error) + + // ReconcileWechatRefund 将跨批次微信退款核销到已存储的清洗后账单 + // 微信退款通过重复上传时"当前状态"字段携带退款信息来触发 + // fullRefund=true 时软删除原记录；否则用 original_amount - cumulativeRefundAmount 计算剩余金额 + // 返回: 是否找到并核销了匹配记录、错误 + ReconcileWechatRefund(transactionID string, fullRefund bool, cumulativeRefundAmount float64) (matched bool, err error) } diff --git a/server/service/bill.go b/server/service/bill.go index d109bb0..348d8ba 100644 --- a/server/service/bill.go +++ b/server/service/bill.go @@ -23,13 +23,21 @@ func getRepo() repository.BillRepository { return repository.GetRepository() } +// WechatRefundUpdate 微信重复行中携带的退款信息（用于跨批次退款核销） +type WechatRefundUpdate struct { + TransactionID string // 原消费行的交易单号 + FullRefund bool // 是否全额退款（已全额退款） + CumulativeRefundAmount float64 // 累计退款金额（已退款(¥X)中的 X，与原始金额相减得剩余） +} + // DeduplicateResult 去重结果 type DeduplicateResult struct { - OriginalCount int // 原始记录数 - DuplicateCount int // 重复记录数 - NewCount int // 新记录数 - DedupFilePath string // 去重后的文件路径（如果有去重则生成新文件） - BillType string // 检测到的账单类型 + OriginalCount int // 原始记录数 + DuplicateCount int // 重复记录数 + NewCount int // 新记录数 + DedupFilePath string // 去重后的文件路径（如果有去重则生成新文件） + BillType string // 检测到的账单类型 + WechatRefundUpdates []WechatRefundUpdate // 微信重复行中检测到的退款状态（用于跨批次核销） } // DeduplicateRawFile 对原始文件进行去重检查，返回去重后的文件路径 @@ -75,6 +83,17 @@ func DeduplicateRawFile(filePath, uploadBatch string) (*DeduplicateResult, error return result, nil } + // 对于微信账单，找到"当前状态"列的索引，用于检测退款状态 + wechatStatusIdx := -1 + if billType == "wechat" { + for i, col := range header { + if col == "当前状态" { + wechatStatusIdx = i + break + } + } + } + // 检查每行是否重复 var newRows [][]string for _, row := range dataRows { @@ -101,6 +120,24 @@ func DeduplicateRawFile(filePath, uploadBatch string) (*DeduplicateResult, error newRows = append(newRows, row) } else { result.DuplicateCount++ + // 微信账单：检查重复行是否携带退款状态（跨批次退款核销） + if wechatStatusIdx >= 0 && len(row) > wechatStatusIdx { + status := strings.TrimSpace(row[wechatStatusIdx]) + if strings.Contains(status, "已全额退款") { + result.WechatRefundUpdates = append(result.WechatRefundUpdates, WechatRefundUpdate{ + TransactionID: transactionID, + FullRefund: true, + }) + } else if strings.Contains(status, "已退款") { + if amount := extractWechatRefundAmount(status); amount > 0 { + result.WechatRefundUpdates = append(result.WechatRefundUpdates, WechatRefundUpdate{ + TransactionID: transactionID, + FullRefund: false, + CumulativeRefundAmount: amount, + }) + } + } + } } } @@ -337,6 +374,7 @@ func saveCleanedBillsFromCSV(filePath, billType, sourceFile, uploadBatch string) bill.ReviewLevel = row[idx] } + bill.OriginalAmount = bill.Amount bills = append(bills, bill) } @@ -431,6 +469,7 @@ func saveCleanedBillsFromJSON(filePath, billType, sourceFile, uploadBatch string bill.ReviewLevel = v } + bill.OriginalAmount = bill.Amount bills = append(bills, bill) } @@ -512,3 +551,28 @@ func parseAmount(s string) float64 { } return 0 } + +// extractWechatRefundAmount 从微信"当前状态"字段中提取累计退款金额 +// 支持格式: "已退款(¥3.00)"、"已退款(￥3.00)"、"已退款￥3.00" +func extractWechatRefundAmount(status string) float64 { + start := strings.Index(status, "(") + end := strings.LastIndex(status, ")") + var inner string + if start >= 0 && end > start { + inner = status[start+1 : end] + } else { + idx := strings.Index(status, "已退款") + if idx < 0 { + return 0 + } + inner = status[idx+len("已退款"):] + } + inner = strings.TrimPrefix(inner, "¥") + inner = strings.TrimPrefix(inner, "￥") + inner = strings.TrimSpace(inner) + v, err := strconv.ParseFloat(inner, 64) + if err != nil { + return 0 + } + return v +} diff --git a/web/src/lib/components/analysis/BillDetailDrawer.svelte b/web/src/lib/components/analysis/BillDetailDrawer.svelte index b0564ad..693c4de 100644 --- a/web/src/lib/components/analysis/BillDetailDrawer.svelte +++ b/web/src/lib/components/analysis/BillDetailDrawer.svelte @@ -263,7 +263,7 @@ {:else}
-
+
¥{record.amount.toFixed(2)}
{record.incomeExpense || '支出'}金额