package handler import ( "crypto/rand" "encoding/hex" "fmt" "io" "net/http" "os" "path/filepath" "strings" "time" "github.com/gin-gonic/gin" "billai-server/config" "billai-server/model" "billai-server/repository" "billai-server/service" ) // Upload 处理账单上传和清理请求 // 支持直接上传 CSV 文件,或上传 ZIP 压缩包(支持密码保护) // ZIP 包内可以是 CSV 或 XLSX 格式的账单文件 func Upload(c *gin.Context) { // 1. 获取上传的文件 file, header, err := c.Request.FormFile("file") if err != nil { c.JSON(http.StatusBadRequest, model.UploadResponse{ Result: false, Message: "请上传账单文件 (参数名: file)", }) return } defer file.Close() // 2. 解析请求参数 var req model.UploadRequest c.ShouldBind(&req) if req.Format == "" { req.Format = "csv" } // 3. 保存上传的文件 timestamp := time.Now().Format("20060102_150405") uniqueID := generateShortID() ext := filepath.Ext(header.Filename) baseName := header.Filename[:len(header.Filename)-len(ext)] inputFileName := fmt.Sprintf("%s_%s_%s%s", timestamp, uniqueID, baseName, ext) uploadDirAbs := config.ResolvePath(config.Global.UploadDir) inputPath := filepath.Join(uploadDirAbs, inputFileName) dst, err := os.Create(inputPath) if err != nil { c.JSON(http.StatusInternalServerError, model.UploadResponse{ Result: false, Message: "保存文件失败: " + err.Error(), }) return } defer dst.Close() if _, err := io.Copy(dst, file); err != nil { c.JSON(http.StatusInternalServerError, model.UploadResponse{ Result: false, Message: "保存文件失败: " + err.Error(), }) return } dst.Close() // 关闭文件以便后续处理 // 4. 处理文件:如果是 ZIP 则解压,否则直接处理 var billFilePath string var billType string var extractedFiles []string var needConvert bool // 是否需要格式转换(xlsx -> csv) if service.IsSupportedArchive(header.Filename) { // 解压 ZIP 文件 fmt.Printf("📦 检测到 ZIP 文件,开始解压...\n") extractResult, err := service.ExtractZip(inputPath, uploadDirAbs, req.Password) if err != nil { c.JSON(http.StatusBadRequest, model.UploadResponse{ Result: false, Message: "解压失败: " + err.Error(), }) return } billFilePath = extractResult.BillFile extractedFiles = extractResult.ExtractedFiles // 使用从文件名检测到的账单类型(如果用户未指定) if req.Type == "" && extractResult.BillType != "" { billType = extractResult.BillType } fmt.Printf(" 解压完成,账单文件: %s\n", filepath.Base(billFilePath)) // ZIP 中提取的文件需要格式转换(xlsx 需要转 csv,csv 可能需要编码转换) needConvert = true } else { // 直接使用上传的文件 billFilePath = inputPath // 检查是否为 xlsx 格式 if strings.HasSuffix(strings.ToLower(header.Filename), ".xlsx") { needConvert = true } } // 5. 如果需要格式/编码转换,调用 analyzer 服务 if needConvert { fmt.Printf("📊 调用分析服务进行格式/编码转换...\n") convertedPath, detectedType, err := service.ConvertBillFile(billFilePath) if err != nil { // 清理临时文件 service.CleanupExtractedFiles(extractedFiles) c.JSON(http.StatusBadRequest, model.UploadResponse{ Result: false, Message: "文件转换失败: " + err.Error(), }) return } // 如果转换后的路径与原路径不同,删除原始文件 if convertedPath != billFilePath { os.Remove(billFilePath) } billFilePath = convertedPath // 使用检测到的账单类型 if req.Type == "" && detectedType != "" { billType = detectedType } fmt.Printf(" 转换完成: %s\n", filepath.Base(convertedPath)) } // 6. 确定账单类型 if req.Type != "" { billType = req.Type } if billType == "" { // 尝试从文件名检测 fileName := strings.ToLower(filepath.Base(billFilePath)) if strings.Contains(fileName, "支付宝") || strings.Contains(fileName, "alipay") { billType = "alipay" } else if strings.Contains(fileName, "微信") || strings.Contains(fileName, "wechat") { billType = "wechat" } else if strings.Contains(fileName, "京东") || strings.Contains(fileName, "jd") { billType = "jd" } } if billType == "" { // 清理临时文件 service.CleanupExtractedFiles(extractedFiles) c.JSON(http.StatusBadRequest, model.UploadResponse{ Result: false, Message: "无法识别账单类型,请指定 type 参数 (alipay/wechat/jd)", }) return } if billType != "alipay" && billType != "wechat" && billType != "jd" { service.CleanupExtractedFiles(extractedFiles) c.JSON(http.StatusBadRequest, model.UploadResponse{ Result: false, Message: "账单类型无效,仅支持 alipay/wechat/jd", }) return } // 7. 对原始数据进行去重检查 fmt.Printf("📋 开始去重检查...\n") dedupResult, dedupErr := service.DeduplicateRawFile(billFilePath, timestamp) if dedupErr != nil { service.CleanupExtractedFiles(extractedFiles) c.JSON(http.StatusInternalServerError, model.UploadResponse{ Result: false, Message: "去重检查失败: " + dedupErr.Error(), }) return } fmt.Printf(" 原始记录: %d 条\n", dedupResult.OriginalCount) if dedupResult.DuplicateCount > 0 { fmt.Printf(" 重复记录: %d 条(已跳过)\n", dedupResult.DuplicateCount) } fmt.Printf(" 新增记录: %d 条\n", dedupResult.NewCount) // 如果全部重复,返回提示 if dedupResult.NewCount == 0 { service.CleanupExtractedFiles(extractedFiles) c.JSON(http.StatusOK, model.UploadResponse{ Result: true, Message: fmt.Sprintf("文件中的 %d 条记录全部已存在,无需重复导入", dedupResult.OriginalCount), Data: &model.UploadData{ BillType: billType, RawCount: 0, CleanedCount: 0, DuplicateCount: dedupResult.DuplicateCount, }, }) return } // 使用去重后的文件路径进行后续处理 processFilePath := dedupResult.DedupFilePath // 8. 构建输出文件路径 outputExt := ".csv" if req.Format == "json" { outputExt = ".json" } outputDirAbs := config.ResolvePath(config.Global.OutputDir) fileSeq := generateFileSequence(outputDirAbs, timestamp, billType, outputExt) outputFileName := fmt.Sprintf("%s_%s_%s%s", timestamp, billType, fileSeq, outputExt) outputPath := filepath.Join(outputDirAbs, outputFileName) // 9. 执行 Python 清洗脚本 cleanOpts := &service.CleanOptions{ Year: req.Year, Month: req.Month, Start: req.Start, End: req.End, Format: req.Format, } _, cleanErr := service.RunCleanScript(processFilePath, outputPath, cleanOpts) if cleanErr != nil { service.CleanupExtractedFiles(extractedFiles) c.JSON(http.StatusInternalServerError, model.UploadResponse{ Result: false, Message: cleanErr.Error(), }) return } // 10. 将去重后的原始数据存入 MongoDB rawCount, rawErr := service.SaveRawBillsFromFile(processFilePath, billType, header.Filename, timestamp) if rawErr != nil { fmt.Printf("⚠️ 存储原始数据到 MongoDB 失败: %v\n", rawErr) } else { fmt.Printf("✅ 已存储 %d 条原始账单记录到 MongoDB\n", rawCount) } // 11. 将清洗后的数据存入 MongoDB cleanedCount, _, cleanedErr := service.SaveCleanedBillsFromFile(outputPath, req.Format, billType, header.Filename, timestamp) if cleanedErr != nil { fmt.Printf("⚠️ 存储清洗后数据到 MongoDB 失败: %v\n", cleanedErr) } else { fmt.Printf("✅ 已存储 %d 条清洗后账单记录到 MongoDB\n", cleanedCount) } // 12. 清理临时文件 if dedupResult.DedupFilePath != inputPath && dedupResult.DedupFilePath != "" { os.Remove(dedupResult.DedupFilePath) } service.CleanupExtractedFiles(extractedFiles) // 13. 如果是京东账单,软删除其他来源中包含"京东-订单编号"的记录 var jdRelatedDeleted int64 if billType == "jd" { repo := repository.GetRepository() if repo != nil { deleted, err := repo.SoftDeleteJDRelatedBills() if err != nil { fmt.Printf("⚠️ 软删除京东关联记录失败: %v\n", err) } else if deleted > 0 { jdRelatedDeleted = deleted fmt.Printf("🗑️ 已软删除 %d 条其他来源中的京东关联记录\n", deleted) } } } // 14. 返回成功响应 message := fmt.Sprintf("处理成功,新增 %d 条记录", cleanedCount) if dedupResult.DuplicateCount > 0 { message = fmt.Sprintf("处理成功,新增 %d 条,跳过 %d 条重复记录", cleanedCount, dedupResult.DuplicateCount) } if jdRelatedDeleted > 0 { message = fmt.Sprintf("%s,标记删除 %d 条重复的京东订单", message, jdRelatedDeleted) } c.JSON(http.StatusOK, model.UploadResponse{ Result: true, Message: message, Data: &model.UploadData{ BillType: billType, FileURL: fmt.Sprintf("/download/%s", outputFileName), FileName: outputFileName, RawCount: rawCount, CleanedCount: cleanedCount, DuplicateCount: dedupResult.DuplicateCount, JDRelatedDeleted: jdRelatedDeleted, }, }) } // generateFileSequence 生成文件序号 func generateFileSequence(dir, timestamp, billType, ext string) string { pattern := fmt.Sprintf("%s_%s_*%s", timestamp, billType, ext) matches, err := filepath.Glob(filepath.Join(dir, pattern)) if err != nil || len(matches) == 0 { return "001" } return fmt.Sprintf("%03d", len(matches)+1) } // generateShortID 生成 6 位随机唯一标识符 func generateShortID() string { bytes := make([]byte, 3) if _, err := rand.Read(bytes); err != nil { return fmt.Sprintf("%06x", time.Now().UnixNano()%0xFFFFFF) } return hex.EncodeToString(bytes) }