295 lines
8.8 KiB
Go
295 lines
8.8 KiB
Go
package handler
|
||
|
||
import (
|
||
"crypto/rand"
|
||
"encoding/hex"
|
||
"fmt"
|
||
"io"
|
||
"net/http"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"time"
|
||
|
||
"github.com/gin-gonic/gin"
|
||
|
||
"billai-server/config"
|
||
"billai-server/model"
|
||
"billai-server/service"
|
||
)
|
||
|
||
// Upload 处理账单上传和清理请求
|
||
// 支持直接上传 CSV 文件，或上传 ZIP 压缩包（支持密码保护）
|
||
// ZIP 包内可以是 CSV 或 XLSX 格式的账单文件
|
||
func Upload(c *gin.Context) {
|
||
// 1. 获取上传的文件
|
||
file, header, err := c.Request.FormFile("file")
|
||
if err != nil {
|
||
c.JSON(http.StatusBadRequest, model.UploadResponse{
|
||
Result: false,
|
||
Message: "请上传账单文件 (参数名: file)",
|
||
})
|
||
return
|
||
}
|
||
defer file.Close()
|
||
|
||
// 2. 解析请求参数
|
||
var req model.UploadRequest
|
||
c.ShouldBind(&req)
|
||
if req.Format == "" {
|
||
req.Format = "csv"
|
||
}
|
||
|
||
// 3. 保存上传的文件
|
||
timestamp := time.Now().Format("20060102_150405")
|
||
uniqueID := generateShortID()
|
||
|
||
ext := filepath.Ext(header.Filename)
|
||
baseName := header.Filename[:len(header.Filename)-len(ext)]
|
||
inputFileName := fmt.Sprintf("%s_%s_%s%s", timestamp, uniqueID, baseName, ext)
|
||
uploadDirAbs := config.ResolvePath(config.Global.UploadDir)
|
||
inputPath := filepath.Join(uploadDirAbs, inputFileName)
|
||
|
||
dst, err := os.Create(inputPath)
|
||
if err != nil {
|
||
c.JSON(http.StatusInternalServerError, model.UploadResponse{
|
||
Result: false,
|
||
Message: "保存文件失败: " + err.Error(),
|
||
})
|
||
return
|
||
}
|
||
defer dst.Close()
|
||
if _, err := io.Copy(dst, file); err != nil {
|
||
c.JSON(http.StatusInternalServerError, model.UploadResponse{
|
||
Result: false,
|
||
Message: "保存文件失败: " + err.Error(),
|
||
})
|
||
return
|
||
}
|
||
dst.Close() // 关闭文件以便后续处理
|
||
|
||
// 4. 处理文件：如果是 ZIP 则解压，否则直接处理
|
||
var billFilePath string
|
||
var billType string
|
||
var extractedFiles []string
|
||
var needConvert bool // 是否需要格式转换（xlsx -> csv）
|
||
|
||
if service.IsSupportedArchive(header.Filename) {
|
||
// 解压 ZIP 文件
|
||
fmt.Printf("📦 检测到 ZIP 文件，开始解压...\n")
|
||
extractResult, err := service.ExtractZip(inputPath, uploadDirAbs, req.Password)
|
||
if err != nil {
|
||
c.JSON(http.StatusBadRequest, model.UploadResponse{
|
||
Result: false,
|
||
Message: "解压失败: " + err.Error(),
|
||
})
|
||
return
|
||
}
|
||
|
||
billFilePath = extractResult.BillFile
|
||
extractedFiles = extractResult.ExtractedFiles
|
||
|
||
// 使用从文件名检测到的账单类型（如果用户未指定）
|
||
if req.Type == "" && extractResult.BillType != "" {
|
||
billType = extractResult.BillType
|
||
}
|
||
|
||
fmt.Printf(" 解压完成，账单文件: %s\n", filepath.Base(billFilePath))
|
||
|
||
// ZIP 中提取的文件需要格式转换（xlsx 需要转 csv，csv 可能需要编码转换）
|
||
needConvert = true
|
||
} else {
|
||
// 直接使用上传的文件
|
||
billFilePath = inputPath
|
||
|
||
// 检查是否为 xlsx 格式
|
||
if strings.HasSuffix(strings.ToLower(header.Filename), ".xlsx") {
|
||
needConvert = true
|
||
}
|
||
}
|
||
|
||
// 5. 如果需要格式/编码转换，调用 analyzer 服务
|
||
if needConvert {
|
||
fmt.Printf("📊 调用分析服务进行格式/编码转换...\n")
|
||
convertedPath, detectedType, err := service.ConvertBillFile(billFilePath)
|
||
if err != nil {
|
||
// 清理临时文件
|
||
service.CleanupExtractedFiles(extractedFiles)
|
||
c.JSON(http.StatusBadRequest, model.UploadResponse{
|
||
Result: false,
|
||
Message: "文件转换失败: " + err.Error(),
|
||
})
|
||
return
|
||
}
|
||
// 如果转换后的路径与原路径不同，删除原始文件
|
||
if convertedPath != billFilePath {
|
||
os.Remove(billFilePath)
|
||
}
|
||
billFilePath = convertedPath
|
||
|
||
// 使用检测到的账单类型
|
||
if req.Type == "" && detectedType != "" {
|
||
billType = detectedType
|
||
}
|
||
fmt.Printf(" 转换完成: %s\n", filepath.Base(convertedPath))
|
||
}
|
||
|
||
// 6. 确定账单类型
|
||
if req.Type != "" {
|
||
billType = req.Type
|
||
}
|
||
if billType == "" {
|
||
// 尝试从文件名检测
|
||
fileName := strings.ToLower(filepath.Base(billFilePath))
|
||
if strings.Contains(fileName, "支付宝") || strings.Contains(fileName, "alipay") {
|
||
billType = "alipay"
|
||
} else if strings.Contains(fileName, "微信") || strings.Contains(fileName, "wechat") {
|
||
billType = "wechat"
|
||
} else if strings.Contains(fileName, "京东") || strings.Contains(fileName, "jd") {
|
||
billType = "jd"
|
||
}
|
||
}
|
||
if billType == "" {
|
||
// 清理临时文件
|
||
service.CleanupExtractedFiles(extractedFiles)
|
||
c.JSON(http.StatusBadRequest, model.UploadResponse{
|
||
Result: false,
|
||
Message: "无法识别账单类型，请指定 type 参数 (alipay/wechat/jd)",
|
||
})
|
||
return
|
||
}
|
||
if billType != "alipay" && billType != "wechat" && billType != "jd" {
|
||
service.CleanupExtractedFiles(extractedFiles)
|
||
c.JSON(http.StatusBadRequest, model.UploadResponse{
|
||
Result: false,
|
||
Message: "账单类型无效，仅支持 alipay/wechat/jd",
|
||
})
|
||
return
|
||
}
|
||
|
||
// 7. 对原始数据进行去重检查
|
||
fmt.Printf("📋 开始去重检查...\n")
|
||
dedupResult, dedupErr := service.DeduplicateRawFile(billFilePath, timestamp)
|
||
if dedupErr != nil {
|
||
service.CleanupExtractedFiles(extractedFiles)
|
||
c.JSON(http.StatusInternalServerError, model.UploadResponse{
|
||
Result: false,
|
||
Message: "去重检查失败: " + dedupErr.Error(),
|
||
})
|
||
return
|
||
}
|
||
|
||
fmt.Printf(" 原始记录: %d 条\n", dedupResult.OriginalCount)
|
||
if dedupResult.DuplicateCount > 0 {
|
||
fmt.Printf(" 重复记录: %d 条（已跳过）\n", dedupResult.DuplicateCount)
|
||
}
|
||
fmt.Printf(" 新增记录: %d 条\n", dedupResult.NewCount)
|
||
|
||
// 如果全部重复，返回提示
|
||
if dedupResult.NewCount == 0 {
|
||
service.CleanupExtractedFiles(extractedFiles)
|
||
c.JSON(http.StatusOK, model.UploadResponse{
|
||
Result: true,
|
||
Message: fmt.Sprintf("文件中的 %d 条记录全部已存在，无需重复导入", dedupResult.OriginalCount),
|
||
Data: &model.UploadData{
|
||
BillType: billType,
|
||
RawCount: 0,
|
||
CleanedCount: 0,
|
||
DuplicateCount: dedupResult.DuplicateCount,
|
||
},
|
||
})
|
||
return
|
||
}
|
||
|
||
// 使用去重后的文件路径进行后续处理
|
||
processFilePath := dedupResult.DedupFilePath
|
||
|
||
// 8. 构建输出文件路径
|
||
outputExt := ".csv"
|
||
if req.Format == "json" {
|
||
outputExt = ".json"
|
||
}
|
||
outputDirAbs := config.ResolvePath(config.Global.OutputDir)
|
||
fileSeq := generateFileSequence(outputDirAbs, timestamp, billType, outputExt)
|
||
outputFileName := fmt.Sprintf("%s_%s_%s%s", timestamp, billType, fileSeq, outputExt)
|
||
outputPath := filepath.Join(outputDirAbs, outputFileName)
|
||
|
||
// 9. 执行 Python 清洗脚本
|
||
cleanOpts := &service.CleanOptions{
|
||
Year: req.Year,
|
||
Month: req.Month,
|
||
Start: req.Start,
|
||
End: req.End,
|
||
Format: req.Format,
|
||
}
|
||
_, cleanErr := service.RunCleanScript(processFilePath, outputPath, cleanOpts)
|
||
if cleanErr != nil {
|
||
service.CleanupExtractedFiles(extractedFiles)
|
||
c.JSON(http.StatusInternalServerError, model.UploadResponse{
|
||
Result: false,
|
||
Message: cleanErr.Error(),
|
||
})
|
||
return
|
||
}
|
||
|
||
// 10. 将去重后的原始数据存入 MongoDB
|
||
rawCount, rawErr := service.SaveRawBillsFromFile(processFilePath, billType, header.Filename, timestamp)
|
||
if rawErr != nil {
|
||
fmt.Printf("⚠️ 存储原始数据到 MongoDB 失败: %v\n", rawErr)
|
||
} else {
|
||
fmt.Printf("✅ 已存储 %d 条原始账单记录到 MongoDB\n", rawCount)
|
||
}
|
||
|
||
// 11. 将清洗后的数据存入 MongoDB
|
||
cleanedCount, _, cleanedErr := service.SaveCleanedBillsFromFile(outputPath, req.Format, billType, header.Filename, timestamp)
|
||
if cleanedErr != nil {
|
||
fmt.Printf("⚠️ 存储清洗后数据到 MongoDB 失败: %v\n", cleanedErr)
|
||
} else {
|
||
fmt.Printf("✅ 已存储 %d 条清洗后账单记录到 MongoDB\n", cleanedCount)
|
||
}
|
||
|
||
// 12. 清理临时文件
|
||
if dedupResult.DedupFilePath != inputPath && dedupResult.DedupFilePath != "" {
|
||
os.Remove(dedupResult.DedupFilePath)
|
||
}
|
||
service.CleanupExtractedFiles(extractedFiles)
|
||
|
||
// 13. 返回成功响应
|
||
message := fmt.Sprintf("处理成功，新增 %d 条记录", cleanedCount)
|
||
if dedupResult.DuplicateCount > 0 {
|
||
message = fmt.Sprintf("处理成功，新增 %d 条，跳过 %d 条重复记录", cleanedCount, dedupResult.DuplicateCount)
|
||
}
|
||
|
||
c.JSON(http.StatusOK, model.UploadResponse{
|
||
Result: true,
|
||
Message: message,
|
||
Data: &model.UploadData{
|
||
BillType: billType,
|
||
FileURL: fmt.Sprintf("/download/%s", outputFileName),
|
||
FileName: outputFileName,
|
||
RawCount: rawCount,
|
||
CleanedCount: cleanedCount,
|
||
DuplicateCount: dedupResult.DuplicateCount,
|
||
},
|
||
})
|
||
}
|
||
|
||
// generateFileSequence 生成文件序号
|
||
func generateFileSequence(dir, timestamp, billType, ext string) string {
|
||
pattern := fmt.Sprintf("%s_%s_*%s", timestamp, billType, ext)
|
||
matches, err := filepath.Glob(filepath.Join(dir, pattern))
|
||
if err != nil || len(matches) == 0 {
|
||
return "001"
|
||
}
|
||
return fmt.Sprintf("%03d", len(matches)+1)
|
||
}
|
||
|
||
// generateShortID 生成 6 位随机唯一标识符
|
||
func generateShortID() string {
|
||
bytes := make([]byte, 3)
|
||
if _, err := rand.Read(bytes); err != nil {
|
||
return fmt.Sprintf("%06x", time.Now().UnixNano()%0xFFFFFF)
|
||
}
|
||
return hex.EncodeToString(bytes)
|
||
}
|