feat: server connect mongo

server/handler/upload.go

@@ -5,7 +5,6 @@ import (
 	"io"
 	"net/http"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"strings"
 	"time"
@@ -14,6 +13,7 @@ import (
 
 	"billai-server/config"
 	"billai-server/model"
+	"billai-server/service"
 )
 
 // Upload 处理账单上传和清理请求
@@ -53,7 +53,45 @@ func Upload(c *gin.Context) {
 	defer dst.Close()
 	io.Copy(dst, file)
 
-	// 4. 构建输出文件路径
+	// 4. 对原始数据进行去重检查
+	fmt.Printf("📋 开始去重检查...\n")
+	dedupResult, dedupErr := service.DeduplicateRawFile(inputPath, timestamp)
+	if dedupErr != nil {
+		c.JSON(http.StatusInternalServerError, model.UploadResponse{
+			Result:  false,
+			Message: "去重检查失败: " + dedupErr.Error(),
+		})
+		return
+	}
+
+	// 账单类型从去重结果获取
+	billType := dedupResult.BillType
+
+	fmt.Printf("   原始记录: %d 条\n", dedupResult.OriginalCount)
+	if dedupResult.DuplicateCount > 0 {
+		fmt.Printf("   重复记录: %d 条（已跳过）\n", dedupResult.DuplicateCount)
+	}
+	fmt.Printf("   新增记录: %d 条\n", dedupResult.NewCount)
+
+	// 如果全部重复，返回提示
+	if dedupResult.NewCount == 0 {
+		c.JSON(http.StatusOK, model.UploadResponse{
+			Result:  true,
+			Message: fmt.Sprintf("文件中的 %d 条记录全部已存在，无需重复导入", dedupResult.OriginalCount),
+			Data: &model.UploadData{
+				BillType:       billType,
+				RawCount:       0,
+				CleanedCount:   0,
+				DuplicateCount: dedupResult.DuplicateCount,
+			},
+		})
+		return
+	}
+
+	// 使用去重后的文件路径进行后续处理
+	processFilePath := dedupResult.DedupFilePath
+
+	// 5. 构建输出文件路径
 	baseName := strings.TrimSuffix(header.Filename, filepath.Ext(header.Filename))
 	outputExt := ".csv"
 	if req.Format == "json" {
@@ -63,57 +101,65 @@ func Upload(c *gin.Context) {
 	outputDirAbs := config.ResolvePath(config.Global.OutputDir)
 	outputPath := filepath.Join(outputDirAbs, outputFileName)
 
-	// 5. 构建命令参数
-	cleanScriptAbs := config.ResolvePath(config.Global.CleanScript)
-	args := []string{cleanScriptAbs, inputPath, outputPath}
-	if req.Year != "" {
-		args = append(args, "--year", req.Year)
+	// 6. 执行 Python 清洗脚本
+	cleanOpts := &service.CleanOptions{
+		Year:   req.Year,
+		Month:  req.Month,
+		Start:  req.Start,
+		End:    req.End,
+		Format: req.Format,
 	}
-	if req.Month != "" {
-		args = append(args, "--month", req.Month)
-	}
-	if req.Start != "" {
-		args = append(args, "--start", req.Start)
-	}
-	if req.End != "" {
-		args = append(args, "--end", req.End)
-	}
-	if req.Format != "" {
-		args = append(args, "--format", req.Format)
-	}
-
-	// 6. 执行 Python 脚本
-	pythonPathAbs := config.ResolvePath(config.Global.PythonPath)
-	cmd := exec.Command(pythonPathAbs, args...)
-	cmd.Dir = config.Global.ProjectRoot
-	output, err := cmd.CombinedOutput()
-	outputStr := string(output)
-
-	if err != nil {
+	cleanResult, cleanErr := service.RunCleanScript(processFilePath, outputPath, cleanOpts)
+	if cleanErr != nil {
 		c.JSON(http.StatusInternalServerError, model.UploadResponse{
 			Result:  false,
-			Message: "处理失败: " + err.Error(),
+			Message: cleanErr.Error(),
 		})
 		return
 	}
 
-	// 7. 检测账单类型
-	billType := ""
-	if strings.Contains(outputStr, "支付宝") {
-		billType = "alipay"
-	} else if strings.Contains(outputStr, "微信") {
-		billType = "wechat"
+	// 7. 如果去重检测没有识别出类型，从 Python 输出中检测
+	if billType == "" {
+		billType = cleanResult.BillType
+	}
+
+	// 8. 将去重后的原始数据存入 MongoDB（原始数据集合）
+	rawCount, rawErr := service.SaveRawBillsFromFile(processFilePath, billType, header.Filename, timestamp)
+	if rawErr != nil {
+		fmt.Printf("⚠️  存储原始数据到 MongoDB 失败: %v\n", rawErr)
+	} else {
+		fmt.Printf("✅ 已存储 %d 条原始账单记录到 MongoDB\n", rawCount)
+	}
+
+	// 9. 将清洗后的数据存入 MongoDB（清洗后数据集合）
+	cleanedCount, _, cleanedErr := service.SaveCleanedBillsFromFile(outputPath, req.Format, billType, header.Filename, timestamp)
+	if cleanedErr != nil {
+		fmt.Printf("⚠️  存储清洗后数据到 MongoDB 失败: %v\n", cleanedErr)
+	} else {
+		fmt.Printf("✅ 已存储 %d 条清洗后账单记录到 MongoDB\n", cleanedCount)
+	}
+
+	// 10. 清理临时的去重文件（如果生成了的话）
+	if dedupResult.DedupFilePath != inputPath && dedupResult.DedupFilePath != "" {
+		os.Remove(dedupResult.DedupFilePath)
+	}
+
+	// 11. 返回成功响应
+	message := fmt.Sprintf("处理成功，新增 %d 条记录", cleanedCount)
+	if dedupResult.DuplicateCount > 0 {
+		message = fmt.Sprintf("处理成功，新增 %d 条，跳过 %d 条重复记录", cleanedCount, dedupResult.DuplicateCount)
 	}
 
-	// 8. 返回成功响应
 	c.JSON(http.StatusOK, model.UploadResponse{
 		Result:  true,
-		Message: "处理成功",
+		Message: message,
 		Data: &model.UploadData{
-			BillType: billType,
-			FileURL:  fmt.Sprintf("/download/%s", outputFileName),
-			FileName: outputFileName,
+			BillType:       billType,
+			FileURL:        fmt.Sprintf("/download/%s", outputFileName),
+			FileName:       outputFileName,
+			RawCount:       rawCount,
+			CleanedCount:   cleanedCount,
+			DuplicateCount: dedupResult.DuplicateCount,
 		},
 	})
 }
-