Compare commits
49 Commits
7be0d705fe
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
18597ba7fa | ||
| 2fafcb9bfd | |||
|
|
b5d177910c | ||
|
|
7df6587fd6 | ||
|
|
94988790f8 | ||
|
|
45dcef5702 | ||
|
|
ed7232e5c0 | ||
|
|
8852ee5a3a | ||
|
|
a7b73b0928 | ||
|
|
e35f3ed684 | ||
|
|
aed09d4341 | ||
|
|
a0cf063ff9 | ||
|
|
323b712c18 | ||
| 6786d174a6 | |||
|
|
de6b5d3960 | ||
|
|
81c2767423 | ||
|
|
a59fbd0edd | ||
|
|
d1a56a2ab3 | ||
|
|
41df42dea4 | ||
|
|
be3e82fc2e | ||
|
|
9e01ee79f1 | ||
|
|
52c9e48a0f | ||
|
|
9b7657cd73 | ||
|
|
a04eedc423 | ||
|
|
a5f1ad153e | ||
|
|
db3beeddb9 | ||
| eabfd83fdf | |||
| 97c3617731 | |||
| ece026bea2 | |||
| b9124451d2 | |||
| 2e158d3fee | |||
| be1047618e | |||
| 3293f1f8a5 | |||
| ff6795b469 | |||
| cb461f0134 | |||
| 7c4dfaba54 | |||
| 5ee1cea0d7 | |||
| a538bd6680 | |||
| cd221719cf | |||
| d0c0d2cbc3 | |||
| 930d782f18 | |||
| bdd21c4b0f | |||
| 0aaafdbaa3 | |||
| 68a1755a83 | |||
| bb7403f700 | |||
| 3a86f811d0 | |||
| 28295f825b | |||
| e0904f5bfb | |||
| 073808eb30 |
1
.gitignore
vendored
1
.gitignore
vendored
@@ -8,3 +8,4 @@ texpixel
|
|||||||
/vendor
|
/vendor
|
||||||
|
|
||||||
dev_deploy.sh
|
dev_deploy.sh
|
||||||
|
speed_take.sh
|
||||||
@@ -1,11 +1,12 @@
|
|||||||
package api
|
package api
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"gitea.com/bitwsd/document_ai/api/v1/formula"
|
"gitea.com/texpixel/document_ai/api/v1/analytics"
|
||||||
"gitea.com/bitwsd/document_ai/api/v1/oss"
|
"gitea.com/texpixel/document_ai/api/v1/formula"
|
||||||
"gitea.com/bitwsd/document_ai/api/v1/task"
|
"gitea.com/texpixel/document_ai/api/v1/oss"
|
||||||
"gitea.com/bitwsd/document_ai/api/v1/user"
|
"gitea.com/texpixel/document_ai/api/v1/task"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/common"
|
"gitea.com/texpixel/document_ai/api/v1/user"
|
||||||
|
"gitea.com/texpixel/document_ai/pkg/common"
|
||||||
"github.com/gin-gonic/gin"
|
"github.com/gin-gonic/gin"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -26,6 +27,7 @@ func SetupRouter(engine *gin.RouterGroup) {
|
|||||||
endpoint := task.NewTaskEndpoint()
|
endpoint := task.NewTaskEndpoint()
|
||||||
taskRouter.POST("/evaluate", endpoint.EvaluateTask)
|
taskRouter.POST("/evaluate", endpoint.EvaluateTask)
|
||||||
taskRouter.GET("/list", common.MustAuthMiddleware(), endpoint.GetTaskList)
|
taskRouter.GET("/list", common.MustAuthMiddleware(), endpoint.GetTaskList)
|
||||||
|
taskRouter.POST("/export", endpoint.ExportTask)
|
||||||
}
|
}
|
||||||
|
|
||||||
ossRouter := v1.Group("/oss", common.GetAuthMiddleware())
|
ossRouter := v1.Group("/oss", common.GetAuthMiddleware())
|
||||||
@@ -36,15 +38,27 @@ func SetupRouter(engine *gin.RouterGroup) {
|
|||||||
ossRouter.POST("/file/upload", endpoint.UploadFile)
|
ossRouter.POST("/file/upload", endpoint.UploadFile)
|
||||||
}
|
}
|
||||||
|
|
||||||
userRouter := v1.Group("/user", common.GetAuthMiddleware())
|
|
||||||
{
|
|
||||||
userEndpoint := user.NewUserEndpoint()
|
userEndpoint := user.NewUserEndpoint()
|
||||||
|
|
||||||
|
userRouter := v1.Group("/user")
|
||||||
{
|
{
|
||||||
userRouter.POST("/sms", userEndpoint.SendVerificationCode)
|
userRouter.POST("/sms", userEndpoint.SendVerificationCode)
|
||||||
userRouter.POST("/register", userEndpoint.RegisterByEmail)
|
userRouter.POST("/register", userEndpoint.RegisterByEmail)
|
||||||
userRouter.POST("/login", userEndpoint.LoginByEmail)
|
userRouter.POST("/login", userEndpoint.LoginByEmail)
|
||||||
userRouter.GET("/info", common.MustAuthMiddleware(), userEndpoint.GetUserInfo)
|
userRouter.GET("/oauth/google/url", userEndpoint.GetGoogleOAuthUrl)
|
||||||
|
userRouter.POST("/oauth/google/callback", userEndpoint.GoogleOAuthCallback)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
userAuthRouter := v1.Group("/user", common.GetAuthMiddleware())
|
||||||
|
{
|
||||||
|
userAuthRouter.GET("/info", common.MustAuthMiddleware(), userEndpoint.GetUserInfo)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 数据埋点路由
|
||||||
|
analyticsRouter := v1.Group("/analytics", common.GetAuthMiddleware())
|
||||||
|
{
|
||||||
|
analyticsHandler := analytics.NewAnalyticsHandler()
|
||||||
|
analyticsRouter.POST("/track", analyticsHandler.TrackEvent)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
50
api/v1/analytics/handler.go
Normal file
50
api/v1/analytics/handler.go
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
package analytics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
|
||||||
|
"gitea.com/texpixel/document_ai/internal/model/analytics"
|
||||||
|
"gitea.com/texpixel/document_ai/internal/service"
|
||||||
|
"gitea.com/texpixel/document_ai/pkg/common"
|
||||||
|
"gitea.com/texpixel/document_ai/pkg/log"
|
||||||
|
"github.com/gin-gonic/gin"
|
||||||
|
)
|
||||||
|
|
||||||
|
type AnalyticsHandler struct {
|
||||||
|
analyticsService *service.AnalyticsService
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewAnalyticsHandler() *AnalyticsHandler {
|
||||||
|
return &AnalyticsHandler{
|
||||||
|
analyticsService: service.NewAnalyticsService(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TrackEvent 记录单个事件
|
||||||
|
// @Summary 记录单个埋点事件
|
||||||
|
// @Description 记录用户行为埋点事件
|
||||||
|
// @Tags Analytics
|
||||||
|
// @Accept json
|
||||||
|
// @Produce json
|
||||||
|
// @Param request body analytics.TrackEventRequest true "事件信息"
|
||||||
|
// @Success 200 {object} common.Response
|
||||||
|
// @Router /api/v1/analytics/track [post]
|
||||||
|
func (h *AnalyticsHandler) TrackEvent(c *gin.Context) {
|
||||||
|
var req analytics.TrackEventRequest
|
||||||
|
if err := c.ShouldBindJSON(&req); err != nil {
|
||||||
|
log.Error(c.Request.Context(), "bind request failed", "error", err)
|
||||||
|
c.JSON(http.StatusOK, common.ErrorResponse(c, common.CodeParamError, "invalid request"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
userID := common.GetUserIDFromContext(c)
|
||||||
|
req.UserID = userID
|
||||||
|
|
||||||
|
if err := h.analyticsService.TrackEvent(c.Request.Context(), &req); err != nil {
|
||||||
|
log.Error(c.Request.Context(), "track event failed", "error", err)
|
||||||
|
c.JSON(http.StatusOK, common.ErrorResponse(c, common.CodeSystemError, "failed to track event"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
c.JSON(http.StatusOK, common.SuccessResponse(c, "success"))
|
||||||
|
}
|
||||||
@@ -5,12 +5,12 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.com/bitwsd/document_ai/internal/model/formula"
|
"gitea.com/texpixel/document_ai/internal/model/formula"
|
||||||
"gitea.com/bitwsd/document_ai/internal/service"
|
"gitea.com/texpixel/document_ai/internal/service"
|
||||||
"gitea.com/bitwsd/document_ai/internal/storage/dao"
|
"gitea.com/texpixel/document_ai/internal/storage/dao"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/common"
|
"gitea.com/texpixel/document_ai/pkg/common"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/constant"
|
"gitea.com/texpixel/document_ai/pkg/constant"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/utils"
|
"gitea.com/texpixel/document_ai/pkg/utils"
|
||||||
|
|
||||||
"github.com/gin-gonic/gin"
|
"github.com/gin-gonic/gin"
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -8,11 +8,11 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.com/bitwsd/document_ai/config"
|
"gitea.com/texpixel/document_ai/config"
|
||||||
"gitea.com/bitwsd/document_ai/internal/storage/dao"
|
"gitea.com/texpixel/document_ai/internal/storage/dao"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/common"
|
"gitea.com/texpixel/document_ai/pkg/common"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/oss"
|
"gitea.com/texpixel/document_ai/pkg/oss"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/utils"
|
"gitea.com/texpixel/document_ai/pkg/utils"
|
||||||
"github.com/gin-gonic/gin"
|
"github.com/gin-gonic/gin"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -3,10 +3,10 @@ package task
|
|||||||
import (
|
import (
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"gitea.com/bitwsd/document_ai/internal/model/task"
|
"gitea.com/texpixel/document_ai/internal/model/task"
|
||||||
"gitea.com/bitwsd/document_ai/internal/service"
|
"gitea.com/texpixel/document_ai/internal/service"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/common"
|
"gitea.com/texpixel/document_ai/pkg/common"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/log"
|
"gitea.com/texpixel/document_ai/pkg/log"
|
||||||
"github.com/gin-gonic/gin"
|
"github.com/gin-gonic/gin"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -61,3 +61,31 @@ func (h *TaskEndpoint) GetTaskList(c *gin.Context) {
|
|||||||
|
|
||||||
c.JSON(http.StatusOK, common.SuccessResponse(c, resp))
|
c.JSON(http.StatusOK, common.SuccessResponse(c, resp))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (h *TaskEndpoint) ExportTask(c *gin.Context) {
|
||||||
|
var req task.ExportTaskRequest
|
||||||
|
if err := c.ShouldBindJSON(&req); err != nil {
|
||||||
|
log.Error(c, "func", "ExportTask", "msg", "Invalid parameters", "error", err)
|
||||||
|
c.JSON(http.StatusOK, common.ErrorResponse(c, common.CodeParamError, "Invalid parameters"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fileData, contentType, err := h.taskService.ExportTask(c, &req)
|
||||||
|
if err != nil {
|
||||||
|
c.JSON(http.StatusOK, common.ErrorResponse(c, common.CodeSystemError, "导出任务失败"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// set filename based on export type
|
||||||
|
var filename string
|
||||||
|
switch req.Type {
|
||||||
|
case "pdf":
|
||||||
|
filename = "texpixel_export.pdf"
|
||||||
|
case "docx":
|
||||||
|
filename = "texpixel_export.docx"
|
||||||
|
default:
|
||||||
|
filename = "texpixel_export"
|
||||||
|
}
|
||||||
|
|
||||||
|
c.Header("Content-Disposition", "attachment; filename="+filename)
|
||||||
|
c.Data(http.StatusOK, contentType, fileData)
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,15 +1,17 @@
|
|||||||
package user
|
package user
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
|
||||||
"gitea.com/bitwsd/document_ai/config"
|
"gitea.com/texpixel/document_ai/config"
|
||||||
model "gitea.com/bitwsd/document_ai/internal/model/user"
|
model "gitea.com/texpixel/document_ai/internal/model/user"
|
||||||
"gitea.com/bitwsd/document_ai/internal/service"
|
"gitea.com/texpixel/document_ai/internal/service"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/common"
|
"gitea.com/texpixel/document_ai/pkg/common"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/constant"
|
"gitea.com/texpixel/document_ai/pkg/constant"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/jwt"
|
"gitea.com/texpixel/document_ai/pkg/jwt"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/log"
|
"gitea.com/texpixel/document_ai/pkg/log"
|
||||||
"github.com/gin-gonic/gin"
|
"github.com/gin-gonic/gin"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -98,15 +100,9 @@ func (h *UserEndpoint) GetUserInfo(ctx *gin.Context) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
status := 0
|
|
||||||
if user.ID > 0 {
|
|
||||||
status = 1
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx.JSON(http.StatusOK, common.SuccessResponse(ctx, model.UserInfoResponse{
|
ctx.JSON(http.StatusOK, common.SuccessResponse(ctx, model.UserInfoResponse{
|
||||||
Username: user.Username,
|
Username: user.Username,
|
||||||
Phone: user.Phone,
|
Email: user.Email,
|
||||||
Status: status,
|
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -169,3 +165,69 @@ func (h *UserEndpoint) LoginByEmail(ctx *gin.Context) {
|
|||||||
ExpiresAt: tokenResult.ExpiresAt,
|
ExpiresAt: tokenResult.ExpiresAt,
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (h *UserEndpoint) GetGoogleOAuthUrl(ctx *gin.Context) {
|
||||||
|
req := model.GoogleAuthUrlRequest{}
|
||||||
|
if err := ctx.ShouldBindQuery(&req); err != nil {
|
||||||
|
ctx.JSON(http.StatusOK, common.ErrorResponse(ctx, common.CodeParamError, common.CodeParamErrorMsg))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
googleConfig := config.GlobalConfig.Google
|
||||||
|
if googleConfig.ClientID == "" {
|
||||||
|
log.Error(ctx, "func", "GetGoogleOAuthUrl", "msg", "Google OAuth not configured")
|
||||||
|
ctx.JSON(http.StatusOK, common.ErrorResponse(ctx, common.CodeSystemError, common.CodeSystemErrorMsg))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
authURL := fmt.Sprintf(
|
||||||
|
"https://accounts.google.com/o/oauth2/v2/auth?client_id=%s&redirect_uri=%s&response_type=code&scope=openid%%20email%%20profile&state=%s",
|
||||||
|
url.QueryEscape(googleConfig.ClientID),
|
||||||
|
url.QueryEscape(req.RedirectURI),
|
||||||
|
url.QueryEscape(req.State),
|
||||||
|
)
|
||||||
|
|
||||||
|
ctx.JSON(http.StatusOK, common.SuccessResponse(ctx, model.GoogleAuthUrlResponse{
|
||||||
|
AuthURL: authURL,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *UserEndpoint) GoogleOAuthCallback(ctx *gin.Context) {
|
||||||
|
req := model.GoogleOAuthCallbackRequest{}
|
||||||
|
if err := ctx.ShouldBindJSON(&req); err != nil {
|
||||||
|
ctx.JSON(http.StatusOK, common.ErrorResponse(ctx, common.CodeParamError, common.CodeParamErrorMsg))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
googleConfig := config.GlobalConfig.Google
|
||||||
|
if googleConfig.ClientID == "" || googleConfig.ClientSecret == "" {
|
||||||
|
log.Error(ctx, "func", "GoogleOAuthCallback", "msg", "Google OAuth not configured")
|
||||||
|
ctx.JSON(http.StatusOK, common.ErrorResponse(ctx, common.CodeSystemError, common.CodeSystemErrorMsg))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
userInfo, err := h.userService.ExchangeGoogleCodeAndGetUserInfo(ctx, googleConfig.ClientID, googleConfig.ClientSecret, req.Code, req.RedirectURI)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "func", "GoogleOAuthCallback", "msg", "exchange code failed", "error", err)
|
||||||
|
ctx.JSON(http.StatusOK, common.ErrorResponse(ctx, common.CodeSystemError, common.CodeSystemErrorMsg))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
uid, err := h.userService.FindOrCreateGoogleUser(ctx, userInfo)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "func", "GoogleOAuthCallback", "msg", "find or create user failed", "error", err)
|
||||||
|
ctx.JSON(http.StatusOK, common.ErrorResponse(ctx, common.CodeSystemError, common.CodeSystemErrorMsg))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
tokenResult, err := jwt.CreateToken(jwt.User{UserId: uid})
|
||||||
|
if err != nil {
|
||||||
|
ctx.JSON(http.StatusOK, common.ErrorResponse(ctx, common.CodeUnauthorized, common.CodeUnauthorizedMsg))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx.JSON(http.StatusOK, common.SuccessResponse(ctx, model.GoogleOAuthCallbackResponse{
|
||||||
|
Token: tokenResult.Token,
|
||||||
|
ExpiresAt: tokenResult.ExpiresAt,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|||||||
73
cmd/migrate/README.md
Normal file
73
cmd/migrate/README.md
Normal file
@@ -0,0 +1,73 @@
|
|||||||
|
# 数据迁移工具
|
||||||
|
|
||||||
|
用于将测试数据库的数据迁移到生产数据库,避免ID冲突,使用事务确保数据一致性。
|
||||||
|
|
||||||
|
## 功能特性
|
||||||
|
|
||||||
|
- ✅ 自动避免ID冲突(使用数据库自增ID)
|
||||||
|
- ✅ 使用事务确保每个任务和结果数据的一致性
|
||||||
|
- ✅ 自动跳过已存在的任务(基于task_uuid)
|
||||||
|
- ✅ 保留原始时间戳
|
||||||
|
- ✅ 处理NULL值
|
||||||
|
- ✅ 详细的日志输出和统计信息
|
||||||
|
|
||||||
|
## 使用方法
|
||||||
|
|
||||||
|
### 基本用法
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 从dev环境迁移到prod环境
|
||||||
|
go run cmd/migrate/main.go -test-env=dev -prod-env=prod
|
||||||
|
|
||||||
|
# 从prod环境迁移到dev环境(测试反向迁移)
|
||||||
|
go run cmd/migrate/main.go -test-env=prod -prod-env=dev
|
||||||
|
```
|
||||||
|
|
||||||
|
### 参数说明
|
||||||
|
|
||||||
|
- `-test-env`: 测试环境配置文件名(dev/prod),默认值:dev
|
||||||
|
- `-prod-env`: 生产环境配置文件名(dev/prod),默认值:prod
|
||||||
|
|
||||||
|
### 编译后使用
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 编译
|
||||||
|
go build -o migrate cmd/migrate/main.go
|
||||||
|
|
||||||
|
# 运行
|
||||||
|
./migrate -test-env=dev -prod-env=prod
|
||||||
|
```
|
||||||
|
|
||||||
|
## 工作原理
|
||||||
|
|
||||||
|
1. **连接数据库**:同时连接测试数据库和生产数据库
|
||||||
|
2. **读取数据**:从测试数据库读取所有任务和结果数据(LEFT JOIN)
|
||||||
|
3. **检查重复**:基于`task_uuid`检查生产数据库中是否已存在
|
||||||
|
4. **事务迁移**:为每个任务创建独立事务:
|
||||||
|
- 创建任务记录(自动生成新ID)
|
||||||
|
- 如果存在结果数据,创建结果记录(关联新任务ID)
|
||||||
|
- 提交事务或回滚
|
||||||
|
5. **统计报告**:输出迁移统计信息
|
||||||
|
|
||||||
|
## 注意事项
|
||||||
|
|
||||||
|
1. **配置文件**:确保`config/config_dev.yaml`和`config/config_prod.yaml`存在且配置正确
|
||||||
|
2. **数据库权限**:确保数据库用户有读写权限
|
||||||
|
3. **网络连接**:确保能同时连接到两个数据库
|
||||||
|
4. **数据备份**:迁移前建议备份生产数据库
|
||||||
|
5. **ID冲突**:脚本会自动处理ID冲突,使用数据库自增ID,不会覆盖现有数据
|
||||||
|
|
||||||
|
## 输出示例
|
||||||
|
|
||||||
|
```
|
||||||
|
从测试数据库读取到 100 条任务记录
|
||||||
|
[1/100] 创建任务成功: task_uuid=xxx, 新ID=1001
|
||||||
|
[1/100] 创建结果成功: task_id=1001
|
||||||
|
[2/100] 跳过已存在的任务: task_uuid=yyy, id=1002
|
||||||
|
...
|
||||||
|
迁移完成统计:
|
||||||
|
成功: 95 条
|
||||||
|
跳过: 3 条
|
||||||
|
失败: 2 条
|
||||||
|
数据迁移完成!
|
||||||
|
```
|
||||||
255
cmd/migrate/main.go
Normal file
255
cmd/migrate/main.go
Normal file
@@ -0,0 +1,255 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.com/texpixel/document_ai/config"
|
||||||
|
"gitea.com/texpixel/document_ai/internal/storage/dao"
|
||||||
|
"github.com/spf13/viper"
|
||||||
|
"gorm.io/driver/mysql"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
"gorm.io/gorm/logger"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
// 解析命令行参数
|
||||||
|
testEnv := flag.String("test-env", "dev", "测试环境配置 (dev/prod)")
|
||||||
|
prodEnv := flag.String("prod-env", "prod", "生产环境配置 (dev/prod)")
|
||||||
|
flag.Parse()
|
||||||
|
|
||||||
|
// 加载测试环境配置
|
||||||
|
testConfigPath := fmt.Sprintf("./config/config_%s.yaml", *testEnv)
|
||||||
|
testConfig, err := loadDatabaseConfig(testConfigPath)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("加载测试环境配置失败: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 连接测试数据库
|
||||||
|
testDSN := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Asia%%2FShanghai",
|
||||||
|
testConfig.Username, testConfig.Password, testConfig.Host, testConfig.Port, testConfig.DBName)
|
||||||
|
testDB, err := gorm.Open(mysql.Open(testDSN), &gorm.Config{
|
||||||
|
Logger: logger.Default.LogMode(logger.Info),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("连接测试数据库失败: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 加载生产环境配置
|
||||||
|
prodConfigPath := fmt.Sprintf("./config/config_%s.yaml", *prodEnv)
|
||||||
|
prodConfig, err := loadDatabaseConfig(prodConfigPath)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("加载生产环境配置失败: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 连接生产数据库
|
||||||
|
prodDSN := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Asia%%2FShanghai",
|
||||||
|
prodConfig.Username, prodConfig.Password, prodConfig.Host, prodConfig.Port, prodConfig.DBName)
|
||||||
|
prodDB, err := gorm.Open(mysql.Open(prodDSN), &gorm.Config{
|
||||||
|
Logger: logger.Default.LogMode(logger.Info),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("连接生产数据库失败: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 执行迁移
|
||||||
|
if err := migrateData(testDB, prodDB); err != nil {
|
||||||
|
log.Fatalf("数据迁移失败: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Println("数据迁移完成!")
|
||||||
|
}
|
||||||
|
|
||||||
|
func migrateData(testDB, prodDB *gorm.DB) error {
|
||||||
|
_ = context.Background() // 保留以备将来使用
|
||||||
|
|
||||||
|
// 从测试数据库读取所有任务数据(包含结果)
|
||||||
|
type TaskWithResult struct {
|
||||||
|
// Task 字段
|
||||||
|
TaskID int64 `gorm:"column:id"`
|
||||||
|
UserID int64 `gorm:"column:user_id"`
|
||||||
|
TaskUUID string `gorm:"column:task_uuid"`
|
||||||
|
FileName string `gorm:"column:file_name"`
|
||||||
|
FileHash string `gorm:"column:file_hash"`
|
||||||
|
FileURL string `gorm:"column:file_url"`
|
||||||
|
TaskType string `gorm:"column:task_type"`
|
||||||
|
Status int `gorm:"column:status"`
|
||||||
|
CompletedAt time.Time `gorm:"column:completed_at"`
|
||||||
|
Remark string `gorm:"column:remark"`
|
||||||
|
IP string `gorm:"column:ip"`
|
||||||
|
TaskCreatedAt time.Time `gorm:"column:created_at"`
|
||||||
|
TaskUpdatedAt time.Time `gorm:"column:updated_at"`
|
||||||
|
// Result 字段
|
||||||
|
ResultID *int64 `gorm:"column:result_id"`
|
||||||
|
ResultTaskID *int64 `gorm:"column:result_task_id"`
|
||||||
|
ResultTaskType *string `gorm:"column:result_task_type"`
|
||||||
|
Latex *string `gorm:"column:latex"`
|
||||||
|
Markdown *string `gorm:"column:markdown"`
|
||||||
|
MathML *string `gorm:"column:mathml"`
|
||||||
|
ResultCreatedAt *time.Time `gorm:"column:result_created_at"`
|
||||||
|
ResultUpdatedAt *time.Time `gorm:"column:result_updated_at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
var tasksWithResults []TaskWithResult
|
||||||
|
query := `
|
||||||
|
SELECT
|
||||||
|
t.id,
|
||||||
|
t.user_id,
|
||||||
|
t.task_uuid,
|
||||||
|
t.file_name,
|
||||||
|
t.file_hash,
|
||||||
|
t.file_url,
|
||||||
|
t.task_type,
|
||||||
|
t.status,
|
||||||
|
t.completed_at,
|
||||||
|
t.remark,
|
||||||
|
t.ip,
|
||||||
|
t.created_at,
|
||||||
|
t.updated_at,
|
||||||
|
r.id as result_id,
|
||||||
|
r.task_id as result_task_id,
|
||||||
|
r.task_type as result_task_type,
|
||||||
|
r.latex,
|
||||||
|
r.markdown,
|
||||||
|
r.mathml,
|
||||||
|
r.created_at as result_created_at,
|
||||||
|
r.updated_at as result_updated_at
|
||||||
|
FROM recognition_tasks t
|
||||||
|
LEFT JOIN recognition_results r ON t.id = r.task_id
|
||||||
|
ORDER BY t.id
|
||||||
|
`
|
||||||
|
|
||||||
|
if err := testDB.Raw(query).Scan(&tasksWithResults).Error; err != nil {
|
||||||
|
return fmt.Errorf("读取测试数据失败: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("从测试数据库读取到 %d 条任务记录", len(tasksWithResults))
|
||||||
|
|
||||||
|
successCount := 0
|
||||||
|
skipCount := 0
|
||||||
|
errorCount := 0
|
||||||
|
|
||||||
|
// 为每个任务使用独立事务,确保单个任务失败不影响其他任务
|
||||||
|
for i, item := range tasksWithResults {
|
||||||
|
// 开始事务
|
||||||
|
tx := prodDB.Begin()
|
||||||
|
|
||||||
|
// 检查生产数据库中是否已存在相同的 task_uuid
|
||||||
|
var existingTask dao.RecognitionTask
|
||||||
|
err := tx.Where("task_uuid = ?", item.TaskUUID).First(&existingTask).Error
|
||||||
|
if err == nil {
|
||||||
|
log.Printf("[%d/%d] 跳过已存在的任务: task_uuid=%s, id=%d", i+1, len(tasksWithResults), item.TaskUUID, existingTask.ID)
|
||||||
|
tx.Rollback()
|
||||||
|
skipCount++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err != gorm.ErrRecordNotFound {
|
||||||
|
log.Printf("[%d/%d] 检查任务是否存在时出错: task_uuid=%s, error=%v", i+1, len(tasksWithResults), item.TaskUUID, err)
|
||||||
|
tx.Rollback()
|
||||||
|
errorCount++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// 创建新任务(不指定ID,让数据库自动生成)
|
||||||
|
newTask := &dao.RecognitionTask{
|
||||||
|
UserID: item.UserID,
|
||||||
|
TaskUUID: item.TaskUUID,
|
||||||
|
FileName: item.FileName,
|
||||||
|
FileHash: item.FileHash,
|
||||||
|
FileURL: item.FileURL,
|
||||||
|
TaskType: dao.TaskType(item.TaskType),
|
||||||
|
Status: dao.TaskStatus(item.Status),
|
||||||
|
CompletedAt: item.CompletedAt,
|
||||||
|
Remark: item.Remark,
|
||||||
|
IP: item.IP,
|
||||||
|
}
|
||||||
|
// 保留原始时间戳
|
||||||
|
newTask.CreatedAt = item.TaskCreatedAt
|
||||||
|
newTask.UpdatedAt = item.TaskUpdatedAt
|
||||||
|
|
||||||
|
if err := tx.Create(newTask).Error; err != nil {
|
||||||
|
log.Printf("[%d/%d] 创建任务失败: task_uuid=%s, error=%v", i+1, len(tasksWithResults), item.TaskUUID, err)
|
||||||
|
tx.Rollback()
|
||||||
|
errorCount++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("[%d/%d] 创建任务成功: task_uuid=%s, 新ID=%d", i+1, len(tasksWithResults), item.TaskUUID, newTask.ID)
|
||||||
|
|
||||||
|
// 如果有结果数据,创建结果记录
|
||||||
|
if item.ResultID != nil {
|
||||||
|
// 处理可能为NULL的字段
|
||||||
|
latex := ""
|
||||||
|
if item.Latex != nil {
|
||||||
|
latex = *item.Latex
|
||||||
|
}
|
||||||
|
markdown := ""
|
||||||
|
if item.Markdown != nil {
|
||||||
|
markdown = *item.Markdown
|
||||||
|
}
|
||||||
|
mathml := ""
|
||||||
|
if item.MathML != nil {
|
||||||
|
mathml = *item.MathML
|
||||||
|
}
|
||||||
|
|
||||||
|
newResult := dao.RecognitionResult{
|
||||||
|
TaskID: newTask.ID, // 使用新任务的ID
|
||||||
|
TaskType: dao.TaskType(item.TaskType),
|
||||||
|
Latex: latex,
|
||||||
|
Markdown: markdown,
|
||||||
|
MathML: mathml,
|
||||||
|
}
|
||||||
|
// 保留原始时间戳
|
||||||
|
if item.ResultCreatedAt != nil {
|
||||||
|
newResult.CreatedAt = *item.ResultCreatedAt
|
||||||
|
}
|
||||||
|
if item.ResultUpdatedAt != nil {
|
||||||
|
newResult.UpdatedAt = *item.ResultUpdatedAt
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := tx.Create(&newResult).Error; err != nil {
|
||||||
|
log.Printf("[%d/%d] 创建结果失败: task_id=%d, error=%v", i+1, len(tasksWithResults), newTask.ID, err)
|
||||||
|
tx.Rollback() // 回滚整个事务(包括任务)
|
||||||
|
errorCount++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("[%d/%d] 创建结果成功: task_id=%d", i+1, len(tasksWithResults), newTask.ID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 提交事务
|
||||||
|
if err := tx.Commit().Error; err != nil {
|
||||||
|
log.Printf("[%d/%d] 提交事务失败: task_uuid=%s, error=%v", i+1, len(tasksWithResults), item.TaskUUID, err)
|
||||||
|
errorCount++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
successCount++
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("迁移完成统计:")
|
||||||
|
log.Printf(" 成功: %d 条", successCount)
|
||||||
|
log.Printf(" 跳过: %d 条", skipCount)
|
||||||
|
log.Printf(" 失败: %d 条", errorCount)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// loadDatabaseConfig 从配置文件加载数据库配置
|
||||||
|
func loadDatabaseConfig(configPath string) (config.DatabaseConfig, error) {
|
||||||
|
v := viper.New()
|
||||||
|
v.SetConfigFile(configPath)
|
||||||
|
if err := v.ReadInConfig(); err != nil {
|
||||||
|
return config.DatabaseConfig{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var dbConfig config.DatabaseConfig
|
||||||
|
if err := v.UnmarshalKey("database", &dbConfig); err != nil {
|
||||||
|
return config.DatabaseConfig{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return dbConfig, nil
|
||||||
|
}
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"gitea.com/bitwsd/document_ai/pkg/log"
|
"gitea.com/texpixel/document_ai/pkg/log"
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -14,6 +14,19 @@ type Config struct {
|
|||||||
Limit LimitConfig `mapstructure:"limit"`
|
Limit LimitConfig `mapstructure:"limit"`
|
||||||
Aliyun AliyunConfig `mapstructure:"aliyun"`
|
Aliyun AliyunConfig `mapstructure:"aliyun"`
|
||||||
Mathpix MathpixConfig `mapstructure:"mathpix"`
|
Mathpix MathpixConfig `mapstructure:"mathpix"`
|
||||||
|
BaiduOCR BaiduOCRConfig `mapstructure:"baidu_ocr"`
|
||||||
|
Google GoogleOAuthConfig `mapstructure:"google"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type BaiduOCRConfig struct {
|
||||||
|
Token string `mapstructure:"token"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type GoogleOAuthConfig struct {
|
||||||
|
ClientID string `mapstructure:"client_id"`
|
||||||
|
ClientSecret string `mapstructure:"client_secret"`
|
||||||
|
RedirectURI string `mapstructure:"redirect_uri"`
|
||||||
|
Proxy string `mapstructure:"proxy"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type MathpixConfig struct {
|
type MathpixConfig struct {
|
||||||
|
|||||||
@@ -4,8 +4,8 @@ server:
|
|||||||
|
|
||||||
database:
|
database:
|
||||||
driver: mysql
|
driver: mysql
|
||||||
host: mysql
|
host: localhost
|
||||||
port: 3306
|
port: 3006
|
||||||
username: root
|
username: root
|
||||||
password: texpixel#pwd123!
|
password: texpixel#pwd123!
|
||||||
dbname: doc_ai
|
dbname: doc_ai
|
||||||
@@ -13,7 +13,7 @@ database:
|
|||||||
max_open: 100
|
max_open: 100
|
||||||
|
|
||||||
redis:
|
redis:
|
||||||
addr: redis:6379
|
addr: localhost:6079
|
||||||
password: yoge@123321!
|
password: yoge@123321!
|
||||||
db: 0
|
db: 0
|
||||||
|
|
||||||
@@ -30,7 +30,6 @@ log:
|
|||||||
maxBackups: 1 # 保留的旧日志文件最大数量
|
maxBackups: 1 # 保留的旧日志文件最大数量
|
||||||
compress: false # 是否压缩旧日志
|
compress: false # 是否压缩旧日志
|
||||||
|
|
||||||
|
|
||||||
aliyun:
|
aliyun:
|
||||||
sms:
|
sms:
|
||||||
access_key_id: "LTAI5tB9ur4ExCF4dYPq7hLz"
|
access_key_id: "LTAI5tB9ur4ExCF4dYPq7hLz"
|
||||||
@@ -43,8 +42,17 @@ aliyun:
|
|||||||
inner_endpoint: oss-cn-beijing-internal.aliyuncs.com
|
inner_endpoint: oss-cn-beijing-internal.aliyuncs.com
|
||||||
access_key_id: LTAI5t8qXhow6NCdYDtu1saF
|
access_key_id: LTAI5t8qXhow6NCdYDtu1saF
|
||||||
access_key_secret: qZ2SwYsNCEBckCVSOszH31yYwXU44A
|
access_key_secret: qZ2SwYsNCEBckCVSOszH31yYwXU44A
|
||||||
bucket_name: texpixel-doc
|
bucket_name: texpixel-doc1
|
||||||
|
|
||||||
mathpix:
|
mathpix:
|
||||||
app_id: "ocr_eede6f_ea9b5c"
|
app_id: "ocr_eede6f_ea9b5c"
|
||||||
app_key: "fb72d251e33ac85c929bfd4eec40d78368d08d82fb2ee1cffb04a8bb967d1db5"
|
app_key: "fb72d251e33ac85c929bfd4eec40d78368d08d82fb2ee1cffb04a8bb967d1db5"
|
||||||
|
|
||||||
|
baidu_ocr:
|
||||||
|
token: "e3a47bd2438f1f38840c203fc5939d17a54482d1"
|
||||||
|
|
||||||
|
google:
|
||||||
|
client_id: "404402221037-nqdsk11bkpk5a7oh396mrg1ieh28u6q1.apps.googleusercontent.com"
|
||||||
|
client_secret: "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"
|
||||||
|
redirect_uri: "https://app.cloud.texpixel.com:10443/auth/google/callback"
|
||||||
|
proxy: "http://localhost:7890"
|
||||||
|
|||||||
@@ -42,9 +42,17 @@ aliyun:
|
|||||||
inner_endpoint: oss-cn-beijing-internal.aliyuncs.com
|
inner_endpoint: oss-cn-beijing-internal.aliyuncs.com
|
||||||
access_key_id: LTAI5t8qXhow6NCdYDtu1saF
|
access_key_id: LTAI5t8qXhow6NCdYDtu1saF
|
||||||
access_key_secret: qZ2SwYsNCEBckCVSOszH31yYwXU44A
|
access_key_secret: qZ2SwYsNCEBckCVSOszH31yYwXU44A
|
||||||
bucket_name: texpixel-doc
|
bucket_name: texpixel-doc1
|
||||||
|
|
||||||
|
|
||||||
mathpix:
|
mathpix:
|
||||||
app_id: "ocr_eede6f_ea9b5c"
|
app_id: "ocr_eede6f_ea9b5c"
|
||||||
app_key: "fb72d251e33ac85c929bfd4eec40d78368d08d82fb2ee1cffb04a8bb967d1db5"
|
app_key: "fb72d251e33ac85c929bfd4eec40d78368d08d82fb2ee1cffb04a8bb967d1db5"
|
||||||
|
|
||||||
|
baidu_ocr:
|
||||||
|
token: "e3a47bd2438f1f38840c203fc5939d17a54482d1"
|
||||||
|
|
||||||
|
google:
|
||||||
|
client_id: "404402221037-nqdsk11bkpk5a7oh396mrg1ieh28u6q1.apps.googleusercontent.com"
|
||||||
|
client_secret: "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"
|
||||||
|
redirect_uri: "https://texpixel.com/auth/google/callback"
|
||||||
|
proxy: "http://100.115.184.74:7890"
|
||||||
|
|||||||
13
deploy_dev.sh
Executable file
13
deploy_dev.sh
Executable file
@@ -0,0 +1,13 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
git push origin test
|
||||||
|
|
||||||
|
ssh ubuntu << 'ENDSSH'
|
||||||
|
cd /home/yoge/Dev/doc_ai_backed
|
||||||
|
git checkout test
|
||||||
|
git pull origin test
|
||||||
|
docker compose -f docker-compose.infra.yml up -d
|
||||||
|
docker compose down
|
||||||
|
docker image rm doc_ai_backed-doc_ai:latest
|
||||||
|
docker compose up -d
|
||||||
|
ENDSSH
|
||||||
10
deploy_prod.sh
Executable file
10
deploy_prod.sh
Executable file
@@ -0,0 +1,10 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
docker build -t crpi-8s2ierii2xan4klg.cn-beijing.personal.cr.aliyuncs.com/texpixel/doc_ai_backend:latest . && docker push crpi-8s2ierii2xan4klg.cn-beijing.personal.cr.aliyuncs.com/texpixel/doc_ai_backend:latest
|
||||||
|
|
||||||
|
ssh ecs << 'ENDSSH'
|
||||||
|
docker stop doc_ai doc_ai_backend 2>/dev/null || true
|
||||||
|
docker rm doc_ai doc_ai_backend 2>/dev/null || true
|
||||||
|
docker pull crpi-8s2ierii2xan4klg.cn-beijing.personal.cr.aliyuncs.com/texpixel/doc_ai_backend:latest
|
||||||
|
docker run -d --name doc_ai -p 8024:8024 --restart unless-stopped crpi-8s2ierii2xan4klg.cn-beijing.personal.cr.aliyuncs.com/texpixel/doc_ai_backend:latest -env=prod
|
||||||
|
ENDSSH
|
||||||
32
docker-compose.infra.yml
Normal file
32
docker-compose.infra.yml
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
services:
|
||||||
|
mysql:
|
||||||
|
image: mysql:8.0
|
||||||
|
container_name: mysql
|
||||||
|
environment:
|
||||||
|
MYSQL_ROOT_PASSWORD: texpixel#pwd123!
|
||||||
|
MYSQL_DATABASE: doc_ai
|
||||||
|
MYSQL_USER: texpixel
|
||||||
|
MYSQL_PASSWORD: texpixel#pwd123!
|
||||||
|
ports:
|
||||||
|
- "3006:3306"
|
||||||
|
volumes:
|
||||||
|
- mysql_data:/var/lib/mysql
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-uroot", "-ptexpixel#pwd123!"]
|
||||||
|
interval: 5s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 10
|
||||||
|
start_period: 30s
|
||||||
|
restart: always
|
||||||
|
|
||||||
|
redis:
|
||||||
|
image: redis:latest
|
||||||
|
container_name: redis
|
||||||
|
command: redis-server --requirepass "yoge@123321!"
|
||||||
|
ports:
|
||||||
|
- "6079:6379"
|
||||||
|
restart: always
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
mysql_data:
|
||||||
|
driver: local
|
||||||
@@ -2,58 +2,9 @@ services:
|
|||||||
doc_ai:
|
doc_ai:
|
||||||
build: .
|
build: .
|
||||||
container_name: doc_ai
|
container_name: doc_ai
|
||||||
ports:
|
network_mode: host
|
||||||
- "8024:8024"
|
|
||||||
volumes:
|
volumes:
|
||||||
- ./config:/app/config
|
- ./config:/app/config
|
||||||
- ./logs:/app/logs
|
- ./logs:/app/logs
|
||||||
networks:
|
|
||||||
- backend
|
|
||||||
depends_on:
|
|
||||||
mysql:
|
|
||||||
condition: service_healthy
|
|
||||||
redis:
|
|
||||||
condition: service_started
|
|
||||||
command: ["-env", "dev"]
|
command: ["-env", "dev"]
|
||||||
restart: always
|
restart: always
|
||||||
|
|
||||||
mysql:
|
|
||||||
image: mysql:8.0
|
|
||||||
container_name: mysql
|
|
||||||
environment:
|
|
||||||
MYSQL_ROOT_PASSWORD: texpixel#pwd123!
|
|
||||||
MYSQL_DATABASE: doc_ai
|
|
||||||
MYSQL_USER: texpixel
|
|
||||||
MYSQL_PASSWORD: texpixel#pwd123!
|
|
||||||
ports:
|
|
||||||
- "3006:3306"
|
|
||||||
volumes:
|
|
||||||
- mysql_data:/var/lib/mysql
|
|
||||||
networks:
|
|
||||||
- backend
|
|
||||||
healthcheck:
|
|
||||||
test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-uroot", "-ptexpixel#pwd123!"]
|
|
||||||
interval: 5s
|
|
||||||
timeout: 5s
|
|
||||||
retries: 10
|
|
||||||
start_period: 30s
|
|
||||||
restart: always
|
|
||||||
|
|
||||||
redis:
|
|
||||||
image: redis:latest
|
|
||||||
container_name: redis
|
|
||||||
command: redis-server --requirepass "yoge@123321!"
|
|
||||||
ports:
|
|
||||||
- "6079:6379"
|
|
||||||
networks:
|
|
||||||
- backend
|
|
||||||
restart: always
|
|
||||||
|
|
||||||
volumes:
|
|
||||||
mysql_data:
|
|
||||||
# 持久化MySQL数据卷
|
|
||||||
driver: local
|
|
||||||
|
|
||||||
networks:
|
|
||||||
backend:
|
|
||||||
driver: bridge
|
|
||||||
|
|||||||
12
go.mod
12
go.mod
@@ -1,4 +1,4 @@
|
|||||||
module gitea.com/bitwsd/document_ai
|
module gitea.com/texpixel/document_ai
|
||||||
|
|
||||||
go 1.20
|
go 1.20
|
||||||
|
|
||||||
@@ -11,18 +11,21 @@ require (
|
|||||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible
|
github.com/dgrijalva/jwt-go v3.2.0+incompatible
|
||||||
github.com/gin-gonic/gin v1.10.0
|
github.com/gin-gonic/gin v1.10.0
|
||||||
github.com/google/uuid v1.6.0
|
github.com/google/uuid v1.6.0
|
||||||
|
github.com/jtolds/gls v4.20.0+incompatible
|
||||||
github.com/redis/go-redis/v9 v9.7.0
|
github.com/redis/go-redis/v9 v9.7.0
|
||||||
github.com/rs/zerolog v1.33.0
|
github.com/rs/zerolog v1.33.0
|
||||||
github.com/spf13/viper v1.19.0
|
github.com/spf13/viper v1.19.0
|
||||||
golang.org/x/crypto v0.23.0
|
golang.org/x/crypto v0.23.0
|
||||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1
|
gopkg.in/natefinch/lumberjack.v2 v2.2.1
|
||||||
|
gorm.io/datatypes v1.2.7
|
||||||
gorm.io/driver/mysql v1.5.7
|
gorm.io/driver/mysql v1.5.7
|
||||||
gorm.io/gorm v1.25.12
|
gorm.io/gorm v1.30.0
|
||||||
)
|
)
|
||||||
|
|
||||||
require github.com/go-sql-driver/mysql v1.7.0 // indirect
|
require github.com/go-sql-driver/mysql v1.8.1 // indirect
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
filippo.io/edwards25519 v1.1.0 // indirect
|
||||||
github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.4 // indirect
|
github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.4 // indirect
|
||||||
github.com/alibabacloud-go/debug v0.0.0-20190504072949-9472017b5c68 // indirect
|
github.com/alibabacloud-go/debug v0.0.0-20190504072949-9472017b5c68 // indirect
|
||||||
github.com/alibabacloud-go/endpoint-util v1.1.0 // indirect
|
github.com/alibabacloud-go/endpoint-util v1.1.0 // indirect
|
||||||
@@ -43,6 +46,7 @@ require (
|
|||||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||||
github.com/go-playground/validator/v10 v10.20.0 // indirect
|
github.com/go-playground/validator/v10 v10.20.0 // indirect
|
||||||
github.com/goccy/go-json v0.10.2 // indirect
|
github.com/goccy/go-json v0.10.2 // indirect
|
||||||
|
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect
|
||||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||||
github.com/jinzhu/now v1.1.5 // indirect
|
github.com/jinzhu/now v1.1.5 // indirect
|
||||||
@@ -72,7 +76,7 @@ require (
|
|||||||
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
|
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
|
||||||
golang.org/x/net v0.25.0 // indirect
|
golang.org/x/net v0.25.0 // indirect
|
||||||
golang.org/x/sys v0.20.0 // indirect
|
golang.org/x/sys v0.20.0 // indirect
|
||||||
golang.org/x/text v0.15.0 // indirect
|
golang.org/x/text v0.20.0 // indirect
|
||||||
golang.org/x/time v0.5.0 // indirect
|
golang.org/x/time v0.5.0 // indirect
|
||||||
google.golang.org/protobuf v1.34.1 // indirect
|
google.golang.org/protobuf v1.34.1 // indirect
|
||||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||||
|
|||||||
29
go.sum
29
go.sum
@@ -1,3 +1,5 @@
|
|||||||
|
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
||||||
|
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||||
github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.4 h1:iC9YFYKDGEy3n/FtqJnOkZsene9olVspKmkX5A2YBEo=
|
github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.4 h1:iC9YFYKDGEy3n/FtqJnOkZsene9olVspKmkX5A2YBEo=
|
||||||
github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.4/go.mod h1:sCavSAvdzOjul4cEqeVtvlSaSScfNsTQ+46HwlTL1hc=
|
github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.4/go.mod h1:sCavSAvdzOjul4cEqeVtvlSaSScfNsTQ+46HwlTL1hc=
|
||||||
github.com/alibabacloud-go/darabonba-openapi v0.1.18/go.mod h1:PB4HffMhJVmAgNKNq3wYbTUlFvPgxJpTzd1F5pTuUsc=
|
github.com/alibabacloud-go/darabonba-openapi v0.1.18/go.mod h1:PB4HffMhJVmAgNKNq3wYbTUlFvPgxJpTzd1F5pTuUsc=
|
||||||
@@ -69,19 +71,27 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
|
|||||||
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
|
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
|
||||||
github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8=
|
github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8=
|
||||||
github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
|
github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
|
||||||
github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
|
|
||||||
github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
|
github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
|
||||||
|
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
|
||||||
|
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
|
||||||
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
|
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
|
||||||
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||||
|
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA=
|
||||||
|
github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A=
|
||||||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||||
|
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0=
|
||||||
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||||
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
|
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
|
||||||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
||||||
|
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||||
|
github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 h1:L0QtFUgDarD7Fpv9jeVMgy/+Ec0mtnmYuImjTz6dtDA=
|
||||||
|
github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw=
|
||||||
|
github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
|
||||||
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||||
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
|
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
|
||||||
@@ -89,6 +99,7 @@ github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/
|
|||||||
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||||
|
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
||||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||||
github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
|
github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
|
||||||
@@ -108,6 +119,8 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/
|
|||||||
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||||
|
github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI=
|
||||||
|
github.com/microsoft/go-mssqldb v1.7.2 h1:CHkFJiObW7ItKTJfHo1QX7QBBD1iV+mn1eOyRP3b/PA=
|
||||||
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
|
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
|
||||||
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||||
@@ -197,6 +210,7 @@ golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
|||||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ=
|
||||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
@@ -209,8 +223,8 @@ golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
|
|||||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||||
golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
|
golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=
|
||||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=
|
||||||
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
|
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
|
||||||
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
|
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
|
||||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
@@ -235,10 +249,15 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
|||||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
|
gorm.io/datatypes v1.2.7 h1:ww9GAhF1aGXZY3EB3cJPJ7//JiuQo7DlQA7NNlVaTdk=
|
||||||
|
gorm.io/datatypes v1.2.7/go.mod h1:M2iO+6S3hhi4nAyYe444Pcb0dcIiOMJ7QHaUXxyiNZY=
|
||||||
gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo=
|
gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo=
|
||||||
gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM=
|
gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM=
|
||||||
|
gorm.io/driver/postgres v1.5.0 h1:u2FXTy14l45qc3UeCJ7QaAXZmZfDDv0YrthvmRq1l0U=
|
||||||
|
gorm.io/driver/sqlite v1.4.3 h1:HBBcZSDnWi5BW3B3rwvVTc510KGkBkexlOg0QrmLUuU=
|
||||||
|
gorm.io/driver/sqlserver v1.6.0 h1:VZOBQVsVhkHU/NzNhRJKoANt5pZGQAS1Bwc6m6dgfnc=
|
||||||
gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
|
gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
|
||||||
gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8=
|
gorm.io/gorm v1.30.0 h1:qbT5aPv1UH8gI99OsRlvDToLxW5zR7FzS9acZDOZcgs=
|
||||||
gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ=
|
gorm.io/gorm v1.30.0/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=
|
||||||
nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
|
nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
|
||||||
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
|
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
|
||||||
|
|||||||
34
internal/model/analytics/request.go
Normal file
34
internal/model/analytics/request.go
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
package analytics
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
|
// TrackEventRequest 埋点事件请求
|
||||||
|
type TrackEventRequest struct {
|
||||||
|
TaskNo string `json:"task_no" binding:"required"`
|
||||||
|
UserID int64 `json:"user_id"`
|
||||||
|
EventName string `json:"event_name" binding:"required"`
|
||||||
|
Properties map[string]interface{} `json:"properties"`
|
||||||
|
DeviceInfo map[string]interface{} `json:"device_info"`
|
||||||
|
MetaData map[string]interface{} `json:"meta_data"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// BatchTrackEventRequest 批量埋点事件请求
|
||||||
|
type BatchTrackEventRequest struct {
|
||||||
|
Events []TrackEventRequest `json:"events" binding:"required,min=1,max=100"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// QueryEventsRequest 查询事件请求
|
||||||
|
type QueryEventsRequest struct {
|
||||||
|
UserID *int64 `json:"user_id" form:"user_id"`
|
||||||
|
EventName string `json:"event_name" form:"event_name"`
|
||||||
|
StartTime *time.Time `json:"start_time" form:"start_time"`
|
||||||
|
EndTime *time.Time `json:"end_time" form:"end_time"`
|
||||||
|
Page int `json:"page" form:"page" binding:"required,min=1"`
|
||||||
|
PageSize int `json:"page_size" form:"page_size" binding:"required,min=1,max=100"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// EventStatsRequest 事件统计请求
|
||||||
|
type EventStatsRequest struct {
|
||||||
|
StartTime time.Time `json:"start_time" form:"start_time" binding:"required"`
|
||||||
|
EndTime time.Time `json:"end_time" form:"end_time" binding:"required"`
|
||||||
|
}
|
||||||
36
internal/model/analytics/response.go
Normal file
36
internal/model/analytics/response.go
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
package analytics
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
|
// EventResponse 事件响应
|
||||||
|
type EventResponse struct {
|
||||||
|
ID int64 `json:"id"`
|
||||||
|
UserID int64 `json:"user_id"`
|
||||||
|
EventName string `json:"event_name"`
|
||||||
|
Properties map[string]interface{} `json:"properties"`
|
||||||
|
DeviceInfo map[string]interface{} `json:"device_info"`
|
||||||
|
MetaData map[string]interface{} `json:"meta_data"`
|
||||||
|
CreatedAt time.Time `json:"created_at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// EventListResponse 事件列表响应
|
||||||
|
type EventListResponse struct {
|
||||||
|
Events []*EventResponse `json:"events"`
|
||||||
|
Total int64 `json:"total"`
|
||||||
|
Page int `json:"page"`
|
||||||
|
Size int `json:"size"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// EventStatsResponse 事件统计响应
|
||||||
|
type EventStatsResponse struct {
|
||||||
|
EventName string `json:"event_name"`
|
||||||
|
Count int64 `json:"count"`
|
||||||
|
UniqueUsers int64 `json:"unique_users"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// EventStatsListResponse 事件统计列表响应
|
||||||
|
type EventStatsListResponse struct {
|
||||||
|
Stats []*EventStatsResponse `json:"stats"`
|
||||||
|
StartTime time.Time `json:"start_time"`
|
||||||
|
EndTime time.Time `json:"end_time"`
|
||||||
|
}
|
||||||
@@ -12,13 +12,18 @@ type GetFormulaTaskResponse struct {
|
|||||||
Latex string `json:"latex"`
|
Latex string `json:"latex"`
|
||||||
Markdown string `json:"markdown"`
|
Markdown string `json:"markdown"`
|
||||||
MathML string `json:"mathml"`
|
MathML string `json:"mathml"`
|
||||||
MathMLMW string `json:"mathml_mw"`
|
MML string `json:"mml"`
|
||||||
ImageBlob string `json:"image_blob"`
|
|
||||||
DocxURL string `json:"docx_url"`
|
|
||||||
PDFURL string `json:"pdf_url"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// FormulaRecognitionResponse 公式识别服务返回的响应
|
// FormulaRecognitionResponse 公式识别服务返回的响应
|
||||||
type FormulaRecognitionResponse struct {
|
type FormulaRecognitionResponse struct {
|
||||||
Result string `json:"result"`
|
Result string `json:"result"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ImageOCRResponse 图片OCR接口返回的响应
|
||||||
|
type ImageOCRResponse struct {
|
||||||
|
Markdown string `json:"markdown"` // Markdown 格式内容
|
||||||
|
Latex string `json:"latex"` // LaTeX 格式内容 (无公式时为空)
|
||||||
|
MathML string `json:"mathml"` // MathML 格式(无公式时为空)
|
||||||
|
MML string `json:"mml"` // MML 格式(无公式时为空)
|
||||||
|
}
|
||||||
|
|||||||
@@ -24,13 +24,15 @@ type TaskListDTO struct {
|
|||||||
Latex string `json:"latex"`
|
Latex string `json:"latex"`
|
||||||
Markdown string `json:"markdown"`
|
Markdown string `json:"markdown"`
|
||||||
MathML string `json:"mathml"`
|
MathML string `json:"mathml"`
|
||||||
MathMLMW string `json:"mathml_mw"`
|
MML string `json:"mml"`
|
||||||
ImageBlob string `json:"image_blob"`
|
|
||||||
DocxURL string `json:"docx_url"`
|
|
||||||
PDFURL string `json:"pdf_url"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type TaskListResponse struct {
|
type TaskListResponse struct {
|
||||||
TaskList []*TaskListDTO `json:"task_list"`
|
TaskList []*TaskListDTO `json:"task_list"`
|
||||||
Total int64 `json:"total"`
|
Total int64 `json:"total"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ExportTaskRequest struct {
|
||||||
|
TaskNo string `json:"task_no" binding:"required"`
|
||||||
|
Type string `json:"type" binding:"required,oneof=pdf docx"`
|
||||||
|
}
|
||||||
|
|||||||
@@ -20,8 +20,7 @@ type PhoneLoginResponse struct {
|
|||||||
|
|
||||||
type UserInfoResponse struct {
|
type UserInfoResponse struct {
|
||||||
Username string `json:"username"`
|
Username string `json:"username"`
|
||||||
Phone string `json:"phone"`
|
Email string `json:"email"`
|
||||||
Status int `json:"status"` // 0: not login, 1: login
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type EmailRegisterRequest struct {
|
type EmailRegisterRequest struct {
|
||||||
@@ -43,3 +42,31 @@ type EmailLoginResponse struct {
|
|||||||
Token string `json:"token"`
|
Token string `json:"token"`
|
||||||
ExpiresAt int64 `json:"expires_at"`
|
ExpiresAt int64 `json:"expires_at"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type GoogleAuthUrlRequest struct {
|
||||||
|
RedirectURI string `form:"redirect_uri" binding:"required"`
|
||||||
|
State string `form:"state" binding:"required"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type GoogleAuthUrlResponse struct {
|
||||||
|
AuthURL string `json:"auth_url"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type GoogleOAuthCallbackRequest struct {
|
||||||
|
Code string `json:"code" binding:"required"`
|
||||||
|
State string `json:"state" binding:"required"`
|
||||||
|
RedirectURI string `json:"redirect_uri" binding:"required"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type GoogleOAuthCallbackResponse struct {
|
||||||
|
Token string `json:"token"`
|
||||||
|
ExpiresAt int64 `json:"expires_at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type GoogleUserInfo struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Email string `json:"email"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Picture string `json:"picture"`
|
||||||
|
VerifiedEmail bool `json:"verified_email"`
|
||||||
|
}
|
||||||
|
|||||||
232
internal/service/analytics_service.go
Normal file
232
internal/service/analytics_service.go
Normal file
@@ -0,0 +1,232 @@
|
|||||||
|
package service
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.com/texpixel/document_ai/internal/model/analytics"
|
||||||
|
"gitea.com/texpixel/document_ai/internal/storage/dao"
|
||||||
|
"gitea.com/texpixel/document_ai/pkg/log"
|
||||||
|
"gorm.io/datatypes"
|
||||||
|
)
|
||||||
|
|
||||||
|
type AnalyticsService struct {
|
||||||
|
eventDao *dao.AnalyticsEventDao
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewAnalyticsService() *AnalyticsService {
|
||||||
|
return &AnalyticsService{
|
||||||
|
eventDao: dao.NewAnalyticsEventDao(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TrackEvent 记录单个事件
|
||||||
|
func (s *AnalyticsService) TrackEvent(ctx context.Context, req *analytics.TrackEventRequest) error {
|
||||||
|
// 将 map 转换为 JSON
|
||||||
|
propertiesJSON, err := json.Marshal(req.Properties)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "marshal properties failed", "error", err)
|
||||||
|
return fmt.Errorf("invalid properties format")
|
||||||
|
}
|
||||||
|
|
||||||
|
deviceInfoJSON, err := json.Marshal(req.DeviceInfo)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "marshal device_info failed", "error", err)
|
||||||
|
return fmt.Errorf("invalid device_info format")
|
||||||
|
}
|
||||||
|
|
||||||
|
metaDataJSON, err := json.Marshal(req.MetaData)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "marshal meta_data failed", "error", err)
|
||||||
|
return fmt.Errorf("invalid meta_data format")
|
||||||
|
}
|
||||||
|
|
||||||
|
event := &dao.AnalyticsEvent{
|
||||||
|
UserID: req.UserID,
|
||||||
|
EventName: req.EventName,
|
||||||
|
Properties: datatypes.JSON(propertiesJSON),
|
||||||
|
DeviceInfo: datatypes.JSON(deviceInfoJSON),
|
||||||
|
MetaData: datatypes.JSON(metaDataJSON),
|
||||||
|
CreatedAt: time.Now(),
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.eventDao.Create(dao.DB.WithContext(ctx), event); err != nil {
|
||||||
|
log.Error(ctx, "create analytics event failed", "error", err)
|
||||||
|
return fmt.Errorf("failed to track event")
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info(ctx, "event tracked successfully",
|
||||||
|
"event_id", event.ID,
|
||||||
|
"user_id", req.UserID,
|
||||||
|
"event_name", req.EventName)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// BatchTrackEvents 批量记录事件
|
||||||
|
func (s *AnalyticsService) BatchTrackEvents(ctx context.Context, req *analytics.BatchTrackEventRequest) error {
|
||||||
|
events := make([]*dao.AnalyticsEvent, 0, len(req.Events))
|
||||||
|
|
||||||
|
for _, eventReq := range req.Events {
|
||||||
|
propertiesJSON, err := json.Marshal(eventReq.Properties)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "marshal properties failed", "error", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
deviceInfoJSON, err := json.Marshal(eventReq.DeviceInfo)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "marshal device_info failed", "error", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
metaDataJSON, err := json.Marshal(eventReq.MetaData)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "marshal meta_data failed", "error", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
event := &dao.AnalyticsEvent{
|
||||||
|
UserID: eventReq.UserID,
|
||||||
|
EventName: eventReq.EventName,
|
||||||
|
Properties: datatypes.JSON(propertiesJSON),
|
||||||
|
DeviceInfo: datatypes.JSON(deviceInfoJSON),
|
||||||
|
MetaData: datatypes.JSON(metaDataJSON),
|
||||||
|
CreatedAt: time.Now(),
|
||||||
|
}
|
||||||
|
events = append(events, event)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(events) == 0 {
|
||||||
|
return fmt.Errorf("no valid events to track")
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.eventDao.BatchCreate(dao.DB.WithContext(ctx), events); err != nil {
|
||||||
|
log.Error(ctx, "batch create analytics events failed", "error", err)
|
||||||
|
return fmt.Errorf("failed to batch track events")
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info(ctx, "batch events tracked successfully", "count", len(events))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// QueryEvents 查询事件
|
||||||
|
func (s *AnalyticsService) QueryEvents(ctx context.Context, req *analytics.QueryEventsRequest) (*analytics.EventListResponse, error) {
|
||||||
|
var events []*dao.AnalyticsEvent
|
||||||
|
var total int64
|
||||||
|
var err error
|
||||||
|
|
||||||
|
// 根据不同条件查询
|
||||||
|
if req.UserID != nil && req.EventName != "" {
|
||||||
|
// 查询用户的指定事件
|
||||||
|
events, total, err = s.eventDao.GetUserEventsByName(dao.DB.WithContext(ctx), *req.UserID, req.EventName, req.Page, req.PageSize)
|
||||||
|
} else if req.UserID != nil {
|
||||||
|
// 查询用户的所有事件
|
||||||
|
events, total, err = s.eventDao.GetUserEvents(dao.DB.WithContext(ctx), *req.UserID, req.Page, req.PageSize)
|
||||||
|
} else if req.EventName != "" {
|
||||||
|
// 查询指定事件
|
||||||
|
events, total, err = s.eventDao.GetEventsByName(dao.DB.WithContext(ctx), req.EventName, req.Page, req.PageSize)
|
||||||
|
} else if req.StartTime != nil && req.EndTime != nil {
|
||||||
|
// 查询时间范围内的事件
|
||||||
|
events, total, err = s.eventDao.GetEventsByTimeRange(dao.DB.WithContext(ctx), *req.StartTime, *req.EndTime, req.Page, req.PageSize)
|
||||||
|
} else {
|
||||||
|
return nil, fmt.Errorf("invalid query parameters")
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "query events failed", "error", err)
|
||||||
|
return nil, fmt.Errorf("failed to query events")
|
||||||
|
}
|
||||||
|
|
||||||
|
// 转换为响应格式
|
||||||
|
eventResponses := make([]*analytics.EventResponse, 0, len(events))
|
||||||
|
for _, event := range events {
|
||||||
|
var properties, deviceInfo, metaData map[string]interface{}
|
||||||
|
|
||||||
|
if len(event.Properties) > 0 {
|
||||||
|
json.Unmarshal(event.Properties, &properties)
|
||||||
|
}
|
||||||
|
if len(event.DeviceInfo) > 0 {
|
||||||
|
json.Unmarshal(event.DeviceInfo, &deviceInfo)
|
||||||
|
}
|
||||||
|
if len(event.MetaData) > 0 {
|
||||||
|
json.Unmarshal(event.MetaData, &metaData)
|
||||||
|
}
|
||||||
|
|
||||||
|
eventResponses = append(eventResponses, &analytics.EventResponse{
|
||||||
|
ID: event.ID,
|
||||||
|
UserID: event.UserID,
|
||||||
|
EventName: event.EventName,
|
||||||
|
Properties: properties,
|
||||||
|
DeviceInfo: deviceInfo,
|
||||||
|
MetaData: metaData,
|
||||||
|
CreatedAt: event.CreatedAt,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return &analytics.EventListResponse{
|
||||||
|
Events: eventResponses,
|
||||||
|
Total: total,
|
||||||
|
Page: req.Page,
|
||||||
|
Size: req.PageSize,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetEventStats 获取事件统计
|
||||||
|
func (s *AnalyticsService) GetEventStats(ctx context.Context, req *analytics.EventStatsRequest) (*analytics.EventStatsListResponse, error) {
|
||||||
|
results, err := s.eventDao.GetEventStats(dao.DB.WithContext(ctx), req.StartTime, req.EndTime)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "get event stats failed", "error", err)
|
||||||
|
return nil, fmt.Errorf("failed to get event stats")
|
||||||
|
}
|
||||||
|
|
||||||
|
stats := make([]*analytics.EventStatsResponse, 0, len(results))
|
||||||
|
for _, result := range results {
|
||||||
|
stats = append(stats, &analytics.EventStatsResponse{
|
||||||
|
EventName: result["event_name"].(string),
|
||||||
|
Count: result["count"].(int64),
|
||||||
|
UniqueUsers: result["unique_users"].(int64),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return &analytics.EventStatsListResponse{
|
||||||
|
Stats: stats,
|
||||||
|
StartTime: req.StartTime,
|
||||||
|
EndTime: req.EndTime,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// CountUserEvents 统计用户事件数量
|
||||||
|
func (s *AnalyticsService) CountUserEvents(ctx context.Context, userID int64) (int64, error) {
|
||||||
|
count, err := s.eventDao.CountUserEvents(dao.DB.WithContext(ctx), userID)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "count user events failed", "error", err, "user_id", userID)
|
||||||
|
return 0, fmt.Errorf("failed to count user events")
|
||||||
|
}
|
||||||
|
return count, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// CountEventsByName 统计指定事件的数量
|
||||||
|
func (s *AnalyticsService) CountEventsByName(ctx context.Context, eventName string) (int64, error) {
|
||||||
|
count, err := s.eventDao.CountEventsByName(dao.DB.WithContext(ctx), eventName)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "count events by name failed", "error", err, "event_name", eventName)
|
||||||
|
return 0, fmt.Errorf("failed to count events")
|
||||||
|
}
|
||||||
|
return count, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// CleanOldEvents 清理旧数据(可以定时执行)
|
||||||
|
func (s *AnalyticsService) CleanOldEvents(ctx context.Context, retentionDays int) error {
|
||||||
|
beforeTime := time.Now().AddDate(0, 0, -retentionDays)
|
||||||
|
|
||||||
|
if err := s.eventDao.DeleteOldEvents(dao.DB.WithContext(ctx), beforeTime); err != nil {
|
||||||
|
log.Error(ctx, "clean old events failed", "error", err, "before_time", beforeTime)
|
||||||
|
return fmt.Errorf("failed to clean old events")
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info(ctx, "old events cleaned successfully", "retention_days", retentionDays)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -7,21 +7,23 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"mime/multipart"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.com/bitwsd/document_ai/config"
|
"gitea.com/texpixel/document_ai/config"
|
||||||
"gitea.com/bitwsd/document_ai/internal/model/formula"
|
"gitea.com/texpixel/document_ai/internal/model/formula"
|
||||||
"gitea.com/bitwsd/document_ai/internal/storage/cache"
|
"gitea.com/texpixel/document_ai/internal/storage/cache"
|
||||||
"gitea.com/bitwsd/document_ai/internal/storage/dao"
|
"gitea.com/texpixel/document_ai/internal/storage/dao"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/log"
|
"gitea.com/texpixel/document_ai/pkg/log"
|
||||||
|
|
||||||
"gitea.com/bitwsd/document_ai/pkg/common"
|
"gitea.com/texpixel/document_ai/pkg/common"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/constant"
|
"gitea.com/texpixel/document_ai/pkg/constant"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/httpclient"
|
"gitea.com/texpixel/document_ai/pkg/httpclient"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/oss"
|
"gitea.com/texpixel/document_ai/pkg/oss"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/utils"
|
"gitea.com/texpixel/document_ai/pkg/requestid"
|
||||||
|
"gitea.com/texpixel/document_ai/pkg/utils"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -178,6 +180,7 @@ func (s *RecognitionService) GetFormualTask(ctx context.Context, taskNo string)
|
|||||||
Latex: taskRet.Latex,
|
Latex: taskRet.Latex,
|
||||||
Markdown: markdown,
|
Markdown: markdown,
|
||||||
MathML: taskRet.MathML,
|
MathML: taskRet.MathML,
|
||||||
|
MML: taskRet.MML,
|
||||||
Status: int(task.Status),
|
Status: int(task.Status),
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
@@ -379,6 +382,44 @@ type MathpixErrorInfo struct {
|
|||||||
Message string `json:"message"`
|
Message string `json:"message"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// BaiduOCRRequest 百度 OCR 版面分析请求结构
|
||||||
|
type BaiduOCRRequest struct {
|
||||||
|
// 文件内容 base64 编码
|
||||||
|
File string `json:"file"`
|
||||||
|
// 文件类型: 0=PDF, 1=图片
|
||||||
|
FileType int `json:"fileType"`
|
||||||
|
// 是否启用文档方向分类
|
||||||
|
UseDocOrientationClassify bool `json:"useDocOrientationClassify"`
|
||||||
|
// 是否启用文档扭曲矫正
|
||||||
|
UseDocUnwarping bool `json:"useDocUnwarping"`
|
||||||
|
// 是否启用图表识别
|
||||||
|
UseChartRecognition bool `json:"useChartRecognition"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// BaiduOCRResponse 百度 OCR 版面分析响应结构
|
||||||
|
type BaiduOCRResponse struct {
|
||||||
|
ErrorCode int `json:"errorCode"`
|
||||||
|
ErrorMsg string `json:"errorMsg"`
|
||||||
|
Result *BaiduOCRResult `json:"result"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// BaiduOCRResult 百度 OCR 响应结果
|
||||||
|
type BaiduOCRResult struct {
|
||||||
|
LayoutParsingResults []BaiduLayoutParsingResult `json:"layoutParsingResults"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// BaiduLayoutParsingResult 单页版面解析结果
|
||||||
|
type BaiduLayoutParsingResult struct {
|
||||||
|
Markdown BaiduMarkdownResult `json:"markdown"`
|
||||||
|
OutputImages map[string]string `json:"outputImages"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// BaiduMarkdownResult markdown 结果
|
||||||
|
type BaiduMarkdownResult struct {
|
||||||
|
Text string `json:"text"`
|
||||||
|
Images map[string]string `json:"images"`
|
||||||
|
}
|
||||||
|
|
||||||
// GetMathML 从响应中获取MathML
|
// GetMathML 从响应中获取MathML
|
||||||
func (r *MathpixResponse) GetMathML() string {
|
func (r *MathpixResponse) GetMathML() string {
|
||||||
for _, item := range r.Data {
|
for _, item := range r.Data {
|
||||||
@@ -472,8 +513,8 @@ func (s *RecognitionService) processFormulaTask(ctx context.Context, taskID int6
|
|||||||
// 设置Content-Type头为application/json
|
// 设置Content-Type头为application/json
|
||||||
headers := map[string]string{"Content-Type": "application/json", utils.RequestIDHeaderKey: utils.GetRequestIDFromContext(ctx)}
|
headers := map[string]string{"Content-Type": "application/json", utils.RequestIDHeaderKey: utils.GetRequestIDFromContext(ctx)}
|
||||||
|
|
||||||
// 发送请求时会使用带超时的context
|
// 发送请求到新的 OCR 接口
|
||||||
resp, err := s.httpClient.RequestWithRetry(ctx, http.MethodPost, "https://cloud.texpixel.com:10443/vlm/formula/predict", bytes.NewReader(jsonData), headers)
|
resp, err := s.httpClient.RequestWithRetry(ctx, http.MethodPost, "https://cloud.texpixel.com:10443/doc_process/v1/image/ocr", bytes.NewReader(jsonData), headers)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if ctx.Err() == context.DeadlineExceeded {
|
if ctx.Err() == context.DeadlineExceeded {
|
||||||
log.Error(ctx, "func", "processFormulaTask", "msg", "请求超时")
|
log.Error(ctx, "func", "processFormulaTask", "msg", "请求超时")
|
||||||
@@ -493,12 +534,19 @@ func (s *RecognitionService) processFormulaTask(ctx context.Context, taskID int6
|
|||||||
log.Info(ctx, "func", "processFormulaTask", "msg", "响应内容", "body", body.String())
|
log.Info(ctx, "func", "processFormulaTask", "msg", "响应内容", "body", body.String())
|
||||||
|
|
||||||
// 解析 JSON 响应
|
// 解析 JSON 响应
|
||||||
var formulaResp formula.FormulaRecognitionResponse
|
var ocrResp formula.ImageOCRResponse
|
||||||
if err := json.Unmarshal(body.Bytes(), &formulaResp); err != nil {
|
if err := json.Unmarshal(body.Bytes(), &ocrResp); err != nil {
|
||||||
log.Error(ctx, "func", "processFormulaTask", "msg", "解析响应JSON失败", "error", err)
|
log.Error(ctx, "func", "processFormulaTask", "msg", "解析响应JSON失败", "error", err)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
err = resultDao.Create(tx, dao.RecognitionResult{TaskID: taskID, TaskType: dao.TaskTypeFormula, Latex: formulaResp.Result})
|
err = resultDao.Create(tx, dao.RecognitionResult{
|
||||||
|
TaskID: taskID,
|
||||||
|
TaskType: dao.TaskTypeFormula,
|
||||||
|
Latex: ocrResp.Latex,
|
||||||
|
Markdown: ocrResp.Markdown,
|
||||||
|
MathML: ocrResp.MathML,
|
||||||
|
MML: ocrResp.MML,
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error(ctx, "func", "processFormulaTask", "msg", "保存任务结果失败", "error", err)
|
log.Error(ctx, "func", "processFormulaTask", "msg", "保存任务结果失败", "error", err)
|
||||||
return err
|
return err
|
||||||
@@ -608,21 +656,20 @@ func (s *RecognitionService) processVLFormulaTask(ctx context.Context, taskID in
|
|||||||
}
|
}
|
||||||
|
|
||||||
resultDao := dao.NewRecognitionResultDao()
|
resultDao := dao.NewRecognitionResultDao()
|
||||||
var formulaRes *dao.RecognitionResult
|
|
||||||
result, err := resultDao.GetByTaskID(dao.DB.WithContext(ctx), taskID)
|
result, err := resultDao.GetByTaskID(dao.DB.WithContext(ctx), taskID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error(ctx, "func", "processVLFormulaTask", "msg", "获取任务结果失败", "error", err)
|
log.Error(ctx, "func", "processVLFormulaTask", "msg", "获取任务结果失败", "error", err)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if result == nil {
|
if result == nil {
|
||||||
formulaRes = &dao.RecognitionResult{TaskID: taskID, TaskType: dao.TaskTypeFormula, Latex: latex}
|
formulaRes := &dao.RecognitionResult{TaskID: taskID, TaskType: dao.TaskTypeFormula, Latex: latex}
|
||||||
err = resultDao.Create(dao.DB.WithContext(ctx), *formulaRes)
|
err = resultDao.Create(dao.DB.WithContext(ctx), *formulaRes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error(ctx, "func", "processVLFormulaTask", "msg", "创建任务结果失败", "error", err)
|
log.Error(ctx, "func", "processVLFormulaTask", "msg", "创建任务结果失败", "error", err)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
formulaRes.Latex = latex
|
result.Latex = latex
|
||||||
err = resultDao.Update(dao.DB.WithContext(ctx), result.ID, map[string]interface{}{"latex": latex})
|
err = resultDao.Update(dao.DB.WithContext(ctx), result.ID, map[string]interface{}{"latex": latex})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error(ctx, "func", "processVLFormulaTask", "msg", "更新任务结果失败", "error", err)
|
log.Error(ctx, "func", "processVLFormulaTask", "msg", "更新任务结果失败", "error", err)
|
||||||
@@ -667,15 +714,19 @@ func (s *RecognitionService) processOneTask(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
ctx = context.WithValue(ctx, utils.RequestIDKey, task.TaskUUID)
|
ctx = context.WithValue(ctx, utils.RequestIDKey, task.TaskUUID)
|
||||||
|
|
||||||
|
// 使用 gls 设置 request_id,确保在整个任务处理过程中可用
|
||||||
|
requestid.SetRequestID(task.TaskUUID, func() {
|
||||||
log.Info(ctx, "func", "processFormulaQueue", "msg", "获取任务成功", "task_id", taskID)
|
log.Info(ctx, "func", "processFormulaQueue", "msg", "获取任务成功", "task_id", taskID)
|
||||||
|
|
||||||
err = s.processMathpixTask(ctx, taskID, task.FileURL)
|
err = s.processFormulaTask(ctx, taskID, task.FileURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error(ctx, "func", "processFormulaQueue", "msg", "处理任务失败", "error", err)
|
log.Error(ctx, "func", "processFormulaQueue", "msg", "处理任务失败", "error", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info(ctx, "func", "processFormulaQueue", "msg", "处理任务成功", "task_id", taskID)
|
log.Info(ctx, "func", "processFormulaQueue", "msg", "处理任务成功", "task_id", taskID)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// processMathpixTask 使用 Mathpix API 处理公式识别任务(用于增强识别)
|
// processMathpixTask 使用 Mathpix API 处理公式识别任务(用于增强识别)
|
||||||
@@ -738,6 +789,10 @@ func (s *RecognitionService) processMathpixTask(ctx context.Context, taskID int6
|
|||||||
|
|
||||||
endpoint := "https://api.mathpix.com/v3/text"
|
endpoint := "https://api.mathpix.com/v3/text"
|
||||||
|
|
||||||
|
startTime := time.Now()
|
||||||
|
|
||||||
|
log.Info(ctx, "func", "processMathpixTask", "msg", "MathpixApi_Start", "start_time", startTime)
|
||||||
|
|
||||||
resp, err := s.httpClient.RequestWithRetry(ctx, http.MethodPost, endpoint, bytes.NewReader(jsonData), headers)
|
resp, err := s.httpClient.RequestWithRetry(ctx, http.MethodPost, endpoint, bytes.NewReader(jsonData), headers)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error(ctx, "func", "processMathpixTask", "msg", "Mathpix API 请求失败", "error", err)
|
log.Error(ctx, "func", "processMathpixTask", "msg", "Mathpix API 请求失败", "error", err)
|
||||||
@@ -745,6 +800,8 @@ func (s *RecognitionService) processMathpixTask(ctx context.Context, taskID int6
|
|||||||
}
|
}
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
log.Info(ctx, "func", "processMathpixTask", "msg", "MathpixApi_End", "end_time", time.Now(), "duration", time.Since(startTime))
|
||||||
|
|
||||||
body := &bytes.Buffer{}
|
body := &bytes.Buffer{}
|
||||||
if _, err = body.ReadFrom(resp.Body); err != nil {
|
if _, err = body.ReadFrom(resp.Body); err != nil {
|
||||||
log.Error(ctx, "func", "processMathpixTask", "msg", "读取响应体失败", "error", err)
|
log.Error(ctx, "func", "processMathpixTask", "msg", "读取响应体失败", "error", err)
|
||||||
@@ -790,6 +847,8 @@ func (s *RecognitionService) processMathpixTask(ctx context.Context, taskID int6
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
log.Info(ctx, "func", "processMathpixTask", "msg", "saveLog", "end_time", time.Now(), "duration", time.Since(startTime))
|
||||||
|
|
||||||
if result == nil {
|
if result == nil {
|
||||||
// 创建新结果
|
// 创建新结果
|
||||||
err = resultDao.Create(dao.DB.WithContext(ctx), dao.RecognitionResult{
|
err = resultDao.Create(dao.DB.WithContext(ctx), dao.RecognitionResult{
|
||||||
@@ -820,6 +879,182 @@ func (s *RecognitionService) processMathpixTask(ctx context.Context, taskID int6
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *RecognitionService) processBaiduOCRTask(ctx context.Context, taskID int64, fileURL string) error {
|
||||||
|
isSuccess := false
|
||||||
|
logDao := dao.NewRecognitionLogDao()
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
if !isSuccess {
|
||||||
|
err := dao.NewRecognitionTaskDao().Update(dao.DB.WithContext(ctx), map[string]interface{}{"id": taskID}, map[string]interface{}{"status": dao.TaskStatusFailed})
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "func", "processBaiduOCRTask", "msg", "更新任务状态失败", "error", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err := dao.NewRecognitionTaskDao().Update(dao.DB.WithContext(ctx), map[string]interface{}{"id": taskID}, map[string]interface{}{"status": dao.TaskStatusCompleted})
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "func", "processBaiduOCRTask", "msg", "更新任务状态失败", "error", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
// 从 OSS 下载文件
|
||||||
|
reader, err := oss.DownloadFile(ctx, fileURL)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "func", "processBaiduOCRTask", "msg", "从OSS下载文件失败", "error", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer reader.Close()
|
||||||
|
|
||||||
|
// 读取文件内容
|
||||||
|
fileBytes, err := io.ReadAll(reader)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "func", "processBaiduOCRTask", "msg", "读取文件内容失败", "error", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Base64 编码
|
||||||
|
fileData := base64.StdEncoding.EncodeToString(fileBytes)
|
||||||
|
|
||||||
|
// 根据文件扩展名确定 fileType: 0=PDF, 1=图片
|
||||||
|
fileType := 1 // 默认为图片
|
||||||
|
lowerFileURL := strings.ToLower(fileURL)
|
||||||
|
if strings.HasSuffix(lowerFileURL, ".pdf") {
|
||||||
|
fileType = 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// 创建百度 OCR API 请求
|
||||||
|
baiduReq := BaiduOCRRequest{
|
||||||
|
File: fileData,
|
||||||
|
FileType: fileType,
|
||||||
|
UseDocOrientationClassify: false,
|
||||||
|
UseDocUnwarping: false,
|
||||||
|
UseChartRecognition: false,
|
||||||
|
}
|
||||||
|
|
||||||
|
jsonData, err := json.Marshal(baiduReq)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "func", "processBaiduOCRTask", "msg", "JSON编码失败", "error", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
headers := map[string]string{
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"Authorization": fmt.Sprintf("token %s", config.GlobalConfig.BaiduOCR.Token),
|
||||||
|
}
|
||||||
|
|
||||||
|
endpoint := "https://j5veh2l2r6ubk6cb.aistudio-app.com/layout-parsing"
|
||||||
|
|
||||||
|
startTime := time.Now()
|
||||||
|
|
||||||
|
log.Info(ctx, "func", "processBaiduOCRTask", "msg", "BaiduOCRApi_Start", "start_time", startTime)
|
||||||
|
|
||||||
|
resp, err := s.httpClient.RequestWithRetry(ctx, http.MethodPost, endpoint, bytes.NewReader(jsonData), headers)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "func", "processBaiduOCRTask", "msg", "百度 OCR API 请求失败", "error", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
log.Info(ctx, "func", "processBaiduOCRTask", "msg", "BaiduOCRApi_End", "end_time", time.Now(), "duration", time.Since(startTime))
|
||||||
|
|
||||||
|
body := &bytes.Buffer{}
|
||||||
|
if _, err = body.ReadFrom(resp.Body); err != nil {
|
||||||
|
log.Error(ctx, "func", "processBaiduOCRTask", "msg", "读取响应体失败", "error", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// 创建日志记录(不记录请求体中的 base64 数据以节省存储)
|
||||||
|
requestLogData := map[string]interface{}{
|
||||||
|
"fileType": fileType,
|
||||||
|
"useDocOrientationClassify": false,
|
||||||
|
"useDocUnwarping": false,
|
||||||
|
"useChartRecognition": false,
|
||||||
|
"fileSize": len(fileBytes),
|
||||||
|
}
|
||||||
|
requestLogBytes, _ := json.Marshal(requestLogData)
|
||||||
|
recognitionLog := &dao.RecognitionLog{
|
||||||
|
TaskID: taskID,
|
||||||
|
Provider: dao.ProviderBaiduOCR,
|
||||||
|
RequestBody: string(requestLogBytes),
|
||||||
|
ResponseBody: body.String(),
|
||||||
|
}
|
||||||
|
|
||||||
|
// 解析响应
|
||||||
|
var baiduResp BaiduOCRResponse
|
||||||
|
if err := json.Unmarshal(body.Bytes(), &baiduResp); err != nil {
|
||||||
|
log.Error(ctx, "func", "processBaiduOCRTask", "msg", "解析响应失败", "error", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// 检查错误
|
||||||
|
if baiduResp.ErrorCode != 0 {
|
||||||
|
errMsg := fmt.Sprintf("errorCode: %d, errorMsg: %s", baiduResp.ErrorCode, baiduResp.ErrorMsg)
|
||||||
|
log.Error(ctx, "func", "processBaiduOCRTask", "msg", "百度 OCR API 返回错误", "error", errMsg)
|
||||||
|
return fmt.Errorf("baidu ocr error: %s", errMsg)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 保存日志
|
||||||
|
err = logDao.Create(dao.DB.WithContext(ctx), recognitionLog)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "func", "processBaiduOCRTask", "msg", "保存日志失败", "error", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 合并所有页面的 markdown 结果
|
||||||
|
var markdownTexts []string
|
||||||
|
if baiduResp.Result != nil && len(baiduResp.Result.LayoutParsingResults) > 0 {
|
||||||
|
for _, res := range baiduResp.Result.LayoutParsingResults {
|
||||||
|
if res.Markdown.Text != "" {
|
||||||
|
markdownTexts = append(markdownTexts, res.Markdown.Text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
markdownResult := strings.Join(markdownTexts, "\n\n---\n\n")
|
||||||
|
|
||||||
|
latex, mml, e := s.HandleConvert(ctx, markdownResult)
|
||||||
|
if e != nil {
|
||||||
|
log.Error(ctx, "func", "processBaiduOCRTask", "msg", "转换失败", "error", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 更新或创建识别结果
|
||||||
|
resultDao := dao.NewRecognitionResultDao()
|
||||||
|
result, err := resultDao.GetByTaskID(dao.DB.WithContext(ctx), taskID)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "func", "processBaiduOCRTask", "msg", "获取任务结果失败", "error", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info(ctx, "func", "processBaiduOCRTask", "msg", "saveLog", "end_time", time.Now(), "duration", time.Since(startTime))
|
||||||
|
|
||||||
|
if result == nil {
|
||||||
|
// 创建新结果
|
||||||
|
err = resultDao.Create(dao.DB.WithContext(ctx), dao.RecognitionResult{
|
||||||
|
TaskID: taskID,
|
||||||
|
TaskType: dao.TaskTypeFormula,
|
||||||
|
Markdown: markdownResult,
|
||||||
|
Latex: latex,
|
||||||
|
MathML: mml,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "func", "processBaiduOCRTask", "msg", "创建任务结果失败", "error", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// 更新现有结果
|
||||||
|
err = resultDao.Update(dao.DB.WithContext(ctx), result.ID, map[string]interface{}{
|
||||||
|
"markdown": markdownResult,
|
||||||
|
"latex": latex,
|
||||||
|
"mathml": mml,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "func", "processBaiduOCRTask", "msg", "更新任务结果失败", "error", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
isSuccess = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (s *RecognitionService) TestProcessMathpixTask(ctx context.Context, taskID int64) error {
|
func (s *RecognitionService) TestProcessMathpixTask(ctx context.Context, taskID int64) error {
|
||||||
task, err := dao.NewRecognitionTaskDao().GetTaskByID(dao.DB.WithContext(ctx), taskID)
|
task, err := dao.NewRecognitionTaskDao().GetTaskByID(dao.DB.WithContext(ctx), taskID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -832,3 +1067,55 @@ func (s *RecognitionService) TestProcessMathpixTask(ctx context.Context, taskID
|
|||||||
}
|
}
|
||||||
return s.processMathpixTask(ctx, taskID, task.FileURL)
|
return s.processMathpixTask(ctx, taskID, task.FileURL)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ConvertResponse Python 接口返回结构
|
||||||
|
type ConvertResponse struct {
|
||||||
|
Latex string `json:"latex"`
|
||||||
|
MathML string `json:"mathml"`
|
||||||
|
Error string `json:"error,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *RecognitionService) HandleConvert(ctx context.Context, markdown string) (latex string, mml string, err error) {
|
||||||
|
url := "https://cloud.texpixel.com:10443/doc_converter/v1/convert"
|
||||||
|
|
||||||
|
// 构建 multipart form
|
||||||
|
body := &bytes.Buffer{}
|
||||||
|
writer := multipart.NewWriter(body)
|
||||||
|
_ = writer.WriteField("markdown_input", markdown)
|
||||||
|
writer.Close()
|
||||||
|
|
||||||
|
// 使用正确的 Content-Type(包含 boundary)
|
||||||
|
headers := map[string]string{
|
||||||
|
"Content-Type": writer.FormDataContentType(),
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := s.httpClient.RequestWithRetry(ctx, http.MethodPost, url, body, headers)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
// 读取响应体
|
||||||
|
respBody, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
// 检查 HTTP 状态码
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return "", "", fmt.Errorf("convert failed: status %d, body: %s", resp.StatusCode, string(respBody))
|
||||||
|
}
|
||||||
|
|
||||||
|
// 解析 JSON 响应
|
||||||
|
var convertResp ConvertResponse
|
||||||
|
if err := json.Unmarshal(respBody, &convertResp); err != nil {
|
||||||
|
return "", "", fmt.Errorf("unmarshal response failed: %v, body: %s", err, string(respBody))
|
||||||
|
}
|
||||||
|
|
||||||
|
// 检查业务错误
|
||||||
|
if convertResp.Error != "" {
|
||||||
|
return "", "", fmt.Errorf("convert error: %s", convertResp.Error)
|
||||||
|
}
|
||||||
|
|
||||||
|
return convertResp.Latex, convertResp.MathML, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,15 +1,19 @@
|
|||||||
package service
|
package service
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.com/bitwsd/document_ai/internal/model/task"
|
"gitea.com/texpixel/document_ai/internal/model/task"
|
||||||
"gitea.com/bitwsd/document_ai/internal/storage/dao"
|
"gitea.com/texpixel/document_ai/internal/storage/dao"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/log"
|
"gitea.com/texpixel/document_ai/pkg/log"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/oss"
|
"gitea.com/texpixel/document_ai/pkg/oss"
|
||||||
)
|
)
|
||||||
|
|
||||||
type TaskService struct {
|
type TaskService struct {
|
||||||
@@ -87,13 +91,14 @@ func (svc *TaskService) GetTaskList(ctx context.Context, req *task.TaskListReque
|
|||||||
for _, item := range tasks {
|
for _, item := range tasks {
|
||||||
var latex string
|
var latex string
|
||||||
var markdown string
|
var markdown string
|
||||||
|
var mathML string
|
||||||
|
var mml string
|
||||||
recognitionResult := recognitionResultMap[item.ID]
|
recognitionResult := recognitionResultMap[item.ID]
|
||||||
if recognitionResult != nil {
|
if recognitionResult != nil {
|
||||||
latex = recognitionResult.Latex
|
latex = recognitionResult.Latex
|
||||||
markdown = recognitionResult.Markdown
|
markdown = recognitionResult.Markdown
|
||||||
if markdown == "" {
|
mathML = recognitionResult.MathML
|
||||||
markdown = fmt.Sprintf("$$%s$$", latex)
|
mml = recognitionResult.MML
|
||||||
}
|
|
||||||
}
|
}
|
||||||
originURL, err := oss.GetDownloadURL(ctx, item.FileURL)
|
originURL, err := oss.GetDownloadURL(ctx, item.FileURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -102,6 +107,8 @@ func (svc *TaskService) GetTaskList(ctx context.Context, req *task.TaskListReque
|
|||||||
resp.TaskList = append(resp.TaskList, &task.TaskListDTO{
|
resp.TaskList = append(resp.TaskList, &task.TaskListDTO{
|
||||||
Latex: latex,
|
Latex: latex,
|
||||||
Markdown: markdown,
|
Markdown: markdown,
|
||||||
|
MathML: mathML,
|
||||||
|
MML: mml,
|
||||||
TaskID: item.TaskUUID,
|
TaskID: item.TaskUUID,
|
||||||
FileName: item.FileName,
|
FileName: item.FileName,
|
||||||
Status: int(item.Status),
|
Status: int(item.Status),
|
||||||
@@ -112,3 +119,87 @@ func (svc *TaskService) GetTaskList(ctx context.Context, req *task.TaskListReque
|
|||||||
}
|
}
|
||||||
return resp, nil
|
return resp, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (svc *TaskService) ExportTask(ctx context.Context, req *task.ExportTaskRequest) ([]byte, string, error) {
|
||||||
|
recognitionTask, err := svc.recognitionTaskDao.GetByTaskNo(dao.DB.WithContext(ctx), req.TaskNo)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "func", "ExportTask", "msg", "get task by task id failed", "error", err)
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
if recognitionTask == nil {
|
||||||
|
log.Error(ctx, "func", "ExportTask", "msg", "task not found")
|
||||||
|
return nil, "", errors.New("task not found")
|
||||||
|
}
|
||||||
|
|
||||||
|
if recognitionTask.Status != dao.TaskStatusCompleted {
|
||||||
|
log.Error(ctx, "func", "ExportTask", "msg", "task not finished")
|
||||||
|
return nil, "", errors.New("task not finished")
|
||||||
|
}
|
||||||
|
|
||||||
|
recognitionResult, err := svc.recognitionResultDao.GetByTaskID(dao.DB.WithContext(ctx), recognitionTask.ID)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "func", "ExportTask", "msg", "get recognition result by task id failed", "error", err)
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
if recognitionResult == nil {
|
||||||
|
log.Error(ctx, "func", "ExportTask", "msg", "recognition result not found")
|
||||||
|
return nil, "", errors.New("recognition result not found")
|
||||||
|
}
|
||||||
|
|
||||||
|
markdown := recognitionResult.Markdown
|
||||||
|
if markdown == "" {
|
||||||
|
log.Error(ctx, "func", "ExportTask", "msg", "markdown not found")
|
||||||
|
return nil, "", errors.New("markdown not found")
|
||||||
|
}
|
||||||
|
|
||||||
|
// 获取文件名(去掉扩展名)
|
||||||
|
filename := strings.TrimSuffix(recognitionTask.FileName, "."+strings.ToLower(strings.Split(recognitionTask.FileName, ".")[len(strings.Split(recognitionTask.FileName, "."))-1]))
|
||||||
|
if filename == "" {
|
||||||
|
filename = "texpixel"
|
||||||
|
}
|
||||||
|
|
||||||
|
// 构建 JSON 请求体
|
||||||
|
requestBody := map[string]string{
|
||||||
|
"markdown": markdown,
|
||||||
|
"filename": filename,
|
||||||
|
}
|
||||||
|
jsonData, err := json.Marshal(requestBody)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "func", "ExportTask", "msg", "json marshal failed", "error", err)
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://cloud.texpixel.com:10443/doc_process/v1/convert/file", bytes.NewReader(jsonData))
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "func", "ExportTask", "msg", "create http request failed", "error", err)
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
httpReq.Header.Set("Content-Type", "application/json")
|
||||||
|
|
||||||
|
client := &http.Client{}
|
||||||
|
resp, err := client.Do(httpReq)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "func", "ExportTask", "msg", "http request failed", "error", err)
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
body, _ := io.ReadAll(resp.Body)
|
||||||
|
log.Error(ctx, "func", "ExportTask", "msg", "http request failed", "status", resp.StatusCode, "body", string(body))
|
||||||
|
return nil, "", fmt.Errorf("export service returned status: %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
fileData, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "func", "ExportTask", "msg", "read response body failed", "error", err)
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
// 新接口只返回 DOCX 格式
|
||||||
|
contentType := "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
|
||||||
|
|
||||||
|
return fileData, contentType, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -2,15 +2,20 @@ package service
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"math/rand"
|
"math/rand"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
|
||||||
"gitea.com/bitwsd/document_ai/internal/storage/cache"
|
"gitea.com/texpixel/document_ai/config"
|
||||||
"gitea.com/bitwsd/document_ai/internal/storage/dao"
|
model "gitea.com/texpixel/document_ai/internal/model/user"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/common"
|
"gitea.com/texpixel/document_ai/internal/storage/cache"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/log"
|
"gitea.com/texpixel/document_ai/internal/storage/dao"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/sms"
|
"gitea.com/texpixel/document_ai/pkg/common"
|
||||||
|
"gitea.com/texpixel/document_ai/pkg/log"
|
||||||
|
"gitea.com/texpixel/document_ai/pkg/sms"
|
||||||
"golang.org/x/crypto/bcrypt"
|
"golang.org/x/crypto/bcrypt"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -159,3 +164,126 @@ func (svc *UserService) LoginByEmail(ctx context.Context, email, password string
|
|||||||
|
|
||||||
return user.ID, nil
|
return user.ID, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type googleTokenResponse struct {
|
||||||
|
AccessToken string `json:"access_token"`
|
||||||
|
IDToken string `json:"id_token"`
|
||||||
|
ExpiresIn int `json:"expires_in"`
|
||||||
|
TokenType string `json:"token_type"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (svc *UserService) googleHTTPClient() *http.Client {
|
||||||
|
if config.GlobalConfig.Google.Proxy == "" {
|
||||||
|
return &http.Client{}
|
||||||
|
}
|
||||||
|
proxyURL, err := url.Parse(config.GlobalConfig.Google.Proxy)
|
||||||
|
if err != nil {
|
||||||
|
return &http.Client{}
|
||||||
|
}
|
||||||
|
return &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (svc *UserService) ExchangeGoogleCodeAndGetUserInfo(ctx context.Context, clientID, clientSecret, code, redirectURI string) (*model.GoogleUserInfo, error) {
|
||||||
|
tokenURL := "https://oauth2.googleapis.com/token"
|
||||||
|
formData := url.Values{
|
||||||
|
"client_id": {clientID},
|
||||||
|
"client_secret": {clientSecret},
|
||||||
|
"code": {code},
|
||||||
|
"grant_type": {"authorization_code"},
|
||||||
|
"redirect_uri": {redirectURI},
|
||||||
|
}
|
||||||
|
|
||||||
|
client := svc.googleHTTPClient()
|
||||||
|
resp, err := client.PostForm(tokenURL, formData)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "func", "ExchangeGoogleCodeAndGetUserInfo", "msg", "exchange code failed", "error", err)
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
var tokenResp googleTokenResponse
|
||||||
|
if err := json.NewDecoder(resp.Body).Decode(&tokenResp); err != nil {
|
||||||
|
log.Error(ctx, "func", "ExchangeGoogleCodeAndGetUserInfo", "msg", "decode token response failed", "error", err)
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if tokenResp.AccessToken == "" {
|
||||||
|
log.Error(ctx, "func", "ExchangeGoogleCodeAndGetUserInfo", "msg", "no access token in response")
|
||||||
|
return nil, errors.New("no access token in response")
|
||||||
|
}
|
||||||
|
|
||||||
|
userInfo, err := svc.getGoogleUserInfo(ctx, tokenResp.AccessToken)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "func", "ExchangeGoogleCodeAndGetUserInfo", "msg", "get user info failed", "error", err)
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return &model.GoogleUserInfo{
|
||||||
|
ID: userInfo.ID,
|
||||||
|
Email: userInfo.Email,
|
||||||
|
Name: userInfo.Name,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (svc *UserService) getGoogleUserInfo(ctx context.Context, accessToken string) (*model.GoogleUserInfo, error) {
|
||||||
|
req, err := http.NewRequestWithContext(ctx, "GET", "https://www.googleapis.com/oauth2/v2/userinfo", nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
req.Header.Set("Authorization", "Bearer "+accessToken)
|
||||||
|
|
||||||
|
client := svc.googleHTTPClient()
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
var userInfo model.GoogleUserInfo
|
||||||
|
if err := json.NewDecoder(resp.Body).Decode(&userInfo); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return &userInfo, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (svc *UserService) FindOrCreateGoogleUser(ctx context.Context, userInfo *model.GoogleUserInfo) (uid int64, err error) {
|
||||||
|
existingUser, err := svc.userDao.GetByGoogleID(dao.DB.WithContext(ctx), userInfo.ID)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "func", "FindOrCreateGoogleUser", "msg", "get user by google id error", "error", err)
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if existingUser != nil {
|
||||||
|
return existingUser.ID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
existingUser, err = svc.userDao.GetByEmail(dao.DB.WithContext(ctx), userInfo.Email)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "func", "FindOrCreateGoogleUser", "msg", "get user by email error", "error", err)
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if existingUser != nil {
|
||||||
|
existingUser.GoogleID = userInfo.ID
|
||||||
|
err = svc.userDao.Update(dao.DB.WithContext(ctx), existingUser)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "func", "FindOrCreateGoogleUser", "msg", "update user google id error", "error", err)
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return existingUser.ID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
user := &dao.User{
|
||||||
|
Email: userInfo.Email,
|
||||||
|
GoogleID: userInfo.ID,
|
||||||
|
Username: userInfo.Name,
|
||||||
|
}
|
||||||
|
err = svc.userDao.Create(dao.DB.WithContext(ctx), user)
|
||||||
|
if err != nil {
|
||||||
|
log.Error(ctx, "func", "FindOrCreateGoogleUser", "msg", "create user error", "error", err)
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return user.ID, nil
|
||||||
|
}
|
||||||
|
|||||||
2
internal/storage/cache/engine.go
vendored
2
internal/storage/cache/engine.go
vendored
@@ -5,7 +5,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.com/bitwsd/document_ai/config"
|
"gitea.com/texpixel/document_ai/config"
|
||||||
"github.com/redis/go-redis/v9"
|
"github.com/redis/go-redis/v9"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
170
internal/storage/dao/analytics_event.go
Normal file
170
internal/storage/dao/analytics_event.go
Normal file
@@ -0,0 +1,170 @@
|
|||||||
|
package dao
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gorm.io/datatypes"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
"gorm.io/gorm/clause"
|
||||||
|
)
|
||||||
|
|
||||||
|
// AnalyticsEvent 数据埋点事件表
|
||||||
|
type AnalyticsEvent struct {
|
||||||
|
ID int64 `gorm:"bigint;primaryKey;autoIncrement;column:id;comment:主键ID" json:"id"`
|
||||||
|
UserID int64 `gorm:"column:user_id;not null;index:idx_user_id;comment:用户ID" json:"user_id"`
|
||||||
|
EventName string `gorm:"column:event_name;varchar(128);not null;index:idx_event_name;comment:事件名称" json:"event_name"`
|
||||||
|
Properties datatypes.JSON `gorm:"column:properties;type:json;comment:事件属性(JSON)" json:"properties"`
|
||||||
|
DeviceInfo datatypes.JSON `gorm:"column:device_info;type:json;comment:设备信息(JSON)" json:"device_info"`
|
||||||
|
MetaData datatypes.JSON `gorm:"column:meta_data;type:json;comment:元数据(JSON,包含task_id等)" json:"meta_data"`
|
||||||
|
CreatedAt time.Time `gorm:"column:created_at;comment:创建时间;not null;default:current_timestamp;index:idx_created_at" json:"created_at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *AnalyticsEvent) TableName() string {
|
||||||
|
return "analytics_events"
|
||||||
|
}
|
||||||
|
|
||||||
|
// AnalyticsEventDao 数据埋点事件DAO
|
||||||
|
type AnalyticsEventDao struct{}
|
||||||
|
|
||||||
|
func NewAnalyticsEventDao() *AnalyticsEventDao {
|
||||||
|
return &AnalyticsEventDao{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create 创建事件记录
|
||||||
|
func (dao *AnalyticsEventDao) Create(tx *gorm.DB, event *AnalyticsEvent) error {
|
||||||
|
return tx.Create(event).Error
|
||||||
|
}
|
||||||
|
|
||||||
|
// BatchCreate 批量创建事件记录
|
||||||
|
func (dao *AnalyticsEventDao) BatchCreate(tx *gorm.DB, events []*AnalyticsEvent) error {
|
||||||
|
if len(events) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return tx.CreateInBatches(events, 100).Error
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetByID 根据ID获取事件
|
||||||
|
func (dao *AnalyticsEventDao) GetByID(tx *gorm.DB, id int64) (*AnalyticsEvent, error) {
|
||||||
|
event := &AnalyticsEvent{}
|
||||||
|
err := tx.Where("id = ?", id).First(event).Error
|
||||||
|
if err != nil {
|
||||||
|
if err == gorm.ErrRecordNotFound {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return event, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetUserEvents 获取用户的事件列表
|
||||||
|
func (dao *AnalyticsEventDao) GetUserEvents(tx *gorm.DB, userID int64, page, pageSize int) ([]*AnalyticsEvent, int64, error) {
|
||||||
|
var events []*AnalyticsEvent
|
||||||
|
var total int64
|
||||||
|
|
||||||
|
offset := (page - 1) * pageSize
|
||||||
|
query := tx.Model(&AnalyticsEvent{}).Where("user_id = ?", userID)
|
||||||
|
|
||||||
|
err := query.Count(&total).Error
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = query.Offset(offset).Limit(pageSize).
|
||||||
|
Order(clause.OrderByColumn{Column: clause.Column{Name: "created_at"}, Desc: true}).
|
||||||
|
Find(&events).Error
|
||||||
|
|
||||||
|
return events, total, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetEventsByName 根据事件名称获取事件列表
|
||||||
|
func (dao *AnalyticsEventDao) GetEventsByName(tx *gorm.DB, eventName string, page, pageSize int) ([]*AnalyticsEvent, int64, error) {
|
||||||
|
var events []*AnalyticsEvent
|
||||||
|
var total int64
|
||||||
|
|
||||||
|
offset := (page - 1) * pageSize
|
||||||
|
query := tx.Model(&AnalyticsEvent{}).Where("event_name = ?", eventName)
|
||||||
|
|
||||||
|
err := query.Count(&total).Error
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = query.Offset(offset).Limit(pageSize).
|
||||||
|
Order(clause.OrderByColumn{Column: clause.Column{Name: "created_at"}, Desc: true}).
|
||||||
|
Find(&events).Error
|
||||||
|
|
||||||
|
return events, total, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetUserEventsByName 获取用户指定事件的列表
|
||||||
|
func (dao *AnalyticsEventDao) GetUserEventsByName(tx *gorm.DB, userID int64, eventName string, page, pageSize int) ([]*AnalyticsEvent, int64, error) {
|
||||||
|
var events []*AnalyticsEvent
|
||||||
|
var total int64
|
||||||
|
|
||||||
|
offset := (page - 1) * pageSize
|
||||||
|
query := tx.Model(&AnalyticsEvent{}).Where("user_id = ? AND event_name = ?", userID, eventName)
|
||||||
|
|
||||||
|
err := query.Count(&total).Error
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = query.Offset(offset).Limit(pageSize).
|
||||||
|
Order(clause.OrderByColumn{Column: clause.Column{Name: "created_at"}, Desc: true}).
|
||||||
|
Find(&events).Error
|
||||||
|
|
||||||
|
return events, total, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetEventsByTimeRange 根据时间范围获取事件列表
|
||||||
|
func (dao *AnalyticsEventDao) GetEventsByTimeRange(tx *gorm.DB, startTime, endTime time.Time, page, pageSize int) ([]*AnalyticsEvent, int64, error) {
|
||||||
|
var events []*AnalyticsEvent
|
||||||
|
var total int64
|
||||||
|
|
||||||
|
offset := (page - 1) * pageSize
|
||||||
|
query := tx.Model(&AnalyticsEvent{}).Where("created_at BETWEEN ? AND ?", startTime, endTime)
|
||||||
|
|
||||||
|
err := query.Count(&total).Error
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = query.Offset(offset).Limit(pageSize).
|
||||||
|
Order(clause.OrderByColumn{Column: clause.Column{Name: "created_at"}, Desc: true}).
|
||||||
|
Find(&events).Error
|
||||||
|
|
||||||
|
return events, total, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// CountEventsByName 统计指定事件的数量
|
||||||
|
func (dao *AnalyticsEventDao) CountEventsByName(tx *gorm.DB, eventName string) (int64, error) {
|
||||||
|
var count int64
|
||||||
|
err := tx.Model(&AnalyticsEvent{}).Where("event_name = ?", eventName).Count(&count).Error
|
||||||
|
return count, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// CountUserEvents 统计用户的事件数量
|
||||||
|
func (dao *AnalyticsEventDao) CountUserEvents(tx *gorm.DB, userID int64) (int64, error) {
|
||||||
|
var count int64
|
||||||
|
err := tx.Model(&AnalyticsEvent{}).Where("user_id = ?", userID).Count(&count).Error
|
||||||
|
return count, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetEventStats 获取事件统计信息(按事件名称分组)
|
||||||
|
func (dao *AnalyticsEventDao) GetEventStats(tx *gorm.DB, startTime, endTime time.Time) ([]map[string]interface{}, error) {
|
||||||
|
var results []map[string]interface{}
|
||||||
|
|
||||||
|
err := tx.Model(&AnalyticsEvent{}).
|
||||||
|
Select("event_name, COUNT(*) as count, COUNT(DISTINCT user_id) as unique_users").
|
||||||
|
Where("created_at BETWEEN ? AND ?", startTime, endTime).
|
||||||
|
Group("event_name").
|
||||||
|
Order("count DESC").
|
||||||
|
Find(&results).Error
|
||||||
|
|
||||||
|
return results, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeleteOldEvents 删除旧事件(数据清理)
|
||||||
|
func (dao *AnalyticsEventDao) DeleteOldEvents(tx *gorm.DB, beforeTime time.Time) error {
|
||||||
|
return tx.Where("created_at < ?", beforeTime).Delete(&AnalyticsEvent{}).Error
|
||||||
|
}
|
||||||
@@ -3,16 +3,19 @@ package dao
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"gitea.com/bitwsd/document_ai/config"
|
"gitea.com/texpixel/document_ai/config"
|
||||||
"gorm.io/driver/mysql"
|
"gorm.io/driver/mysql"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
|
"gorm.io/gorm/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
var DB *gorm.DB
|
var DB *gorm.DB
|
||||||
|
|
||||||
func InitDB(conf config.DatabaseConfig) {
|
func InitDB(conf config.DatabaseConfig) {
|
||||||
dns := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Asia%%2FShanghai", conf.Username, conf.Password, conf.Host, conf.Port, conf.DBName)
|
dns := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Asia%%2FShanghai", conf.Username, conf.Password, conf.Host, conf.Port, conf.DBName)
|
||||||
db, err := gorm.Open(mysql.Open(dns), &gorm.Config{})
|
db, err := gorm.Open(mysql.Open(dns), &gorm.Config{
|
||||||
|
Logger: logger.Default.LogMode(logger.Silent), // 禁用 GORM 日志输出
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ const (
|
|||||||
ProviderMathpix RecognitionLogProvider = "mathpix"
|
ProviderMathpix RecognitionLogProvider = "mathpix"
|
||||||
ProviderSiliconflow RecognitionLogProvider = "siliconflow"
|
ProviderSiliconflow RecognitionLogProvider = "siliconflow"
|
||||||
ProviderTexpixel RecognitionLogProvider = "texpixel"
|
ProviderTexpixel RecognitionLogProvider = "texpixel"
|
||||||
|
ProviderBaiduOCR RecognitionLogProvider = "baidu_ocr"
|
||||||
)
|
)
|
||||||
|
|
||||||
// RecognitionLog 识别调用日志表,记录第三方API调用请求和响应
|
// RecognitionLog 识别调用日志表,记录第三方API调用请求和响应
|
||||||
|
|||||||
@@ -9,8 +9,9 @@ type RecognitionResult struct {
|
|||||||
TaskID int64 `gorm:"column:task_id;bigint;not null;default:0;comment:任务ID" json:"task_id"`
|
TaskID int64 `gorm:"column:task_id;bigint;not null;default:0;comment:任务ID" json:"task_id"`
|
||||||
TaskType TaskType `gorm:"column:task_type;varchar(16);not null;comment:任务类型;default:''" json:"task_type"`
|
TaskType TaskType `gorm:"column:task_type;varchar(16);not null;comment:任务类型;default:''" json:"task_type"`
|
||||||
Latex string `json:"latex" gorm:"column:latex;type:text;not null;default:''"`
|
Latex string `json:"latex" gorm:"column:latex;type:text;not null;default:''"`
|
||||||
Markdown string `json:"markdown" gorm:"column:markdown;type:text;not null;default:''"` // Mathpix Markdown 格式
|
Markdown string `json:"markdown" gorm:"column:markdown;type:text;not null;default:''"` // Markdown 格式
|
||||||
MathML string `json:"mathml" gorm:"column:mathml;type:text;not null;default:''"` // MathML 格式
|
MathML string `json:"mathml" gorm:"column:mathml;type:text;not null;default:''"` // MathML 格式
|
||||||
|
MML string `json:"mml" gorm:"column:mml;type:text;not null;default:''"` // MML 格式
|
||||||
}
|
}
|
||||||
|
|
||||||
type RecognitionResultDao struct {
|
type RecognitionResultDao struct {
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ type User struct {
|
|||||||
Password string `gorm:"column:password" json:"password"`
|
Password string `gorm:"column:password" json:"password"`
|
||||||
WechatOpenID string `gorm:"column:wechat_open_id" json:"wechat_open_id"`
|
WechatOpenID string `gorm:"column:wechat_open_id" json:"wechat_open_id"`
|
||||||
WechatUnionID string `gorm:"column:wechat_union_id" json:"wechat_union_id"`
|
WechatUnionID string `gorm:"column:wechat_union_id" json:"wechat_union_id"`
|
||||||
|
GoogleID string `gorm:"column:google_id" json:"google_id"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (u *User) TableName() string {
|
func (u *User) TableName() string {
|
||||||
@@ -63,3 +64,18 @@ func (dao *UserDao) GetByEmail(tx *gorm.DB, email string) (*User, error) {
|
|||||||
}
|
}
|
||||||
return &user, nil
|
return &user, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (dao *UserDao) GetByGoogleID(tx *gorm.DB, googleID string) (*User, error) {
|
||||||
|
var user User
|
||||||
|
if err := tx.Where("google_id = ?", googleID).First(&user).Error; err != nil {
|
||||||
|
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &user, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (dao *UserDao) Update(tx *gorm.DB, user *User) error {
|
||||||
|
return tx.Save(user).Error
|
||||||
|
}
|
||||||
|
|||||||
23
main.go
23
main.go
@@ -10,23 +10,26 @@ import (
|
|||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.com/bitwsd/document_ai/api"
|
"gitea.com/texpixel/document_ai/api"
|
||||||
"gitea.com/bitwsd/document_ai/config"
|
"gitea.com/texpixel/document_ai/config"
|
||||||
"gitea.com/bitwsd/document_ai/internal/storage/cache"
|
"gitea.com/texpixel/document_ai/internal/storage/cache"
|
||||||
"gitea.com/bitwsd/document_ai/internal/storage/dao"
|
"gitea.com/texpixel/document_ai/internal/storage/dao"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/common"
|
"gitea.com/texpixel/document_ai/pkg/common"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/cors"
|
"gitea.com/texpixel/document_ai/pkg/cors"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/log"
|
"gitea.com/texpixel/document_ai/pkg/log"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/middleware"
|
"gitea.com/texpixel/document_ai/pkg/middleware"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/sms"
|
"gitea.com/texpixel/document_ai/pkg/sms"
|
||||||
"github.com/gin-gonic/gin"
|
"github.com/gin-gonic/gin"
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
// 加载配置
|
// 加载配置
|
||||||
env := "dev"
|
env := ""
|
||||||
flag.StringVar(&env, "env", "dev", "environment (dev/prod)")
|
flag.StringVar(&env, "env", "dev", "environment (dev/prod)")
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
|
fmt.Println("env:", env)
|
||||||
|
|
||||||
configPath := fmt.Sprintf("./config/config_%s.yaml", env)
|
configPath := fmt.Sprintf("./config/config_%s.yaml", env)
|
||||||
if err := config.Init(configPath); err != nil {
|
if err := config.Init(configPath); err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
|
|||||||
18
migrations/analytics_events.sql
Normal file
18
migrations/analytics_events.sql
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
-- 数据埋点事件表
|
||||||
|
CREATE TABLE IF NOT EXISTS `analytics_events` (
|
||||||
|
`id` BIGINT NOT NULL AUTO_INCREMENT COMMENT '主键ID',
|
||||||
|
`user_id` BIGINT NOT NULL COMMENT '用户ID',
|
||||||
|
`event_name` VARCHAR(128) NOT NULL COMMENT '事件名称',
|
||||||
|
`properties` JSON DEFAULT NULL COMMENT '事件属性(JSON)',
|
||||||
|
`device_info` JSON DEFAULT NULL COMMENT '设备信息(JSON)',
|
||||||
|
`meta_data` JSON DEFAULT NULL COMMENT '元数据(JSON,包含task_id等)',
|
||||||
|
`created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
||||||
|
PRIMARY KEY (`id`),
|
||||||
|
INDEX `idx_user_id` (`user_id`),
|
||||||
|
INDEX `idx_event_name` (`event_name`),
|
||||||
|
INDEX `idx_created_at` (`created_at`)
|
||||||
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='数据埋点事件表';
|
||||||
|
|
||||||
|
-- 创建复合索引以提高查询性能
|
||||||
|
CREATE INDEX `idx_user_event` ON `analytics_events` (`user_id`, `event_name`);
|
||||||
|
CREATE INDEX `idx_event_time` ON `analytics_events` (`event_name`, `created_at`);
|
||||||
@@ -6,8 +6,8 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.com/bitwsd/document_ai/pkg/constant"
|
"gitea.com/texpixel/document_ai/pkg/constant"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/jwt"
|
"gitea.com/texpixel/document_ai/pkg/jwt"
|
||||||
"github.com/gin-gonic/gin"
|
"github.com/gin-gonic/gin"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ package common
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
|
||||||
"gitea.com/bitwsd/document_ai/pkg/constant"
|
"gitea.com/texpixel/document_ai/pkg/constant"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Response struct {
|
type Response struct {
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.com/bitwsd/document_ai/pkg/log"
|
"gitea.com/texpixel/document_ai/pkg/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
// RetryConfig 重试配置
|
// RetryConfig 重试配置
|
||||||
@@ -23,9 +23,9 @@ type RetryConfig struct {
|
|||||||
|
|
||||||
// DefaultRetryConfig 默认重试配置
|
// DefaultRetryConfig 默认重试配置
|
||||||
var DefaultRetryConfig = RetryConfig{
|
var DefaultRetryConfig = RetryConfig{
|
||||||
MaxRetries: 2,
|
MaxRetries: 1,
|
||||||
InitialInterval: 100 * time.Millisecond,
|
InitialInterval: 100 * time.Millisecond,
|
||||||
MaxInterval: 5 * time.Second,
|
MaxInterval: 30 * time.Second,
|
||||||
SkipTLSVerify: true,
|
SkipTLSVerify: true,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ import (
|
|||||||
"runtime"
|
"runtime"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gitea.com/texpixel/document_ai/pkg/requestid"
|
||||||
|
|
||||||
"github.com/rs/zerolog"
|
"github.com/rs/zerolog"
|
||||||
"gopkg.in/natefinch/lumberjack.v2"
|
"gopkg.in/natefinch/lumberjack.v2"
|
||||||
)
|
)
|
||||||
@@ -67,8 +69,13 @@ func log(ctx context.Context, level zerolog.Level, logType LogType, kv ...interf
|
|||||||
// 添加日志类型
|
// 添加日志类型
|
||||||
event.Str("type", string(logType))
|
event.Str("type", string(logType))
|
||||||
|
|
||||||
// 添加请求ID
|
reqID := requestid.GetRequestID()
|
||||||
if reqID, exists := ctx.Value("request_id").(string); exists {
|
if reqID == "" {
|
||||||
|
if id, exists := ctx.Value("request_id").(string); exists {
|
||||||
|
reqID = id
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if reqID != "" {
|
||||||
event.Str("request_id", reqID)
|
event.Str("request_id", reqID)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -149,4 +156,3 @@ func Fatal(ctx context.Context, kv ...interface{}) {
|
|||||||
func Access(ctx context.Context, kv ...interface{}) {
|
func Access(ctx context.Context, kv ...interface{}) {
|
||||||
log(ctx, zerolog.InfoLevel, TypeAccess, kv...)
|
log(ctx, zerolog.InfoLevel, TypeAccess, kv...)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.com/bitwsd/document_ai/pkg/log"
|
"gitea.com/texpixel/document_ai/pkg/log"
|
||||||
"github.com/gin-gonic/gin"
|
"github.com/gin-gonic/gin"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -72,4 +72,3 @@ func AccessLog() gin.HandlerFunc {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,19 +1,23 @@
|
|||||||
package middleware
|
package middleware
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"gitea.com/texpixel/document_ai/pkg/requestid"
|
||||||
|
|
||||||
"github.com/gin-gonic/gin"
|
"github.com/gin-gonic/gin"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
)
|
)
|
||||||
|
|
||||||
func RequestID() gin.HandlerFunc {
|
func RequestID() gin.HandlerFunc {
|
||||||
return func(c *gin.Context) {
|
return func(c *gin.Context) {
|
||||||
requestID := c.Request.Header.Get("X-Request-ID")
|
reqID := c.Request.Header.Get("X-Request-ID")
|
||||||
if requestID == "" {
|
if reqID == "" {
|
||||||
requestID = uuid.New().String()
|
reqID = uuid.New().String()
|
||||||
}
|
}
|
||||||
c.Request.Header.Set("X-Request-ID", requestID)
|
c.Request.Header.Set("X-Request-ID", reqID)
|
||||||
c.Set("request_id", requestID)
|
c.Set("request_id", reqID)
|
||||||
|
|
||||||
|
requestid.SetRequestID(reqID, func() {
|
||||||
c.Next()
|
c.Next()
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -12,8 +12,8 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.com/bitwsd/document_ai/config"
|
"gitea.com/texpixel/document_ai/config"
|
||||||
"gitea.com/bitwsd/document_ai/pkg/log"
|
"gitea.com/texpixel/document_ai/pkg/log"
|
||||||
"github.com/aliyun/aliyun-oss-go-sdk/oss"
|
"github.com/aliyun/aliyun-oss-go-sdk/oss"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
27
pkg/requestid/requestid.go
Normal file
27
pkg/requestid/requestid.go
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
package requestid
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/jtolds/gls"
|
||||||
|
)
|
||||||
|
|
||||||
|
// requestIDKey 是 gls 中存储 request_id 的 key
|
||||||
|
var requestIDKey = gls.GenSym()
|
||||||
|
|
||||||
|
// glsMgr 是 gls 管理器
|
||||||
|
var glsMgr = gls.NewContextManager()
|
||||||
|
|
||||||
|
// SetRequestID 在 gls 中设置 request_id,并在 fn 执行期间保持有效
|
||||||
|
func SetRequestID(requestID string, fn func()) {
|
||||||
|
glsMgr.SetValues(gls.Values{requestIDKey: requestID}, fn)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetRequestID 从 gls 中获取当前 goroutine 的 request_id
|
||||||
|
func GetRequestID() string {
|
||||||
|
val, ok := glsMgr.GetValue(requestIDKey)
|
||||||
|
if !ok {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
reqID, _ := val.(string)
|
||||||
|
return reqID
|
||||||
|
}
|
||||||
|
|
||||||
@@ -4,7 +4,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"gitea.com/bitwsd/document_ai/config"
|
"gitea.com/texpixel/document_ai/config"
|
||||||
openapi "github.com/alibabacloud-go/darabonba-openapi/client"
|
openapi "github.com/alibabacloud-go/darabonba-openapi/client"
|
||||||
dysmsapi "github.com/alibabacloud-go/dysmsapi-20170525/v2/client"
|
dysmsapi "github.com/alibabacloud-go/dysmsapi-20170525/v2/client"
|
||||||
aliutil "github.com/alibabacloud-go/tea-utils/service"
|
aliutil "github.com/alibabacloud-go/tea-utils/service"
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ package utils
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
|
||||||
"gitea.com/bitwsd/document_ai/pkg/log"
|
"gitea.com/texpixel/document_ai/pkg/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
func SafeGo(fn func()) {
|
func SafeGo(fn func()) {
|
||||||
|
|||||||
Reference in New Issue
Block a user