Merge branch 'feature/pdf-recognition' into test

This commit is contained in:
2026-03-31 19:30:35 +08:00
31 changed files with 1146 additions and 114 deletions

27
internal/storage/cache/pdf.go vendored Normal file
View File

@@ -0,0 +1,27 @@
package cache
import (
"context"
"strconv"
)
const (
PDFRecognitionTaskQueue = "pdf_recognition_queue"
PDFRecognitionDistLock = "pdf_recognition_dist_lock"
)
func PushPDFTask(ctx context.Context, taskID int64) (int64, error) {
return RedisClient.LPush(ctx, PDFRecognitionTaskQueue, taskID).Result()
}
func PopPDFTask(ctx context.Context) (int64, error) {
result, err := RedisClient.BRPop(ctx, 0, PDFRecognitionTaskQueue).Result()
if err != nil {
return 0, err
}
return strconv.ParseInt(result[1], 10, 64)
}
func GetPDFDistributedLock(ctx context.Context) (bool, error) {
return RedisClient.SetNX(ctx, PDFRecognitionDistLock, "locked", DefaultLockTimeout).Result()
}

View File

@@ -88,10 +88,6 @@ func SetUserEmailCode(ctx context.Context, email, code string) error {
return RedisClient.Set(ctx, fmt.Sprintf(UserEmailCodePrefix, email), code, UserEmailCodeTTL).Err()
}
func DeleteUserEmailCode(ctx context.Context, email string) error {
return RedisClient.Del(ctx, fmt.Sprintf(UserEmailCodePrefix, email)).Err()
}
func GetUserSendEmailLimit(ctx context.Context, email string) (int, error) {
limit, err := RedisClient.Get(ctx, fmt.Sprintf(UserSendEmailLimit, email)).Result()
if err != nil {
@@ -104,13 +100,16 @@ func GetUserSendEmailLimit(ctx context.Context, email string) (int, error) {
}
func SetUserSendEmailLimit(ctx context.Context, email string) error {
key := fmt.Sprintf(UserSendEmailLimit, email)
count, err := RedisClient.Incr(ctx, key).Result()
count, err := RedisClient.Incr(ctx, fmt.Sprintf(UserSendEmailLimit, email)).Result()
if err != nil {
return err
}
if count > UserSendEmailLimitCount {
return errors.New("send email limit")
}
return RedisClient.Expire(ctx, key, UserSendEmailLimitTTL).Err()
return RedisClient.Expire(ctx, fmt.Sprintf(UserSendEmailLimit, email), UserSendEmailLimitTTL).Err()
}
func DeleteUserEmailCode(ctx context.Context, email string) error {
return RedisClient.Del(ctx, fmt.Sprintf(UserEmailCodePrefix, email)).Err()
}

View File

@@ -0,0 +1,50 @@
package dao
import (
"gorm.io/gorm"
)
type EmailSendStatus int8
const (
EmailSendStatusSent EmailSendStatus = 0 // 已发送,用户未注册
EmailSendStatusRegistered EmailSendStatus = 1 // 用户已完成注册
)
type EmailSendLog struct {
BaseModel
Email string `gorm:"column:email;type:varchar(255);not null;comment:邮箱地址" json:"email"`
Status EmailSendStatus `gorm:"column:status;type:tinyint;not null;default:0;comment:状态: 0=已发送未注册 1=已注册" json:"status"`
}
func (e *EmailSendLog) TableName() string {
return "email_send_log"
}
type EmailSendLogDao struct{}
func NewEmailSendLogDao() *EmailSendLogDao {
return &EmailSendLogDao{}
}
func (d *EmailSendLogDao) Create(tx *gorm.DB, log *EmailSendLog) error {
return tx.Create(log).Error
}
func (d *EmailSendLogDao) GetLatestByEmail(tx *gorm.DB, email string) (*EmailSendLog, error) {
var record EmailSendLog
err := tx.Where("email = ?", email).Order("id DESC").First(&record).Error
if err != nil {
if err == gorm.ErrRecordNotFound {
return nil, nil
}
return nil, err
}
return &record, nil
}
func (d *EmailSendLogDao) MarkRegistered(tx *gorm.DB, email string) error {
return tx.Model(&EmailSendLog{}).
Where("email = ? AND status = ?", email, EmailSendStatusSent).
Update("status", EmailSendStatusRegistered).Error
}

View File

@@ -1,45 +1,104 @@
package dao
import (
"encoding/json"
"gorm.io/gorm"
)
type RecognitionResult struct {
BaseModel
TaskID int64 `gorm:"column:task_id;bigint;not null;default:0;comment:任务ID" json:"task_id"`
TaskType TaskType `gorm:"column:task_type;varchar(16);not null;comment:任务类型;default:''" json:"task_type"`
Latex string `json:"latex" gorm:"column:latex;type:text;not null;default:''"`
Markdown string `json:"markdown" gorm:"column:markdown;type:text;not null;default:''"` // Markdown 格式
MathML string `json:"mathml" gorm:"column:mathml;type:text;not null;default:''"` // MathML 格式
MML string `json:"mml" gorm:"column:mml;type:text;not null;default:''"` // MML 格式
// FormulaContent 公式识别的 content 字段结构
type FormulaContent struct {
Latex string `json:"latex"`
Markdown string `json:"markdown"`
MathML string `json:"mathml"`
MML string `json:"mml"`
}
type RecognitionResultDao struct {
// PDFPageContent PDF 单页识别结果
type PDFPageContent struct {
PageNumber int `json:"page_number"`
Markdown string `json:"markdown"`
}
// ResultMetaData recognition_results.meta_data 字段结构
type ResultMetaData struct {
TotalNum int `json:"total_num"`
}
// RecognitionResult recognition_results 表模型
type RecognitionResult struct {
BaseModel
TaskID int64 `gorm:"column:task_id;bigint;not null;default:0;index;comment:任务ID" json:"task_id"`
TaskType TaskType `gorm:"column:task_type;varchar(16);not null;comment:任务类型;default:''" json:"task_type"`
MetaData string `gorm:"column:meta_data;type:json;comment:元数据" json:"meta_data"`
Content string `gorm:"column:content;type:json;comment:识别内容JSON" json:"content"`
}
// SetMetaData 序列化并写入 MetaData 字段
func (r *RecognitionResult) SetMetaData(meta ResultMetaData) error {
b, err := json.Marshal(meta)
if err != nil {
return err
}
r.MetaData = string(b)
return nil
}
// GetFormulaContent 从 Content 字段反序列化公式结果
func (r *RecognitionResult) GetFormulaContent() (*FormulaContent, error) {
var c FormulaContent
if err := json.Unmarshal([]byte(r.Content), &c); err != nil {
return nil, err
}
return &c, nil
}
// GetPDFContent 从 Content 字段反序列化 PDF 分页结果
func (r *RecognitionResult) GetPDFContent() ([]PDFPageContent, error) {
var pages []PDFPageContent
if err := json.Unmarshal([]byte(r.Content), &pages); err != nil {
return nil, err
}
return pages, nil
}
// MarshalFormulaContent 将公式结果序列化为 JSON 字符串(供写入 Content
func MarshalFormulaContent(c FormulaContent) (string, error) {
b, err := json.Marshal(c)
return string(b), err
}
// MarshalPDFContent 将 PDF 分页结果序列化为 JSON 字符串(供写入 Content
func MarshalPDFContent(pages []PDFPageContent) (string, error) {
b, err := json.Marshal(pages)
return string(b), err
}
type RecognitionResultDao struct{}
func NewRecognitionResultDao() *RecognitionResultDao {
return &RecognitionResultDao{}
}
// 模型方法
func (dao *RecognitionResultDao) Create(tx *gorm.DB, data RecognitionResult) error {
return tx.Create(&data).Error
}
func (dao *RecognitionResultDao) GetByTaskID(tx *gorm.DB, taskID int64) (result *RecognitionResult, err error) {
result = &RecognitionResult{}
err = tx.Where("task_id = ?", taskID).First(result).Error
func (dao *RecognitionResultDao) GetByTaskID(tx *gorm.DB, taskID int64) (*RecognitionResult, error) {
result := &RecognitionResult{}
err := tx.Where("task_id = ?", taskID).First(result).Error
if err != nil && err == gorm.ErrRecordNotFound {
return nil, nil
}
return
}
func (dao *RecognitionResultDao) GetByTaskIDs(tx *gorm.DB, taskIDs []int64) (results []*RecognitionResult, err error) {
err = tx.Where("task_id IN (?)", taskIDs).Find(&results).Error
return
return result, err
}
func (dao *RecognitionResultDao) Update(tx *gorm.DB, id int64, updates map[string]interface{}) error {
return tx.Model(&RecognitionResult{}).Where("id = ?", id).Updates(updates).Error
}
func (dao *RecognitionResultDao) GetByTaskIDs(tx *gorm.DB, taskIDs []int64) ([]*RecognitionResult, error) {
var results []*RecognitionResult
err := tx.Where("task_id IN (?)", taskIDs).Find(&results).Error
return results, err
}

View File

@@ -20,6 +20,7 @@ const (
TaskTypeText TaskType = "TEXT"
TaskTypeTable TaskType = "TABLE"
TaskTypeLayout TaskType = "LAYOUT"
TaskTypePDF TaskType = "PDF"
)
func (t TaskType) String() string {