Merge branch 'feature/pdf-recognition' into test
This commit is contained in:
27
internal/storage/cache/pdf.go
vendored
Normal file
27
internal/storage/cache/pdf.go
vendored
Normal file
@@ -0,0 +1,27 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
const (
|
||||
PDFRecognitionTaskQueue = "pdf_recognition_queue"
|
||||
PDFRecognitionDistLock = "pdf_recognition_dist_lock"
|
||||
)
|
||||
|
||||
func PushPDFTask(ctx context.Context, taskID int64) (int64, error) {
|
||||
return RedisClient.LPush(ctx, PDFRecognitionTaskQueue, taskID).Result()
|
||||
}
|
||||
|
||||
func PopPDFTask(ctx context.Context) (int64, error) {
|
||||
result, err := RedisClient.BRPop(ctx, 0, PDFRecognitionTaskQueue).Result()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return strconv.ParseInt(result[1], 10, 64)
|
||||
}
|
||||
|
||||
func GetPDFDistributedLock(ctx context.Context) (bool, error) {
|
||||
return RedisClient.SetNX(ctx, PDFRecognitionDistLock, "locked", DefaultLockTimeout).Result()
|
||||
}
|
||||
13
internal/storage/cache/user.go
vendored
13
internal/storage/cache/user.go
vendored
@@ -88,10 +88,6 @@ func SetUserEmailCode(ctx context.Context, email, code string) error {
|
||||
return RedisClient.Set(ctx, fmt.Sprintf(UserEmailCodePrefix, email), code, UserEmailCodeTTL).Err()
|
||||
}
|
||||
|
||||
func DeleteUserEmailCode(ctx context.Context, email string) error {
|
||||
return RedisClient.Del(ctx, fmt.Sprintf(UserEmailCodePrefix, email)).Err()
|
||||
}
|
||||
|
||||
func GetUserSendEmailLimit(ctx context.Context, email string) (int, error) {
|
||||
limit, err := RedisClient.Get(ctx, fmt.Sprintf(UserSendEmailLimit, email)).Result()
|
||||
if err != nil {
|
||||
@@ -104,13 +100,16 @@ func GetUserSendEmailLimit(ctx context.Context, email string) (int, error) {
|
||||
}
|
||||
|
||||
func SetUserSendEmailLimit(ctx context.Context, email string) error {
|
||||
key := fmt.Sprintf(UserSendEmailLimit, email)
|
||||
count, err := RedisClient.Incr(ctx, key).Result()
|
||||
count, err := RedisClient.Incr(ctx, fmt.Sprintf(UserSendEmailLimit, email)).Result()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if count > UserSendEmailLimitCount {
|
||||
return errors.New("send email limit")
|
||||
}
|
||||
return RedisClient.Expire(ctx, key, UserSendEmailLimitTTL).Err()
|
||||
return RedisClient.Expire(ctx, fmt.Sprintf(UserSendEmailLimit, email), UserSendEmailLimitTTL).Err()
|
||||
}
|
||||
|
||||
func DeleteUserEmailCode(ctx context.Context, email string) error {
|
||||
return RedisClient.Del(ctx, fmt.Sprintf(UserEmailCodePrefix, email)).Err()
|
||||
}
|
||||
|
||||
50
internal/storage/dao/email_send_log.go
Normal file
50
internal/storage/dao/email_send_log.go
Normal file
@@ -0,0 +1,50 @@
|
||||
package dao
|
||||
|
||||
import (
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type EmailSendStatus int8
|
||||
|
||||
const (
|
||||
EmailSendStatusSent EmailSendStatus = 0 // 已发送,用户未注册
|
||||
EmailSendStatusRegistered EmailSendStatus = 1 // 用户已完成注册
|
||||
)
|
||||
|
||||
type EmailSendLog struct {
|
||||
BaseModel
|
||||
Email string `gorm:"column:email;type:varchar(255);not null;comment:邮箱地址" json:"email"`
|
||||
Status EmailSendStatus `gorm:"column:status;type:tinyint;not null;default:0;comment:状态: 0=已发送未注册 1=已注册" json:"status"`
|
||||
}
|
||||
|
||||
func (e *EmailSendLog) TableName() string {
|
||||
return "email_send_log"
|
||||
}
|
||||
|
||||
type EmailSendLogDao struct{}
|
||||
|
||||
func NewEmailSendLogDao() *EmailSendLogDao {
|
||||
return &EmailSendLogDao{}
|
||||
}
|
||||
|
||||
func (d *EmailSendLogDao) Create(tx *gorm.DB, log *EmailSendLog) error {
|
||||
return tx.Create(log).Error
|
||||
}
|
||||
|
||||
func (d *EmailSendLogDao) GetLatestByEmail(tx *gorm.DB, email string) (*EmailSendLog, error) {
|
||||
var record EmailSendLog
|
||||
err := tx.Where("email = ?", email).Order("id DESC").First(&record).Error
|
||||
if err != nil {
|
||||
if err == gorm.ErrRecordNotFound {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return &record, nil
|
||||
}
|
||||
|
||||
func (d *EmailSendLogDao) MarkRegistered(tx *gorm.DB, email string) error {
|
||||
return tx.Model(&EmailSendLog{}).
|
||||
Where("email = ? AND status = ?", email, EmailSendStatusSent).
|
||||
Update("status", EmailSendStatusRegistered).Error
|
||||
}
|
||||
@@ -1,45 +1,104 @@
|
||||
package dao
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type RecognitionResult struct {
|
||||
BaseModel
|
||||
TaskID int64 `gorm:"column:task_id;bigint;not null;default:0;comment:任务ID" json:"task_id"`
|
||||
TaskType TaskType `gorm:"column:task_type;varchar(16);not null;comment:任务类型;default:''" json:"task_type"`
|
||||
Latex string `json:"latex" gorm:"column:latex;type:text;not null;default:''"`
|
||||
Markdown string `json:"markdown" gorm:"column:markdown;type:text;not null;default:''"` // Markdown 格式
|
||||
MathML string `json:"mathml" gorm:"column:mathml;type:text;not null;default:''"` // MathML 格式
|
||||
MML string `json:"mml" gorm:"column:mml;type:text;not null;default:''"` // MML 格式
|
||||
// FormulaContent 公式识别的 content 字段结构
|
||||
type FormulaContent struct {
|
||||
Latex string `json:"latex"`
|
||||
Markdown string `json:"markdown"`
|
||||
MathML string `json:"mathml"`
|
||||
MML string `json:"mml"`
|
||||
}
|
||||
|
||||
type RecognitionResultDao struct {
|
||||
// PDFPageContent PDF 单页识别结果
|
||||
type PDFPageContent struct {
|
||||
PageNumber int `json:"page_number"`
|
||||
Markdown string `json:"markdown"`
|
||||
}
|
||||
|
||||
// ResultMetaData recognition_results.meta_data 字段结构
|
||||
type ResultMetaData struct {
|
||||
TotalNum int `json:"total_num"`
|
||||
}
|
||||
|
||||
// RecognitionResult recognition_results 表模型
|
||||
type RecognitionResult struct {
|
||||
BaseModel
|
||||
TaskID int64 `gorm:"column:task_id;bigint;not null;default:0;index;comment:任务ID" json:"task_id"`
|
||||
TaskType TaskType `gorm:"column:task_type;varchar(16);not null;comment:任务类型;default:''" json:"task_type"`
|
||||
MetaData string `gorm:"column:meta_data;type:json;comment:元数据" json:"meta_data"`
|
||||
Content string `gorm:"column:content;type:json;comment:识别内容JSON" json:"content"`
|
||||
}
|
||||
|
||||
// SetMetaData 序列化并写入 MetaData 字段
|
||||
func (r *RecognitionResult) SetMetaData(meta ResultMetaData) error {
|
||||
b, err := json.Marshal(meta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r.MetaData = string(b)
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetFormulaContent 从 Content 字段反序列化公式结果
|
||||
func (r *RecognitionResult) GetFormulaContent() (*FormulaContent, error) {
|
||||
var c FormulaContent
|
||||
if err := json.Unmarshal([]byte(r.Content), &c); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
// GetPDFContent 从 Content 字段反序列化 PDF 分页结果
|
||||
func (r *RecognitionResult) GetPDFContent() ([]PDFPageContent, error) {
|
||||
var pages []PDFPageContent
|
||||
if err := json.Unmarshal([]byte(r.Content), &pages); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return pages, nil
|
||||
}
|
||||
|
||||
// MarshalFormulaContent 将公式结果序列化为 JSON 字符串(供写入 Content)
|
||||
func MarshalFormulaContent(c FormulaContent) (string, error) {
|
||||
b, err := json.Marshal(c)
|
||||
return string(b), err
|
||||
}
|
||||
|
||||
// MarshalPDFContent 将 PDF 分页结果序列化为 JSON 字符串(供写入 Content)
|
||||
func MarshalPDFContent(pages []PDFPageContent) (string, error) {
|
||||
b, err := json.Marshal(pages)
|
||||
return string(b), err
|
||||
}
|
||||
|
||||
type RecognitionResultDao struct{}
|
||||
|
||||
func NewRecognitionResultDao() *RecognitionResultDao {
|
||||
return &RecognitionResultDao{}
|
||||
}
|
||||
|
||||
// 模型方法
|
||||
func (dao *RecognitionResultDao) Create(tx *gorm.DB, data RecognitionResult) error {
|
||||
return tx.Create(&data).Error
|
||||
}
|
||||
|
||||
func (dao *RecognitionResultDao) GetByTaskID(tx *gorm.DB, taskID int64) (result *RecognitionResult, err error) {
|
||||
result = &RecognitionResult{}
|
||||
err = tx.Where("task_id = ?", taskID).First(result).Error
|
||||
func (dao *RecognitionResultDao) GetByTaskID(tx *gorm.DB, taskID int64) (*RecognitionResult, error) {
|
||||
result := &RecognitionResult{}
|
||||
err := tx.Where("task_id = ?", taskID).First(result).Error
|
||||
if err != nil && err == gorm.ErrRecordNotFound {
|
||||
return nil, nil
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (dao *RecognitionResultDao) GetByTaskIDs(tx *gorm.DB, taskIDs []int64) (results []*RecognitionResult, err error) {
|
||||
err = tx.Where("task_id IN (?)", taskIDs).Find(&results).Error
|
||||
return
|
||||
return result, err
|
||||
}
|
||||
|
||||
func (dao *RecognitionResultDao) Update(tx *gorm.DB, id int64, updates map[string]interface{}) error {
|
||||
return tx.Model(&RecognitionResult{}).Where("id = ?", id).Updates(updates).Error
|
||||
}
|
||||
|
||||
func (dao *RecognitionResultDao) GetByTaskIDs(tx *gorm.DB, taskIDs []int64) ([]*RecognitionResult, error) {
|
||||
var results []*RecognitionResult
|
||||
err := tx.Where("task_id IN (?)", taskIDs).Find(&results).Error
|
||||
return results, err
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ const (
|
||||
TaskTypeText TaskType = "TEXT"
|
||||
TaskTypeTable TaskType = "TABLE"
|
||||
TaskTypeLayout TaskType = "LAYOUT"
|
||||
TaskTypePDF TaskType = "PDF"
|
||||
)
|
||||
|
||||
func (t TaskType) String() string {
|
||||
|
||||
Reference in New Issue
Block a user