feat(task): introduce task timeout configuration and cleanup unfinished tasks

- Added TaskTimeoutMinutes constant to configure the timeout duration for asynchronous tasks.
- Implemented sweepTimedOutTasks function to identify and handle unfinished tasks that exceed the timeout limit, marking them as failed and processing refunds if applicable.
- Enhanced task polling loop to include the new timeout handling logic, ensuring timely cleanup of stale tasks.
This commit is contained in:
CaIon
2026-02-22 17:59:38 +08:00
parent fb494c12d6
commit bc7c5cf9cf
4 changed files with 75 additions and 0 deletions

View File

@@ -35,12 +35,65 @@ type TaskPollingAdaptor interface {
// 打破 service -> relay -> relay/channel -> service 的循环依赖。
var GetTaskAdaptorFunc func(platform constant.TaskPlatform) TaskPollingAdaptor
// sweepTimedOutTasks 在主轮询之前独立清理超时任务。
// 每次最多处理 100 条,剩余的下个周期继续处理。
// 使用 per-task CAS (UpdateWithStatus) 防止覆盖被正常轮询已推进的任务。
func sweepTimedOutTasks(ctx context.Context) {
if constant.TaskTimeoutMinutes <= 0 {
return
}
cutoff := time.Now().Unix() - int64(constant.TaskTimeoutMinutes)*60
tasks := model.GetTimedOutUnfinishedTasks(cutoff, 100)
if len(tasks) == 0 {
return
}
const legacyTaskCutoff int64 = 1740182400 // 2026-02-22 00:00:00 UTC
reason := fmt.Sprintf("任务超时(%d分钟", constant.TaskTimeoutMinutes)
legacyReason := "任务超时(旧系统遗留任务,不进行退款,请联系管理员)"
now := time.Now().Unix()
timedOutCount := 0
for _, task := range tasks {
isLegacy := task.SubmitTime > 0 && task.SubmitTime < legacyTaskCutoff
oldStatus := task.Status
task.Status = model.TaskStatusFailure
task.Progress = "100%"
task.FinishTime = now
if isLegacy {
task.FailReason = legacyReason
} else {
task.FailReason = reason
}
won, err := task.UpdateWithStatus(oldStatus)
if err != nil {
logger.LogError(ctx, fmt.Sprintf("sweepTimedOutTasks CAS update error for task %s: %v", task.TaskID, err))
continue
}
if !won {
logger.LogInfo(ctx, fmt.Sprintf("sweepTimedOutTasks: task %s already transitioned, skip", task.TaskID))
continue
}
timedOutCount++
if !isLegacy && task.Quota != 0 {
RefundTaskQuota(ctx, task, reason)
}
}
if timedOutCount > 0 {
logger.LogInfo(ctx, fmt.Sprintf("sweepTimedOutTasks: timed out %d tasks", timedOutCount))
}
}
// TaskPollingLoop 主轮询循环,每 15 秒检查一次未完成的任务
func TaskPollingLoop() {
for {
time.Sleep(time.Duration(15) * time.Second)
common.SysLog("任务进度轮询开始")
ctx := context.TODO()
sweepTimedOutTasks(ctx)
allTasks := model.GetAllUnFinishSyncTasks(constant.TaskQueryLimit)
platformTask := make(map[constant.TaskPlatform][]*model.Task)
for _, t := range allTasks {