feat: 支持基于 crontab 的定时账号测试
每个测试计划绑定一个账号和一个模型，按 cron 表达式定期执行测试， 保存历史结果并在前端账号管理页面中提供完整的增删改查和结果查看功能。 主要变更： - 新增 scheduled_test_plans / scheduled_test_results 两张表及迁移 - 后端 service 层：CRUD 服务 + 后台 cron runner（每分钟扫描到期计划并发执行） - RunTestBackground 方法通过 httptest 在内存中执行账号测试并解析 SSE 输出 - Redis leader lock + pg_try_advisory_lock 双重保障多实例部署只执行一次 - REST API：5 个管理端点（计划 CRUD + 结果查询） - 前端 ScheduledTestsPanel 组件：计划管理、启用开关、内联编辑、结果展开查看 - 中英文 i18n 支持 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
guoyongchang
@@ -12,6 +12,7 @@ import (
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -1560,3 +1561,65 @@ func (s *AccountTestService) sendErrorAndEnd(c *gin.Context, errorMsg string) er
|
||||
s.sendEvent(c, TestEvent{Type: "error", Error: errorMsg})
|
||||
return fmt.Errorf("%s", errorMsg)
|
||||
}
|
||||
|
||||
// RunTestBackground executes an account test in-memory (no real HTTP client),
|
||||
// capturing SSE output via httptest.NewRecorder, then parses the result.
|
||||
func (s *AccountTestService) RunTestBackground(ctx context.Context, accountID int64, modelID string) (*ScheduledTestOutcome, error) {
|
||||
startedAt := time.Now()
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
ginCtx, _ := gin.CreateTestContext(w)
|
||||
ginCtx.Request = (&http.Request{}).WithContext(ctx)
|
||||
|
||||
testErr := s.TestAccountConnection(ginCtx, accountID, modelID)
|
||||
|
||||
finishedAt := time.Now()
|
||||
latencyMs := finishedAt.Sub(startedAt).Milliseconds()
|
||||
|
||||
body := w.Body.String()
|
||||
responseText, errMsg := parseTestSSEOutput(body)
|
||||
|
||||
outcome := &ScheduledTestOutcome{
|
||||
Status: "success",
|
||||
ResponseText: responseText,
|
||||
ErrorMessage: errMsg,
|
||||
LatencyMs: latencyMs,
|
||||
StartedAt: startedAt,
|
||||
FinishedAt: finishedAt,
|
||||
}
|
||||
|
||||
if testErr != nil || errMsg != "" {
|
||||
outcome.Status = "failed"
|
||||
if errMsg == "" && testErr != nil {
|
||||
outcome.ErrorMessage = testErr.Error()
|
||||
}
|
||||
}
|
||||
|
||||
return outcome, nil
|
||||
}
|
||||
|
||||
// parseTestSSEOutput extracts response text and error message from captured SSE output.
|
||||
func parseTestSSEOutput(body string) (responseText, errMsg string) {
|
||||
var texts []string
|
||||
for _, line := range strings.Split(body, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if !strings.HasPrefix(line, "data: ") {
|
||||
continue
|
||||
}
|
||||
jsonStr := strings.TrimPrefix(line, "data: ")
|
||||
var event TestEvent
|
||||
if err := json.Unmarshal([]byte(jsonStr), &event); err != nil {
|
||||
continue
|
||||
}
|
||||
switch event.Type {
|
||||
case "content":
|
||||
if event.Text != "" {
|
||||
texts = append(texts, event.Text)
|
||||
}
|
||||
case "error":
|
||||
errMsg = event.Error
|
||||
}
|
||||
}
|
||||
responseText = strings.Join(texts, "")
|
||||
return
|
||||
}
|
||||
|
||||
61
backend/internal/service/scheduled_test_port.go
Normal file
61
backend/internal/service/scheduled_test_port.go
Normal file
@@ -0,0 +1,61 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ScheduledTestPlan represents a scheduled test plan domain model.
|
||||
type ScheduledTestPlan struct {
|
||||
ID int64 `json:"id"`
|
||||
AccountID int64 `json:"account_id"`
|
||||
ModelID string `json:"model_id"`
|
||||
CronExpression string `json:"cron_expression"`
|
||||
Enabled bool `json:"enabled"`
|
||||
MaxResults int `json:"max_results"`
|
||||
LastRunAt *time.Time `json:"last_run_at"`
|
||||
NextRunAt *time.Time `json:"next_run_at"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// ScheduledTestResult represents a single test execution result.
|
||||
type ScheduledTestResult struct {
|
||||
ID int64 `json:"id"`
|
||||
PlanID int64 `json:"plan_id"`
|
||||
Status string `json:"status"`
|
||||
ResponseText string `json:"response_text"`
|
||||
ErrorMessage string `json:"error_message"`
|
||||
LatencyMs int64 `json:"latency_ms"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
FinishedAt time.Time `json:"finished_at"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
// ScheduledTestOutcome is returned by RunTestBackground.
|
||||
type ScheduledTestOutcome struct {
|
||||
Status string
|
||||
ResponseText string
|
||||
ErrorMessage string
|
||||
LatencyMs int64
|
||||
StartedAt time.Time
|
||||
FinishedAt time.Time
|
||||
}
|
||||
|
||||
// ScheduledTestPlanRepository defines the data access interface for test plans.
|
||||
type ScheduledTestPlanRepository interface {
|
||||
Create(ctx context.Context, plan *ScheduledTestPlan) (*ScheduledTestPlan, error)
|
||||
GetByID(ctx context.Context, id int64) (*ScheduledTestPlan, error)
|
||||
ListByAccountID(ctx context.Context, accountID int64) ([]*ScheduledTestPlan, error)
|
||||
ListDue(ctx context.Context, now time.Time) ([]*ScheduledTestPlan, error)
|
||||
Update(ctx context.Context, plan *ScheduledTestPlan) (*ScheduledTestPlan, error)
|
||||
Delete(ctx context.Context, id int64) error
|
||||
UpdateAfterRun(ctx context.Context, id int64, lastRunAt time.Time, nextRunAt time.Time) error
|
||||
}
|
||||
|
||||
// ScheduledTestResultRepository defines the data access interface for test results.
|
||||
type ScheduledTestResultRepository interface {
|
||||
Create(ctx context.Context, result *ScheduledTestResult) (*ScheduledTestResult, error)
|
||||
ListByPlanID(ctx context.Context, planID int64, limit int) ([]*ScheduledTestResult, error)
|
||||
PruneOldResults(ctx context.Context, planID int64, keepCount int) error
|
||||
}
|
||||
207
backend/internal/service/scheduled_test_runner_service.go
Normal file
207
backend/internal/service/scheduled_test_runner_service.go
Normal file
@@ -0,0 +1,207 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/config"
|
||||
"github.com/Wei-Shaw/sub2api/internal/pkg/logger"
|
||||
"github.com/google/uuid"
|
||||
"github.com/redis/go-redis/v9"
|
||||
"github.com/robfig/cron/v3"
|
||||
)
|
||||
|
||||
const (
|
||||
scheduledTestLeaderLockKey = "scheduled_test:runner:leader"
|
||||
scheduledTestLeaderLockTTL = 2 * time.Minute
|
||||
scheduledTestDefaultMaxWorkers = 10
|
||||
)
|
||||
|
||||
var scheduledTestReleaseScript = redis.NewScript(`
|
||||
if redis.call("GET", KEYS[1]) == ARGV[1] then
|
||||
return redis.call("DEL", KEYS[1])
|
||||
end
|
||||
return 0
|
||||
`)
|
||||
|
||||
// ScheduledTestRunnerService periodically scans due test plans and executes them.
|
||||
type ScheduledTestRunnerService struct {
|
||||
planRepo ScheduledTestPlanRepository
|
||||
scheduledSvc *ScheduledTestService
|
||||
accountTestSvc *AccountTestService
|
||||
db *sql.DB
|
||||
redisClient *redis.Client
|
||||
cfg *config.Config
|
||||
|
||||
instanceID string
|
||||
cron *cron.Cron
|
||||
startOnce sync.Once
|
||||
stopOnce sync.Once
|
||||
|
||||
warnNoRedisOnce sync.Once
|
||||
}
|
||||
|
||||
// NewScheduledTestRunnerService creates a new runner.
|
||||
func NewScheduledTestRunnerService(
|
||||
planRepo ScheduledTestPlanRepository,
|
||||
scheduledSvc *ScheduledTestService,
|
||||
accountTestSvc *AccountTestService,
|
||||
db *sql.DB,
|
||||
redisClient *redis.Client,
|
||||
cfg *config.Config,
|
||||
) *ScheduledTestRunnerService {
|
||||
return &ScheduledTestRunnerService{
|
||||
planRepo: planRepo,
|
||||
scheduledSvc: scheduledSvc,
|
||||
accountTestSvc: accountTestSvc,
|
||||
db: db,
|
||||
redisClient: redisClient,
|
||||
cfg: cfg,
|
||||
instanceID: uuid.NewString(),
|
||||
}
|
||||
}
|
||||
|
||||
// Start begins the cron ticker (every minute).
|
||||
func (s *ScheduledTestRunnerService) Start() {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.startOnce.Do(func() {
|
||||
loc := time.Local
|
||||
if s.cfg != nil {
|
||||
if parsed, err := time.LoadLocation(s.cfg.Timezone); err == nil && parsed != nil {
|
||||
loc = parsed
|
||||
}
|
||||
}
|
||||
|
||||
c := cron.New(cron.WithParser(scheduledTestCronParser), cron.WithLocation(loc))
|
||||
_, err := c.AddFunc("* * * * *", func() { s.runScheduled() })
|
||||
if err != nil {
|
||||
logger.LegacyPrintf("service.scheduled_test_runner", "[ScheduledTestRunner] not started (invalid schedule): %v", err)
|
||||
return
|
||||
}
|
||||
s.cron = c
|
||||
s.cron.Start()
|
||||
logger.LegacyPrintf("service.scheduled_test_runner", "[ScheduledTestRunner] started (tick=every minute)")
|
||||
})
|
||||
}
|
||||
|
||||
// Stop gracefully shuts down the cron scheduler.
|
||||
func (s *ScheduledTestRunnerService) Stop() {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.stopOnce.Do(func() {
|
||||
if s.cron != nil {
|
||||
ctx := s.cron.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(3 * time.Second):
|
||||
logger.LegacyPrintf("service.scheduled_test_runner", "[ScheduledTestRunner] cron stop timed out")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func (s *ScheduledTestRunnerService) runScheduled() {
|
||||
// Delay 10s so execution lands at ~:10 of each minute instead of :00.
|
||||
time.Sleep(10 * time.Second)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
release, ok := s.tryAcquireLeaderLock(ctx)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if release != nil {
|
||||
defer release()
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
plans, err := s.planRepo.ListDue(ctx, now)
|
||||
if err != nil {
|
||||
logger.LegacyPrintf("service.scheduled_test_runner", "[ScheduledTestRunner] ListDue error: %v", err)
|
||||
return
|
||||
}
|
||||
if len(plans) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
logger.LegacyPrintf("service.scheduled_test_runner", "[ScheduledTestRunner] found %d due plans", len(plans))
|
||||
|
||||
maxWorkers := scheduledTestDefaultMaxWorkers
|
||||
sem := make(chan struct{}, maxWorkers)
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for _, plan := range plans {
|
||||
sem <- struct{}{}
|
||||
wg.Add(1)
|
||||
go func(p *ScheduledTestPlan) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
s.runOnePlan(ctx, p)
|
||||
}(plan)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func (s *ScheduledTestRunnerService) runOnePlan(ctx context.Context, plan *ScheduledTestPlan) {
|
||||
outcome, err := s.accountTestSvc.RunTestBackground(ctx, plan.AccountID, plan.ModelID)
|
||||
if err != nil {
|
||||
logger.LegacyPrintf("service.scheduled_test_runner", "[ScheduledTestRunner] plan=%d RunTestBackground error: %v", plan.ID, err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := s.scheduledSvc.SaveResult(ctx, plan.ID, plan.MaxResults, outcome); err != nil {
|
||||
logger.LegacyPrintf("service.scheduled_test_runner", "[ScheduledTestRunner] plan=%d SaveResult error: %v", plan.ID, err)
|
||||
}
|
||||
|
||||
// Compute next run
|
||||
nextRun, err := computeNextRun(plan.CronExpression, time.Now())
|
||||
if err != nil {
|
||||
logger.LegacyPrintf("service.scheduled_test_runner", "[ScheduledTestRunner] plan=%d computeNextRun error: %v", plan.ID, err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := s.planRepo.UpdateAfterRun(ctx, plan.ID, time.Now(), nextRun); err != nil {
|
||||
logger.LegacyPrintf("service.scheduled_test_runner", "[ScheduledTestRunner] plan=%d UpdateAfterRun error: %v", plan.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ScheduledTestRunnerService) tryAcquireLeaderLock(ctx context.Context) (func(), bool) {
|
||||
if s.cfg != nil && s.cfg.RunMode == config.RunModeSimple {
|
||||
return nil, true
|
||||
}
|
||||
|
||||
key := scheduledTestLeaderLockKey
|
||||
ttl := scheduledTestLeaderLockTTL
|
||||
|
||||
if s.redisClient != nil {
|
||||
ok, err := s.redisClient.SetNX(ctx, key, s.instanceID, ttl).Result()
|
||||
if err == nil {
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
return func() {
|
||||
_, _ = scheduledTestReleaseScript.Run(ctx, s.redisClient, []string{key}, s.instanceID).Result()
|
||||
}, true
|
||||
}
|
||||
s.warnNoRedisOnce.Do(func() {
|
||||
logger.LegacyPrintf("service.scheduled_test_runner", "[ScheduledTestRunner] Redis SetNX failed; falling back to DB advisory lock: %v", err)
|
||||
})
|
||||
} else {
|
||||
s.warnNoRedisOnce.Do(func() {
|
||||
logger.LegacyPrintf("service.scheduled_test_runner", "[ScheduledTestRunner] Redis not configured; using DB advisory lock")
|
||||
})
|
||||
}
|
||||
|
||||
release, ok := tryAcquireDBAdvisoryLock(ctx, s.db, hashAdvisoryLockID(key))
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
return release, true
|
||||
}
|
||||
102
backend/internal/service/scheduled_test_service.go
Normal file
102
backend/internal/service/scheduled_test_service.go
Normal file
@@ -0,0 +1,102 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/robfig/cron/v3"
|
||||
)
|
||||
|
||||
var scheduledTestCronParser = cron.NewParser(cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)
|
||||
|
||||
// ScheduledTestService provides CRUD operations for scheduled test plans and results.
|
||||
type ScheduledTestService struct {
|
||||
planRepo ScheduledTestPlanRepository
|
||||
resultRepo ScheduledTestResultRepository
|
||||
}
|
||||
|
||||
// NewScheduledTestService creates a new ScheduledTestService.
|
||||
func NewScheduledTestService(
|
||||
planRepo ScheduledTestPlanRepository,
|
||||
resultRepo ScheduledTestResultRepository,
|
||||
) *ScheduledTestService {
|
||||
return &ScheduledTestService{
|
||||
planRepo: planRepo,
|
||||
resultRepo: resultRepo,
|
||||
}
|
||||
}
|
||||
|
||||
// CreatePlan validates the cron expression, computes next_run_at, and persists the plan.
|
||||
func (s *ScheduledTestService) CreatePlan(ctx context.Context, plan *ScheduledTestPlan) (*ScheduledTestPlan, error) {
|
||||
nextRun, err := computeNextRun(plan.CronExpression, time.Now())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid cron expression: %w", err)
|
||||
}
|
||||
plan.NextRunAt = &nextRun
|
||||
|
||||
if plan.MaxResults <= 0 {
|
||||
plan.MaxResults = 50
|
||||
}
|
||||
|
||||
return s.planRepo.Create(ctx, plan)
|
||||
}
|
||||
|
||||
// GetPlan retrieves a plan by ID.
|
||||
func (s *ScheduledTestService) GetPlan(ctx context.Context, id int64) (*ScheduledTestPlan, error) {
|
||||
return s.planRepo.GetByID(ctx, id)
|
||||
}
|
||||
|
||||
// ListPlansByAccount returns all plans for a given account.
|
||||
func (s *ScheduledTestService) ListPlansByAccount(ctx context.Context, accountID int64) ([]*ScheduledTestPlan, error) {
|
||||
return s.planRepo.ListByAccountID(ctx, accountID)
|
||||
}
|
||||
|
||||
// UpdatePlan validates cron and updates the plan.
|
||||
func (s *ScheduledTestService) UpdatePlan(ctx context.Context, plan *ScheduledTestPlan) (*ScheduledTestPlan, error) {
|
||||
nextRun, err := computeNextRun(plan.CronExpression, time.Now())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid cron expression: %w", err)
|
||||
}
|
||||
plan.NextRunAt = &nextRun
|
||||
|
||||
return s.planRepo.Update(ctx, plan)
|
||||
}
|
||||
|
||||
// DeletePlan removes a plan and its results (via CASCADE).
|
||||
func (s *ScheduledTestService) DeletePlan(ctx context.Context, id int64) error {
|
||||
return s.planRepo.Delete(ctx, id)
|
||||
}
|
||||
|
||||
// ListResults returns the most recent results for a plan.
|
||||
func (s *ScheduledTestService) ListResults(ctx context.Context, planID int64, limit int) ([]*ScheduledTestResult, error) {
|
||||
if limit <= 0 {
|
||||
limit = 50
|
||||
}
|
||||
return s.resultRepo.ListByPlanID(ctx, planID, limit)
|
||||
}
|
||||
|
||||
// SaveResult inserts a result and prunes old entries beyond maxResults.
|
||||
func (s *ScheduledTestService) SaveResult(ctx context.Context, planID int64, maxResults int, outcome *ScheduledTestOutcome) error {
|
||||
result := &ScheduledTestResult{
|
||||
PlanID: planID,
|
||||
Status: outcome.Status,
|
||||
ResponseText: outcome.ResponseText,
|
||||
ErrorMessage: outcome.ErrorMessage,
|
||||
LatencyMs: outcome.LatencyMs,
|
||||
StartedAt: outcome.StartedAt,
|
||||
FinishedAt: outcome.FinishedAt,
|
||||
}
|
||||
if _, err := s.resultRepo.Create(ctx, result); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.resultRepo.PruneOldResults(ctx, planID, maxResults)
|
||||
}
|
||||
|
||||
func computeNextRun(cronExpr string, from time.Time) (time.Time, error) {
|
||||
sched, err := scheduledTestCronParser.Parse(cronExpr)
|
||||
if err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
return sched.Next(from), nil
|
||||
}
|
||||
@@ -274,6 +274,28 @@ func ProvideIdempotencyCleanupService(repo IdempotencyRepository, cfg *config.Co
|
||||
return svc
|
||||
}
|
||||
|
||||
// ProvideScheduledTestService creates ScheduledTestService.
|
||||
func ProvideScheduledTestService(
|
||||
planRepo ScheduledTestPlanRepository,
|
||||
resultRepo ScheduledTestResultRepository,
|
||||
) *ScheduledTestService {
|
||||
return NewScheduledTestService(planRepo, resultRepo)
|
||||
}
|
||||
|
||||
// ProvideScheduledTestRunnerService creates and starts ScheduledTestRunnerService.
|
||||
func ProvideScheduledTestRunnerService(
|
||||
planRepo ScheduledTestPlanRepository,
|
||||
scheduledSvc *ScheduledTestService,
|
||||
accountTestSvc *AccountTestService,
|
||||
db *sql.DB,
|
||||
redisClient *redis.Client,
|
||||
cfg *config.Config,
|
||||
) *ScheduledTestRunnerService {
|
||||
svc := NewScheduledTestRunnerService(planRepo, scheduledSvc, accountTestSvc, db, redisClient, cfg)
|
||||
svc.Start()
|
||||
return svc
|
||||
}
|
||||
|
||||
// ProvideOpsScheduledReportService creates and starts OpsScheduledReportService.
|
||||
func ProvideOpsScheduledReportService(
|
||||
opsService *OpsService,
|
||||
@@ -380,4 +402,6 @@ var ProviderSet = wire.NewSet(
|
||||
ProvideIdempotencyCoordinator,
|
||||
ProvideSystemOperationLockService,
|
||||
ProvideIdempotencyCleanupService,
|
||||
ProvideScheduledTestService,
|
||||
ProvideScheduledTestRunnerService,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user