//go:build unit package service import ( "bytes" "context" "io" "net/http" "strings" "testing" "github.com/stretchr/testify/require" ) // stubSmartRetryCache 用于 handleSmartRetry 测试的 GatewayCache mock // 仅关注 DeleteSessionAccountID 的调用记录 type stubSmartRetryCache struct { GatewayCache // 嵌入接口,未实现的方法 panic(确保只调用预期方法) deleteCalls []deleteSessionCall } type deleteSessionCall struct { groupID int64 sessionHash string } func (c *stubSmartRetryCache) DeleteSessionAccountID(_ context.Context, groupID int64, sessionHash string) error { c.deleteCalls = append(c.deleteCalls, deleteSessionCall{groupID: groupID, sessionHash: sessionHash}) return nil } // mockSmartRetryUpstream 用于 handleSmartRetry 测试的 mock upstream type mockSmartRetryUpstream struct { responses []*http.Response errors []error callIdx int calls []string } func (m *mockSmartRetryUpstream) Do(req *http.Request, proxyURL string, accountID int64, accountConcurrency int) (*http.Response, error) { idx := m.callIdx m.calls = append(m.calls, req.URL.String()) m.callIdx++ if idx < len(m.responses) { return m.responses[idx], m.errors[idx] } return nil, nil } func (m *mockSmartRetryUpstream) DoWithTLS(req *http.Request, proxyURL string, accountID int64, accountConcurrency int, enableTLSFingerprint bool) (*http.Response, error) { return m.Do(req, proxyURL, accountID, accountConcurrency) } // TestHandleSmartRetry_URLLevelRateLimit 测试 URL 级别限流切换 func TestHandleSmartRetry_URLLevelRateLimit(t *testing.T) { account := &Account{ ID: 1, Name: "acc-1", Type: AccountTypeOAuth, Platform: PlatformAntigravity, } respBody := []byte(`{"error":{"message":"Resource has been exhausted"}}`) resp := &http.Response{ StatusCode: http.StatusTooManyRequests, Header: http.Header{}, Body: io.NopCloser(bytes.NewReader(respBody)), } params := antigravityRetryLoopParams{ ctx: context.Background(), prefix: "[test]", account: account, accessToken: "token", action: "generateContent", body: []byte(`{"input":"test"}`), handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult { return nil }, } availableURLs := []string{"https://ag-1.test", "https://ag-2.test"} svc := &AntigravityGatewayService{} result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs) require.NotNil(t, result) require.Equal(t, smartRetryActionContinueURL, result.action) require.Nil(t, result.resp) require.Nil(t, result.err) require.Nil(t, result.switchError) } // TestHandleSmartRetry_LongDelay_ReturnsSwitchError 测试 retryDelay >= 阈值时返回 switchError func TestHandleSmartRetry_LongDelay_ReturnsSwitchError(t *testing.T) { repo := &stubAntigravityAccountRepo{} account := &Account{ ID: 1, Name: "acc-1", Type: AccountTypeOAuth, Platform: PlatformAntigravity, } // 15s >= 7s 阈值,应该返回 switchError respBody := []byte(`{ "error": { "status": "RESOURCE_EXHAUSTED", "details": [ {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "claude-sonnet-4-5"}, "reason": "RATE_LIMIT_EXCEEDED"}, {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "15s"} ] } }`) resp := &http.Response{ StatusCode: http.StatusTooManyRequests, Header: http.Header{}, Body: io.NopCloser(bytes.NewReader(respBody)), } params := antigravityRetryLoopParams{ ctx: context.Background(), prefix: "[test]", account: account, accessToken: "token", action: "generateContent", body: []byte(`{"input":"test"}`), accountRepo: repo, isStickySession: true, handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult { return nil }, } availableURLs := []string{"https://ag-1.test"} svc := &AntigravityGatewayService{} result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs) require.NotNil(t, result) require.Equal(t, smartRetryActionBreakWithResp, result.action) require.Nil(t, result.resp, "should not return resp when switchError is set") require.Nil(t, result.err) require.NotNil(t, result.switchError, "should return switchError for long delay") require.Equal(t, account.ID, result.switchError.OriginalAccountID) require.Equal(t, "claude-sonnet-4-5", result.switchError.RateLimitedModel) require.True(t, result.switchError.IsStickySession) // 验证模型限流已设置 require.Len(t, repo.modelRateLimitCalls, 1) require.Equal(t, "claude-sonnet-4-5", repo.modelRateLimitCalls[0].modelKey) } // TestHandleSmartRetry_ShortDelay_SmartRetrySuccess 测试智能重试成功 func TestHandleSmartRetry_ShortDelay_SmartRetrySuccess(t *testing.T) { successResp := &http.Response{ StatusCode: http.StatusOK, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(`{"result":"ok"}`)), } upstream := &mockSmartRetryUpstream{ responses: []*http.Response{successResp}, errors: []error{nil}, } account := &Account{ ID: 1, Name: "acc-1", Type: AccountTypeOAuth, Platform: PlatformAntigravity, } // 0.5s < 7s 阈值,应该触发智能重试 respBody := []byte(`{ "error": { "status": "RESOURCE_EXHAUSTED", "details": [ {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "claude-opus-4"}, "reason": "RATE_LIMIT_EXCEEDED"}, {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.5s"} ] } }`) resp := &http.Response{ StatusCode: http.StatusTooManyRequests, Header: http.Header{}, Body: io.NopCloser(bytes.NewReader(respBody)), } params := antigravityRetryLoopParams{ ctx: context.Background(), prefix: "[test]", account: account, accessToken: "token", action: "generateContent", body: []byte(`{"input":"test"}`), httpUpstream: upstream, handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult { return nil }, } availableURLs := []string{"https://ag-1.test"} svc := &AntigravityGatewayService{} result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs) require.NotNil(t, result) require.Equal(t, smartRetryActionBreakWithResp, result.action) require.NotNil(t, result.resp, "should return successful response") require.Equal(t, http.StatusOK, result.resp.StatusCode) require.Nil(t, result.err) require.Nil(t, result.switchError, "should not return switchError on success") require.Len(t, upstream.calls, 1, "should have made one retry call") } // TestHandleSmartRetry_ShortDelay_SmartRetryFailed_ReturnsSwitchError 测试智能重试失败后返回 switchError func TestHandleSmartRetry_ShortDelay_SmartRetryFailed_ReturnsSwitchError(t *testing.T) { // 智能重试后仍然返回 429(需要提供 1 个响应,因为智能重试最多 1 次) failRespBody := `{ "error": { "status": "RESOURCE_EXHAUSTED", "details": [ {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "gemini-3-flash"}, "reason": "RATE_LIMIT_EXCEEDED"}, {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"} ] } }` failResp1 := &http.Response{ StatusCode: http.StatusTooManyRequests, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(failRespBody)), } upstream := &mockSmartRetryUpstream{ responses: []*http.Response{failResp1}, errors: []error{nil}, } repo := &stubAntigravityAccountRepo{} account := &Account{ ID: 2, Name: "acc-2", Type: AccountTypeOAuth, Platform: PlatformAntigravity, } // 3s < 7s 阈值,应该触发智能重试(最多 1 次) respBody := []byte(`{ "error": { "status": "RESOURCE_EXHAUSTED", "details": [ {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "gemini-3-flash"}, "reason": "RATE_LIMIT_EXCEEDED"}, {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"} ] } }`) resp := &http.Response{ StatusCode: http.StatusTooManyRequests, Header: http.Header{}, Body: io.NopCloser(bytes.NewReader(respBody)), } params := antigravityRetryLoopParams{ ctx: context.Background(), prefix: "[test]", account: account, accessToken: "token", action: "generateContent", body: []byte(`{"input":"test"}`), httpUpstream: upstream, accountRepo: repo, isStickySession: false, handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult { return nil }, } availableURLs := []string{"https://ag-1.test"} svc := &AntigravityGatewayService{} result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs) require.NotNil(t, result) require.Equal(t, smartRetryActionBreakWithResp, result.action) require.Nil(t, result.resp, "should not return resp when switchError is set") require.Nil(t, result.err) require.NotNil(t, result.switchError, "should return switchError after smart retry failed") require.Equal(t, account.ID, result.switchError.OriginalAccountID) require.Equal(t, "gemini-3-flash", result.switchError.RateLimitedModel) require.False(t, result.switchError.IsStickySession) // 验证模型限流已设置 require.Len(t, repo.modelRateLimitCalls, 1) require.Equal(t, "gemini-3-flash", repo.modelRateLimitCalls[0].modelKey) require.Len(t, upstream.calls, 1, "should have made one retry call (max attempts)") } // TestHandleSmartRetry_503_ModelCapacityExhausted_ReturnsSwitchError 测试 503 MODEL_CAPACITY_EXHAUSTED 返回 switchError func TestHandleSmartRetry_503_ModelCapacityExhausted_ReturnsSwitchError(t *testing.T) { repo := &stubAntigravityAccountRepo{} account := &Account{ ID: 3, Name: "acc-3", Type: AccountTypeOAuth, Platform: PlatformAntigravity, } // 503 + MODEL_CAPACITY_EXHAUSTED + 39s >= 7s 阈值 respBody := []byte(`{ "error": { "code": 503, "status": "UNAVAILABLE", "details": [ {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "gemini-3-pro-high"}, "reason": "MODEL_CAPACITY_EXHAUSTED"}, {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "39s"} ], "message": "No capacity available for model gemini-3-pro-high on the server" } }`) resp := &http.Response{ StatusCode: http.StatusServiceUnavailable, Header: http.Header{}, Body: io.NopCloser(bytes.NewReader(respBody)), } params := antigravityRetryLoopParams{ ctx: context.Background(), prefix: "[test]", account: account, accessToken: "token", action: "generateContent", body: []byte(`{"input":"test"}`), accountRepo: repo, isStickySession: true, handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult { return nil }, } availableURLs := []string{"https://ag-1.test"} svc := &AntigravityGatewayService{} result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs) require.NotNil(t, result) require.Equal(t, smartRetryActionBreakWithResp, result.action) require.Nil(t, result.resp) require.Nil(t, result.err) require.NotNil(t, result.switchError, "should return switchError for 503 model capacity exhausted") require.Equal(t, account.ID, result.switchError.OriginalAccountID) require.Equal(t, "gemini-3-pro-high", result.switchError.RateLimitedModel) require.True(t, result.switchError.IsStickySession) // 验证模型限流已设置 require.Len(t, repo.modelRateLimitCalls, 1) require.Equal(t, "gemini-3-pro-high", repo.modelRateLimitCalls[0].modelKey) } // TestHandleSmartRetry_NonAntigravityAccount_ContinuesDefaultLogic 测试非 Antigravity 平台账号走默认逻辑 func TestHandleSmartRetry_NonAntigravityAccount_ContinuesDefaultLogic(t *testing.T) { account := &Account{ ID: 4, Name: "acc-4", Type: AccountTypeAPIKey, // 非 Antigravity 平台账号 Platform: PlatformAnthropic, } // 即使是模型限流响应,非 OAuth 账号也应该走默认逻辑 respBody := []byte(`{ "error": { "status": "RESOURCE_EXHAUSTED", "details": [ {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "claude-sonnet-4-5"}, "reason": "RATE_LIMIT_EXCEEDED"}, {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "15s"} ] } }`) resp := &http.Response{ StatusCode: http.StatusTooManyRequests, Header: http.Header{}, Body: io.NopCloser(bytes.NewReader(respBody)), } params := antigravityRetryLoopParams{ ctx: context.Background(), prefix: "[test]", account: account, accessToken: "token", action: "generateContent", body: []byte(`{"input":"test"}`), handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult { return nil }, } availableURLs := []string{"https://ag-1.test"} svc := &AntigravityGatewayService{} result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs) require.NotNil(t, result) require.Equal(t, smartRetryActionContinue, result.action, "non-Antigravity platform account should continue default logic") require.Nil(t, result.resp) require.Nil(t, result.err) require.Nil(t, result.switchError) } // TestHandleSmartRetry_NonModelRateLimit_ContinuesDefaultLogic 测试非模型限流响应走默认逻辑 func TestHandleSmartRetry_NonModelRateLimit_ContinuesDefaultLogic(t *testing.T) { account := &Account{ ID: 5, Name: "acc-5", Type: AccountTypeOAuth, Platform: PlatformAntigravity, } // 429 但没有 RATE_LIMIT_EXCEEDED 或 MODEL_CAPACITY_EXHAUSTED respBody := []byte(`{ "error": { "status": "RESOURCE_EXHAUSTED", "details": [ {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "5s"} ], "message": "Quota exceeded" } }`) resp := &http.Response{ StatusCode: http.StatusTooManyRequests, Header: http.Header{}, Body: io.NopCloser(bytes.NewReader(respBody)), } params := antigravityRetryLoopParams{ ctx: context.Background(), prefix: "[test]", account: account, accessToken: "token", action: "generateContent", body: []byte(`{"input":"test"}`), handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult { return nil }, } availableURLs := []string{"https://ag-1.test"} svc := &AntigravityGatewayService{} result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs) require.NotNil(t, result) require.Equal(t, smartRetryActionContinue, result.action, "non-model rate limit should continue default logic") require.Nil(t, result.resp) require.Nil(t, result.err) require.Nil(t, result.switchError) } // TestHandleSmartRetry_ExactlyAtThreshold_ReturnsSwitchError 测试刚好等于阈值时返回 switchError func TestHandleSmartRetry_ExactlyAtThreshold_ReturnsSwitchError(t *testing.T) { repo := &stubAntigravityAccountRepo{} account := &Account{ ID: 6, Name: "acc-6", Type: AccountTypeOAuth, Platform: PlatformAntigravity, } // 刚好 7s = 7s 阈值,应该返回 switchError respBody := []byte(`{ "error": { "status": "RESOURCE_EXHAUSTED", "details": [ {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "gemini-pro"}, "reason": "RATE_LIMIT_EXCEEDED"}, {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "7s"} ] } }`) resp := &http.Response{ StatusCode: http.StatusTooManyRequests, Header: http.Header{}, Body: io.NopCloser(bytes.NewReader(respBody)), } params := antigravityRetryLoopParams{ ctx: context.Background(), prefix: "[test]", account: account, accessToken: "token", action: "generateContent", body: []byte(`{"input":"test"}`), accountRepo: repo, handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult { return nil }, } availableURLs := []string{"https://ag-1.test"} svc := &AntigravityGatewayService{} result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs) require.NotNil(t, result) require.Equal(t, smartRetryActionBreakWithResp, result.action) require.Nil(t, result.resp) require.NotNil(t, result.switchError, "exactly at threshold should return switchError") require.Equal(t, "gemini-pro", result.switchError.RateLimitedModel) } // TestAntigravityRetryLoop_HandleSmartRetry_SwitchError_Propagates 测试 switchError 正确传播到上层 func TestAntigravityRetryLoop_HandleSmartRetry_SwitchError_Propagates(t *testing.T) { // 模拟 429 + 长延迟的响应 respBody := []byte(`{ "error": { "status": "RESOURCE_EXHAUSTED", "details": [ {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "claude-opus-4-6"}, "reason": "RATE_LIMIT_EXCEEDED"}, {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "30s"} ] } }`) rateLimitResp := &http.Response{ StatusCode: http.StatusTooManyRequests, Header: http.Header{}, Body: io.NopCloser(bytes.NewReader(respBody)), } upstream := &mockSmartRetryUpstream{ responses: []*http.Response{rateLimitResp}, errors: []error{nil}, } repo := &stubAntigravityAccountRepo{} account := &Account{ ID: 7, Name: "acc-7", Type: AccountTypeOAuth, Platform: PlatformAntigravity, Schedulable: true, Status: StatusActive, Concurrency: 1, } svc := &AntigravityGatewayService{} result, err := svc.antigravityRetryLoop(antigravityRetryLoopParams{ ctx: context.Background(), prefix: "[test]", account: account, accessToken: "token", action: "generateContent", body: []byte(`{"input":"test"}`), httpUpstream: upstream, accountRepo: repo, isStickySession: true, handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult { return nil }, }) require.Nil(t, result, "should not return result when switchError") require.NotNil(t, err, "should return error") var switchErr *AntigravityAccountSwitchError require.ErrorAs(t, err, &switchErr, "error should be AntigravityAccountSwitchError") require.Equal(t, account.ID, switchErr.OriginalAccountID) require.Equal(t, "claude-opus-4-6", switchErr.RateLimitedModel) require.True(t, switchErr.IsStickySession) } // TestHandleSmartRetry_NetworkError_ExhaustsRetry 测试网络错误时(maxAttempts=1)直接耗尽重试并切换账号 func TestHandleSmartRetry_NetworkError_ExhaustsRetry(t *testing.T) { // 唯一一次重试遇到网络错误(nil response) upstream := &mockSmartRetryUpstream{ responses: []*http.Response{nil}, // 返回 nil(模拟网络错误) errors: []error{nil}, // mock 不返回 error,靠 nil response 触发 } repo := &stubAntigravityAccountRepo{} account := &Account{ ID: 8, Name: "acc-8", Type: AccountTypeOAuth, Platform: PlatformAntigravity, } // 0.1s < 7s 阈值,应该触发智能重试 respBody := []byte(`{ "error": { "status": "RESOURCE_EXHAUSTED", "details": [ {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "claude-sonnet-4-5"}, "reason": "RATE_LIMIT_EXCEEDED"}, {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"} ] } }`) resp := &http.Response{ StatusCode: http.StatusTooManyRequests, Header: http.Header{}, Body: io.NopCloser(bytes.NewReader(respBody)), } params := antigravityRetryLoopParams{ ctx: context.Background(), prefix: "[test]", account: account, accessToken: "token", action: "generateContent", body: []byte(`{"input":"test"}`), httpUpstream: upstream, accountRepo: repo, handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult { return nil }, } availableURLs := []string{"https://ag-1.test"} svc := &AntigravityGatewayService{} result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs) require.NotNil(t, result) require.Equal(t, smartRetryActionBreakWithResp, result.action) require.Nil(t, result.resp, "should not return resp when switchError is set") require.NotNil(t, result.switchError, "should return switchError after network error exhausted retry") require.Equal(t, account.ID, result.switchError.OriginalAccountID) require.Equal(t, "claude-sonnet-4-5", result.switchError.RateLimitedModel) require.Len(t, upstream.calls, 1, "should have made one retry call") // 验证模型限流已设置 require.Len(t, repo.modelRateLimitCalls, 1) require.Equal(t, "claude-sonnet-4-5", repo.modelRateLimitCalls[0].modelKey) } // TestHandleSmartRetry_NoRetryDelay_UsesDefaultRateLimit 测试无 retryDelay 时使用默认 1 分钟限流 func TestHandleSmartRetry_NoRetryDelay_UsesDefaultRateLimit(t *testing.T) { repo := &stubAntigravityAccountRepo{} account := &Account{ ID: 9, Name: "acc-9", Type: AccountTypeOAuth, Platform: PlatformAntigravity, } // 429 + RATE_LIMIT_EXCEEDED + 无 retryDelay → 使用默认 1 分钟限流 respBody := []byte(`{ "error": { "status": "RESOURCE_EXHAUSTED", "details": [ {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "claude-sonnet-4-5"}, "reason": "RATE_LIMIT_EXCEEDED"} ], "message": "You have exhausted your capacity on this model." } }`) resp := &http.Response{ StatusCode: http.StatusTooManyRequests, Header: http.Header{}, Body: io.NopCloser(bytes.NewReader(respBody)), } params := antigravityRetryLoopParams{ ctx: context.Background(), prefix: "[test]", account: account, accessToken: "token", action: "generateContent", body: []byte(`{"input":"test"}`), accountRepo: repo, isStickySession: true, handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult { return nil }, } availableURLs := []string{"https://ag-1.test"} svc := &AntigravityGatewayService{} result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs) require.NotNil(t, result) require.Equal(t, smartRetryActionBreakWithResp, result.action) require.Nil(t, result.resp, "should not return resp when switchError is set") require.NotNil(t, result.switchError, "should return switchError for no retryDelay") require.Equal(t, "claude-sonnet-4-5", result.switchError.RateLimitedModel) require.True(t, result.switchError.IsStickySession) // 验证模型限流已设置 require.Len(t, repo.modelRateLimitCalls, 1) require.Equal(t, "claude-sonnet-4-5", repo.modelRateLimitCalls[0].modelKey) } // --------------------------------------------------------------------------- // 以下测试覆盖本次改动: // 1. antigravitySmartRetryMaxAttempts = 1(仅重试 1 次) // 2. 智能重试失败后清除粘性会话绑定(DeleteSessionAccountID) // --------------------------------------------------------------------------- // TestSmartRetryMaxAttempts_VerifyConstant 验证常量值为 1 func TestSmartRetryMaxAttempts_VerifyConstant(t *testing.T) { require.Equal(t, 1, antigravitySmartRetryMaxAttempts, "antigravitySmartRetryMaxAttempts should be 1 to prevent repeated rate limiting") } // TestHandleSmartRetry_ShortDelay_StickySession_FailedRetry_ClearsSession // 核心场景:粘性会话 + 短延迟重试失败 → 必须清除粘性绑定 func TestHandleSmartRetry_ShortDelay_StickySession_FailedRetry_ClearsSession(t *testing.T) { failRespBody := `{ "error": { "status": "RESOURCE_EXHAUSTED", "details": [ {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "claude-sonnet-4-5"}, "reason": "RATE_LIMIT_EXCEEDED"}, {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"} ] } }` failResp := &http.Response{ StatusCode: http.StatusTooManyRequests, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(failRespBody)), } upstream := &mockSmartRetryUpstream{ responses: []*http.Response{failResp}, errors: []error{nil}, } repo := &stubAntigravityAccountRepo{} cache := &stubSmartRetryCache{} account := &Account{ ID: 10, Name: "acc-10", Type: AccountTypeOAuth, Platform: PlatformAntigravity, } respBody := []byte(`{ "error": { "status": "RESOURCE_EXHAUSTED", "details": [ {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "claude-sonnet-4-5"}, "reason": "RATE_LIMIT_EXCEEDED"}, {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"} ] } }`) resp := &http.Response{ StatusCode: http.StatusTooManyRequests, Header: http.Header{}, Body: io.NopCloser(bytes.NewReader(respBody)), } params := antigravityRetryLoopParams{ ctx: context.Background(), prefix: "[test]", account: account, accessToken: "token", action: "generateContent", body: []byte(`{"input":"test"}`), httpUpstream: upstream, accountRepo: repo, isStickySession: true, groupID: 42, sessionHash: "sticky-hash-abc", handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult { return nil }, } availableURLs := []string{"https://ag-1.test"} svc := &AntigravityGatewayService{cache: cache} result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs) // 验证返回 switchError require.NotNil(t, result) require.Equal(t, smartRetryActionBreakWithResp, result.action) require.NotNil(t, result.switchError) require.True(t, result.switchError.IsStickySession, "switchError should carry IsStickySession=true") require.Equal(t, account.ID, result.switchError.OriginalAccountID) // 核心断言:DeleteSessionAccountID 被调用,且参数正确 require.Len(t, cache.deleteCalls, 1, "should call DeleteSessionAccountID exactly once") require.Equal(t, int64(42), cache.deleteCalls[0].groupID) require.Equal(t, "sticky-hash-abc", cache.deleteCalls[0].sessionHash) // 验证仅重试 1 次 require.Len(t, upstream.calls, 1, "should make exactly 1 retry call (maxAttempts=1)") // 验证模型限流已设置 require.Len(t, repo.modelRateLimitCalls, 1) require.Equal(t, "claude-sonnet-4-5", repo.modelRateLimitCalls[0].modelKey) } // TestHandleSmartRetry_ShortDelay_NonStickySession_FailedRetry_NoDeleteSession // 非粘性会话 + 短延迟重试失败 → 不应调用 DeleteSessionAccountID(sessionHash 为空) func TestHandleSmartRetry_ShortDelay_NonStickySession_FailedRetry_NoDeleteSession(t *testing.T) { failRespBody := `{ "error": { "status": "RESOURCE_EXHAUSTED", "details": [ {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "gemini-3-flash"}, "reason": "RATE_LIMIT_EXCEEDED"}, {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"} ] } }` failResp := &http.Response{ StatusCode: http.StatusTooManyRequests, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(failRespBody)), } upstream := &mockSmartRetryUpstream{ responses: []*http.Response{failResp}, errors: []error{nil}, } repo := &stubAntigravityAccountRepo{} cache := &stubSmartRetryCache{} account := &Account{ ID: 11, Name: "acc-11", Type: AccountTypeOAuth, Platform: PlatformAntigravity, } respBody := []byte(`{ "error": { "status": "RESOURCE_EXHAUSTED", "details": [ {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "gemini-3-flash"}, "reason": "RATE_LIMIT_EXCEEDED"}, {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"} ] } }`) resp := &http.Response{ StatusCode: http.StatusTooManyRequests, Header: http.Header{}, Body: io.NopCloser(bytes.NewReader(respBody)), } params := antigravityRetryLoopParams{ ctx: context.Background(), prefix: "[test]", account: account, accessToken: "token", action: "generateContent", body: []byte(`{"input":"test"}`), httpUpstream: upstream, accountRepo: repo, isStickySession: false, groupID: 42, sessionHash: "", // 非粘性会话,sessionHash 为空 handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult { return nil }, } availableURLs := []string{"https://ag-1.test"} svc := &AntigravityGatewayService{cache: cache} result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs) require.NotNil(t, result) require.Equal(t, smartRetryActionBreakWithResp, result.action) require.NotNil(t, result.switchError) require.False(t, result.switchError.IsStickySession) // 核心断言:sessionHash 为空时不应调用 DeleteSessionAccountID require.Len(t, cache.deleteCalls, 0, "should NOT call DeleteSessionAccountID when sessionHash is empty") } // TestHandleSmartRetry_ShortDelay_StickySession_FailedRetry_NilCache_NoPanic // 边界:cache 为 nil 时不应 panic func TestHandleSmartRetry_ShortDelay_StickySession_FailedRetry_NilCache_NoPanic(t *testing.T) { failRespBody := `{ "error": { "status": "RESOURCE_EXHAUSTED", "details": [ {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "claude-sonnet-4-5"}, "reason": "RATE_LIMIT_EXCEEDED"}, {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"} ] } }` failResp := &http.Response{ StatusCode: http.StatusTooManyRequests, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(failRespBody)), } upstream := &mockSmartRetryUpstream{ responses: []*http.Response{failResp}, errors: []error{nil}, } repo := &stubAntigravityAccountRepo{} account := &Account{ ID: 12, Name: "acc-12", Type: AccountTypeOAuth, Platform: PlatformAntigravity, } respBody := []byte(`{ "error": { "status": "RESOURCE_EXHAUSTED", "details": [ {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "claude-sonnet-4-5"}, "reason": "RATE_LIMIT_EXCEEDED"}, {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"} ] } }`) resp := &http.Response{ StatusCode: http.StatusTooManyRequests, Header: http.Header{}, Body: io.NopCloser(bytes.NewReader(respBody)), } params := antigravityRetryLoopParams{ ctx: context.Background(), prefix: "[test]", account: account, accessToken: "token", action: "generateContent", body: []byte(`{"input":"test"}`), httpUpstream: upstream, accountRepo: repo, isStickySession: true, groupID: 42, sessionHash: "sticky-hash-nil-cache", handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult { return nil }, } availableURLs := []string{"https://ag-1.test"} // cache 为 nil,不应 panic svc := &AntigravityGatewayService{cache: nil} require.NotPanics(t, func() { result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs) require.NotNil(t, result) require.Equal(t, smartRetryActionBreakWithResp, result.action) require.NotNil(t, result.switchError) require.True(t, result.switchError.IsStickySession) }) } // TestHandleSmartRetry_ShortDelay_StickySession_SuccessRetry_NoDeleteSession // 重试成功时不应清除粘性会话(只有失败才清除) func TestHandleSmartRetry_ShortDelay_StickySession_SuccessRetry_NoDeleteSession(t *testing.T) { successResp := &http.Response{ StatusCode: http.StatusOK, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(`{"result":"ok"}`)), } upstream := &mockSmartRetryUpstream{ responses: []*http.Response{successResp}, errors: []error{nil}, } cache := &stubSmartRetryCache{} account := &Account{ ID: 13, Name: "acc-13", Type: AccountTypeOAuth, Platform: PlatformAntigravity, } respBody := []byte(`{ "error": { "status": "RESOURCE_EXHAUSTED", "details": [ {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "claude-opus-4"}, "reason": "RATE_LIMIT_EXCEEDED"}, {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.5s"} ] } }`) resp := &http.Response{ StatusCode: http.StatusTooManyRequests, Header: http.Header{}, Body: io.NopCloser(bytes.NewReader(respBody)), } params := antigravityRetryLoopParams{ ctx: context.Background(), prefix: "[test]", account: account, accessToken: "token", action: "generateContent", body: []byte(`{"input":"test"}`), httpUpstream: upstream, isStickySession: true, groupID: 42, sessionHash: "sticky-hash-success", handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult { return nil }, } availableURLs := []string{"https://ag-1.test"} svc := &AntigravityGatewayService{cache: cache} result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs) require.NotNil(t, result) require.Equal(t, smartRetryActionBreakWithResp, result.action) require.NotNil(t, result.resp, "should return successful response") require.Equal(t, http.StatusOK, result.resp.StatusCode) require.Nil(t, result.switchError, "should not return switchError on success") // 核心断言:重试成功时不应清除粘性会话 require.Len(t, cache.deleteCalls, 0, "should NOT call DeleteSessionAccountID on successful retry") } // TestHandleSmartRetry_LongDelay_StickySession_NoDeleteInHandleSmartRetry // 长延迟路径(情况1)在 handleSmartRetry 中不直接调用 DeleteSessionAccountID // (清除由 handler 层的 shouldClearStickySession 在下次请求时处理) func TestHandleSmartRetry_LongDelay_StickySession_NoDeleteInHandleSmartRetry(t *testing.T) { repo := &stubAntigravityAccountRepo{} cache := &stubSmartRetryCache{} account := &Account{ ID: 14, Name: "acc-14", Type: AccountTypeOAuth, Platform: PlatformAntigravity, } // 15s >= 7s 阈值 → 走长延迟路径 respBody := []byte(`{ "error": { "status": "RESOURCE_EXHAUSTED", "details": [ {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "claude-sonnet-4-5"}, "reason": "RATE_LIMIT_EXCEEDED"}, {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "15s"} ] } }`) resp := &http.Response{ StatusCode: http.StatusTooManyRequests, Header: http.Header{}, Body: io.NopCloser(bytes.NewReader(respBody)), } params := antigravityRetryLoopParams{ ctx: context.Background(), prefix: "[test]", account: account, accessToken: "token", action: "generateContent", body: []byte(`{"input":"test"}`), accountRepo: repo, isStickySession: true, groupID: 42, sessionHash: "sticky-hash-long-delay", handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult { return nil }, } availableURLs := []string{"https://ag-1.test"} svc := &AntigravityGatewayService{cache: cache} result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs) require.NotNil(t, result) require.Equal(t, smartRetryActionBreakWithResp, result.action) require.NotNil(t, result.switchError) require.True(t, result.switchError.IsStickySession) // 长延迟路径不在 handleSmartRetry 中调用 DeleteSessionAccountID // (由上游 handler 的 shouldClearStickySession 处理) require.Len(t, cache.deleteCalls, 0, "long delay path should NOT call DeleteSessionAccountID in handleSmartRetry (handled by handler layer)") } // TestHandleSmartRetry_ShortDelay_NetworkError_StickySession_ClearsSession // 网络错误耗尽重试 + 粘性会话 → 也应清除粘性绑定 func TestHandleSmartRetry_ShortDelay_NetworkError_StickySession_ClearsSession(t *testing.T) { upstream := &mockSmartRetryUpstream{ responses: []*http.Response{nil}, // 网络错误 errors: []error{nil}, } repo := &stubAntigravityAccountRepo{} cache := &stubSmartRetryCache{} account := &Account{ ID: 15, Name: "acc-15", Type: AccountTypeOAuth, Platform: PlatformAntigravity, } respBody := []byte(`{ "error": { "status": "RESOURCE_EXHAUSTED", "details": [ {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "gemini-3-flash"}, "reason": "RATE_LIMIT_EXCEEDED"}, {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"} ] } }`) resp := &http.Response{ StatusCode: http.StatusTooManyRequests, Header: http.Header{}, Body: io.NopCloser(bytes.NewReader(respBody)), } params := antigravityRetryLoopParams{ ctx: context.Background(), prefix: "[test]", account: account, accessToken: "token", action: "generateContent", body: []byte(`{"input":"test"}`), httpUpstream: upstream, accountRepo: repo, isStickySession: true, groupID: 99, sessionHash: "sticky-net-error", handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult { return nil }, } availableURLs := []string{"https://ag-1.test"} svc := &AntigravityGatewayService{cache: cache} result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs) require.NotNil(t, result) require.NotNil(t, result.switchError) require.True(t, result.switchError.IsStickySession) // 核心断言:网络错误耗尽重试后也应清除粘性绑定 require.Len(t, cache.deleteCalls, 1, "should call DeleteSessionAccountID after network error exhausts retry") require.Equal(t, int64(99), cache.deleteCalls[0].groupID) require.Equal(t, "sticky-net-error", cache.deleteCalls[0].sessionHash) } // TestHandleSmartRetry_ShortDelay_503_StickySession_FailedRetry_ClearsSession // 503 + 短延迟 + 粘性会话 + 重试失败 → 清除粘性绑定 func TestHandleSmartRetry_ShortDelay_503_StickySession_FailedRetry_ClearsSession(t *testing.T) { failRespBody := `{ "error": { "code": 503, "status": "UNAVAILABLE", "details": [ {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "gemini-3-pro"}, "reason": "MODEL_CAPACITY_EXHAUSTED"}, {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.5s"} ] } }` failResp := &http.Response{ StatusCode: http.StatusServiceUnavailable, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(failRespBody)), } upstream := &mockSmartRetryUpstream{ responses: []*http.Response{failResp}, errors: []error{nil}, } repo := &stubAntigravityAccountRepo{} cache := &stubSmartRetryCache{} account := &Account{ ID: 16, Name: "acc-16", Type: AccountTypeOAuth, Platform: PlatformAntigravity, } respBody := []byte(`{ "error": { "code": 503, "status": "UNAVAILABLE", "details": [ {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "gemini-3-pro"}, "reason": "MODEL_CAPACITY_EXHAUSTED"}, {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.5s"} ] } }`) resp := &http.Response{ StatusCode: http.StatusServiceUnavailable, Header: http.Header{}, Body: io.NopCloser(bytes.NewReader(respBody)), } params := antigravityRetryLoopParams{ ctx: context.Background(), prefix: "[test]", account: account, accessToken: "token", action: "generateContent", body: []byte(`{"input":"test"}`), httpUpstream: upstream, accountRepo: repo, isStickySession: true, groupID: 77, sessionHash: "sticky-503-short", handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult { return nil }, } availableURLs := []string{"https://ag-1.test"} svc := &AntigravityGatewayService{cache: cache} result := svc.handleSmartRetry(params, resp, respBody, "https://ag-1.test", 0, availableURLs) require.NotNil(t, result) require.NotNil(t, result.switchError) require.True(t, result.switchError.IsStickySession) // 验证粘性绑定被清除 require.Len(t, cache.deleteCalls, 1) require.Equal(t, int64(77), cache.deleteCalls[0].groupID) require.Equal(t, "sticky-503-short", cache.deleteCalls[0].sessionHash) // 验证模型限流已设置 require.Len(t, repo.modelRateLimitCalls, 1) require.Equal(t, "gemini-3-pro", repo.modelRateLimitCalls[0].modelKey) } // TestAntigravityRetryLoop_SmartRetryFailed_StickySession_SwitchErrorPropagates // 集成测试:antigravityRetryLoop → handleSmartRetry → switchError 传播 // 验证 IsStickySession 正确传递到上层,且粘性绑定被清除 func TestAntigravityRetryLoop_SmartRetryFailed_StickySession_SwitchErrorPropagates(t *testing.T) { // 初始 429 响应 initialRespBody := []byte(`{ "error": { "status": "RESOURCE_EXHAUSTED", "details": [ {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "claude-opus-4-6"}, "reason": "RATE_LIMIT_EXCEEDED"}, {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"} ] } }`) initialResp := &http.Response{ StatusCode: http.StatusTooManyRequests, Header: http.Header{}, Body: io.NopCloser(bytes.NewReader(initialRespBody)), } // 智能重试也返回 429 retryRespBody := `{ "error": { "status": "RESOURCE_EXHAUSTED", "details": [ {"@type": "type.googleapis.com/google.rpc.ErrorInfo", "metadata": {"model": "claude-opus-4-6"}, "reason": "RATE_LIMIT_EXCEEDED"}, {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "0.1s"} ] } }` retryResp := &http.Response{ StatusCode: http.StatusTooManyRequests, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(retryRespBody)), } upstream := &mockSmartRetryUpstream{ responses: []*http.Response{initialResp, retryResp}, errors: []error{nil, nil}, } repo := &stubAntigravityAccountRepo{} cache := &stubSmartRetryCache{} account := &Account{ ID: 17, Name: "acc-17", Type: AccountTypeOAuth, Platform: PlatformAntigravity, Schedulable: true, Status: StatusActive, Concurrency: 1, } svc := &AntigravityGatewayService{cache: cache} result, err := svc.antigravityRetryLoop(antigravityRetryLoopParams{ ctx: context.Background(), prefix: "[test]", account: account, accessToken: "token", action: "generateContent", body: []byte(`{"input":"test"}`), httpUpstream: upstream, accountRepo: repo, isStickySession: true, groupID: 55, sessionHash: "sticky-loop-test", handleError: func(ctx context.Context, prefix string, account *Account, statusCode int, headers http.Header, body []byte, requestedModel string, groupID int64, sessionHash string, isStickySession bool) *handleModelRateLimitResult { return nil }, }) require.Nil(t, result, "should not return result when switchError") require.NotNil(t, err, "should return error") var switchErr *AntigravityAccountSwitchError require.ErrorAs(t, err, &switchErr, "error should be AntigravityAccountSwitchError") require.Equal(t, account.ID, switchErr.OriginalAccountID) require.Equal(t, "claude-opus-4-6", switchErr.RateLimitedModel) require.True(t, switchErr.IsStickySession, "IsStickySession must propagate through retryLoop") // 验证粘性绑定被清除 require.Len(t, cache.deleteCalls, 1, "should clear sticky session in handleSmartRetry") require.Equal(t, int64(55), cache.deleteCalls[0].groupID) require.Equal(t, "sticky-loop-test", cache.deleteCalls[0].sessionHash) }