feat: make antigravity max retries configurable
This commit is contained in:
@@ -24,13 +24,14 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
antigravityStickySessionTTL = time.Hour
|
antigravityStickySessionTTL = time.Hour
|
||||||
antigravityMaxRetries = 3
|
antigravityDefaultMaxRetries = 5
|
||||||
antigravityRetryBaseDelay = 1 * time.Second
|
antigravityRetryBaseDelay = 1 * time.Second
|
||||||
antigravityRetryMaxDelay = 16 * time.Second
|
antigravityRetryMaxDelay = 16 * time.Second
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
antigravityMaxRetriesEnv = "GATEWAY_ANTIGRAVITY_MAX_RETRIES"
|
||||||
antigravityScopeRateLimitEnv = "GATEWAY_ANTIGRAVITY_429_SCOPE_LIMIT"
|
antigravityScopeRateLimitEnv = "GATEWAY_ANTIGRAVITY_429_SCOPE_LIMIT"
|
||||||
antigravityBillingModelEnv = "GATEWAY_ANTIGRAVITY_BILL_WITH_MAPPED_MODEL"
|
antigravityBillingModelEnv = "GATEWAY_ANTIGRAVITY_BILL_WITH_MAPPED_MODEL"
|
||||||
antigravityFallbackSecondsEnv = "GATEWAY_ANTIGRAVITY_FALLBACK_COOLDOWN_SECONDS"
|
antigravityFallbackSecondsEnv = "GATEWAY_ANTIGRAVITY_FALLBACK_COOLDOWN_SECONDS"
|
||||||
@@ -63,6 +64,7 @@ func antigravityRetryLoop(p antigravityRetryLoopParams) (*antigravityRetryLoopRe
|
|||||||
if len(availableURLs) == 0 {
|
if len(availableURLs) == 0 {
|
||||||
availableURLs = antigravity.BaseURLs
|
availableURLs = antigravity.BaseURLs
|
||||||
}
|
}
|
||||||
|
maxRetries := antigravityMaxRetries()
|
||||||
|
|
||||||
var resp *http.Response
|
var resp *http.Response
|
||||||
var usedBaseURL string
|
var usedBaseURL string
|
||||||
@@ -81,7 +83,7 @@ func antigravityRetryLoop(p antigravityRetryLoopParams) (*antigravityRetryLoopRe
|
|||||||
urlFallbackLoop:
|
urlFallbackLoop:
|
||||||
for urlIdx, baseURL := range availableURLs {
|
for urlIdx, baseURL := range availableURLs {
|
||||||
usedBaseURL = baseURL
|
usedBaseURL = baseURL
|
||||||
for attempt := 1; attempt <= antigravityMaxRetries; attempt++ {
|
for attempt := 1; attempt <= maxRetries; attempt++ {
|
||||||
select {
|
select {
|
||||||
case <-p.ctx.Done():
|
case <-p.ctx.Done():
|
||||||
log.Printf("%s status=context_canceled error=%v", p.prefix, p.ctx.Err())
|
log.Printf("%s status=context_canceled error=%v", p.prefix, p.ctx.Err())
|
||||||
@@ -114,8 +116,8 @@ urlFallbackLoop:
|
|||||||
log.Printf("%s URL fallback (connection error): %s -> %s", p.prefix, baseURL, availableURLs[urlIdx+1])
|
log.Printf("%s URL fallback (connection error): %s -> %s", p.prefix, baseURL, availableURLs[urlIdx+1])
|
||||||
continue urlFallbackLoop
|
continue urlFallbackLoop
|
||||||
}
|
}
|
||||||
if attempt < antigravityMaxRetries {
|
if attempt < maxRetries {
|
||||||
log.Printf("%s status=request_failed retry=%d/%d error=%v", p.prefix, attempt, antigravityMaxRetries, err)
|
log.Printf("%s status=request_failed retry=%d/%d error=%v", p.prefix, attempt, maxRetries, err)
|
||||||
if !sleepAntigravityBackoffWithContext(p.ctx, attempt) {
|
if !sleepAntigravityBackoffWithContext(p.ctx, attempt) {
|
||||||
log.Printf("%s status=context_canceled_during_backoff", p.prefix)
|
log.Printf("%s status=context_canceled_during_backoff", p.prefix)
|
||||||
return nil, p.ctx.Err()
|
return nil, p.ctx.Err()
|
||||||
@@ -138,8 +140,8 @@ urlFallbackLoop:
|
|||||||
continue urlFallbackLoop
|
continue urlFallbackLoop
|
||||||
}
|
}
|
||||||
|
|
||||||
// 账户/模型配额限流,重试 3 次(指数退避)
|
// 账户/模型配额限流,按最大重试次数做指数退避
|
||||||
if attempt < antigravityMaxRetries {
|
if attempt < maxRetries {
|
||||||
upstreamMsg := strings.TrimSpace(extractAntigravityErrorMessage(respBody))
|
upstreamMsg := strings.TrimSpace(extractAntigravityErrorMessage(respBody))
|
||||||
upstreamMsg = sanitizeUpstreamErrorMessage(upstreamMsg)
|
upstreamMsg = sanitizeUpstreamErrorMessage(upstreamMsg)
|
||||||
appendOpsUpstreamError(p.c, OpsUpstreamErrorEvent{
|
appendOpsUpstreamError(p.c, OpsUpstreamErrorEvent{
|
||||||
@@ -152,7 +154,7 @@ urlFallbackLoop:
|
|||||||
Message: upstreamMsg,
|
Message: upstreamMsg,
|
||||||
Detail: getUpstreamDetail(respBody),
|
Detail: getUpstreamDetail(respBody),
|
||||||
})
|
})
|
||||||
log.Printf("%s status=429 retry=%d/%d body=%s", p.prefix, attempt, antigravityMaxRetries, truncateForLog(respBody, 200))
|
log.Printf("%s status=429 retry=%d/%d body=%s", p.prefix, attempt, maxRetries, truncateForLog(respBody, 200))
|
||||||
if !sleepAntigravityBackoffWithContext(p.ctx, attempt) {
|
if !sleepAntigravityBackoffWithContext(p.ctx, attempt) {
|
||||||
log.Printf("%s status=context_canceled_during_backoff", p.prefix)
|
log.Printf("%s status=context_canceled_during_backoff", p.prefix)
|
||||||
return nil, p.ctx.Err()
|
return nil, p.ctx.Err()
|
||||||
@@ -176,7 +178,7 @@ urlFallbackLoop:
|
|||||||
respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 2<<20))
|
respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 2<<20))
|
||||||
_ = resp.Body.Close()
|
_ = resp.Body.Close()
|
||||||
|
|
||||||
if attempt < antigravityMaxRetries {
|
if attempt < maxRetries {
|
||||||
upstreamMsg := strings.TrimSpace(extractAntigravityErrorMessage(respBody))
|
upstreamMsg := strings.TrimSpace(extractAntigravityErrorMessage(respBody))
|
||||||
upstreamMsg = sanitizeUpstreamErrorMessage(upstreamMsg)
|
upstreamMsg = sanitizeUpstreamErrorMessage(upstreamMsg)
|
||||||
appendOpsUpstreamError(p.c, OpsUpstreamErrorEvent{
|
appendOpsUpstreamError(p.c, OpsUpstreamErrorEvent{
|
||||||
@@ -189,7 +191,7 @@ urlFallbackLoop:
|
|||||||
Message: upstreamMsg,
|
Message: upstreamMsg,
|
||||||
Detail: getUpstreamDetail(respBody),
|
Detail: getUpstreamDetail(respBody),
|
||||||
})
|
})
|
||||||
log.Printf("%s status=%d retry=%d/%d body=%s", p.prefix, resp.StatusCode, attempt, antigravityMaxRetries, truncateForLog(respBody, 500))
|
log.Printf("%s status=%d retry=%d/%d body=%s", p.prefix, resp.StatusCode, attempt, maxRetries, truncateForLog(respBody, 500))
|
||||||
if !sleepAntigravityBackoffWithContext(p.ctx, attempt) {
|
if !sleepAntigravityBackoffWithContext(p.ctx, attempt) {
|
||||||
log.Printf("%s status=context_canceled_during_backoff", p.prefix)
|
log.Printf("%s status=context_canceled_during_backoff", p.prefix)
|
||||||
return nil, p.ctx.Err()
|
return nil, p.ctx.Err()
|
||||||
@@ -1538,6 +1540,18 @@ func antigravityUseScopeRateLimit() bool {
|
|||||||
return v == "1" || v == "true" || v == "yes" || v == "on"
|
return v == "1" || v == "true" || v == "yes" || v == "on"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func antigravityMaxRetries() int {
|
||||||
|
raw := strings.TrimSpace(os.Getenv(antigravityMaxRetriesEnv))
|
||||||
|
if raw == "" {
|
||||||
|
return antigravityDefaultMaxRetries
|
||||||
|
}
|
||||||
|
value, err := strconv.Atoi(raw)
|
||||||
|
if err != nil || value <= 0 {
|
||||||
|
return antigravityDefaultMaxRetries
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
func antigravityUseMappedModelForBilling() bool {
|
func antigravityUseMappedModelForBilling() bool {
|
||||||
v := strings.ToLower(strings.TrimSpace(os.Getenv(antigravityBillingModelEnv)))
|
v := strings.ToLower(strings.TrimSpace(os.Getenv(antigravityBillingModelEnv)))
|
||||||
return v == "1" || v == "true" || v == "yes" || v == "on"
|
return v == "1" || v == "true" || v == "yes" || v == "on"
|
||||||
|
|||||||
Reference in New Issue
Block a user