From 36a1a7998b5d54147fdbf66803b16c6d2cb8e5d4 Mon Sep 17 00:00:00 2001 From: yangjianbo Date: Thu, 19 Feb 2026 20:29:31 +0800 Subject: [PATCH] =?UTF-8?q?feat(sora):=20=E5=BC=BA=E5=88=B6Sora=E8=B5=B0cu?= =?UTF-8?q?rl=5Fcffi=20sidecar=E5=B9=B6=E5=AE=8C=E5=96=84=E6=A0=A1?= =?UTF-8?q?=E9=AA=8C=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/internal/config/config.go | 46 +++- backend/internal/config/config_test.go | 49 ++++ backend/internal/service/sora_client.go | 8 + backend/internal/service/sora_client_test.go | 142 +++++++++++ .../service/sora_curl_cffi_sidecar.go | 238 ++++++++++++++++++ deploy/config.example.yaml | 15 ++ 6 files changed, 486 insertions(+), 12 deletions(-) create mode 100644 backend/internal/service/sora_curl_cffi_sidecar.go diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index 8efcb550..94388a0d 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -271,18 +271,27 @@ type SoraConfig struct { // SoraClientConfig 直连 Sora 客户端配置 type SoraClientConfig struct { - BaseURL string `mapstructure:"base_url"` - TimeoutSeconds int `mapstructure:"timeout_seconds"` - MaxRetries int `mapstructure:"max_retries"` - PollIntervalSeconds int `mapstructure:"poll_interval_seconds"` - MaxPollAttempts int `mapstructure:"max_poll_attempts"` - RecentTaskLimit int `mapstructure:"recent_task_limit"` - RecentTaskLimitMax int `mapstructure:"recent_task_limit_max"` - Debug bool `mapstructure:"debug"` - UseOpenAITokenProvider bool `mapstructure:"use_openai_token_provider"` - Headers map[string]string `mapstructure:"headers"` - UserAgent string `mapstructure:"user_agent"` - DisableTLSFingerprint bool `mapstructure:"disable_tls_fingerprint"` + BaseURL string `mapstructure:"base_url"` + TimeoutSeconds int `mapstructure:"timeout_seconds"` + MaxRetries int `mapstructure:"max_retries"` + PollIntervalSeconds int `mapstructure:"poll_interval_seconds"` + MaxPollAttempts int `mapstructure:"max_poll_attempts"` + RecentTaskLimit int `mapstructure:"recent_task_limit"` + RecentTaskLimitMax int `mapstructure:"recent_task_limit_max"` + Debug bool `mapstructure:"debug"` + UseOpenAITokenProvider bool `mapstructure:"use_openai_token_provider"` + Headers map[string]string `mapstructure:"headers"` + UserAgent string `mapstructure:"user_agent"` + DisableTLSFingerprint bool `mapstructure:"disable_tls_fingerprint"` + CurlCFFISidecar SoraCurlCFFISidecarConfig `mapstructure:"curl_cffi_sidecar"` +} + +// SoraCurlCFFISidecarConfig Sora 专用 curl_cffi sidecar 配置 +type SoraCurlCFFISidecarConfig struct { + Enabled bool `mapstructure:"enabled"` + BaseURL string `mapstructure:"base_url"` + Impersonate string `mapstructure:"impersonate"` + TimeoutSeconds int `mapstructure:"timeout_seconds"` } // SoraStorageConfig 媒体存储配置 @@ -1123,6 +1132,10 @@ func setDefaults() { viper.SetDefault("sora.client.headers", map[string]string{}) viper.SetDefault("sora.client.user_agent", "Sora/1.2026.007 (Android 15; 24122RKC7C; build 2600700)") viper.SetDefault("sora.client.disable_tls_fingerprint", false) + viper.SetDefault("sora.client.curl_cffi_sidecar.enabled", true) + viper.SetDefault("sora.client.curl_cffi_sidecar.base_url", "http://sora-curl-cffi-sidecar:8080") + viper.SetDefault("sora.client.curl_cffi_sidecar.impersonate", "chrome131") + viper.SetDefault("sora.client.curl_cffi_sidecar.timeout_seconds", 60) viper.SetDefault("sora.storage.type", "local") viper.SetDefault("sora.storage.local_path", "") @@ -1526,6 +1539,15 @@ func (c *Config) Validate() error { c.Sora.Client.RecentTaskLimitMax < c.Sora.Client.RecentTaskLimit { c.Sora.Client.RecentTaskLimitMax = c.Sora.Client.RecentTaskLimit } + if c.Sora.Client.CurlCFFISidecar.TimeoutSeconds < 0 { + return fmt.Errorf("sora.client.curl_cffi_sidecar.timeout_seconds must be non-negative") + } + if !c.Sora.Client.CurlCFFISidecar.Enabled { + return fmt.Errorf("sora.client.curl_cffi_sidecar.enabled must be true") + } + if strings.TrimSpace(c.Sora.Client.CurlCFFISidecar.BaseURL) == "" { + return fmt.Errorf("sora.client.curl_cffi_sidecar.base_url is required") + } if c.Sora.Storage.MaxConcurrentDownloads < 0 { return fmt.Errorf("sora.storage.max_concurrent_downloads must be non-negative") } diff --git a/backend/internal/config/config_test.go b/backend/internal/config/config_test.go index a3c65c41..e81f70b0 100644 --- a/backend/internal/config/config_test.go +++ b/backend/internal/config/config_test.go @@ -1024,3 +1024,52 @@ func TestValidateConfigErrors(t *testing.T) { }) } } + +func TestSoraCurlCFFISidecarDefaults(t *testing.T) { + resetViperWithJWTSecret(t) + + cfg, err := Load() + if err != nil { + t.Fatalf("Load() error: %v", err) + } + + if !cfg.Sora.Client.CurlCFFISidecar.Enabled { + t.Fatalf("Sora curl_cffi sidecar should be enabled by default") + } + if cfg.Sora.Client.CurlCFFISidecar.BaseURL == "" { + t.Fatalf("Sora curl_cffi sidecar base_url should not be empty by default") + } + if cfg.Sora.Client.CurlCFFISidecar.Impersonate == "" { + t.Fatalf("Sora curl_cffi sidecar impersonate should not be empty by default") + } +} + +func TestValidateSoraCurlCFFISidecarRequired(t *testing.T) { + resetViperWithJWTSecret(t) + + cfg, err := Load() + if err != nil { + t.Fatalf("Load() error: %v", err) + } + + cfg.Sora.Client.CurlCFFISidecar.Enabled = false + err = cfg.Validate() + if err == nil || !strings.Contains(err.Error(), "sora.client.curl_cffi_sidecar.enabled must be true") { + t.Fatalf("Validate() error = %v, want sidecar enabled error", err) + } +} + +func TestValidateSoraCurlCFFISidecarBaseURLRequired(t *testing.T) { + resetViperWithJWTSecret(t) + + cfg, err := Load() + if err != nil { + t.Fatalf("Load() error: %v", err) + } + + cfg.Sora.Client.CurlCFFISidecar.BaseURL = " " + err = cfg.Validate() + if err == nil || !strings.Contains(err.Error(), "sora.client.curl_cffi_sidecar.base_url is required") { + t.Fatalf("Validate() error = %v, want sidecar base_url required error", err) + } +} diff --git a/backend/internal/service/sora_client.go b/backend/internal/service/sora_client.go index e1af5ead..77d099ab 100644 --- a/backend/internal/service/sora_client.go +++ b/backend/internal/service/sora_client.go @@ -1630,6 +1630,14 @@ func shouldAttemptSoraTokenRecover(statusCode int, rawURL string) bool { } func (c *SoraDirectClient) doHTTP(req *http.Request, proxyURL string, account *Account) (*http.Response, error) { + if c != nil && c.cfg != nil && c.cfg.Sora.Client.CurlCFFISidecar.Enabled { + resp, err := c.doHTTPViaCurlCFFISidecar(req, proxyURL) + if err != nil { + return nil, err + } + return resp, nil + } + enableTLS := c == nil || c.cfg == nil || !c.cfg.Sora.Client.DisableTLSFingerprint if c.httpUpstream != nil { accountID := int64(0) diff --git a/backend/internal/service/sora_client_test.go b/backend/internal/service/sora_client_test.go index 9e528f97..ae92782e 100644 --- a/backend/internal/service/sora_client_test.go +++ b/backend/internal/service/sora_client_test.go @@ -4,6 +4,7 @@ package service import ( "context" + "encoding/base64" "encoding/json" "errors" "io" @@ -639,3 +640,144 @@ func TestSoraDirectClient_PostVideoForWatermarkFree(t *testing.T) { require.Equal(t, "/backend-api/sentinel/req", upstream.calls[0].Path) require.Equal(t, "/backend/project_y/post", upstream.calls[1].Path) } + +type soraClientFallbackUpstream struct { + doWithTLSCalls int32 + respBody string + respStatusCode int + err error +} + +func (u *soraClientFallbackUpstream) Do(_ *http.Request, _ string, _ int64, _ int) (*http.Response, error) { + return nil, errors.New("unexpected Do call") +} + +func (u *soraClientFallbackUpstream) DoWithTLS(_ *http.Request, _ string, _ int64, _ int, _ bool) (*http.Response, error) { + atomic.AddInt32(&u.doWithTLSCalls, 1) + if u.err != nil { + return nil, u.err + } + statusCode := u.respStatusCode + if statusCode <= 0 { + statusCode = http.StatusOK + } + body := u.respBody + if body == "" { + body = `{"ok":true}` + } + return newSoraClientMockResponse(statusCode, body), nil +} + +func TestSoraDirectClient_DoHTTP_UsesCurlCFFISidecarWhenEnabled(t *testing.T) { + var captured soraCurlCFFISidecarRequest + sidecar := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, http.MethodPost, r.Method) + require.Equal(t, "/request", r.URL.Path) + raw, err := io.ReadAll(r.Body) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(raw, &captured)) + _ = json.NewEncoder(w).Encode(map[string]any{ + "status_code": http.StatusOK, + "headers": map[string]any{ + "Content-Type": "application/json", + "X-Sidecar": []string{"yes"}, + }, + "body_base64": base64.StdEncoding.EncodeToString([]byte(`{"ok":true}`)), + }) + })) + defer sidecar.Close() + + upstream := &soraClientFallbackUpstream{} + cfg := &config.Config{ + Sora: config.SoraConfig{ + Client: config.SoraClientConfig{ + BaseURL: "https://sora.chatgpt.com/backend", + CurlCFFISidecar: config.SoraCurlCFFISidecarConfig{ + Enabled: true, + BaseURL: sidecar.URL, + Impersonate: "chrome131", + TimeoutSeconds: 15, + }, + }, + }, + } + client := NewSoraDirectClient(cfg, upstream, nil) + req, err := http.NewRequest(http.MethodPost, "https://sora.chatgpt.com/backend/me", strings.NewReader("hello-sidecar")) + require.NoError(t, err) + req.Header.Set("User-Agent", "test-ua") + + resp, err := client.doHTTP(req, "http://127.0.0.1:18080", &Account{ID: 1}) + require.NoError(t, err) + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + require.JSONEq(t, `{"ok":true}`, string(body)) + require.Equal(t, int32(0), atomic.LoadInt32(&upstream.doWithTLSCalls)) + require.Equal(t, "http://127.0.0.1:18080", captured.ProxyURL) + require.Equal(t, "chrome131", captured.Impersonate) + require.Equal(t, "https://sora.chatgpt.com/backend/me", captured.URL) + decodedReqBody, err := base64.StdEncoding.DecodeString(captured.BodyBase64) + require.NoError(t, err) + require.Equal(t, "hello-sidecar", string(decodedReqBody)) +} + +func TestSoraDirectClient_DoHTTP_CurlCFFISidecarFailureReturnsError(t *testing.T) { + sidecar := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadGateway) + _, _ = w.Write([]byte(`{"error":"boom"}`)) + })) + defer sidecar.Close() + + upstream := &soraClientFallbackUpstream{respBody: `{"fallback":true}`} + cfg := &config.Config{ + Sora: config.SoraConfig{ + Client: config.SoraClientConfig{ + BaseURL: "https://sora.chatgpt.com/backend", + CurlCFFISidecar: config.SoraCurlCFFISidecarConfig{ + Enabled: true, + BaseURL: sidecar.URL, + }, + }, + }, + } + client := NewSoraDirectClient(cfg, upstream, nil) + req, err := http.NewRequest(http.MethodGet, "https://sora.chatgpt.com/backend/me", nil) + require.NoError(t, err) + + _, err = client.doHTTP(req, "", &Account{ID: 2}) + require.Error(t, err) + require.Contains(t, err.Error(), "sora curl_cffi sidecar") + require.Equal(t, int32(0), atomic.LoadInt32(&upstream.doWithTLSCalls)) +} + +func TestSoraDirectClient_DoHTTP_CurlCFFISidecarDisabledUsesLegacyStack(t *testing.T) { + upstream := &soraClientFallbackUpstream{respBody: `{"legacy":true}`} + cfg := &config.Config{ + Sora: config.SoraConfig{ + Client: config.SoraClientConfig{ + BaseURL: "https://sora.chatgpt.com/backend", + CurlCFFISidecar: config.SoraCurlCFFISidecarConfig{ + Enabled: false, + BaseURL: "http://127.0.0.1:18080", + }, + }, + }, + } + client := NewSoraDirectClient(cfg, upstream, nil) + req, err := http.NewRequest(http.MethodGet, "https://sora.chatgpt.com/backend/me", nil) + require.NoError(t, err) + + resp, err := client.doHTTP(req, "", &Account{ID: 3}) + require.NoError(t, err) + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.JSONEq(t, `{"legacy":true}`, string(body)) + require.Equal(t, int32(1), atomic.LoadInt32(&upstream.doWithTLSCalls)) +} + +func TestConvertSidecarHeaderValue_NilAndSlice(t *testing.T) { + require.Nil(t, convertSidecarHeaderValue(nil)) + require.Equal(t, []string{"a", "b"}, convertSidecarHeaderValue([]any{"a", " ", "b"})) +} diff --git a/backend/internal/service/sora_curl_cffi_sidecar.go b/backend/internal/service/sora_curl_cffi_sidecar.go new file mode 100644 index 00000000..6c83a97b --- /dev/null +++ b/backend/internal/service/sora_curl_cffi_sidecar.go @@ -0,0 +1,238 @@ +package service + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/Wei-Shaw/sub2api/internal/util/logredact" +) + +const soraCurlCFFISidecarDefaultTimeoutSeconds = 60 + +type soraCurlCFFISidecarRequest struct { + Method string `json:"method"` + URL string `json:"url"` + Headers map[string][]string `json:"headers,omitempty"` + BodyBase64 string `json:"body_base64,omitempty"` + ProxyURL string `json:"proxy_url,omitempty"` + Impersonate string `json:"impersonate,omitempty"` + TimeoutSeconds int `json:"timeout_seconds,omitempty"` +} + +type soraCurlCFFISidecarResponse struct { + StatusCode int `json:"status_code"` + Status int `json:"status"` + Headers map[string]any `json:"headers"` + BodyBase64 string `json:"body_base64"` + Body string `json:"body"` + Error string `json:"error"` +} + +func (c *SoraDirectClient) doHTTPViaCurlCFFISidecar(req *http.Request, proxyURL string) (*http.Response, error) { + if req == nil || req.URL == nil { + return nil, errors.New("request url is nil") + } + if c == nil || c.cfg == nil { + return nil, errors.New("sora curl_cffi sidecar config is nil") + } + if !c.cfg.Sora.Client.CurlCFFISidecar.Enabled { + return nil, errors.New("sora curl_cffi sidecar is disabled") + } + endpoint := c.curlCFFISidecarEndpoint() + if endpoint == "" { + return nil, errors.New("sora curl_cffi sidecar base_url is empty") + } + + bodyBytes, err := readAndRestoreRequestBody(req) + if err != nil { + return nil, fmt.Errorf("sora curl_cffi sidecar read request body failed: %w", err) + } + + headers := make(map[string][]string, len(req.Header)+1) + for key, vals := range req.Header { + copied := make([]string, len(vals)) + copy(copied, vals) + headers[key] = copied + } + if strings.TrimSpace(req.Host) != "" { + if _, ok := headers["Host"]; !ok { + headers["Host"] = []string{req.Host} + } + } + + payload := soraCurlCFFISidecarRequest{ + Method: req.Method, + URL: req.URL.String(), + Headers: headers, + ProxyURL: strings.TrimSpace(proxyURL), + Impersonate: c.curlCFFIImpersonate(), + TimeoutSeconds: c.curlCFFISidecarTimeoutSeconds(), + } + if len(bodyBytes) > 0 { + payload.BodyBase64 = base64.StdEncoding.EncodeToString(bodyBytes) + } + + encoded, err := json.Marshal(payload) + if err != nil { + return nil, fmt.Errorf("sora curl_cffi sidecar marshal request failed: %w", err) + } + + sidecarReq, err := http.NewRequestWithContext(req.Context(), http.MethodPost, endpoint, bytes.NewReader(encoded)) + if err != nil { + return nil, fmt.Errorf("sora curl_cffi sidecar build request failed: %w", err) + } + sidecarReq.Header.Set("Content-Type", "application/json") + sidecarReq.Header.Set("Accept", "application/json") + + httpClient := &http.Client{Timeout: time.Duration(payload.TimeoutSeconds) * time.Second} + sidecarResp, err := httpClient.Do(sidecarReq) + if err != nil { + return nil, fmt.Errorf("sora curl_cffi sidecar request failed: %w", err) + } + defer sidecarResp.Body.Close() + + sidecarRespBody, err := io.ReadAll(io.LimitReader(sidecarResp.Body, 8<<20)) + if err != nil { + return nil, fmt.Errorf("sora curl_cffi sidecar read response failed: %w", err) + } + if sidecarResp.StatusCode != http.StatusOK { + redacted := truncateForLog([]byte(logredact.RedactText(string(sidecarRespBody))), 512) + return nil, fmt.Errorf("sora curl_cffi sidecar http status=%d body=%s", sidecarResp.StatusCode, redacted) + } + + var payloadResp soraCurlCFFISidecarResponse + if err := json.Unmarshal(sidecarRespBody, &payloadResp); err != nil { + return nil, fmt.Errorf("sora curl_cffi sidecar parse response failed: %w", err) + } + if msg := strings.TrimSpace(payloadResp.Error); msg != "" { + return nil, fmt.Errorf("sora curl_cffi sidecar upstream error: %s", msg) + } + statusCode := payloadResp.StatusCode + if statusCode <= 0 { + statusCode = payloadResp.Status + } + if statusCode <= 0 { + return nil, errors.New("sora curl_cffi sidecar response missing status code") + } + + responseBody := []byte(payloadResp.Body) + if strings.TrimSpace(payloadResp.BodyBase64) != "" { + decoded, err := base64.StdEncoding.DecodeString(payloadResp.BodyBase64) + if err != nil { + return nil, fmt.Errorf("sora curl_cffi sidecar decode body failed: %w", err) + } + responseBody = decoded + } + + respHeaders := make(http.Header) + for key, rawVal := range payloadResp.Headers { + for _, v := range convertSidecarHeaderValue(rawVal) { + respHeaders.Add(key, v) + } + } + + return &http.Response{ + StatusCode: statusCode, + Header: respHeaders, + Body: io.NopCloser(bytes.NewReader(responseBody)), + ContentLength: int64(len(responseBody)), + Request: req, + }, nil +} + +func readAndRestoreRequestBody(req *http.Request) ([]byte, error) { + if req == nil || req.Body == nil { + return nil, nil + } + bodyBytes, err := io.ReadAll(req.Body) + if err != nil { + return nil, err + } + _ = req.Body.Close() + req.Body = io.NopCloser(bytes.NewReader(bodyBytes)) + req.ContentLength = int64(len(bodyBytes)) + return bodyBytes, nil +} + +func (c *SoraDirectClient) curlCFFISidecarEndpoint() string { + if c == nil || c.cfg == nil { + return "" + } + raw := strings.TrimSpace(c.cfg.Sora.Client.CurlCFFISidecar.BaseURL) + if raw == "" { + return "" + } + parsed, err := url.Parse(raw) + if err != nil || strings.TrimSpace(parsed.Scheme) == "" || strings.TrimSpace(parsed.Host) == "" { + return raw + } + if path := strings.TrimSpace(parsed.Path); path == "" || path == "/" { + parsed.Path = "/request" + } + return parsed.String() +} + +func (c *SoraDirectClient) curlCFFISidecarTimeoutSeconds() int { + if c == nil || c.cfg == nil { + return soraCurlCFFISidecarDefaultTimeoutSeconds + } + timeoutSeconds := c.cfg.Sora.Client.CurlCFFISidecar.TimeoutSeconds + if timeoutSeconds <= 0 { + return soraCurlCFFISidecarDefaultTimeoutSeconds + } + return timeoutSeconds +} + +func (c *SoraDirectClient) curlCFFIImpersonate() string { + if c == nil || c.cfg == nil { + return "chrome131" + } + impersonate := strings.TrimSpace(c.cfg.Sora.Client.CurlCFFISidecar.Impersonate) + if impersonate == "" { + return "chrome131" + } + return impersonate +} + +func convertSidecarHeaderValue(raw any) []string { + switch val := raw.(type) { + case nil: + return nil + case string: + if strings.TrimSpace(val) == "" { + return nil + } + return []string{val} + case []any: + out := make([]string, 0, len(val)) + for _, item := range val { + s := strings.TrimSpace(fmt.Sprint(item)) + if s != "" { + out = append(out, s) + } + } + return out + case []string: + out := make([]string, 0, len(val)) + for _, item := range val { + if strings.TrimSpace(item) != "" { + out = append(out, item) + } + } + return out + default: + s := strings.TrimSpace(fmt.Sprint(val)) + if s == "" { + return nil + } + return []string{s} + } +} diff --git a/deploy/config.example.yaml b/deploy/config.example.yaml index 0ff1ec02..27e29d34 100644 --- a/deploy/config.example.yaml +++ b/deploy/config.example.yaml @@ -402,6 +402,21 @@ sora: # Disable TLS fingerprint for Sora upstream # 关闭 Sora 上游 TLS 指纹伪装 disable_tls_fingerprint: false + # curl_cffi sidecar for Sora only (required) + # 仅 Sora 链路使用的 curl_cffi sidecar（必需） + curl_cffi_sidecar: + # Sora 强制通过 sidecar 请求，必须启用 + # Sora is forced to use sidecar only; keep enabled=true + enabled: true + # Sidecar base URL (default endpoint: /request) + # sidecar 基础地址（默认请求端点：/request） + base_url: "http://sora-curl-cffi-sidecar:8080" + # curl_cffi impersonate profile, e.g. chrome131/chrome124/safari18_0 + # curl_cffi 指纹伪装 profile，例如 chrome131/chrome124/safari18_0 + impersonate: "chrome131" + # Sidecar request timeout (seconds) + # sidecar 请求超时（秒） + timeout_seconds: 60 storage: # Storage type (local only for now) # 存储类型（首发仅支持 local）