使用codex的生图接口代替web2api
This commit is contained in:
@@ -930,10 +930,8 @@ func (a *Account) SupportsOpenAIImageCapability(capability OpenAIImagesCapabilit
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
switch capability {
|
switch capability {
|
||||||
case OpenAIImagesCapabilityBasic:
|
case OpenAIImagesCapabilityBasic, OpenAIImagesCapabilityNative:
|
||||||
return a.Type == AccountTypeOAuth || a.Type == AccountTypeAPIKey
|
return a.Type == AccountTypeOAuth || a.Type == AccountTypeAPIKey
|
||||||
case OpenAIImagesCapabilityNative:
|
|
||||||
return a.Type == AccountTypeAPIKey
|
|
||||||
default:
|
default:
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
"encoding/base64"
|
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
@@ -1138,7 +1137,7 @@ func (s *AccountTestService) testOpenAIImageAPIKey(c *gin.Context, ctx context.C
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// testOpenAIImageOAuth tests OpenAI image generation using an OAuth account via ChatGPT backend API.
|
// testOpenAIImageOAuth tests OpenAI image generation using an OAuth account via Codex /responses API.
|
||||||
func (s *AccountTestService) testOpenAIImageOAuth(c *gin.Context, ctx context.Context, account *Account, modelID, prompt string) error {
|
func (s *AccountTestService) testOpenAIImageOAuth(c *gin.Context, ctx context.Context, account *Account, modelID, prompt string) error {
|
||||||
authToken := account.GetOpenAIAccessToken()
|
authToken := account.GetOpenAIAccessToken()
|
||||||
if authToken == "" {
|
if authToken == "" {
|
||||||
@@ -1153,69 +1152,46 @@ func (s *AccountTestService) testOpenAIImageOAuth(c *gin.Context, ctx context.Co
|
|||||||
c.Writer.Flush()
|
c.Writer.Flush()
|
||||||
|
|
||||||
s.sendEvent(c, TestEvent{Type: "test_start", Model: modelID})
|
s.sendEvent(c, TestEvent{Type: "test_start", Model: modelID})
|
||||||
s.sendEvent(c, TestEvent{Type: "content", Text: "Initializing ChatGPT backend...\n"})
|
s.sendEvent(c, TestEvent{Type: "content", Text: "Calling Codex /responses image tool...\n"})
|
||||||
|
|
||||||
// Build headers (replicating buildOpenAIBackendAPIHeaders logic)
|
parsed := &OpenAIImagesRequest{
|
||||||
headers := buildOpenAIBackendAPIHeadersForTest(ctx, account, authToken, s.accountRepo)
|
Endpoint: openAIImagesGenerationsEndpoint,
|
||||||
|
Model: strings.TrimSpace(modelID),
|
||||||
|
Prompt: prompt,
|
||||||
|
}
|
||||||
|
applyOpenAIImagesDefaults(parsed)
|
||||||
|
|
||||||
|
responsesBody, err := buildOpenAIImagesResponsesRequest(parsed, parsed.Model)
|
||||||
|
if err != nil {
|
||||||
|
return s.sendErrorAndEnd(c, fmt.Sprintf("Failed to build image request: %s", err.Error()))
|
||||||
|
}
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, chatgptCodexAPIURL, bytes.NewReader(responsesBody))
|
||||||
|
if err != nil {
|
||||||
|
return s.sendErrorAndEnd(c, "Failed to create request")
|
||||||
|
}
|
||||||
|
req.Host = "chatgpt.com"
|
||||||
|
req.Header.Set("Authorization", "Bearer "+authToken)
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
req.Header.Set("Accept", "text/event-stream")
|
||||||
|
req.Header.Set("OpenAI-Beta", "responses=experimental")
|
||||||
|
req.Header.Set("originator", "opencode")
|
||||||
|
if customUA := strings.TrimSpace(account.GetOpenAIUserAgent()); customUA != "" {
|
||||||
|
req.Header.Set("User-Agent", customUA)
|
||||||
|
} else {
|
||||||
|
req.Header.Set("User-Agent", codexCLIUserAgent)
|
||||||
|
}
|
||||||
|
if chatgptAccountID := strings.TrimSpace(account.GetChatGPTAccountID()); chatgptAccountID != "" {
|
||||||
|
req.Header.Set("chatgpt-account-id", chatgptAccountID)
|
||||||
|
}
|
||||||
|
|
||||||
proxyURL := ""
|
proxyURL := ""
|
||||||
if account.ProxyID != nil && account.Proxy != nil {
|
if account.ProxyID != nil && account.Proxy != nil {
|
||||||
proxyURL = account.Proxy.URL()
|
proxyURL = account.Proxy.URL()
|
||||||
}
|
}
|
||||||
|
resp, err := s.httpUpstream.Do(req, proxyURL, account.ID, account.Concurrency)
|
||||||
client, err := newOpenAIBackendAPIClient(proxyURL)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return s.sendErrorAndEnd(c, fmt.Sprintf("Failed to create client: %s", err.Error()))
|
return s.sendErrorAndEnd(c, fmt.Sprintf("Responses API request failed: %s", err.Error()))
|
||||||
}
|
|
||||||
|
|
||||||
// Bootstrap
|
|
||||||
if bootstrapErr := bootstrapOpenAIBackendAPI(ctx, client, headers); bootstrapErr != nil {
|
|
||||||
log.Printf("OpenAI image test bootstrap warning: %v", bootstrapErr)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fetch chat requirements
|
|
||||||
s.sendEvent(c, TestEvent{Type: "content", Text: "Fetching chat requirements...\n"})
|
|
||||||
chatReqs, err := fetchOpenAIChatRequirements(ctx, client, headers)
|
|
||||||
if err != nil {
|
|
||||||
return s.sendErrorAndEnd(c, fmt.Sprintf("Chat requirements failed: %s", err.Error()))
|
|
||||||
}
|
|
||||||
if chatReqs.Arkose.Required {
|
|
||||||
return s.sendErrorAndEnd(c, "Unsupported challenge: arkose required")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Initialize and prepare conversation
|
|
||||||
s.sendEvent(c, TestEvent{Type: "content", Text: "Preparing image conversation...\n"})
|
|
||||||
parentMessageID := uuid.NewString()
|
|
||||||
proofToken := generateOpenAIProofToken(chatReqs.ProofOfWork.Required, chatReqs.ProofOfWork.Seed, chatReqs.ProofOfWork.Difficulty, headers.Get("User-Agent"))
|
|
||||||
_ = initializeOpenAIImageConversation(ctx, client, headers)
|
|
||||||
conduitToken, err := prepareOpenAIImageConversation(ctx, client, headers, prompt, parentMessageID, chatReqs.Token, proofToken)
|
|
||||||
if err != nil {
|
|
||||||
return s.sendErrorAndEnd(c, fmt.Sprintf("Conversation prepare failed: %s", err.Error()))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Build simplified conversation request (no file uploads)
|
|
||||||
convReq := buildOpenAIImageTestConversationRequest(prompt, parentMessageID)
|
|
||||||
convHeaders := cloneHTTPHeader(headers)
|
|
||||||
convHeaders.Set("Accept", "text/event-stream")
|
|
||||||
convHeaders.Set("Content-Type", "application/json")
|
|
||||||
convHeaders.Set("openai-sentinel-chat-requirements-token", chatReqs.Token)
|
|
||||||
if conduitToken != "" {
|
|
||||||
convHeaders.Set("x-conduit-token", conduitToken)
|
|
||||||
}
|
|
||||||
if proofToken != "" {
|
|
||||||
convHeaders.Set("openai-sentinel-proof-token", proofToken)
|
|
||||||
}
|
|
||||||
|
|
||||||
s.sendEvent(c, TestEvent{Type: "content", Text: "Generating image...\n"})
|
|
||||||
|
|
||||||
resp, err := client.R().
|
|
||||||
SetContext(ctx).
|
|
||||||
DisableAutoReadResponse().
|
|
||||||
SetHeaders(headerToMap(convHeaders)).
|
|
||||||
SetBodyJsonMarshal(convReq).
|
|
||||||
Post(openAIChatGPTConversationURL)
|
|
||||||
if err != nil {
|
|
||||||
return s.sendErrorAndEnd(c, fmt.Sprintf("Conversation request failed: %s", err.Error()))
|
|
||||||
}
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
if resp != nil && resp.Body != nil {
|
if resp != nil && resp.Body != nil {
|
||||||
@@ -1223,49 +1199,35 @@ func (s *AccountTestService) testOpenAIImageOAuth(c *gin.Context, ctx context.Co
|
|||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
if resp.StatusCode >= 400 {
|
if resp.StatusCode >= 400 {
|
||||||
return s.sendErrorAndEnd(c, fmt.Sprintf("Conversation API returned %d", resp.StatusCode))
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 2<<20))
|
||||||
|
message := strings.TrimSpace(extractUpstreamErrorMessage(body))
|
||||||
|
if message == "" {
|
||||||
|
message = fmt.Sprintf("Responses API returned %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
return s.sendErrorAndEnd(c, message)
|
||||||
}
|
}
|
||||||
|
|
||||||
startTime := time.Now()
|
body, err := io.ReadAll(resp.Body)
|
||||||
conversationID, pointerInfos, _, _, err := readOpenAIImageConversationStream(resp, startTime)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return s.sendErrorAndEnd(c, fmt.Sprintf("Stream read failed: %s", err.Error()))
|
return s.sendErrorAndEnd(c, fmt.Sprintf("Failed to read image response: %s", err.Error()))
|
||||||
}
|
}
|
||||||
|
|
||||||
pointerInfos = mergeOpenAIImagePointerInfos(pointerInfos, nil)
|
results, _, _, _, _, err := collectOpenAIImagesFromResponsesBody(body)
|
||||||
if conversationID != "" && !hasOpenAIFileServicePointerInfos(pointerInfos) {
|
if err != nil {
|
||||||
s.sendEvent(c, TestEvent{Type: "content", Text: "Waiting for image generation to complete...\n"})
|
return s.sendErrorAndEnd(c, fmt.Sprintf("Failed to parse image response: %s", err.Error()))
|
||||||
polledPointers, pollErr := pollOpenAIImageConversation(ctx, client, headers, conversationID)
|
|
||||||
if pollErr != nil {
|
|
||||||
return s.sendErrorAndEnd(c, fmt.Sprintf("Poll failed: %s", pollErr.Error()))
|
|
||||||
}
|
|
||||||
pointerInfos = mergeOpenAIImagePointerInfos(pointerInfos, polledPointers)
|
|
||||||
}
|
}
|
||||||
pointerInfos = preferOpenAIFileServicePointerInfos(pointerInfos)
|
if len(results) == 0 {
|
||||||
if len(pointerInfos) == 0 {
|
return s.sendErrorAndEnd(c, "No images returned from responses API")
|
||||||
return s.sendErrorAndEnd(c, "No images returned from conversation")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
s.sendEvent(c, TestEvent{Type: "content", Text: "Downloading generated image...\n"})
|
for _, item := range results {
|
||||||
|
if item.RevisedPrompt != "" {
|
||||||
// Download and encode each image
|
s.sendEvent(c, TestEvent{Type: "content", Text: item.RevisedPrompt})
|
||||||
for _, pointer := range pointerInfos {
|
|
||||||
downloadURL, err := fetchOpenAIImageDownloadURL(ctx, client, headers, conversationID, pointer.Pointer)
|
|
||||||
if err != nil {
|
|
||||||
return s.sendErrorAndEnd(c, fmt.Sprintf("Download URL fetch failed: %s", err.Error()))
|
|
||||||
}
|
|
||||||
data, err := downloadOpenAIImageBytes(ctx, client, headers, downloadURL)
|
|
||||||
if err != nil {
|
|
||||||
return s.sendErrorAndEnd(c, fmt.Sprintf("Image download failed: %s", err.Error()))
|
|
||||||
}
|
|
||||||
b64 := base64.StdEncoding.EncodeToString(data)
|
|
||||||
mimeType := http.DetectContentType(data)
|
|
||||||
if pointer.Prompt != "" {
|
|
||||||
s.sendEvent(c, TestEvent{Type: "content", Text: pointer.Prompt})
|
|
||||||
}
|
}
|
||||||
|
mimeType := openAIImageOutputMIMEType(item.OutputFormat)
|
||||||
s.sendEvent(c, TestEvent{
|
s.sendEvent(c, TestEvent{
|
||||||
Type: "image",
|
Type: "image",
|
||||||
ImageURL: "data:" + mimeType + ";base64," + b64,
|
ImageURL: "data:" + mimeType + ";base64," + item.Result,
|
||||||
MimeType: mimeType,
|
MimeType: mimeType,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -1274,107 +1236,6 @@ func (s *AccountTestService) testOpenAIImageOAuth(c *gin.Context, ctx context.Co
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// buildOpenAIBackendAPIHeadersForTest builds ChatGPT backend API headers for test purposes.
|
|
||||||
// Replicates the logic from OpenAIGatewayService.buildOpenAIBackendAPIHeaders without
|
|
||||||
// requiring the full gateway service dependency.
|
|
||||||
func buildOpenAIBackendAPIHeadersForTest(ctx context.Context, account *Account, token string, repo AccountRepository) http.Header {
|
|
||||||
// Ensure device and session IDs exist
|
|
||||||
deviceID := account.GetOpenAIDeviceID()
|
|
||||||
sessionID := account.GetOpenAISessionID()
|
|
||||||
if deviceID == "" || sessionID == "" {
|
|
||||||
updates := map[string]any{}
|
|
||||||
if deviceID == "" {
|
|
||||||
deviceID = uuid.NewString()
|
|
||||||
updates["openai_device_id"] = deviceID
|
|
||||||
}
|
|
||||||
if sessionID == "" {
|
|
||||||
sessionID = uuid.NewString()
|
|
||||||
updates["openai_session_id"] = sessionID
|
|
||||||
}
|
|
||||||
if account.Extra == nil {
|
|
||||||
account.Extra = map[string]any{}
|
|
||||||
}
|
|
||||||
for key, value := range updates {
|
|
||||||
account.Extra[key] = value
|
|
||||||
}
|
|
||||||
if repo != nil {
|
|
||||||
updateCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
_ = repo.UpdateExtra(updateCtx, account.ID, updates)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
headers := make(http.Header)
|
|
||||||
headers.Set("Authorization", "Bearer "+token)
|
|
||||||
headers.Set("Accept", "application/json")
|
|
||||||
headers.Set("Origin", "https://chatgpt.com")
|
|
||||||
headers.Set("Referer", "https://chatgpt.com/")
|
|
||||||
headers.Set("Sec-Fetch-Dest", "empty")
|
|
||||||
headers.Set("Sec-Fetch-Mode", "cors")
|
|
||||||
headers.Set("Sec-Fetch-Site", "same-origin")
|
|
||||||
headers.Set("User-Agent", openAIImageBackendUserAgent)
|
|
||||||
if customUA := strings.TrimSpace(account.GetOpenAIUserAgent()); customUA != "" {
|
|
||||||
headers.Set("User-Agent", customUA)
|
|
||||||
}
|
|
||||||
if chatgptAccountID := strings.TrimSpace(account.GetChatGPTAccountID()); chatgptAccountID != "" {
|
|
||||||
headers.Set("chatgpt-account-id", chatgptAccountID)
|
|
||||||
}
|
|
||||||
if deviceID != "" {
|
|
||||||
headers.Set("oai-device-id", deviceID)
|
|
||||||
headers.Set("Cookie", "oai-did="+deviceID)
|
|
||||||
}
|
|
||||||
if sessionID != "" {
|
|
||||||
headers.Set("oai-session-id", sessionID)
|
|
||||||
}
|
|
||||||
return headers
|
|
||||||
}
|
|
||||||
|
|
||||||
// buildOpenAIImageTestConversationRequest creates a simplified image generation conversation request.
|
|
||||||
func buildOpenAIImageTestConversationRequest(prompt, parentMessageID string) map[string]any {
|
|
||||||
promptText := strings.TrimSpace(prompt)
|
|
||||||
if promptText == "" {
|
|
||||||
promptText = "Generate an image."
|
|
||||||
}
|
|
||||||
metadata := map[string]any{
|
|
||||||
"developer_mode_connector_ids": []any{},
|
|
||||||
"selected_github_repos": []any{},
|
|
||||||
"selected_all_github_repos": false,
|
|
||||||
"system_hints": []string{"picture_v2"},
|
|
||||||
"serialization_metadata": map[string]any{
|
|
||||||
"custom_symbol_offsets": []any{},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
message := map[string]any{
|
|
||||||
"id": uuid.NewString(),
|
|
||||||
"author": map[string]any{"role": "user"},
|
|
||||||
"content": map[string]any{
|
|
||||||
"content_type": "text",
|
|
||||||
"parts": []any{promptText},
|
|
||||||
},
|
|
||||||
"metadata": metadata,
|
|
||||||
"create_time": float64(time.Now().UnixMilli()) / 1000,
|
|
||||||
}
|
|
||||||
return map[string]any{
|
|
||||||
"action": "next",
|
|
||||||
"client_prepare_state": "sent",
|
|
||||||
"parent_message_id": parentMessageID,
|
|
||||||
"messages": []any{message},
|
|
||||||
"model": "auto",
|
|
||||||
"timezone_offset_min": openAITimezoneOffsetMinutes(),
|
|
||||||
"timezone": openAITimezoneName(),
|
|
||||||
"conversation_mode": map[string]any{"kind": "primary_assistant"},
|
|
||||||
"system_hints": []string{"picture_v2"},
|
|
||||||
"supports_buffering": true,
|
|
||||||
"supported_encodings": []string{"v1"},
|
|
||||||
"client_contextual_info": map[string]any{"app_name": "chatgpt.com"},
|
|
||||||
"force_nulligen": false,
|
|
||||||
"force_paragen": false,
|
|
||||||
"force_paragen_model_slug": "",
|
|
||||||
"force_rate_limit": false,
|
|
||||||
"websocket_request_id": uuid.NewString(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *AccountTestService) sendEvent(c *gin.Context, event TestEvent) {
|
func (s *AccountTestService) sendEvent(c *gin.Context, event TestEvent) {
|
||||||
eventJSON, _ := json.Marshal(event)
|
eventJSON, _ := json.Marshal(event)
|
||||||
if _, err := fmt.Fprintf(c.Writer, "data: %s\n\n", eventJSON); err != nil {
|
if _, err := fmt.Fprintf(c.Writer, "data: %s\n\n", eventJSON); err != nil {
|
||||||
|
|||||||
@@ -0,0 +1,50 @@
|
|||||||
|
package service
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/gin-gonic/gin"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAccountTestService_OpenAIImageOAuthHandlesOutputItemDoneFallback(t *testing.T) {
|
||||||
|
gin.SetMode(gin.TestMode)
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
c, _ := gin.CreateTestContext(rec)
|
||||||
|
c.Request = httptest.NewRequest(http.MethodPost, "/api/v1/admin/accounts/1/test", nil)
|
||||||
|
|
||||||
|
upstream := &httpUpstreamRecorder{
|
||||||
|
resp: &http.Response{
|
||||||
|
StatusCode: http.StatusOK,
|
||||||
|
Header: http.Header{
|
||||||
|
"Content-Type": []string{"text/event-stream"},
|
||||||
|
},
|
||||||
|
Body: io.NopCloser(strings.NewReader(
|
||||||
|
"data: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"ig_123\",\"type\":\"image_generation_call\",\"result\":\"aGVsbG8=\",\"revised_prompt\":\"draw a cat\",\"output_format\":\"png\"}}\n\n" +
|
||||||
|
"data: {\"type\":\"response.completed\",\"response\":{\"created_at\":1710000006,\"tool_usage\":{\"image_gen\":{\"images\":1}},\"output\":[]}}\n\n" +
|
||||||
|
"data: [DONE]\n\n",
|
||||||
|
)),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
svc := &AccountTestService{httpUpstream: upstream}
|
||||||
|
account := &Account{
|
||||||
|
ID: 53,
|
||||||
|
Name: "openai-oauth",
|
||||||
|
Platform: PlatformOpenAI,
|
||||||
|
Type: AccountTypeOAuth,
|
||||||
|
Credentials: map[string]any{
|
||||||
|
"access_token": "token-123",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
err := svc.testOpenAIImageOAuth(c, context.Background(), account, "gpt-image-2", "draw a cat")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Contains(t, rec.Body.String(), "Calling Codex /responses image tool")
|
||||||
|
require.Contains(t, rec.Body.String(), "data:image/png;base64,aGVsbG8=")
|
||||||
|
require.Contains(t, rec.Body.String(), "\"success\":true")
|
||||||
|
}
|
||||||
@@ -50,6 +50,7 @@ const (
|
|||||||
openAIImageLifecycleTimeout = 2 * time.Minute
|
openAIImageLifecycleTimeout = 2 * time.Minute
|
||||||
openAIImageMaxDownloadBytes = 20 << 20 // 20MB per image download
|
openAIImageMaxDownloadBytes = 20 << 20 // 20MB per image download
|
||||||
openAIImageMaxUploadPartSize = 20 << 20 // 20MB per multipart upload part
|
openAIImageMaxUploadPartSize = 20 << 20 // 20MB per multipart upload part
|
||||||
|
openAIImagesResponsesMainModel = "gpt-5.4-mini"
|
||||||
)
|
)
|
||||||
|
|
||||||
type OpenAIImagesCapability string
|
type OpenAIImagesCapability string
|
||||||
@@ -81,10 +82,21 @@ type OpenAIImagesRequest struct {
|
|||||||
ExplicitSize bool
|
ExplicitSize bool
|
||||||
SizeTier string
|
SizeTier string
|
||||||
ResponseFormat string
|
ResponseFormat string
|
||||||
|
Quality string
|
||||||
|
Background string
|
||||||
|
OutputFormat string
|
||||||
|
Moderation string
|
||||||
|
InputFidelity string
|
||||||
|
Style string
|
||||||
|
OutputCompression *int
|
||||||
|
PartialImages *int
|
||||||
HasMask bool
|
HasMask bool
|
||||||
HasNativeOptions bool
|
HasNativeOptions bool
|
||||||
RequiredCapability OpenAIImagesCapability
|
RequiredCapability OpenAIImagesCapability
|
||||||
|
InputImageURLs []string
|
||||||
|
MaskImageURL string
|
||||||
Uploads []OpenAIImagesUpload
|
Uploads []OpenAIImagesUpload
|
||||||
|
MaskUpload *OpenAIImagesUpload
|
||||||
Body []byte
|
Body []byte
|
||||||
bodyHash string
|
bodyHash string
|
||||||
}
|
}
|
||||||
@@ -188,7 +200,54 @@ func parseOpenAIImagesJSONRequest(body []byte, req *OpenAIImagesRequest) error {
|
|||||||
req.ExplicitSize = req.Size != ""
|
req.ExplicitSize = req.Size != ""
|
||||||
}
|
}
|
||||||
req.ResponseFormat = strings.ToLower(strings.TrimSpace(gjson.GetBytes(body, "response_format").String()))
|
req.ResponseFormat = strings.ToLower(strings.TrimSpace(gjson.GetBytes(body, "response_format").String()))
|
||||||
|
req.Quality = strings.TrimSpace(gjson.GetBytes(body, "quality").String())
|
||||||
|
req.Background = strings.TrimSpace(gjson.GetBytes(body, "background").String())
|
||||||
|
req.OutputFormat = strings.TrimSpace(gjson.GetBytes(body, "output_format").String())
|
||||||
|
req.Moderation = strings.TrimSpace(gjson.GetBytes(body, "moderation").String())
|
||||||
|
req.InputFidelity = strings.TrimSpace(gjson.GetBytes(body, "input_fidelity").String())
|
||||||
|
req.Style = strings.TrimSpace(gjson.GetBytes(body, "style").String())
|
||||||
req.HasMask = gjson.GetBytes(body, "mask").Exists()
|
req.HasMask = gjson.GetBytes(body, "mask").Exists()
|
||||||
|
if outputCompression := gjson.GetBytes(body, "output_compression"); outputCompression.Exists() {
|
||||||
|
if outputCompression.Type != gjson.Number {
|
||||||
|
return fmt.Errorf("invalid output_compression field type")
|
||||||
|
}
|
||||||
|
v := int(outputCompression.Int())
|
||||||
|
req.OutputCompression = &v
|
||||||
|
}
|
||||||
|
if partialImages := gjson.GetBytes(body, "partial_images"); partialImages.Exists() {
|
||||||
|
if partialImages.Type != gjson.Number {
|
||||||
|
return fmt.Errorf("invalid partial_images field type")
|
||||||
|
}
|
||||||
|
v := int(partialImages.Int())
|
||||||
|
req.PartialImages = &v
|
||||||
|
}
|
||||||
|
if req.IsEdits() {
|
||||||
|
images := gjson.GetBytes(body, "images")
|
||||||
|
if images.Exists() {
|
||||||
|
if !images.IsArray() {
|
||||||
|
return fmt.Errorf("invalid images field type")
|
||||||
|
}
|
||||||
|
for _, item := range images.Array() {
|
||||||
|
if imageURL := strings.TrimSpace(item.Get("image_url").String()); imageURL != "" {
|
||||||
|
req.InputImageURLs = append(req.InputImageURLs, imageURL)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if item.Get("file_id").Exists() {
|
||||||
|
return fmt.Errorf("images[].file_id is not supported (use images[].image_url instead)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if maskImageURL := strings.TrimSpace(gjson.GetBytes(body, "mask.image_url").String()); maskImageURL != "" {
|
||||||
|
req.MaskImageURL = maskImageURL
|
||||||
|
req.HasMask = true
|
||||||
|
}
|
||||||
|
if gjson.GetBytes(body, "mask.file_id").Exists() {
|
||||||
|
return fmt.Errorf("mask.file_id is not supported (use mask.image_url instead)")
|
||||||
|
}
|
||||||
|
if len(req.InputImageURLs) == 0 {
|
||||||
|
return fmt.Errorf("images[].image_url is required")
|
||||||
|
}
|
||||||
|
}
|
||||||
req.HasNativeOptions = hasOpenAINativeImageOptions(func(path string) bool {
|
req.HasNativeOptions = hasOpenAINativeImageOptions(func(path string) bool {
|
||||||
return gjson.GetBytes(body, path).Exists()
|
return gjson.GetBytes(body, path).Exists()
|
||||||
})
|
})
|
||||||
@@ -231,6 +290,16 @@ func parseOpenAIImagesMultipartRequest(body []byte, contentType string, req *Ope
|
|||||||
partContentType := strings.TrimSpace(part.Header.Get("Content-Type"))
|
partContentType := strings.TrimSpace(part.Header.Get("Content-Type"))
|
||||||
if name == "mask" && len(data) > 0 {
|
if name == "mask" && len(data) > 0 {
|
||||||
req.HasMask = true
|
req.HasMask = true
|
||||||
|
width, height := parseOpenAIImageDimensions(part.Header)
|
||||||
|
maskUpload := OpenAIImagesUpload{
|
||||||
|
FieldName: name,
|
||||||
|
FileName: fileName,
|
||||||
|
ContentType: partContentType,
|
||||||
|
Data: data,
|
||||||
|
Width: width,
|
||||||
|
Height: height,
|
||||||
|
}
|
||||||
|
req.MaskUpload = &maskUpload
|
||||||
}
|
}
|
||||||
if name == "image" || strings.HasPrefix(name, "image[") {
|
if name == "image" || strings.HasPrefix(name, "image[") {
|
||||||
width, height := parseOpenAIImageDimensions(part.Header)
|
width, height := parseOpenAIImageDimensions(part.Header)
|
||||||
@@ -270,6 +339,38 @@ func parseOpenAIImagesMultipartRequest(body []byte, contentType string, req *Ope
|
|||||||
return fmt.Errorf("n must be a positive integer")
|
return fmt.Errorf("n must be a positive integer")
|
||||||
}
|
}
|
||||||
req.N = n
|
req.N = n
|
||||||
|
case "quality":
|
||||||
|
req.Quality = value
|
||||||
|
req.HasNativeOptions = true
|
||||||
|
case "background":
|
||||||
|
req.Background = value
|
||||||
|
req.HasNativeOptions = true
|
||||||
|
case "output_format":
|
||||||
|
req.OutputFormat = value
|
||||||
|
req.HasNativeOptions = true
|
||||||
|
case "moderation":
|
||||||
|
req.Moderation = value
|
||||||
|
req.HasNativeOptions = true
|
||||||
|
case "input_fidelity":
|
||||||
|
req.InputFidelity = value
|
||||||
|
req.HasNativeOptions = true
|
||||||
|
case "style":
|
||||||
|
req.Style = value
|
||||||
|
req.HasNativeOptions = true
|
||||||
|
case "output_compression":
|
||||||
|
n, err := strconv.Atoi(value)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("invalid output_compression field value")
|
||||||
|
}
|
||||||
|
req.OutputCompression = &n
|
||||||
|
req.HasNativeOptions = true
|
||||||
|
case "partial_images":
|
||||||
|
n, err := strconv.Atoi(value)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("invalid partial_images field value")
|
||||||
|
}
|
||||||
|
req.PartialImages = &n
|
||||||
|
req.HasNativeOptions = true
|
||||||
default:
|
default:
|
||||||
if isOpenAINativeImageOption(name) && value != "" {
|
if isOpenAINativeImageOption(name) && value != "" {
|
||||||
req.HasNativeOptions = true
|
req.HasNativeOptions = true
|
||||||
@@ -359,6 +460,8 @@ func hasOpenAINativeImageOptions(exists func(path string) bool) bool {
|
|||||||
"output_format",
|
"output_format",
|
||||||
"output_compression",
|
"output_compression",
|
||||||
"moderation",
|
"moderation",
|
||||||
|
"input_fidelity",
|
||||||
|
"partial_images",
|
||||||
} {
|
} {
|
||||||
if exists(path) {
|
if exists(path) {
|
||||||
return true
|
return true
|
||||||
@@ -369,7 +472,7 @@ func hasOpenAINativeImageOptions(exists func(path string) bool) bool {
|
|||||||
|
|
||||||
func isOpenAINativeImageOption(name string) bool {
|
func isOpenAINativeImageOption(name string) bool {
|
||||||
switch strings.TrimSpace(strings.ToLower(name)) {
|
switch strings.TrimSpace(strings.ToLower(name)) {
|
||||||
case "background", "quality", "style", "output_format", "output_compression", "moderation":
|
case "background", "quality", "style", "output_format", "output_compression", "moderation", "input_fidelity", "partial_images":
|
||||||
return true
|
return true
|
||||||
default:
|
default:
|
||||||
return false
|
return false
|
||||||
@@ -782,156 +885,6 @@ func extractOpenAIImageCountFromJSONBytes(body []byte) int {
|
|||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *OpenAIGatewayService) forwardOpenAIImagesOAuth(
|
|
||||||
ctx context.Context,
|
|
||||||
c *gin.Context,
|
|
||||||
account *Account,
|
|
||||||
parsed *OpenAIImagesRequest,
|
|
||||||
channelMappedModel string,
|
|
||||||
) (*OpenAIForwardResult, error) {
|
|
||||||
startTime := time.Now()
|
|
||||||
requestModel := strings.TrimSpace(parsed.Model)
|
|
||||||
if mapped := strings.TrimSpace(channelMappedModel); mapped != "" {
|
|
||||||
requestModel = mapped
|
|
||||||
}
|
|
||||||
if err := validateOpenAIImagesModel(requestModel); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
logger.LegacyPrintf(
|
|
||||||
"service.openai_gateway",
|
|
||||||
"[OpenAI] Images request routing request_model=%s endpoint=%s account_type=%s uploads=%d",
|
|
||||||
requestModel,
|
|
||||||
parsed.Endpoint,
|
|
||||||
account.Type,
|
|
||||||
len(parsed.Uploads),
|
|
||||||
)
|
|
||||||
|
|
||||||
token, _, err := s.GetAccessToken(ctx, account)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
client, err := newOpenAIBackendAPIClient(resolveOpenAIProxyURL(account))
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
headers, err := s.buildOpenAIBackendAPIHeaders(account, token)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if bootstrapErr := bootstrapOpenAIBackendAPI(ctx, client, headers); bootstrapErr != nil {
|
|
||||||
logger.LegacyPrintf("service.openai_gateway", "OpenAI image bootstrap failed: %v", bootstrapErr)
|
|
||||||
}
|
|
||||||
|
|
||||||
chatReqs, err := fetchOpenAIChatRequirements(ctx, client, headers)
|
|
||||||
if err != nil {
|
|
||||||
return nil, s.wrapOpenAIImageBackendError(ctx, c, account, err)
|
|
||||||
}
|
|
||||||
if chatReqs.Arkose.Required {
|
|
||||||
return nil, s.wrapOpenAIImageBackendError(
|
|
||||||
ctx,
|
|
||||||
c,
|
|
||||||
account,
|
|
||||||
newOpenAIImageSyntheticStatusError(
|
|
||||||
http.StatusForbidden,
|
|
||||||
"chat-requirements requires unsupported challenge (arkose)",
|
|
||||||
openAIChatGPTChatRequirementsURL,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
parentMessageID := uuid.NewString()
|
|
||||||
proofToken := generateOpenAIProofToken(chatReqs.ProofOfWork.Required, chatReqs.ProofOfWork.Seed, chatReqs.ProofOfWork.Difficulty, headers.Get("User-Agent"))
|
|
||||||
_ = initializeOpenAIImageConversation(ctx, client, headers)
|
|
||||||
conduitToken, err := prepareOpenAIImageConversation(ctx, client, headers, parsed.Prompt, parentMessageID, chatReqs.Token, proofToken)
|
|
||||||
if err != nil {
|
|
||||||
return nil, s.wrapOpenAIImageBackendError(ctx, c, account, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
uploads, err := uploadOpenAIImageFiles(ctx, client, headers, parsed.Uploads)
|
|
||||||
if err != nil {
|
|
||||||
return nil, s.wrapOpenAIImageBackendError(ctx, c, account, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
convReq := buildOpenAIImageConversationRequest(parsed, parentMessageID, uploads)
|
|
||||||
if parsedContent, err := json.Marshal(convReq); err == nil {
|
|
||||||
setOpsUpstreamRequestBody(c, parsedContent)
|
|
||||||
}
|
|
||||||
convHeaders := cloneHTTPHeader(headers)
|
|
||||||
convHeaders.Set("Accept", "text/event-stream")
|
|
||||||
convHeaders.Set("Content-Type", "application/json")
|
|
||||||
convHeaders.Set("openai-sentinel-chat-requirements-token", chatReqs.Token)
|
|
||||||
if conduitToken != "" {
|
|
||||||
convHeaders.Set("x-conduit-token", conduitToken)
|
|
||||||
}
|
|
||||||
if proofToken != "" {
|
|
||||||
convHeaders.Set("openai-sentinel-proof-token", proofToken)
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := client.R().
|
|
||||||
SetContext(ctx).
|
|
||||||
DisableAutoReadResponse().
|
|
||||||
SetHeaders(headerToMap(convHeaders)).
|
|
||||||
SetBodyJsonMarshal(convReq).
|
|
||||||
Post(openAIChatGPTConversationURL)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("openai image conversation request failed: %w", err)
|
|
||||||
}
|
|
||||||
defer func() {
|
|
||||||
if resp != nil && resp.Body != nil {
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
if resp.StatusCode >= 400 {
|
|
||||||
return nil, s.wrapOpenAIImageBackendError(ctx, c, account, handleOpenAIImageBackendError(resp))
|
|
||||||
}
|
|
||||||
|
|
||||||
conversationID, pointerInfos, usage, firstTokenMs, err := readOpenAIImageConversationStream(resp, startTime)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
pointerInfos = mergeOpenAIImagePointerInfos(pointerInfos, nil)
|
|
||||||
logger.LegacyPrintf(
|
|
||||||
"service.openai_gateway",
|
|
||||||
"[OpenAI] Image extraction stream conversation_id=%s total_assets=%d file_service_assets=%d direct_assets=%d",
|
|
||||||
conversationID,
|
|
||||||
len(pointerInfos),
|
|
||||||
countOpenAIFileServicePointerInfos(pointerInfos),
|
|
||||||
countOpenAIDirectImageAssets(pointerInfos),
|
|
||||||
)
|
|
||||||
lifecycleCtx, releaseLifecycleCtx := detachOpenAIImageLifecycleContext(ctx, openAIImageLifecycleTimeout)
|
|
||||||
defer releaseLifecycleCtx()
|
|
||||||
if conversationID != "" && !hasOpenAIFileServicePointerInfos(pointerInfos) {
|
|
||||||
polledPointers, pollErr := pollOpenAIImageConversation(lifecycleCtx, client, headers, conversationID)
|
|
||||||
if pollErr != nil {
|
|
||||||
return nil, s.wrapOpenAIImageBackendError(ctx, c, account, pollErr)
|
|
||||||
}
|
|
||||||
pointerInfos = mergeOpenAIImagePointerInfos(pointerInfos, polledPointers)
|
|
||||||
}
|
|
||||||
pointerInfos = preferOpenAIFileServicePointerInfos(pointerInfos)
|
|
||||||
if len(pointerInfos) == 0 {
|
|
||||||
logger.LegacyPrintf("service.openai_gateway", "[OpenAI] Image extraction yielded no assets conversation_id=%s", conversationID)
|
|
||||||
return nil, fmt.Errorf("openai image conversation returned no downloadable images")
|
|
||||||
}
|
|
||||||
|
|
||||||
responseBody, imageCount, err := buildOpenAIImageResponse(lifecycleCtx, client, headers, conversationID, pointerInfos)
|
|
||||||
if err != nil {
|
|
||||||
return nil, s.wrapOpenAIImageBackendError(ctx, c, account, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
c.Data(http.StatusOK, "application/json; charset=utf-8", responseBody)
|
|
||||||
return &OpenAIForwardResult{
|
|
||||||
RequestID: resp.Header.Get("x-request-id"),
|
|
||||||
Usage: usage,
|
|
||||||
Model: requestModel,
|
|
||||||
UpstreamModel: requestModel,
|
|
||||||
Stream: false,
|
|
||||||
Duration: time.Since(startTime),
|
|
||||||
FirstTokenMs: firstTokenMs,
|
|
||||||
ImageCount: imageCount,
|
|
||||||
ImageSize: parsed.SizeTier,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func resolveOpenAIProxyURL(account *Account) string {
|
func resolveOpenAIProxyURL(account *Account) string {
|
||||||
if account != nil && account.ProxyID != nil && account.Proxy != nil {
|
if account != nil && account.ProxyID != nil && account.Proxy != nil {
|
||||||
return account.Proxy.URL()
|
return account.Proxy.URL()
|
||||||
|
|||||||
692
backend/internal/service/openai_images_responses.go
Normal file
692
backend/internal/service/openai_images_responses.go
Normal file
@@ -0,0 +1,692 @@
|
|||||||
|
package service
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/Wei-Shaw/sub2api/internal/pkg/logger"
|
||||||
|
"github.com/Wei-Shaw/sub2api/internal/util/responseheaders"
|
||||||
|
"github.com/gin-gonic/gin"
|
||||||
|
"github.com/tidwall/gjson"
|
||||||
|
"github.com/tidwall/sjson"
|
||||||
|
)
|
||||||
|
|
||||||
|
type openAIResponsesImageResult struct {
|
||||||
|
Result string
|
||||||
|
RevisedPrompt string
|
||||||
|
OutputFormat string
|
||||||
|
Size string
|
||||||
|
Background string
|
||||||
|
Quality string
|
||||||
|
}
|
||||||
|
|
||||||
|
func openAIResponsesImageResultKey(itemID string, result openAIResponsesImageResult) string {
|
||||||
|
if strings.TrimSpace(result.Result) != "" {
|
||||||
|
return strings.TrimSpace(result.OutputFormat) + "|" + strings.TrimSpace(result.Result)
|
||||||
|
}
|
||||||
|
return "item:" + strings.TrimSpace(itemID)
|
||||||
|
}
|
||||||
|
|
||||||
|
func appendOpenAIResponsesImageResultDedup(results *[]openAIResponsesImageResult, seen map[string]struct{}, itemID string, result openAIResponsesImageResult) bool {
|
||||||
|
if results == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
key := openAIResponsesImageResultKey(itemID, result)
|
||||||
|
if key != "" {
|
||||||
|
if _, exists := seen[key]; exists {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
seen[key] = struct{}{}
|
||||||
|
}
|
||||||
|
*results = append(*results, result)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func openAIImageOutputMIMEType(outputFormat string) string {
|
||||||
|
if outputFormat == "" {
|
||||||
|
return "image/png"
|
||||||
|
}
|
||||||
|
if strings.Contains(outputFormat, "/") {
|
||||||
|
return outputFormat
|
||||||
|
}
|
||||||
|
switch strings.ToLower(strings.TrimSpace(outputFormat)) {
|
||||||
|
case "png":
|
||||||
|
return "image/png"
|
||||||
|
case "jpg", "jpeg":
|
||||||
|
return "image/jpeg"
|
||||||
|
case "webp":
|
||||||
|
return "image/webp"
|
||||||
|
default:
|
||||||
|
return "image/png"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func openAIImageUploadToDataURL(upload OpenAIImagesUpload) (string, error) {
|
||||||
|
if len(upload.Data) == 0 {
|
||||||
|
return "", fmt.Errorf("upload %q is empty", strings.TrimSpace(upload.FileName))
|
||||||
|
}
|
||||||
|
contentType := strings.TrimSpace(upload.ContentType)
|
||||||
|
if contentType == "" {
|
||||||
|
contentType = http.DetectContentType(upload.Data)
|
||||||
|
}
|
||||||
|
return "data:" + contentType + ";base64," + base64.StdEncoding.EncodeToString(upload.Data), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildOpenAIImagesResponsesRequest(parsed *OpenAIImagesRequest, toolModel string) ([]byte, error) {
|
||||||
|
if parsed == nil {
|
||||||
|
return nil, fmt.Errorf("parsed images request is required")
|
||||||
|
}
|
||||||
|
prompt := strings.TrimSpace(parsed.Prompt)
|
||||||
|
if prompt == "" {
|
||||||
|
return nil, fmt.Errorf("prompt is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
inputImages := make([]string, 0, len(parsed.InputImageURLs)+len(parsed.Uploads))
|
||||||
|
for _, imageURL := range parsed.InputImageURLs {
|
||||||
|
if trimmed := strings.TrimSpace(imageURL); trimmed != "" {
|
||||||
|
inputImages = append(inputImages, trimmed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, upload := range parsed.Uploads {
|
||||||
|
dataURL, err := openAIImageUploadToDataURL(upload)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
inputImages = append(inputImages, dataURL)
|
||||||
|
}
|
||||||
|
if parsed.IsEdits() && len(inputImages) == 0 {
|
||||||
|
return nil, fmt.Errorf("image input is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
req := []byte(`{"instructions":"","stream":true,"reasoning":{"effort":"medium","summary":"auto"},"parallel_tool_calls":true,"include":["reasoning.encrypted_content"],"model":"","store":false,"tool_choice":{"type":"image_generation"}}`)
|
||||||
|
req, _ = sjson.SetBytes(req, "model", openAIImagesResponsesMainModel)
|
||||||
|
|
||||||
|
input := []byte(`[{"type":"message","role":"user","content":[{"type":"input_text","text":""}]}]`)
|
||||||
|
input, _ = sjson.SetBytes(input, "0.content.0.text", prompt)
|
||||||
|
for index, imageURL := range inputImages {
|
||||||
|
part := []byte(`{"type":"input_image","image_url":""}`)
|
||||||
|
part, _ = sjson.SetBytes(part, "image_url", imageURL)
|
||||||
|
input, _ = sjson.SetRawBytes(input, fmt.Sprintf("0.content.%d", index+1), part)
|
||||||
|
}
|
||||||
|
req, _ = sjson.SetRawBytes(req, "input", input)
|
||||||
|
|
||||||
|
action := "generate"
|
||||||
|
if parsed.IsEdits() {
|
||||||
|
action = "edit"
|
||||||
|
}
|
||||||
|
tool := []byte(`{"type":"image_generation","action":"","model":""}`)
|
||||||
|
tool, _ = sjson.SetBytes(tool, "action", action)
|
||||||
|
tool, _ = sjson.SetBytes(tool, "model", strings.TrimSpace(toolModel))
|
||||||
|
|
||||||
|
for _, field := range []struct {
|
||||||
|
path string
|
||||||
|
value string
|
||||||
|
}{
|
||||||
|
{path: "size", value: parsed.Size},
|
||||||
|
{path: "quality", value: parsed.Quality},
|
||||||
|
{path: "background", value: parsed.Background},
|
||||||
|
{path: "output_format", value: parsed.OutputFormat},
|
||||||
|
{path: "moderation", value: parsed.Moderation},
|
||||||
|
{path: "input_fidelity", value: parsed.InputFidelity},
|
||||||
|
{path: "style", value: parsed.Style},
|
||||||
|
} {
|
||||||
|
if trimmed := strings.TrimSpace(field.value); trimmed != "" {
|
||||||
|
tool, _ = sjson.SetBytes(tool, field.path, trimmed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if parsed.N > 1 {
|
||||||
|
return nil, fmt.Errorf("codex /responses image tool currently supports only n=1")
|
||||||
|
}
|
||||||
|
if parsed.OutputCompression != nil {
|
||||||
|
tool, _ = sjson.SetBytes(tool, "output_compression", *parsed.OutputCompression)
|
||||||
|
}
|
||||||
|
if parsed.PartialImages != nil {
|
||||||
|
tool, _ = sjson.SetBytes(tool, "partial_images", *parsed.PartialImages)
|
||||||
|
}
|
||||||
|
|
||||||
|
maskImageURL := strings.TrimSpace(parsed.MaskImageURL)
|
||||||
|
if parsed.MaskUpload != nil {
|
||||||
|
dataURL, err := openAIImageUploadToDataURL(*parsed.MaskUpload)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
maskImageURL = dataURL
|
||||||
|
}
|
||||||
|
if maskImageURL != "" {
|
||||||
|
tool, _ = sjson.SetBytes(tool, "input_image_mask.image_url", maskImageURL)
|
||||||
|
}
|
||||||
|
|
||||||
|
req, _ = sjson.SetRawBytes(req, "tools", []byte(`[]`))
|
||||||
|
req, _ = sjson.SetRawBytes(req, "tools.-1", tool)
|
||||||
|
return req, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractOpenAIImagesFromResponsesCompleted(payload []byte) ([]openAIResponsesImageResult, int64, []byte, openAIResponsesImageResult, error) {
|
||||||
|
if gjson.GetBytes(payload, "type").String() != "response.completed" {
|
||||||
|
return nil, 0, nil, openAIResponsesImageResult{}, fmt.Errorf("unexpected event type")
|
||||||
|
}
|
||||||
|
|
||||||
|
createdAt := gjson.GetBytes(payload, "response.created_at").Int()
|
||||||
|
if createdAt <= 0 {
|
||||||
|
createdAt = time.Now().Unix()
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
results []openAIResponsesImageResult
|
||||||
|
firstMeta openAIResponsesImageResult
|
||||||
|
)
|
||||||
|
output := gjson.GetBytes(payload, "response.output")
|
||||||
|
if output.IsArray() {
|
||||||
|
for _, item := range output.Array() {
|
||||||
|
if item.Get("type").String() != "image_generation_call" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
result := strings.TrimSpace(item.Get("result").String())
|
||||||
|
if result == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
entry := openAIResponsesImageResult{
|
||||||
|
Result: result,
|
||||||
|
RevisedPrompt: strings.TrimSpace(item.Get("revised_prompt").String()),
|
||||||
|
OutputFormat: strings.TrimSpace(item.Get("output_format").String()),
|
||||||
|
Size: strings.TrimSpace(item.Get("size").String()),
|
||||||
|
Background: strings.TrimSpace(item.Get("background").String()),
|
||||||
|
Quality: strings.TrimSpace(item.Get("quality").String()),
|
||||||
|
}
|
||||||
|
if len(results) == 0 {
|
||||||
|
firstMeta = entry
|
||||||
|
}
|
||||||
|
results = append(results, entry)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var usageRaw []byte
|
||||||
|
if usage := gjson.GetBytes(payload, "response.tool_usage.image_gen"); usage.Exists() && usage.IsObject() {
|
||||||
|
usageRaw = []byte(usage.Raw)
|
||||||
|
}
|
||||||
|
return results, createdAt, usageRaw, firstMeta, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractOpenAIImageFromResponsesOutputItemDone(payload []byte) (openAIResponsesImageResult, string, bool, error) {
|
||||||
|
if gjson.GetBytes(payload, "type").String() != "response.output_item.done" {
|
||||||
|
return openAIResponsesImageResult{}, "", false, fmt.Errorf("unexpected event type")
|
||||||
|
}
|
||||||
|
|
||||||
|
item := gjson.GetBytes(payload, "item")
|
||||||
|
if !item.Exists() || item.Get("type").String() != "image_generation_call" {
|
||||||
|
return openAIResponsesImageResult{}, "", false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
result := strings.TrimSpace(item.Get("result").String())
|
||||||
|
if result == "" {
|
||||||
|
return openAIResponsesImageResult{}, "", false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
entry := openAIResponsesImageResult{
|
||||||
|
Result: result,
|
||||||
|
RevisedPrompt: strings.TrimSpace(item.Get("revised_prompt").String()),
|
||||||
|
OutputFormat: strings.TrimSpace(item.Get("output_format").String()),
|
||||||
|
Size: strings.TrimSpace(item.Get("size").String()),
|
||||||
|
Background: strings.TrimSpace(item.Get("background").String()),
|
||||||
|
Quality: strings.TrimSpace(item.Get("quality").String()),
|
||||||
|
}
|
||||||
|
return entry, strings.TrimSpace(item.Get("id").String()), true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func collectOpenAIImagesFromResponsesBody(body []byte) ([]openAIResponsesImageResult, int64, []byte, openAIResponsesImageResult, bool, error) {
|
||||||
|
var (
|
||||||
|
fallbackResults []openAIResponsesImageResult
|
||||||
|
fallbackSeen = make(map[string]struct{})
|
||||||
|
createdAt int64
|
||||||
|
usageRaw []byte
|
||||||
|
foundFinal bool
|
||||||
|
)
|
||||||
|
|
||||||
|
for _, line := range bytes.Split(body, []byte("\n")) {
|
||||||
|
line = bytes.TrimRight(line, "\r")
|
||||||
|
data, ok := extractOpenAISSEDataLine(string(line))
|
||||||
|
if !ok || data == "" || data == "[DONE]" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
payload := []byte(data)
|
||||||
|
if !gjson.ValidBytes(payload) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
switch gjson.GetBytes(payload, "type").String() {
|
||||||
|
case "response.created":
|
||||||
|
if createdAt <= 0 {
|
||||||
|
createdAt = gjson.GetBytes(payload, "response.created_at").Int()
|
||||||
|
}
|
||||||
|
case "response.output_item.done":
|
||||||
|
result, itemID, ok, err := extractOpenAIImageFromResponsesOutputItemDone(payload)
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, nil, openAIResponsesImageResult{}, false, err
|
||||||
|
}
|
||||||
|
if ok {
|
||||||
|
appendOpenAIResponsesImageResultDedup(&fallbackResults, fallbackSeen, itemID, result)
|
||||||
|
}
|
||||||
|
case "response.completed":
|
||||||
|
results, completedAt, completedUsageRaw, firstMeta, err := extractOpenAIImagesFromResponsesCompleted(payload)
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, nil, openAIResponsesImageResult{}, false, err
|
||||||
|
}
|
||||||
|
foundFinal = true
|
||||||
|
if completedAt > 0 {
|
||||||
|
createdAt = completedAt
|
||||||
|
}
|
||||||
|
if len(completedUsageRaw) > 0 {
|
||||||
|
usageRaw = completedUsageRaw
|
||||||
|
}
|
||||||
|
if len(results) > 0 {
|
||||||
|
return results, createdAt, usageRaw, firstMeta, true, nil
|
||||||
|
}
|
||||||
|
if len(fallbackResults) > 0 {
|
||||||
|
return fallbackResults, createdAt, usageRaw, fallbackResults[0], true, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(fallbackResults) > 0 {
|
||||||
|
return fallbackResults, createdAt, usageRaw, fallbackResults[0], foundFinal, nil
|
||||||
|
}
|
||||||
|
return nil, createdAt, usageRaw, openAIResponsesImageResult{}, foundFinal, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildOpenAIImagesAPIResponse(
|
||||||
|
results []openAIResponsesImageResult,
|
||||||
|
createdAt int64,
|
||||||
|
usageRaw []byte,
|
||||||
|
firstMeta openAIResponsesImageResult,
|
||||||
|
responseFormat string,
|
||||||
|
) ([]byte, error) {
|
||||||
|
if createdAt <= 0 {
|
||||||
|
createdAt = time.Now().Unix()
|
||||||
|
}
|
||||||
|
out := []byte(`{"created":0,"data":[]}`)
|
||||||
|
out, _ = sjson.SetBytes(out, "created", createdAt)
|
||||||
|
|
||||||
|
format := strings.ToLower(strings.TrimSpace(responseFormat))
|
||||||
|
if format == "" {
|
||||||
|
format = "b64_json"
|
||||||
|
}
|
||||||
|
for _, img := range results {
|
||||||
|
item := []byte(`{}`)
|
||||||
|
if format == "url" {
|
||||||
|
item, _ = sjson.SetBytes(item, "url", "data:"+openAIImageOutputMIMEType(img.OutputFormat)+";base64,"+img.Result)
|
||||||
|
} else {
|
||||||
|
item, _ = sjson.SetBytes(item, "b64_json", img.Result)
|
||||||
|
}
|
||||||
|
if img.RevisedPrompt != "" {
|
||||||
|
item, _ = sjson.SetBytes(item, "revised_prompt", img.RevisedPrompt)
|
||||||
|
}
|
||||||
|
out, _ = sjson.SetRawBytes(out, "data.-1", item)
|
||||||
|
}
|
||||||
|
if firstMeta.Background != "" {
|
||||||
|
out, _ = sjson.SetBytes(out, "background", firstMeta.Background)
|
||||||
|
}
|
||||||
|
if firstMeta.OutputFormat != "" {
|
||||||
|
out, _ = sjson.SetBytes(out, "output_format", firstMeta.OutputFormat)
|
||||||
|
}
|
||||||
|
if firstMeta.Quality != "" {
|
||||||
|
out, _ = sjson.SetBytes(out, "quality", firstMeta.Quality)
|
||||||
|
}
|
||||||
|
if firstMeta.Size != "" {
|
||||||
|
out, _ = sjson.SetBytes(out, "size", firstMeta.Size)
|
||||||
|
}
|
||||||
|
if len(usageRaw) > 0 && gjson.ValidBytes(usageRaw) {
|
||||||
|
out, _ = sjson.SetRawBytes(out, "usage", usageRaw)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func openAIImagesStreamPrefix(parsed *OpenAIImagesRequest) string {
|
||||||
|
if parsed != nil && parsed.IsEdits() {
|
||||||
|
return "image_edit"
|
||||||
|
}
|
||||||
|
return "image_generation"
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildOpenAIImagesStreamErrorBody(message string) []byte {
|
||||||
|
body := []byte(`{"type":"error","error":{"type":"upstream_error","message":""}}`)
|
||||||
|
if strings.TrimSpace(message) == "" {
|
||||||
|
message = "upstream request failed"
|
||||||
|
}
|
||||||
|
body, _ = sjson.SetBytes(body, "error.message", message)
|
||||||
|
return body
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *OpenAIGatewayService) writeOpenAIImagesStreamEvent(c *gin.Context, flusher http.Flusher, eventName string, payload []byte) error {
|
||||||
|
if strings.TrimSpace(eventName) != "" {
|
||||||
|
if _, err := fmt.Fprintf(c.Writer, "event: %s\n", eventName); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := fmt.Fprintf(c.Writer, "data: %s\n\n", payload); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
flusher.Flush()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *OpenAIGatewayService) handleOpenAIImagesOAuthNonStreamingResponse(
|
||||||
|
resp *http.Response,
|
||||||
|
c *gin.Context,
|
||||||
|
responseFormat string,
|
||||||
|
) (OpenAIUsage, int, error) {
|
||||||
|
body, err := ReadUpstreamResponseBody(resp.Body, s.cfg, c, openAITooLargeError)
|
||||||
|
if err != nil {
|
||||||
|
return OpenAIUsage{}, 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var usage OpenAIUsage
|
||||||
|
for _, line := range bytes.Split(body, []byte("\n")) {
|
||||||
|
line = bytes.TrimRight(line, "\r")
|
||||||
|
data, ok := extractOpenAISSEDataLine(string(line))
|
||||||
|
if !ok || data == "" || data == "[DONE]" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
dataBytes := []byte(data)
|
||||||
|
s.parseSSEUsageBytes(dataBytes, &usage)
|
||||||
|
}
|
||||||
|
results, createdAt, usageRaw, firstMeta, _, err := collectOpenAIImagesFromResponsesBody(body)
|
||||||
|
if err != nil {
|
||||||
|
return OpenAIUsage{}, 0, err
|
||||||
|
}
|
||||||
|
if len(results) == 0 {
|
||||||
|
return OpenAIUsage{}, 0, fmt.Errorf("upstream did not return image output")
|
||||||
|
}
|
||||||
|
|
||||||
|
responseBody, err := buildOpenAIImagesAPIResponse(results, createdAt, usageRaw, firstMeta, responseFormat)
|
||||||
|
if err != nil {
|
||||||
|
return OpenAIUsage{}, 0, err
|
||||||
|
}
|
||||||
|
responseheaders.WriteFilteredHeaders(c.Writer.Header(), resp.Header, s.responseHeaderFilter)
|
||||||
|
c.Data(resp.StatusCode, "application/json; charset=utf-8", responseBody)
|
||||||
|
return usage, len(results), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *OpenAIGatewayService) handleOpenAIImagesOAuthStreamingResponse(
|
||||||
|
resp *http.Response,
|
||||||
|
c *gin.Context,
|
||||||
|
startTime time.Time,
|
||||||
|
responseFormat string,
|
||||||
|
streamPrefix string,
|
||||||
|
) (OpenAIUsage, int, *int, error) {
|
||||||
|
responseheaders.WriteFilteredHeaders(c.Writer.Header(), resp.Header, s.responseHeaderFilter)
|
||||||
|
c.Header("Content-Type", "text/event-stream")
|
||||||
|
c.Header("Cache-Control", "no-cache")
|
||||||
|
c.Header("Connection", "keep-alive")
|
||||||
|
c.Status(resp.StatusCode)
|
||||||
|
|
||||||
|
flusher, ok := c.Writer.(http.Flusher)
|
||||||
|
if !ok {
|
||||||
|
return OpenAIUsage{}, 0, nil, fmt.Errorf("streaming is not supported by response writer")
|
||||||
|
}
|
||||||
|
|
||||||
|
format := strings.ToLower(strings.TrimSpace(responseFormat))
|
||||||
|
if format == "" {
|
||||||
|
format = "b64_json"
|
||||||
|
}
|
||||||
|
|
||||||
|
reader := bufio.NewReader(resp.Body)
|
||||||
|
usage := OpenAIUsage{}
|
||||||
|
imageCount := 0
|
||||||
|
var firstTokenMs *int
|
||||||
|
emitted := make(map[string]struct{})
|
||||||
|
|
||||||
|
for {
|
||||||
|
line, err := reader.ReadBytes('\n')
|
||||||
|
if len(line) > 0 {
|
||||||
|
trimmedLine := strings.TrimRight(string(line), "\r\n")
|
||||||
|
data, ok := extractOpenAISSEDataLine(trimmedLine)
|
||||||
|
if ok && data != "" && data != "[DONE]" {
|
||||||
|
if firstTokenMs == nil {
|
||||||
|
ms := int(time.Since(startTime).Milliseconds())
|
||||||
|
firstTokenMs = &ms
|
||||||
|
}
|
||||||
|
dataBytes := []byte(data)
|
||||||
|
s.parseSSEUsageBytes(dataBytes, &usage)
|
||||||
|
if gjson.ValidBytes(dataBytes) {
|
||||||
|
switch gjson.GetBytes(dataBytes, "type").String() {
|
||||||
|
case "response.image_generation_call.partial_image":
|
||||||
|
b64 := strings.TrimSpace(gjson.GetBytes(dataBytes, "partial_image_b64").String())
|
||||||
|
if b64 != "" {
|
||||||
|
eventName := streamPrefix + ".partial_image"
|
||||||
|
payload := []byte(`{"type":"","partial_image_index":0}`)
|
||||||
|
payload, _ = sjson.SetBytes(payload, "type", eventName)
|
||||||
|
payload, _ = sjson.SetBytes(payload, "partial_image_index", gjson.GetBytes(dataBytes, "partial_image_index").Int())
|
||||||
|
if format == "url" {
|
||||||
|
outputFormat := strings.TrimSpace(gjson.GetBytes(dataBytes, "output_format").String())
|
||||||
|
payload, _ = sjson.SetBytes(payload, "url", "data:"+openAIImageOutputMIMEType(outputFormat)+";base64,"+b64)
|
||||||
|
} else {
|
||||||
|
payload, _ = sjson.SetBytes(payload, "b64_json", b64)
|
||||||
|
}
|
||||||
|
if writeErr := s.writeOpenAIImagesStreamEvent(c, flusher, eventName, payload); writeErr != nil {
|
||||||
|
return OpenAIUsage{}, imageCount, firstTokenMs, writeErr
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "response.output_item.done":
|
||||||
|
img, itemID, ok, extractErr := extractOpenAIImageFromResponsesOutputItemDone(dataBytes)
|
||||||
|
if extractErr != nil {
|
||||||
|
_ = s.writeOpenAIImagesStreamEvent(c, flusher, "error", buildOpenAIImagesStreamErrorBody(extractErr.Error()))
|
||||||
|
return OpenAIUsage{}, imageCount, firstTokenMs, extractErr
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
key := openAIResponsesImageResultKey(itemID, img)
|
||||||
|
if _, exists := emitted[key]; exists {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
eventName := streamPrefix + ".completed"
|
||||||
|
payload := []byte(`{"type":""}`)
|
||||||
|
payload, _ = sjson.SetBytes(payload, "type", eventName)
|
||||||
|
if format == "url" {
|
||||||
|
payload, _ = sjson.SetBytes(payload, "url", "data:"+openAIImageOutputMIMEType(img.OutputFormat)+";base64,"+img.Result)
|
||||||
|
} else {
|
||||||
|
payload, _ = sjson.SetBytes(payload, "b64_json", img.Result)
|
||||||
|
}
|
||||||
|
if img.RevisedPrompt != "" {
|
||||||
|
payload, _ = sjson.SetBytes(payload, "revised_prompt", img.RevisedPrompt)
|
||||||
|
}
|
||||||
|
if writeErr := s.writeOpenAIImagesStreamEvent(c, flusher, eventName, payload); writeErr != nil {
|
||||||
|
return OpenAIUsage{}, imageCount, firstTokenMs, writeErr
|
||||||
|
}
|
||||||
|
emitted[key] = struct{}{}
|
||||||
|
imageCount = len(emitted)
|
||||||
|
case "response.completed":
|
||||||
|
results, _, usageRaw, _, extractErr := extractOpenAIImagesFromResponsesCompleted(dataBytes)
|
||||||
|
if extractErr != nil {
|
||||||
|
_ = s.writeOpenAIImagesStreamEvent(c, flusher, "error", buildOpenAIImagesStreamErrorBody(extractErr.Error()))
|
||||||
|
return OpenAIUsage{}, imageCount, firstTokenMs, extractErr
|
||||||
|
}
|
||||||
|
if len(results) == 0 {
|
||||||
|
if imageCount > 0 {
|
||||||
|
return usage, imageCount, firstTokenMs, nil
|
||||||
|
}
|
||||||
|
err = fmt.Errorf("upstream did not return image output")
|
||||||
|
_ = s.writeOpenAIImagesStreamEvent(c, flusher, "error", buildOpenAIImagesStreamErrorBody(err.Error()))
|
||||||
|
return OpenAIUsage{}, imageCount, firstTokenMs, err
|
||||||
|
}
|
||||||
|
eventName := streamPrefix + ".completed"
|
||||||
|
for _, img := range results {
|
||||||
|
key := openAIResponsesImageResultKey("", img)
|
||||||
|
if _, exists := emitted[key]; exists {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
payload := []byte(`{"type":""}`)
|
||||||
|
payload, _ = sjson.SetBytes(payload, "type", eventName)
|
||||||
|
if format == "url" {
|
||||||
|
payload, _ = sjson.SetBytes(payload, "url", "data:"+openAIImageOutputMIMEType(img.OutputFormat)+";base64,"+img.Result)
|
||||||
|
} else {
|
||||||
|
payload, _ = sjson.SetBytes(payload, "b64_json", img.Result)
|
||||||
|
}
|
||||||
|
if img.RevisedPrompt != "" {
|
||||||
|
payload, _ = sjson.SetBytes(payload, "revised_prompt", img.RevisedPrompt)
|
||||||
|
}
|
||||||
|
if len(usageRaw) > 0 && gjson.ValidBytes(usageRaw) {
|
||||||
|
payload, _ = sjson.SetRawBytes(payload, "usage", usageRaw)
|
||||||
|
}
|
||||||
|
if writeErr := s.writeOpenAIImagesStreamEvent(c, flusher, eventName, payload); writeErr != nil {
|
||||||
|
return OpenAIUsage{}, imageCount, firstTokenMs, writeErr
|
||||||
|
}
|
||||||
|
emitted[key] = struct{}{}
|
||||||
|
}
|
||||||
|
imageCount = len(emitted)
|
||||||
|
return usage, imageCount, firstTokenMs, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err == io.EOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
_ = s.writeOpenAIImagesStreamEvent(c, flusher, "error", buildOpenAIImagesStreamErrorBody(err.Error()))
|
||||||
|
return OpenAIUsage{}, imageCount, firstTokenMs, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if imageCount > 0 {
|
||||||
|
return usage, imageCount, firstTokenMs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
streamErr := fmt.Errorf("stream disconnected before image generation completed")
|
||||||
|
_ = s.writeOpenAIImagesStreamEvent(c, flusher, "error", buildOpenAIImagesStreamErrorBody(streamErr.Error()))
|
||||||
|
return OpenAIUsage{}, imageCount, firstTokenMs, streamErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *OpenAIGatewayService) forwardOpenAIImagesOAuth(
|
||||||
|
ctx context.Context,
|
||||||
|
c *gin.Context,
|
||||||
|
account *Account,
|
||||||
|
parsed *OpenAIImagesRequest,
|
||||||
|
channelMappedModel string,
|
||||||
|
) (*OpenAIForwardResult, error) {
|
||||||
|
startTime := time.Now()
|
||||||
|
requestModel := strings.TrimSpace(parsed.Model)
|
||||||
|
if mapped := strings.TrimSpace(channelMappedModel); mapped != "" {
|
||||||
|
requestModel = mapped
|
||||||
|
}
|
||||||
|
if requestModel == "" {
|
||||||
|
requestModel = "gpt-image-2"
|
||||||
|
}
|
||||||
|
if err := validateOpenAIImagesModel(requestModel); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
logger.LegacyPrintf(
|
||||||
|
"service.openai_gateway",
|
||||||
|
"[OpenAI] Images request routing request_model=%s endpoint=%s account_type=%s uploads=%d",
|
||||||
|
requestModel,
|
||||||
|
parsed.Endpoint,
|
||||||
|
account.Type,
|
||||||
|
len(parsed.Uploads),
|
||||||
|
)
|
||||||
|
|
||||||
|
token, _, err := s.GetAccessToken(ctx, account)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
responsesBody, err := buildOpenAIImagesResponsesRequest(parsed, requestModel)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
setOpsUpstreamRequestBody(c, responsesBody)
|
||||||
|
|
||||||
|
upstreamReq, err := s.buildUpstreamRequest(ctx, c, account, responsesBody, token, true, parsed.StickySessionSeed(), false)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
upstreamReq.Header.Set("Content-Type", "application/json")
|
||||||
|
upstreamReq.Header.Set("Accept", "text/event-stream")
|
||||||
|
|
||||||
|
proxyURL := ""
|
||||||
|
if account.ProxyID != nil && account.Proxy != nil {
|
||||||
|
proxyURL = account.Proxy.URL()
|
||||||
|
}
|
||||||
|
upstreamStart := time.Now()
|
||||||
|
resp, err := s.httpUpstream.Do(upstreamReq, proxyURL, account.ID, account.Concurrency)
|
||||||
|
SetOpsLatencyMs(c, OpsUpstreamLatencyMsKey, time.Since(upstreamStart).Milliseconds())
|
||||||
|
if err != nil {
|
||||||
|
safeErr := sanitizeUpstreamErrorMessage(err.Error())
|
||||||
|
setOpsUpstreamError(c, 0, safeErr, "")
|
||||||
|
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
|
||||||
|
Platform: account.Platform,
|
||||||
|
AccountID: account.ID,
|
||||||
|
AccountName: account.Name,
|
||||||
|
UpstreamStatusCode: 0,
|
||||||
|
UpstreamURL: safeUpstreamURL(upstreamReq.URL.String()),
|
||||||
|
Kind: "request_error",
|
||||||
|
Message: safeErr,
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("upstream request failed: %s", safeErr)
|
||||||
|
}
|
||||||
|
if resp.StatusCode >= 400 {
|
||||||
|
respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 2<<20))
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
resp.Body = io.NopCloser(bytes.NewReader(respBody))
|
||||||
|
upstreamMsg := strings.TrimSpace(extractUpstreamErrorMessage(respBody))
|
||||||
|
upstreamMsg = sanitizeUpstreamErrorMessage(upstreamMsg)
|
||||||
|
if s.shouldFailoverOpenAIUpstreamResponse(resp.StatusCode, upstreamMsg, respBody) {
|
||||||
|
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
|
||||||
|
Platform: account.Platform,
|
||||||
|
AccountID: account.ID,
|
||||||
|
AccountName: account.Name,
|
||||||
|
UpstreamStatusCode: resp.StatusCode,
|
||||||
|
UpstreamRequestID: resp.Header.Get("x-request-id"),
|
||||||
|
UpstreamURL: safeUpstreamURL(upstreamReq.URL.String()),
|
||||||
|
Kind: "failover",
|
||||||
|
Message: upstreamMsg,
|
||||||
|
})
|
||||||
|
s.handleFailoverSideEffects(ctx, resp, account)
|
||||||
|
return nil, &UpstreamFailoverError{
|
||||||
|
StatusCode: resp.StatusCode,
|
||||||
|
ResponseBody: respBody,
|
||||||
|
RetryableOnSameAccount: account.IsPoolMode() && isPoolModeRetryableStatus(resp.StatusCode),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return s.handleErrorResponse(ctx, resp, c, account, responsesBody)
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
|
||||||
|
var (
|
||||||
|
usage OpenAIUsage
|
||||||
|
imageCount = parsed.N
|
||||||
|
firstTokenMs *int
|
||||||
|
)
|
||||||
|
if parsed.Stream {
|
||||||
|
usage, imageCount, firstTokenMs, err = s.handleOpenAIImagesOAuthStreamingResponse(resp, c, startTime, parsed.ResponseFormat, openAIImagesStreamPrefix(parsed))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
usage, imageCount, err = s.handleOpenAIImagesOAuthNonStreamingResponse(resp, c, parsed.ResponseFormat)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if imageCount <= 0 {
|
||||||
|
imageCount = parsed.N
|
||||||
|
}
|
||||||
|
return &OpenAIForwardResult{
|
||||||
|
RequestID: resp.Header.Get("x-request-id"),
|
||||||
|
Usage: usage,
|
||||||
|
Model: requestModel,
|
||||||
|
UpstreamModel: requestModel,
|
||||||
|
Stream: parsed.Stream,
|
||||||
|
ResponseHeaders: resp.Header.Clone(),
|
||||||
|
Duration: time.Since(startTime),
|
||||||
|
FirstTokenMs: firstTokenMs,
|
||||||
|
ImageCount: imageCount,
|
||||||
|
ImageSize: parsed.SizeTier,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
@@ -3,13 +3,17 @@ package service
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"io"
|
||||||
"mime/multipart"
|
"mime/multipart"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
|
"net/textproto"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/gin-gonic/gin"
|
"github.com/gin-gonic/gin"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
"github.com/tidwall/gjson"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestOpenAIGatewayServiceParseOpenAIImagesRequest_JSON(t *testing.T) {
|
func TestOpenAIGatewayServiceParseOpenAIImagesRequest_JSON(t *testing.T) {
|
||||||
@@ -70,6 +74,58 @@ func TestOpenAIGatewayServiceParseOpenAIImagesRequest_MultipartEdit(t *testing.T
|
|||||||
require.Equal(t, OpenAIImagesCapabilityNative, parsed.RequiredCapability)
|
require.Equal(t, OpenAIImagesCapabilityNative, parsed.RequiredCapability)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestOpenAIGatewayServiceParseOpenAIImagesRequest_MultipartEditWithMaskAndNativeOptions(t *testing.T) {
|
||||||
|
gin.SetMode(gin.TestMode)
|
||||||
|
|
||||||
|
var body bytes.Buffer
|
||||||
|
writer := multipart.NewWriter(&body)
|
||||||
|
require.NoError(t, writer.WriteField("model", "gpt-image-2"))
|
||||||
|
require.NoError(t, writer.WriteField("prompt", "replace foreground"))
|
||||||
|
require.NoError(t, writer.WriteField("output_format", "png"))
|
||||||
|
require.NoError(t, writer.WriteField("input_fidelity", "high"))
|
||||||
|
require.NoError(t, writer.WriteField("output_compression", "80"))
|
||||||
|
require.NoError(t, writer.WriteField("partial_images", "2"))
|
||||||
|
|
||||||
|
imageHeader := make(textproto.MIMEHeader)
|
||||||
|
imageHeader.Set("Content-Disposition", `form-data; name="image"; filename="source.png"`)
|
||||||
|
imageHeader.Set("Content-Type", "image/png")
|
||||||
|
imagePart, err := writer.CreatePart(imageHeader)
|
||||||
|
require.NoError(t, err)
|
||||||
|
_, err = imagePart.Write([]byte("source-image-bytes"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
maskHeader := make(textproto.MIMEHeader)
|
||||||
|
maskHeader.Set("Content-Disposition", `form-data; name="mask"; filename="mask.png"`)
|
||||||
|
maskHeader.Set("Content-Type", "image/png")
|
||||||
|
maskPart, err := writer.CreatePart(maskHeader)
|
||||||
|
require.NoError(t, err)
|
||||||
|
_, err = maskPart.Write([]byte("mask-image-bytes"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
require.NoError(t, writer.Close())
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/images/edits", bytes.NewReader(body.Bytes()))
|
||||||
|
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
c, _ := gin.CreateTestContext(rec)
|
||||||
|
c.Request = req
|
||||||
|
|
||||||
|
svc := &OpenAIGatewayService{}
|
||||||
|
parsed, err := svc.ParseOpenAIImagesRequest(c, body.Bytes())
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, parsed)
|
||||||
|
require.Len(t, parsed.Uploads, 1)
|
||||||
|
require.NotNil(t, parsed.MaskUpload)
|
||||||
|
require.True(t, parsed.HasMask)
|
||||||
|
require.Equal(t, "png", parsed.OutputFormat)
|
||||||
|
require.Equal(t, "high", parsed.InputFidelity)
|
||||||
|
require.NotNil(t, parsed.OutputCompression)
|
||||||
|
require.Equal(t, 80, *parsed.OutputCompression)
|
||||||
|
require.NotNil(t, parsed.PartialImages)
|
||||||
|
require.Equal(t, 2, *parsed.PartialImages)
|
||||||
|
require.Equal(t, OpenAIImagesCapabilityNative, parsed.RequiredCapability)
|
||||||
|
}
|
||||||
|
|
||||||
func TestOpenAIGatewayServiceParseOpenAIImagesRequest_PromptOnlyDefaultsRemainBasic(t *testing.T) {
|
func TestOpenAIGatewayServiceParseOpenAIImagesRequest_PromptOnlyDefaultsRemainBasic(t *testing.T) {
|
||||||
gin.SetMode(gin.TestMode)
|
gin.SetMode(gin.TestMode)
|
||||||
body := []byte(`{"prompt":"draw a cat"}`)
|
body := []byte(`{"prompt":"draw a cat"}`)
|
||||||
@@ -121,6 +177,40 @@ func TestOpenAIGatewayServiceParseOpenAIImagesRequest_RejectsNonImageModel(t *te
|
|||||||
require.ErrorContains(t, err, `images endpoint requires an image model, got "gpt-5.4"`)
|
require.ErrorContains(t, err, `images endpoint requires an image model, got "gpt-5.4"`)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestOpenAIGatewayServiceParseOpenAIImagesRequest_JSONEditURLs(t *testing.T) {
|
||||||
|
gin.SetMode(gin.TestMode)
|
||||||
|
body := []byte(`{
|
||||||
|
"model":"gpt-image-2",
|
||||||
|
"prompt":"replace the background",
|
||||||
|
"images":[{"image_url":"https://example.com/source.png"}],
|
||||||
|
"mask":{"image_url":"https://example.com/mask.png"},
|
||||||
|
"input_fidelity":"high",
|
||||||
|
"output_compression":90,
|
||||||
|
"partial_images":2,
|
||||||
|
"response_format":"url"
|
||||||
|
}`)
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/images/edits", bytes.NewReader(body))
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
c, _ := gin.CreateTestContext(rec)
|
||||||
|
c.Request = req
|
||||||
|
|
||||||
|
svc := &OpenAIGatewayService{}
|
||||||
|
parsed, err := svc.ParseOpenAIImagesRequest(c, body)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, parsed)
|
||||||
|
require.Equal(t, []string{"https://example.com/source.png"}, parsed.InputImageURLs)
|
||||||
|
require.Equal(t, "https://example.com/mask.png", parsed.MaskImageURL)
|
||||||
|
require.Equal(t, "high", parsed.InputFidelity)
|
||||||
|
require.NotNil(t, parsed.OutputCompression)
|
||||||
|
require.Equal(t, 90, *parsed.OutputCompression)
|
||||||
|
require.NotNil(t, parsed.PartialImages)
|
||||||
|
require.Equal(t, 2, *parsed.PartialImages)
|
||||||
|
require.True(t, parsed.HasMask)
|
||||||
|
require.Equal(t, OpenAIImagesCapabilityNative, parsed.RequiredCapability)
|
||||||
|
}
|
||||||
|
|
||||||
func TestCollectOpenAIImagePointers_RecognizesDirectAssets(t *testing.T) {
|
func TestCollectOpenAIImagePointers_RecognizesDirectAssets(t *testing.T) {
|
||||||
items := collectOpenAIImagePointers([]byte(`{
|
items := collectOpenAIImagePointers([]byte(`{
|
||||||
"revised_prompt": "cat astronaut",
|
"revised_prompt": "cat astronaut",
|
||||||
@@ -157,3 +247,365 @@ func TestResolveOpenAIImageBytes_PrefersInlineBase64(t *testing.T) {
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Equal(t, []byte("ABC"), data)
|
require.Equal(t, []byte("ABC"), data)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestAccountSupportsOpenAIImageCapability_OAuthSupportsNative(t *testing.T) {
|
||||||
|
account := &Account{
|
||||||
|
Platform: PlatformOpenAI,
|
||||||
|
Type: AccountTypeOAuth,
|
||||||
|
}
|
||||||
|
|
||||||
|
require.True(t, account.SupportsOpenAIImageCapability(OpenAIImagesCapabilityBasic))
|
||||||
|
require.True(t, account.SupportsOpenAIImageCapability(OpenAIImagesCapabilityNative))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAIGatewayServiceForwardImages_OAuthUsesResponsesAPI(t *testing.T) {
|
||||||
|
gin.SetMode(gin.TestMode)
|
||||||
|
body := []byte(`{"model":"gpt-image-2","prompt":"draw a cat","size":"1024x1024","quality":"high"}`)
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/images/generations", bytes.NewReader(body))
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
c, _ := gin.CreateTestContext(rec)
|
||||||
|
c.Request = req
|
||||||
|
c.Set("api_key", &APIKey{ID: 42})
|
||||||
|
|
||||||
|
svc := &OpenAIGatewayService{}
|
||||||
|
parsed, err := svc.ParseOpenAIImagesRequest(c, body)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
upstream := &httpUpstreamRecorder{
|
||||||
|
resp: &http.Response{
|
||||||
|
StatusCode: http.StatusOK,
|
||||||
|
Header: http.Header{
|
||||||
|
"Content-Type": []string{"text/event-stream"},
|
||||||
|
"X-Request-Id": []string{"req_img_123"},
|
||||||
|
},
|
||||||
|
Body: io.NopCloser(strings.NewReader(
|
||||||
|
"data: {\"type\":\"response.completed\",\"response\":{\"created_at\":1710000000,\"usage\":{\"input_tokens\":11,\"output_tokens\":22,\"input_tokens_details\":{\"cached_tokens\":3},\"output_tokens_details\":{\"image_tokens\":7}},\"tool_usage\":{\"image_gen\":{\"images\":1}},\"output\":[{\"type\":\"image_generation_call\",\"result\":\"aGVsbG8=\",\"revised_prompt\":\"draw a cat\",\"output_format\":\"png\",\"quality\":\"high\",\"size\":\"1024x1024\"}]}}\n\n" +
|
||||||
|
"data: [DONE]\n\n",
|
||||||
|
)),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
svc.httpUpstream = upstream
|
||||||
|
|
||||||
|
account := &Account{
|
||||||
|
ID: 1,
|
||||||
|
Name: "openai-oauth",
|
||||||
|
Platform: PlatformOpenAI,
|
||||||
|
Type: AccountTypeOAuth,
|
||||||
|
Credentials: map[string]any{
|
||||||
|
"access_token": "token-123",
|
||||||
|
"chatgpt_account_id": "acct-123",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := svc.ForwardImages(context.Background(), c, account, body, parsed, "")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, result)
|
||||||
|
require.Equal(t, "gpt-image-2", result.Model)
|
||||||
|
require.Equal(t, "gpt-image-2", result.UpstreamModel)
|
||||||
|
require.Equal(t, 1, result.ImageCount)
|
||||||
|
require.Equal(t, 11, result.Usage.InputTokens)
|
||||||
|
require.Equal(t, 22, result.Usage.OutputTokens)
|
||||||
|
require.Equal(t, 7, result.Usage.ImageOutputTokens)
|
||||||
|
|
||||||
|
require.NotNil(t, upstream.lastReq)
|
||||||
|
require.Equal(t, chatgptCodexURL, upstream.lastReq.URL.String())
|
||||||
|
require.Equal(t, "chatgpt.com", upstream.lastReq.Host)
|
||||||
|
require.Equal(t, "application/json", upstream.lastReq.Header.Get("Content-Type"))
|
||||||
|
require.Equal(t, "text/event-stream", upstream.lastReq.Header.Get("Accept"))
|
||||||
|
require.Equal(t, "acct-123", upstream.lastReq.Header.Get("chatgpt-account-id"))
|
||||||
|
require.Equal(t, "responses=experimental", upstream.lastReq.Header.Get("OpenAI-Beta"))
|
||||||
|
|
||||||
|
require.Equal(t, openAIImagesResponsesMainModel, gjson.GetBytes(upstream.lastBody, "model").String())
|
||||||
|
require.True(t, gjson.GetBytes(upstream.lastBody, "stream").Bool())
|
||||||
|
require.Equal(t, "image_generation", gjson.GetBytes(upstream.lastBody, "tools.0.type").String())
|
||||||
|
require.Equal(t, "generate", gjson.GetBytes(upstream.lastBody, "tools.0.action").String())
|
||||||
|
require.Equal(t, "gpt-image-2", gjson.GetBytes(upstream.lastBody, "tools.0.model").String())
|
||||||
|
require.Equal(t, "1024x1024", gjson.GetBytes(upstream.lastBody, "tools.0.size").String())
|
||||||
|
require.Equal(t, "high", gjson.GetBytes(upstream.lastBody, "tools.0.quality").String())
|
||||||
|
require.False(t, gjson.GetBytes(upstream.lastBody, "tools.0.n").Exists())
|
||||||
|
require.Equal(t, "draw a cat", gjson.GetBytes(upstream.lastBody, "input.0.content.0.text").String())
|
||||||
|
|
||||||
|
require.Equal(t, http.StatusOK, rec.Code)
|
||||||
|
require.Equal(t, "aGVsbG8=", gjson.Get(rec.Body.String(), "data.0.b64_json").String())
|
||||||
|
require.Equal(t, "draw a cat", gjson.Get(rec.Body.String(), "data.0.revised_prompt").String())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAIGatewayServiceForwardImages_OAuthStreamingTransformsEvents(t *testing.T) {
|
||||||
|
gin.SetMode(gin.TestMode)
|
||||||
|
body := []byte(`{"model":"gpt-image-2","prompt":"draw a cat","stream":true,"response_format":"url"}`)
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/images/generations", bytes.NewReader(body))
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
c, _ := gin.CreateTestContext(rec)
|
||||||
|
c.Request = req
|
||||||
|
|
||||||
|
svc := &OpenAIGatewayService{}
|
||||||
|
parsed, err := svc.ParseOpenAIImagesRequest(c, body)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
upstream := &httpUpstreamRecorder{
|
||||||
|
resp: &http.Response{
|
||||||
|
StatusCode: http.StatusOK,
|
||||||
|
Header: http.Header{
|
||||||
|
"Content-Type": []string{"text/event-stream"},
|
||||||
|
"X-Request-Id": []string{"req_img_stream"},
|
||||||
|
},
|
||||||
|
Body: io.NopCloser(strings.NewReader(
|
||||||
|
"data: {\"type\":\"response.image_generation_call.partial_image\",\"partial_image_b64\":\"cGFydGlhbA==\",\"partial_image_index\":0,\"output_format\":\"png\"}\n\n" +
|
||||||
|
"data: {\"type\":\"response.completed\",\"response\":{\"created_at\":1710000001,\"usage\":{\"input_tokens\":5,\"output_tokens\":9,\"output_tokens_details\":{\"image_tokens\":4}},\"tool_usage\":{\"image_gen\":{\"images\":1}},\"output\":[{\"type\":\"image_generation_call\",\"result\":\"ZmluYWw=\",\"output_format\":\"png\"}]}}\n\n" +
|
||||||
|
"data: [DONE]\n\n",
|
||||||
|
)),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
svc.httpUpstream = upstream
|
||||||
|
|
||||||
|
account := &Account{
|
||||||
|
ID: 2,
|
||||||
|
Name: "openai-oauth",
|
||||||
|
Platform: PlatformOpenAI,
|
||||||
|
Type: AccountTypeOAuth,
|
||||||
|
Credentials: map[string]any{
|
||||||
|
"access_token": "token-123",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := svc.ForwardImages(context.Background(), c, account, body, parsed, "")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, result)
|
||||||
|
require.True(t, result.Stream)
|
||||||
|
require.Equal(t, 1, result.ImageCount)
|
||||||
|
require.Contains(t, rec.Body.String(), "event: image_generation.partial_image")
|
||||||
|
require.Contains(t, rec.Body.String(), "event: image_generation.completed")
|
||||||
|
require.Contains(t, rec.Body.String(), "\"type\":\"image_generation.partial_image\"")
|
||||||
|
require.Contains(t, rec.Body.String(), "\"type\":\"image_generation.completed\"")
|
||||||
|
require.Contains(t, rec.Body.String(), "\"url\":\"data:image/png;base64,cGFydGlhbA==\"")
|
||||||
|
require.Contains(t, rec.Body.String(), "\"url\":\"data:image/png;base64,ZmluYWw=\"")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAIGatewayServiceForwardImages_OAuthEditsMultipartUsesResponsesAPI(t *testing.T) {
|
||||||
|
gin.SetMode(gin.TestMode)
|
||||||
|
|
||||||
|
var body bytes.Buffer
|
||||||
|
writer := multipart.NewWriter(&body)
|
||||||
|
require.NoError(t, writer.WriteField("model", "gpt-image-2"))
|
||||||
|
require.NoError(t, writer.WriteField("prompt", "replace background with aurora"))
|
||||||
|
require.NoError(t, writer.WriteField("input_fidelity", "high"))
|
||||||
|
require.NoError(t, writer.WriteField("output_format", "webp"))
|
||||||
|
require.NoError(t, writer.WriteField("quality", "high"))
|
||||||
|
|
||||||
|
imageHeader := make(textproto.MIMEHeader)
|
||||||
|
imageHeader.Set("Content-Disposition", `form-data; name="image"; filename="source.png"`)
|
||||||
|
imageHeader.Set("Content-Type", "image/png")
|
||||||
|
imagePart, err := writer.CreatePart(imageHeader)
|
||||||
|
require.NoError(t, err)
|
||||||
|
_, err = imagePart.Write([]byte("png-image-content"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
maskHeader := make(textproto.MIMEHeader)
|
||||||
|
maskHeader.Set("Content-Disposition", `form-data; name="mask"; filename="mask.png"`)
|
||||||
|
maskHeader.Set("Content-Type", "image/png")
|
||||||
|
maskPart, err := writer.CreatePart(maskHeader)
|
||||||
|
require.NoError(t, err)
|
||||||
|
_, err = maskPart.Write([]byte("png-mask-content"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
require.NoError(t, writer.Close())
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/images/edits", bytes.NewReader(body.Bytes()))
|
||||||
|
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
c, _ := gin.CreateTestContext(rec)
|
||||||
|
c.Request = req
|
||||||
|
c.Set("api_key", &APIKey{ID: 100})
|
||||||
|
|
||||||
|
svc := &OpenAIGatewayService{}
|
||||||
|
parsed, err := svc.ParseOpenAIImagesRequest(c, body.Bytes())
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
upstream := &httpUpstreamRecorder{
|
||||||
|
resp: &http.Response{
|
||||||
|
StatusCode: http.StatusOK,
|
||||||
|
Header: http.Header{
|
||||||
|
"Content-Type": []string{"text/event-stream"},
|
||||||
|
"X-Request-Id": []string{"req_img_edit_123"},
|
||||||
|
},
|
||||||
|
Body: io.NopCloser(strings.NewReader(
|
||||||
|
"data: {\"type\":\"response.completed\",\"response\":{\"created_at\":1710000002,\"usage\":{\"input_tokens\":13,\"output_tokens\":21,\"output_tokens_details\":{\"image_tokens\":8}},\"tool_usage\":{\"image_gen\":{\"images\":1}},\"output\":[{\"type\":\"image_generation_call\",\"result\":\"ZWRpdGVk\",\"revised_prompt\":\"replace background with aurora\",\"output_format\":\"webp\",\"quality\":\"high\"}]}}\n\n" +
|
||||||
|
"data: [DONE]\n\n",
|
||||||
|
)),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
svc.httpUpstream = upstream
|
||||||
|
|
||||||
|
account := &Account{
|
||||||
|
ID: 3,
|
||||||
|
Name: "openai-oauth",
|
||||||
|
Platform: PlatformOpenAI,
|
||||||
|
Type: AccountTypeOAuth,
|
||||||
|
Credentials: map[string]any{
|
||||||
|
"access_token": "token-123",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := svc.ForwardImages(context.Background(), c, account, body.Bytes(), parsed, "")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, result)
|
||||||
|
require.Equal(t, 1, result.ImageCount)
|
||||||
|
require.Equal(t, "gpt-image-2", gjson.GetBytes(upstream.lastBody, "tools.0.model").String())
|
||||||
|
require.Equal(t, "edit", gjson.GetBytes(upstream.lastBody, "tools.0.action").String())
|
||||||
|
require.Equal(t, "high", gjson.GetBytes(upstream.lastBody, "tools.0.input_fidelity").String())
|
||||||
|
require.Equal(t, "webp", gjson.GetBytes(upstream.lastBody, "tools.0.output_format").String())
|
||||||
|
require.True(t, strings.HasPrefix(gjson.GetBytes(upstream.lastBody, "input.0.content.1.image_url").String(), "data:image/png;base64,"))
|
||||||
|
require.True(t, strings.HasPrefix(gjson.GetBytes(upstream.lastBody, "tools.0.input_image_mask.image_url").String(), "data:image/png;base64,"))
|
||||||
|
require.Equal(t, "replace background with aurora", gjson.GetBytes(upstream.lastBody, "input.0.content.0.text").String())
|
||||||
|
require.Equal(t, "ZWRpdGVk", gjson.Get(rec.Body.String(), "data.0.b64_json").String())
|
||||||
|
require.Equal(t, "replace background with aurora", gjson.Get(rec.Body.String(), "data.0.revised_prompt").String())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAIGatewayServiceForwardImages_OAuthEditsStreamingTransformsEvents(t *testing.T) {
|
||||||
|
gin.SetMode(gin.TestMode)
|
||||||
|
body := []byte(`{
|
||||||
|
"model":"gpt-image-2",
|
||||||
|
"prompt":"replace background with aurora",
|
||||||
|
"images":[{"image_url":"https://example.com/source.png"}],
|
||||||
|
"mask":{"image_url":"https://example.com/mask.png"},
|
||||||
|
"stream":true,
|
||||||
|
"response_format":"url"
|
||||||
|
}`)
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/images/edits", bytes.NewReader(body))
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
c, _ := gin.CreateTestContext(rec)
|
||||||
|
c.Request = req
|
||||||
|
|
||||||
|
svc := &OpenAIGatewayService{}
|
||||||
|
parsed, err := svc.ParseOpenAIImagesRequest(c, body)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
upstream := &httpUpstreamRecorder{
|
||||||
|
resp: &http.Response{
|
||||||
|
StatusCode: http.StatusOK,
|
||||||
|
Header: http.Header{
|
||||||
|
"Content-Type": []string{"text/event-stream"},
|
||||||
|
},
|
||||||
|
Body: io.NopCloser(strings.NewReader(
|
||||||
|
"data: {\"type\":\"response.image_generation_call.partial_image\",\"partial_image_b64\":\"cGFydGlhbA==\",\"partial_image_index\":0,\"output_format\":\"webp\"}\n\n" +
|
||||||
|
"data: {\"type\":\"response.completed\",\"response\":{\"created_at\":1710000003,\"usage\":{\"input_tokens\":7,\"output_tokens\":10,\"output_tokens_details\":{\"image_tokens\":5}},\"tool_usage\":{\"image_gen\":{\"images\":1}},\"output\":[{\"type\":\"image_generation_call\",\"result\":\"ZWRpdGVk\",\"revised_prompt\":\"replace background with aurora\",\"output_format\":\"webp\"}]}}\n\n" +
|
||||||
|
"data: [DONE]\n\n",
|
||||||
|
)),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
svc.httpUpstream = upstream
|
||||||
|
|
||||||
|
account := &Account{
|
||||||
|
ID: 4,
|
||||||
|
Name: "openai-oauth",
|
||||||
|
Platform: PlatformOpenAI,
|
||||||
|
Type: AccountTypeOAuth,
|
||||||
|
Credentials: map[string]any{
|
||||||
|
"access_token": "token-123",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := svc.ForwardImages(context.Background(), c, account, body, parsed, "")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, result)
|
||||||
|
require.Equal(t, 1, result.ImageCount)
|
||||||
|
require.Equal(t, "edit", gjson.GetBytes(upstream.lastBody, "tools.0.action").String())
|
||||||
|
require.Equal(t, "https://example.com/source.png", gjson.GetBytes(upstream.lastBody, "input.0.content.1.image_url").String())
|
||||||
|
require.Equal(t, "https://example.com/mask.png", gjson.GetBytes(upstream.lastBody, "tools.0.input_image_mask.image_url").String())
|
||||||
|
require.Contains(t, rec.Body.String(), "event: image_edit.partial_image")
|
||||||
|
require.Contains(t, rec.Body.String(), "event: image_edit.completed")
|
||||||
|
require.Contains(t, rec.Body.String(), "\"type\":\"image_edit.partial_image\"")
|
||||||
|
require.Contains(t, rec.Body.String(), "\"type\":\"image_edit.completed\"")
|
||||||
|
require.Contains(t, rec.Body.String(), "\"url\":\"data:image/webp;base64,cGFydGlhbA==\"")
|
||||||
|
require.Contains(t, rec.Body.String(), "\"url\":\"data:image/webp;base64,ZWRpdGVk\"")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildOpenAIImagesResponsesRequest_RejectsMultipleImages(t *testing.T) {
|
||||||
|
parsed := &OpenAIImagesRequest{
|
||||||
|
Endpoint: openAIImagesGenerationsEndpoint,
|
||||||
|
Model: "gpt-image-2",
|
||||||
|
Prompt: "draw a cat",
|
||||||
|
N: 2,
|
||||||
|
}
|
||||||
|
|
||||||
|
body, err := buildOpenAIImagesResponsesRequest(parsed, "gpt-image-2")
|
||||||
|
require.Error(t, err)
|
||||||
|
require.Nil(t, body)
|
||||||
|
require.Contains(t, err.Error(), "only n=1")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCollectOpenAIImagesFromResponsesBody_FallsBackToOutputItemDone(t *testing.T) {
|
||||||
|
body := []byte(
|
||||||
|
"data: {\"type\":\"response.created\",\"response\":{\"created_at\":1710000004}}\n\n" +
|
||||||
|
"data: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"ig_123\",\"type\":\"image_generation_call\",\"result\":\"aGVsbG8=\",\"revised_prompt\":\"draw a cat\",\"output_format\":\"png\",\"quality\":\"high\"}}\n\n" +
|
||||||
|
"data: {\"type\":\"response.completed\",\"response\":{\"created_at\":1710000004,\"tool_usage\":{\"image_gen\":{\"images\":1}},\"output\":[]}}\n\n" +
|
||||||
|
"data: [DONE]\n\n",
|
||||||
|
)
|
||||||
|
|
||||||
|
results, createdAt, usageRaw, firstMeta, foundFinal, err := collectOpenAIImagesFromResponsesBody(body)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.True(t, foundFinal)
|
||||||
|
require.Equal(t, int64(1710000004), createdAt)
|
||||||
|
require.Len(t, results, 1)
|
||||||
|
require.Equal(t, "aGVsbG8=", results[0].Result)
|
||||||
|
require.Equal(t, "draw a cat", results[0].RevisedPrompt)
|
||||||
|
require.Equal(t, "png", firstMeta.OutputFormat)
|
||||||
|
require.JSONEq(t, `{"images":1}`, string(usageRaw))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOpenAIGatewayServiceForwardImages_OAuthStreamingHandlesOutputItemDoneFallback(t *testing.T) {
|
||||||
|
gin.SetMode(gin.TestMode)
|
||||||
|
body := []byte(`{"model":"gpt-image-2","prompt":"draw a cat","stream":true,"response_format":"url"}`)
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/images/generations", bytes.NewReader(body))
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
c, _ := gin.CreateTestContext(rec)
|
||||||
|
c.Request = req
|
||||||
|
|
||||||
|
svc := &OpenAIGatewayService{}
|
||||||
|
parsed, err := svc.ParseOpenAIImagesRequest(c, body)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
upstream := &httpUpstreamRecorder{
|
||||||
|
resp: &http.Response{
|
||||||
|
StatusCode: http.StatusOK,
|
||||||
|
Header: http.Header{
|
||||||
|
"Content-Type": []string{"text/event-stream"},
|
||||||
|
"X-Request-Id": []string{"req_img_stream_output_item_done"},
|
||||||
|
},
|
||||||
|
Body: io.NopCloser(strings.NewReader(
|
||||||
|
"data: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"ig_123\",\"type\":\"image_generation_call\",\"result\":\"ZmluYWw=\",\"revised_prompt\":\"draw a cat\",\"output_format\":\"png\"}}\n\n" +
|
||||||
|
"data: {\"type\":\"response.completed\",\"response\":{\"created_at\":1710000005,\"usage\":{\"input_tokens\":5,\"output_tokens\":9,\"output_tokens_details\":{\"image_tokens\":4}},\"tool_usage\":{\"image_gen\":{\"images\":1}},\"output\":[]}}\n\n" +
|
||||||
|
"data: [DONE]\n\n",
|
||||||
|
)),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
svc.httpUpstream = upstream
|
||||||
|
|
||||||
|
account := &Account{
|
||||||
|
ID: 5,
|
||||||
|
Name: "openai-oauth",
|
||||||
|
Platform: PlatformOpenAI,
|
||||||
|
Type: AccountTypeOAuth,
|
||||||
|
Credentials: map[string]any{
|
||||||
|
"access_token": "token-123",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := svc.ForwardImages(context.Background(), c, account, body, parsed, "")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, result)
|
||||||
|
require.True(t, result.Stream)
|
||||||
|
require.Equal(t, 1, result.ImageCount)
|
||||||
|
require.Contains(t, rec.Body.String(), "event: image_generation.completed")
|
||||||
|
require.Contains(t, rec.Body.String(), "\"type\":\"image_generation.completed\"")
|
||||||
|
require.Contains(t, rec.Body.String(), "\"url\":\"data:image/png;base64,ZmluYWw=\"")
|
||||||
|
require.NotContains(t, rec.Body.String(), "event: error")
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user