From c5aac1251d5496ea21f2b1bc09bcf7cee5eea687 Mon Sep 17 00:00:00 2001 From: YanzheL Date: Thu, 2 Apr 2026 00:11:06 +0800 Subject: [PATCH 1/3] fix(gateway): add content-based session hash fallback for non-Codex clients When no explicit session signals (session_id, conversation_id, prompt_cache_key) are provided, derive a stable session seed from the request body content (model + tools + system prompt + first user message) to enable sticky routing and prompt caching for non-Codex clients using the Chat Completions API. This mirrors the content-based fallback already present in GatewayService. GenerateSessionHash, adapted for the OpenAI gateway's request formats (both Chat Completions messages and Responses API input). JSON fragments are canonicalized via normalizeCompatSeedJSON to ensure semantically identical requests produce the same seed regardless of whitespace or key ordering. Closes #1421 --- .../service/openai_content_session_seed.go | 107 ++++++++++++++++++ .../service/openai_gateway_service.go | 4 + 2 files changed, 111 insertions(+) create mode 100644 backend/internal/service/openai_content_session_seed.go diff --git a/backend/internal/service/openai_content_session_seed.go b/backend/internal/service/openai_content_session_seed.go new file mode 100644 index 00000000..cb8fcb84 --- /dev/null +++ b/backend/internal/service/openai_content_session_seed.go @@ -0,0 +1,107 @@ +package service + +import ( + "encoding/json" + "strings" + + "github.com/tidwall/gjson" +) + +// contentSessionSeedPrefix prevents collisions between content-derived seeds +// and explicit session IDs (e.g. "sess-xxx" or "compat_cc_xxx"). +const contentSessionSeedPrefix = "compat_cs_" + +// deriveOpenAIContentSessionSeed builds a stable session seed from an +// OpenAI-format request body. Only fields constant across conversation turns +// are included: model, tools/functions definitions, system/developer prompts, +// instructions (Responses API), and the first user message. +// Supports both Chat Completions (messages) and Responses API (input). +func deriveOpenAIContentSessionSeed(body []byte) string { + if len(body) == 0 { + return "" + } + + var b strings.Builder + + if model := gjson.GetBytes(body, "model").String(); model != "" { + b.WriteString("model=") + b.WriteString(model) + } + + if tools := gjson.GetBytes(body, "tools"); tools.Exists() && tools.IsArray() && tools.Raw != "[]" { + b.WriteString("|tools=") + b.WriteString(normalizeCompatSeedJSON(json.RawMessage(tools.Raw))) + } + + if funcs := gjson.GetBytes(body, "functions"); funcs.Exists() && funcs.IsArray() && funcs.Raw != "[]" { + b.WriteString("|functions=") + b.WriteString(normalizeCompatSeedJSON(json.RawMessage(funcs.Raw))) + } + + if instr := gjson.GetBytes(body, "instructions").String(); instr != "" { + b.WriteString("|instructions=") + b.WriteString(instr) + } + + firstUserCaptured := false + + msgs := gjson.GetBytes(body, "messages") + if msgs.Exists() && msgs.IsArray() { + msgs.ForEach(func(_, msg gjson.Result) bool { + role := msg.Get("role").String() + switch role { + case "system", "developer": + b.WriteString("|system=") + if c := msg.Get("content"); c.Exists() { + b.WriteString(normalizeCompatSeedJSON(json.RawMessage(c.Raw))) + } + case "user": + if !firstUserCaptured { + b.WriteString("|first_user=") + if c := msg.Get("content"); c.Exists() { + b.WriteString(normalizeCompatSeedJSON(json.RawMessage(c.Raw))) + } + firstUserCaptured = true + } + } + return true + }) + } else if inp := gjson.GetBytes(body, "input"); inp.Exists() { + if inp.Type == gjson.String { + b.WriteString("|input=") + b.WriteString(inp.String()) + } else if inp.IsArray() { + inp.ForEach(func(_, item gjson.Result) bool { + role := item.Get("role").String() + switch role { + case "system", "developer": + b.WriteString("|system=") + if c := item.Get("content"); c.Exists() { + b.WriteString(normalizeCompatSeedJSON(json.RawMessage(c.Raw))) + } + case "user": + if !firstUserCaptured { + b.WriteString("|first_user=") + if c := item.Get("content"); c.Exists() { + b.WriteString(normalizeCompatSeedJSON(json.RawMessage(c.Raw))) + } + firstUserCaptured = true + } + } + if !firstUserCaptured && item.Get("type").String() == "input_text" { + b.WriteString("|first_user=") + if text := item.Get("text").String(); text != "" { + b.WriteString(text) + } + firstUserCaptured = true + } + return true + }) + } + } + + if b.Len() == 0 { + return "" + } + return contentSessionSeedPrefix + b.String() +} diff --git a/backend/internal/service/openai_gateway_service.go b/backend/internal/service/openai_gateway_service.go index 0a959615..b9f42cd7 100644 --- a/backend/internal/service/openai_gateway_service.go +++ b/backend/internal/service/openai_gateway_service.go @@ -1044,6 +1044,7 @@ func (s *OpenAIGatewayService) ExtractSessionID(c *gin.Context, body []byte) str // 1. Header: session_id // 2. Header: conversation_id // 3. Body: prompt_cache_key (opencode) +// 4. Body: content-based fallback (model + system + tools + first user message) func (s *OpenAIGatewayService) GenerateSessionHash(c *gin.Context, body []byte) string { if c == nil { return "" @@ -1056,6 +1057,9 @@ func (s *OpenAIGatewayService) GenerateSessionHash(c *gin.Context, body []byte) if sessionID == "" && len(body) > 0 { sessionID = strings.TrimSpace(gjson.GetBytes(body, "prompt_cache_key").String()) } + if sessionID == "" && len(body) > 0 { + sessionID = deriveOpenAIContentSessionSeed(body) + } if sessionID == "" { return "" } From 4fb16030016d4450b6551184982073b718687bcc Mon Sep 17 00:00:00 2001 From: YanzheL Date: Thu, 2 Apr 2026 00:11:17 +0800 Subject: [PATCH 2/3] test(gateway): add tests for content-based session hash fallback - 20 unit tests for deriveOpenAIContentSessionSeed covering: - Empty/nil inputs, model-only, stable across turns - Different model/system/first-user produce different seeds - Tools, functions, developer role, structured content - Responses API: input string, input array, instructions, input_text typed items - JSON canonicalization (whitespace/key-order insensitive) - Prefix presence, empty tools ignored, messages preferred over input - 3 integration tests for GenerateSessionHash content fallback: - Content fallback produces stable hash - Explicit signals override content fallback - Empty body still returns empty hash --- .../openai_content_session_seed_test.go | 218 ++++++++++++++++++ .../service/openai_gateway_service_test.go | 54 +++++ 2 files changed, 272 insertions(+) create mode 100644 backend/internal/service/openai_content_session_seed_test.go diff --git a/backend/internal/service/openai_content_session_seed_test.go b/backend/internal/service/openai_content_session_seed_test.go new file mode 100644 index 00000000..65a0bf18 --- /dev/null +++ b/backend/internal/service/openai_content_session_seed_test.go @@ -0,0 +1,218 @@ +package service + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDeriveOpenAIContentSessionSeed_EmptyInputs(t *testing.T) { + require.Empty(t, deriveOpenAIContentSessionSeed(nil)) + require.Empty(t, deriveOpenAIContentSessionSeed([]byte{})) + require.Empty(t, deriveOpenAIContentSessionSeed([]byte(`{}`))) +} + +func TestDeriveOpenAIContentSessionSeed_ModelOnly(t *testing.T) { + seed := deriveOpenAIContentSessionSeed([]byte(`{"model":"gpt-5.4"}`)) + require.Contains(t, seed, contentSessionSeedPrefix) + require.Contains(t, seed, "model=gpt-5.4") +} + +func TestDeriveOpenAIContentSessionSeed_ChatCompletions_StableAcrossTurns(t *testing.T) { + turn1 := []byte(`{ + "model": "gpt-5.4", + "messages": [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "Hello"} + ] + }`) + turn2 := []byte(`{ + "model": "gpt-5.4", + "messages": [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi there!"}, + {"role": "user", "content": "How are you?"} + ] + }`) + s1 := deriveOpenAIContentSessionSeed(turn1) + s2 := deriveOpenAIContentSessionSeed(turn2) + require.Equal(t, s1, s2, "seed should be stable across later turns") + require.NotEmpty(t, s1) +} + +func TestDeriveOpenAIContentSessionSeed_ChatCompletions_DifferentFirstUserDiffers(t *testing.T) { + req1 := []byte(`{"model":"gpt-5.4","messages":[{"role":"user","content":"Question A"}]}`) + req2 := []byte(`{"model":"gpt-5.4","messages":[{"role":"user","content":"Question B"}]}`) + s1 := deriveOpenAIContentSessionSeed(req1) + s2 := deriveOpenAIContentSessionSeed(req2) + require.NotEqual(t, s1, s2) +} + +func TestDeriveOpenAIContentSessionSeed_ChatCompletions_DifferentSystemDiffers(t *testing.T) { + req1 := []byte(`{"model":"gpt-5.4","messages":[{"role":"system","content":"A"},{"role":"user","content":"Hi"}]}`) + req2 := []byte(`{"model":"gpt-5.4","messages":[{"role":"system","content":"B"},{"role":"user","content":"Hi"}]}`) + s1 := deriveOpenAIContentSessionSeed(req1) + s2 := deriveOpenAIContentSessionSeed(req2) + require.NotEqual(t, s1, s2) +} + +func TestDeriveOpenAIContentSessionSeed_ChatCompletions_DifferentModelDiffers(t *testing.T) { + req1 := []byte(`{"model":"gpt-5.4","messages":[{"role":"user","content":"Hi"}]}`) + req2 := []byte(`{"model":"gpt-4o","messages":[{"role":"user","content":"Hi"}]}`) + s1 := deriveOpenAIContentSessionSeed(req1) + s2 := deriveOpenAIContentSessionSeed(req2) + require.NotEqual(t, s1, s2) +} + +func TestDeriveOpenAIContentSessionSeed_ChatCompletions_WithTools(t *testing.T) { + withTools := []byte(`{ + "model": "gpt-5.4", + "tools": [{"type":"function","function":{"name":"get_weather"}}], + "messages": [{"role": "user", "content": "Hello"}] + }`) + withoutTools := []byte(`{ + "model": "gpt-5.4", + "messages": [{"role": "user", "content": "Hello"}] + }`) + s1 := deriveOpenAIContentSessionSeed(withTools) + s2 := deriveOpenAIContentSessionSeed(withoutTools) + require.NotEqual(t, s1, s2, "tools should affect the seed") + require.Contains(t, s1, "|tools=") +} + +func TestDeriveOpenAIContentSessionSeed_ChatCompletions_WithFunctions(t *testing.T) { + body := []byte(`{ + "model": "gpt-5.4", + "functions": [{"name":"get_weather","parameters":{}}], + "messages": [{"role": "user", "content": "Hello"}] + }`) + seed := deriveOpenAIContentSessionSeed(body) + require.Contains(t, seed, "|functions=") +} + +func TestDeriveOpenAIContentSessionSeed_ChatCompletions_DeveloperRole(t *testing.T) { + body := []byte(`{ + "model": "gpt-5.4", + "messages": [ + {"role": "developer", "content": "You are helpful."}, + {"role": "user", "content": "Hello"} + ] + }`) + seed := deriveOpenAIContentSessionSeed(body) + require.Contains(t, seed, "|system=") + require.Contains(t, seed, "|first_user=") +} + +func TestDeriveOpenAIContentSessionSeed_ChatCompletions_StructuredContent(t *testing.T) { + body := []byte(`{ + "model": "gpt-5.4", + "messages": [ + {"role": "user", "content": [{"type":"text","text":"Hello"}]} + ] + }`) + seed := deriveOpenAIContentSessionSeed(body) + require.NotEmpty(t, seed) + require.Contains(t, seed, "|first_user=") +} + +func TestDeriveOpenAIContentSessionSeed_ResponsesAPI_InputString(t *testing.T) { + body := []byte(`{"model":"gpt-5.4","input":"Hello, how are you?"}`) + seed := deriveOpenAIContentSessionSeed(body) + require.Contains(t, seed, "|input=Hello, how are you?") +} + +func TestDeriveOpenAIContentSessionSeed_ResponsesAPI_InputArray(t *testing.T) { + body := []byte(`{ + "model": "gpt-5.4", + "input": [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "Hello"} + ] + }`) + seed := deriveOpenAIContentSessionSeed(body) + require.Contains(t, seed, "|system=") + require.Contains(t, seed, "|first_user=") +} + +func TestDeriveOpenAIContentSessionSeed_ResponsesAPI_WithInstructions(t *testing.T) { + body := []byte(`{ + "model": "gpt-5.4", + "instructions": "You are a coding assistant.", + "input": "Write a hello world" + }`) + seed := deriveOpenAIContentSessionSeed(body) + require.Contains(t, seed, "|instructions=You are a coding assistant.") + require.Contains(t, seed, "|input=Write a hello world") +} + +func TestDeriveOpenAIContentSessionSeed_Deterministic(t *testing.T) { + body := []byte(`{ + "model": "gpt-5.4", + "messages": [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "Hello"} + ] + }`) + s1 := deriveOpenAIContentSessionSeed(body) + s2 := deriveOpenAIContentSessionSeed(body) + require.Equal(t, s1, s2, "seed must be deterministic") +} + +func TestDeriveOpenAIContentSessionSeed_PrefixPresent(t *testing.T) { + body := []byte(`{"model":"gpt-5.4","messages":[{"role":"user","content":"Hi"}]}`) + seed := deriveOpenAIContentSessionSeed(body) + require.True(t, len(seed) > len(contentSessionSeedPrefix)) + require.Equal(t, contentSessionSeedPrefix, seed[:len(contentSessionSeedPrefix)]) +} + +func TestDeriveOpenAIContentSessionSeed_EmptyToolsIgnored(t *testing.T) { + body := []byte(`{"model":"gpt-5.4","tools":[],"messages":[{"role":"user","content":"Hi"}]}`) + seed := deriveOpenAIContentSessionSeed(body) + require.NotContains(t, seed, "|tools=") +} + +func TestDeriveOpenAIContentSessionSeed_MessagesPreferredOverInput(t *testing.T) { + body := []byte(`{ + "model": "gpt-5.4", + "messages": [{"role": "user", "content": "from messages"}], + "input": "from input" + }`) + seed := deriveOpenAIContentSessionSeed(body) + require.Contains(t, seed, "|first_user=") + require.NotContains(t, seed, "|input=") +} + +func TestDeriveOpenAIContentSessionSeed_JSONCanonicalisation(t *testing.T) { + compact := []byte(`{"model":"gpt-5.4","tools":[{"type":"function","function":{"name":"get_weather","description":"Get weather"}}],"messages":[{"role":"user","content":"Hi"}]}`) + spaced := []byte(`{ + "model": "gpt-5.4", + "tools": [ + { "type" : "function", "function": { "description": "Get weather", "name": "get_weather" } } + ], + "messages": [ { "role": "user", "content": "Hi" } ] + }`) + s1 := deriveOpenAIContentSessionSeed(compact) + s2 := deriveOpenAIContentSessionSeed(spaced) + require.Equal(t, s1, s2, "different formatting of identical JSON should produce the same seed") +} + +func TestDeriveOpenAIContentSessionSeed_ResponsesAPI_InputTextTypedItem(t *testing.T) { + body := []byte(`{ + "model": "gpt-5.4", + "input": [{"type": "input_text", "text": "Hello world"}] + }`) + seed := deriveOpenAIContentSessionSeed(body) + require.Contains(t, seed, "|first_user=") + require.Contains(t, seed, "Hello world") +} + +func TestDeriveOpenAIContentSessionSeed_ResponsesAPI_TypedMessageItem(t *testing.T) { + body := []byte(`{ + "model": "gpt-5.4", + "input": [{"type": "message", "role": "user", "content": "Hello from typed message"}] + }`) + seed := deriveOpenAIContentSessionSeed(body) + require.Contains(t, seed, "|first_user=") + require.Contains(t, seed, "Hello from typed message") +} diff --git a/backend/internal/service/openai_gateway_service_test.go b/backend/internal/service/openai_gateway_service_test.go index 9e2f33f2..71b7acf1 100644 --- a/backend/internal/service/openai_gateway_service_test.go +++ b/backend/internal/service/openai_gateway_service_test.go @@ -237,6 +237,60 @@ func TestOpenAIGatewayService_GenerateSessionHashWithFallback(t *testing.T) { require.Equal(t, "", empty) } +func TestOpenAIGatewayService_GenerateSessionHash_ContentFallback(t *testing.T) { + gin.SetMode(gin.TestMode) + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + c.Request = httptest.NewRequest(http.MethodPost, "/openai/v1/chat/completions", nil) + + svc := &OpenAIGatewayService{} + + body := []byte(`{"model":"gpt-5.4","messages":[{"role":"system","content":"You are helpful."},{"role":"user","content":"Hello"}]}`) + + hash := svc.GenerateSessionHash(c, body) + require.NotEmpty(t, hash, "content-based fallback should produce a hash") + + hash2 := svc.GenerateSessionHash(c, body) + require.Equal(t, hash, hash2, "same content should produce same hash") + + bodyExtended := []byte(`{"model":"gpt-5.4","messages":[{"role":"system","content":"You are helpful."},{"role":"user","content":"Hello"},{"role":"assistant","content":"Hi!"},{"role":"user","content":"How are you?"}]}`) + hashExtended := svc.GenerateSessionHash(c, bodyExtended) + require.Equal(t, hash, hashExtended, "hash should be stable across later turns") + + bodyDifferent := []byte(`{"model":"gpt-5.4","messages":[{"role":"user","content":"Different question"}]}`) + hashDifferent := svc.GenerateSessionHash(c, bodyDifferent) + require.NotEqual(t, hash, hashDifferent, "different content should produce different hash") +} + +func TestOpenAIGatewayService_GenerateSessionHash_ExplicitSignalWinsOverContent(t *testing.T) { + gin.SetMode(gin.TestMode) + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + c.Request = httptest.NewRequest(http.MethodPost, "/openai/v1/chat/completions", nil) + + svc := &OpenAIGatewayService{} + body := []byte(`{"model":"gpt-5.4","messages":[{"role":"user","content":"Hello"}]}`) + + contentHash := svc.GenerateSessionHash(c, body) + require.NotEmpty(t, contentHash) + + c.Request.Header.Set("session_id", "explicit-session") + explicitHash := svc.GenerateSessionHash(c, body) + require.NotEmpty(t, explicitHash) + require.NotEqual(t, contentHash, explicitHash, "explicit session_id should override content fallback") +} + +func TestOpenAIGatewayService_GenerateSessionHash_EmptyBodyStillEmpty(t *testing.T) { + gin.SetMode(gin.TestMode) + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + c.Request = httptest.NewRequest(http.MethodPost, "/openai/v1/chat/completions", nil) + + svc := &OpenAIGatewayService{} + require.Empty(t, svc.GenerateSessionHash(c, []byte(`{}`))) + require.Empty(t, svc.GenerateSessionHash(c, nil)) +} + func (c stubConcurrencyCache) GetAccountWaitingCount(ctx context.Context, accountID int64) (int, error) { if c.waitCounts != nil { if count, ok := c.waitCounts[accountID]; ok { From cf9efefd963bb8d6daff210b50d2a0db98b3d5d2 Mon Sep 17 00:00:00 2001 From: YanzheL Date: Thu, 2 Apr 2026 01:03:22 +0800 Subject: [PATCH 3/3] fix(lint): satisfy errcheck for strings.Builder.WriteString calls --- .../service/openai_content_session_seed.go | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/backend/internal/service/openai_content_session_seed.go b/backend/internal/service/openai_content_session_seed.go index cb8fcb84..7c2ba251 100644 --- a/backend/internal/service/openai_content_session_seed.go +++ b/backend/internal/service/openai_content_session_seed.go @@ -24,23 +24,23 @@ func deriveOpenAIContentSessionSeed(body []byte) string { var b strings.Builder if model := gjson.GetBytes(body, "model").String(); model != "" { - b.WriteString("model=") - b.WriteString(model) + _, _ = b.WriteString("model=") + _, _ = b.WriteString(model) } if tools := gjson.GetBytes(body, "tools"); tools.Exists() && tools.IsArray() && tools.Raw != "[]" { - b.WriteString("|tools=") - b.WriteString(normalizeCompatSeedJSON(json.RawMessage(tools.Raw))) + _, _ = b.WriteString("|tools=") + _, _ = b.WriteString(normalizeCompatSeedJSON(json.RawMessage(tools.Raw))) } if funcs := gjson.GetBytes(body, "functions"); funcs.Exists() && funcs.IsArray() && funcs.Raw != "[]" { - b.WriteString("|functions=") - b.WriteString(normalizeCompatSeedJSON(json.RawMessage(funcs.Raw))) + _, _ = b.WriteString("|functions=") + _, _ = b.WriteString(normalizeCompatSeedJSON(json.RawMessage(funcs.Raw))) } if instr := gjson.GetBytes(body, "instructions").String(); instr != "" { - b.WriteString("|instructions=") - b.WriteString(instr) + _, _ = b.WriteString("|instructions=") + _, _ = b.WriteString(instr) } firstUserCaptured := false @@ -51,15 +51,15 @@ func deriveOpenAIContentSessionSeed(body []byte) string { role := msg.Get("role").String() switch role { case "system", "developer": - b.WriteString("|system=") + _, _ = b.WriteString("|system=") if c := msg.Get("content"); c.Exists() { - b.WriteString(normalizeCompatSeedJSON(json.RawMessage(c.Raw))) + _, _ = b.WriteString(normalizeCompatSeedJSON(json.RawMessage(c.Raw))) } case "user": if !firstUserCaptured { - b.WriteString("|first_user=") + _, _ = b.WriteString("|first_user=") if c := msg.Get("content"); c.Exists() { - b.WriteString(normalizeCompatSeedJSON(json.RawMessage(c.Raw))) + _, _ = b.WriteString(normalizeCompatSeedJSON(json.RawMessage(c.Raw))) } firstUserCaptured = true } @@ -68,30 +68,30 @@ func deriveOpenAIContentSessionSeed(body []byte) string { }) } else if inp := gjson.GetBytes(body, "input"); inp.Exists() { if inp.Type == gjson.String { - b.WriteString("|input=") - b.WriteString(inp.String()) + _, _ = b.WriteString("|input=") + _, _ = b.WriteString(inp.String()) } else if inp.IsArray() { inp.ForEach(func(_, item gjson.Result) bool { role := item.Get("role").String() switch role { case "system", "developer": - b.WriteString("|system=") + _, _ = b.WriteString("|system=") if c := item.Get("content"); c.Exists() { - b.WriteString(normalizeCompatSeedJSON(json.RawMessage(c.Raw))) + _, _ = b.WriteString(normalizeCompatSeedJSON(json.RawMessage(c.Raw))) } case "user": if !firstUserCaptured { - b.WriteString("|first_user=") + _, _ = b.WriteString("|first_user=") if c := item.Get("content"); c.Exists() { - b.WriteString(normalizeCompatSeedJSON(json.RawMessage(c.Raw))) + _, _ = b.WriteString(normalizeCompatSeedJSON(json.RawMessage(c.Raw))) } firstUserCaptured = true } } if !firstUserCaptured && item.Get("type").String() == "input_text" { - b.WriteString("|first_user=") + _, _ = b.WriteString("|first_user=") if text := item.Get("text").String(); text != "" { - b.WriteString(text) + _, _ = b.WriteString(text) } firstUserCaptured = true }