From 8a81085a72dd4624cc958de165da3e8bb2ba358e Mon Sep 17 00:00:00 2001 From: Nick Nyanjui Date: Thu, 10 Sep 2026 22:53:57 +0700 Subject: [PATCH] fix(claude): cap re-anchored cache_control at the 4-marker budget and keep single-object content turns Anthropic accepts at most 4 blocks carrying cache_control per request. When the client had already spent that budget, the re-anchor added a 5th marker and the request was rejected with a non-retryable 400 that the failure path treated as an account problem, retrying the same malformed body across the whole pool until every account locked. anchorClaudeCache now normalizes bare-object content, strips the invalid cache_control carried by defer_loading tools, pins the 1h head anchors on the last system block and last cacheable tool, then trims an over-budget body to 4 markers. The trim holds those head anchors and fills the remaining slots with the tail-most message markers: a plain "keep the last four in document order" rule drops the anchors first even though they lead document order, and skipping the re-anchor at a spent budget left system/tools on the 5m default instead of 1h. Some clients send content as a single block object rather than a one-element array. Such a turn was dropped or zeroed on every leg that reads messages, silently losing conversation history. normalizeMessageContent wraps it as a one-block array on all four paths, and hasValidContent keeps it. --- open-sse/translator/formats/claude.js | 106 ++++++++- .../translator/request/claude-to-openai.js | 7 + .../claude-cache-budget-single-object.test.js | 222 ++++++++++++++++++ 3 files changed, 330 insertions(+), 5 deletions(-) create mode 100644 tests/unit/claude-cache-budget-single-object.test.js diff --git a/open-sse/translator/formats/claude.js b/open-sse/translator/formats/claude.js index 930c761a..f57972da 100644 --- a/open-sse/translator/formats/claude.js +++ b/open-sse/translator/formats/claude.js @@ -27,6 +27,14 @@ export function lastCacheableToolIndex(tools) { // Check if message has valid non-empty content export function hasValidContent(msg) { if (typeof msg.content === "string" && msg.content.trim()) return true; + if (msg.content && typeof msg.content === "object" && !Array.isArray(msg.content)) { + const block = msg.content; + return !!((block.type === CLAUDE_BLOCK.TEXT && block.text?.trim()) || + block.type === CLAUDE_BLOCK.TOOL_USE || + block.type === CLAUDE_BLOCK.TOOL_RESULT || + block.type === CLAUDE_BLOCK.IMAGE || + block.type === CLAUDE_BLOCK.DOCUMENT); + } if (Array.isArray(msg.content)) { return msg.content.some(block => (block.type === CLAUDE_BLOCK.TEXT && block.text?.trim()) || @@ -38,6 +46,60 @@ export function hasValidContent(msg) { } return false; } +// Content may arrive as a single content block object (spec allows string | array; +// some clients send the bare object). Wrap it as a one-block array and strip any +// client-placed cache_control: a bare-object marker must never survive +// normalization, on any path, guard or no guard. +function normalizeMessageContent(msg) { + const c = msg?.content; + if (c && typeof c === "object" && !Array.isArray(c)) { + delete c.cache_control; + msg.content = [c]; + } + return msg; +} + +// Total blocks carrying cache_control across system, tools, and messages — the +// upstream Messages API allows at most 4 markers per request. +function countCacheControlBlocks(body) { + let n = 0; + if (Array.isArray(body?.system)) for (const b of body.system) if (b?.cache_control) n++; + if (Array.isArray(body?.tools)) for (const t of body.tools) if (t?.cache_control) n++; + if (Array.isArray(body?.messages)) { + for (const m of body.messages) { + if (Array.isArray(m?.content)) { + for (const b of m.content) if (b?.cache_control) n++; + } else if (m?.content && typeof m.content === "object" && m.content.cache_control) n++; + } + } + return n; +} +// Trim every marker past the 4-marker budget. The head anchors (last system +// block, last cacheable tool) are held; the remaining slots go to the tail-most +// of the other markers in document order. A plain "keep the last 4 in document +// order" rule would drop the head anchors first — they lead document order, yet +// they are exactly what re-anchoring exists to pin. +function capCacheControlBlocks(body) { + const isHead = (b) => { + const sys = Array.isArray(body?.system) ? body.system : []; + if (sys.length && sys[sys.length - 1] === b) return true; + const tools = Array.isArray(body?.tools) ? body.tools : []; + const lastTool = lastCacheableToolIndex(tools); + return lastTool >= 0 && tools[lastTool] === b; + }; + const marked = []; + if (Array.isArray(body?.system)) for (const b of body.system) if (b?.cache_control) marked.push(b); + if (Array.isArray(body?.tools)) for (const t of body.tools) if (t?.cache_control) marked.push(t); + if (Array.isArray(body?.messages)) { + for (const m of body.messages) { + if (Array.isArray(m?.content)) for (const b of m.content) if (b?.cache_control) marked.push(b); + } + } + const head = marked.filter(isHead); + const rest = marked.filter(b => !isHead(b)); + const keep = Math.max(0, 4 - head.length); + for (const b of rest.slice(0, Math.max(0, rest.length - keep))) delete b.cache_control; +} // Fix tool_use/tool_result ordering for Claude API // 1. Assistant message with tool_use: remove text AFTER tool_use (Claude doesn't allow) @@ -136,8 +198,9 @@ function hasForeignServerToolUseId(block) { // Newer Cowork/Claude Code clients emit beta-only shapes that OAuth endpoints reject: // 1. thinking.type "adaptive" → unsupported on Haiku // 2. output_config.effort → unsupported on Haiku -// 3. role "system" messages (mid-conversation-system beta) → only top-level system is allowed -// 4. server_tool_use blocks carrying a foreign (non-srvtoolu_) id → rejected outright +// 3. bare content-block objects (content: {block} instead of [{block}]) → wrapped first +// 4. role "system" messages (mid-conversation-system beta) → only top-level system is allowed +// 5. server_tool_use blocks carrying a foreign (non-srvtoolu_) id → rejected outright export function normalizeClaudePassthrough(body, model = "") { if (!body || typeof body !== "object") return body; @@ -152,7 +215,15 @@ export function normalizeClaudePassthrough(body, model = "") { if (Object.keys(body.output_config).length === 0) delete body.output_config; } - // 2. Fold mid-conversation system messages into the neighbouring turn. + // 3. Wrap bare content-block objects as one-element arrays before folding. + // Some clients send content: {block} instead of content: [{block}]; the + // mid-conversation-system fold below assumes the array shape, so it must + // run first — a bare-object neighbor would otherwise be zeroed to []. + if (Array.isArray(body.messages)) { + for (const msg of body.messages) normalizeMessageContent(msg); + } + + // 4. Fold mid-conversation system messages into the neighbouring turn. // Hoisting them into body.system would insert volatile content (token counters, // reminders) ahead of the whole conversation and invalidate the prefix cache on // every request. Folding in place keeps the cached prefix stable. @@ -186,7 +257,7 @@ export function normalizeClaudePassthrough(body, model = "") { body.messages = messages; } - // 3. Drop thinking blocks whose signature is not Claude's (combo mixes models, + // 5. Drop thinking blocks whose signature is not Claude's (combo mixes models, // so foreign signatures leak into history and Anthropic rejects them). const thinkingEnabled = body.thinking?.type === "enabled"; const droppedServerToolUseIds = new Set(); @@ -233,7 +304,7 @@ export function normalizeClaudePassthrough(body, model = "") { } } - // 5. Drop empty text blocks and any message left with no content at all. + // 6. Drop empty text blocks and any message left with no content at all. // Anthropic rejects `messages.N.content` blocks with empty text (400 // "text content blocks must be non-empty"); a message whose blocks were all // stripped above must be dropped, not padded with an empty placeholder. @@ -271,7 +342,22 @@ function markLastCacheableBlock(msg) { // (normalize, tool dedupe, token savers) — otherwise the anchor drifts off the tail. export function anchorClaudeCache(body) { if (!body || typeof body !== "object") return body; + if (Array.isArray(body.messages)) { + for (const msg of body.messages) normalizeMessageContent(msg); + } + // Invalid markers first, whatever the budget: Anthropic rejects a tool that + // carries BOTH defer_loading and cache_control (#3567). The re-anchor path + // below strips them anyway; the over-budget early return used to forward + // them untouched. + if (Array.isArray(body.tools)) { + for (const t of body.tools) { + if (t?.defer_loading === true) delete t.cache_control; + } + } + // Head anchors first, before any budget guard: the 1h TTL on system/tools is + // the point of re-anchoring, and skipping it because the client spent its + // budget would silently downgrade a cache hit to the 5m default. if (Array.isArray(body.system)) { const last = body.system.length - 1; body.system.forEach((block, i) => { @@ -289,6 +375,15 @@ export function anchorClaudeCache(body) { }); } + // Budget guard AFTER the head anchors: with the last system block and last + // tool pinned, at most 2 slots remain. At >= 4 markers the client has spent + // the rest of the budget and every remaining marker is itself a valid + // breakpoint — re-anchoring the tail could only exceed 4, so trim instead. + if (countCacheControlBlocks(body) >= 4) { + capCacheControlBlocks(body); + return body; + } + if (Array.isArray(body.messages)) { let anchored = null; for (let i = body.messages.length - 1; i >= 0; i--) { @@ -368,6 +463,7 @@ export function prepareClaudeRequest(body, provider = null, apiKey = null, conne // Pass 1: remove cache_control + filter empty messages for (let i = 0; i < len; i++) { const msg = body.messages[i]; + normalizeMessageContent(msg); // Remove cache_control from content blocks if (Array.isArray(msg.content)) { diff --git a/open-sse/translator/request/claude-to-openai.js b/open-sse/translator/request/claude-to-openai.js index 3956f828..38976226 100644 --- a/open-sse/translator/request/claude-to-openai.js +++ b/open-sse/translator/request/claude-to-openai.js @@ -142,6 +142,13 @@ function systemReminderText(content) { // Convert single Claude message - returns single message or array of messages function convertClaudeMessage(msg) { + // Some clients send content as a single block object; normalize to the + // one-element array every branch below (the system-reminder fold included) + // expects. Must run BEFORE the role branch: systemReminderText only reads + // arrays and strings, so a bare-object system turn was dropped outright. + if (msg.content && typeof msg.content === "object" && !Array.isArray(msg.content)) { + msg.content = [msg.content]; + } // Mid-conversation system message -> user (per Anthropic placement rules) if (msg.role === ROLE.SYSTEM) { const text = systemReminderText(msg.content); diff --git a/tests/unit/claude-cache-budget-single-object.test.js b/tests/unit/claude-cache-budget-single-object.test.js new file mode 100644 index 00000000..e2680723 --- /dev/null +++ b/tests/unit/claude-cache-budget-single-object.test.js @@ -0,0 +1,222 @@ +// #3795 — a proxy must not dispatch more cache_control blocks than Anthropic +// accepts, and must not lose turns that use single-object content (#3567 interplay). +import { describe, it, expect } from "vitest"; +import { + anchorClaudeCache, + normalizeClaudePassthrough, + prepareClaudeRequest, +} from "../../open-sse/translator/formats/claude.js"; +import { claudeToOpenAIRequest } from "../../open-sse/translator/request/claude-to-openai.js"; + +const CC = { type: "ephemeral" }; +const text = (t, extra = {}) => ({ type: "text", text: t, ...extra }); +const tool = (name, extra = {}) => ({ name, description: "d", input_schema: {}, ...extra }); + +// counts markers incl. single-object content — mirrors the upstream contract +function countMarkers(body) { + let n = 0; + if (Array.isArray(body.system)) for (const b of body.system) if (b?.cache_control) n++; + if (Array.isArray(body.tools)) for (const t of body.tools) if (t?.cache_control) n++; + if (Array.isArray(body.messages)) for (const m of body.messages) { + if (Array.isArray(m?.content)) { + for (const b of m.content) if (b?.cache_control) n++; + } else if (m?.content && typeof m.content === "object" && m.content.cache_control) n++; + } + return n; +} + +describe("cache marker budget and single-block content", () => { + it("never emits more than four markers when the client already spent its budget", () => { + const out = anchorClaudeCache({ + system: [text("s1"), text("s2", { cache_control: CC })], + tools: [tool("t1"), tool("t2", { cache_control: CC })], + messages: [ + { role: "user", content: text("u1", { cache_control: CC }) }, + { role: "assistant", content: text("a1", { cache_control: CC }) }, + { role: "user", content: [text("q")] }, + ], + }); + expect(countMarkers(out)).toBeLessThanOrEqual(4); // base: 5 + }); + + it("normalizes a single-object turn in passthrough and anchors it", () => { + const body = { + messages: [ + { role: "user", content: [text("u1")] }, + { role: "assistant", content: text("a1") }, // single object, no marker + { role: "user", content: [text("q")] }, + ], + }; + normalizeClaudePassthrough(body); + const assistant = body.messages.find(m => m.role === "assistant"); + expect(assistant).toBeDefined(); + expect(Array.isArray(assistant.content)).toBe(true); // base: still bare object + expect(assistant.content).toHaveLength(1); + const out = anchorClaudeCache(body); + expect(countMarkers(out)).toBe(1); + }); + + it("keeps a turn whose content is a single object and strips its marker", () => { + const out = prepareClaudeRequest({ + model: "claude-sonnet-5", max_tokens: 100, + system: [text("s1")], + messages: [ + { role: "user", content: text("u1", { cache_control: CC }) }, + { role: "assistant", content: [text("a1")] }, + { role: "user", content: [text("q")] }, + ], + }, "claude"); + const kept = out.messages.filter(m => JSON.stringify(m.content).includes("u1")); + expect(kept.length).toBe(1); // base: 0 (dropped) + expect(kept[0].content).toHaveLength(1); // normalized to array + expect(kept[0].content[0].cache_control).toBeUndefined(); + }); + + it("drops no conversation turn when content is a single text object", () => { + const out = prepareClaudeRequest({ + model: "claude-sonnet-5", max_tokens: 100, + messages: [ + { role: "user", content: text("u1") }, + { role: "assistant", content: text("a1") }, + { role: "user", content: [text("q")] }, + ], + }, "claude"); + expect(out.messages.length).toBe(3); // base: 1 + expect(Array.isArray(out.messages[0].content)).toBe(true); + }); + + it("re-anchors the last assistant turn even when it uses single-object content", () => { + const out = prepareClaudeRequest({ + model: "claude-sonnet-5", max_tokens: 100, + messages: [ + { role: "user", content: [text("u1")] }, + { role: "assistant", content: text("a1") }, + { role: "user", content: [text("q")] }, + ], + }, "claude"); + expect(countMarkers(out)).toBe(1); // base: 0 + }); + + it("keeps a marked single-object turn when the marker budget is spent", () => { + const body = { + system: [text("s1", { cache_control: CC })], + tools: [tool("t1", { cache_control: CC })], + messages: [ + { role: "user", content: [text("c1"), text("c2")] }, + { role: "assistant", content: [text("a1", { cache_control: CC })] }, + { role: "user", content: text("u1", { cache_control: CC }) }, + { role: "user", content: [text("q")] }, + ], + }; + normalizeClaudePassthrough(body); + const out = anchorClaudeCache(body); + const kept = out.messages.filter(m => JSON.stringify(m.content).includes("u1")); + expect(kept.length).toBe(1); + expect(Array.isArray(kept[0].content)).toBe(true); // base: bare object survives + expect(kept[0].content).toHaveLength(1); + expect(kept[0].content[0].cache_control).toBeUndefined(); + const ctx = out.messages.find(m => JSON.stringify(m.content).includes("c1")); + expect(ctx.content).toEqual([text("c1"), text("c2")]); + expect(countMarkers(out)).toBeLessThanOrEqual(4); // fixed: 3 + }); + + it("keeps single-object turns on the claude-to-openai leg", () => { + const out = claudeToOpenAIRequest("m", { + messages: [ + { role: "user", content: text("u1") }, + { role: "assistant", content: { type: "image", source: { type: "base64", media_type: "image/png", data: "iVBORw0KGgo=" } } }, + ], + }, false); + expect(out.messages.some(m => JSON.stringify(m.content).includes("u1"))).toBe(true); // base: dropped + const img = out.messages.find(m => m.role === "assistant"); + expect(JSON.stringify(img.content)).toContain("image_url"); // base: dropped + }); + + it("keeps a bare-object user turn folded with a mid-conversation system message", () => { + const body = { + messages: [ + { role: "user", content: text("u1") }, + { role: "system", content: [text("reminder")] }, + { role: "user", content: [text("q")] }, + ], + }; + normalizeClaudePassthrough(body); + const first = body.messages[0]; + expect(Array.isArray(first.content)).toBe(true); + expect(JSON.stringify(first.content).includes("u1")).toBe(true); // pre-hoist: fold zeroes bare-object content + expect(JSON.stringify(first.content).includes("reminder")).toBe(true); + }); + it("prunes a client body that already carries five markers down to four", () => { + const out = anchorClaudeCache({ + system: [text("s1"), text("s2", { cache_control: CC })], + tools: [tool("t1", { cache_control: CC }), tool("t2", { cache_control: CC })], + messages: [ + { role: "user", content: [text("u1", { cache_control: CC })] }, + { role: "assistant", content: [text("a1", { cache_control: CC })] }, + { role: "user", content: [text("q")] }, + ], + }); + expect(countMarkers(out)).toBe(4); // pre-fix: 5 forwarded unchanged + expect(out.system[0].cache_control).toBeUndefined(); // earliest marker pruned + }); + + // A spent budget must not cost the head anchors their 1h TTL: system/tools are + // the whole point of re-anchoring, and a 5m fallback silently halves the cache + // lifetime on exactly the requests that already cached aggressively. + it("keeps the 1h head anchors when the client spent the whole budget", () => { + const out = anchorClaudeCache({ + system: [text("s1"), text("s2", { cache_control: CC })], + tools: [tool("t1", { cache_control: CC }), tool("t2")], + messages: [ + { role: "user", content: [text("u1", { cache_control: CC })] }, + { role: "assistant", content: [text("a1", { cache_control: CC })] }, + { role: "user", content: [text("q")] }, + ], + }); + expect(countMarkers(out)).toBeLessThanOrEqual(4); + expect(out.system.at(-1).cache_control?.ttl).toBe("1h"); // pre-fix: fell back to 5m + expect(out.tools.at(-1).cache_control?.ttl).toBe("1h"); // pre-fix: fell back to 5m + }); + + it("keeps the 1h head anchors on an over-budget body", () => { + const out = anchorClaudeCache({ + system: [text("s1", { cache_control: CC })], + tools: [tool("t1", { cache_control: CC }), tool("t2")], + messages: [ + { role: "user", content: [text("u1", { cache_control: CC })] }, + { role: "assistant", content: [text("a1", { cache_control: CC })] }, + { role: "user", content: [text("u2", { cache_control: CC })] }, + { role: "assistant", content: [text("a2", { cache_control: CC })] }, + { role: "user", content: [text("q")] }, + ], + }); + expect(countMarkers(out)).toBe(4); + expect(out.system.at(-1).cache_control?.ttl).toBe("1h"); + expect(out.tools.at(-1).cache_control?.ttl).toBe("1h"); + }); + + it("strips a marker from a deferred tool even when the budget is spent", () => { + const out = anchorClaudeCache({ + system: [text("s1", { cache_control: CC })], + tools: [tool("t1", { cache_control: CC, defer_loading: true })], + messages: [ + { role: "user", content: [text("u1", { cache_control: CC })] }, + { role: "assistant", content: [text("a1", { cache_control: CC })] }, + { role: "user", content: [text("q")] }, + ], + }); + expect(countMarkers(out)).toBeLessThanOrEqual(4); + const deferred = out.tools.find(t => t.defer_loading); + expect(deferred?.cache_control).toBeUndefined(); // pre-fix: invalid marker forwarded + }); + + it("keeps a bare-object system reminder on the claude-to-openai leg", () => { + const out = claudeToOpenAIRequest("m", { + messages: [ + { role: "user", content: "hi" }, + { role: "system", content: text("be brief") }, + ], + }, false); + expect(JSON.stringify(out.messages)).toContain("be brief"); // pre-fix: turn dropped + }); +});