From 781c18d83746a9b3a9dd7d401ca0ed2c6db1ce69 Mon Sep 17 00:00:00 2001 From: anhtran-ai Date: Thu, 10 Sep 2026 22:25:23 +0700 Subject: [PATCH] fix(codex): strip Unicode-property tool schema patterns Codex rejects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex's /responses validator has no Unicode property escapes, so a tool `pattern` containing `\p{...}` 400s the whole request with `Invalid schema for function ... is not a 'regex'` — identically on every account, costing a full combo failover per turn (#3922). - Add open-sse/utils/codexToolSchema.js: copy-on-write walk that drops only `pattern` values carrying a property escape, returning the original reference when nothing changed so the caller's schema stays intact for a retry against another provider - Treat `properties` keys as property names, so a field literally called `pattern` is never read as the schema keyword; skip escaped literals via backslash-parity counting - Apply it in normalizeCodexTools for both function and namespace sub-tool parameters, and log the strip count via dbg - Add three cases to tests/unit/codex-tool-normalization.test.js --- open-sse/executors/codex.js | 12 +++- open-sse/utils/codexToolSchema.js | 80 +++++++++++++++++++++ tests/unit/codex-tool-normalization.test.js | 63 ++++++++++++++++ 3 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 open-sse/utils/codexToolSchema.js diff --git a/open-sse/executors/codex.js b/open-sse/executors/codex.js index 4d9acbd2..de2af822 100644 --- a/open-sse/executors/codex.js +++ b/open-sse/executors/codex.js @@ -12,6 +12,7 @@ import { getThinkingLevels } from "../providers/thinkingLevels.js"; import { DEFAULT_RETRY_CONFIG, HTTP_STATUS, resolveRetryEntry } from "../config/runtimeConfig.js"; import { dbg } from "../utils/debugLog.js"; import { resolveSessionId } from "../utils/sessionManager.js"; +import { stripCodexUnsupportedPatterns } from "../utils/codexToolSchema.js"; // SSE error patterns inside 200-OK bodies. Some retry same account first; capacity rotates accounts. const CODEX_SSE_RETRY_PATTERNS = ["server_is_overloaded", "service_unavailable_error"]; @@ -72,6 +73,9 @@ function stripStoredItemReferences(body) { function normalizeCodexTools(body) { if (!Array.isArray(body.tools)) return; const validNames = new Set(); + // Codex's schema validator has no Unicode property escapes; a `pattern` + // carrying `\p{...}` 400s the whole request on every account (#3922). + const patternStats = { removed: 0 }; body.tools = body.tools.filter((tool) => { if (!tool || typeof tool !== "object" || Array.isArray(tool)) return false; const type = typeof tool.type === "string" ? tool.type : ""; @@ -80,6 +84,9 @@ function normalizeCodexTools(body) { for (const st of tool.tools) { const n = typeof st?.name === "string" ? st.name.trim().slice(0, 128) : ""; if (n) validNames.add(n); + if (st?.parameters && typeof st.parameters === "object") { + st.parameters = stripCodexUnsupportedPatterns(st.parameters, patternStats); + } } } return true; @@ -101,10 +108,13 @@ function normalizeCodexTools(body) { tool.type = "function"; tool.name = name.slice(0, 128); if (description) tool.description = description; - tool.parameters = parameters; + tool.parameters = stripCodexUnsupportedPatterns(parameters, patternStats); validNames.add(name); return true; }); + if (patternStats.removed > 0) { + dbg("CODEX", `stripped ${patternStats.removed} unsupported tool schema pattern(s)`); + } // Drop tool_choice if it references an unknown function name if (body.tool_choice && typeof body.tool_choice === "object" && !Array.isArray(body.tool_choice)) { if (body.tool_choice.type === "function") { diff --git a/open-sse/utils/codexToolSchema.js b/open-sse/utils/codexToolSchema.js new file mode 100644 index 00000000..0c017b61 --- /dev/null +++ b/open-sse/utils/codexToolSchema.js @@ -0,0 +1,80 @@ +// Codex-specific tool JSON Schema compatibility. +// +// `https://chatgpt.com/backend-api/codex/responses` validates every function +// tool's `parameters` with a regex engine that does not implement Unicode +// property escapes. A `pattern` such as +// +// "^(?!__.*__$)[^\\p{Cc}\\p{Cf}\\p{Zl}\\p{Zp}\"\\\\./\\[\\]]{1,200}$" +// +// is a perfectly valid ECMAScript `u`-mode regex, but Codex answers +// +// 400 Invalid schema for function 'Artifact': '^\p{Cc}...' is not a 'regex' +// param: tools[0].parameters +// +// The request is deterministically malformed for this provider, so every +// account fails identically and the combo pays a full failover before landing +// somewhere that accepts it (#3922). +// +// Scope guardrail (#3667): this is NOT a global schema sanitizer. Providers +// that do support `\p{...}` keep the constraint untouched — the strip runs only +// on the Codex dispatch path, and only on `pattern` strings that actually +// contain a property escape. Everything else in the schema (including valid +// patterns) passes through byte-identical. + +// `\p{...}` / `\P{...}` with an odd number of preceding backslashes — an even +// count means the backslash itself is escaped, so `\\p{Cc}` is a literal "p". +const UNICODE_PROPERTY_ESCAPE = /(^|[^\\])(\\\\)*\\[pP]\{/; + +export function hasUnicodePropertyEscape(pattern) { + return typeof pattern === "string" && UNICODE_PROPERTY_ESCAPE.test(pattern); +} + +// Copy-on-write walk: returns the original reference when nothing changed, so +// untouched schemas keep object identity and callers can cheaply detect a no-op. +// `properties` is special-cased because its keys are arbitrary property *names* +// (which may themselves be "pattern" or "properties") and must never be read as +// schema keywords; every other key recurses as an ordinary schema node. +function stripNode(node, stats) { + if (Array.isArray(node)) { + let changed = false; + const next = node.map((item) => { + const cleaned = stripNode(item, stats); + if (cleaned !== item) changed = true; + return cleaned; + }); + return changed ? next : node; + } + if (!node || typeof node !== "object") return node; + + let changed = false; + const next = {}; + for (const [key, value] of Object.entries(node)) { + if (key === "pattern" && hasUnicodePropertyEscape(value)) { + stats.removed++; + changed = true; + continue; + } + if (key === "properties" && value && typeof value === "object" && !Array.isArray(value)) { + let propsChanged = false; + const props = {}; + for (const [propName, propSchema] of Object.entries(value)) { + const cleaned = stripNode(propSchema, stats); + if (cleaned !== propSchema) propsChanged = true; + props[propName] = cleaned; + } + if (propsChanged) changed = true; + next[key] = propsChanged ? props : value; + continue; + } + const cleaned = stripNode(value, stats); + if (cleaned !== value) changed = true; + next[key] = cleaned; + } + return changed ? next : node; +} + +// Remove only the `pattern` constraints Codex's validator rejects. +// Returns the same reference when the schema is already compatible. +export function stripCodexUnsupportedPatterns(schema, stats = { removed: 0 }) { + return stripNode(schema, stats); +} diff --git a/tests/unit/codex-tool-normalization.test.js b/tests/unit/codex-tool-normalization.test.js index d1b5901e..c7193898 100644 --- a/tests/unit/codex-tool-normalization.test.js +++ b/tests/unit/codex-tool-normalization.test.js @@ -118,6 +118,69 @@ describe("CodexExecutor tool normalization", () => { ]); }); + it("strips only Unicode-property patterns rejected by Codex", () => { + const unicodePattern = "^(?!__.*__$)[^\\p{Cc}\\p{Cf}\\p{Zl}\\p{Zp}]{1,200}$"; + const validPattern = "^[a-z][a-z0-9_-]{0,31}$"; + const sourceParameters = { + type: "object", + properties: { + artifact: { + type: "object", + properties: { + name: { type: "string", pattern: unicodePattern }, + slug: { type: "string", pattern: validPattern }, + }, + }, + // A property named "pattern" is data, not the schema keyword. + pattern: { type: "string", pattern: validPattern }, + }, + allOf: [{ properties: { title: { type: "string", pattern: unicodePattern } } }], + }; + const tools = normalizeTools([{ + type: "function", + name: "Artifact", + parameters: sourceParameters, + }]); + + expect(tools[0].parameters.properties.artifact.properties.name.pattern).toBeUndefined(); + expect(tools[0].parameters.properties.artifact.properties.slug.pattern).toBe(validPattern); + expect(tools[0].parameters.properties.pattern.pattern).toBe(validPattern); + expect(tools[0].parameters.allOf[0].properties.title.pattern).toBeUndefined(); + // Copy-on-write: the caller's schema remains available for another provider. + expect(sourceParameters.properties.artifact.properties.name.pattern).toBe(unicodePattern); + }); + + it("keeps escaped literal property text and schema identity when no strip is needed", () => { + const parameters = { + type: "object", + properties: { + literal: { type: "string", pattern: "^\\\\p{Cc}$" }, + simple: { type: "string", pattern: "^[A-Z]+$" }, + }, + }; + const tools = normalizeTools([{ type: "function", name: "probe", parameters }]); + + expect(tools[0].parameters).toBe(parameters); + expect(tools[0].parameters.properties.literal.pattern).toBe("^\\\\p{Cc}$"); + }); + + it("sanitizes nested namespace function schemas", () => { + const tools = normalizeTools([{ + type: "namespace", + name: "agent", + tools: [{ + type: "function", + name: "Artifact", + parameters: { + type: "object", + properties: { name: { type: "string", pattern: "^\\p{Cc}+$" } }, + }, + }], + }]); + + expect(tools[0].tools[0].parameters.properties.name.pattern).toBeUndefined(); + }); + it("preserves custom freeform tools with format payloads", () => { const tools = normalizeTools([ {