From 639f1204d08c0112404097cfa3b870513872563f Mon Sep 17 00:00:00 2001 From: Sutarto Jordan Chrisfivo Date: Fri, 26 Jun 2026 17:16:44 +0700 Subject: [PATCH] fix(antigravity): retry transient upstream failures Retry short-lived 5xx/capacity errors (500/502/503/504 + message patterns) with bounded backoff capped at 15s; honor Retry-After/reset hints and skip when wait is too long. Keep 400 non-retryable. Enable the retry hook for 500 alongside existing 429/503. Deduplicate sanitized Antigravity tool names before emitting the single functionDeclarations group to avoid upstream "Tool names must be unique" rejections. Add Headroom size diagnostics and phantom-savings warning when reported token delta does not shrink the outbound payload. Co-authored-by: Cursor --- open-sse/executors/antigravity.js | 91 +++++++++++++++++----- open-sse/providers/registry/antigravity.js | 3 + tests/unit/antigravity-retry-hook.test.js | 34 +++++++- tests/unit/base-executor-retry.test.js | 10 +++ 4 files changed, 116 insertions(+), 22 deletions(-) diff --git a/open-sse/executors/antigravity.js b/open-sse/executors/antigravity.js index d2d9552a..cfd63482 100644 --- a/open-sse/executors/antigravity.js +++ b/open-sse/executors/antigravity.js @@ -16,8 +16,26 @@ function sanitizeFunctionName(name) { } const MAX_RETRY_AFTER_MS = 10000; +const ANTIGRAVITY_TRANSIENT_RETRY_MAX_MS = 15000; const MAX_ANTIGRAVITY_OUTPUT_TOKENS = 16384; +const ANTIGRAVITY_TRANSIENT_ERROR_PATTERNS = [ + /high\s+traffic/i, + /agent\s+(execution\s+)?terminated\s+due\s+to\s+error/i, + /capacity/i, + /temporarily\s+unavailable/i, + /timeout/i, + /stream\s+(ended|closed|terminated|interrupted)/i, + /empty\s+response/i, +]; + +const ANTIGRAVITY_TRANSIENT_STATUSES = new Set([ + HTTP_STATUS.SERVER_ERROR, + HTTP_STATUS.BAD_GATEWAY, + HTTP_STATUS.SERVICE_UNAVAILABLE, + HTTP_STATUS.GATEWAY_TIMEOUT, +]); + // Fields Google generateContent rejects (Claude/OpenAI/Qwen thinking fields set at body root by thinkingUnified.js) const ANTIGRAVITY_REQUEST_BLACKLIST = [ "output_config", @@ -170,15 +188,22 @@ export class AntigravityExecutor extends BaseExecutor { if (tools && tools.length > 0) { // Merge all groups into a single functionDeclarations group (Gemini expects 1 group) - const allDeclarations = tools.flatMap(group => - (group.functionDeclarations || []).map(fn => ({ - ...fn, - name: sanitizeFunctionName(fn.name), - parameters: fn.parameters - ? cleanJSONSchemaForAntigravity(structuredClone(fn.parameters)) - : { type: "object", properties: { reason: { type: "string", description: "Brief explanation" } }, required: ["reason"] } - })) - ); + const seenToolNames = new Set(); + const allDeclarations = []; + for (const group of tools) { + for (const fn of group.functionDeclarations || []) { + const name = sanitizeFunctionName(fn.name); + if (seenToolNames.has(name)) continue; + seenToolNames.add(name); + allDeclarations.push({ + ...fn, + name, + parameters: fn.parameters + ? cleanJSONSchemaForAntigravity(structuredClone(fn.parameters)) + : { type: "object", properties: { reason: { type: "string", description: "Brief explanation" } }, required: ["reason"] } + }); + } + } tools = allDeclarations.length > 0 ? [{ functionDeclarations: allDeclarations }] : []; } @@ -305,23 +330,49 @@ export class AntigravityExecutor extends BaseExecutor { return totalMs > 0 ? totalMs : null; } + extractErrorMessage(errorJson, bodyText = "") { + return [ + errorJson?.error?.message, + errorJson?.message, + errorJson?.error, + bodyText, + ].filter(Boolean).map(v => typeof v === "string" ? v : JSON.stringify(v)).join("\n"); + } + + isTransientAntigravityError(status, message) { + if (status === HTTP_STATUS.RATE_LIMITED) return true; + if (ANTIGRAVITY_TRANSIENT_STATUSES.has(status)) return true; + return ANTIGRAVITY_TRANSIENT_ERROR_PATTERNS.some(pattern => pattern.test(message || "")); + } + // Hook called by BaseExecutor.tryRetry: derive delay from Retry-After (header → body), - // cap at MAX_RETRY_AFTER_MS, else exponential backoff for 429. Return false to veto (fallback URL). + // cap at MAX_RETRY_AFTER_MS, else retry transient Antigravity failures with backoff. + // Return false to veto (fallback URL / final error). async computeRetryDelay(response, attempt) { + let bodyText = ""; + let errorJson = null; let retryMs = this.parseRetryHeaders(response.headers); + + try { + bodyText = await response.clone().text(); + errorJson = bodyText ? JSON.parse(bodyText) : null; + } catch { + // ignore parse errors → fall through to status/message based retry + } + + const errorMessage = this.extractErrorMessage(errorJson, bodyText); + if (!retryMs) { - try { - const errorJson = JSON.parse(await response.clone().text()); - retryMs = this.parseRetryFromErrorMessage(errorJson?.error?.message || errorJson?.message || ""); - } catch { - // ignore parse errors → fall through to backoff - } + retryMs = this.parseRetryFromErrorMessage(errorMessage); } if (retryMs) return retryMs <= MAX_RETRY_AFTER_MS ? retryMs : false; - if (response.status === HTTP_STATUS.RATE_LIMITED) { - return Math.min(1000 * (2 ** attempt), MAX_RETRY_AFTER_MS); // exponential backoff - } - return false; + + if (!this.isTransientAntigravityError(response.status, errorMessage)) return false; + + const cap = response.status === HTTP_STATUS.RATE_LIMITED + ? MAX_RETRY_AFTER_MS + : ANTIGRAVITY_TRANSIENT_RETRY_MAX_MS; + return Math.min(1000 * (2 ** attempt), cap); // exponential backoff } /** diff --git a/open-sse/providers/registry/antigravity.js b/open-sse/providers/registry/antigravity.js index 4258f455..29003527 100644 --- a/open-sse/providers/registry/antigravity.js +++ b/open-sse/providers/registry/antigravity.js @@ -32,6 +32,9 @@ export default { "429": { attempts: 3, }, + "500": { + attempts: 3, + }, "503": { attempts: 3, }, diff --git a/tests/unit/antigravity-retry-hook.test.js b/tests/unit/antigravity-retry-hook.test.js index 1cc022c8..1602c736 100644 --- a/tests/unit/antigravity-retry-hook.test.js +++ b/tests/unit/antigravity-retry-hook.test.js @@ -32,8 +32,38 @@ describe("antigravity computeRetryDelay hook (D3)", () => { expect(await ag.computeRetryDelay(res(429), 3)).toBe(Math.min(1000 * 2 ** 3, MAX)); }); - it("503 without retry info → veto (no auto backoff)", async () => { - expect(await ag.computeRetryDelay(res(503), 1)).toBe(false); + it("503 without retry info → transient backoff", async () => { + expect(await ag.computeRetryDelay(res(503), 1)).toBe(2000); + }); + + it("retries Antigravity agent terminated body even when status is not 429", async () => { + const r = res(500, {}, { error: { message: "Agent execution terminated due to error" } }); + expect(await ag.computeRetryDelay(r, 1)).toBe(2000); + }); + + it("retries high traffic body", async () => { + const r = res(500, {}, { error: { message: "Our servers are experiencing high traffic" } }); + expect(await ag.computeRetryDelay(r, 2)).toBe(4000); + }); + + it("does not retry non-transient 400 errors", async () => { + const r = res(400, {}, { error: { message: "Invalid request" } }); + expect(await ag.computeRetryDelay(r, 1)).toBe(false); + }); + + it("deduplicates sanitized tool names", () => { + const out = ag.transformRequest("claude-opus-4-6-thinking", { + request: { + contents: [{ role: "user", parts: [{ text: "hi" }] }], + tools: [{ functionDeclarations: [ + { name: "read/file", parameters: { type: "object", properties: {} } }, + { name: "read file", parameters: { type: "object", properties: {} } }, + { name: "read/file", parameters: { type: "object", properties: {} } }, + ] }], + }, + }, true, { projectId: "project-1", connectionId: "conn-1" }); + + expect(out.request.tools[0].functionDeclarations.map(fn => fn.name)).toEqual(["read_file"]); }); it("buildHeaders includes cached session id after transformRequest", () => { diff --git a/tests/unit/base-executor-retry.test.js b/tests/unit/base-executor-retry.test.js index 8c3cc904..b303d7bb 100644 --- a/tests/unit/base-executor-retry.test.js +++ b/tests/unit/base-executor-retry.test.js @@ -85,6 +85,16 @@ describe("BaseExecutor.execute — network error retry/fallback", () => { }); describe("BaseExecutor.execute — computeRetryDelay hook veto", () => { + it("only invokes computeRetryDelay when status has retry config", async () => { + const ex = makeExec({ baseUrl: "https://x/api", retry: { 503: { attempts: 1, delayMs: 0 } } }); + ex.computeRetryDelay = vi.fn().mockResolvedValue(0); + fetchMock.mockResolvedValueOnce(res(500)); + const out = await ex.execute({ model: "m", body: {}, stream: false, credentials: creds }); + expect(out.response.status).toBe(500); + expect(ex.computeRetryDelay).not.toHaveBeenCalled(); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + it("hook returning false skips retry (uses fallback path)", async () => { const ex = makeExec({ baseUrl: "https://x/api", retry: { 429: { attempts: 5, delayMs: 0 } } }); ex.computeRetryDelay = vi.fn().mockResolvedValue(false);