fix(antigravity): strike-break optimistic quota readings that keep 429ing

Google's quota API can report remaining quota while generation endpoints
keep returning 429 (sprint/weekly dual-pool mismatch). handleAntigravityQuotaError
trusted remainingPercentage > 0 as healthy and returned null, causing 429 retry
loops across multi-account pools.

Add a strike-based circuit breaker to the optimistic and unavailable quota paths:
- After 3 strikes (429/409) within 60s for the same connection+model, cache-block
  that pair for 15 minutes by synthesizing an entry in the shared RAM quota cache.
- Re-assert active strike blocks across refreshes so optimistic readings cannot
  resurrect a broken pair prematurely.
- Reset strikes and clear synthesized cache entry upon successful request.
- Keep exact-resetAt handling for genuine 0% exhausted readings.

Closes #3681
This commit is contained in:
louis-cai
2026-09-03 09:34:05 +07:00
parent a58902e4a7
commit ac98dd9d32
3 changed files with 233 additions and 4 deletions

View File

@@ -27,7 +27,7 @@ vi.mock("open-sse/services/usage/google.js", () => ({
}));
vi.mock("@/sse/utils/logger.js", () => ({ debug: vi.fn(), info: vi.fn(), warn: vi.fn() }));
const { getAntigravityQuotaCache, handleAntigravityQuotaError, refreshAntigravityQuota } = await import("@/sse/services/antigravityQuota.js");
const { getAntigravityQuotaCache, handleAntigravityQuotaError, refreshAntigravityQuota, clearAntigravityStrikes } = await import("@/sse/services/antigravityQuota.js");
const { getProviderCredentials } = await import("@/sse/services/auth.js");
const MODEL = "claude-opus-4-6-thinking";
@@ -169,4 +169,143 @@ describe("Antigravity quota-aware routing", () => {
vi.useRealTimers();
}
});
it("strike-breaks after 3 optimistic 429s within 60s and cache-blocks 15 minutes", async () => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-08-26T00:00:00.000Z"));
// Quota API lies: reports 90% remaining while generation keeps 429ing.
mocks.getAntigravityUsage.mockResolvedValue({ quotas: {
[MODEL]: { remainingPercentage: 90, resetAt: FUTURE_RESET },
} });
try {
const first = await handleAntigravityQuotaError("ag-strike", 429, MODEL, "token", {});
expect(first).toBeNull();
const second = await handleAntigravityQuotaError("ag-strike", 429, MODEL, "token", {});
expect(second).toBeNull();
const third = await handleAntigravityQuotaError("ag-strike", 429, MODEL, "token", {});
expect(third).toBe(Date.parse("2026-08-26T00:15:00.000Z"));
} finally {
vi.useRealTimers();
}
});
it("resets the strike counter when strikes fall outside the 60s window", async () => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-08-26T00:00:00.000Z"));
mocks.getAntigravityUsage.mockResolvedValue({ quotas: {
[MODEL]: { remainingPercentage: 90, resetAt: FUTURE_RESET },
} });
try {
await handleAntigravityQuotaError("ag-window", 429, MODEL, "token", {});
await handleAntigravityQuotaError("ag-window", 429, MODEL, "token", {});
await vi.advanceTimersByTimeAsync(61_000);
const result = await handleAntigravityQuotaError("ag-window", 429, MODEL, "token", {});
expect(result).toBeNull(); // window lapsed — counter restarted at 1
} finally {
vi.useRealTimers();
}
});
it("strike-breaks when the quota API is unavailable (null reading) too", async () => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-08-26T00:00:00.000Z"));
// Quota endpoint failing/forbidden => quota unknown. Strikes must still count.
mocks.getAntigravityUsage.mockResolvedValue({ message: "forbidden", quotas: {} });
try {
await handleAntigravityQuotaError("ag-null", 429, MODEL, "token", {});
await handleAntigravityQuotaError("ag-null", 429, MODEL, "token", {});
const third = await handleAntigravityQuotaError("ag-null", 429, MODEL, "token", {});
expect(third).toBe(Date.parse("2026-08-26T00:15:00.000Z"));
} finally {
vi.useRealTimers();
}
});
it("persists the block into the shared cache so the next request skips the pair", async () => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-08-26T00:00:00.000Z"));
mocks.getAntigravityUsage.mockResolvedValue({ quotas: {
[MODEL]: { remainingPercentage: 90, resetAt: FUTURE_RESET },
} });
try {
await handleAntigravityQuotaError("ag-persist", 429, MODEL, "token", {});
await handleAntigravityQuotaError("ag-persist", 429, MODEL, "token", {});
await handleAntigravityQuotaError("ag-persist", 429, MODEL, "token", {});
// The synthesized entry must be visible to the auth pre-filter reading
// the shared cache — and must survive an optimistic upstream refresh.
const cached = getAntigravityQuotaCache().get("ag-persist")?.[MODEL];
expect(cached).toMatchObject({ remainingPercentage: 0 });
expect(Date.parse(cached.resetAt)).toBe(Date.parse("2026-08-26T00:15:00.000Z"));
await refreshAntigravityQuota("ag-persist", "token", {});
expect(getAntigravityQuotaCache().get("ag-persist")?.[MODEL]).toMatchObject({
remainingPercentage: 0,
});
} finally {
vi.useRealTimers();
}
});
it("clears strike state and the synthesized block after a successful request", async () => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-08-26T00:00:00.000Z"));
mocks.getAntigravityUsage.mockResolvedValue({ quotas: {
[MODEL]: { remainingPercentage: 90, resetAt: FUTURE_RESET },
} });
try {
await handleAntigravityQuotaError("ag-clear", 429, MODEL, "token", {});
await handleAntigravityQuotaError("ag-clear", 429, MODEL, "token", {});
await handleAntigravityQuotaError("ag-clear", 429, MODEL, "token", {});
expect(getAntigravityQuotaCache().get("ag-clear")?.[MODEL]?.remainingPercentage).toBe(0);
clearAntigravityStrikes("ag-clear", MODEL);
// Synthesized entry gone — pair selectable again immediately.
expect(getAntigravityQuotaCache().get("ag-clear")?.[MODEL]).toBeUndefined();
// Two more 429s do NOT inherit earlier strikes: no block on the third-in-episode.
await handleAntigravityQuotaError("ag-clear", 429, MODEL, "token", {});
await expect(handleAntigravityQuotaError("ag-clear", 429, MODEL, "token", {})).resolves.toBeNull();
} finally {
vi.useRealTimers();
}
});
it("anchors the window at the first strike: 3 strikes spread over 90s do not trip", async () => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-08-26T00:00:00.000Z"));
mocks.getAntigravityUsage.mockResolvedValue({ quotas: {
[MODEL]: { remainingPercentage: 90, resetAt: FUTURE_RESET },
} });
try {
await handleAntigravityQuotaError("ag-anchor", 429, MODEL, "token", {}); // t=0
await vi.advanceTimersByTimeAsync(45_000);
await handleAntigravityQuotaError("ag-anchor", 429, MODEL, "token", {}); // t=45s
await vi.advanceTimersByTimeAsync(45_000);
// t=90s: within 60s of strike #2 but outside 60s of strike #1 => new window
const result = await handleAntigravityQuotaError("ag-anchor", 429, MODEL, "token", {});
expect(result).toBeNull();
} finally {
vi.useRealTimers();
}
});
it("keeps the optimistic path null without touching the quota cache", async () => {
mocks.getAntigravityUsage.mockResolvedValue({ quotas: {
[MODEL]: { remainingPercentage: 90, resetAt: FUTURE_RESET },
} });
await expect(handleAntigravityQuotaError("ag-optimistic", 429, MODEL, "token", {}))
.resolves.toBeNull();
// Optimistic reading must NOT poison the shared cache (auth pre-filter
// treats cached 0% as exhausted).
expect(getAntigravityQuotaCache().get("ag-optimistic")?.[MODEL]?.remainingPercentage).toBe(90);
});
});