fix(antigravity): strike-break optimistic quota readings that keep 429ing
Google's quota API can report remaining quota while generation endpoints keep returning 429 (sprint/weekly dual-pool mismatch). handleAntigravityQuotaError trusted remainingPercentage > 0 as healthy and returned null, causing 429 retry loops across multi-account pools. Add a strike-based circuit breaker to the optimistic and unavailable quota paths: - After 3 strikes (429/409) within 60s for the same connection+model, cache-block that pair for 15 minutes by synthesizing an entry in the shared RAM quota cache. - Re-assert active strike blocks across refreshes so optimistic readings cannot resurrect a broken pair prematurely. - Reset strikes and clear synthesized cache entry upon successful request. - Keep exact-resetAt handling for genuine 0% exhausted readings. Closes #3681
This commit is contained in:
@@ -27,7 +27,7 @@ vi.mock("open-sse/services/usage/google.js", () => ({
|
||||
}));
|
||||
vi.mock("@/sse/utils/logger.js", () => ({ debug: vi.fn(), info: vi.fn(), warn: vi.fn() }));
|
||||
|
||||
const { getAntigravityQuotaCache, handleAntigravityQuotaError, refreshAntigravityQuota } = await import("@/sse/services/antigravityQuota.js");
|
||||
const { getAntigravityQuotaCache, handleAntigravityQuotaError, refreshAntigravityQuota, clearAntigravityStrikes } = await import("@/sse/services/antigravityQuota.js");
|
||||
const { getProviderCredentials } = await import("@/sse/services/auth.js");
|
||||
|
||||
const MODEL = "claude-opus-4-6-thinking";
|
||||
@@ -169,4 +169,143 @@ describe("Antigravity quota-aware routing", () => {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("strike-breaks after 3 optimistic 429s within 60s and cache-blocks 15 minutes", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-08-26T00:00:00.000Z"));
|
||||
// Quota API lies: reports 90% remaining while generation keeps 429ing.
|
||||
mocks.getAntigravityUsage.mockResolvedValue({ quotas: {
|
||||
[MODEL]: { remainingPercentage: 90, resetAt: FUTURE_RESET },
|
||||
} });
|
||||
|
||||
try {
|
||||
const first = await handleAntigravityQuotaError("ag-strike", 429, MODEL, "token", {});
|
||||
expect(first).toBeNull();
|
||||
const second = await handleAntigravityQuotaError("ag-strike", 429, MODEL, "token", {});
|
||||
expect(second).toBeNull();
|
||||
|
||||
const third = await handleAntigravityQuotaError("ag-strike", 429, MODEL, "token", {});
|
||||
expect(third).toBe(Date.parse("2026-08-26T00:15:00.000Z"));
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("resets the strike counter when strikes fall outside the 60s window", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-08-26T00:00:00.000Z"));
|
||||
mocks.getAntigravityUsage.mockResolvedValue({ quotas: {
|
||||
[MODEL]: { remainingPercentage: 90, resetAt: FUTURE_RESET },
|
||||
} });
|
||||
|
||||
try {
|
||||
await handleAntigravityQuotaError("ag-window", 429, MODEL, "token", {});
|
||||
await handleAntigravityQuotaError("ag-window", 429, MODEL, "token", {});
|
||||
await vi.advanceTimersByTimeAsync(61_000);
|
||||
const result = await handleAntigravityQuotaError("ag-window", 429, MODEL, "token", {});
|
||||
expect(result).toBeNull(); // window lapsed — counter restarted at 1
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("strike-breaks when the quota API is unavailable (null reading) too", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-08-26T00:00:00.000Z"));
|
||||
// Quota endpoint failing/forbidden => quota unknown. Strikes must still count.
|
||||
mocks.getAntigravityUsage.mockResolvedValue({ message: "forbidden", quotas: {} });
|
||||
|
||||
try {
|
||||
await handleAntigravityQuotaError("ag-null", 429, MODEL, "token", {});
|
||||
await handleAntigravityQuotaError("ag-null", 429, MODEL, "token", {});
|
||||
const third = await handleAntigravityQuotaError("ag-null", 429, MODEL, "token", {});
|
||||
expect(third).toBe(Date.parse("2026-08-26T00:15:00.000Z"));
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("persists the block into the shared cache so the next request skips the pair", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-08-26T00:00:00.000Z"));
|
||||
mocks.getAntigravityUsage.mockResolvedValue({ quotas: {
|
||||
[MODEL]: { remainingPercentage: 90, resetAt: FUTURE_RESET },
|
||||
} });
|
||||
|
||||
try {
|
||||
await handleAntigravityQuotaError("ag-persist", 429, MODEL, "token", {});
|
||||
await handleAntigravityQuotaError("ag-persist", 429, MODEL, "token", {});
|
||||
await handleAntigravityQuotaError("ag-persist", 429, MODEL, "token", {});
|
||||
|
||||
// The synthesized entry must be visible to the auth pre-filter reading
|
||||
// the shared cache — and must survive an optimistic upstream refresh.
|
||||
const cached = getAntigravityQuotaCache().get("ag-persist")?.[MODEL];
|
||||
expect(cached).toMatchObject({ remainingPercentage: 0 });
|
||||
expect(Date.parse(cached.resetAt)).toBe(Date.parse("2026-08-26T00:15:00.000Z"));
|
||||
|
||||
await refreshAntigravityQuota("ag-persist", "token", {});
|
||||
expect(getAntigravityQuotaCache().get("ag-persist")?.[MODEL]).toMatchObject({
|
||||
remainingPercentage: 0,
|
||||
});
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("clears strike state and the synthesized block after a successful request", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-08-26T00:00:00.000Z"));
|
||||
mocks.getAntigravityUsage.mockResolvedValue({ quotas: {
|
||||
[MODEL]: { remainingPercentage: 90, resetAt: FUTURE_RESET },
|
||||
} });
|
||||
|
||||
try {
|
||||
await handleAntigravityQuotaError("ag-clear", 429, MODEL, "token", {});
|
||||
await handleAntigravityQuotaError("ag-clear", 429, MODEL, "token", {});
|
||||
await handleAntigravityQuotaError("ag-clear", 429, MODEL, "token", {});
|
||||
expect(getAntigravityQuotaCache().get("ag-clear")?.[MODEL]?.remainingPercentage).toBe(0);
|
||||
|
||||
clearAntigravityStrikes("ag-clear", MODEL);
|
||||
// Synthesized entry gone — pair selectable again immediately.
|
||||
expect(getAntigravityQuotaCache().get("ag-clear")?.[MODEL]).toBeUndefined();
|
||||
|
||||
// Two more 429s do NOT inherit earlier strikes: no block on the third-in-episode.
|
||||
await handleAntigravityQuotaError("ag-clear", 429, MODEL, "token", {});
|
||||
await expect(handleAntigravityQuotaError("ag-clear", 429, MODEL, "token", {})).resolves.toBeNull();
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("anchors the window at the first strike: 3 strikes spread over 90s do not trip", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-08-26T00:00:00.000Z"));
|
||||
mocks.getAntigravityUsage.mockResolvedValue({ quotas: {
|
||||
[MODEL]: { remainingPercentage: 90, resetAt: FUTURE_RESET },
|
||||
} });
|
||||
|
||||
try {
|
||||
await handleAntigravityQuotaError("ag-anchor", 429, MODEL, "token", {}); // t=0
|
||||
await vi.advanceTimersByTimeAsync(45_000);
|
||||
await handleAntigravityQuotaError("ag-anchor", 429, MODEL, "token", {}); // t=45s
|
||||
await vi.advanceTimersByTimeAsync(45_000);
|
||||
// t=90s: within 60s of strike #2 but outside 60s of strike #1 => new window
|
||||
const result = await handleAntigravityQuotaError("ag-anchor", 429, MODEL, "token", {});
|
||||
expect(result).toBeNull();
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps the optimistic path null without touching the quota cache", async () => {
|
||||
mocks.getAntigravityUsage.mockResolvedValue({ quotas: {
|
||||
[MODEL]: { remainingPercentage: 90, resetAt: FUTURE_RESET },
|
||||
} });
|
||||
|
||||
await expect(handleAntigravityQuotaError("ag-optimistic", 429, MODEL, "token", {}))
|
||||
.resolves.toBeNull();
|
||||
// Optimistic reading must NOT poison the shared cache (auth pre-filter
|
||||
// treats cached 0% as exhausted).
|
||||
expect(getAntigravityQuotaCache().get("ag-optimistic")?.[MODEL]?.remainingPercentage).toBe(90);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user