From 1442cc73ce12d4d30f8a8e6a50a00fbdacb90f18 Mon Sep 17 00:00:00 2001 From: Hifzi Date: Sat, 5 Sep 2026 21:27:21 +0700 Subject: [PATCH] fix(antigravity): prevent Google anti-abuse rate limits on multi-account refresh (#3813) --- open-sse/services/projectId.js | 13 ++++--- src/sse/services/backgroundTokenRefresh.js | 44 +++++++++++++++------- src/sse/services/tokenRefresh.js | 31 ++++++++------- 3 files changed, 57 insertions(+), 31 deletions(-) diff --git a/open-sse/services/projectId.js b/open-sse/services/projectId.js index 84ab5a2b..f582c4af 100644 --- a/open-sse/services/projectId.js +++ b/open-sse/services/projectId.js @@ -203,7 +203,8 @@ async function onboardUser(accessToken, tierID, externalSignal, endpoints, provi const reqBody = { tierId: tierID, metadata: LOAD_CODE_ASSIST_METADATA }; const headers = provider === "antigravity" ? ANTIGRAVITY_LOAD_CODE_ASSIST_HEADERS : LOAD_CODE_ASSIST_HEADERS; - const MAX_ATTEMPTS = 5; + const MAX_ATTEMPTS = Number(process.env.ONBOARD_MAX_ATTEMPTS) || 2; + const BASE_RETRY_DELAY_MS = Number(process.env.ONBOARD_RETRY_DELAY_MS) || 12_000; for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { // Bail out immediately if the connection was removed @@ -241,9 +242,10 @@ async function onboardUser(accessToken, tierID, externalSignal, endpoints, provi throw new Error("onboardUser done but no project_id in response"); } - // Server not done yet – wait and retry + // Server not done yet – wait and retry with jitter + const jitter = Math.floor(Math.random() * 5000); console.log(`[ProjectId] Onboard attempt ${attempt}/${MAX_ATTEMPTS}: not done yet, waiting...`); - await new Promise(resolve => setTimeout(resolve, 2000)); + await new Promise(resolve => setTimeout(resolve, BASE_RETRY_DELAY_MS + jitter)); } catch (error) { clearTimeout(timeoutId); @@ -256,9 +258,10 @@ async function onboardUser(accessToken, tierID, externalSignal, endpoints, provi console.warn(`[ProjectId] onboardUser failed after ${MAX_ATTEMPTS} attempts: ${error.message}`); return null; } - // Continue to next attempt instead of throwing (which would skip remaining retries) + // Wait with jitter before retrying + const jitter = Math.floor(Math.random() * 5000); console.warn(`[ProjectId] onboardUser attempt ${attempt} failed: ${error.message}, retrying...`); - await new Promise(resolve => setTimeout(resolve, 2000)); + await new Promise(resolve => setTimeout(resolve, BASE_RETRY_DELAY_MS + jitter)); } finally { clearTimeout(timeoutId); externalSignal?.removeEventListener("abort", forwardAbort); diff --git a/src/sse/services/backgroundTokenRefresh.js b/src/sse/services/backgroundTokenRefresh.js index 5810e049..b83bf335 100644 --- a/src/sse/services/backgroundTokenRefresh.js +++ b/src/sse/services/backgroundTokenRefresh.js @@ -9,6 +9,7 @@ import { getCredentialExpiryMs } from "open-sse/services/oauthCredentialManager. export const BACKGROUND_REFRESH_LEAD_MS = 30 * 60 * 1000; const DEFAULT_INTERVAL_MS = 5 * 60 * 1000; const INITIAL_DELAY_MS = 10 * 1000; +const SENSITIVE_PROVIDERS = new Set(["antigravity", "gemini-cli"]); let started = false; let intervalHandle = null; @@ -92,25 +93,42 @@ export async function runBackgroundTokenRefreshTick(deps = {}) { try { const load = deps.loadConnections || loadActiveConnections; const refresh = deps.refreshConnection || refreshOne; + const sleep = deps.sleep || ((ms) => new Promise((res) => setTimeout(res, ms))); const connections = await load(); const due = selectConnectionsNeedingRefresh(connections, Date.now()); if (due.length === 0) return; - await Promise.allSettled( - due.map(async (conn) => { - try { - await refresh(conn); - } catch (err) { - log.warn("BG_TOKEN_REFRESH", "Connection refresh failed (swallowed)", { - id: conn?.id, - provider: conn?.provider, - error: err?.message ?? String(err), - }); - } - }) - ); + const baseSensitiveDelay = Number(process.env.BG_REFRESH_GOOGLE_DELAY_MS) || 12_000; + const baseNormalDelay = Number(process.env.BG_REFRESH_DELAY_MS) || 1_500; + + for (let i = 0; i < due.length; i++) { + const conn = due[i]; + try { + await refresh(conn); + log.info("BG_TOKEN_REFRESH", "Connection refresh finished", { + id: conn.id, + email: conn.email || conn.name || conn.id, + provider: conn.provider, + }); + } catch (err) { + log.warn("BG_TOKEN_REFRESH", "Connection refresh failed (swallowed)", { + id: conn?.id, + email: conn?.email || conn?.name || conn?.id, + provider: conn?.provider, + error: err?.message ?? String(err), + }); + } + + // Sequential delay between accounts to prevent bursting upstream providers (especially Google Cloud) + if (i < due.length - 1) { + const isSensitive = SENSITIVE_PROVIDERS.has(conn.provider); + const baseDelay = isSensitive ? baseSensitiveDelay : baseNormalDelay; + const jitter = isSensitive ? Math.floor(Math.random() * 4000) : 200; + await sleep(baseDelay + jitter); + } + } } catch (err) { log.warn("BG_TOKEN_REFRESH", "Tick failed (swallowed)", { error: err?.message ?? String(err), diff --git a/src/sse/services/tokenRefresh.js b/src/sse/services/tokenRefresh.js index 6c808f07..58a6f870 100644 --- a/src/sse/services/tokenRefresh.js +++ b/src/sse/services/tokenRefresh.js @@ -124,25 +124,30 @@ function needsProjectId(provider) { function _refreshProjectId(provider, connectionId, accessToken) { if (!needsProjectId(provider) || !connectionId || !accessToken) return; - // Evict the stale cached entry so getProjectIdForConnection does a real fetch + // Invalidate the stale cached entry so getProjectIdForConnection does a real fetch invalidateProjectId(connectionId); - getProjectIdForConnection(connectionId, accessToken) - .then((projectId) => { - if (!projectId) return; - updateProviderCredentials(connectionId, { projectId }).catch((err) => { - log.debug("TOKEN_REFRESH", "Failed to persist refreshed projectId", { + // Lazy resolution: Do not eagerly trigger onboardUser during background token refresh. + // Eagerly fetching projectId across multiple accounts simultaneously triggers Google Cloud anti-abuse / rate limits. + // Runtime handlers (e.g. chat handler) will lazily call getProjectIdForConnection() on demand. + if (process.env.EAGER_PROJECT_ID_REFRESH === "true") { + getProjectIdForConnection(connectionId, accessToken, provider) + .then((projectId) => { + if (!projectId) return; + updateProviderCredentials(connectionId, { projectId }).catch((err) => { + log.debug("TOKEN_REFRESH", "Failed to persist refreshed projectId", { + connectionId, + error: err?.message ?? err, + }); + }); + }) + .catch((err) => { + log.debug("TOKEN_REFRESH", "Failed to fetch projectId after token refresh", { connectionId, error: err?.message ?? err, }); }); - }) - .catch((err) => { - log.debug("TOKEN_REFRESH", "Failed to fetch projectId after token refresh", { - connectionId, - error: err?.message ?? err, - }); - }); + } } // ─── Local-specific: persist credentials to localDb ──────────────────────────