luulam/9router
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(antigravity): prevent Google anti-abuse rate limits on multi-account refresh (#3813)

Browse Source
This commit is contained in:
Hifzi
2026-09-05 21:27:21 +07:00
parent fb9fab0206
commit 1442cc73ce
3 changed files with 57 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
open-sse/services/projectId.js
View File

@@ -203,7 +203,8 @@ async function onboardUser(accessToken, tierID, externalSignal, endpoints, provi
const reqBody = { tierId: tierID, metadata: LOAD_CODE_ASSIST_METADATA };
const headers = provider === "antigravity" ? ANTIGRAVITY_LOAD_CODE_ASSIST_HEADERS : LOAD_CODE_ASSIST_HEADERS;
const MAX_ATTEMPTS = 5;
const MAX_ATTEMPTS = Number(process.env.ONBOARD_MAX_ATTEMPTS) || 2;
const BASE_RETRY_DELAY_MS = Number(process.env.ONBOARD_RETRY_DELAY_MS) || 12_000;
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
// Bail out immediately if the connection was removed
@@ -241,9 +242,10 @@ async function onboardUser(accessToken, tierID, externalSignal, endpoints, provi
throw new Error("onboardUser done but no project_id in response");
}
// Server not done yet – wait and retry
// Server not done yet – wait and retry with jitter
const jitter = Math.floor(Math.random() * 5000);
console.log(`[ProjectId] Onboard attempt ${attempt}/${MAX_ATTEMPTS}: not done yet, waiting...`);
await new Promise(resolve => setTimeout(resolve, 2000));
await new Promise(resolve => setTimeout(resolve, BASE_RETRY_DELAY_MS + jitter));
} catch (error) {
clearTimeout(timeoutId);
@@ -256,9 +258,10 @@ async function onboardUser(accessToken, tierID, externalSignal, endpoints, provi
console.warn(`[ProjectId] onboardUser failed after ${MAX_ATTEMPTS} attempts: ${error.message}`);
return null;
}
// Continue to next attempt instead of throwing (which would skip remaining retries)
// Wait with jitter before retrying
const jitter = Math.floor(Math.random() * 5000);
console.warn(`[ProjectId] onboardUser attempt ${attempt} failed: ${error.message}, retrying...`);
await new Promise(resolve => setTimeout(resolve, 2000));
await new Promise(resolve => setTimeout(resolve, BASE_RETRY_DELAY_MS + jitter));
} finally {
clearTimeout(timeoutId);
externalSignal?.removeEventListener("abort", forwardAbort);

44
src/sse/services/backgroundTokenRefresh.js
View File

@@ -9,6 +9,7 @@ import { getCredentialExpiryMs } from "open-sse/services/oauthCredentialManager.
export const BACKGROUND_REFRESH_LEAD_MS = 30 * 60 * 1000;
const DEFAULT_INTERVAL_MS = 5 * 60 * 1000;
const INITIAL_DELAY_MS = 10 * 1000;
const SENSITIVE_PROVIDERS = new Set(["antigravity", "gemini-cli"]);
let started = false;
let intervalHandle = null;
@@ -92,25 +93,42 @@ export async function runBackgroundTokenRefreshTick(deps = {}) {
try {
const load = deps.loadConnections || loadActiveConnections;
const refresh = deps.refreshConnection || refreshOne;
const sleep = deps.sleep || ((ms) => new Promise((res) => setTimeout(res, ms)));
const connections = await load();
const due = selectConnectionsNeedingRefresh(connections, Date.now());
if (due.length === 0) return;
await Promise.allSettled(
due.map(async (conn) => {
try {
await refresh(conn);
} catch (err) {
log.warn("BG_TOKEN_REFRESH", "Connection refresh failed (swallowed)", {
id: conn?.id,
provider: conn?.provider,
error: err?.message ?? String(err),
});
}
})
);
const baseSensitiveDelay = Number(process.env.BG_REFRESH_GOOGLE_DELAY_MS) || 12_000;
const baseNormalDelay = Number(process.env.BG_REFRESH_DELAY_MS) || 1_500;
for (let i = 0; i < due.length; i++) {
const conn = due[i];
try {
await refresh(conn);
log.info("BG_TOKEN_REFRESH", "Connection refresh finished", {
id: conn.id,
email: conn.email || conn.name || conn.id,
provider: conn.provider,
});
} catch (err) {
log.warn("BG_TOKEN_REFRESH", "Connection refresh failed (swallowed)", {
id: conn?.id,
email: conn?.email || conn?.name || conn?.id,
provider: conn?.provider,
error: err?.message ?? String(err),
});
}
// Sequential delay between accounts to prevent bursting upstream providers (especially Google Cloud)
if (i < due.length - 1) {
const isSensitive = SENSITIVE_PROVIDERS.has(conn.provider);
const baseDelay = isSensitive ? baseSensitiveDelay : baseNormalDelay;
const jitter = isSensitive ? Math.floor(Math.random() * 4000) : 200;
await sleep(baseDelay + jitter);
}
}
} catch (err) {
log.warn("BG_TOKEN_REFRESH", "Tick failed (swallowed)", {
error: err?.message ?? String(err),

31
src/sse/services/tokenRefresh.js
View File

@@ -124,25 +124,30 @@ function needsProjectId(provider) {
function _refreshProjectId(provider, connectionId, accessToken) {
if (!needsProjectId(provider) || !connectionId || !accessToken) return;
// Evict the stale cached entry so getProjectIdForConnection does a real fetch
// Invalidate the stale cached entry so getProjectIdForConnection does a real fetch
invalidateProjectId(connectionId);
getProjectIdForConnection(connectionId, accessToken)
.then((projectId) => {
if (!projectId) return;
updateProviderCredentials(connectionId, { projectId }).catch((err) => {
log.debug("TOKEN_REFRESH", "Failed to persist refreshed projectId", {
// Lazy resolution: Do not eagerly trigger onboardUser during background token refresh.
// Eagerly fetching projectId across multiple accounts simultaneously triggers Google Cloud anti-abuse / rate limits.
// Runtime handlers (e.g. chat handler) will lazily call getProjectIdForConnection() on demand.
if (process.env.EAGER_PROJECT_ID_REFRESH === "true") {
getProjectIdForConnection(connectionId, accessToken, provider)
.then((projectId) => {
if (!projectId) return;
updateProviderCredentials(connectionId, { projectId }).catch((err) => {
log.debug("TOKEN_REFRESH", "Failed to persist refreshed projectId", {
connectionId,
error: err?.message ?? err,
});
});
})
.catch((err) => {
log.debug("TOKEN_REFRESH", "Failed to fetch projectId after token refresh", {
connectionId,
error: err?.message ?? err,
});
});
})
.catch((err) => {
log.debug("TOKEN_REFRESH", "Failed to fetch projectId after token refresh", {
connectionId,
error: err?.message ?? err,
});
});
}
}
// ─── Local-specific: persist credentials to localDb ──────────────────────────