fix(antigravity): prevent Google anti-abuse rate limits on multi-account refresh (#3813)
This commit is contained in:
@@ -9,6 +9,7 @@ import { getCredentialExpiryMs } from "open-sse/services/oauthCredentialManager.
|
||||
export const BACKGROUND_REFRESH_LEAD_MS = 30 * 60 * 1000;
|
||||
const DEFAULT_INTERVAL_MS = 5 * 60 * 1000;
|
||||
const INITIAL_DELAY_MS = 10 * 1000;
|
||||
const SENSITIVE_PROVIDERS = new Set(["antigravity", "gemini-cli"]);
|
||||
|
||||
let started = false;
|
||||
let intervalHandle = null;
|
||||
@@ -92,25 +93,42 @@ export async function runBackgroundTokenRefreshTick(deps = {}) {
|
||||
try {
|
||||
const load = deps.loadConnections || loadActiveConnections;
|
||||
const refresh = deps.refreshConnection || refreshOne;
|
||||
const sleep = deps.sleep || ((ms) => new Promise((res) => setTimeout(res, ms)));
|
||||
|
||||
const connections = await load();
|
||||
const due = selectConnectionsNeedingRefresh(connections, Date.now());
|
||||
|
||||
if (due.length === 0) return;
|
||||
|
||||
await Promise.allSettled(
|
||||
due.map(async (conn) => {
|
||||
try {
|
||||
await refresh(conn);
|
||||
} catch (err) {
|
||||
log.warn("BG_TOKEN_REFRESH", "Connection refresh failed (swallowed)", {
|
||||
id: conn?.id,
|
||||
provider: conn?.provider,
|
||||
error: err?.message ?? String(err),
|
||||
});
|
||||
}
|
||||
})
|
||||
);
|
||||
const baseSensitiveDelay = Number(process.env.BG_REFRESH_GOOGLE_DELAY_MS) || 12_000;
|
||||
const baseNormalDelay = Number(process.env.BG_REFRESH_DELAY_MS) || 1_500;
|
||||
|
||||
for (let i = 0; i < due.length; i++) {
|
||||
const conn = due[i];
|
||||
try {
|
||||
await refresh(conn);
|
||||
log.info("BG_TOKEN_REFRESH", "Connection refresh finished", {
|
||||
id: conn.id,
|
||||
email: conn.email || conn.name || conn.id,
|
||||
provider: conn.provider,
|
||||
});
|
||||
} catch (err) {
|
||||
log.warn("BG_TOKEN_REFRESH", "Connection refresh failed (swallowed)", {
|
||||
id: conn?.id,
|
||||
email: conn?.email || conn?.name || conn?.id,
|
||||
provider: conn?.provider,
|
||||
error: err?.message ?? String(err),
|
||||
});
|
||||
}
|
||||
|
||||
// Sequential delay between accounts to prevent bursting upstream providers (especially Google Cloud)
|
||||
if (i < due.length - 1) {
|
||||
const isSensitive = SENSITIVE_PROVIDERS.has(conn.provider);
|
||||
const baseDelay = isSensitive ? baseSensitiveDelay : baseNormalDelay;
|
||||
const jitter = isSensitive ? Math.floor(Math.random() * 4000) : 200;
|
||||
await sleep(baseDelay + jitter);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
log.warn("BG_TOKEN_REFRESH", "Tick failed (swallowed)", {
|
||||
error: err?.message ?? String(err),
|
||||
|
||||
@@ -124,25 +124,30 @@ function needsProjectId(provider) {
|
||||
function _refreshProjectId(provider, connectionId, accessToken) {
|
||||
if (!needsProjectId(provider) || !connectionId || !accessToken) return;
|
||||
|
||||
// Evict the stale cached entry so getProjectIdForConnection does a real fetch
|
||||
// Invalidate the stale cached entry so getProjectIdForConnection does a real fetch
|
||||
invalidateProjectId(connectionId);
|
||||
|
||||
getProjectIdForConnection(connectionId, accessToken)
|
||||
.then((projectId) => {
|
||||
if (!projectId) return;
|
||||
updateProviderCredentials(connectionId, { projectId }).catch((err) => {
|
||||
log.debug("TOKEN_REFRESH", "Failed to persist refreshed projectId", {
|
||||
// Lazy resolution: Do not eagerly trigger onboardUser during background token refresh.
|
||||
// Eagerly fetching projectId across multiple accounts simultaneously triggers Google Cloud anti-abuse / rate limits.
|
||||
// Runtime handlers (e.g. chat handler) will lazily call getProjectIdForConnection() on demand.
|
||||
if (process.env.EAGER_PROJECT_ID_REFRESH === "true") {
|
||||
getProjectIdForConnection(connectionId, accessToken, provider)
|
||||
.then((projectId) => {
|
||||
if (!projectId) return;
|
||||
updateProviderCredentials(connectionId, { projectId }).catch((err) => {
|
||||
log.debug("TOKEN_REFRESH", "Failed to persist refreshed projectId", {
|
||||
connectionId,
|
||||
error: err?.message ?? err,
|
||||
});
|
||||
});
|
||||
})
|
||||
.catch((err) => {
|
||||
log.debug("TOKEN_REFRESH", "Failed to fetch projectId after token refresh", {
|
||||
connectionId,
|
||||
error: err?.message ?? err,
|
||||
});
|
||||
});
|
||||
})
|
||||
.catch((err) => {
|
||||
log.debug("TOKEN_REFRESH", "Failed to fetch projectId after token refresh", {
|
||||
connectionId,
|
||||
error: err?.message ?? err,
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Local-specific: persist credentials to localDb ──────────────────────────
|
||||
|
||||
Reference in New Issue
Block a user