feat: Ollama Cloud quota tracker + proactive background OAuth refresh
Ollama: replace informational stub with real quota tracker hitting ollama.com/api/usage (session 5h + weekly 7d, 0..1 ratio) and /api/me plan label; bind handler to apiKey + add features.usageApikey so apikey connections work.
Token refresh: add backgroundTokenRefresh scheduler that refreshes OAuth connections within max(provider lead, 30min) of expiry, independent of inbound traffic (10s after boot, then every 5min, unref'd timers, DISABLE_BACKGROUND_TOKEN_REFRESH kill-switch, fail-open per tick/connection). Registered from custom-server.js (listening) and initializeApp.js. checkAndRefreshToken gains opt-in {force} for the scheduler; request path unchanged.
This commit is contained in:
@@ -522,6 +522,22 @@ export function parseQuotaData(provider, data) {
|
||||
}
|
||||
break;
|
||||
|
||||
case "ollama":
|
||||
// Session (5h) / Weekly (7d) usage % from ollama.com/api/usage.
|
||||
// remainingPercentage only — no absolute remaining (UI treats remaining as %).
|
||||
if (data.quotas) {
|
||||
Object.entries(data.quotas).forEach(([name, quota]) => {
|
||||
normalizedQuotas.push({
|
||||
name,
|
||||
used: quota.used || 0,
|
||||
total: quota.total || 0,
|
||||
resetAt: quota.resetAt || null,
|
||||
remainingPercentage: quota.remainingPercentage,
|
||||
});
|
||||
});
|
||||
}
|
||||
break;
|
||||
|
||||
default:
|
||||
// Generic fallback for unknown providers
|
||||
if (data.quotas) {
|
||||
|
||||
@@ -112,6 +112,12 @@ async function runHeavyStartup() {
|
||||
.then(({ startQuotaAutoPing }) => startQuotaAutoPing())
|
||||
.catch((e) => console.log("[AutoPing] scheduler start failed:", e.message));
|
||||
}
|
||||
|
||||
// Proactive OAuth token refresh (e.g. grok-cli ~6h TTL). Module is idempotent
|
||||
// and also started from custom-server.js when that entry is used.
|
||||
import("@/sse/services/backgroundTokenRefresh.js")
|
||||
.then(({ startBackgroundTokenRefresh }) => startBackgroundTokenRefresh())
|
||||
.catch((e) => console.log("[BackgroundTokenRefresh] scheduler start failed:", e.message));
|
||||
}
|
||||
|
||||
function hasQuotaAutoPingEnabled(settings) {
|
||||
|
||||
195
src/sse/services/backgroundTokenRefresh.js
Normal file
195
src/sse/services/backgroundTokenRefresh.js
Normal file
@@ -0,0 +1,195 @@
|
||||
// Background proactive OAuth token refresh — independent of inbound requests.
|
||||
// Fail-open everywhere: tick errors and per-connection failures never kill the interval.
|
||||
|
||||
import * as log from "../utils/logger.js";
|
||||
import { getRefreshLeadMs } from "open-sse/services/tokenRefresh.js";
|
||||
import { getCredentialExpiryMs } from "open-sse/services/oauthCredentialManager.js";
|
||||
|
||||
/** Refresh when expiry is within 30 minutes (or the provider on-request lead, whichever larger). */
|
||||
export const BACKGROUND_REFRESH_LEAD_MS = 30 * 60 * 1000;
|
||||
const DEFAULT_INTERVAL_MS = 5 * 60 * 1000;
|
||||
const INITIAL_DELAY_MS = 10 * 1000;
|
||||
|
||||
let started = false;
|
||||
let intervalHandle = null;
|
||||
let initialTimeoutHandle = null;
|
||||
let tickRunning = false;
|
||||
|
||||
function isTruthyEnv(value) {
|
||||
if (value == null || value === "") return false;
|
||||
const v = String(value).trim().toLowerCase();
|
||||
return v === "1" || v === "true" || v === "yes" || v === "on";
|
||||
}
|
||||
|
||||
function isNonServerRuntime() {
|
||||
if (typeof window !== "undefined") return true;
|
||||
const phase = process.env.NEXT_PHASE || "";
|
||||
if (
|
||||
phase === "phase-production-build" ||
|
||||
phase === "phase-export" ||
|
||||
phase === "phase-static"
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
// Next.js build / static generation markers
|
||||
if (process.env.NEXT_RUNTIME === "edge") return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure selection: OAuth connections with a refreshToken whose access token
|
||||
* expires within max(provider on-request lead, BACKGROUND_REFRESH_LEAD_MS).
|
||||
*
|
||||
* @param {Array<object>} connections
|
||||
* @param {number} [nowMs]
|
||||
* @returns {Array<object>}
|
||||
*/
|
||||
export function selectConnectionsNeedingRefresh(connections, nowMs = Date.now()) {
|
||||
if (!Array.isArray(connections) || connections.length === 0) return [];
|
||||
|
||||
const out = [];
|
||||
for (const conn of connections) {
|
||||
if (!conn) continue;
|
||||
|
||||
const authType = String(conn.authType || "").toLowerCase().replace(/_/g, "");
|
||||
if (authType !== "oauth") continue;
|
||||
if (!conn.refreshToken) continue;
|
||||
|
||||
const expiresAtMs = getCredentialExpiryMs(conn);
|
||||
if (expiresAtMs === null) continue;
|
||||
|
||||
const providerLead = getRefreshLeadMs(conn.provider);
|
||||
const leadMs = Math.max(
|
||||
Number.isFinite(providerLead) ? providerLead : 0,
|
||||
BACKGROUND_REFRESH_LEAD_MS
|
||||
);
|
||||
|
||||
if (expiresAtMs - nowMs < leadMs) {
|
||||
out.push(conn);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function loadActiveConnections() {
|
||||
// Dynamic import avoids circular load with db / app graph at module eval time.
|
||||
const { getProviderConnections } = await import("../../lib/db/repos/connectionsRepo.js");
|
||||
return getProviderConnections({ isActive: true });
|
||||
}
|
||||
|
||||
async function refreshOne(connection) {
|
||||
const { checkAndRefreshToken } = await import("./tokenRefresh.js");
|
||||
return checkAndRefreshToken(connection.provider, connection, { force: true });
|
||||
}
|
||||
|
||||
/**
|
||||
* One scheduler tick. Fail-open at top level and per connection.
|
||||
* @param {{ loadConnections?: Function, refreshConnection?: Function }} [deps]
|
||||
*/
|
||||
export async function runBackgroundTokenRefreshTick(deps = {}) {
|
||||
if (tickRunning) {
|
||||
log.debug("BG_TOKEN_REFRESH", "Tick already running, skip");
|
||||
return;
|
||||
}
|
||||
tickRunning = true;
|
||||
try {
|
||||
const load = deps.loadConnections || loadActiveConnections;
|
||||
const refresh = deps.refreshConnection || refreshOne;
|
||||
|
||||
const connections = await load();
|
||||
const due = selectConnectionsNeedingRefresh(connections, Date.now());
|
||||
|
||||
if (due.length === 0) {
|
||||
log.debug("BG_TOKEN_REFRESH", "No connections due for refresh", {
|
||||
active: Array.isArray(connections) ? connections.length : 0,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
log.info("BG_TOKEN_REFRESH", "Refreshing due OAuth connections", {
|
||||
due: due.length,
|
||||
ids: due.map((c) => c.id).filter(Boolean),
|
||||
});
|
||||
|
||||
await Promise.allSettled(
|
||||
due.map(async (conn) => {
|
||||
try {
|
||||
await refresh(conn);
|
||||
log.info("BG_TOKEN_REFRESH", "Connection refresh finished", {
|
||||
id: conn.id,
|
||||
provider: conn.provider,
|
||||
});
|
||||
} catch (err) {
|
||||
log.warn("BG_TOKEN_REFRESH", "Connection refresh failed (swallowed)", {
|
||||
id: conn?.id,
|
||||
provider: conn?.provider,
|
||||
error: err?.message ?? String(err),
|
||||
});
|
||||
}
|
||||
})
|
||||
);
|
||||
} catch (err) {
|
||||
log.warn("BG_TOKEN_REFRESH", "Tick failed (swallowed)", {
|
||||
error: err?.message ?? String(err),
|
||||
});
|
||||
} finally {
|
||||
tickRunning = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the background interval. Safe to call multiple times (no-op if already started).
|
||||
* @param {{ intervalMs?: number }} [opts]
|
||||
* @returns {boolean} true if started this call
|
||||
*/
|
||||
export function startBackgroundTokenRefresh({ intervalMs } = {}) {
|
||||
if (started) return false;
|
||||
if (isTruthyEnv(process.env.DISABLE_BACKGROUND_TOKEN_REFRESH)) {
|
||||
log.info("BG_TOKEN_REFRESH", "Disabled via DISABLE_BACKGROUND_TOKEN_REFRESH");
|
||||
return false;
|
||||
}
|
||||
if (isNonServerRuntime()) {
|
||||
log.debug("BG_TOKEN_REFRESH", "Skip start outside long-running server runtime");
|
||||
return false;
|
||||
}
|
||||
|
||||
started = true;
|
||||
const period = Number.isFinite(intervalMs) && intervalMs > 0 ? intervalMs : DEFAULT_INTERVAL_MS;
|
||||
|
||||
const safeTick = () => {
|
||||
runBackgroundTokenRefreshTick().catch((err) => {
|
||||
log.warn("BG_TOKEN_REFRESH", "Unhandled tick rejection (swallowed)", {
|
||||
error: err?.message ?? String(err),
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
// First pass soon after boot so idle connections don't wait a full interval.
|
||||
initialTimeoutHandle = setTimeout(safeTick, INITIAL_DELAY_MS);
|
||||
if (initialTimeoutHandle.unref) initialTimeoutHandle.unref();
|
||||
|
||||
intervalHandle = setInterval(safeTick, period);
|
||||
if (intervalHandle.unref) intervalHandle.unref();
|
||||
|
||||
log.info("BG_TOKEN_REFRESH", "Scheduler started", {
|
||||
intervalMs: period,
|
||||
initialDelayMs: INITIAL_DELAY_MS,
|
||||
leadMs: BACKGROUND_REFRESH_LEAD_MS,
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
export function stopBackgroundTokenRefresh() {
|
||||
if (initialTimeoutHandle) {
|
||||
clearTimeout(initialTimeoutHandle);
|
||||
initialTimeoutHandle = null;
|
||||
}
|
||||
if (intervalHandle) {
|
||||
clearInterval(intervalHandle);
|
||||
intervalHandle = null;
|
||||
}
|
||||
if (started) {
|
||||
started = false;
|
||||
log.info("BG_TOKEN_REFRESH", "Scheduler stopped");
|
||||
}
|
||||
}
|
||||
@@ -216,16 +216,20 @@ export async function updateProviderCredentials(connectionId, newCredentials) {
|
||||
*
|
||||
* @param {string} provider
|
||||
* @param {object} credentials
|
||||
* @param {{ force?: boolean }} [options] force=true skips the on-request lead check
|
||||
* (used by background scheduler which applies a larger lead). Request path omits this.
|
||||
* @returns {Promise<object>} updated credentials object
|
||||
*/
|
||||
export async function checkAndRefreshToken(provider, credentials) {
|
||||
export async function checkAndRefreshToken(provider, credentials, options = {}) {
|
||||
let creds = { ...credentials };
|
||||
if (!creds.connectionId && creds.id) {
|
||||
creds.connectionId = creds.id;
|
||||
}
|
||||
|
||||
const force = options?.force === true;
|
||||
|
||||
// ── 1. Regular access-token expiry ────────────────────────────────────────
|
||||
if (_shouldRefreshCredentials(provider, creds)) {
|
||||
if (force || _shouldRefreshCredentials(provider, creds)) {
|
||||
const expiresAt = creds.expiresAt ? new Date(creds.expiresAt).getTime() : null;
|
||||
const remaining = expiresAt ? expiresAt - Date.now() : null;
|
||||
const refreshLead = _getRefreshLeadMs(provider);
|
||||
|
||||
Reference in New Issue
Block a user