feat: Ollama Cloud quota tracker + proactive background OAuth refresh

Ollama: replace informational stub with real quota tracker hitting ollama.com/api/usage (session 5h + weekly 7d, 0..1 ratio) and /api/me plan label; bind handler to apiKey + add features.usageApikey so apikey connections work.

Token refresh: add backgroundTokenRefresh scheduler that refreshes OAuth connections within max(provider lead, 30min) of expiry, independent of inbound traffic (10s after boot, then every 5min, unref'd timers, DISABLE_BACKGROUND_TOKEN_REFRESH kill-switch, fail-open per tick/connection). Registered from custom-server.js (listening) and initializeApp.js. checkAndRefreshToken gains opt-in {force} for the scheduler; request path unchanged.
This commit is contained in:
B1nh M1nh
2026-08-02 09:34:27 +07:00
parent 6fcd27337a
commit f260a1817b
10 changed files with 710 additions and 18 deletions

View File

@@ -522,6 +522,22 @@ export function parseQuotaData(provider, data) {
}
break;
case "ollama":
// Session (5h) / Weekly (7d) usage % from ollama.com/api/usage.
// remainingPercentage only — no absolute remaining (UI treats remaining as %).
if (data.quotas) {
Object.entries(data.quotas).forEach(([name, quota]) => {
normalizedQuotas.push({
name,
used: quota.used || 0,
total: quota.total || 0,
resetAt: quota.resetAt || null,
remainingPercentage: quota.remainingPercentage,
});
});
}
break;
default:
// Generic fallback for unknown providers
if (data.quotas) {

View File

@@ -112,6 +112,12 @@ async function runHeavyStartup() {
.then(({ startQuotaAutoPing }) => startQuotaAutoPing())
.catch((e) => console.log("[AutoPing] scheduler start failed:", e.message));
}
// Proactive OAuth token refresh (e.g. grok-cli ~6h TTL). Module is idempotent
// and also started from custom-server.js when that entry is used.
import("@/sse/services/backgroundTokenRefresh.js")
.then(({ startBackgroundTokenRefresh }) => startBackgroundTokenRefresh())
.catch((e) => console.log("[BackgroundTokenRefresh] scheduler start failed:", e.message));
}
function hasQuotaAutoPingEnabled(settings) {

View File

@@ -0,0 +1,195 @@
// Background proactive OAuth token refresh — independent of inbound requests.
// Fail-open everywhere: tick errors and per-connection failures never kill the interval.
import * as log from "../utils/logger.js";
import { getRefreshLeadMs } from "open-sse/services/tokenRefresh.js";
import { getCredentialExpiryMs } from "open-sse/services/oauthCredentialManager.js";
/** Refresh when expiry is within 30 minutes (or the provider on-request lead, whichever larger). */
export const BACKGROUND_REFRESH_LEAD_MS = 30 * 60 * 1000;
const DEFAULT_INTERVAL_MS = 5 * 60 * 1000;
const INITIAL_DELAY_MS = 10 * 1000;
let started = false;
let intervalHandle = null;
let initialTimeoutHandle = null;
let tickRunning = false;
function isTruthyEnv(value) {
if (value == null || value === "") return false;
const v = String(value).trim().toLowerCase();
return v === "1" || v === "true" || v === "yes" || v === "on";
}
function isNonServerRuntime() {
if (typeof window !== "undefined") return true;
const phase = process.env.NEXT_PHASE || "";
if (
phase === "phase-production-build" ||
phase === "phase-export" ||
phase === "phase-static"
) {
return true;
}
// Next.js build / static generation markers
if (process.env.NEXT_RUNTIME === "edge") return true;
return false;
}
/**
* Pure selection: OAuth connections with a refreshToken whose access token
* expires within max(provider on-request lead, BACKGROUND_REFRESH_LEAD_MS).
*
* @param {Array<object>} connections
* @param {number} [nowMs]
* @returns {Array<object>}
*/
export function selectConnectionsNeedingRefresh(connections, nowMs = Date.now()) {
if (!Array.isArray(connections) || connections.length === 0) return [];
const out = [];
for (const conn of connections) {
if (!conn) continue;
const authType = String(conn.authType || "").toLowerCase().replace(/_/g, "");
if (authType !== "oauth") continue;
if (!conn.refreshToken) continue;
const expiresAtMs = getCredentialExpiryMs(conn);
if (expiresAtMs === null) continue;
const providerLead = getRefreshLeadMs(conn.provider);
const leadMs = Math.max(
Number.isFinite(providerLead) ? providerLead : 0,
BACKGROUND_REFRESH_LEAD_MS
);
if (expiresAtMs - nowMs < leadMs) {
out.push(conn);
}
}
return out;
}
async function loadActiveConnections() {
// Dynamic import avoids circular load with db / app graph at module eval time.
const { getProviderConnections } = await import("../../lib/db/repos/connectionsRepo.js");
return getProviderConnections({ isActive: true });
}
async function refreshOne(connection) {
const { checkAndRefreshToken } = await import("./tokenRefresh.js");
return checkAndRefreshToken(connection.provider, connection, { force: true });
}
/**
* One scheduler tick. Fail-open at top level and per connection.
* @param {{ loadConnections?: Function, refreshConnection?: Function }} [deps]
*/
export async function runBackgroundTokenRefreshTick(deps = {}) {
if (tickRunning) {
log.debug("BG_TOKEN_REFRESH", "Tick already running, skip");
return;
}
tickRunning = true;
try {
const load = deps.loadConnections || loadActiveConnections;
const refresh = deps.refreshConnection || refreshOne;
const connections = await load();
const due = selectConnectionsNeedingRefresh(connections, Date.now());
if (due.length === 0) {
log.debug("BG_TOKEN_REFRESH", "No connections due for refresh", {
active: Array.isArray(connections) ? connections.length : 0,
});
return;
}
log.info("BG_TOKEN_REFRESH", "Refreshing due OAuth connections", {
due: due.length,
ids: due.map((c) => c.id).filter(Boolean),
});
await Promise.allSettled(
due.map(async (conn) => {
try {
await refresh(conn);
log.info("BG_TOKEN_REFRESH", "Connection refresh finished", {
id: conn.id,
provider: conn.provider,
});
} catch (err) {
log.warn("BG_TOKEN_REFRESH", "Connection refresh failed (swallowed)", {
id: conn?.id,
provider: conn?.provider,
error: err?.message ?? String(err),
});
}
})
);
} catch (err) {
log.warn("BG_TOKEN_REFRESH", "Tick failed (swallowed)", {
error: err?.message ?? String(err),
});
} finally {
tickRunning = false;
}
}
/**
* Start the background interval. Safe to call multiple times (no-op if already started).
* @param {{ intervalMs?: number }} [opts]
* @returns {boolean} true if started this call
*/
export function startBackgroundTokenRefresh({ intervalMs } = {}) {
if (started) return false;
if (isTruthyEnv(process.env.DISABLE_BACKGROUND_TOKEN_REFRESH)) {
log.info("BG_TOKEN_REFRESH", "Disabled via DISABLE_BACKGROUND_TOKEN_REFRESH");
return false;
}
if (isNonServerRuntime()) {
log.debug("BG_TOKEN_REFRESH", "Skip start outside long-running server runtime");
return false;
}
started = true;
const period = Number.isFinite(intervalMs) && intervalMs > 0 ? intervalMs : DEFAULT_INTERVAL_MS;
const safeTick = () => {
runBackgroundTokenRefreshTick().catch((err) => {
log.warn("BG_TOKEN_REFRESH", "Unhandled tick rejection (swallowed)", {
error: err?.message ?? String(err),
});
});
};
// First pass soon after boot so idle connections don't wait a full interval.
initialTimeoutHandle = setTimeout(safeTick, INITIAL_DELAY_MS);
if (initialTimeoutHandle.unref) initialTimeoutHandle.unref();
intervalHandle = setInterval(safeTick, period);
if (intervalHandle.unref) intervalHandle.unref();
log.info("BG_TOKEN_REFRESH", "Scheduler started", {
intervalMs: period,
initialDelayMs: INITIAL_DELAY_MS,
leadMs: BACKGROUND_REFRESH_LEAD_MS,
});
return true;
}
export function stopBackgroundTokenRefresh() {
if (initialTimeoutHandle) {
clearTimeout(initialTimeoutHandle);
initialTimeoutHandle = null;
}
if (intervalHandle) {
clearInterval(intervalHandle);
intervalHandle = null;
}
if (started) {
started = false;
log.info("BG_TOKEN_REFRESH", "Scheduler stopped");
}
}

View File

@@ -216,16 +216,20 @@ export async function updateProviderCredentials(connectionId, newCredentials) {
*
* @param {string} provider
* @param {object} credentials
* @param {{ force?: boolean }} [options] force=true skips the on-request lead check
* (used by background scheduler which applies a larger lead). Request path omits this.
* @returns {Promise<object>} updated credentials object
*/
export async function checkAndRefreshToken(provider, credentials) {
export async function checkAndRefreshToken(provider, credentials, options = {}) {
let creds = { ...credentials };
if (!creds.connectionId && creds.id) {
creds.connectionId = creds.id;
}
const force = options?.force === true;
// ── 1. Regular access-token expiry ────────────────────────────────────────
if (_shouldRefreshCredentials(provider, creds)) {
if (force || _shouldRefreshCredentials(provider, creds)) {
const expiresAt = creds.expiresAt ? new Date(creds.expiresAt).getTime() : null;
const remaining = expiresAt ? expiresAt - Date.now() : null;
const refreshLead = _getRefreshLeadMs(provider);