feat(models): refresh model capabilities from models.dev in the background

Capability tables are hand-maintained, so a model gains vision or a wider
context only when someone notices and edits the file. This adds a daily
sync that fills the gap for models already in the registry.

How it decides:

- Modalities (vision/pdf/audio/video) belong to the MODEL — every gateway
  serving glm-5.3-flash serves the same weights — so they are keyed by
  model id and shared. A majority of sources must declare one, which keeps
  out lone mis-declarations: minimax-m2.5 (1 of 45), glm-4.7 (1 of 44) and
  gpt-oss-120b (2 of 76) are text-only despite a reseller claiming vision.
- Context/output limits belong to the GATEWAY — each truncates differently
  (glm-5 ships as 202752/16384 on one host and 204800/131072 on another) —
  so they are keyed by provider + model and only the matching provider's
  own numbers are trusted.

Both layers are strictly additive and sit BELOW the hand-written tables,
which short-circuit first. A capability already true stays true.

Mechanics: worker thread (the 4MB parse would block the loop ~20ms),
ETag so an unchanged catalog costs one empty request, 60s startup delay,
30min backoff on failure, MODEL_CATALOG_SYNC=off to disable. Only the
~57KB delta is kept; lookups cost ~0.1us via an mtime-guarded cache.

capabilities.js is bundled into the browser through useModelCaps, so it
cannot import node:fs — the server injects the reader via
setCatalogSource() from instrumentation.

visionPatterns.js is the last resort: a model nobody has catalogued yet
still accepts images when its id says so (qwen3-vl-plus, glm-4.6v, llava),
with image-generation and embedding ids excluded.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

open-sse/providers/capabilities.js

@@ -6,6 +6,16 @@
 //   3. PATTERN_CAPABILITIES                     — glob match, ordered specific -> generic
 //   4. DEFAULT_CAPABILITIES                     — safe floor (always returned)
 //
+// Two extra layers then refine the result, and neither can override the hand
+// written tables above (steps 1-2 short-circuit before they are consulted):
+//   • the synced catalog — modalities keyed by model, limits keyed by provider
+//     + model, refreshed from models.dev in the background. It reads a file, so
+//     the server installs it via setCatalogSource(); this module stays free of
+//     node:fs because the dashboard bundles it into the browser too.
+//   • visionPatterns.js — name-based vision detection, last resort so a model
+//     nobody has catalogued yet still accepts images.
+// Both only ever turn a capability ON.
+//
 // ── HOW TO ADD / UPDATE A MODEL ──────────────────────────────────────
 // Authoritative data source: https://models.dev/api.json (145 providers, 4000+
 // models, MIT). Each model exposes the exact fields we map below:
@@ -23,6 +33,7 @@
 // 2.0+, Grok, Perplexity). Verify with: curl -s https://models.dev/api.json
 
 import { matchPattern } from "./pricing.js";
+import { looksLikeVisionModel } from "./visionPatterns.js";
 
 /**
  * Safe floor — every resolved result is merged over this so consumers
@@ -94,8 +105,8 @@ export const MODEL_CAPABILITIES = {
   // Gemini image-gen / OpenAI image / xai image variants
   "gpt-image-1":       { imageOutput: true, tools: false },
 
-  // GLM vision variants (text GLM has no vision) — 5.3-Flash is natively
+  // GLM vision variants (text GLM has no vision) — 5.3-Flash and 5V-Turbo are
-  // multimodal per z.ai and carries the full 1M window.
+  // natively multimodal per z.ai, and 5.3-Flash carries the full 1M window.
   "glm-5.3-flash":     { vision: true, videoInput: true, pdf: true, reasoning: true, thinkingFormat: "zai", contextWindow: 1000000, maxOutput: 131072 },
   "glm-4.6v":          { vision: true, videoInput: true, reasoning: true, thinkingFormat: "zai", contextWindow: 128000, maxOutput: 32768 },
   "glm-4.5v":          { vision: true, videoInput: true, reasoning: true, thinkingFormat: "zai", contextWindow: 64000, maxOutput: 16384 },
@@ -333,6 +344,46 @@ export const PATTERN_CAPABILITIES = [
  * @param {string} model
  * @returns {object} full capabilities object
  */
+const MODALITY_KEYS = ["vision", "pdf", "audioInput", "videoInput"];
+
+// Catalog lookups, installed by the server at startup. Left as no-ops in the
+// browser bundle, where there is no file to read.
+let catalogSource = null;
+
+/**
+ * Install the synced catalog reader (server only).
+ * @param {{ getModalities: Function, getLimits: Function } | null} source
+ */
+export function setCatalogSource(source) {
+  catalogSource = source;
+}
+
+// Apply the synced catalog + name heuristic on top of a table-resolved result.
+// Strictly additive: a capability already true stays true, and a false one only
+// flips when an outside source positively declares support.
+function refine(base, provider, model) {
+  const result = { ...DEFAULT_CAPABILITIES, ...base };
+
+  if (catalogSource) {
+    const modalities = catalogSource.getModalities(model);
+    if (modalities) {
+      for (const key of MODALITY_KEYS) {
+        if (modalities[key] === true) result[key] = true;
+      }
+    }
+
+    const limits = catalogSource.getLimits(provider, model);
+    if (limits) {
+      if (limits.contextWindow > 0) result.contextWindow = limits.contextWindow;
+      if (limits.maxOutput > 0) result.maxOutput = limits.maxOutput;
+    }
+  }
+
+  if (!result.vision && looksLikeVisionModel(model)) result.vision = true;
+
+  return result;
+}
+
 export function getCapabilitiesForModel(provider, model) {
   if (!model) return { ...DEFAULT_CAPABILITIES };
 
@@ -350,13 +401,13 @@ export function getCapabilitiesForModel(provider, model) {
   if (MODEL_CAPABILITIES[baseModel]) return { ...DEFAULT_CAPABILITIES, ...MODEL_CAPABILITIES[baseModel] };
   if (MODEL_CAPABILITIES[model]) return { ...DEFAULT_CAPABILITIES, ...MODEL_CAPABILITIES[model] };
 
-  // 3. Pattern match (first match wins)
+  // 3. Pattern match (first match wins), refined by catalog + name heuristic
   for (const { pattern, caps } of PATTERN_CAPABILITIES) {
     if (matchPattern(pattern, baseModel) || matchPattern(pattern, model)) {
-      return { ...DEFAULT_CAPABILITIES, ...caps };
+      return refine(caps, provider, model);
     }
   }
 
   // 4. Floor
-  return { ...DEFAULT_CAPABILITIES };
+  return refine(null, provider, model);
 }

open-sse/providers/catalogOverride.js

@@ -0,0 +1,72 @@
+// Read side of the model catalog synced from models.dev.
+//
+// The file is the source of truth; the only thing held in memory is a parsed
+// copy dropped as soon as the file's mtime changes. getCapabilitiesForModel is
+// synchronous and runs per request, so the hot path is one stat (~1us) and the
+// parse (~0.1ms on a ~18KB file) only reruns after a sync.
+
+import fs from "node:fs";
+import path from "node:path";
+import { DATA_DIR } from "@/lib/dataDir.js";
+
+export const CATALOG_FILE = path.join(DATA_DIR, "model-catalog.json");
+// Trimmed upstream catalog, read by the add-models skill (not by the router).
+export const CATALOG_RAW_FILE = path.join(DATA_DIR, "model-catalog-raw.json");
+
+const EMPTY = { models: {}, providers: {} };
+let cache = EMPTY;
+let cachedMtime = -1;
+
+// "zai-org/GLM-4.6V:free" -> "glm-4.6v"
+function baseId(model) {
+  if (!model) return "";
+  const withoutVendor = model.includes("/") ? model.split("/").pop() : model;
+  return withoutVendor.toLowerCase().split(":")[0];
+}
+
+function load() {
+  let mtime;
+  try {
+    mtime = fs.statSync(CATALOG_FILE).mtimeMs;
+  } catch {
+    cache = EMPTY;
+    cachedMtime = -1;
+    return cache;
+  }
+  if (mtime === cachedMtime) return cache;
+
+  cachedMtime = mtime;
+  try {
+    const parsed = JSON.parse(fs.readFileSync(CATALOG_FILE, "utf8"));
+    cache = { models: parsed?.models || {}, providers: parsed?.providers || {} };
+  } catch {
+    cache = EMPTY;
+  }
+  return cache;
+}
+
+// Modality is a property of the model itself — any gateway serving it inherits
+// the same image/video/pdf support, so this is keyed by model id alone.
+export function getCatalogModalities(model) {
+  return load().models[baseId(model)] || null;
+}
+
+// Context and output limits are a property of the gateway, not the model: each
+// one truncates differently, so these stay keyed by provider + model.
+export function getCatalogLimits(provider, model) {
+  const byProvider = provider && load().providers[provider];
+  if (!byProvider) return null;
+  return byProvider[model] || byProvider[baseId(model)] || null;
+}
+
+// Force a re-read on the next lookup (called right after a sync writes the file).
+export function invalidateCatalog() {
+  cachedMtime = -1;
+}
+
+// Hand the reader to capabilities.js. That module is bundled into the browser
+// too, so it cannot import this file directly — the server pushes it in.
+export async function installCatalogSource() {
+  const { setCatalogSource } = await import("./capabilities.js");
+  setCatalogSource({ getModalities: getCatalogModalities, getLimits: getCatalogLimits });
+}

open-sse/providers/visionPatterns.js

@@ -0,0 +1,42 @@
+// Name-based vision detection — last resort when neither the catalog file nor
+// the capability tables know a model. Vendors put the modality in the id
+// ("qwen3-vl-plus", "glm-4.6v", "deepseek-v4-flash-vision-exp"), so a custom or
+// freshly released model still gets image input instead of silently dropping it.
+//
+// Only ever turns vision ON. Never used to turn a declared capability off.
+
+const SEP = "[-_/:.]";
+
+// Image GENERATION, video generation, and non-chat models also carry these
+// words but take no image input — checked first so they can never match.
+const NOT_VISION = new RegExp(
+  [
+    `(^|${SEP})(image|img)(${SEP}|$)`,
+    "stable-image", "gen[0-9]_image", "nanobanana", "imagine",
+    "t2v", "i2v", "flux", "dall", "sdxl", "diffusion",
+    "embed", "rerank", "guard", "moderation",
+    "tts", "stt", "whisper", "voice", "speech", "audio",
+  ].join("|"),
+  "i"
+);
+
+// Explicit modality words, plus the "<digit>v" suffix vendors use for vision
+// variants (glm-4.6v, glm-5v-turbo). The digit-v branch requires a dotted
+// version so the never-shipped `gpt-4v` cannot match.
+const VISION_NAME = new RegExp(
+  [
+    `(^|${SEP})(vision|vl|vlm|multimodal|omni|visual)(${SEP}|$)`,
+    `[0-9]\\.[0-9]+v(${SEP}|$)`,
+    `(^|${SEP})glm-[0-9]+v(${SEP}|$)`,
+    "(^|[-_/:.])(llava|pixtral|internvl|cogvlm|minicpm-v|moondream|idefics|fuyu)",
+  ].join("|"),
+  "i"
+);
+
+// Does this model id look like a vision model? Name signal only.
+export function looksLikeVisionModel(modelId) {
+  if (!modelId) return false;
+  const id = String(modelId).toLowerCase();
+  if (NOT_VISION.test(id)) return false;
+  return VISION_NAME.test(id);
+}

src/app/api/models/catalog-sync/route.js

@@ -0,0 +1,31 @@
+import { NextResponse } from "next/server";
+import fs from "node:fs";
+import { getSyncState, syncModelCatalog } from "@/lib/modelCatalog/sync.js";
+import { CATALOG_FILE } from "open-sse/providers/catalogOverride.js";
+
+// GET /api/models/catalog-sync - Sync status and what the catalog currently holds
+export async function GET() {
+  const state = getSyncState();
+  let catalog = null;
+  try {
+    const parsed = JSON.parse(fs.readFileSync(CATALOG_FILE, "utf8"));
+    catalog = {
+      syncedAt: parsed.syncedAt,
+      models: Object.keys(parsed.models || {}).length,
+      providers: Object.keys(parsed.providers || {}).length,
+      bytes: fs.statSync(CATALOG_FILE).size,
+    };
+  } catch {
+    catalog = null;
+  }
+  return NextResponse.json({ ...state, catalog });
+}
+
+// POST /api/models/catalog-sync - Run a sync now instead of waiting for the timer
+export async function POST() {
+  const result = await syncModelCatalog();
+  if (!result) {
+    return NextResponse.json({ error: getSyncState().lastError || "sync in progress" }, { status: 503 });
+  }
+  return NextResponse.json({ success: true, result });
+}

src/instrumentation.js

@@ -2,5 +2,13 @@ export async function register() {
   if (process.env.NEXT_RUNTIME === "nodejs") {
     const { initConsoleLogCapture } = await import("@/lib/consoleLogBuffer");
     initConsoleLogCapture();
+
+    // Server-only: lets capabilities.js read the synced catalog without pulling
+    // node:fs into the dashboard's browser bundle.
+    const { installCatalogSource } = await import("open-sse/providers/catalogOverride.js");
+    await installCatalogSource();
+
+    const { startModelCatalogSync } = await import("@/lib/modelCatalog/sync.js");
+    startModelCatalogSync();
   }
 }

src/lib/modelCatalog/sync.js

@@ -0,0 +1,140 @@
+// Background refresh of model capabilities from models.dev.
+//
+// Failures are swallowed on purpose: a stale or missing catalog just means the
+// hand-written capability tables keep deciding on their own.
+
+import path from "node:path";
+import { Worker } from "node:worker_threads";
+import { fileURLToPath } from "node:url";
+import { CATALOG_FILE, CATALOG_RAW_FILE, invalidateCatalog, installCatalogSource } from "open-sse/providers/catalogOverride.js";
+
+const CATALOG_URL = "https://models.dev/api.json";
+const WORKER_FILE = path.join(path.dirname(fileURLToPath(import.meta.url)), "worker.js");
+
+// 9router provider id -> models.dev provider id, for context/maxOutput only.
+// Providers absent here keep whatever the local pattern table resolves; the
+// names that already match are resolved automatically.
+const PROVIDER_ALIASES = {
+  "glm": "zai",
+  "glm-cn": "zhipuai",
+  "claude": "anthropic",
+  "gemini": "google",
+  "kimi": "moonshotai",
+  "kimi-cn": "moonshotai-cn",
+  "qwen": "alibaba",
+  "qwen-cn": "alibaba-cn",
+  "zhipu": "zhipuai",
+  "hunyuan": "tencent",
+  "doubao": "volcengine",
+  "cloudflare-ai": "cloudflare-workers-ai",
+};
+
+export const SYNC_INTERVAL_MS = 24 * 60 * 60 * 1000;
+const STARTUP_DELAY_MS = 60 * 1000;   // let the server boot and serve first requests
+const RETRY_DELAY_MS = 30 * 60 * 1000;
+const WORKER_TIMEOUT_MS = 120000;
+
+let state = { running: false, lastSync: null, lastError: null, lastResult: null, etag: null };
+let timer = null;
+
+export function getSyncState() {
+  return { ...state, file: CATALOG_FILE, url: CATALOG_URL, intervalMs: SYNC_INTERVAL_MS };
+}
+
+// Snapshot every registered model with its currently resolved capabilities, so
+// the worker can compute a delta without importing app modules (it cannot
+// resolve the bundler-only "open-sse/*" alias).
+async function collectEntries() {
+  const [{ default: registry }, { getCapabilitiesForModel }] = await Promise.all([
+    import("open-sse/providers/registry/index.js"),
+    import("open-sse/providers/capabilities.js"),
+  ]);
+  await installCatalogSource();
+
+  const entries = [];
+  for (const provider of registry) {
+    for (const model of provider.models || []) {
+      entries.push({
+        provider: provider.id,
+        model: model.id,
+        contextLength: model.contextLength,
+        current: getCapabilitiesForModel(provider.id, model.id),
+      });
+    }
+  }
+  return entries;
+}
+
+function runWorker(entries) {
+  return new Promise((resolve, reject) => {
+    const worker = new Worker(WORKER_FILE, {
+      workerData: {
+        url: CATALOG_URL,
+        etag: state.etag,
+        outFile: CATALOG_FILE,
+        rawFile: CATALOG_RAW_FILE,
+        entries,
+        providerAliases: PROVIDER_ALIASES,
+      },
+      resourceLimits: { maxOldGenerationSizeMb: 512 },
+    });
+
+    let settled = false;
+    const finish = (fn, value) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timeout);
+      fn(value);
+    };
+    const timeout = setTimeout(() => {
+      worker.terminate();
+      finish(reject, new Error("sync timed out"));
+    }, WORKER_TIMEOUT_MS);
+
+    worker.on("message", (msg) => {
+      if (msg?.ok) finish(resolve, msg.result);
+      else finish(reject, new Error(msg?.error || "sync failed"));
+    });
+    worker.on("error", (err) => finish(reject, err));
+    worker.on("exit", (code) => finish(reject, new Error(`worker exited with ${code}`)));
+  });
+}
+
+// Run one sync. Returns the worker summary, or null when it could not complete.
+export async function syncModelCatalog() {
+  if (state.running) return null;
+  state.running = true;
+  try {
+    const result = await runWorker(await collectEntries());
+    if (result.status === "updated") {
+      state.etag = result.etag;
+      invalidateCatalog();
+      console.log(`[modelCatalog] ${result.models} models, ${result.providers} providers, ${(result.bytes / 1024).toFixed(1)}KB`);
+    }
+    state.lastSync = Date.now();
+    state.lastError = null;
+    state.lastResult = result;
+    return result;
+  } catch (error) {
+    state.lastError = error?.message || String(error);
+    console.log(`[modelCatalog] sync failed: ${state.lastError}`);
+    return null;
+  } finally {
+    state.running = false;
+  }
+}
+
+// Schedule the recurring sync. Disable entirely with MODEL_CATALOG_SYNC=off.
+export function startModelCatalogSync() {
+  if (timer) return;
+  if (String(process.env.MODEL_CATALOG_SYNC || "").toLowerCase() === "off") return;
+
+  const schedule = (delay) => {
+    timer = setTimeout(async () => {
+      const result = await syncModelCatalog();
+      schedule(result ? SYNC_INTERVAL_MS : RETRY_DELAY_MS);
+    }, delay);
+    timer.unref?.();
+  };
+  schedule(STARTUP_DELAY_MS);
+}

src/lib/modelCatalog/worker.js

@@ -0,0 +1,132 @@
+// Downloads models.dev and writes the capability deltas 9router reads.
+// Runs in a worker thread: the 4MB parse would otherwise block requests for ~20ms.
+
+import { parentPort, workerData } from "node:worker_threads";
+import fs from "node:fs";
+import path from "node:path";
+
+const FETCH_TIMEOUT_MS = 60000;
+const MODALITY_BY_INPUT = { image: "vision", pdf: "pdf", audio: "audioInput", video: "videoInput" };
+// Gateways disagree about the same model, so a modality needs a majority of
+// them to declare it — one reseller mislabelling a text model must not win.
+const MIN_SHARE = 0.5;
+// Ignore limit differences below this: gateways round 200000 vs 202752.
+const LIMIT_TOLERANCE = 0.1;
+
+// "zai-org/GLM-4.6V:free" -> "glm-4.6v"
+function baseId(modelId) {
+  const withoutVendor = modelId.includes("/") ? modelId.split("/").pop() : modelId;
+  return withoutVendor.toLowerCase().split(":")[0];
+}
+
+function build(catalog, entries, providerAliases) {
+  // Index the catalog once: per provider for limits, and tallied for modalities.
+  const byProvider = {};
+  const tally = {};
+  for (const [providerId, provider] of Object.entries(catalog)) {
+    const models = {};
+    for (const [modelId, model] of Object.entries(provider?.models || {})) {
+      const id = baseId(modelId);
+      models[id] = model;
+      const counts = tally[id] || (tally[id] = { total: 0 });
+      counts.total++;
+      for (const input of model?.modalities?.input || []) {
+        const key = MODALITY_BY_INPUT[input];
+        if (key) counts[key] = (counts[key] || 0) + 1;
+      }
+    }
+    byProvider[providerId] = models;
+  }
+
+  // Modalities belong to the model — every gateway serving it has the same
+  // weights — so they are keyed by model id and shared across providers.
+  const models = {};
+  for (const [id, counts] of Object.entries(tally)) {
+    const declared = {};
+    for (const key of Object.values(MODALITY_BY_INPUT)) {
+      if ((counts[key] || 0) / counts.total >= MIN_SHARE) declared[key] = true;
+    }
+    if (Object.keys(declared).length) models[id] = declared;
+  }
+
+  // Limits belong to the gateway — each truncates differently — so only the
+  // matching provider's own numbers are used, keyed by provider + model.
+  const providers = {};
+  for (const { provider, model, contextLength, current } of entries) {
+    const alias = providerAliases[provider];
+    const upstream = catalog[provider] ? provider : (alias && catalog[alias] ? alias : null);
+    const entry = upstream && byProvider[upstream]?.[baseId(model)];
+    if (!entry) continue;
+
+    const delta = {};
+    const { context, output } = entry.limit || {};
+    if (context > 0 && !contextLength
+      && Math.abs(context - current.contextWindow) / current.contextWindow > LIMIT_TOLERANCE) {
+      delta.contextWindow = context;
+    }
+    if (output > 0
+      && Math.abs(output - current.maxOutput) / current.maxOutput > LIMIT_TOLERANCE) {
+      delta.maxOutput = output;
+    }
+    if (Object.keys(delta).length) (providers[provider] || (providers[provider] = {}))[model] = delta;
+  }
+
+  return { models, providers };
+}
+
+// Trimmed copy of the upstream catalog, kept for the add-models skill: same
+// 7348 models, 470KB instead of 4.3MB, so a scan reads it in ~5ms.
+function slim(catalog) {
+  const out = {};
+  for (const [providerId, provider] of Object.entries(catalog)) {
+    const models = {};
+    for (const [modelId, model] of Object.entries(provider?.models || {})) {
+      models[modelId] = {
+        i: (model?.modalities?.input || []).filter((x) => x !== "text"),
+        c: model?.limit?.context,
+        o: model?.limit?.output,
+        r: model?.reasoning || undefined,
+      };
+    }
+    out[providerId] = models;
+  }
+  return out;
+}
+
+function writeAtomic(file, contents) {
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.writeFileSync(`${file}.tmp`, contents, "utf8");
+  fs.renameSync(`${file}.tmp`, file);
+}
+
+async function run() {
+  const { url, etag, outFile, rawFile, entries, providerAliases } = workerData;
+
+  const headers = { accept: "application/json" };
+  if (etag) headers["if-none-match"] = etag;
+  const response = await fetch(url, { headers, signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) });
+
+  if (response.status === 304) return { status: "unchanged" };
+  if (!response.ok) throw new Error(`HTTP ${response.status}`);
+
+  const catalog = await response.json();
+  const nextEtag = response.headers.get("etag") || null;
+  const { models, providers } = build(catalog, entries, providerAliases);
+  const serialized = JSON.stringify({ v: 1, etag: nextEtag, syncedAt: Date.now(), models, providers });
+
+  writeAtomic(outFile, serialized);
+  if (rawFile) writeAtomic(rawFile, JSON.stringify(slim(catalog)));
+
+  return {
+    status: "updated",
+    etag: nextEtag,
+    bytes: Buffer.byteLength(serialized),
+    models: Object.keys(models).length,
+    providers: Object.keys(providers).length,
+  };
+}
+
+run().then(
+  (result) => parentPort?.postMessage({ ok: true, result }),
+  (error) => parentPort?.postMessage({ ok: false, error: error?.message || String(error) })
+);