From 0532f00d84b131b4e31424cf5045df5dd8b09b44 Mon Sep 17 00:00:00 2001 From: decolua Date: Thu, 27 Aug 2026 17:53:53 +0700 Subject: [PATCH] feat(models): refresh model capabilities from models.dev in the background MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Capability tables are hand-maintained, so a model gains vision or a wider context only when someone notices and edits the file. This adds a daily sync that fills the gap for models already in the registry. How it decides: - Modalities (vision/pdf/audio/video) belong to the MODEL — every gateway serving glm-5.3-flash serves the same weights — so they are keyed by model id and shared. A majority of sources must declare one, which keeps out lone mis-declarations: minimax-m2.5 (1 of 45), glm-4.7 (1 of 44) and gpt-oss-120b (2 of 76) are text-only despite a reseller claiming vision. - Context/output limits belong to the GATEWAY — each truncates differently (glm-5 ships as 202752/16384 on one host and 204800/131072 on another) — so they are keyed by provider + model and only the matching provider's own numbers are trusted. Both layers are strictly additive and sit BELOW the hand-written tables, which short-circuit first. A capability already true stays true. Mechanics: worker thread (the 4MB parse would block the loop ~20ms), ETag so an unchanged catalog costs one empty request, 60s startup delay, 30min backoff on failure, MODEL_CATALOG_SYNC=off to disable. Only the ~57KB delta is kept; lookups cost ~0.1us via an mtime-guarded cache. capabilities.js is bundled into the browser through useModelCaps, so it cannot import node:fs — the server injects the reader via setCatalogSource() from instrumentation. visionPatterns.js is the last resort: a model nobody has catalogued yet still accepts images when its id says so (qwen3-vl-plus, glm-4.6v, llava), with image-generation and embedding ids excluded. Co-Authored-By: Claude Fable 5 --- open-sse/providers/capabilities.js | 61 +++++++++- open-sse/providers/catalogOverride.js | 72 ++++++++++++ open-sse/providers/visionPatterns.js | 42 +++++++ src/app/api/models/catalog-sync/route.js | 31 +++++ src/instrumentation.js | 8 ++ src/lib/modelCatalog/sync.js | 140 +++++++++++++++++++++++ src/lib/modelCatalog/worker.js | 132 +++++++++++++++++++++ 7 files changed, 481 insertions(+), 5 deletions(-) create mode 100644 open-sse/providers/catalogOverride.js create mode 100644 open-sse/providers/visionPatterns.js create mode 100644 src/app/api/models/catalog-sync/route.js create mode 100644 src/lib/modelCatalog/sync.js create mode 100644 src/lib/modelCatalog/worker.js diff --git a/open-sse/providers/capabilities.js b/open-sse/providers/capabilities.js index 2b21afbc..24434a86 100644 --- a/open-sse/providers/capabilities.js +++ b/open-sse/providers/capabilities.js @@ -6,6 +6,16 @@ // 3. PATTERN_CAPABILITIES — glob match, ordered specific -> generic // 4. DEFAULT_CAPABILITIES — safe floor (always returned) // +// Two extra layers then refine the result, and neither can override the hand +// written tables above (steps 1-2 short-circuit before they are consulted): +// • the synced catalog — modalities keyed by model, limits keyed by provider +// + model, refreshed from models.dev in the background. It reads a file, so +// the server installs it via setCatalogSource(); this module stays free of +// node:fs because the dashboard bundles it into the browser too. +// • visionPatterns.js — name-based vision detection, last resort so a model +// nobody has catalogued yet still accepts images. +// Both only ever turn a capability ON. +// // ── HOW TO ADD / UPDATE A MODEL ────────────────────────────────────── // Authoritative data source: https://models.dev/api.json (145 providers, 4000+ // models, MIT). Each model exposes the exact fields we map below: @@ -23,6 +33,7 @@ // 2.0+, Grok, Perplexity). Verify with: curl -s https://models.dev/api.json import { matchPattern } from "./pricing.js"; +import { looksLikeVisionModel } from "./visionPatterns.js"; /** * Safe floor — every resolved result is merged over this so consumers @@ -94,8 +105,8 @@ export const MODEL_CAPABILITIES = { // Gemini image-gen / OpenAI image / xai image variants "gpt-image-1": { imageOutput: true, tools: false }, - // GLM vision variants (text GLM has no vision) — 5.3-Flash is natively - // multimodal per z.ai and carries the full 1M window. + // GLM vision variants (text GLM has no vision) — 5.3-Flash and 5V-Turbo are + // natively multimodal per z.ai, and 5.3-Flash carries the full 1M window. "glm-5.3-flash": { vision: true, videoInput: true, pdf: true, reasoning: true, thinkingFormat: "zai", contextWindow: 1000000, maxOutput: 131072 }, "glm-4.6v": { vision: true, videoInput: true, reasoning: true, thinkingFormat: "zai", contextWindow: 128000, maxOutput: 32768 }, "glm-4.5v": { vision: true, videoInput: true, reasoning: true, thinkingFormat: "zai", contextWindow: 64000, maxOutput: 16384 }, @@ -333,6 +344,46 @@ export const PATTERN_CAPABILITIES = [ * @param {string} model * @returns {object} full capabilities object */ +const MODALITY_KEYS = ["vision", "pdf", "audioInput", "videoInput"]; + +// Catalog lookups, installed by the server at startup. Left as no-ops in the +// browser bundle, where there is no file to read. +let catalogSource = null; + +/** + * Install the synced catalog reader (server only). + * @param {{ getModalities: Function, getLimits: Function } | null} source + */ +export function setCatalogSource(source) { + catalogSource = source; +} + +// Apply the synced catalog + name heuristic on top of a table-resolved result. +// Strictly additive: a capability already true stays true, and a false one only +// flips when an outside source positively declares support. +function refine(base, provider, model) { + const result = { ...DEFAULT_CAPABILITIES, ...base }; + + if (catalogSource) { + const modalities = catalogSource.getModalities(model); + if (modalities) { + for (const key of MODALITY_KEYS) { + if (modalities[key] === true) result[key] = true; + } + } + + const limits = catalogSource.getLimits(provider, model); + if (limits) { + if (limits.contextWindow > 0) result.contextWindow = limits.contextWindow; + if (limits.maxOutput > 0) result.maxOutput = limits.maxOutput; + } + } + + if (!result.vision && looksLikeVisionModel(model)) result.vision = true; + + return result; +} + export function getCapabilitiesForModel(provider, model) { if (!model) return { ...DEFAULT_CAPABILITIES }; @@ -350,13 +401,13 @@ export function getCapabilitiesForModel(provider, model) { if (MODEL_CAPABILITIES[baseModel]) return { ...DEFAULT_CAPABILITIES, ...MODEL_CAPABILITIES[baseModel] }; if (MODEL_CAPABILITIES[model]) return { ...DEFAULT_CAPABILITIES, ...MODEL_CAPABILITIES[model] }; - // 3. Pattern match (first match wins) + // 3. Pattern match (first match wins), refined by catalog + name heuristic for (const { pattern, caps } of PATTERN_CAPABILITIES) { if (matchPattern(pattern, baseModel) || matchPattern(pattern, model)) { - return { ...DEFAULT_CAPABILITIES, ...caps }; + return refine(caps, provider, model); } } // 4. Floor - return { ...DEFAULT_CAPABILITIES }; + return refine(null, provider, model); } diff --git a/open-sse/providers/catalogOverride.js b/open-sse/providers/catalogOverride.js new file mode 100644 index 00000000..12914b7d --- /dev/null +++ b/open-sse/providers/catalogOverride.js @@ -0,0 +1,72 @@ +// Read side of the model catalog synced from models.dev. +// +// The file is the source of truth; the only thing held in memory is a parsed +// copy dropped as soon as the file's mtime changes. getCapabilitiesForModel is +// synchronous and runs per request, so the hot path is one stat (~1us) and the +// parse (~0.1ms on a ~18KB file) only reruns after a sync. + +import fs from "node:fs"; +import path from "node:path"; +import { DATA_DIR } from "@/lib/dataDir.js"; + +export const CATALOG_FILE = path.join(DATA_DIR, "model-catalog.json"); +// Trimmed upstream catalog, read by the add-models skill (not by the router). +export const CATALOG_RAW_FILE = path.join(DATA_DIR, "model-catalog-raw.json"); + +const EMPTY = { models: {}, providers: {} }; +let cache = EMPTY; +let cachedMtime = -1; + +// "zai-org/GLM-4.6V:free" -> "glm-4.6v" +function baseId(model) { + if (!model) return ""; + const withoutVendor = model.includes("/") ? model.split("/").pop() : model; + return withoutVendor.toLowerCase().split(":")[0]; +} + +function load() { + let mtime; + try { + mtime = fs.statSync(CATALOG_FILE).mtimeMs; + } catch { + cache = EMPTY; + cachedMtime = -1; + return cache; + } + if (mtime === cachedMtime) return cache; + + cachedMtime = mtime; + try { + const parsed = JSON.parse(fs.readFileSync(CATALOG_FILE, "utf8")); + cache = { models: parsed?.models || {}, providers: parsed?.providers || {} }; + } catch { + cache = EMPTY; + } + return cache; +} + +// Modality is a property of the model itself — any gateway serving it inherits +// the same image/video/pdf support, so this is keyed by model id alone. +export function getCatalogModalities(model) { + return load().models[baseId(model)] || null; +} + +// Context and output limits are a property of the gateway, not the model: each +// one truncates differently, so these stay keyed by provider + model. +export function getCatalogLimits(provider, model) { + const byProvider = provider && load().providers[provider]; + if (!byProvider) return null; + return byProvider[model] || byProvider[baseId(model)] || null; +} + +// Force a re-read on the next lookup (called right after a sync writes the file). +export function invalidateCatalog() { + cachedMtime = -1; +} + +// Hand the reader to capabilities.js. That module is bundled into the browser +// too, so it cannot import this file directly — the server pushes it in. +export async function installCatalogSource() { + const { setCatalogSource } = await import("./capabilities.js"); + setCatalogSource({ getModalities: getCatalogModalities, getLimits: getCatalogLimits }); +} diff --git a/open-sse/providers/visionPatterns.js b/open-sse/providers/visionPatterns.js new file mode 100644 index 00000000..3c93afe6 --- /dev/null +++ b/open-sse/providers/visionPatterns.js @@ -0,0 +1,42 @@ +// Name-based vision detection — last resort when neither the catalog file nor +// the capability tables know a model. Vendors put the modality in the id +// ("qwen3-vl-plus", "glm-4.6v", "deepseek-v4-flash-vision-exp"), so a custom or +// freshly released model still gets image input instead of silently dropping it. +// +// Only ever turns vision ON. Never used to turn a declared capability off. + +const SEP = "[-_/:.]"; + +// Image GENERATION, video generation, and non-chat models also carry these +// words but take no image input — checked first so they can never match. +const NOT_VISION = new RegExp( + [ + `(^|${SEP})(image|img)(${SEP}|$)`, + "stable-image", "gen[0-9]_image", "nanobanana", "imagine", + "t2v", "i2v", "flux", "dall", "sdxl", "diffusion", + "embed", "rerank", "guard", "moderation", + "tts", "stt", "whisper", "voice", "speech", "audio", + ].join("|"), + "i" +); + +// Explicit modality words, plus the "v" suffix vendors use for vision +// variants (glm-4.6v, glm-5v-turbo). The digit-v branch requires a dotted +// version so the never-shipped `gpt-4v` cannot match. +const VISION_NAME = new RegExp( + [ + `(^|${SEP})(vision|vl|vlm|multimodal|omni|visual)(${SEP}|$)`, + `[0-9]\\.[0-9]+v(${SEP}|$)`, + `(^|${SEP})glm-[0-9]+v(${SEP}|$)`, + "(^|[-_/:.])(llava|pixtral|internvl|cogvlm|minicpm-v|moondream|idefics|fuyu)", + ].join("|"), + "i" +); + +// Does this model id look like a vision model? Name signal only. +export function looksLikeVisionModel(modelId) { + if (!modelId) return false; + const id = String(modelId).toLowerCase(); + if (NOT_VISION.test(id)) return false; + return VISION_NAME.test(id); +} diff --git a/src/app/api/models/catalog-sync/route.js b/src/app/api/models/catalog-sync/route.js new file mode 100644 index 00000000..bc5482b5 --- /dev/null +++ b/src/app/api/models/catalog-sync/route.js @@ -0,0 +1,31 @@ +import { NextResponse } from "next/server"; +import fs from "node:fs"; +import { getSyncState, syncModelCatalog } from "@/lib/modelCatalog/sync.js"; +import { CATALOG_FILE } from "open-sse/providers/catalogOverride.js"; + +// GET /api/models/catalog-sync - Sync status and what the catalog currently holds +export async function GET() { + const state = getSyncState(); + let catalog = null; + try { + const parsed = JSON.parse(fs.readFileSync(CATALOG_FILE, "utf8")); + catalog = { + syncedAt: parsed.syncedAt, + models: Object.keys(parsed.models || {}).length, + providers: Object.keys(parsed.providers || {}).length, + bytes: fs.statSync(CATALOG_FILE).size, + }; + } catch { + catalog = null; + } + return NextResponse.json({ ...state, catalog }); +} + +// POST /api/models/catalog-sync - Run a sync now instead of waiting for the timer +export async function POST() { + const result = await syncModelCatalog(); + if (!result) { + return NextResponse.json({ error: getSyncState().lastError || "sync in progress" }, { status: 503 }); + } + return NextResponse.json({ success: true, result }); +} diff --git a/src/instrumentation.js b/src/instrumentation.js index f511eae2..57ea03f1 100644 --- a/src/instrumentation.js +++ b/src/instrumentation.js @@ -2,5 +2,13 @@ export async function register() { if (process.env.NEXT_RUNTIME === "nodejs") { const { initConsoleLogCapture } = await import("@/lib/consoleLogBuffer"); initConsoleLogCapture(); + + // Server-only: lets capabilities.js read the synced catalog without pulling + // node:fs into the dashboard's browser bundle. + const { installCatalogSource } = await import("open-sse/providers/catalogOverride.js"); + await installCatalogSource(); + + const { startModelCatalogSync } = await import("@/lib/modelCatalog/sync.js"); + startModelCatalogSync(); } } diff --git a/src/lib/modelCatalog/sync.js b/src/lib/modelCatalog/sync.js new file mode 100644 index 00000000..d30adc71 --- /dev/null +++ b/src/lib/modelCatalog/sync.js @@ -0,0 +1,140 @@ +// Background refresh of model capabilities from models.dev. +// +// Failures are swallowed on purpose: a stale or missing catalog just means the +// hand-written capability tables keep deciding on their own. + +import path from "node:path"; +import { Worker } from "node:worker_threads"; +import { fileURLToPath } from "node:url"; +import { CATALOG_FILE, CATALOG_RAW_FILE, invalidateCatalog, installCatalogSource } from "open-sse/providers/catalogOverride.js"; + +const CATALOG_URL = "https://models.dev/api.json"; +const WORKER_FILE = path.join(path.dirname(fileURLToPath(import.meta.url)), "worker.js"); + +// 9router provider id -> models.dev provider id, for context/maxOutput only. +// Providers absent here keep whatever the local pattern table resolves; the +// names that already match are resolved automatically. +const PROVIDER_ALIASES = { + "glm": "zai", + "glm-cn": "zhipuai", + "claude": "anthropic", + "gemini": "google", + "kimi": "moonshotai", + "kimi-cn": "moonshotai-cn", + "qwen": "alibaba", + "qwen-cn": "alibaba-cn", + "zhipu": "zhipuai", + "hunyuan": "tencent", + "doubao": "volcengine", + "cloudflare-ai": "cloudflare-workers-ai", +}; + +export const SYNC_INTERVAL_MS = 24 * 60 * 60 * 1000; +const STARTUP_DELAY_MS = 60 * 1000; // let the server boot and serve first requests +const RETRY_DELAY_MS = 30 * 60 * 1000; +const WORKER_TIMEOUT_MS = 120000; + +let state = { running: false, lastSync: null, lastError: null, lastResult: null, etag: null }; +let timer = null; + +export function getSyncState() { + return { ...state, file: CATALOG_FILE, url: CATALOG_URL, intervalMs: SYNC_INTERVAL_MS }; +} + +// Snapshot every registered model with its currently resolved capabilities, so +// the worker can compute a delta without importing app modules (it cannot +// resolve the bundler-only "open-sse/*" alias). +async function collectEntries() { + const [{ default: registry }, { getCapabilitiesForModel }] = await Promise.all([ + import("open-sse/providers/registry/index.js"), + import("open-sse/providers/capabilities.js"), + ]); + await installCatalogSource(); + + const entries = []; + for (const provider of registry) { + for (const model of provider.models || []) { + entries.push({ + provider: provider.id, + model: model.id, + contextLength: model.contextLength, + current: getCapabilitiesForModel(provider.id, model.id), + }); + } + } + return entries; +} + +function runWorker(entries) { + return new Promise((resolve, reject) => { + const worker = new Worker(WORKER_FILE, { + workerData: { + url: CATALOG_URL, + etag: state.etag, + outFile: CATALOG_FILE, + rawFile: CATALOG_RAW_FILE, + entries, + providerAliases: PROVIDER_ALIASES, + }, + resourceLimits: { maxOldGenerationSizeMb: 512 }, + }); + + let settled = false; + const finish = (fn, value) => { + if (settled) return; + settled = true; + clearTimeout(timeout); + fn(value); + }; + const timeout = setTimeout(() => { + worker.terminate(); + finish(reject, new Error("sync timed out")); + }, WORKER_TIMEOUT_MS); + + worker.on("message", (msg) => { + if (msg?.ok) finish(resolve, msg.result); + else finish(reject, new Error(msg?.error || "sync failed")); + }); + worker.on("error", (err) => finish(reject, err)); + worker.on("exit", (code) => finish(reject, new Error(`worker exited with ${code}`))); + }); +} + +// Run one sync. Returns the worker summary, or null when it could not complete. +export async function syncModelCatalog() { + if (state.running) return null; + state.running = true; + try { + const result = await runWorker(await collectEntries()); + if (result.status === "updated") { + state.etag = result.etag; + invalidateCatalog(); + console.log(`[modelCatalog] ${result.models} models, ${result.providers} providers, ${(result.bytes / 1024).toFixed(1)}KB`); + } + state.lastSync = Date.now(); + state.lastError = null; + state.lastResult = result; + return result; + } catch (error) { + state.lastError = error?.message || String(error); + console.log(`[modelCatalog] sync failed: ${state.lastError}`); + return null; + } finally { + state.running = false; + } +} + +// Schedule the recurring sync. Disable entirely with MODEL_CATALOG_SYNC=off. +export function startModelCatalogSync() { + if (timer) return; + if (String(process.env.MODEL_CATALOG_SYNC || "").toLowerCase() === "off") return; + + const schedule = (delay) => { + timer = setTimeout(async () => { + const result = await syncModelCatalog(); + schedule(result ? SYNC_INTERVAL_MS : RETRY_DELAY_MS); + }, delay); + timer.unref?.(); + }; + schedule(STARTUP_DELAY_MS); +} diff --git a/src/lib/modelCatalog/worker.js b/src/lib/modelCatalog/worker.js new file mode 100644 index 00000000..39629ea0 --- /dev/null +++ b/src/lib/modelCatalog/worker.js @@ -0,0 +1,132 @@ +// Downloads models.dev and writes the capability deltas 9router reads. +// Runs in a worker thread: the 4MB parse would otherwise block requests for ~20ms. + +import { parentPort, workerData } from "node:worker_threads"; +import fs from "node:fs"; +import path from "node:path"; + +const FETCH_TIMEOUT_MS = 60000; +const MODALITY_BY_INPUT = { image: "vision", pdf: "pdf", audio: "audioInput", video: "videoInput" }; +// Gateways disagree about the same model, so a modality needs a majority of +// them to declare it — one reseller mislabelling a text model must not win. +const MIN_SHARE = 0.5; +// Ignore limit differences below this: gateways round 200000 vs 202752. +const LIMIT_TOLERANCE = 0.1; + +// "zai-org/GLM-4.6V:free" -> "glm-4.6v" +function baseId(modelId) { + const withoutVendor = modelId.includes("/") ? modelId.split("/").pop() : modelId; + return withoutVendor.toLowerCase().split(":")[0]; +} + +function build(catalog, entries, providerAliases) { + // Index the catalog once: per provider for limits, and tallied for modalities. + const byProvider = {}; + const tally = {}; + for (const [providerId, provider] of Object.entries(catalog)) { + const models = {}; + for (const [modelId, model] of Object.entries(provider?.models || {})) { + const id = baseId(modelId); + models[id] = model; + const counts = tally[id] || (tally[id] = { total: 0 }); + counts.total++; + for (const input of model?.modalities?.input || []) { + const key = MODALITY_BY_INPUT[input]; + if (key) counts[key] = (counts[key] || 0) + 1; + } + } + byProvider[providerId] = models; + } + + // Modalities belong to the model — every gateway serving it has the same + // weights — so they are keyed by model id and shared across providers. + const models = {}; + for (const [id, counts] of Object.entries(tally)) { + const declared = {}; + for (const key of Object.values(MODALITY_BY_INPUT)) { + if ((counts[key] || 0) / counts.total >= MIN_SHARE) declared[key] = true; + } + if (Object.keys(declared).length) models[id] = declared; + } + + // Limits belong to the gateway — each truncates differently — so only the + // matching provider's own numbers are used, keyed by provider + model. + const providers = {}; + for (const { provider, model, contextLength, current } of entries) { + const alias = providerAliases[provider]; + const upstream = catalog[provider] ? provider : (alias && catalog[alias] ? alias : null); + const entry = upstream && byProvider[upstream]?.[baseId(model)]; + if (!entry) continue; + + const delta = {}; + const { context, output } = entry.limit || {}; + if (context > 0 && !contextLength + && Math.abs(context - current.contextWindow) / current.contextWindow > LIMIT_TOLERANCE) { + delta.contextWindow = context; + } + if (output > 0 + && Math.abs(output - current.maxOutput) / current.maxOutput > LIMIT_TOLERANCE) { + delta.maxOutput = output; + } + if (Object.keys(delta).length) (providers[provider] || (providers[provider] = {}))[model] = delta; + } + + return { models, providers }; +} + +// Trimmed copy of the upstream catalog, kept for the add-models skill: same +// 7348 models, 470KB instead of 4.3MB, so a scan reads it in ~5ms. +function slim(catalog) { + const out = {}; + for (const [providerId, provider] of Object.entries(catalog)) { + const models = {}; + for (const [modelId, model] of Object.entries(provider?.models || {})) { + models[modelId] = { + i: (model?.modalities?.input || []).filter((x) => x !== "text"), + c: model?.limit?.context, + o: model?.limit?.output, + r: model?.reasoning || undefined, + }; + } + out[providerId] = models; + } + return out; +} + +function writeAtomic(file, contents) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(`${file}.tmp`, contents, "utf8"); + fs.renameSync(`${file}.tmp`, file); +} + +async function run() { + const { url, etag, outFile, rawFile, entries, providerAliases } = workerData; + + const headers = { accept: "application/json" }; + if (etag) headers["if-none-match"] = etag; + const response = await fetch(url, { headers, signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) }); + + if (response.status === 304) return { status: "unchanged" }; + if (!response.ok) throw new Error(`HTTP ${response.status}`); + + const catalog = await response.json(); + const nextEtag = response.headers.get("etag") || null; + const { models, providers } = build(catalog, entries, providerAliases); + const serialized = JSON.stringify({ v: 1, etag: nextEtag, syncedAt: Date.now(), models, providers }); + + writeAtomic(outFile, serialized); + if (rawFile) writeAtomic(rawFile, JSON.stringify(slim(catalog))); + + return { + status: "updated", + etag: nextEtag, + bytes: Buffer.byteLength(serialized), + models: Object.keys(models).length, + providers: Object.keys(providers).length, + }; +} + +run().then( + (result) => parentPort?.postMessage({ ok: true, result }), + (error) => parentPort?.postMessage({ ok: false, error: error?.message || String(error) }) +);