feat(fetch): add Ollama Cloud web fetch provider

This commit is contained in:
Sami Basra
2026-09-03 10:12:41 +07:00
committed by decolua
parent 6ab9ca9eb1
commit e0ffc7e2a1
6 changed files with 255 additions and 8 deletions

View File

@@ -1,4 +1,4 @@
// Web Fetch handler — dispatches to firecrawl, jina-reader, tavily, exa
// Web Fetch handler — dispatches to firecrawl, jina-reader, tavily, exa, ollama
// Returns normalized shape across all providers
const DEFAULT_TIMEOUT_MS = 15000;
@@ -56,8 +56,8 @@ function parseJinaTitle(text) {
return m ? m[1].trim() : null;
}
function buildData({ provider, url, title, format, text, costUsd, responseMs, upstreamMs }) {
return {
function buildData({ provider, url, title, format, text, links, costUsd, responseMs, upstreamMs }) {
const data = {
provider,
url,
title: title || null,
@@ -66,6 +66,8 @@ function buildData({ provider, url, title, format, text, costUsd, responseMs, up
usage: { fetch_cost_usd: costUsd ?? null },
metrics: { response_time_ms: responseMs, upstream_latency_ms: upstreamMs }
};
if (Array.isArray(links)) data.links = links;
return data;
}
async function readJsonOrText(res) {
@@ -115,6 +117,18 @@ export async function handleFetchCore({ url, format, maxCharacters, provider, pr
if (provider === "exa") {
return await runExa({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery, startedAt });
}
if (provider === "ollama") {
return await runOllama({
url,
fmt,
timeoutMs,
apiKey,
maxCharacters,
costPerQuery,
startedAt,
baseUrl: providerConfig?.baseUrl,
});
}
return { success: false, status: 400, error: `Unsupported provider: ${provider}` };
} catch (err) {
log?.("fetch handler error:", err?.message || err);
@@ -241,3 +255,56 @@ async function runExa({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery
})
};
}
async function runOllama({
url,
fmt,
timeoutMs,
apiKey,
maxCharacters,
costPerQuery,
startedAt,
baseUrl,
}) {
const upstreamStart = Date.now();
const r = await tryFetch(baseUrl, {
method: "POST",
headers: {
"content-type": "application/json",
...(apiKey ? { authorization: `Bearer ${apiKey}` } : {})
},
body: JSON.stringify({ url })
}, timeoutMs);
if (!r.ok) {
return { success: false, status: r.timeout ? 504 : 502, error: r.error };
}
const upstreamMs = Date.now() - upstreamStart;
const { json, text: responseText } = await readJsonOrText(r.res);
if (!r.res.ok) {
const error = json?.error
|| json?.message
|| responseText?.slice(0, 500)
|| `Ollama error: ${r.res.status}`;
return { success: false, status: r.res.status, error };
}
if (!json || typeof json.content !== "string") {
return { success: false, status: 502, error: "Ollama returned an empty or invalid web fetch response" };
}
const text = truncate(json.content, maxCharacters);
return {
success: true,
data: buildData({
provider: "ollama",
url,
title: json.title || null,
format: fmt,
text,
links: json.links,
costUsd: costPerQuery,
responseMs: Date.now() - startedAt,
upstreamMs
})
};
}

View File

@@ -31,7 +31,16 @@ export default {
{ id: "qwen3.5", name: "Qwen3.5" },
{ id: "minimax-m3", name: "MiniMax M3" },
],
serviceKinds: ["llm"],
serviceKinds: ["llm", "webFetch"],
fetchConfig: {
baseUrl: "https://ollama.com/api/web_fetch",
method: "POST",
authType: "apikey",
authHeader: "bearer",
formats: ["markdown"],
maxCharacters: 200000,
timeoutMs: 30000,
},
features: {
usage: true,
usageApikey: true,