feat(fetch): add Ollama Cloud web fetch provider
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
// Web Fetch handler — dispatches to firecrawl, jina-reader, tavily, exa
|
||||
// Web Fetch handler — dispatches to firecrawl, jina-reader, tavily, exa, ollama
|
||||
// Returns normalized shape across all providers
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 15000;
|
||||
@@ -56,8 +56,8 @@ function parseJinaTitle(text) {
|
||||
return m ? m[1].trim() : null;
|
||||
}
|
||||
|
||||
function buildData({ provider, url, title, format, text, costUsd, responseMs, upstreamMs }) {
|
||||
return {
|
||||
function buildData({ provider, url, title, format, text, links, costUsd, responseMs, upstreamMs }) {
|
||||
const data = {
|
||||
provider,
|
||||
url,
|
||||
title: title || null,
|
||||
@@ -66,6 +66,8 @@ function buildData({ provider, url, title, format, text, costUsd, responseMs, up
|
||||
usage: { fetch_cost_usd: costUsd ?? null },
|
||||
metrics: { response_time_ms: responseMs, upstream_latency_ms: upstreamMs }
|
||||
};
|
||||
if (Array.isArray(links)) data.links = links;
|
||||
return data;
|
||||
}
|
||||
|
||||
async function readJsonOrText(res) {
|
||||
@@ -115,6 +117,18 @@ export async function handleFetchCore({ url, format, maxCharacters, provider, pr
|
||||
if (provider === "exa") {
|
||||
return await runExa({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery, startedAt });
|
||||
}
|
||||
if (provider === "ollama") {
|
||||
return await runOllama({
|
||||
url,
|
||||
fmt,
|
||||
timeoutMs,
|
||||
apiKey,
|
||||
maxCharacters,
|
||||
costPerQuery,
|
||||
startedAt,
|
||||
baseUrl: providerConfig?.baseUrl,
|
||||
});
|
||||
}
|
||||
return { success: false, status: 400, error: `Unsupported provider: ${provider}` };
|
||||
} catch (err) {
|
||||
log?.("fetch handler error:", err?.message || err);
|
||||
@@ -241,3 +255,56 @@ async function runExa({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery
|
||||
})
|
||||
};
|
||||
}
|
||||
|
||||
async function runOllama({
|
||||
url,
|
||||
fmt,
|
||||
timeoutMs,
|
||||
apiKey,
|
||||
maxCharacters,
|
||||
costPerQuery,
|
||||
startedAt,
|
||||
baseUrl,
|
||||
}) {
|
||||
const upstreamStart = Date.now();
|
||||
const r = await tryFetch(baseUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
...(apiKey ? { authorization: `Bearer ${apiKey}` } : {})
|
||||
},
|
||||
body: JSON.stringify({ url })
|
||||
}, timeoutMs);
|
||||
|
||||
if (!r.ok) {
|
||||
return { success: false, status: r.timeout ? 504 : 502, error: r.error };
|
||||
}
|
||||
const upstreamMs = Date.now() - upstreamStart;
|
||||
const { json, text: responseText } = await readJsonOrText(r.res);
|
||||
if (!r.res.ok) {
|
||||
const error = json?.error
|
||||
|| json?.message
|
||||
|| responseText?.slice(0, 500)
|
||||
|| `Ollama error: ${r.res.status}`;
|
||||
return { success: false, status: r.res.status, error };
|
||||
}
|
||||
if (!json || typeof json.content !== "string") {
|
||||
return { success: false, status: 502, error: "Ollama returned an empty or invalid web fetch response" };
|
||||
}
|
||||
|
||||
const text = truncate(json.content, maxCharacters);
|
||||
return {
|
||||
success: true,
|
||||
data: buildData({
|
||||
provider: "ollama",
|
||||
url,
|
||||
title: json.title || null,
|
||||
format: fmt,
|
||||
text,
|
||||
links: json.links,
|
||||
costUsd: costPerQuery,
|
||||
responseMs: Date.now() - startedAt,
|
||||
upstreamMs
|
||||
})
|
||||
};
|
||||
}
|
||||
|
||||
@@ -31,7 +31,16 @@ export default {
|
||||
{ id: "qwen3.5", name: "Qwen3.5" },
|
||||
{ id: "minimax-m3", name: "MiniMax M3" },
|
||||
],
|
||||
serviceKinds: ["llm"],
|
||||
serviceKinds: ["llm", "webFetch"],
|
||||
fetchConfig: {
|
||||
baseUrl: "https://ollama.com/api/web_fetch",
|
||||
method: "POST",
|
||||
authType: "apikey",
|
||||
authHeader: "bearer",
|
||||
formats: ["markdown"],
|
||||
maxCharacters: 200000,
|
||||
timeoutMs: 30000,
|
||||
},
|
||||
features: {
|
||||
usage: true,
|
||||
usageApikey: true,
|
||||
|
||||
Reference in New Issue
Block a user