From 60e890d7f18b5a82f3690cbeb92f243e17d49f04 Mon Sep 17 00:00:00 2001 From: Emirhan Date: Mon, 28 Sep 2026 12:39:11 +0700 Subject: [PATCH] feat(web): add TinyFish search and fetch provider --- CHANGELOG.md | 1 + open-sse/handlers/fetch/index.js | 32 +++++++- open-sse/handlers/search/callers.js | 24 ++++++ open-sse/handlers/search/index.js | 5 +- open-sse/handlers/search/normalizers.js | 14 ++++ open-sse/providers/registry/index.js | 2 + open-sse/providers/registry/tinyfish.js | 36 +++++++++ public/providers/tinyfish.png | Bin 0 -> 7463 bytes tests/unit/tinyfish-web-provider.test.js | 91 +++++++++++++++++++++++ 9 files changed, 203 insertions(+), 2 deletions(-) create mode 100644 open-sse/providers/registry/tinyfish.js create mode 100644 public/providers/tinyfish.png create mode 100644 tests/unit/tinyfish-web-provider.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cb89240..b6d8cd61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # v0.5.91 (2026-09-26) ## Features +- **Web Search & Fetch**: add TinyFish Search and Fetch with one API-key connection, normalized results, and official provider icon - **Providers**: add Token Harbor provider and four OpenAI-compatible aggregator providers (dahl, atria, agnes, bai) - **Claude**: forward `x-claude-code-session-id` on OAuth requests; merge client `anthropic-beta` flags and forward rate-limit headers; return thinking text to OpenAI-format clients - **Codex**: add GPT-6 Sol and Luna support diff --git a/open-sse/handlers/fetch/index.js b/open-sse/handlers/fetch/index.js index 9bad7e18..364caa5d 100644 --- a/open-sse/handlers/fetch/index.js +++ b/open-sse/handlers/fetch/index.js @@ -1,4 +1,4 @@ -// Web Fetch handler — dispatches to firecrawl, jina-reader, tavily, exa, ollama +// Web Fetch handler — dispatches to firecrawl, jina-reader, tavily, exa, ollama, tinyfish // Returns normalized shape across all providers const DEFAULT_TIMEOUT_MS = 15000; @@ -129,6 +129,9 @@ export async function handleFetchCore({ url, format, maxCharacters, provider, pr baseUrl: providerConfig?.baseUrl, }); } + if (provider === "tinyfish") { + return await runTinyfish({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery, startedAt, baseUrl: providerConfig?.baseUrl }); + } return { success: false, status: 400, error: `Unsupported provider: ${provider}` }; } catch (err) { log?.("fetch handler error:", err?.message || err); @@ -136,6 +139,33 @@ export async function handleFetchCore({ url, format, maxCharacters, provider, pr } } +async function runTinyfish({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery, startedAt, baseUrl }) { + if (!["markdown", "html"].includes(fmt)) return { success: false, status: 400, error: `Unsupported TinyFish format: ${fmt}` }; + const upstreamStart = Date.now(); + const r = await tryFetch(baseUrl, { + method: "POST", + headers: { "content-type": "application/json", "X-API-Key": apiKey }, + body: JSON.stringify({ urls: [url], format: fmt }), + }, timeoutMs); + if (!r.ok) return { success: false, status: r.timeout ? 504 : 502, error: r.error }; + const upstreamMs = Date.now() - upstreamStart; + const { json } = await readJsonOrText(r.res); + if (!r.res.ok) return { success: false, status: r.res.status, error: json?.error?.message || `TinyFish error: ${r.res.status}` }; + const failure = json?.errors?.[0]; + if (failure) return { success: false, status: failure.status || 502, error: `TinyFish fetch failed: ${failure.error || "unknown error"}` }; + const page = json?.results?.[0]; + if (!page || typeof page.text !== "string") return { success: false, status: 502, error: "TinyFish returned no extractable content" }; + const text = truncate(page.text, maxCharacters); + return { + success: true, + data: { + ...buildData({ provider: "tinyfish", url, title: page.title, format: fmt, text, links: page.links, + costUsd: costPerQuery, responseMs: Date.now() - startedAt, upstreamMs }), + metadata: { author: page.author || null, published_at: page.published_date || null, language: page.language || null }, + }, + }; +} + async function runFirecrawl({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery, startedAt }) { const upstreamStart = Date.now(); const r = await tryFetch("https://api.firecrawl.dev/v1/scrape", { diff --git a/open-sse/handlers/search/callers.js b/open-sse/handlers/search/callers.js index 5e3b3c09..56d08e4f 100644 --- a/open-sse/handlers/search/callers.js +++ b/open-sse/handlers/search/callers.js @@ -374,6 +374,29 @@ function buildXquikRequest(config, params) { }; } +function buildTinyfishRequest(config, params) { + if (params.searchType && !["web", "news", "research_paper"].includes(params.searchType)) { + throw new Error("Unsupported TinyFish search type"); + } + const qp = new URLSearchParams({ query: params.query }); + if (params.searchType && params.searchType !== "web") qp.set("domain_type", params.searchType); + if (params.country) qp.set("location", params.country); + if (params.language) qp.set("language", params.language); + const { includes, excludes } = parseDomainFilter(params.domainFilter); + if (includes.length) qp.set("include_domains", includes.join(",")); + if (excludes.length) qp.set("exclude_domains", excludes.join(",")); + if (Number.isInteger(params.offset) && params.offset > 0) { + if (params.offset >= 110) throw new Error("TinyFish search offset exceeds available pages"); + if (params.offset % 10 + params.maxResults > 10) throw new Error("TinyFish search offset and max_results must fit within one page"); + qp.set("page", String(Math.floor(params.offset / 10))); + } + return { + // Keep API-key endpoint fixed; client baseUrl overrides must not receive the key. + url: `${config.baseUrl}?${qp}`, + init: { method: "GET", headers: { Accept: "application/json", "X-API-Key": params.token } }, + }; +} + // ── Ollama Cloud web_search ────────────────────────────────────────────── // POST https://ollama.com/api/web_search { query, max_results } // Response: { results: [{ title, url, content, published_at? }] } @@ -436,6 +459,7 @@ const BUILDERS = { "youcom": buildYouComRequest, "searxng": buildSearxngRequest, "xquik": buildXquikRequest, + "tinyfish": buildTinyfishRequest, "ollama-search": buildOllamaSearchRequest, "glm": buildGlmSearchRequest, }; diff --git a/open-sse/handlers/search/index.js b/open-sse/handlers/search/index.js index 70c3c749..fa6d146d 100644 --- a/open-sse/handlers/search/index.js +++ b/open-sse/handlers/search/index.js @@ -110,7 +110,10 @@ async function tryDedicatedProvider({ provider, providerConfig, body, credential } const data = await resp.json(); const normalized = normalizeSearchResponse(provider.id, data, params.query, params.searchType); - const results = normalized.results.slice(0, params.maxResults); + // TinyFish uses fixed 10-result pages; offset within a page is applied locally. + const pageOffset = provider.id === "tinyfish" && Number.isInteger(params.offset) && params.offset > 0 + ? params.offset % 10 : 0; + const results = normalized.results.slice(pageOffset, pageOffset + params.maxResults); const duration = Date.now() - startTime; const usage = { queries_used: 1, diff --git a/open-sse/handlers/search/normalizers.js b/open-sse/handlers/search/normalizers.js index 3b415ae5..5ef7f36f 100644 --- a/open-sse/handlers/search/normalizers.js +++ b/open-sse/handlers/search/normalizers.js @@ -107,6 +107,19 @@ function normalizeTavily(data, _query, _searchType) { return { results, totalResults: results.length }; } +function normalizeTinyfish(data) { + const now = new Date().toISOString(); + const items = Array.isArray(data?.results) ? data.results : []; + return { + results: items.map((item, idx) => makeResult("tinyfish", { + title: item.title, url: item.url, snippet: item.snippet, + published_at: item.date, source_type: item.publisher || null, + author: Array.isArray(item.authors) ? item.authors.join(", ") : null, + }, idx, now)), + totalResults: data?.total_results ?? null, + }; +} + function normalizeGooglePse(data, _query, _searchType) { const now = new Date().toISOString(); const items = Array.isArray(data.items) ? data.items : []; @@ -294,6 +307,7 @@ const NORMALIZERS = { "youcom": normalizeYouCom, "searxng": normalizeSearxng, "xquik": normalizeXquik, + "tinyfish": normalizeTinyfish, "ollama-search": normalizeOllamaSearch, "glm": normalizeGlmSearch, }; diff --git a/open-sse/providers/registry/index.js b/open-sse/providers/registry/index.js index 07b71d2c..6c9d1f41 100644 --- a/open-sse/providers/registry/index.js +++ b/open-sse/providers/registry/index.js @@ -130,6 +130,7 @@ import p126 from "./dahl.js"; import p127 from "./atria.js"; import p129 from "./agnes.js"; import p130 from "./bai.js"; +import p131 from "./tinyfish.js"; export default [ p0, p1, @@ -260,4 +261,5 @@ export default [ p127, p129, p130, + p131, ]; diff --git a/open-sse/providers/registry/tinyfish.js b/open-sse/providers/registry/tinyfish.js new file mode 100644 index 00000000..c6875251 --- /dev/null +++ b/open-sse/providers/registry/tinyfish.js @@ -0,0 +1,36 @@ +export default { + id: "tinyfish", + alias: "tinyfish", + display: { + name: "TinyFish", + color: "#FF6700", + textIcon: "TF", + website: "https://www.tinyfish.ai/", + notice: { apiKeyUrl: "https://agent.tinyfish.ai/api-keys" }, + }, + category: "apikey", + authType: "apikey", + serviceKinds: ["webSearch", "webFetch"], + searchConfig: { + baseUrl: "https://api.search.tinyfish.ai", + validateUrl: "https://api.search.tinyfish.ai/usage?limit=1", + method: "GET", + authType: "apikey", + authHeader: "x-api-key", + costPerQuery: 0, + searchTypes: ["web", "news", "research_paper"], + defaultMaxResults: 5, + maxMaxResults: 10, + timeoutMs: 10000, + }, + fetchConfig: { + baseUrl: "https://api.fetch.tinyfish.ai", + method: "POST", + authType: "apikey", + authHeader: "x-api-key", + costPerQuery: 0, + formats: ["markdown", "html"], + maxCharacters: 100000, + timeoutMs: 150000, + }, +}; diff --git a/public/providers/tinyfish.png b/public/providers/tinyfish.png new file mode 100644 index 0000000000000000000000000000000000000000..9768f27a6e76364115f24cec0b6e05b09c00957d GIT binary patch literal 7463 zcmV+?9oXWDP)Q}zxj1L$oPExW88d#rIluQ^Ywfk(wbtJ6yVkqj^{%~-4f(%Zh---F zh^fRQ#5u%Qh&K@xGF(*o~x zeiZfZyA{`$D2&MP5R#Y>&9EVHIq@6ft;8T=o(1#4^6cFK1$l3!i3s92@7&;4eDYFl zyxjZ&yw%|m{5rf77LV+Lp}m^nsYmWYk;0K86-pv6F)@qSl^8?JlaSX1pwTPDC9*Gd z&lcEbQ01?m3!FQ{9N?Dgi=tiQ`|xeQmRLezJoo6`2n{i-P-M7`M4u9WA>K*MbCA~s zaF!f9PDEN7uxNiWjvqS(B7=Q#IZ!B>ofMy+xxgOEOnm+B(MV)iem?0 z%=-@5#3n9i(+mU$+0dQ~^?0ri)$WicGN~VDIE8o@G1r7(W9NAZ8t`UhwcO0uCsR^5 z;Nvfn!WU?L*tih*jeD}B^Cp#ML!D~3VDcwz@OaHTU>Iie+OToTz$F!v_$mfhQ2>6Q zO84>~Qd#w7}pu~%*NZ9cw&deN*)2B`&{_F*uO-w>5s-fX;Hb>1W zYy9A>Ye)NFoyYZDKN4_`NBZwia-Z?gIeAK2>)Ff82d0#!EFh> zK-b*1)4&X>{(*l2y;cKVmILjmg<6l_i`KoqLz}lJpu_u<(fzZj_(DC=%MLA z{h2@;GQ1c4rGqxZG*_qn1I*qJ%=zO8W-Q-|$qP1N)bzFZbnIe$HE{(J&wJ>xhU|9$ zkz!|fl^`z(AfC@m8;79-G^)9NBAj9=NZTcZcYBm7ec~>l7MWi0C*Ym+!1`mr;d8(l z@>A=BqdKInXb3^|f|PI;FiuWMMM^4l71Iu2FOYS#hu2>brJ6L))C@0k2KY!h({L7A zO<*04SB4JQPitp8Rm&xg_BGsu{bwyuk0bp#uCJ?(0x8UQkeX2pHOf83fMz!V-ROAe zR}J{EW*7#vs)7%@Jc_qFHo&W`YNP$X?!#jbR={1iUWe#{;jpjAW)ljEFz80)rULS= z>byaVbdCrjPyxutE(_)Vcw;gR8#AVYV3>U%cFJetYEicS=xuh@(uAAc!Bj0fC=eF` z69u?yc#1aD6yDwX=D_U7fiYYoKCfv*|K=6ZyKiT7f8}ZPYF{7yJ2u2;-JifP?vUlO zm^-v1){O0rpFV1Xwog<;i5TtE*r*-2)SG*dFL=q`@hszP2Y&+4O>}3kdx`UjLx{tP zGl;Rol|Aj;Pb#_GQS0d zaB~z;vOo}`iWNu5BW++bXb%+Q;0@u_;GJotYflgn7J}$P5#}bX{0+tN6g{b9-gy?g zr}x3n1KQznwql0B8W>Wwz@6h`h`I zq0D)ZD7S%T$dyR5T78;JUI-HEv9t8?q$t}R# zIcvwsr30RKt`@{)EPA`U605*%d97ZUla!uJc+dYiGr}V-ePSd$V zI-3A9ftCZvkX=O;8?S_ zTeesM^nCU~+kvf(RwUe#%ptWEu53C9#`Q~ta@2bXC%y~oWe9&d$z8FS%Y3gv1s;f zZ&%EhXH~I1CljHS05r|arow_rc8QyoQ#8Jha(kI@5=h~$hovFGLAbSSG0Y$OGMd+` z?6e$M0eX``>w=g}$_$`sqn)QrTk~l`SxdTJqrTR4Kxj~gI+ru3^HJv+KvvFGV7{~C z(2yWhuY99<>Qz`CFlYs*t*(Wg#KYQ8nvt;(hUtvuD3dW3KMDlgT z|A1Ipr#dSnnasL@|89aNwJY+<1y3>cCNcd&F6PU=?#MDzscI9|`)-&f9z%Wk9jI=cTe=4Q@8EQ?UjWDX`rp5Mqy0Ee8lV~@w<+$Ng*j!}=q z1Q@%1hq35KU?+V6njIuATPhl(-f9Usx_Bv2``wsmNOXcs)o*!0BOS7!AbmfOx8Iin zkjsH()=-c-s?tLM1z}B?_0sbflCfj|Fd^9U&P%0*o)5cEO6gx&I0@8Ni-FV?D+puS+%2mA7l=dr7_x@5N^B-oqzRx6{|S zqs3<^*W@EiS@36izmZ8NA!u>_iDwLYZf5EU^cqYZrX>(Y)?g)obh`0 zauN(3d}t=<=~!^%busAo(j(@%EH4Fe_jYDNyK23!6U;Qql82+n{HJfxRbZG__70%z z?ST=eqdI&n9s|cO!`ma~VaBqpNcD8axv1ayWtg5K3RVxKfY+`>QC zuqql>cj;)|nWknx)-xA(jeMz{rZixf7Z?PE09EP&hShDu>raOmfWW1Vei_cXx0_GT*d_!Z}g<-2=#fqc`4|9~fN6 zsS;fxFxqrfzpmM7j_PFkt2I&2@Pus0Pb)Bb#yV#k6VDP``eC7>1`BpE&OiocFOPvnu4`ZczO;C51H^JWR9%D*^Dl z39DF}8vvNEfz0HQAFq7_AJo`FUs~K{xBltm!HPEF?7ZfT~$e!;ArHZtUI4(0p z+7rC~d|iZcufRg_R(l-9yz~xMg<|{)R+AtY0pD#0bl`1X_zF^Jwym8BFy6g&JKmpn z5Y7o+ib|+gRiH@b2Qd&90mPK_)vITIx~t&G7`!uh9M-K{hr)#mdz7ZqdYd?Jqdo5u z?>*_XIr2-DDuuAHFnhiuyvS3u(+RHakt5eLs(5or$UKGpN5dMHtDKbp)Wq)$ju1`w z`xM>4uHAu|qmu^3Q94WhI;_r3yk(6YwV-hil2n#`(5}q@N*&ZNM^qvaW zL$oa5oClv8K}>rEe*#dYZy4BLxMbo_ibRLtOjZ+R9{e=_s}BJh&!krm4;I(`@ubWW@HZ~U$Wq?K25`ICT9hvm%5 zIutZ*?5x$l#Es11mDoCaAeJp%oS}`IN*96Sz{ORjH>}5h&>9_^RyV)Ik=M7+i}hUP zR*BLjqMS28WThE+E+))yLLdSVtv--5X93Met&d+%L{l~B!21B z6Qrf$-tsYYpFd1BXdl&1k~U9Nb0$6Fdn%N{pq@=IiLQosJ2ga9fpCv2(r1xb|foY(Hg+|Aj>fgfH?~oLyPeV*OhW| z5FL6rJ9>x=4?+Kz8)7z3xAX?Y4|E(ByL`Bk9bK5rce`ax?V7uU000coNklxOtql&rIfgkV+k%jlG0&G*`a06lzty2{DBtvaw}*Qx`6# zAocR@Q@j(}3)6*x3P9Dg3c7TeL>lF~QgIF8!6{Zh1i^9R&;VF;?W`V#dzkHEfI?nDE)fw+?iJjikJT$Q-R_Fgei0q_V4G$;|G zX5_(qhlYdaDf!%`+bX3RDyFAK1SIa;JP??Rnw$zm~&mCp__M1kB-Ut-)V-m{;bl#x`7aRnZ1mHtrAHLzJ>|Q;OZ<0Vt zzxidO&xt51Ycx(D1Ald6PKgM=hN}wcMG8pH;(ey(W;&{WZj;Td*6!6UKV>I9xEvT`xRuq%b^Kh zKbx-txBO?BCfgYjZsLYiz8xkl;owWKh_GPXSE(#?j_a?w zm_DLU3!ZJePPr_r>^^h?!>6otr;!UM9T(h-0e=Ee&vGW2we+b^t6R*ZWpIuoUq_fu z733s7M^)tfq{x{Ua7?@0I6^0sWgVZ zl*3F5&|!YxVRPI^{~siuPqYMo0?>($%K?6W3819rgk_V!DH;G9c!una>=-r4lXGEy z*X8pq3vU&brx|dTwyn}vfY+Cd#)$v4Fu$Caf5_QXcxW(vM1|4q*;<%A_$A2dF%19A z*O~Lri#KEL%I(gk^!mgHyim-@H&_Wko39G2>auql)B54HU-*rxdavhUl1Cj0;ptz2 zMYN1EC4^*3cB_u5J(|nXg$~c>4ebd1@l&T3HSk*N+8F$LQ!E_O34?k*34I}h0Dt=q zpTcL~FG2h{jxIo^+7eb>MDz<*0*GL3YJ8YOK!o*O%7`TvpwEi@F;>2$j`Xjt$fS9% zS0W*c$fEy1i}dwNueJ}9;Z2}dN;)>H4jrZhT3>X@$*Jh_$^Wo%J4cf;01dt2Ov(ac zWDVl0FA9x(7t2dT0L6*Eh?OtVPo=qTabl(^sXI00Bg?44EMPqCghZ|zYQrz4av|A3 z)q2TtZS&iVlR)}CEF>82s>oU9FP5*Ub-(eLzj}xBOj#_@_j5B@S7}DpU?qT1R@I?0 zBS3F7__KcWt?maKOmbfV*$w%WxC+=wWD#j5^jimL_iWF#4vj$WgR<<^M4&WDIntuP|F1lXi^t zDH9_=qkky(i;d?1Tjjbkb1j04+}Uqg4kWqkZ42V;FFHvt>2rO@;m6qy&U(H=^otMAHuWNQd?+H1& z;x-*UnfCn#FAKeal>m}if!isDghTWK$i8RN1J09uKT`JX7IXDc4Ubau;yA-siGLE^ zL5oxmW(*^m<^yBMZPAm4&T|WZS^EL)9h9$r^fJK9%YfQE8tSkl4L>g4j(MwHrk}`M zD2qf+wp5J{S8mo5ai`AW(=m%o7xiDeo#V)CPt4aFaGKE?Y-VPlft3I>zq#|VH+7g- z-=vf8K?+6xO44-&;N!hfF}6KhknZ=U7i9zWVR;9l1jUG&g_QFT0?#jm`3D^H4ggP3 zJ?YOl^gj_=Ya${-@owiw(U~TsSt(uc^^{d;`uZ3g4A6(4uDa_|qh8Lt2Ct07d!rWM zV4Q1AiiUTIR!sa%ncOcTs7yB4cVy1Imp9#&Ru`T1eL~ z6$*5a-+=iV8D<+ydxP`LoIq?&Pf9~ z?nBS3TgxMFej|J9p5^h@q9OliRjswnfd}(7x z7l+w+N-~nppU38%M=*5KN@$U-OI<%}#WuQ@^++a_0u30wEvNy*os7BhQht`u=BF05 zFkT=H+0D>D$i~+fFtT!JSY~=;aGc{^cB7B`JcJkigcSU57KJR5^mk)}3M4f84Kz~uJZw^>y4RQOCcwmodNvW}|WZN#P?j0vnu=YSACT@<$ z2TM<)>Esi*VfZmr7#W8ibB<&AjuSX?A^}O4O-uEY#^4lj3b6`t6fuFAE#OZ8&Q4;P z&|+Gjaa&>+qE>$8J;t>DWHGC_ zu#j_6e`O}>*7V9$CHpE(0;*31WMOIJH=XN?tUGa7bY4);L;ZzkSSGVs1CYt=gFpqK z9<`9}U}8E^H}xH3J`OZPs8D&8JDGuAi(*8zU)r}js`eWIRtr408($RM=cr^i5B%L zyT9n_D)UI_kY+30Fv}<5_rpn$x$IG%xeM*mKgnE`tW)PD0fg`|9wEA*ba)D?MsW3n zg_b_cp}u?y4C?*ZGfC*N@`~O=(q`%{-hf6vqnwAlBmk|jk?xONvA_j{t(*er5Jrra z^G%L8Ewr-bH!gtj^A||ny$L%{B%8bCl}7<>8r|CoFfHc)kF?qpT1cD}*$8<_0Ahg0 zgUkJK@q8kTl~ZBN91df__b?WZhcSON4E<@yhIu%A>H-FG|J-0Eu+inr$P$M*cTByn zDVL=NvQjB82_S(FK;x021H{m&djWIDeU_&9k=?+7ZAgpThrP!V(RVGNKW&vssly#d zHAV74DGn5Mh$2&y+vGs2WDi|l5`aek7(SL_Mlk<~wKe@tb?E(+!%tpPIpnHN2Fgza zKB57i)+Q=uWC@fi1_6*mO*TK1R~Pb<0QBV2!K;*TqDH?C`G@n2 z;LAUOAL;tmhl0v?(l}zNuxU)o2eqDYMz4@&CGeF2pVlgc7zGYgry7YDpruz|dMSrv3D~7x(06mzztqYit0Q6^1ZJ3tY zkP}d626K3wsRMA*=Q+sB0&p@*sw9yYS|6k~WMaB3CFWn#a4iVnnr5)So6i@3KMv(^ zdA entry.id === "tinyfish"); +const url = "https://example.com/article"; + +afterEach(() => vi.unstubAllGlobals()); + +describe("TinyFish Search and Fetch", () => { + it("registers both kinds on one API-key provider", () => { + expect(provider.serviceKinds).toEqual(["webSearch", "webFetch"]); + expect(getProvidersByKind("webSearch").some((p) => p.id === "tinyfish")).toBe(true); + expect(getProvidersByKind("webFetch").some((p) => p.id === "tinyfish")).toBe(true); + expect(provider.searchConfig.validateUrl).toContain("/usage?limit=1"); + }); + + it("builds GET search with header auth and maps results", () => { + const request = buildSearchRequest({ id: "tinyfish", ...provider.searchConfig }, { + query: "latest release", searchType: "news", token: "secret", maxResults: 5, + country: "TR", language: "tr", domainFilter: ["example.com", "-other.com"], offset: 5, + }); + const parsed = new URL(request.url); + expect(Object.fromEntries(parsed.searchParams)).toEqual({ + query: "latest release", domain_type: "news", location: "TR", language: "tr", + include_domains: "example.com", exclude_domains: "other.com", page: "0", + }); + expect(request.url).not.toContain("secret"); + expect(request.init.headers["X-API-Key"]).toBe("secret"); + expect(buildSearchRequest({ id: "tinyfish", ...provider.searchConfig }, { + query: "release", searchType: "web", token: "secret", maxResults: 5, + providerOptions: { baseUrl: "https://example.org/collect" }, + }).url.startsWith(provider.searchConfig.baseUrl)).toBe(true); + const normalized = normalizeSearchResponse("tinyfish", { + total_results: 1, results: [{ title: "Article", url, snippet: "Preview", date: "2026-09-01" }], + }, "latest release", "news"); + expect(normalized.totalResults).toBe(1); + expect(normalized.results[0]).toMatchObject({ title: "Article", url, snippet: "Preview", published_at: "2026-09-01" }); + }); + + it("applies offset within TinyFish fixed-size result pages", async () => { + const entries = Array.from({ length: 10 }, (_, i) => ({ title: `Article ${i}`, url: `${url}/${i}`, snippet: "Preview" })); + vi.stubGlobal("fetch", vi.fn(async () => new Response(JSON.stringify({ results: entries, total_results: 10 }), { + headers: { "content-type": "application/json" }, + }))); + const result = await handleSearchCore({ + body: { query: "release", max_results: 2, offset: 3 }, + provider: { id: "tinyfish" }, providerConfig: provider.searchConfig, + credentials: { apiKey: "secret" }, + }); + expect(result.success).toBe(true); + expect(result.data.results.map((entry) => entry.title)).toEqual(["Article 3", "Article 4"]); + expect(new URL(vi.mocked(fetch).mock.calls[0][0]).searchParams.get("page")).toBe("0"); + }); + + it("rejects offsets that would silently omit requested results", () => { + const params = { query: "release", searchType: "web", token: "secret", maxResults: 5, offset: 8 }; + expect(() => buildSearchRequest({ id: "tinyfish", ...provider.searchConfig }, params)) + .toThrow("TinyFish search offset and max_results must fit within one page"); + }); + + it("maps fetched content and treats HTTP 200 per-URL errors as failures", async () => { + const fetchMock = vi.fn(async () => new Response(JSON.stringify({ + results: [{ url, title: "Article", text: "# Article", language: "en" }], errors: [], + }), { headers: { "content-type": "application/json" } })); + vi.stubGlobal("fetch", fetchMock); + const input = { url, provider: "tinyfish", providerConfig: provider.fetchConfig, credentials: { apiKey: "secret" } }; + const result = await handleFetchCore(input); + expect(result.data).toMatchObject({ provider: "tinyfish", title: "Article", content: { text: "# Article" }, metadata: { language: "en" } }); + expect(fetchMock.mock.calls[0][0]).toBe(provider.fetchConfig.baseUrl); + expect(fetchMock.mock.calls[0][1].headers["X-API-Key"]).toBe("secret"); + expect(JSON.parse(fetchMock.mock.calls[0][1].body)).toEqual({ urls: [url], format: "markdown" }); + + fetchMock.mockResolvedValueOnce(new Response(JSON.stringify({ results: [], errors: [{ url, error: "bot_blocked" }] }), { + headers: { "content-type": "application/json" }, + })); + expect(await handleFetchCore(input)).toMatchObject({ success: false, status: 502, error: "TinyFish fetch failed: bot_blocked" }); + + expect(await handleFetchCore({ ...input, format: "text" })).toMatchObject({ success: false, status: 400 }); + expect(fetchMock).toHaveBeenCalledTimes(2); + + fetchMock.mockResolvedValueOnce(new Response(JSON.stringify({ error: { message: "Invalid API key" } }), { + status: 401, headers: { "content-type": "application/json" }, + })); + expect(await handleFetchCore(input)).toMatchObject({ success: false, status: 401 }); + }); +});