From 3c17d3406b974926f745bb6f1cc4543d5fbdd4aa Mon Sep 17 00:00:00 2001 From: jacardl Date: Thu, 23 Jul 2026 16:24:07 +0700 Subject: [PATCH] fix(jina-reader): recover after transient errors and use JSON POST API Clear stale provider error code and account lock after a successful web fetch (the core fetch handler never consumed the onRequestSuccess callback), switch Jina Reader to its documented JSON POST request, and parse the Title: metadata line before falling back to a Markdown heading. --- open-sse/handlers/fetch/index.js | 16 +++- src/sse/handlers/fetch.js | 4 +- src/sse/services/auth.js | 8 +- .../unit/fetch-success-clears-account.test.js | 91 +++++++++++++++++++ tests/unit/jina-reader-fetch.test.js | 66 ++++++++++++++ 5 files changed, 176 insertions(+), 9 deletions(-) create mode 100644 tests/unit/fetch-success-clears-account.test.js create mode 100644 tests/unit/jina-reader-fetch.test.js diff --git a/open-sse/handlers/fetch/index.js b/open-sse/handlers/fetch/index.js index da1c2303..187bd60e 100644 --- a/open-sse/handlers/fetch/index.js +++ b/open-sse/handlers/fetch/index.js @@ -49,7 +49,10 @@ function truncate(text, max) { } function parseJinaTitle(text) { - const m = String(text || "").match(/^\s*#\s+(.+)$/m); + const source = String(text || ""); + const metadataTitle = source.match(/^\s*Title:\s*(.+)$/mi); + if (metadataTitle) return metadataTitle[1].trim(); + const m = source.match(/^\s*#\s+(.+)$/m); return m ? m[1].trim() : null; } @@ -151,11 +154,14 @@ async function runFirecrawl({ url, fmt, timeoutMs, apiKey, maxCharacters, costPe } async function runJina({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery, startedAt }) { - const target = `https://r.jina.ai/${encodeURIComponent(url)}`; const upstreamStart = Date.now(); - const r = await tryFetch(target, { - method: "GET", - headers: apiKey ? { authorization: `Bearer ${apiKey}` } : {} + const r = await tryFetch("https://r.jina.ai/", { + method: "POST", + headers: { + "content-type": "application/json", + ...(apiKey ? { authorization: `Bearer ${apiKey}` } : {}) + }, + body: JSON.stringify({ url }) }, timeoutMs); if (!r.ok) { diff --git a/src/sse/handlers/fetch.js b/src/sse/handlers/fetch.js index db62a8c7..0005095c 100644 --- a/src/sse/handlers/fetch.js +++ b/src/sse/handlers/fetch.js @@ -195,13 +195,11 @@ async function handleSingleProviderFetch(body, providerInput, request, apiKey, s providerSpecificData: newCreds.providerSpecificData, testStatus: "active" }); - }, - onRequestSuccess: async () => { - await clearAccountError(credentials.connectionId, credentials); } }); if (result.success) { + await clearAccountError(credentials.connectionId, credentials); return new Response(JSON.stringify(result.data), { headers: { "Content-Type": "application/json", "Access-Control-Allow-Origin": "*" } }); diff --git a/src/sse/services/auth.js b/src/sse/services/auth.js index f931209b..36fd6c49 100644 --- a/src/sse/services/auth.js +++ b/src/sse/services/auth.js @@ -285,7 +285,13 @@ export async function clearAccountError(connectionId, currentConnection, model = // Only reset error state if no active locks remain if (remainingActiveLocks.length === 0) { - Object.assign(clearObj, { testStatus: "active", lastError: null, lastErrorAt: null, backoffLevel: 0 }); + Object.assign(clearObj, { + testStatus: "active", + lastError: null, + errorCode: null, + lastErrorAt: null, + backoffLevel: 0 + }); } await updateProviderConnection(connectionId, clearObj); diff --git a/tests/unit/fetch-success-clears-account.test.js b/tests/unit/fetch-success-clears-account.test.js new file mode 100644 index 00000000..eb8aa267 --- /dev/null +++ b/tests/unit/fetch-success-clears-account.test.js @@ -0,0 +1,91 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + getProviderCredentials: vi.fn(), + markAccountUnavailable: vi.fn(), + clearAccountError: vi.fn(), + extractApiKey: vi.fn(() => null), + isValidApiKey: vi.fn(), + getSettings: vi.fn(), + getCombos: vi.fn(), + handleFetchCore: vi.fn(), + checkAndRefreshToken: vi.fn(), +})); + +vi.mock("@/sse/services/auth.js", () => ({ + getProviderCredentials: mocks.getProviderCredentials, + markAccountUnavailable: mocks.markAccountUnavailable, + clearAccountError: mocks.clearAccountError, + extractApiKey: mocks.extractApiKey, + isValidApiKey: mocks.isValidApiKey, +})); + +vi.mock("@/lib/localDb", () => ({ + getSettings: mocks.getSettings, + getCombos: mocks.getCombos, +})); + +vi.mock("open-sse/handlers/fetch/index.js", () => ({ + handleFetchCore: mocks.handleFetchCore, +})); + +vi.mock("@/sse/services/tokenRefresh.js", () => ({ + checkAndRefreshToken: mocks.checkAndRefreshToken, + updateProviderCredentials: vi.fn(), +})); + +vi.mock("@/sse/utils/logger.js", () => ({ + request: vi.fn(), + info: vi.fn(), + debug: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + maskKey: vi.fn(() => "masked"), +})); + +vi.mock("@/shared/utils/ssrfGuard.js", () => ({ + assertPublicUrl: vi.fn(), +})); + +import { handleFetch } from "@/sse/handlers/fetch.js"; + +describe("web fetch account state", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.getSettings.mockResolvedValue({ requireApiKey: false }); + mocks.getCombos.mockResolvedValue([]); + mocks.getProviderCredentials.mockResolvedValue({ + apiKey: "jina-test-key", + connectionId: "jina-connection", + connectionName: "Jina Test", + _connection: { + testStatus: "unavailable", + lastError: "old error", + modelLock___all: "2026-01-01T00:00:00.000Z", + }, + }); + mocks.checkAndRefreshToken.mockImplementation(async (_provider, credentials) => credentials); + mocks.handleFetchCore.mockResolvedValue({ + success: true, + data: { provider: "jina-reader", content: { text: "ok" } }, + }); + }); + + it("clears a stale provider lock after a successful fetch", async () => { + const response = await handleFetch(new Request("http://localhost/v1/web/fetch", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + provider: "jina-reader", + url: "https://example.com/article", + }), + })); + + expect(response.status).toBe(200); + expect(mocks.clearAccountError).toHaveBeenCalledWith( + "jina-connection", + expect.objectContaining({ connectionName: "Jina Test" }), + ); + expect(mocks.markAccountUnavailable).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/unit/jina-reader-fetch.test.js b/tests/unit/jina-reader-fetch.test.js new file mode 100644 index 00000000..3a6512da --- /dev/null +++ b/tests/unit/jina-reader-fetch.test.js @@ -0,0 +1,66 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { handleFetchCore } from "../../open-sse/handlers/fetch/index.js"; + +const originalFetch = global.fetch; + +describe("Jina Reader fetch", () => { + beforeEach(() => { + global.fetch = vi.fn(); + }); + + afterEach(() => { + global.fetch = originalFetch; + }); + + it("uses Jina's JSON POST API instead of embedding the URL in the path", async () => { + global.fetch.mockResolvedValueOnce(new Response([ + "Title: Example page", + "", + "URL Source: https://example.com/article", + "", + "Markdown Content:", + "Hello", + ].join("\n"))); + + const result = await handleFetchCore({ + url: "https://example.com/article", + format: "markdown", + provider: "jina-reader", + providerConfig: { timeoutMs: 30000 }, + credentials: { apiKey: "jina-test-key" }, + }); + + expect(result.success).toBe(true); + expect(result.data.title).toBe("Example page"); + expect(global.fetch).toHaveBeenCalledTimes(1); + + const [requestUrl, init] = global.fetch.mock.calls[0]; + expect(requestUrl).toBe("https://r.jina.ai/"); + expect(init.method).toBe("POST"); + expect(init.headers).toEqual({ + "content-type": "application/json", + authorization: "Bearer jina-test-key", + }); + expect(JSON.parse(init.body)).toEqual({ url: "https://example.com/article" }); + }); + + it("returns the upstream status and error body", async () => { + global.fetch.mockResolvedValueOnce(new Response( + JSON.stringify({ detail: "Payment required" }), + { status: 402, headers: { "Content-Type": "application/json" } }, + )); + + const result = await handleFetchCore({ + url: "https://example.com/article", + provider: "jina-reader", + providerConfig: { timeoutMs: 30000 }, + credentials: { apiKey: "jina-test-key" }, + }); + + expect(result).toMatchObject({ + success: false, + status: 402, + }); + expect(result.error).toContain("Payment required"); + }); +});