fix(jina-reader): recover after transient errors and use JSON POST API

Clear stale provider error code and account lock after a successful web
fetch (the core fetch handler never consumed the onRequestSuccess
callback), switch Jina Reader to its documented JSON POST request, and
parse the Title: metadata line before falling back to a Markdown heading.
This commit is contained in:
jacardl
2026-07-23 16:24:07 +07:00
parent 007d372724
commit 3c17d3406b
5 changed files with 176 additions and 9 deletions

View File

@@ -49,7 +49,10 @@ function truncate(text, max) {
}
function parseJinaTitle(text) {
const m = String(text || "").match(/^\s*#\s+(.+)$/m);
const source = String(text || "");
const metadataTitle = source.match(/^\s*Title:\s*(.+)$/mi);
if (metadataTitle) return metadataTitle[1].trim();
const m = source.match(/^\s*#\s+(.+)$/m);
return m ? m[1].trim() : null;
}
@@ -151,11 +154,14 @@ async function runFirecrawl({ url, fmt, timeoutMs, apiKey, maxCharacters, costPe
}
async function runJina({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery, startedAt }) {
const target = `https://r.jina.ai/${encodeURIComponent(url)}`;
const upstreamStart = Date.now();
const r = await tryFetch(target, {
method: "GET",
headers: apiKey ? { authorization: `Bearer ${apiKey}` } : {}
const r = await tryFetch("https://r.jina.ai/", {
method: "POST",
headers: {
"content-type": "application/json",
...(apiKey ? { authorization: `Bearer ${apiKey}` } : {})
},
body: JSON.stringify({ url })
}, timeoutMs);
if (!r.ok) {

View File

@@ -195,13 +195,11 @@ async function handleSingleProviderFetch(body, providerInput, request, apiKey, s
providerSpecificData: newCreds.providerSpecificData,
testStatus: "active"
});
},
onRequestSuccess: async () => {
await clearAccountError(credentials.connectionId, credentials);
}
});
if (result.success) {
await clearAccountError(credentials.connectionId, credentials);
return new Response(JSON.stringify(result.data), {
headers: { "Content-Type": "application/json", "Access-Control-Allow-Origin": "*" }
});

View File

@@ -285,7 +285,13 @@ export async function clearAccountError(connectionId, currentConnection, model =
// Only reset error state if no active locks remain
if (remainingActiveLocks.length === 0) {
Object.assign(clearObj, { testStatus: "active", lastError: null, lastErrorAt: null, backoffLevel: 0 });
Object.assign(clearObj, {
testStatus: "active",
lastError: null,
errorCode: null,
lastErrorAt: null,
backoffLevel: 0
});
}
await updateProviderConnection(connectionId, clearObj);

View File

@@ -0,0 +1,91 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
getProviderCredentials: vi.fn(),
markAccountUnavailable: vi.fn(),
clearAccountError: vi.fn(),
extractApiKey: vi.fn(() => null),
isValidApiKey: vi.fn(),
getSettings: vi.fn(),
getCombos: vi.fn(),
handleFetchCore: vi.fn(),
checkAndRefreshToken: vi.fn(),
}));
vi.mock("@/sse/services/auth.js", () => ({
getProviderCredentials: mocks.getProviderCredentials,
markAccountUnavailable: mocks.markAccountUnavailable,
clearAccountError: mocks.clearAccountError,
extractApiKey: mocks.extractApiKey,
isValidApiKey: mocks.isValidApiKey,
}));
vi.mock("@/lib/localDb", () => ({
getSettings: mocks.getSettings,
getCombos: mocks.getCombos,
}));
vi.mock("open-sse/handlers/fetch/index.js", () => ({
handleFetchCore: mocks.handleFetchCore,
}));
vi.mock("@/sse/services/tokenRefresh.js", () => ({
checkAndRefreshToken: mocks.checkAndRefreshToken,
updateProviderCredentials: vi.fn(),
}));
vi.mock("@/sse/utils/logger.js", () => ({
request: vi.fn(),
info: vi.fn(),
debug: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
maskKey: vi.fn(() => "masked"),
}));
vi.mock("@/shared/utils/ssrfGuard.js", () => ({
assertPublicUrl: vi.fn(),
}));
import { handleFetch } from "@/sse/handlers/fetch.js";
describe("web fetch account state", () => {
beforeEach(() => {
vi.clearAllMocks();
mocks.getSettings.mockResolvedValue({ requireApiKey: false });
mocks.getCombos.mockResolvedValue([]);
mocks.getProviderCredentials.mockResolvedValue({
apiKey: "jina-test-key",
connectionId: "jina-connection",
connectionName: "Jina Test",
_connection: {
testStatus: "unavailable",
lastError: "old error",
modelLock___all: "2026-01-01T00:00:00.000Z",
},
});
mocks.checkAndRefreshToken.mockImplementation(async (_provider, credentials) => credentials);
mocks.handleFetchCore.mockResolvedValue({
success: true,
data: { provider: "jina-reader", content: { text: "ok" } },
});
});
it("clears a stale provider lock after a successful fetch", async () => {
const response = await handleFetch(new Request("http://localhost/v1/web/fetch", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
provider: "jina-reader",
url: "https://example.com/article",
}),
}));
expect(response.status).toBe(200);
expect(mocks.clearAccountError).toHaveBeenCalledWith(
"jina-connection",
expect.objectContaining({ connectionName: "Jina Test" }),
);
expect(mocks.markAccountUnavailable).not.toHaveBeenCalled();
});
});

View File

@@ -0,0 +1,66 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { handleFetchCore } from "../../open-sse/handlers/fetch/index.js";
const originalFetch = global.fetch;
describe("Jina Reader fetch", () => {
beforeEach(() => {
global.fetch = vi.fn();
});
afterEach(() => {
global.fetch = originalFetch;
});
it("uses Jina's JSON POST API instead of embedding the URL in the path", async () => {
global.fetch.mockResolvedValueOnce(new Response([
"Title: Example page",
"",
"URL Source: https://example.com/article",
"",
"Markdown Content:",
"Hello",
].join("\n")));
const result = await handleFetchCore({
url: "https://example.com/article",
format: "markdown",
provider: "jina-reader",
providerConfig: { timeoutMs: 30000 },
credentials: { apiKey: "jina-test-key" },
});
expect(result.success).toBe(true);
expect(result.data.title).toBe("Example page");
expect(global.fetch).toHaveBeenCalledTimes(1);
const [requestUrl, init] = global.fetch.mock.calls[0];
expect(requestUrl).toBe("https://r.jina.ai/");
expect(init.method).toBe("POST");
expect(init.headers).toEqual({
"content-type": "application/json",
authorization: "Bearer jina-test-key",
});
expect(JSON.parse(init.body)).toEqual({ url: "https://example.com/article" });
});
it("returns the upstream status and error body", async () => {
global.fetch.mockResolvedValueOnce(new Response(
JSON.stringify({ detail: "Payment required" }),
{ status: 402, headers: { "Content-Type": "application/json" } },
));
const result = await handleFetchCore({
url: "https://example.com/article",
provider: "jina-reader",
providerConfig: { timeoutMs: 30000 },
credentials: { apiKey: "jina-test-key" },
});
expect(result).toMatchObject({
success: false,
status: 402,
});
expect(result.error).toContain("Payment required");
});
});