fix(jina-reader): recover after transient errors and use JSON POST API
Clear stale provider error code and account lock after a successful web fetch (the core fetch handler never consumed the onRequestSuccess callback), switch Jina Reader to its documented JSON POST request, and parse the Title: metadata line before falling back to a Markdown heading.
This commit is contained in:
@@ -49,7 +49,10 @@ function truncate(text, max) {
|
||||
}
|
||||
|
||||
function parseJinaTitle(text) {
|
||||
const m = String(text || "").match(/^\s*#\s+(.+)$/m);
|
||||
const source = String(text || "");
|
||||
const metadataTitle = source.match(/^\s*Title:\s*(.+)$/mi);
|
||||
if (metadataTitle) return metadataTitle[1].trim();
|
||||
const m = source.match(/^\s*#\s+(.+)$/m);
|
||||
return m ? m[1].trim() : null;
|
||||
}
|
||||
|
||||
@@ -151,11 +154,14 @@ async function runFirecrawl({ url, fmt, timeoutMs, apiKey, maxCharacters, costPe
|
||||
}
|
||||
|
||||
async function runJina({ url, fmt, timeoutMs, apiKey, maxCharacters, costPerQuery, startedAt }) {
|
||||
const target = `https://r.jina.ai/${encodeURIComponent(url)}`;
|
||||
const upstreamStart = Date.now();
|
||||
const r = await tryFetch(target, {
|
||||
method: "GET",
|
||||
headers: apiKey ? { authorization: `Bearer ${apiKey}` } : {}
|
||||
const r = await tryFetch("https://r.jina.ai/", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
...(apiKey ? { authorization: `Bearer ${apiKey}` } : {})
|
||||
},
|
||||
body: JSON.stringify({ url })
|
||||
}, timeoutMs);
|
||||
|
||||
if (!r.ok) {
|
||||
|
||||
@@ -195,13 +195,11 @@ async function handleSingleProviderFetch(body, providerInput, request, apiKey, s
|
||||
providerSpecificData: newCreds.providerSpecificData,
|
||||
testStatus: "active"
|
||||
});
|
||||
},
|
||||
onRequestSuccess: async () => {
|
||||
await clearAccountError(credentials.connectionId, credentials);
|
||||
}
|
||||
});
|
||||
|
||||
if (result.success) {
|
||||
await clearAccountError(credentials.connectionId, credentials);
|
||||
return new Response(JSON.stringify(result.data), {
|
||||
headers: { "Content-Type": "application/json", "Access-Control-Allow-Origin": "*" }
|
||||
});
|
||||
|
||||
@@ -285,7 +285,13 @@ export async function clearAccountError(connectionId, currentConnection, model =
|
||||
|
||||
// Only reset error state if no active locks remain
|
||||
if (remainingActiveLocks.length === 0) {
|
||||
Object.assign(clearObj, { testStatus: "active", lastError: null, lastErrorAt: null, backoffLevel: 0 });
|
||||
Object.assign(clearObj, {
|
||||
testStatus: "active",
|
||||
lastError: null,
|
||||
errorCode: null,
|
||||
lastErrorAt: null,
|
||||
backoffLevel: 0
|
||||
});
|
||||
}
|
||||
|
||||
await updateProviderConnection(connectionId, clearObj);
|
||||
|
||||
91
tests/unit/fetch-success-clears-account.test.js
Normal file
91
tests/unit/fetch-success-clears-account.test.js
Normal file
@@ -0,0 +1,91 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
getProviderCredentials: vi.fn(),
|
||||
markAccountUnavailable: vi.fn(),
|
||||
clearAccountError: vi.fn(),
|
||||
extractApiKey: vi.fn(() => null),
|
||||
isValidApiKey: vi.fn(),
|
||||
getSettings: vi.fn(),
|
||||
getCombos: vi.fn(),
|
||||
handleFetchCore: vi.fn(),
|
||||
checkAndRefreshToken: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/sse/services/auth.js", () => ({
|
||||
getProviderCredentials: mocks.getProviderCredentials,
|
||||
markAccountUnavailable: mocks.markAccountUnavailable,
|
||||
clearAccountError: mocks.clearAccountError,
|
||||
extractApiKey: mocks.extractApiKey,
|
||||
isValidApiKey: mocks.isValidApiKey,
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/localDb", () => ({
|
||||
getSettings: mocks.getSettings,
|
||||
getCombos: mocks.getCombos,
|
||||
}));
|
||||
|
||||
vi.mock("open-sse/handlers/fetch/index.js", () => ({
|
||||
handleFetchCore: mocks.handleFetchCore,
|
||||
}));
|
||||
|
||||
vi.mock("@/sse/services/tokenRefresh.js", () => ({
|
||||
checkAndRefreshToken: mocks.checkAndRefreshToken,
|
||||
updateProviderCredentials: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("@/sse/utils/logger.js", () => ({
|
||||
request: vi.fn(),
|
||||
info: vi.fn(),
|
||||
debug: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
maskKey: vi.fn(() => "masked"),
|
||||
}));
|
||||
|
||||
vi.mock("@/shared/utils/ssrfGuard.js", () => ({
|
||||
assertPublicUrl: vi.fn(),
|
||||
}));
|
||||
|
||||
import { handleFetch } from "@/sse/handlers/fetch.js";
|
||||
|
||||
describe("web fetch account state", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mocks.getSettings.mockResolvedValue({ requireApiKey: false });
|
||||
mocks.getCombos.mockResolvedValue([]);
|
||||
mocks.getProviderCredentials.mockResolvedValue({
|
||||
apiKey: "jina-test-key",
|
||||
connectionId: "jina-connection",
|
||||
connectionName: "Jina Test",
|
||||
_connection: {
|
||||
testStatus: "unavailable",
|
||||
lastError: "old error",
|
||||
modelLock___all: "2026-01-01T00:00:00.000Z",
|
||||
},
|
||||
});
|
||||
mocks.checkAndRefreshToken.mockImplementation(async (_provider, credentials) => credentials);
|
||||
mocks.handleFetchCore.mockResolvedValue({
|
||||
success: true,
|
||||
data: { provider: "jina-reader", content: { text: "ok" } },
|
||||
});
|
||||
});
|
||||
|
||||
it("clears a stale provider lock after a successful fetch", async () => {
|
||||
const response = await handleFetch(new Request("http://localhost/v1/web/fetch", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
provider: "jina-reader",
|
||||
url: "https://example.com/article",
|
||||
}),
|
||||
}));
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(mocks.clearAccountError).toHaveBeenCalledWith(
|
||||
"jina-connection",
|
||||
expect.objectContaining({ connectionName: "Jina Test" }),
|
||||
);
|
||||
expect(mocks.markAccountUnavailable).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
66
tests/unit/jina-reader-fetch.test.js
Normal file
66
tests/unit/jina-reader-fetch.test.js
Normal file
@@ -0,0 +1,66 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { handleFetchCore } from "../../open-sse/handlers/fetch/index.js";
|
||||
|
||||
const originalFetch = global.fetch;
|
||||
|
||||
describe("Jina Reader fetch", () => {
|
||||
beforeEach(() => {
|
||||
global.fetch = vi.fn();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
global.fetch = originalFetch;
|
||||
});
|
||||
|
||||
it("uses Jina's JSON POST API instead of embedding the URL in the path", async () => {
|
||||
global.fetch.mockResolvedValueOnce(new Response([
|
||||
"Title: Example page",
|
||||
"",
|
||||
"URL Source: https://example.com/article",
|
||||
"",
|
||||
"Markdown Content:",
|
||||
"Hello",
|
||||
].join("\n")));
|
||||
|
||||
const result = await handleFetchCore({
|
||||
url: "https://example.com/article",
|
||||
format: "markdown",
|
||||
provider: "jina-reader",
|
||||
providerConfig: { timeoutMs: 30000 },
|
||||
credentials: { apiKey: "jina-test-key" },
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.data.title).toBe("Example page");
|
||||
expect(global.fetch).toHaveBeenCalledTimes(1);
|
||||
|
||||
const [requestUrl, init] = global.fetch.mock.calls[0];
|
||||
expect(requestUrl).toBe("https://r.jina.ai/");
|
||||
expect(init.method).toBe("POST");
|
||||
expect(init.headers).toEqual({
|
||||
"content-type": "application/json",
|
||||
authorization: "Bearer jina-test-key",
|
||||
});
|
||||
expect(JSON.parse(init.body)).toEqual({ url: "https://example.com/article" });
|
||||
});
|
||||
|
||||
it("returns the upstream status and error body", async () => {
|
||||
global.fetch.mockResolvedValueOnce(new Response(
|
||||
JSON.stringify({ detail: "Payment required" }),
|
||||
{ status: 402, headers: { "Content-Type": "application/json" } },
|
||||
));
|
||||
|
||||
const result = await handleFetchCore({
|
||||
url: "https://example.com/article",
|
||||
provider: "jina-reader",
|
||||
providerConfig: { timeoutMs: 30000 },
|
||||
credentials: { apiKey: "jina-test-key" },
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
success: false,
|
||||
status: 402,
|
||||
});
|
||||
expect(result.error).toContain("Payment required");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user