Merge remote-tracking branch 'origin/master' into gitea/new_feature

# Conflicts: # open-sse/executors/qoder.js # open-sse/handlers/chatCore.js # open-sse/handlers/chatCore/sseToJsonHandler.js # open-sse/providers/registry/commandcode.js # src/app/(dashboard)/dashboard/combos/page.js # src/app/api/v1/models/route.js # src/lib/db/repos/usageRepo.js # src/shared/components/UsageStats.js
2026-09-25 10:25:56 +07:00
parent 11089ab137 39e36d3d0c
commit 272dbcb9cc
154 changed files with 10246 additions and 1241 deletions
--- a/open-sse/handlers/chatCore.js
+++ b/open-sse/handlers/chatCore.js
@@ -20,6 +20,7 @@ import { handleNonStreamingResponse } from "./chatCore/nonStreamingHandler.js";
 import { handleStreamingResponse, buildOnStreamComplete } from "./chatCore/streamingHandler.js";
 import { detectClientTool, isNativePassthrough } from "../utils/clientDetector.js";
 import { dedupeTools } from "../utils/toolDeduper.js";
+import { takeRenamedToolNames } from "../utils/opencodeFingerprint.js";
 import { injectCaveman } from "../rtk/caveman.js";
 import { injectPonytail } from "../rtk/ponytail.js";
 import { compressMessages, formatRtkLog } from "../rtk/index.js";
@@ -116,6 +117,19 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred
    }
  }

+  // Per-request opt-out: client can bypass all token savers via header
+  const tokenSaverEnabled = clientRawRequest?.headers?.[TOKEN_SAVER_HEADER]?.toLowerCase() !== "off";
+
+  // Cursor's translator rewrites tool_result into user text, so RTK must run on
+  // the source body before translation. Every other pair translates the tool
+  // shapes 1:1 — keep the post-translate pass there so those providers are
+  // untouched (and a retry never re-compresses an already-compressed body).
+  const preTranslateRtk = provider === "cursor"
+    ? compressMessages(body, tokenSaverEnabled && rtkEnabled)
+    : null;
+  const preTranslateRtkLine = formatRtkLog(preTranslateRtk);
+  if (preTranslateRtkLine) console.log(preTranslateRtkLine);
+
  const clientRequestedStreaming = body.stream === true || sourceFormat === FORMATS.ANTIGRAVITY || sourceFormat === FORMATS.GEMINI || sourceFormat === FORMATS.GEMINI_CLI;
  const providerRequiresStreaming = PROVIDERS[provider]?.forceStream === true;
  let stream = providerRequiresStreaming ? true : (body.stream !== false);
@@ -254,11 +268,8 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred
    translatedBody.tools = defaultClaudeToolType(translatedBody.tools);
  }

-  // Per-request opt-out: client can bypass all token savers via header
-  const tokenSaverEnabled = clientRawRequest?.headers?.[TOKEN_SAVER_HEADER]?.toLowerCase() !== "off";
-
-  // RTK: compress tool_result content
-  const rtkStats = compressMessages(translatedBody, tokenSaverEnabled && rtkEnabled);
+  // RTK: compress tool_result content. Skipped when already done pre-translate.
+  const rtkStats = preTranslateRtk || compressMessages(translatedBody, tokenSaverEnabled && rtkEnabled);
  const rtkLine = formatRtkLog(rtkStats);
  if (rtkLine) log?.info?.("RTK", rtkLine.replace(/^\[RTK\] /, ""));

@@ -277,6 +288,8 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred
  // Token-saver flags accumulator for the single "⚙" log line below.
  const xf = [];

+  if (rtkStats?.hits?.length) xf.push(`RTK:${rtkStats.hits.length}`);
+
  // Caveman: inject terse-style system prompt
  if (tokenSaverEnabled && cavemanEnabled && cavemanLevel) {
    injectCaveman(translatedBody, finalFormat, cavemanLevel);
@@ -378,6 +391,10 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred
    providerHeaders = result.headers;
    finalBody = result.transformedBody;
    providerResponseFormat = result.responseFormat || targetFormat;
+    const renamedToolNames = takeRenamedToolNames(translatedBody);
+    if (renamedToolNames?.size) {
+      toolNameMap = new Map([...(toolNameMap || []), ...renamedToolNames]);
+    }
    reqLogger.logTargetRequest(providerUrl, providerHeaders, finalBody);
  } catch (error) {
    trackPendingRequest(model, provider, connectionId, false, true);
@@ -508,7 +525,7 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred

  // Provider forced streaming but client wants JSON
  if (!clientRequestedStreaming && providerRequiresStreaming) {
-    const result = await handleForcedSSEToJson({ ...sharedCtx, providerResponse, sourceFormat, targetFormat: providerResponseFormat, customToolNames, trackDone, appendLog });
+    const result = await handleForcedSSEToJson({ ...sharedCtx, providerResponse, sourceFormat, targetFormat: providerResponseFormat, customToolNames, toolNameMap, trackDone, appendLog });
    if (result) { streamController.handleComplete(); return result; }
  }

--- a/open-sse/handlers/chatCore/nonStreamingHandler.js
+++ b/open-sse/handlers/chatCore/nonStreamingHandler.js
@@ -11,6 +11,7 @@ import { buildRequestDetail, extractRequestConfig, extractUsageFromResponse, sav
 import { saveRequestDetail } from "@/lib/usageDb.js";
 import { matchStreamErrorPatterns } from "../../utils/streamErrorPatterns.js";
 import { decloakToolNames } from "../../utils/claudeCloaking.js";
+import { restoreToolNames } from "../../utils/opencodeFingerprint.js";
 import { ROLE, RESPONSES_ITEM } from "../../translator/schema/index.js";

 function parseToolArguments(value) {
@@ -415,7 +416,7 @@ export async function handleNonStreamingResponse({ providerResponse, provider, m

  return {
    success: true,
-    response: new Response(JSON.stringify(translatedResponse), {
+    response: new Response(JSON.stringify(restoreToolNames(translatedResponse, toolNameMap)), {
      headers: { "Content-Type": "application/json", "Access-Control-Allow-Origin": "*" }
    })
  };
--- a/open-sse/handlers/chatCore/sseToJsonHandler.js
+++ b/open-sse/handlers/chatCore/sseToJsonHandler.js
@@ -1,5 +1,6 @@
 import { convertResponsesStreamToJson } from "../../transformer/streamToJsonConverter.js";
 import { matchStreamErrorPatterns } from "../../utils/streamErrorPatterns.js";
+import { restoreToolNames } from "../../utils/opencodeFingerprint.js";
 import { createErrorResult } from "../../utils/error.js";
 import { HTTP_STATUS } from "../../config/runtimeConfig.js";
 import { FORMATS } from "../../translator/formats.js";
@@ -215,17 +216,13 @@ export async function handleForcedSSEToJson({
 	clientRawRequest,
 	onRequestSuccess,
 	customToolNames,
+	toolNameMap,
 	trackDone,
 	appendLog,
 	reqTag,
 	log,
 	streamErrorPatterns,
 }) {
-	const contentType = providerResponse.headers.get("content-type") || "";
-	const isSSE =
-		contentType.includes("text/event-stream") ||
-		(contentType === "" && isResponsesProvider(provider));
-	if (!isSSE) return null; // not handled here

 	trackDone();

@@ -306,7 +303,7 @@ export async function handleForcedSSEToJson({
 			if (sourceFormat === FORMATS.OPENAI_RESPONSES) {
 				return {
 					success: true,
-					response: new Response(JSON.stringify(jsonResponse), {
+					response: new Response(JSON.stringify(restoreToolNames(jsonResponse, toolNameMap)), {
 						headers: {
 							"Content-Type": "application/json",
 							"Access-Control-Allow-Origin": "*",
@@ -406,7 +403,7 @@ export async function handleForcedSSEToJson({

 			return {
 				success: true,
-				response: new Response(JSON.stringify(finalResp), {
+				response: new Response(JSON.stringify(restoreToolNames(finalResp, toolNameMap)), {
 					headers: {
 						"Content-Type": "application/json",
 						"Access-Control-Allow-Origin": "*",
@@ -432,8 +429,16 @@ export async function handleForcedSSEToJson({
 				"Invalid SSE response for non-streaming request",
 			);
 		if (parsed.error) {
+			// Structured error chunks may carry the real upstream status (e.g. the
+			// Qoder executor emits status 403 for billing envelopes). Preserve it so
+			// the account loop locks/falls back on the right status instead of a
+			// generic 502. Anything outside 400-599 still maps to 502.
+			const upstreamStatus = Number(parsed.error.status);
+			const status = Number.isInteger(upstreamStatus) && upstreamStatus >= 400 && upstreamStatus <= 599
+				? upstreamStatus
+				: HTTP_STATUS.BAD_GATEWAY;
 			return createErrorResult(
-				HTTP_STATUS.BAD_GATEWAY,
+				status,
 				parsed.error.message || "Upstream SSE stream failed",
 			);
 		}
@@ -508,7 +513,7 @@ export async function handleForcedSSEToJson({

 		return {
 			success: true,
-			response: new Response(JSON.stringify(finalBody), {
+			response: new Response(JSON.stringify(restoreToolNames(finalBody, toolNameMap)), {
 				headers: {
 					"Content-Type": "application/json",
 					"Access-Control-Allow-Origin": "*",
--- a/open-sse/handlers/imageProviders/huggingface.js
+++ b/open-sse/handlers/imageProviders/huggingface.js
@@ -1,18 +1,93 @@
-// HuggingFace Inference API — returns binary image
-import { nowSec } from "./_base.js";
+// HuggingFace Inference Providers router — returns binary image
+//
+// The router is a switchboard in front of many inference providers and is
+// addressed as `<baseUrl>/<provider>/<providerModelId>`. `providerModelId` is
+// the id the *provider* uses, which is not the Hub model id, so it is resolved
+// through `imageConfig.modelMap` (built from the Hub API's
+// inferenceProviderMapping and limited to providers the router forwards to).
+//
+// The legacy `api-inference.huggingface.co` host is gone (DNS ENOTFOUND) and is
+// deliberately not referenced anywhere here.
+import { nowSec, urlToBase64 } from "./_base.js";
 import { PROVIDER_MEDIA } from "../../providers/index.js";

-const BASE_URL = PROVIDER_MEDIA["huggingface"]?.imageConfig?.baseUrl;
+const imageConfig = () => PROVIDER_MEDIA["huggingface"]?.imageConfig || {};
+const BASE_URL = imageConfig().baseUrl;
+const MODEL_MAP = imageConfig().modelMap || {};
+
+// A plain-object lookup returns inherited truthy values for keys like "toString" or
+// "constructor", which would build nonsense URLs. Resolve own keys only.
+const lookup = (model) => (Object.hasOwn(MODEL_MAP, model) ? MODEL_MAP[model] : undefined);
+
+// modelMap values are either a bare path (text-to-image) or { path, task }.
+const mappingPath = (entry) => (typeof entry === "string" ? entry : entry.path);
+const mappingTask = (entry) => (typeof entry === "string" ? "text-to-image" : entry.task || "text-to-image");
+
+// A connection may point at its own endpoint (self-hosted Text Generation
+// Inference / TGI container). That endpoint already knows its own model ids, so
+// the router mapping does not apply and the Hub id is passed through verbatim.
+function customBaseUrl(creds) {
+  const url = creds?.providerSpecificData?.baseUrl;
+  return typeof url === "string" && url.trim() ? url.trim().replace(/\/+$/, "") : null;
+}
+
+// The router's image-to-image payload wants raw base64 — not a data URL, not a URL.
+// Accept every shape our own callers use (data URL, bare base64, remote URL, array).
+async function sourceImage(body) {
+  const raw = body?.image || (Array.isArray(body?.images) ? body.images[0] : null);
+  if (typeof raw !== "string" || !raw.trim()) return null;
+  const value = raw.trim();
+  if (/^https?:\/\//i.test(value)) return await urlToBase64(value);
+  const match = /^data:image\/[^;]+;base64,(.+)$/i.exec(value);
+  return match ? match[1] : value;
+}

 export default {
-  buildUrl: (model) => `${BASE_URL}/${model}`,
+  buildUrl: (model, creds) => {
+    const override = customBaseUrl(creds);
+    if (override) {
+      // The model id is client-controlled; on a custom endpoint it lands in a URL
+      // path verbatim, so reject traversal/query injection (mirrors sttCore's guard).
+      if (model.includes("..") || model.includes("//") || /[?#]/.test(model)) {
+        throw new Error(`HuggingFace: invalid model ID "${model}"`);
+      }
+      return `${override}/${model}`;
+    }
+
+    const entry = lookup(model);
+    if (!entry) {
+      throw new Error(
+        `HuggingFace: no HuggingFace router mapping for model "${model}". ` +
+          `Add it to imageConfig.modelMap in open-sse/providers/registry/huggingface.js, ` +
+          `or set a custom base URL on the connection.`
+      );
+    }
+    return `${BASE_URL}/${mappingPath(entry)}`;
+  },
  buildHeaders: (creds) => {
    const headers = { "Content-Type": "application/json" };
    const key = creds?.apiKey || creds?.accessToken;
    if (key) headers["Authorization"] = `Bearer ${key}`;
    return headers;
  },
-  buildBody: (_model, body) => ({ inputs: body.prompt }),
+  buildBody: async (model, body) => {
+    const entry = lookup(model);
+    const task = mappingTask(entry || "");
+
+    if (task === "image-to-image") {
+      const image = await sourceImage(body);
+      if (!image) {
+        throw new Error(
+          `HuggingFace: model "${model}" requires a source image. ` +
+            `Send it as "image" (or "images") in the request body.`
+        );
+      }
+      // inputs carries the source image; the prompt moves under parameters.
+      return { inputs: image, parameters: { prompt: body.prompt } };
+    }
+
+    return { inputs: body.prompt };
+  },
  // HF returns raw image bytes — convert to b64_json
  async parseResponse(response) {
    const buf = await response.arrayBuffer();
--- a/open-sse/handlers/systemoneCore.js
+++ b/open-sse/handlers/systemoneCore.js
@@ -0,0 +1,95 @@
+import { createErrorResult, parseUpstreamError, formatProviderError } from "../utils/error.js";
+import { HTTP_STATUS, FETCH_CONNECT_TIMEOUT_MS } from "../config/runtimeConfig.js";
+import { PROVIDER_MEDIA } from "../providers/index.js";
+import { generateSessionId } from "../executors/opencode-zen.js";
+
+/**
+ * Core System One (Jev) handler — native decision payload pass-through.
+ * URL/headers come from the registry's systemoneConfig; body and JSON response
+ * are forwarded untouched (decision models have no chat translation layer).
+ *
+ * @returns {Promise<{ success: boolean, response: Response, usage?: object, status?: number, error?: string }>}
+ */
+export async function handleSystemoneCore({
+  body,
+  modelInfo,
+  credentials,
+  log,
+  onRequestSuccess,
+}) {
+  const { provider, model } = modelInfo;
+  const cfg = PROVIDER_MEDIA[provider]?.systemoneConfig;
+  if (!cfg?.baseUrl) {
+    return createErrorResult(
+      HTTP_STATUS.BAD_REQUEST,
+      `Provider '${provider}' does not support System One.`
+    );
+  }
+
+  // Validate input at the trust boundary; question-level shape is upstream's job.
+  if (body.state === undefined || body.state === null) {
+    return createErrorResult(HTTP_STATUS.BAD_REQUEST, "Missing required field: state");
+  }
+  if (!body.questions || typeof body.questions !== "object" || Array.isArray(body.questions)) {
+    return createErrorResult(HTTP_STATUS.BAD_REQUEST, "Missing required field: questions");
+  }
+
+  // noAuth free lanes carry accessToken "public" from the credential stub.
+  const token = credentials?.apiKey || credentials?.accessToken;
+  const headers = {
+    "Content-Type": "application/json",
+    ...(token ? { Authorization: `Bearer ${token}` } : {}),
+    ...(cfg.headers || {}),
+    // Zen lanes expect the official client session header on every request.
+    "x-opencode-session": generateSessionId(),
+  };
+  const requestBody = { ...body, model };
+
+  log?.debug?.("SYSTEMONE", `${provider.toUpperCase()} | ${model}`);
+
+  let providerResponse;
+  try {
+    providerResponse = await fetch(cfg.baseUrl, {
+      method: "POST",
+      headers,
+      body: JSON.stringify(requestBody),
+      ...(typeof AbortSignal?.timeout === "function"
+        ? { signal: AbortSignal.timeout(FETCH_CONNECT_TIMEOUT_MS) }
+        : {}),
+    });
+  } catch (error) {
+    const errMsg = formatProviderError(error, provider, model, HTTP_STATUS.BAD_GATEWAY);
+    log?.debug?.("SYSTEMONE", `Fetch error: ${errMsg}`);
+    return createErrorResult(HTTP_STATUS.BAD_GATEWAY, errMsg);
+  }
+
+  if (!providerResponse.ok) {
+    const { statusCode, message } = await parseUpstreamError(providerResponse);
+    const errMsg = formatProviderError(new Error(message), provider, model, statusCode);
+    log?.debug?.("SYSTEMONE", `Provider error: ${errMsg}`);
+    return createErrorResult(statusCode, errMsg);
+  }
+
+  let responseBody;
+  try {
+    responseBody = await providerResponse.json();
+  } catch {
+    return createErrorResult(HTTP_STATUS.BAD_GATEWAY, `Invalid JSON response from ${provider}`);
+  }
+
+  if (onRequestSuccess) await onRequestSuccess();
+
+  const usage = responseBody?.usage;
+  return {
+    success: true,
+    usage: usage
+      ? { prompt_tokens: usage.input_tokens || 0, completion_tokens: usage.output_tokens || 0 }
+      : null,
+    response: new Response(JSON.stringify(responseBody), {
+      headers: {
+        "Content-Type": "application/json",
+        "Access-Control-Allow-Origin": "*",
+      },
+    }),
+  };
+}