Merge remote-tracking branch 'origin/master' into gitea/feature/end

Resolved conflicts taking origin/master (v0.5.55) as canonical, with local
features re-applied:
- runtime log level (LOG_LEVEL env + dashboard Settings → Logging, applied
  immediately and persisted across restarts)
- free/noAuth provider enable/disable toggle via providerStrategies.enabled
- parallel model testing (Test All Models / Test Selected Keys)
This commit is contained in:
2026-08-17 00:38:31 +07:00
parent e7470e955e
commit 7e45ead2ac
557 changed files with 53396 additions and 6935 deletions

View File

@@ -1,9 +1,10 @@
// Port of auto_detect_filter (rtk/src/cmds/system/pipe_cmd.rs:132-188) + JS extras
// Order: git-diff → git-status → build-output → grep → find → tree → ls → search-list
// → read-numbered → dedup-log → smart-truncate → null
// Detection order: git-log → git-diff → git-status → build-output → grep → find → tree → ls → search-list
// → read-numbered → dedup-log → smart-truncate → null
import { DETECT_WINDOW, READ_NUMBERED_MIN_HIT_RATIO, SMART_TRUNCATE_MIN_LINES } from "./constants.js";
import { gitDiff } from "./filters/gitDiff.js";
import { gitStatus } from "./filters/gitStatus.js";
import { gitLog } from "./filters/gitLog.js";
import { buildOutput } from "./filters/buildOutput.js";
import { grep } from "./filters/grep.js";
import { find } from "./filters/find.js";
@@ -17,6 +18,7 @@ import { searchList, SEARCH_LIST_HEADER_RE } from "./filters/searchList.js";
const RE_GIT_DIFF = /^diff --git /m;
const RE_GIT_DIFF_HUNK = /^@@ /m;
const RE_GIT_STATUS = /^On branch |^nothing to commit|^Changes (not |to be )|^Untracked files:/m;
const RE_GIT_LOG = /^[*|/\\ ]*commit [0-9a-f]{7,40}$/m;
const RE_PORCELAIN = /^[ MADRCU?!][ MADRCU?!] \S/m;
const RE_BUILD_OUTPUT = /^(npm (warn|error|ERR!)|yarn (warn|error)|\s*Compiling\s+\S+|\s*Downloading\s+\S+|added \d+ package|\[ERROR\]|BUILD (SUCCESS|FAILED)|\s*Finished\s+|Successfully (installed|built)|ERROR:)/im;
const RE_TREE_GLYPH = /[├└]──|│ /;
@@ -27,6 +29,7 @@ export function autoDetectFilter(text) {
// Rust: floor_char_boundary to avoid UTF-8 split — JS .slice() by char is safe
const head = text.length > DETECT_WINDOW ? text.slice(0, DETECT_WINDOW) : text;
if (RE_GIT_LOG.test(head)) return gitLog;
if (RE_GIT_DIFF.test(head) || RE_GIT_DIFF_HUNK.test(head)) return gitDiff;
if (RE_GIT_STATUS.test(head)) return gitStatus;
@@ -81,6 +84,11 @@ function isGrepLine(line) {
function isPathLike(line) {
const t = line.trim();
if (t.length === 0) return false;
// A drive-letter prefix (e.g. "C:\Users\me" or "C:/Users/me") marks a
// Windows absolute path, so treat the whole line as path-like. Trailing
// colons (e.g. "C:\path\file.js:10") are tolerated, matching grep-style
// suffixes on Windows dumps.
if (/^[A-Za-z]:[\\/]/.test(t)) return true;
if (t.includes(":")) return false;
return t.startsWith(".") || t.startsWith("/") || t.includes("/");
}

View File

@@ -18,6 +18,14 @@ const SHARED_AUTO_CLARITY = "Auto-Clarity: drop caveman for security warnings, i
const SHARED_PERSISTENCE = "ACTIVE EVERY RESPONSE. No revert after many turns. No filler drift. Still active if unsure.";
const SHARED_NO_INVENTED_ABBREV = "No invented abbreviations. Standard well-known tech acronyms (DB, API, HTTP, URL, JSON, ID, OS, CPU) OK. Names of code symbols, function names, API names, error strings: keep verbatim.";
const SHARED_PRESERVE_LANGUAGE = "Preserve the user's dominant language. User wrote Vietnamese, reply Vietnamese. User wrote English, reply English. Wenyan/classical-Chinese levels override this language-preservation rule. Code identifiers, error strings, file paths, commands: keep in their original form regardless of language.";
const SHARED_NO_SELF_REFERENCE = 'No self-reference. Do not name or announce the style (no "caveman mode", no "me caveman think", no "compressed mode active"). Just respond.';
const SHARED_NO_DECORATION = 'No decorative emoji. No narrating tool calls ("I will now search", "I used X to find Y"). No status phrases ("Sure!", "Of course!", "I\'d be happy to"). No causal arrow shorthand ("A -> B -> fails"). State the thing, the action, the reason. Then next step.';
export const CAVEMAN_PROMPTS = {
[CAVEMAN_LEVELS.LITE]: [
"Respond tersely. Keep grammar and full sentences but drop filler, hedging and pleasantries (just/really/basically/sure/of course/I'd be happy to).",
@@ -26,6 +34,10 @@ export const CAVEMAN_PROMPTS = {
SHARED_BOUNDARIES,
SHARED_AUTO_CLARITY,
SHARED_PERSISTENCE,
SHARED_NO_INVENTED_ABBREV,
SHARED_PRESERVE_LANGUAGE,
SHARED_NO_SELF_REFERENCE,
SHARED_NO_DECORATION,
].join(" "),
[CAVEMAN_LEVELS.FULL]: [
@@ -36,16 +48,24 @@ export const CAVEMAN_PROMPTS = {
SHARED_BOUNDARIES,
SHARED_AUTO_CLARITY,
SHARED_PERSISTENCE,
SHARED_NO_INVENTED_ABBREV,
SHARED_PRESERVE_LANGUAGE,
SHARED_NO_SELF_REFERENCE,
SHARED_NO_DECORATION,
].join(" "),
[CAVEMAN_LEVELS.ULTRA]: [
"Respond ultra-terse. Maximum compression. Telegraphic.",
"Abbreviate (DB/auth/config/req/res/fn/impl), strip conjunctions, use arrows for causality (X → Y). One word when one word enough.",
"Pattern: [thing] → [result]. [fix].",
"Strip conjunctions. One word when one word enough.",
"Pattern: [thing] [action] [reason]. [next step].",
SHARED_EXAMPLES,
SHARED_BOUNDARIES,
SHARED_AUTO_CLARITY,
SHARED_PERSISTENCE,
SHARED_NO_INVENTED_ABBREV,
SHARED_PRESERVE_LANGUAGE,
SHARED_NO_SELF_REFERENCE,
SHARED_NO_DECORATION,
].join(" "),
[CAVEMAN_LEVELS.WENYAN_LITE]: [
@@ -55,6 +75,10 @@ export const CAVEMAN_PROMPTS = {
SHARED_BOUNDARIES,
SHARED_AUTO_CLARITY,
SHARED_PERSISTENCE,
SHARED_NO_INVENTED_ABBREV,
SHARED_PRESERVE_LANGUAGE,
SHARED_NO_SELF_REFERENCE,
SHARED_NO_DECORATION,
].join(" "),
[CAVEMAN_LEVELS.WENYAN]: [
@@ -65,6 +89,10 @@ export const CAVEMAN_PROMPTS = {
SHARED_BOUNDARIES,
SHARED_AUTO_CLARITY,
SHARED_PERSISTENCE,
SHARED_NO_INVENTED_ABBREV,
SHARED_PRESERVE_LANGUAGE,
SHARED_NO_SELF_REFERENCE,
SHARED_NO_DECORATION,
].join(" "),
[CAVEMAN_LEVELS.WENYAN_ULTRA]: [
@@ -74,5 +102,9 @@ export const CAVEMAN_PROMPTS = {
SHARED_BOUNDARIES,
SHARED_AUTO_CLARITY,
SHARED_PERSISTENCE,
SHARED_NO_INVENTED_ABBREV,
SHARED_PRESERVE_LANGUAGE,
SHARED_NO_SELF_REFERENCE,
SHARED_NO_DECORATION,
].join(" "),
};

View File

@@ -4,6 +4,7 @@ export const MIN_COMPRESS_SIZE = 500; // bytes; skip tiny blobs
export const DETECT_WINDOW = 1024; // autodetect peeks first N chars
export const GIT_DIFF_HUNK_MAX_LINES = 100; // per-hunk line cap
export const GIT_DIFF_CONTEXT_KEEP = 3; // context lines around changes
export const GIT_LOG_MAX_LINES = 200; // gitLog line cap
export const DEDUP_LINE_MAX = 2000; // dedupLog truncation cap
// Rust pipe_cmd.rs parity caps

View File

@@ -9,16 +9,17 @@ export function find(input) {
const byDir = new Map();
for (const path of lines) {
const lastSlash = path.lastIndexOf("/");
// Accept both Unix ("/a/b") and Windows ("C:\a\b") separators
const lastSep = Math.max(path.lastIndexOf("/"), path.lastIndexOf("\\"));
let dir;
let basename;
if (lastSlash === -1) {
if (lastSep === -1) {
dir = ".";
basename = path;
} else {
// Rust: PathBuf::from(path).parent().display() + file_name().display()
dir = path.slice(0, lastSlash) || "/";
basename = path.slice(lastSlash + 1);
dir = path.slice(0, lastSep) || "/";
basename = path.slice(lastSep + 1);
}
if (!byDir.has(dir)) byDir.set(dir, []);
byDir.get(dir).push(basename);
@@ -31,7 +32,8 @@ export function find(input) {
const showDirs = dirs.slice(0, FIND_TOTAL_DIR_MAX);
for (const dir of showDirs) {
const files = byDir.get(dir);
out += `${dir}/ (${files.length})\n`;
const dirLabel = dir.replace(/\\/g, "/");
out += `${dirLabel}/ (${files.length})\n`;
const showFiles = files.slice(0, FIND_PER_DIR_MAX);
for (const f of showFiles) out += ` ${f}\n`;
if (files.length > FIND_PER_DIR_MAX) {

View File

@@ -0,0 +1,99 @@
// JS-native git-log filter
// Compresses `git log` output: keeps commit headers, subjects, Author/Date;
// drops body padding, decoration, embedded diff lines.
import { GIT_LOG_MAX_LINES } from "../constants.js";
export function gitLog(text, maxLines = GIT_LOG_MAX_LINES) {
if (!text) return "";
const input = String(text);
const lines = input.split("\n");
const out = [];
let skipped = 0;
let inCommit = false;
let subjectSeen = false;
function pushLine(l) {
if (out.length < maxLines) {
out.push(l);
return true;
}
skipped++;
return false;
}
for (let i = 0; i < lines.length; i++) {
const raw = lines[i];
const line = raw.trimEnd();
const trimmed = line.trim();
// commit <sha> header — starts new commit entry
// Also matched with leading graph decoration (`* commit abc1234...` — --graph without --oneline)
if (/^commit [0-9a-f]{7,40}$/i.test(trimmed) || /^[*|/\\ ]+commit [0-9a-f]{7,40}/i.test(trimmed)) {
inCommit = true;
subjectSeen = false;
pushLine(line);
continue;
}
if (inCommit) {
// Author / Date — keep as-is (already column 0 in raw, or graph-prefix stripped by commit-header match)
if (/^[*|/\\ ]*(Author|Date):/i.test(trimmed)) {
pushLine(trimmed);
continue;
}
// blank — skip
if (trimmed === "") continue;
// indented subject (4 spaces, optionally preceded by graph decoration) — first one is subject
if (!subjectSeen && /^[*|/\\ ]* \S/.test(line)) {
pushLine(" Subject: " + trimmed);
subjectSeen = true;
continue;
}
// stat summary: "N file(s) changed, N insertions(+), N deletions(-)"
if (/^\d+ file\w* changed/.test(trimmed)) {
pushLine(" " + trimmed);
continue;
}
// embedded diff header — one-line marker
if (/^diff --git /.test(trimmed)) {
pushLine(" ... diff body omitted");
continue;
}
// everything else in commit body — drop
continue;
}
// Not in a commit block (--oneline / --graph modes):
// Graph decoration + sha + subject: "*|/\\ <sha7> <subject>"
const graphMatch = trimmed.match(/^[*|/\\ ]+([0-9a-f]{7,40}\s+.+)/i);
if (graphMatch) {
pushLine(graphMatch[1]);
continue;
}
// Plain oneline: "<sha7> <subject>"
if (/^[0-9a-f]{7,40}\s+/.test(trimmed)) {
pushLine(trimmed);
continue;
}
// Pure graph decoration (no sha) — drop
if (/^[*|/\\ ]+$/.test(trimmed) && /[*|/\\]/.test(trimmed)) {
continue;
}
// catch-all pass-through
pushLine(trimmed);
}
if (skipped > 0) out.push(`... (${skipped} more lines)`);
const result = out.join("\n");
if (!result && input) return input;
if (result.length > input.length) return input;
return result;
}
gitLog.filterName = "git-log";

View File

@@ -18,14 +18,24 @@ function jsonBytes(value) {
function messagePayload(body) {
if (Array.isArray(body?.messages)) return body.messages;
if (Array.isArray(body?.input)) return body.input;
const kiro = collectKiroHeadroomMessages(body);
if (kiro) return kiro.messages;
return null;
}
function captureSizeSnapshot(body) {
const messages = messagePayload(body);
const toolHistory = messages?.filter((message) =>
message?.role === "tool"
|| message?.role === "function"
|| message?.tool_calls?.length
|| message?.content?.some?.((part) => part?.type === "tool_use" || part?.type === "tool_result")
) || [];
return {
bodyBytes: jsonBytes(body),
messageBytes: messages ? jsonBytes(messages) : 0,
toolSchemaBytes: jsonBytes(body?.tools || []),
toolHistoryBytes: jsonBytes(toolHistory),
};
}
@@ -81,6 +91,121 @@ function hasUnsafeResponsesInputForCompression(body) {
});
}
function collectKiroHeadroomMessages(body) {
const state = body?.conversationState;
if (!state || typeof state !== "object") return null;
const messages = [];
const targets = [];
const addTextTarget = (role, text, target, extra = {}) => {
if (typeof text !== "string") return;
messages.push({ role, content: text, ...extra });
targets.push(target);
};
const toToolCalls = (toolUses) => {
if (!Array.isArray(toolUses) || toolUses.length === 0) return undefined;
const calls = toolUses.map((toolUse) => ({
id: toolUse?.toolUseId,
type: "function",
function: {
name: toolUse?.name || "",
arguments: JSON.stringify(toolUse?.input || {}),
},
})).filter((call) => call.id || call.function.name);
return calls.length > 0 ? calls : undefined;
};
const visit = (item) => {
const user = item?.userInputMessage;
if (user) {
addTextTarget("system", user.systemInstruction, { object: user, key: "systemInstruction" });
addTextTarget("user", user.content, { object: user, key: "content" });
const toolResults = user.userInputMessageContext?.toolResults;
if (Array.isArray(toolResults)) {
for (const toolResult of toolResults) {
const content = toolResult?.content;
if (!Array.isArray(content)) continue;
for (const part of content) {
addTextTarget(
"tool",
part?.text,
{ object: part, key: "text" },
toolResult?.toolUseId ? { tool_call_id: toolResult.toolUseId } : {}
);
}
}
}
return;
}
const assistant = item?.assistantResponseMessage;
if (assistant) {
const toolCalls = toToolCalls(assistant.toolUses);
addTextTarget(
"assistant",
assistant.content,
{ object: assistant, key: "content" },
toolCalls ? { tool_calls: toolCalls } : {}
);
}
};
if (Array.isArray(state.history)) {
for (const item of state.history) visit(item);
}
if (state.currentMessage) visit(state.currentMessage);
return messages.length > 0 ? { messages, targets } : null;
}
function textFromHeadroomMessage(message) {
const content = message?.content;
if (typeof content === "string") return content;
if (!Array.isArray(content)) return null;
const parts = [];
for (const part of content) {
if (typeof part === "string") {
parts.push(part);
} else if (typeof part?.text === "string") {
parts.push(part.text);
}
}
return parts.length > 0 ? parts.join("\n") : null;
}
function applyKiroHeadroomMessages(projection, compressedMessages, diagnostics) {
if (!Array.isArray(compressedMessages) || compressedMessages.length !== projection.messages.length) {
setDiagnostic(diagnostics, "proxy response did not match Kiro message count");
return false;
}
const updates = [];
for (let i = 0; i < projection.messages.length; i++) {
const expected = projection.messages[i];
const actual = compressedMessages[i];
if (!actual || actual.role !== expected.role) {
setDiagnostic(diagnostics, "proxy response did not preserve Kiro message order");
return false;
}
const text = textFromHeadroomMessage(actual);
if (text === null) {
setDiagnostic(diagnostics, "proxy response missing Kiro text content");
return false;
}
updates.push({ target: projection.targets[i], text });
}
for (const update of updates) {
update.target.object[update.target.key] = update.text;
}
return true;
}
// POST messages to Headroom /v1/compress; returns compressed messages + stats or null.
async function callCompress(url, messages, model, timeoutMs, compressUserMessages, diagnostics) {
const endpoint = buildCompressEndpoint(url);
@@ -171,6 +296,22 @@ export async function compressWithHeadroom(body, { enabled, url, model, format,
return data;
}
// Kiro shape: conversationState.history/currentMessage are projected to
// OpenAI messages for the proxy, then copied back into the original Kiro
// fields. Keep the provider payload shape intact for Kiro's executor.
if (format === "kiro") {
const projection = collectKiroHeadroomMessages(body);
if (!projection) {
setDiagnostic(diagnostics, "Kiro request did not project to messages[]");
return null;
}
const data = await callCompress(url, projection.messages, model, timeoutMs, compressUserMessages, diagnostics || {});
if (!data) return null;
if (!applyKiroHeadroomMessages(projection, data.messages, diagnostics)) return null;
if (diagnostics) diagnostics.after = captureSizeSnapshot(body);
return data;
}
// OpenAI shape: messages/input go straight to the proxy.
const key = Array.isArray(body.messages) ? "messages"
: Array.isArray(body.input) ? "input"
@@ -203,7 +344,10 @@ export function formatHeadroomSizeLog(diagnostics) {
const before = diagnostics?.before;
const after = diagnostics?.after;
if (!before || !after) return "";
return `body=${before.bodyBytes}B→${after.bodyBytes}B messages=${before.messageBytes}B→${after.messageBytes}B`;
const effective = before.bodyBytes > 0
? (((before.bodyBytes - after.bodyBytes) / before.bodyBytes) * 100).toFixed(1)
: "0.0";
return `body=${before.bodyBytes}B→${after.bodyBytes}B messages=${before.messageBytes}B→${after.messageBytes}B tools=${before.toolSchemaBytes || 0}B→${after.toolSchemaBytes || 0}B toolHistory=${before.toolHistoryBytes || 0}B→${after.toolHistoryBytes || 0}B effective=${effective}%`;
}
export function isHeadroomPhantomSavings(stats, diagnostics, minShrinkRatio = 0.05) {

104
open-sse/rtk/pxpipe.js Normal file
View File

@@ -0,0 +1,104 @@
// PXPIPE: render bulky Claude-format context as dense PNGs via pxpipe-proxy's
// library API (transformAnthropicMessages). Fail-open like every token saver:
// any error/timeout returns { body: null, summary } and leaves the request untouched.
import { FORMATS } from "../translator/formats.js";
const DEFAULT_TIMEOUT_MS = 15000;
const DEFAULT_MIN_CHARS = 25000;
// pxpipe's own profitability gate assumes ~4 chars/token; reuse it for the
// estimated before/after numbers surfaced in stats (marked "estimated" in UI).
const EST_CHARS_PER_TOKEN = 4;
function bodyChars(body) {
try {
return JSON.stringify(body)?.length || 0;
} catch {
return 0;
}
}
function estTokens(chars) {
return Math.round(chars / EST_CHARS_PER_TOKEN);
}
function skipped(reason, extra = {}) {
return { body: null, summary: { applied: false, reason, ...extra } };
}
// Transform a Claude-format request body through pxpipe. Returns
// { body: <new body object> | null, summary } — body is null when nothing changed.
// opts.transform is injected by the host (src side) so open-sse stays free of
// filesystem/install concerns and remains usable standalone.
export async function compressWithPxpipe(body, { enabled, format, model, minChars, timeoutMs, transform } = {}) {
if (!enabled) return skipped("disabled");
if (typeof transform !== "function") return skipped("not_installed");
if (!body) return skipped("missing_body");
if (format !== FORMATS.CLAUDE) return skipped("unsupported_format", { detail: format });
const startedAt = Date.now();
const originalChars = bodyChars(body);
const threshold = Number(minChars) > 0 ? Number(minChars) : DEFAULT_MIN_CHARS;
if (originalChars < threshold) {
return skipped("below_threshold", { originalChars, threshold });
}
try {
const encoded = new TextEncoder().encode(JSON.stringify(body));
const budget = Number(timeoutMs) > 0 ? Number(timeoutMs) : DEFAULT_TIMEOUT_MS;
// transformAnthropicMessages is local CPU work and can't be aborted; race a
// timer and discard the result if it loses (input body is never mutated).
const result = await Promise.race([
transform({
body: encoded,
model,
options: { minCompressChars: threshold },
}),
new Promise((resolve) => setTimeout(() => resolve(null), budget)),
]);
if (!result) return skipped("timeout", { originalChars, durationMs: Date.now() - startedAt });
if (!result.applied) {
return skipped(result.reason || "passthrough", {
detail: result.detail,
originalChars,
durationMs: Date.now() - startedAt,
});
}
const newBody = JSON.parse(new TextDecoder().decode(result.body));
const compressedBodyChars = bodyChars(newBody);
const info = result.info || {};
const imagedChars = info.compressedChars || 0;
// The transformed body is BIGGER in bytes (base64 PNGs) but cheaper in tokens:
// images bill by pixels (Anthropic: pixels/750), not by encoded length. So the
// after-estimate is remaining-text tokens + image tokens — never chars/4 of the
// new body. Provider-billed usage recorded per request stays the ground truth.
const imageTokensEst = info.imageTokens
|| (info.imagePixels ? Math.round(info.imagePixels / 750) : (info.imageCount || 0) * 4761);
const summary = {
applied: true,
reason: "applied",
originalChars,
compressedBodyChars,
imagedChars,
imageCount: info.imageCount || 0,
imageBytes: info.imageBytes || 0,
tokensBeforeEst: info.baselineTokens || estTokens(originalChars),
tokensAfterEst: estTokens(Math.max(0, originalChars - imagedChars)) + imageTokensEst,
durationMs: Date.now() - startedAt,
cacheOwnsControl: result.cache?.ownsCacheControl === true,
};
summary.tokensSavedEst = Math.max(0, summary.tokensBeforeEst - summary.tokensAfterEst);
summary.savedPct = summary.tokensBeforeEst > 0
? +((summary.tokensSavedEst / summary.tokensBeforeEst) * 100).toFixed(2)
: 0;
return { body: newBody, summary };
} catch (e) {
return skipped("transform_error", { detail: e?.message || String(e), originalChars, durationMs: Date.now() - startedAt });
}
}
export function formatPxpipeLog(summary) {
if (!summary) return null;
if (!summary.applied) return null;
return `imaged ${summary.imagedChars}ch → ${summary.imageCount} image(s) | est ${summary.tokensBeforeEst}→${summary.tokensAfterEst} tokens (-${summary.savedPct}%) | ${summary.durationMs}ms`;
}

View File

@@ -1,6 +1,7 @@
import { FILTERS } from "./constants.js";
import { gitDiff } from "./filters/gitDiff.js";
import { gitStatus } from "./filters/gitStatus.js";
import { gitLog } from "./filters/gitLog.js";
import { grep } from "./filters/grep.js";
import { find } from "./filters/find.js";
import { dedupLog } from "./filters/dedupLog.js";
@@ -13,6 +14,7 @@ import { searchList } from "./filters/searchList.js";
const REGISTRY = {
[FILTERS.GIT_DIFF]: gitDiff,
[FILTERS.GIT_STATUS]: gitStatus,
[FILTERS.GIT_LOG]: gitLog,
[FILTERS.GREP]: grep,
[FILTERS.FIND]: find,
[FILTERS.DEDUP_LOG]: dedupLog,