Merge remote-tracking branch 'upstream/master'

# Conflicts:
#	.gitignore
#	open-sse/handlers/chatCore.js

src/lib/db/backup.js

@@ -1,9 +1,20 @@
+// DB safety backups — taken ONLY before a schema change (see migrate.js).
+//
+// ⚠️ AGENT/DEV NOTES:
+// - Backups are a best-effort safety net before schema migrations. There is NO
+//   automated restore path; recovery is manual (copy a backup file back).
+// - Backups intentionally EXCLUDE the `requestDetails` table (observability log,
+//   auto-pruned, non-critical) so a multi-hundred-MB DB backs up as a few MB.
+// - Only the newest KEEP_BACKUPS are kept; older ones are pruned automatically.
 import fs from "node:fs";
 import path from "node:path";
 import { BACKUPS_DIR, ensureDirs } from "./paths.js";
 import { timestampSlug, getAppVersion } from "./version.js";
 
-const KEEP_BACKUPS = 5;
+const KEEP_BACKUPS = 3;
+
+// Tables excluded from safety backups (large, non-critical, reproducible).
+const BACKUP_EXCLUDE_TABLES = ["requestDetails"];
 
 export function makeBackupDir(label) {
   ensureDirs();
@@ -22,6 +33,35 @@ export function backupFile(srcPath, destDir, destName = null) {
   return dest;
 }
 
+// Lightweight DB backup via ATTACH: create an empty sqlite file, copy every
+// table EXCEPT the excluded ones into it. Avoids duplicating the huge
+// observability log, so the backup stays small regardless of DB size.
+export function backupDbLite(adapter, destDir, destName = "data.sqlite") {
+  const dest = path.join(destDir, destName);
+  try { fs.rmSync(dest, { force: true }); } catch {}
+  const escaped = dest.replace(/'/g, "''");
+
+  adapter.exec(`ATTACH DATABASE '${escaped}' AS bak`);
+  try {
+    const excluded = new Set(BACKUP_EXCLUDE_TABLES);
+    const tables = adapter
+      .all(`SELECT name, sql FROM main.sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'`)
+      .filter((t) => !excluded.has(t.name));
+
+    adapter.transaction(() => {
+      for (const t of tables) {
+        // Recreate table structure in backup DB, then copy rows.
+        const createSql = t.sql.replace(/CREATE TABLE\s+/i, "CREATE TABLE bak.");
+        adapter.exec(createSql);
+        adapter.exec(`INSERT INTO bak.${t.name} SELECT * FROM main.${t.name}`);
+      }
+    });
+  } finally {
+    try { adapter.exec("DETACH DATABASE bak"); } catch {}
+  }
+  return dest;
+}
+
 export function pruneOldBackups() {
   if (!fs.existsSync(BACKUPS_DIR)) return;
   const entries = fs.readdirSync(BACKUPS_DIR, { withFileTypes: true })

src/lib/db/index.js

@@ -64,7 +64,7 @@ export {
 
 // Request details
 export {
-  saveRequestDetail, getRequestDetails, getRequestDetailById,
+  saveRequestDetail, getRequestDetails, getRequestDetailById, getDistinctProviders,
 } from "./repos/requestDetailsRepo.js";
 
 // Export/import full DB

src/lib/db/migrate.js

@@ -1,10 +1,10 @@
 import fs from "node:fs";
 import path from "node:path";
-import { LEGACY_FILES, DB_DIR, DATA_FILE } from "./paths.js";
-import { TABLES, buildCreateTableSql } from "./schema.js";
+import { LEGACY_FILES, DB_DIR } from "./paths.js";
+import { TABLES, buildCreateTableSql, SCHEMA_VERSION } from "./schema.js";
 import { MIGRATIONS, latestVersion } from "./migrations/index.js";
 import { getMetaSync, setMetaSync } from "./helpers/metaStore.js";
-import { makeBackupDir, backupFile, pruneOldBackups } from "./backup.js";
+import { makeBackupDir, backupFile, backupDbLite, pruneOldBackups } from "./backup.js";
 import { getAppVersion } from "./version.js";
 import { stringifyJson } from "./helpers/jsonCol.js";
 
@@ -221,12 +221,37 @@ export async function runMigrationOnce(adapter) {
   // a brand-new DB as non-fresh once schemaVersion is written).
   const fresh = isFreshDb(adapter);
 
+  // Prune stale backups every boot so old oversized backups shrink to KEEP.
+  pruneOldBackups();
+
+  // Bootstrap _meta so we can read the stored backup schema version below
+  // (runVersionedMigrations also ensures this, but we need it earlier here).
+  adapter.exec(buildCreateTableSql("_meta", TABLES._meta));
+
+  // Detect a pending schema change via the central SCHEMA_VERSION const.
+  // A lightweight backup is taken BEFORE any schema mutation below.
+  const storedSchemaVer = parseInt(getMetaSync(adapter, "backupSchemaVersion", "0"), 10) || 0;
+  const schemaChanging = !fresh && storedSchemaVer < SCHEMA_VERSION;
+  if (schemaChanging) {
+    try {
+      const backupDir = makeBackupDir(`schema-${storedSchemaVer}-to-${SCHEMA_VERSION}`);
+      backupDbLite(adapter, backupDir);
+      pruneOldBackups();
+      console.log(`[DB][migrate] pre-schema backup ${storedSchemaVer} → ${SCHEMA_VERSION}: ${backupDir}`);
+    } catch (e) {
+      console.warn(`[DB][migrate] pre-schema backup failed (continuing): ${e.message}`);
+    }
+  }
+
   // 1. Always run versioned migrations chain (skip-version safe)
   const migInfo = runVersionedMigrations(adapter);
 
   // 2. Additive sync (auto add missing columns/indexes declared in TABLES)
   syncSchemaFromTables(adapter);
 
+  // Stamp the schema version we just reached so future boots skip re-backup.
+  setMetaSync(adapter, "backupSchemaVersion", SCHEMA_VERSION);
+
   // 3. One-time legacy JSON import (only if DB was fresh on entry)
   const alreadyImported = fs.existsSync(MIGRATED_MARKER);
   const legacyMain = readJsonSafe(LEGACY_FILES.main);
@@ -247,6 +272,7 @@ export async function runMigrationOnce(adapter) {
         importLegacyDisabled(adapter, legacyDisabled);
         importLegacyDetails(adapter, legacyDetails);
         setMetaSync(adapter, "appVersion", getAppVersion());
+        setMetaSync(adapter, "backupSchemaVersion", SCHEMA_VERSION);
         setMetaSync(adapter, "migratedAt", new Date().toISOString());
       });
     } catch (err) {
@@ -263,24 +289,9 @@ export async function runMigrationOnce(adapter) {
     return;
   }
 
-  if (fresh) {
-    setMetaSync(adapter, "appVersion", getAppVersion());
-    return;
-  }
-
-  // 4. App version bump → backup data.sqlite (safety net before user-side upgrade)
-  const oldVer = getMetaSync(adapter, "appVersion", null);
+  // Track app version for informational purposes only. App version bumps no
+  // longer trigger a DB backup — only real schema changes (SCHEMA_VERSION) do.
   const newVer = getAppVersion();
-  if (oldVer && oldVer !== newVer) {
-    const backupDir = makeBackupDir(`upgrade-${oldVer}-to-${newVer}`);
-    try { backupFile(DATA_FILE, backupDir); } catch {}
-    setMetaSync(adapter, "appVersion", newVer);
-    pruneOldBackups();
-    console.log(`[DB][migrate] App ${oldVer} → ${newVer} | schema ${migInfo.from} → ${migInfo.to} | backup: ${backupDir}`);
-  } else if (migInfo.applied > 0) {
-    // Schema upgrade without app version bump — still backup
-    const backupDir = makeBackupDir(`schema-${migInfo.from}-to-${migInfo.to}`);
-    try { backupFile(DATA_FILE, backupDir); } catch {}
-    pruneOldBackups();
-  }
+  const oldVer = getMetaSync(adapter, "appVersion", null);
+  if (oldVer !== newVer) setMetaSync(adapter, "appVersion", newVer);
 }

src/lib/db/repos/connectionsRepo.js

@@ -56,6 +56,17 @@ function upsert(db, c) {
   );
 }
 
+function deriveConnectionName(data, fallbackName) {
+  if (data.provider === "github") {
+    return data.providerSpecificData?.githubLogin
+      || data.providerSpecificData?.githubEmail
+      || data.email
+      || data.providerSpecificData?.githubName
+      || fallbackName;
+  }
+  return fallbackName;
+}
+
 export async function getProviderConnections(filter = {}) {
   const db = await getAdapter();
   const where = [];
@@ -102,7 +113,18 @@ export async function createProviderConnection(data) {
       const incomingWs = data.providerSpecificData?.chatgptAccountId;
       existing = all.find(c => {
         if (c.authType !== "oauth" || c.email !== data.email) return false;
-        // Workspace providers (Codex) use workspace ID when both sides have it
+
+        // Codex/OpenAI can issue multiple OAuth grants for the same email.
+        // Refresh tokens are rotated single-use; collapsing a new login onto an
+        // existing bare-email row overwrites the first account's token pair and
+        // makes it look "invalid" after adding a second account. Only update an
+        // existing Codex row when both rows expose the same ChatGPT account ID.
+        if (data.provider === "codex") {
+          const existingWs = c.providerSpecificData?.chatgptAccountId;
+          return !!incomingWs && !!existingWs && incomingWs === existingWs;
+        }
+
+        // Workspace providers use workspace ID when both sides have it
         const existingWs = c.providerSpecificData?.chatgptAccountId;
         if (incomingWs && existingWs) return incomingWs === existingWs;
         if (incomingWs && !existingWs) return false;
@@ -133,7 +155,7 @@ export async function createProviderConnection(data) {
 
     let connectionName = data.name || null;
     if (!connectionName && (data.authType === "oauth" || data.authType === "access_token")) {
-      connectionName = data.email || `Account ${all.length + 1}`;
+      connectionName = deriveConnectionName(data, data.email || `Account ${all.length + 1}`);
     }
     let connectionPriority = data.priority;
     if (!connectionPriority) {

src/lib/db/repos/requestDetailsRepo.js

@@ -98,6 +98,7 @@ async function flushToDatabase() {
             providerRequest: truncateField(item.providerRequest, config.maxJsonSize),
             providerResponse: truncateField(item.providerResponse, config.maxJsonSize),
             response: truncateField(item.response, config.maxJsonSize),
+            pxpipe: item.pxpipe || undefined,
           };
 
           db.run(
@@ -174,6 +175,12 @@ export async function getRequestDetails(filter = {}) {
   };
 }
 
+export async function getDistinctProviders() {
+  const db = await getAdapter();
+  const rows = db.all(`SELECT DISTINCT provider FROM requestDetails WHERE provider IS NOT NULL ORDER BY provider ASC`);
+  return rows.map((r) => r.provider);
+}
+
 export async function getRequestDetailById(id) {
   const db = await getAdapter();
   const row = db.get(`SELECT data FROM requestDetails WHERE id = ?`, [id]);

src/lib/db/repos/settingsRepo.js

@@ -43,6 +43,10 @@ const DEFAULT_SETTINGS = {
   cavemanLevel: "full",
   ponytailEnabled: false,
   ponytailLevel: "full",
+  pxpipeEnabled: false,
+  pxpipeAutoInstall: true,
+  pxpipeMinChars: 25000,
+  pxpipeTimeoutMs: 15000,
 };
 
 async function readRaw() {

src/lib/db/repos/usageRepo.js

@@ -189,8 +189,7 @@ export function trackPendingRequest(model, provider, connectionId, started, erro
     lastErrorProvider.ts = Date.now();
   }
 
-  const t = new Date().toLocaleTimeString("en-US", { hour12: false, hour: "2-digit", minute: "2-digit", second: "2-digit" });
-  console.log(`[${t}] [PENDING] ${started ? "START" : "END"}${error ? " (ERROR)" : ""} | provider=${provider} | model=${model}`);
+  // [PENDING] console line removed; lifecycle is visible via "▶" and "📊 done" lines
   scheduleStatsEvent("pending");
 }
 

src/lib/db/schema.js

@@ -1,4 +1,8 @@
-// Latest schema version — bumped when a migration is added in ./migrations/
+// ⚠️ AGENT/DEV: Bump this by +1 EVERY TIME you change the schema below
+// (add/remove/alter a table, column, or index in TABLES). It drives the
+// pre-change safety backup in migrate.js: when the stored version is lower,
+// one lightweight DB backup is taken before applying schema changes. Forgetting
+// to bump only skips that backup — it does NOT break the additive auto-sync.
 export const SCHEMA_VERSION = 1;
 
 export const PRAGMA_SQL = `