Fix tunnel health check
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "9router",
|
||||
"version": "0.4.58",
|
||||
"version": "0.4.59",
|
||||
"description": "9Router CLI - Start and manage 9Router server",
|
||||
"bin": {
|
||||
"9router": "./cli.js"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "9router-app",
|
||||
"version": "0.4.58",
|
||||
"version": "0.4.59",
|
||||
"description": "9Router web dashboard",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
|
||||
@@ -21,17 +21,17 @@ const CLIENT_PING_FAST_MS = 10000;
|
||||
const CLIENT_PING_SLOW_MS = 60000;
|
||||
const CLIENT_PING_TIMEOUT_MS = 5000;
|
||||
|
||||
// Browser-side health probe: bypasses backend DNS issues (1.1.1.1 vs OS resolver).
|
||||
// Uses no-cors → opaque response means TLS+DNS reach succeeded, which is enough.
|
||||
// Browser-side health probe: must reach origin (not just CF edge).
|
||||
// CORS mode → res.ok=false for 5xx (Cloudflare Error 1033 when origin dead).
|
||||
async function clientPingUrl(url) {
|
||||
if (!url) return false;
|
||||
try {
|
||||
await fetch(`${url}/api/health`, {
|
||||
mode: "no-cors",
|
||||
const res = await fetch(`${url}/api/health`, {
|
||||
mode: "cors",
|
||||
cache: "no-store",
|
||||
signal: AbortSignal.timeout(CLIENT_PING_TIMEOUT_MS),
|
||||
});
|
||||
return true;
|
||||
return res.ok;
|
||||
} catch { return false; }
|
||||
}
|
||||
|
||||
@@ -162,6 +162,7 @@ export default function APIPageClient({ machineId }) {
|
||||
const ok = await clientPingAny(tunnelPublicUrl, tunnelUrl);
|
||||
tunnelClientReachableRef.current = ok;
|
||||
if (ok) { tunnelMissRef.current = 0; setTunnelReachable(true); if (!tunnelEverReachableRef.current) { tunnelEverReachableRef.current = true; setTunnelEverReachable(true); } }
|
||||
else { tunnelMissRef.current += 1; if (tunnelMissRef.current >= REACHABLE_MISS_THRESHOLD) setTunnelReachable(false); }
|
||||
} else {
|
||||
tunnelClientReachableRef.current = false;
|
||||
}
|
||||
@@ -169,6 +170,7 @@ export default function APIPageClient({ machineId }) {
|
||||
const ok = await clientPingUrl(tsUrl);
|
||||
tsClientReachableRef.current = ok;
|
||||
if (ok) { tsMissRef.current = 0; setTsReachable(true); if (!tsEverReachableRef.current) { tsEverReachableRef.current = true; setTsEverReachable(true); } }
|
||||
else { tsMissRef.current += 1; if (tsMissRef.current >= REACHABLE_MISS_THRESHOLD) setTsReachable(false); }
|
||||
} else {
|
||||
tsClientReachableRef.current = false;
|
||||
}
|
||||
@@ -345,8 +347,8 @@ export default function APIPageClient({ machineId }) {
|
||||
while (Date.now() - start < TUNNEL_PING_MAX_MS) {
|
||||
await new Promise((r) => setTimeout(r, TUNNEL_PING_INTERVAL_MS));
|
||||
const ok = await Promise.any(targets.map(async (h) => {
|
||||
const p = await fetch(h, { mode: "no-cors", cache: "no-store" });
|
||||
if (p.ok || p.type === "opaque") return true;
|
||||
const p = await fetch(h, { mode: "cors", cache: "no-store" });
|
||||
if (p.ok) return true;
|
||||
throw new Error("not ready");
|
||||
})).catch(() => false);
|
||||
if (ok) {
|
||||
|
||||
@@ -1,5 +1,15 @@
|
||||
import { NextResponse } from "next/server";
|
||||
|
||||
const CORS_HEADERS = {
|
||||
"Access-Control-Allow-Origin": "*",
|
||||
"Access-Control-Allow-Methods": "GET, OPTIONS",
|
||||
"Access-Control-Allow-Headers": "*",
|
||||
};
|
||||
|
||||
export async function GET() {
|
||||
return NextResponse.json({ ok: true });
|
||||
return NextResponse.json({ ok: true }, { headers: CORS_HEADERS });
|
||||
}
|
||||
|
||||
export async function OPTIONS() {
|
||||
return new NextResponse(null, { status: 204, headers: CORS_HEADERS });
|
||||
}
|
||||
|
||||
@@ -27,10 +27,56 @@ import {
|
||||
getOAuthClientMetadata,
|
||||
} from "./constants/oauth";
|
||||
import { XAI_CONFIG, XAI_PKCE_VERIFIER_BYTES } from "./constants/xai";
|
||||
import {
|
||||
decodeIdTokenEmail as decodeXaiIdTokenEmail,
|
||||
discoverEndpoints as discoverXaiEndpoints,
|
||||
} from "./services/xai";
|
||||
|
||||
// Inlined from services/xai.js to keep web route bundle free of `open` (CLI-only) package
|
||||
let cachedXaiDiscovery = null;
|
||||
|
||||
function validateXaiOAuthEndpoint(rawUrl, field) {
|
||||
const value = String(rawUrl || "").trim();
|
||||
if (!value) throw new Error(`xai discovery ${field} is empty`);
|
||||
let parsed;
|
||||
try { parsed = new URL(value); } catch (err) {
|
||||
throw new Error(`xai discovery ${field} is invalid: ${err.message}`);
|
||||
}
|
||||
if (parsed.protocol !== "https:") throw new Error(`xai discovery ${field} must use https: ${value}`);
|
||||
const host = parsed.hostname.toLowerCase().trim();
|
||||
if (host !== "x.ai" && !host.endsWith(".x.ai")) {
|
||||
throw new Error(`xai discovery ${field} host ${host} is not on x.ai`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
async function discoverXaiEndpoints() {
|
||||
if (cachedXaiDiscovery) return cachedXaiDiscovery;
|
||||
try {
|
||||
const res = await fetch(XAI_CONFIG.discoveryUrl, { headers: { Accept: "application/json" } });
|
||||
if (res.ok) {
|
||||
const data = await res.json();
|
||||
cachedXaiDiscovery = {
|
||||
authorizeUrl: validateXaiOAuthEndpoint(data.authorization_endpoint, "authorization_endpoint"),
|
||||
tokenUrl: validateXaiOAuthEndpoint(data.token_endpoint, "token_endpoint"),
|
||||
};
|
||||
return cachedXaiDiscovery;
|
||||
}
|
||||
} catch { /* fall through to static fallback */ }
|
||||
cachedXaiDiscovery = { authorizeUrl: XAI_CONFIG.authorizeUrl, tokenUrl: XAI_CONFIG.tokenUrl };
|
||||
return cachedXaiDiscovery;
|
||||
}
|
||||
|
||||
function decodeXaiIdTokenEmail(idToken) {
|
||||
if (!idToken || typeof idToken !== "string") return undefined;
|
||||
const parts = idToken.split(".");
|
||||
if (parts.length !== 3) return undefined;
|
||||
try {
|
||||
const base64 = parts[1].replace(/-/g, "+").replace(/_/g, "/");
|
||||
const padding = (BASE64_BLOCK_SIZE - (base64.length % BASE64_BLOCK_SIZE)) % BASE64_BLOCK_SIZE;
|
||||
const json = Buffer.from(base64 + "=".repeat(padding), "base64").toString("utf8");
|
||||
const payload = JSON.parse(json);
|
||||
return payload.email || payload.preferred_username || payload.sub || undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
const BASE64_BLOCK_SIZE = 4;
|
||||
|
||||
|
||||
@@ -289,7 +289,7 @@ export async function spawnQuickTunnel(localPort, onUrlUpdate) {
|
||||
|
||||
const requestedProtocol = String(process.env.TUNNEL_TRANSPORT_PROTOCOL || process.env.CLOUDFLARED_PROTOCOL || DEFAULT_QUICK_TUNNEL_PROTOCOL).trim().toLowerCase();
|
||||
const tunnelProtocol = QUICK_TUNNEL_PROTOCOLS.has(requestedProtocol) ? requestedProtocol : DEFAULT_QUICK_TUNNEL_PROTOCOL;
|
||||
const child = spawn(binaryPath, ["tunnel", "--url", `http://127.0.0.1:${localPort}`, "--config", configPath, "--no-autoupdate"], {
|
||||
const child = spawn(binaryPath, ["tunnel", "--url", `http://127.0.0.1:${localPort}`, "--config", configPath, "--no-autoupdate", "--retries", "99"], {
|
||||
detached: false,
|
||||
windowsHide: true,
|
||||
cwd: os.tmpdir(),
|
||||
|
||||
@@ -12,7 +12,7 @@ export const INTERNET_CHECK = {
|
||||
timeoutMs: 3000,
|
||||
};
|
||||
|
||||
export const RESTART_COOLDOWN_MS = 180000;
|
||||
export const RESTART_COOLDOWN_MS = 120000;
|
||||
export const NETWORK_SETTLE_MS = 2500;
|
||||
export const WATCHDOG_INTERVAL_MS = 60000;
|
||||
export const NETWORK_CHECK_INTERVAL_MS = 5000;
|
||||
|
||||
@@ -31,6 +31,10 @@ export function isTunnelManuallyDisabled() { return tunnelSvc.cancelToken.cancel
|
||||
export function isTunnelReconnecting() { return tunnelSvc.spawnInProgress; }
|
||||
export function isTailscaleReconnecting() { return tailscaleSvc.spawnInProgress; }
|
||||
|
||||
// Callback invoked when cloudflared exits unexpectedly (set by initializeApp)
|
||||
let onTunnelUnexpectedExit = null;
|
||||
export function setTunnelUnexpectedExitCallback(cb) { onTunnelUnexpectedExit = cb; }
|
||||
|
||||
// ─── Cloudflare Tunnel ───────────────────────────────────────────────────────
|
||||
|
||||
async function registerTunnelUrl(shortId, tunnelUrl) {
|
||||
@@ -55,10 +59,18 @@ export async function enableTunnel(localPort = 20128) {
|
||||
try {
|
||||
if (isCloudflaredRunning()) {
|
||||
const existing = loadState();
|
||||
if (existing?.tunnelUrl && await probeUrlAlive(existing.tunnelUrl)) {
|
||||
if (existing?.tunnelUrl && existing?.shortId) {
|
||||
const publicUrl = `https://r${existing.shortId}.abc-tunnel.us`;
|
||||
console.log(`[Tunnel] already running, reuse: ${existing.tunnelUrl}`);
|
||||
return { success: true, tunnelUrl: existing.tunnelUrl, shortId: existing.shortId, publicUrl, alreadyRunning: true };
|
||||
// Reuse only if BOTH direct + public URL alive (avoid stale socket after network change)
|
||||
const [directOk, publicOk] = await Promise.all([
|
||||
probeUrlAlive(existing.tunnelUrl),
|
||||
probeUrlAlive(publicUrl),
|
||||
]);
|
||||
if (directOk && publicOk) {
|
||||
console.log(`[Tunnel] already running, reuse: ${existing.tunnelUrl}`);
|
||||
return { success: true, tunnelUrl: existing.tunnelUrl, shortId: existing.shortId, publicUrl, alreadyRunning: true };
|
||||
}
|
||||
console.log(`[Tunnel] stale (direct=${directOk} public=${publicOk}), respawn`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -77,6 +89,12 @@ export async function enableTunnel(localPort = 20128) {
|
||||
await updateSettings({ tunnelEnabled: true, tunnelUrl: url });
|
||||
};
|
||||
|
||||
// Register exit handler BEFORE spawn so it fires even on early exit
|
||||
setUnexpectedExitHandler(() => {
|
||||
console.warn("[Tunnel] cloudflared exited unexpectedly, scheduling respawn");
|
||||
if (onTunnelUnexpectedExit) onTunnelUnexpectedExit();
|
||||
});
|
||||
|
||||
const { tunnelUrl } = await spawnQuickTunnel(localPort, onUrlUpdate);
|
||||
console.log(`[Tunnel] spawned: ${tunnelUrl}`);
|
||||
throwIfCancelled(token, "tunnel");
|
||||
|
||||
@@ -6,7 +6,7 @@ import { cleanupProviderConnections, getSettings, updateSettings, getApiKeys } f
|
||||
import {
|
||||
enableTunnel, enableTailscale,
|
||||
isTunnelManuallyDisabled, isTunnelReconnecting, isTailscaleReconnecting,
|
||||
getTunnelService, getTailscaleService,
|
||||
getTunnelService, getTailscaleService, setTunnelUnexpectedExitCallback,
|
||||
} from "@/lib/tunnel/tunnelManager";
|
||||
import { killCloudflared, isCloudflaredRunning, ensureCloudflared } from "@/lib/tunnel/cloudflared";
|
||||
import { isTailscaleRunning } from "@/lib/tunnel/tailscale";
|
||||
@@ -83,6 +83,11 @@ export async function initializeApp() {
|
||||
// Sync mitmAlias DB → JSON cache so standalone MITM server can read it
|
||||
syncMitmAliasCache().catch(() => {});
|
||||
|
||||
// Auto-respawn tunnel when cloudflared exits unexpectedly (e.g. network change drop)
|
||||
setTunnelUnexpectedExitCallback(() => {
|
||||
safeRestartTunnel("unexpected-exit").catch(() => {});
|
||||
});
|
||||
|
||||
startWatchdog();
|
||||
startNetworkMonitor();
|
||||
autoStartMitm();
|
||||
@@ -133,13 +138,22 @@ async function safeRestartTunnel(reason) {
|
||||
if (!settings.tunnelEnabled) return;
|
||||
if (svc.cancelToken.cancelled) return;
|
||||
if (svc.spawnInProgress) return;
|
||||
if (Date.now() - svc.lastRestartAt < RESTART_COOLDOWN_MS) return;
|
||||
// Bypass cooldown when process is dead (real respawn, not restart-loop guard)
|
||||
const processDead = !isCloudflaredRunning();
|
||||
if (!processDead && Date.now() - svc.lastRestartAt < RESTART_COOLDOWN_MS) return;
|
||||
|
||||
// Alive check: process up + URL responds → skip
|
||||
// Alive check: process up + BOTH direct & public URL respond → skip
|
||||
if (isCloudflaredRunning()) {
|
||||
const state = loadState();
|
||||
const publicUrl = state?.shortId ? `https://r${state.shortId}.abc-tunnel.us` : null;
|
||||
if (publicUrl && await probeUrlAlive(publicUrl)) return;
|
||||
const directUrl = state?.tunnelUrl || null;
|
||||
if (publicUrl && directUrl) {
|
||||
const [publicOk, directOk] = await Promise.all([
|
||||
probeUrlAlive(publicUrl),
|
||||
probeUrlAlive(directUrl),
|
||||
]);
|
||||
if (publicOk && directOk) return;
|
||||
}
|
||||
}
|
||||
|
||||
if (!await checkInternet()) return;
|
||||
|
||||
Reference in New Issue
Block a user