diff --git a/electron/backendCrashReport.js b/electron/backendCrashReport.js new file mode 100644 index 00000000..11364eed --- /dev/null +++ b/electron/backendCrashReport.js @@ -0,0 +1,332 @@ +"use strict"; + +const fs = require("node:fs"); +const path = require("node:path"); + +const REPORT_FILENAME = "last-backend-crash.json"; + +/** + * @param {string} storageDir + * @returns {string} + */ +function getLogsDir(storageDir) { + return path.join(storageDir, "logs"); +} + +/** + * @param {string} storageDir + * @returns {string} + */ +function getCrashReportPath(storageDir) { + return path.join(getLogsDir(storageDir), REPORT_FILENAME); +} + +/** + * @param {string} storageDir + * @returns {string} + */ +function getBackendLogPath(storageDir) { + return path.join(getLogsDir(storageDir), "meshchatx.log"); +} + +/** + * @param {string} text + * @returns {string} + */ +function tailText(text, maxChars = 12000) { + if (!text || typeof text !== "string") { + return ""; + } + if (text.length <= maxChars) { + return text; + } + return text.slice(-maxChars); +} + +/** + * @param {object} crash + * @returns {object} + */ +function normalizeCrash(crash) { + if (!crash || typeof crash !== "object") { + return null; + } + const code = crash.code != null ? Number(crash.code) : null; + const stdout = tailText(crash.stdout || ""); + const stderr = tailText(crash.stderr || ""); + const at = crash.at != null ? Number(crash.at) : Date.now(); + return { + code, + stdout, + stderr, + at, + platform: crash.platform || process.platform, + pid: crash.pid != null ? Number(crash.pid) : null, + }; +} + +/** + * @param {string} storageDir + * @param {object} crash + */ +function persistCrashReport(storageDir, crash) { + if (!storageDir) { + return null; + } + const normalized = normalizeCrash(crash); + if (!normalized) { + return null; + } + const logsDir = getLogsDir(storageDir); + fs.mkdirSync(logsDir, { recursive: true }); + const reportPath = getCrashReportPath(storageDir); + const payload = { + ...normalized, + savedAt: Date.now(), + reportPath, + backendLogPath: getBackendLogPath(storageDir), + }; + fs.writeFileSync(reportPath, `${JSON.stringify(payload, null, 2)}\n`, "utf8"); + return payload; +} + +/** + * @param {string} storageDir + * @returns {object|null} + */ +function loadCrashReport(storageDir) { + if (!storageDir) { + return null; + } + try { + const raw = fs.readFileSync(getCrashReportPath(storageDir), "utf8"); + const parsed = JSON.parse(raw); + return normalizeCrash(parsed); + } catch { + return null; + } +} + +/** + * @param {string} storageDir + */ +function clearCrashReport(storageDir) { + if (!storageDir) { + return; + } + try { + fs.unlinkSync(getCrashReportPath(storageDir)); + } catch { + /* already absent */ + } +} + +/** + * @param {string} text + * @returns {object|null} + */ +function parseMemoryLogLine(text) { + if (!text || typeof text !== "string") { + return null; + } + for (const rawLine of text.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line.startsWith("MESHCHAT_MEMORY:")) { + continue; + } + const parsed = {}; + const payload = line.slice("MESHCHAT_MEMORY:".length).trim(); + for (const token of payload.split(/\s+/)) { + const idx = token.indexOf("="); + if (idx <= 0) { + continue; + } + const key = token.slice(0, idx); + const value = token.slice(idx + 1); + if (key === "total_mb" || key === "available_mb" || key === "percent_used") { + const num = Number(value); + if (Number.isFinite(num)) { + parsed[key] = num; + } + } else { + parsed[key] = value; + } + } + return Object.keys(parsed).length > 0 ? parsed : null; + } + return null; +} + +/** + * @param {string} stderr + * @param {string} stdout + * @param {number|null} exitCode + * @returns {{ category: string, summary: string, hints: string[] }} + */ +function diagnoseBackendCrash(stderr, stdout, exitCode) { + const combined = `${stdout || ""}\n${stderr || ""}`.toLowerCase(); + const rawCombined = `${stdout || ""}\n${stderr || ""}`; + const hints = []; + const memoryLine = parseMemoryLogLine(rawCombined); + + if (memoryLine) { + const available = Number(memoryLine.available_mb); + const action = String(memoryLine.action || ""); + if (action === "abort" || (Number.isFinite(available) && available < 200)) { + hints.push("Close other applications to free RAM, then relaunch."); + hints.push("On low-memory devices, try Emergency Mode from the crash screen."); + return { + category: "memory", + summary: Number.isFinite(available) + ? `The backend reported critically low memory (${available.toFixed(0)} MB available).` + : "The backend aborted startup due to critically low memory.", + hints, + }; + } + if (action === "warn") { + hints.push("Emergency Mode reduces startup memory use on 4 GB devices."); + return { + category: "memory", + summary: Number.isFinite(available) + ? `The backend detected limited memory (${available.toFixed(0)} MB available).` + : "The backend detected limited system memory during startup.", + hints, + }; + } + } + + if ( + combined.includes("memoryerror") || + combined.includes("out of memory") || + combined.includes("cannot allocate memory") || + combined.includes("oom") + ) { + hints.push("Close other applications to free RAM, then relaunch."); + hints.push("On low-memory devices, try Emergency Mode from the crash screen."); + return { + category: "memory", + summary: "The backend likely ran out of memory during startup.", + hints, + }; + } + + if ( + combined.includes("eacces") || + combined.includes("eperm") || + combined.includes("permission denied") || + combined.includes("access is denied") + ) { + hints.push("Check that MeshChatX can write to its data folder."); + hints.push("Avoid running from protected locations; portable installs should stay on a writable drive."); + return { + category: "permission", + summary: "The backend could not read or write a required file or folder.", + hints, + }; + } + + if ( + combined.includes("address already in use") || + combined.includes("eaddrinuse") || + combined.includes("only one usage of each socket") || + combined.includes("port 9337") + ) { + hints.push( + "Another backend copy may still be running. End ReticulumMeshChatX.exe in Task Manager, then relaunch." + ); + return { + category: "port-conflict", + summary: "Port 9337 is already in use by another process.", + hints, + }; + } + + if ( + combined.includes("dll load failed") || + combined.includes("vcruntime") || + combined.includes("api-ms-win") || + combined.includes("the specified module could not be found") + ) { + hints.push("Install the latest Microsoft Visual C++ Redistributable (x64), then relaunch."); + return { + category: "runtime", + summary: "A required Windows runtime library may be missing.", + hints, + }; + } + + if (combined.includes("database") && (combined.includes("corrupt") || combined.includes("malformed"))) { + hints.push("Try Emergency Mode, or rename the storage folder after backing it up."); + return { + category: "database", + summary: "The local database may be corrupted.", + hints, + }; + } + + if (combined.includes("!!! application crash detected !!!")) { + hints.push("Open the saved crash report and meshchatx.log for the full diagnostic trace."); + return { + category: "python-crash", + summary: "The Python backend crash recovery system captured a fatal error.", + hints, + }; + } + + if (exitCode === 3221226505 || combined.includes("access violation") || combined.includes("segfault")) { + hints.push( + "This can happen on older GPUs or low-memory systems. Try creating a disable-gpu file in the data folder." + ); + return { + category: "native-crash", + summary: "The backend process crashed natively (access violation).", + hints, + }; + } + + if (exitCode != null && exitCode !== 0) { + hints.push("Open the logs folder below and send meshchatx.log plus the crash report when asking for support."); + return { + category: "exit-code", + summary: `The backend exited with code ${exitCode}.`, + hints, + }; + } + + return { + category: "unknown", + summary: "The backend stopped before the UI could connect.", + hints: ["Review the saved crash report and meshchatx.log for details."], + }; +} + +/** + * @param {string} storageDir + * @param {string} reticulumConfigDir + * @returns {object} + */ +function getDiagnosticPaths(storageDir, reticulumConfigDir) { + const logsDir = storageDir ? getLogsDir(storageDir) : null; + return { + storageDir: storageDir || null, + reticulumConfigDir: reticulumConfigDir || null, + logsDir, + backendLogPath: storageDir ? getBackendLogPath(storageDir) : null, + crashReportPath: storageDir ? getCrashReportPath(storageDir) : null, + }; +} + +module.exports = { + REPORT_FILENAME, + clearCrashReport, + diagnoseBackendCrash, + getBackendLogPath, + getCrashReportPath, + getDiagnosticPaths, + getLogsDir, + loadCrashReport, + normalizeCrash, + parseMemoryLogLine, + persistCrashReport, + tailText, +}; diff --git a/electron/backendProcess.js b/electron/backendProcess.js index 47b63d15..2b452ce3 100644 --- a/electron/backendProcess.js +++ b/electron/backendProcess.js @@ -2,8 +2,16 @@ const path = require("node:path"); const { spawn: defaultSpawn } = require("child_process"); const { verifyBackendIntegrity } = require("./backendIntegrity"); +const { + clearCrashReport, + getDiagnosticPaths, + getLogsDir, + loadCrashReport, + persistCrashReport, +} = require("./backendCrashReport"); +const { killOrphanBackendProcesses } = require("./backendProcessWin"); -const LOG_LINE_CAP = 100; +const LOG_LINE_CAP = 200; function createInitialRuntimeState() { return { @@ -34,6 +42,17 @@ function createBackendProcessManager(deps) { let resolvedExePath = null; let userProvidedArguments = []; + const storageDir = () => getDefaultStorageDir(); + const reticulumConfigDir = () => getDefaultReticulumConfigDir(); + + function hydratePersistedCrash() { + const persisted = loadCrashReport(storageDir()); + if (persisted) { + lastCrash = persisted; + } + } + hydratePersistedCrash(); + function isRunning() { return !!childProcess && childProcess.exitCode === null && childProcess.signalCode === null; } @@ -63,6 +82,15 @@ function createBackendProcessManager(deps) { return lastCrash; } + function getStartupDiagnostics() { + const paths = getDiagnosticPaths(storageDir(), reticulumConfigDir()); + return { + runtime: getRuntimeState(), + crash: lastCrash, + paths, + }; + } + function setUserProvidedArguments(args) { userProvidedArguments = Array.isArray(args) ? args : []; } @@ -72,6 +100,35 @@ function createBackendProcessManager(deps) { return resolvedExePath; } + function recordCrash(code) { + const logs = getJoinedLogs(); + lastCrash = { + code, + stdout: logs.stdout, + stderr: logs.stderr, + at: Date.now(), + pid: runtimeState.pid, + platform: process.platform, + }; + try { + persistCrashReport(storageDir(), lastCrash); + } catch (error) { + log(`Failed to persist backend crash report: ${error && error.message ? error.message : error}`); + } + } + + function notifyStartupFailure(code) { + const paths = getDiagnosticPaths(storageDir(), reticulumConfigDir()); + notifyRenderer("backend-startup-failed", { + code, + at: lastCrash?.at ?? Date.now(), + stdout: lastCrash?.stdout || "", + stderr: lastCrash?.stderr || "", + paths, + }); + notifyRenderer("backend-process-exited", { code, at: lastCrash?.at ?? Date.now() }); + } + function attachChildHandlers(proc) { logBuffers = { stdout: [], stderr: [] }; @@ -105,18 +162,17 @@ function createBackendProcessManager(deps) { return; } - const logs = getJoinedLogs(); - lastCrash = { - code, - stdout: logs.stdout, - stderr: logs.stderr, - at: Date.now(), - }; + recordCrash(code); + + const page = getMainWindowPageKind(); + if (page === "loading") { + notifyStartupFailure(code); + return; + } notifyRenderer("backend-process-exited", { code, at: lastCrash.at }); - const page = getMainWindowPageKind(); - if (page === "loading" || page === "app") { + if (page === "app") { return; } @@ -128,6 +184,16 @@ function createBackendProcessManager(deps) { }); } + function buildSpawnEnv() { + const logsDir = getLogsDir(storageDir()); + return { + ...process.env, + MESHCHAT_LOG_DIR: logsDir, + MESHCHAT_STORAGE_DIR: storageDir(), + MESHCHAT_RETICULUM_CONFIG_DIR: reticulumConfigDir(), + }; + } + async function spawnBackend(exePath, integrityStatusRef) { if (!exePath) { throw new Error("Backend executable path is not set."); @@ -136,6 +202,11 @@ function createBackendProcessManager(deps) { return { ok: true, alreadyRunning: true }; } + const removed = killOrphanBackendProcesses(null); + if (removed > 0) { + log(`Removed ${removed} orphan backend process(es) before startup.`); + } + resolvedExePath = exePath; const exeDir = path.dirname(exePath); integrityStatusRef.backend = verifyBackendIntegrity(exeDir); @@ -154,13 +225,16 @@ function createBackendProcessManager(deps) { const requiredArguments = ["--headless", "--port", "9337"]; if (!userProvidedArguments.includes("--reticulum-config-dir")) { - requiredArguments.push("--reticulum-config-dir", getDefaultReticulumConfigDir()); + requiredArguments.push("--reticulum-config-dir", reticulumConfigDir()); } if (!userProvidedArguments.includes("--storage-dir")) { - requiredArguments.push("--storage-dir", getDefaultStorageDir()); + requiredArguments.push("--storage-dir", storageDir()); } - const proc = spawnFn(exePath, [...requiredArguments, ...userProvidedArguments]); + const proc = spawnFn(exePath, [...requiredArguments, ...userProvidedArguments], { + env: buildSpawnEnv(), + windowsHide: true, + }); if (!proc || !proc.pid) { throw new Error("Failed to start backend process (no PID)."); } @@ -189,6 +263,18 @@ function createBackendProcessManager(deps) { if (childProcess.exitCode !== null || childProcess.signalCode !== null) { return; } + if (process.platform === "win32") { + try { + const { execFileSync } = require("node:child_process"); + execFileSync("taskkill", ["/F", "/T", "/PID", String(childProcess.pid)], { + stdio: "ignore", + windowsHide: true, + }); + } catch (error) { + log(error); + } + return; + } childProcess.kill(signal); } @@ -215,6 +301,13 @@ function createBackendProcessManager(deps) { return { ok: true }; } + function markBackendHealthy() { + runtimeState.lastExitCode = null; + runtimeState.lastError = ""; + clearCrashReport(storageDir()); + lastCrash = null; + } + return { createInitialRuntimeState, setUserProvidedArguments, @@ -224,6 +317,8 @@ function createBackendProcessManager(deps) { openCrashReport, getRuntimeState, getLastCrash, + getStartupDiagnostics, + markBackendHealthy, getChildProcess, isRunning, killChild, diff --git a/electron/backendProcessWin.js b/electron/backendProcessWin.js new file mode 100644 index 00000000..a84756df --- /dev/null +++ b/electron/backendProcessWin.js @@ -0,0 +1,79 @@ +"use strict"; + +const { execFileSync } = require("node:child_process"); + +const BACKEND_IMAGE = "ReticulumMeshChatX.exe"; + +/** + * @param {number|null|undefined} ownPid + * @returns {number[]} + */ +function listBackendPids(ownPid = null) { + if (process.platform !== "win32") { + return []; + } + try { + const out = execFileSync("tasklist", ["/FI", `IMAGENAME eq ${BACKEND_IMAGE}`, "/FO", "CSV", "/NH"], { + encoding: "utf8", + windowsHide: true, + }); + const pids = []; + for (const line of out.split(/\r?\n/)) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("INFO:")) { + continue; + } + const match = trimmed.match(/"[^"]+","(\d+)"/); + if (!match) { + continue; + } + const pid = Number(match[1]); + if (!Number.isFinite(pid) || pid <= 0) { + continue; + } + if (ownPid != null && pid === ownPid) { + continue; + } + pids.push(pid); + } + return pids; + } catch { + return []; + } +} + +/** + * @param {number[]} pids + */ +function killPids(pids) { + if (process.platform !== "win32" || !Array.isArray(pids)) { + return; + } + for (const pid of pids) { + try { + execFileSync("taskkill", ["/F", "/T", "/PID", String(pid)], { + stdio: "ignore", + windowsHide: true, + }); + } catch { + /* process may already be gone */ + } + } +} + +/** + * @param {number|null|undefined} ownPid + * @returns {number} + */ +function killOrphanBackendProcesses(ownPid = null) { + const pids = listBackendPids(ownPid); + killPids(pids); + return pids.length; +} + +module.exports = { + BACKEND_IMAGE, + killOrphanBackendProcesses, + killPids, + listBackendPids, +}; diff --git a/electron/crash.html b/electron/crash.html index 85374ced..6e0c4e31 100644 --- a/electron/crash.html +++ b/electron/crash.html @@ -78,6 +78,8 @@ +

+

{ const btn = event && event.currentTarget ? event.currentTarget : null; diff --git a/electron/loading.html b/electron/loading.html index 537a9ff9..3fe5a81b 100644 --- a/electron/loading.html +++ b/electron/loading.html @@ -59,6 +59,53 @@ id="connection-notice" class="mt-2 min-h-[2rem] text-center text-xs leading-relaxed text-amber-700 dark:text-amber-300" >

+

v0.0.0

@@ -72,6 +119,14 @@ const statusLine = document.getElementById("status-line"); const attemptHint = document.getElementById("attempt-hint"); const connectionNotice = document.getElementById("connection-notice"); + const startupDiagnostics = document.getElementById("startup-diagnostics"); + const startupDiagnosisSummary = document.getElementById("startup-diagnosis-summary"); + const startupDiagnosisLog = document.getElementById("startup-diagnosis-log"); + const startupLogPath = document.getElementById("startup-log-path"); + const btnViewCrashReport = document.getElementById("btn-view-crash-report"); + const btnOpenLogs = document.getElementById("btn-open-logs"); + const btnRestartBackend = document.getElementById("btn-restart-backend"); + const btnEmergencyRelaunch = document.getElementById("btn-emergency-relaunch"); const API_HOST = "127.0.0.1"; const API_PORT = "9337"; @@ -90,6 +145,18 @@ (async function bootstrap() { applyStartupErrorHint(); + wireStartupActions(); + if (window.electron && typeof window.electron.onBackendStartupFailed === "function") { + window.electron.onBackendStartupFailed((payload) => { + handleStartupFailure(payload); + }); + } + try { + if (window.electron && typeof window.electron.backendStartupDiagnostics === "function") { + cachedDiagnostics = await window.electron.backendStartupDiagnostics(); + cachedCrash = cachedDiagnostics?.crash || null; + } + } catch (e) {} try { if (window.electron && typeof window.electron.backendHttpOnly === "function") { const httpOnly = await window.electron.backendHttpOnly(); @@ -101,6 +168,105 @@ check(); })(); + function wireStartupActions() { + if (btnViewCrashReport) { + btnViewCrashReport.addEventListener("click", async () => { + if (!window.electron?.openBackendCrashReport) { + return; + } + await window.electron.openBackendCrashReport(); + }); + } + if (btnOpenLogs) { + btnOpenLogs.addEventListener("click", async () => { + const logsDir = cachedDiagnostics?.paths?.logsDir; + if (!logsDir || !window.electron?.openPath) { + return; + } + await window.electron.openPath(logsDir); + }); + } + if (btnRestartBackend) { + btnRestartBackend.addEventListener("click", async () => { + if (!window.electron?.restartBackend) { + return; + } + startupFailed = false; + if (startupDiagnostics) { + startupDiagnostics.classList.add("hidden"); + } + statusLine.textContent = "Retrying backend startup…"; + connectionNotice.textContent = ""; + const result = await window.electron.restartBackend(); + if (!result?.ok) { + connectionNotice.textContent = result?.error || "Backend restart failed."; + return; + } + attemptCount = 0; + recentFailures.length = 0; + scheduleCheck(250); + }); + } + if (btnEmergencyRelaunch) { + btnEmergencyRelaunch.addEventListener("click", async () => { + if (!window.electron?.relaunchEmergency) { + return; + } + await window.electron.relaunchEmergency(); + }); + } + } + + function scheduleCheck(delayMs) { + if (pollTimer != null) { + clearTimeout(pollTimer); + } + pollTimer = setTimeout(check, delayMs); + } + + function formatCrashExcerpt(crash) { + if (!crash || typeof crash !== "object") { + return "No backend output was captured."; + } + const stderr = (crash.stderr || "").trim(); + const stdout = (crash.stdout || "").trim(); + const chunk = stderr || stdout; + if (!chunk) { + return "No backend output was captured."; + } + if (chunk.length <= 1800) { + return chunk; + } + return chunk.slice(-1800); + } + + function showStartupDiagnostics(issue) { + if (!startupDiagnostics || !issue) { + return; + } + startupDiagnostics.classList.remove("hidden"); + if (startupDiagnosisSummary) { + const codeText = issue.exitCode != null ? `Exit code ${issue.exitCode}. ` : ""; + startupDiagnosisSummary.textContent = `${codeText}${issue.detail || issue.headline || ""}`; + } + if (startupDiagnosisLog) { + startupDiagnosisLog.textContent = formatCrashExcerpt(issue.crash || cachedCrash); + } + if (startupLogPath) { + startupLogPath.textContent = issue.logHint || ""; + } + } + + function handleStartupFailure(payload) { + startupFailed = true; + cachedCrash = payload || cachedCrash; + if (pollTimer != null) { + clearTimeout(pollTimer); + pollTimer = null; + } + updateStartupNotice(true); + } + function applyStartupErrorHint() { const startupError = startupParams.get("startup_error"); if (startupError !== "backend_unreachable") { @@ -151,6 +317,10 @@ let attemptCount = 0; let runtimeProbeAttempt = 0; let cachedRuntimeState = null; + let cachedDiagnostics = null; + let cachedCrash = null; + let startupFailed = false; + let pollTimer = null; const recentFailures = []; function parseStatusJson(text) { @@ -229,6 +399,8 @@ return helper.classifyConnectionIssue(recentFailures, cachedRuntimeState, { attemptCount: attemptCount, networkWarnAfterAttempts: NETWORK_WARNING_AFTER_ATTEMPTS, + crash: cachedCrash, + paths: cachedDiagnostics?.paths || null, }); } return { @@ -238,18 +410,36 @@ }; } - async function updateStartupNotice() { + async function updateStartupNotice(forceDiagnostics) { + if (startupFailed || forceDiagnostics) { + await refreshBackendRuntimeStateIfNeeded(); + const issue = classifyConnectionIssue(); + statusLine.textContent = issue.headline; + connectionNotice.textContent = issue.detail; + showStartupDiagnostics(issue); + return; + } if (attemptCount < NOTICE_AFTER_ATTEMPTS) { connectionNotice.textContent = ""; + if (startupDiagnostics) { + startupDiagnostics.classList.add("hidden"); + } return; } await refreshBackendRuntimeStateIfNeeded(); const issue = classifyConnectionIssue(); statusLine.textContent = issue.headline; connectionNotice.textContent = issue.detail; + if (issue.reason === "backend-exited") { + startupFailed = true; + showStartupDiagnostics(issue); + } } async function check() { + if (startupFailed) { + return; + } attemptCount += 1; if (attemptCount === 1) { attemptHint.textContent = ""; @@ -262,6 +452,11 @@ for (const protocol of protocolOrder) { const result = await tryOnce(protocol); if (result.ok) { + if (window.electron && typeof window.electron.markBackendHealthy === "function") { + try { + await window.electron.markBackendHealthy(); + } catch (e) {} + } detectedProtocol = result.protocol; statusLine.textContent = "Opening the app…"; attemptHint.textContent = ""; @@ -274,8 +469,8 @@ rememberFailure(result.failure); } } - await updateStartupNotice(); - setTimeout(check, 350); + await updateStartupNotice(false); + scheduleCheck(350); } function onReady() { diff --git a/electron/loadingStatusNotice.js b/electron/loadingStatusNotice.js index 2463b7d0..1bcc9b36 100644 --- a/electron/loadingStatusNotice.js +++ b/electron/loadingStatusNotice.js @@ -12,18 +12,187 @@ return String(value).toLowerCase(); } + function parseMemoryLogLine(text) { + if (!text || typeof text !== "string") { + return null; + } + for (const rawLine of text.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line.startsWith("MESHCHAT_MEMORY:")) { + continue; + } + const parsed = {}; + const payload = line.slice("MESHCHAT_MEMORY:".length).trim(); + for (const token of payload.split(/\s+/)) { + const idx = token.indexOf("="); + if (idx <= 0) { + continue; + } + const key = token.slice(0, idx); + const value = token.slice(idx + 1); + if (key === "total_mb" || key === "available_mb" || key === "percent_used") { + const num = Number(value); + if (Number.isFinite(num)) { + parsed[key] = num; + } + } else { + parsed[key] = value; + } + } + return Object.keys(parsed).length > 0 ? parsed : null; + } + return null; + } + + /** + * @param {string} stderr + * @param {string} stdout + * @param {number|null|undefined} exitCode + * @returns {{ category: string, summary: string, hints: string[] }} + */ + function diagnoseBackendCrash(stderr, stdout, exitCode) { + const combined = `${stdout || ""}\n${stderr || ""}`.toLowerCase(); + const rawCombined = `${stdout || ""}\n${stderr || ""}`; + const hints = []; + const memoryLine = parseMemoryLogLine(rawCombined); + + if (memoryLine) { + const available = Number(memoryLine.available_mb); + const action = String(memoryLine.action || ""); + if (action === "abort" || (Number.isFinite(available) && available < 200)) { + hints.push("Close other apps to free RAM, then relaunch."); + hints.push("Try Emergency Mode if the device has 4 GB RAM or less."); + return { + category: "memory", + summary: Number.isFinite(available) + ? `The backend reported critically low memory (${available.toFixed(0)} MB available).` + : "The backend aborted startup due to critically low memory.", + hints, + }; + } + if (action === "warn") { + hints.push("Try Emergency Mode if the device has 4 GB RAM or less."); + return { + category: "memory", + summary: Number.isFinite(available) + ? `The backend detected limited memory (${available.toFixed(0)} MB available).` + : "The backend detected limited system memory during startup.", + hints, + }; + } + } + + if ( + combined.includes("memoryerror") || + combined.includes("out of memory") || + combined.includes("cannot allocate memory") + ) { + hints.push("Close other apps to free RAM, then relaunch."); + hints.push("Try Emergency Mode if the device has 4 GB RAM or less."); + return { + category: "memory", + summary: "The backend likely ran out of memory during startup.", + hints, + }; + } + + if ( + combined.includes("address already in use") || + combined.includes("eaddrinuse") || + combined.includes("only one usage of each socket") + ) { + hints.push("End any ReticulumMeshChatX.exe process in Task Manager, then relaunch."); + return { + category: "port-conflict", + summary: "Port 9337 is already in use.", + hints, + }; + } + + if ( + combined.includes("eacces") || + combined.includes("eperm") || + combined.includes("permission denied") || + combined.includes("access is denied") + ) { + hints.push("Ensure the MeshChatX data folder is writable."); + return { + category: "permission", + summary: "The backend could not access a required file or folder.", + hints, + }; + } + + if (combined.includes("!!! application crash detected !!!")) { + hints.push("Open meshchatx.log for the full diagnostic trace from the crash recovery engine."); + return { + category: "python-crash", + summary: "The backend crash recovery system reported a fatal Python error.", + hints, + }; + } + + if (exitCode != null && exitCode !== 0) { + hints.push("Open the logs folder below and review meshchatx.log and last-backend-crash.json."); + return { + category: "exit-code", + summary: `The backend exited with code ${exitCode}.`, + hints, + }; + } + + return { + category: "unknown", + summary: "The backend stopped before the UI could connect.", + hints: ["Review the saved crash report and meshchatx.log for details."], + }; + } + + function formatLogPathHint(paths) { + if (!paths || typeof paths !== "object") { + return ""; + } + if (paths.backendLogPath) { + return `Logs: ${paths.backendLogPath}`; + } + if (paths.logsDir) { + return `Logs folder: ${paths.logsDir}`; + } + return ""; + } + + /** + * @param {unknown[]} failures + * @param {object|null} runtimeState + * @param {object} options + * @returns {object} + */ function classifyConnectionIssue(failures, runtimeState, options) { const entries = Array.isArray(failures) ? failures : []; const state = runtimeState && typeof runtimeState === "object" ? runtimeState : null; const opts = options && typeof options === "object" ? options : {}; const attemptCount = Number(opts.attemptCount) || 0; const networkWarnAfterAttempts = Number(opts.networkWarnAfterAttempts) || 24; + const crash = opts.crash && typeof opts.crash === "object" ? opts.crash : null; + const paths = opts.paths && typeof opts.paths === "object" ? opts.paths : null; if (state && state.running === false && state.lastExitCode != null) { + const diagnosis = diagnoseBackendCrash(crash?.stderr || "", crash?.stdout || "", state.lastExitCode); + const logHint = formatLogPathHint(paths); + const detailParts = [diagnosis.summary, ...diagnosis.hints]; + if (logHint) { + detailParts.push(logHint); + } return { reason: "backend-exited", headline: "The backend process stopped unexpectedly.", - detail: "Please restart MeshChatX. If this keeps happening, review crash logs.", + detail: detailParts.join(" "), + category: diagnosis.category, + exitCode: state.lastExitCode, + hints: diagnosis.hints, + logHint, + crash, + paths, }; } @@ -92,7 +261,9 @@ } return { - classifyConnectionIssue: classifyConnectionIssue, - classifyFetchError: classifyFetchError, + classifyConnectionIssue, + classifyFetchError, + diagnoseBackendCrash, + formatLogPathHint, }; }); diff --git a/electron/main.js b/electron/main.js index d12ffae9..04534c01 100644 --- a/electron/main.js +++ b/electron/main.js @@ -181,6 +181,15 @@ ipcMain.handle("backend-runtime-state", () => { return getBackendManager().getRuntimeState(); }); +ipcMain.handle("backend-startup-diagnostics", () => { + return getBackendManager().getStartupDiagnostics(); +}); + +ipcMain.handle("mark-backend-healthy", () => { + getBackendManager().markBackendHealthy(); + return { ok: true }; +}); + ipcMain.handle("restart-backend", async () => { return await getBackendManager().restartBackend(integrityStatus); }); @@ -479,6 +488,7 @@ async function loadBackendCrashPage(crash) { const stdoutBase64 = Buffer.from((crash && crash.stdout) || "").toString("base64"); const stderrBase64 = Buffer.from((crash && crash.stderr) || "").toString("base64"); const code = crash && crash.code != null ? String(crash.code) : ""; + const paths = getBackendManager().getStartupDiagnostics().paths; if (!mainWindow || mainWindow.isDestroyed()) { await dialog.showMessageBox({ @@ -497,6 +507,8 @@ async function loadBackendCrashPage(crash) { code: code, stdout: stdoutBase64, stderr: stderrBase64, + logPath: paths?.backendLogPath || "", + crashReportPath: paths?.crashReportPath || "", }, }); } diff --git a/electron/preload.js b/electron/preload.js index eda33b85..3b686904 100644 --- a/electron/preload.js +++ b/electron/preload.js @@ -99,6 +99,12 @@ contextBridge.exposeInMainWorld("electron", { backendRuntimeState: async function () { return await ipcRenderer.invoke("backend-runtime-state"); }, + backendStartupDiagnostics: async function () { + return await ipcRenderer.invoke("backend-startup-diagnostics"); + }, + markBackendHealthy: async function () { + return await ipcRenderer.invoke("mark-backend-healthy"); + }, restartBackend: async function () { return await ipcRenderer.invoke("restart-backend"); }, @@ -111,4 +117,10 @@ contextBridge.exposeInMainWorld("electron", { } ipcRenderer.on("backend-process-exited", (_event, payload) => callback(payload)); }, + onBackendStartupFailed: function (callback) { + if (typeof callback !== "function") { + return; + } + ipcRenderer.on("backend-startup-failed", (_event, payload) => callback(payload)); + }, }); diff --git a/meshchatx/meshchat.py b/meshchatx/meshchat.py index 728cb378..18720321 100644 --- a/meshchatx/meshchat.py +++ b/meshchatx/meshchat.py @@ -142,7 +142,12 @@ from meshchatx.src.backend.nomadnet_utils import ( ) from meshchatx.src.backend.page_node_manager import PageNodeManager from meshchatx.src.backend.persistent_log_handler import PersistentLogHandler -from meshchatx.src.backend.recovery import CrashRecovery, HealthMonitor +from meshchatx.src.backend.recovery import ( + CrashRecovery, + HealthMonitor, + evaluate_startup_memory, + format_memory_log_line, +) from meshchatx.src.backend import reticulum_pathfinding from meshchatx.src.backend.rnprobe_handler import RNProbeHandler from meshchatx.src.backend.sideband_commands import SidebandCommands @@ -19100,6 +19105,19 @@ def main(): # init app (allow optional one-shot backup/restore before running) rns_log_cli = (args.rns_log_level or "").strip() or None + mem_check = evaluate_startup_memory(args.emergency) + print(format_memory_log_line(mem_check), flush=True) + if mem_check.get("message"): + print(mem_check["message"], flush=True) + if mem_check["action"] == "abort": + print( + "Startup aborted due to critically low memory. " + "Free RAM or relaunch with --emergency.", + file=sys.stderr, + flush=True, + ) + sys.exit(1) + reticulum_meshchat = ReticulumMeshChat( identity, args.storage_dir, diff --git a/meshchatx/src/backend/recovery/__init__.py b/meshchatx/src/backend/recovery/__init__.py index 63afd44b..1ac18bab 100644 --- a/meshchatx/src/backend/recovery/__init__.py +++ b/meshchatx/src/backend/recovery/__init__.py @@ -2,5 +2,16 @@ from .crash_recovery import CrashRecovery from .health_monitor import HealthMonitor +from .memory_preflight import ( + evaluate_startup_memory, + format_memory_log_line, + parse_memory_log_line, +) -__all__ = ["CrashRecovery", "HealthMonitor"] +__all__ = [ + "CrashRecovery", + "HealthMonitor", + "evaluate_startup_memory", + "format_memory_log_line", + "parse_memory_log_line", +] diff --git a/meshchatx/src/backend/recovery/crash_recovery.py b/meshchatx/src/backend/recovery/crash_recovery.py index 200b8563..a60ae328 100644 --- a/meshchatx/src/backend/recovery/crash_recovery.py +++ b/meshchatx/src/backend/recovery/crash_recovery.py @@ -379,6 +379,7 @@ class CrashRecovery: "reasoning": "Available system memory is extremely low, leading to allocation failures.", "suggestions": [ "Close other memory-intensive applications.", + "Relaunch in Emergency Mode (--emergency) to reduce startup memory use.", "Add more RAM or swap space to the system.", ], }, @@ -518,12 +519,24 @@ class CrashRecovery: else: potential_causes["ASYNC_RACE"]["probability"] = 0.45 + if exc_type is MemoryError or "memoryerror" in error_type: + potential_causes["OOM"]["probability"] = 0.95 + potential_causes["OOM"]["reasoning"] += ( + " Python raised MemoryError during allocation." + ) + if symptoms["low_mem"]: # If we have a DB error and low memory, OOM is highly likely as the true cause if symptoms["sqlite_in_msg"]: - potential_causes["OOM"]["probability"] = 0.85 + potential_causes["OOM"]["probability"] = max( + potential_causes["OOM"]["probability"], + 0.85, + ) else: - potential_causes["OOM"]["probability"] = 0.75 + potential_causes["OOM"]["probability"] = max( + potential_causes["OOM"]["probability"], + 0.75, + ) if symptoms["rns_config_missing"]: potential_causes["CONFIG_MISSING"]["probability"] = 0.99 @@ -739,7 +752,7 @@ class CrashRecovery: file.write( f"- Memory: {mem.percent}% used ({results['available_mem_mb']:.1f} MB available)\n", ) - if mem.percent > 95: + if mem.percent > 95 or results["available_mem_mb"] < 400: results["low_memory"] = True file.write(" [CRITICAL] System memory is dangerously low!\n") diff --git a/meshchatx/src/backend/recovery/memory_preflight.py b/meshchatx/src/backend/recovery/memory_preflight.py new file mode 100644 index 00000000..cc91b3a7 --- /dev/null +++ b/meshchatx/src/backend/recovery/memory_preflight.py @@ -0,0 +1,106 @@ +# SPDX-License-Identifier: 0BSD + +"""Startup memory checks before heavy backend initialization.""" + +from __future__ import annotations + +import contextlib + +import psutil + +MEMORY_LOG_PREFIX = "MESHCHAT_MEMORY:" + +_ABORT_AVAILABLE_MB = 150 +_CRITICAL_WARN_AVAILABLE_MB = 400 +_WARN_AVAILABLE_MB = 700 +_LOW_TOTAL_MB = 5120 +_CRITICAL_WARN_PERCENT = 90 +_WARN_PERCENT = 85 + + +def get_system_memory_snapshot() -> dict: + """Return host memory metrics in megabytes and percent used.""" + mem = psutil.virtual_memory() + return { + "total_mb": mem.total / (1024**2), + "available_mb": mem.available / (1024**2), + "percent_used": float(mem.percent), + } + + +def evaluate_startup_memory(emergency: bool = False) -> dict: + """Classify startup memory pressure and choose a guard action.""" + snapshot = get_system_memory_snapshot() + available_mb = snapshot["available_mb"] + total_mb = snapshot["total_mb"] + percent_used = snapshot["percent_used"] + + action = "ok" + message = "" + + if available_mb < _ABORT_AVAILABLE_MB: + action = "abort" + message = ( + "Available memory is critically low " + f"({available_mb:.0f} MB). Close other apps or relaunch in Emergency Mode." + ) + elif available_mb < _CRITICAL_WARN_AVAILABLE_MB or ( + total_mb <= _LOW_TOTAL_MB and percent_used >= _CRITICAL_WARN_PERCENT + ): + action = "warn" + message = ( + "Low system memory detected " + f"({available_mb:.0f} MB available). Emergency Mode is strongly recommended." + ) + elif available_mb < _WARN_AVAILABLE_MB or ( + total_mb <= _LOW_TOTAL_MB and percent_used >= _WARN_PERCENT + ): + action = "warn" + message = ( + "System memory is limited " + f"({available_mb:.0f} MB available). Emergency Mode is recommended on 4 GB devices." + ) + + return { + **snapshot, + "action": action, + "message": message, + "emergency_requested": emergency, + "low_memory": action in ("abort", "warn"), + } + + +def format_memory_log_line(result: dict) -> str: + """Emit a single machine-readable startup memory line for Electron logs.""" + return ( + f"{MEMORY_LOG_PREFIX} total_mb={result['total_mb']:.1f} " + f"available_mb={result['available_mb']:.1f} " + f"percent_used={result['percent_used']:.1f} " + f"action={result['action']} " + f"emergency={str(result.get('emergency_requested', False)).lower()}" + ) + + +def parse_memory_log_line(text: str) -> dict | None: + """Parse a MESHCHAT_MEMORY startup line from captured backend output.""" + if not text: + return None + for raw_line in text.splitlines(): + line = raw_line.strip() + if not line.startswith(MEMORY_LOG_PREFIX): + continue + payload = line[len(MEMORY_LOG_PREFIX) :].strip() + parsed: dict[str, str | float | bool] = {} + for token in payload.split(): + if "=" not in token: + continue + key, value = token.split("=", 1) + if key in ("total_mb", "available_mb", "percent_used"): + with contextlib.suppress(ValueError): + parsed[key] = float(value) + elif key == "emergency": + parsed[key] = value.lower() in ("true", "1", "yes") + else: + parsed[key] = value + return parsed if parsed else None + return None diff --git a/meshchatx/src/path_utils.py b/meshchatx/src/path_utils.py index 3099579c..055cc962 100644 --- a/meshchatx/src/path_utils.py +++ b/meshchatx/src/path_utils.py @@ -16,6 +16,10 @@ def resolve_log_dir(): if env_dir: candidates.append(env_dir) + storage_dir = os.environ.get("MESHCHAT_STORAGE_DIR") + if storage_dir: + candidates.append(os.path.join(storage_dir, "logs")) + candidates.append("/config/logs") if os.name == "nt": diff --git a/tests/backend/test_crash_recovery.py b/tests/backend/test_crash_recovery.py index afd82799..3d3c973a 100644 --- a/tests/backend/test_crash_recovery.py +++ b/tests/backend/test_crash_recovery.py @@ -171,6 +171,16 @@ class TestCrashRecovery(unittest.TestCase): self.assertIsNotNone(oom_cause) self.assertEqual(oom_cause["probability"], 85) + def test_heuristic_analysis_memoryerror(self): + causes = self.recovery._analyze_cause( + MemoryError, + MemoryError("unable to allocate array"), + {"low_memory": False, "available_mem_mb": 1024}, + ) + oom_cause = next((c for c in causes if "OOM" in c["description"]), None) + self.assertIsNotNone(oom_cause) + self.assertEqual(oom_cause["probability"], 95) + def test_heuristic_analysis_rns_missing(self): """Verify high confidence for missing RNS config.""" exc_type = RuntimeError diff --git a/tests/backend/test_memory_preflight.py b/tests/backend/test_memory_preflight.py new file mode 100644 index 00000000..f900b954 --- /dev/null +++ b/tests/backend/test_memory_preflight.py @@ -0,0 +1,82 @@ +# SPDX-License-Identifier: 0BSD + +from unittest.mock import MagicMock, patch + +from meshchatx.src.backend.recovery.memory_preflight import ( + evaluate_startup_memory, + format_memory_log_line, + parse_memory_log_line, +) + + +def _mock_memory(total_mb, available_mb, percent): + mem = MagicMock() + mem.total = int(total_mb * 1024**2) + mem.available = int(available_mb * 1024**2) + mem.percent = percent + return mem + + +def test_evaluate_startup_memory_ok(): + with patch( + "meshchatx.src.backend.recovery.memory_preflight.psutil.virtual_memory", + return_value=_mock_memory(16384, 8192, 50), + ): + result = evaluate_startup_memory(emergency=False) + assert result["action"] == "ok" + assert result["low_memory"] is False + + +def test_evaluate_startup_memory_warn_on_low_total(): + with patch( + "meshchatx.src.backend.recovery.memory_preflight.psutil.virtual_memory", + return_value=_mock_memory(4096, 900, 78), + ): + result = evaluate_startup_memory(emergency=False) + assert result["action"] == "warn" + assert result["low_memory"] is True + + +def test_evaluate_startup_memory_critical_warn(): + with patch( + "meshchatx.src.backend.recovery.memory_preflight.psutil.virtual_memory", + return_value=_mock_memory(4096, 320, 92), + ): + result = evaluate_startup_memory(emergency=False) + assert result["action"] == "warn" + assert "strongly recommended" in result["message"] + + +def test_evaluate_startup_memory_abort(): + with patch( + "meshchatx.src.backend.recovery.memory_preflight.psutil.virtual_memory", + return_value=_mock_memory(4096, 120, 97), + ): + result = evaluate_startup_memory(emergency=False) + assert result["action"] == "abort" + + +def test_evaluate_startup_memory_already_in_emergency(): + with patch( + "meshchatx.src.backend.recovery.memory_preflight.psutil.virtual_memory", + return_value=_mock_memory(4096, 320, 92), + ): + result = evaluate_startup_memory(emergency=True) + assert result["action"] == "warn" + assert result["emergency_requested"] is True + + +def test_format_and_parse_memory_log_line(): + result = { + "total_mb": 4096.0, + "available_mb": 512.5, + "percent_used": 87.5, + "action": "warn", + "emergency_requested": False, + } + line = format_memory_log_line(result) + assert line.startswith("MESHCHAT_MEMORY:") + parsed = parse_memory_log_line(f"booting\n{line}\n") + assert parsed["available_mb"] == 512.5 + assert parsed["action"] == "warn" + assert parsed["emergency"] == "false" diff --git a/tests/electron/backendCrashReport.test.js b/tests/electron/backendCrashReport.test.js new file mode 100644 index 00000000..4cfdac60 --- /dev/null +++ b/tests/electron/backendCrashReport.test.js @@ -0,0 +1,53 @@ +import { describe, expect, it } from "vitest"; +import { + clearCrashReport, + diagnoseBackendCrash, + getCrashReportPath, + loadCrashReport, + persistCrashReport, +} from "../../electron/backendCrashReport.js"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +describe("electron/backendCrashReport", () => { + it("persists and reloads crash reports under storage logs", () => { + const storageDir = fs.mkdtempSync(path.join(os.tmpdir(), "meshchatx-crash-")); + try { + const saved = persistCrashReport(storageDir, { + code: 17, + stdout: "ok", + stderr: "MemoryError: out of memory", + at: Date.now(), + }); + expect(saved).toBeTruthy(); + expect(fs.existsSync(getCrashReportPath(storageDir))).toBe(true); + const loaded = loadCrashReport(storageDir); + expect(loaded.code).toBe(17); + expect(loaded.stderr).toContain("MemoryError"); + clearCrashReport(storageDir); + expect(loadCrashReport(storageDir)).toBeNull(); + } finally { + fs.rmSync(storageDir, { recursive: true, force: true }); + } + }); + + it("diagnoseBackendCrash detects memory failures", () => { + const diagnosis = diagnoseBackendCrash("MemoryError: allocation failed", "", 1); + expect(diagnosis.category).toBe("memory"); + expect(diagnosis.summary).toContain("memory"); + }); + + it("diagnoseBackendCrash parses MESHCHAT_MEMORY startup lines", () => { + const stdout = + "MESHCHAT_MEMORY: total_mb=4096.0 available_mb=120.0 percent_used=97.0 action=abort emergency=false"; + const diagnosis = diagnoseBackendCrash("", stdout, 1); + expect(diagnosis.category).toBe("memory"); + expect(diagnosis.summary).toContain("120"); + }); + + it("diagnoseBackendCrash detects port conflicts", () => { + const diagnosis = diagnoseBackendCrash("", "OSError: [Errno 98] Address already in use", 1); + expect(diagnosis.category).toBe("port-conflict"); + }); +}); diff --git a/tests/electron/backendProcess.test.js b/tests/electron/backendProcess.test.js index 351da891..21c98f4f 100644 --- a/tests/electron/backendProcess.test.js +++ b/tests/electron/backendProcess.test.js @@ -49,6 +49,33 @@ describe("electron/backendProcess", () => { expect(manager.getRuntimeState().lastExitCode).toBe(255); }); + it("notifies loading screen when backend exits during startup", async () => { + const notifyRenderer = vi.fn(); + const showCrashPage = vi.fn(); + const manager = createBackendProcessManager({ + log: vi.fn(), + getDefaultStorageDir: () => "/tmp/storage", + getDefaultReticulumConfigDir: () => "/tmp/reticulum", + getMainWindowPageKind: () => "loading", + isQuiting: () => false, + notifyRenderer, + showCrashPage, + spawn: spawnMock, + }); + + manager.setUserProvidedArguments([]); + await manager.spawnBackend("/tmp/ReticulumMeshChatX", { backend: { ok: true, issues: [] } }); + fakeProc.emit("exit", 9); + await new Promise((resolve) => setImmediate(resolve)); + + expect(notifyRenderer).toHaveBeenCalledWith( + "backend-startup-failed", + expect.objectContaining({ code: 9, paths: expect.any(Object) }) + ); + expect(showCrashPage).not.toHaveBeenCalled(); + expect(manager.getLastCrash()).toEqual(expect.objectContaining({ code: 9 })); + }); + it("opens the crash page when backend exits outside the main shell", async () => { const notifyRenderer = vi.fn(); const showCrashPage = vi.fn(); diff --git a/tests/electron/loadingStatusNotice.test.js b/tests/electron/loadingStatusNotice.test.js index c3a306a5..544db178 100644 --- a/tests/electron/loadingStatusNotice.test.js +++ b/tests/electron/loadingStatusNotice.test.js @@ -6,11 +6,38 @@ const notice = require("../../electron/loadingStatusNotice.js"); describe("electron/loadingStatusNotice", () => { it("classifyConnectionIssue prefers backend-exited when runtime not running", () => { - const r = notice.classifyConnectionIssue([], { - running: false, - lastExitCode: 1, - }); + const r = notice.classifyConnectionIssue( + [], + { + running: false, + lastExitCode: 1, + }, + { + crash: { stderr: "MemoryError: out of memory", stdout: "" }, + paths: { backendLogPath: "C:\\Users\\x\\.reticulum-meshchatx\\logs\\meshchatx.log" }, + } + ); expect(r.reason).toBe("backend-exited"); + expect(r.category).toBe("memory"); + expect(r.logHint).toContain("meshchatx.log"); + }); + + it("classifyConnectionIssue parses MESHCHAT_MEMORY startup lines", () => { + const r = notice.classifyConnectionIssue( + [], + { + running: false, + lastExitCode: 1, + }, + { + crash: { + stderr: "", + stdout: "MESHCHAT_MEMORY: total_mb=4096.0 available_mb=120.0 percent_used=97.0 action=abort emergency=false", + }, + } + ); + expect(r.category).toBe("memory"); + expect(r.detail).toContain("120"); }); it("classifyConnectionIssue flags loopback blocked", () => {