mirror of
https://github.com/Quad4-Software/MeshChatX.git
synced 2026-08-18 09:49:09 -04:00
feat(crash-reporting): implement backend crash reporting and diagnostics, including memory checks and better logging
This commit is contained in:
Ivan
parent
4b6850f3ee
commit
88193cc22e
18 changed files with 1291 additions and 29 deletions
332
electron/backendCrashReport.js
Normal file
332
electron/backendCrashReport.js
Normal file
|
|
@ -0,0 +1,332 @@
|
|||
"use strict";
|
||||
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
|
||||
const REPORT_FILENAME = "last-backend-crash.json";
|
||||
|
||||
/**
|
||||
* @param {string} storageDir
|
||||
* @returns {string}
|
||||
*/
|
||||
function getLogsDir(storageDir) {
|
||||
return path.join(storageDir, "logs");
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} storageDir
|
||||
* @returns {string}
|
||||
*/
|
||||
function getCrashReportPath(storageDir) {
|
||||
return path.join(getLogsDir(storageDir), REPORT_FILENAME);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} storageDir
|
||||
* @returns {string}
|
||||
*/
|
||||
function getBackendLogPath(storageDir) {
|
||||
return path.join(getLogsDir(storageDir), "meshchatx.log");
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} text
|
||||
* @returns {string}
|
||||
*/
|
||||
function tailText(text, maxChars = 12000) {
|
||||
if (!text || typeof text !== "string") {
|
||||
return "";
|
||||
}
|
||||
if (text.length <= maxChars) {
|
||||
return text;
|
||||
}
|
||||
return text.slice(-maxChars);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {object} crash
|
||||
* @returns {object}
|
||||
*/
|
||||
function normalizeCrash(crash) {
|
||||
if (!crash || typeof crash !== "object") {
|
||||
return null;
|
||||
}
|
||||
const code = crash.code != null ? Number(crash.code) : null;
|
||||
const stdout = tailText(crash.stdout || "");
|
||||
const stderr = tailText(crash.stderr || "");
|
||||
const at = crash.at != null ? Number(crash.at) : Date.now();
|
||||
return {
|
||||
code,
|
||||
stdout,
|
||||
stderr,
|
||||
at,
|
||||
platform: crash.platform || process.platform,
|
||||
pid: crash.pid != null ? Number(crash.pid) : null,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} storageDir
|
||||
* @param {object} crash
|
||||
*/
|
||||
function persistCrashReport(storageDir, crash) {
|
||||
if (!storageDir) {
|
||||
return null;
|
||||
}
|
||||
const normalized = normalizeCrash(crash);
|
||||
if (!normalized) {
|
||||
return null;
|
||||
}
|
||||
const logsDir = getLogsDir(storageDir);
|
||||
fs.mkdirSync(logsDir, { recursive: true });
|
||||
const reportPath = getCrashReportPath(storageDir);
|
||||
const payload = {
|
||||
...normalized,
|
||||
savedAt: Date.now(),
|
||||
reportPath,
|
||||
backendLogPath: getBackendLogPath(storageDir),
|
||||
};
|
||||
fs.writeFileSync(reportPath, `${JSON.stringify(payload, null, 2)}\n`, "utf8");
|
||||
return payload;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} storageDir
|
||||
* @returns {object|null}
|
||||
*/
|
||||
function loadCrashReport(storageDir) {
|
||||
if (!storageDir) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
const raw = fs.readFileSync(getCrashReportPath(storageDir), "utf8");
|
||||
const parsed = JSON.parse(raw);
|
||||
return normalizeCrash(parsed);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} storageDir
|
||||
*/
|
||||
function clearCrashReport(storageDir) {
|
||||
if (!storageDir) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
fs.unlinkSync(getCrashReportPath(storageDir));
|
||||
} catch {
|
||||
/* already absent */
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} text
|
||||
* @returns {object|null}
|
||||
*/
|
||||
function parseMemoryLogLine(text) {
|
||||
if (!text || typeof text !== "string") {
|
||||
return null;
|
||||
}
|
||||
for (const rawLine of text.split(/\r?\n/)) {
|
||||
const line = rawLine.trim();
|
||||
if (!line.startsWith("MESHCHAT_MEMORY:")) {
|
||||
continue;
|
||||
}
|
||||
const parsed = {};
|
||||
const payload = line.slice("MESHCHAT_MEMORY:".length).trim();
|
||||
for (const token of payload.split(/\s+/)) {
|
||||
const idx = token.indexOf("=");
|
||||
if (idx <= 0) {
|
||||
continue;
|
||||
}
|
||||
const key = token.slice(0, idx);
|
||||
const value = token.slice(idx + 1);
|
||||
if (key === "total_mb" || key === "available_mb" || key === "percent_used") {
|
||||
const num = Number(value);
|
||||
if (Number.isFinite(num)) {
|
||||
parsed[key] = num;
|
||||
}
|
||||
} else {
|
||||
parsed[key] = value;
|
||||
}
|
||||
}
|
||||
return Object.keys(parsed).length > 0 ? parsed : null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} stderr
|
||||
* @param {string} stdout
|
||||
* @param {number|null} exitCode
|
||||
* @returns {{ category: string, summary: string, hints: string[] }}
|
||||
*/
|
||||
function diagnoseBackendCrash(stderr, stdout, exitCode) {
|
||||
const combined = `${stdout || ""}\n${stderr || ""}`.toLowerCase();
|
||||
const rawCombined = `${stdout || ""}\n${stderr || ""}`;
|
||||
const hints = [];
|
||||
const memoryLine = parseMemoryLogLine(rawCombined);
|
||||
|
||||
if (memoryLine) {
|
||||
const available = Number(memoryLine.available_mb);
|
||||
const action = String(memoryLine.action || "");
|
||||
if (action === "abort" || (Number.isFinite(available) && available < 200)) {
|
||||
hints.push("Close other applications to free RAM, then relaunch.");
|
||||
hints.push("On low-memory devices, try Emergency Mode from the crash screen.");
|
||||
return {
|
||||
category: "memory",
|
||||
summary: Number.isFinite(available)
|
||||
? `The backend reported critically low memory (${available.toFixed(0)} MB available).`
|
||||
: "The backend aborted startup due to critically low memory.",
|
||||
hints,
|
||||
};
|
||||
}
|
||||
if (action === "warn") {
|
||||
hints.push("Emergency Mode reduces startup memory use on 4 GB devices.");
|
||||
return {
|
||||
category: "memory",
|
||||
summary: Number.isFinite(available)
|
||||
? `The backend detected limited memory (${available.toFixed(0)} MB available).`
|
||||
: "The backend detected limited system memory during startup.",
|
||||
hints,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
combined.includes("memoryerror") ||
|
||||
combined.includes("out of memory") ||
|
||||
combined.includes("cannot allocate memory") ||
|
||||
combined.includes("oom")
|
||||
) {
|
||||
hints.push("Close other applications to free RAM, then relaunch.");
|
||||
hints.push("On low-memory devices, try Emergency Mode from the crash screen.");
|
||||
return {
|
||||
category: "memory",
|
||||
summary: "The backend likely ran out of memory during startup.",
|
||||
hints,
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
combined.includes("eacces") ||
|
||||
combined.includes("eperm") ||
|
||||
combined.includes("permission denied") ||
|
||||
combined.includes("access is denied")
|
||||
) {
|
||||
hints.push("Check that MeshChatX can write to its data folder.");
|
||||
hints.push("Avoid running from protected locations; portable installs should stay on a writable drive.");
|
||||
return {
|
||||
category: "permission",
|
||||
summary: "The backend could not read or write a required file or folder.",
|
||||
hints,
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
combined.includes("address already in use") ||
|
||||
combined.includes("eaddrinuse") ||
|
||||
combined.includes("only one usage of each socket") ||
|
||||
combined.includes("port 9337")
|
||||
) {
|
||||
hints.push(
|
||||
"Another backend copy may still be running. End ReticulumMeshChatX.exe in Task Manager, then relaunch."
|
||||
);
|
||||
return {
|
||||
category: "port-conflict",
|
||||
summary: "Port 9337 is already in use by another process.",
|
||||
hints,
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
combined.includes("dll load failed") ||
|
||||
combined.includes("vcruntime") ||
|
||||
combined.includes("api-ms-win") ||
|
||||
combined.includes("the specified module could not be found")
|
||||
) {
|
||||
hints.push("Install the latest Microsoft Visual C++ Redistributable (x64), then relaunch.");
|
||||
return {
|
||||
category: "runtime",
|
||||
summary: "A required Windows runtime library may be missing.",
|
||||
hints,
|
||||
};
|
||||
}
|
||||
|
||||
if (combined.includes("database") && (combined.includes("corrupt") || combined.includes("malformed"))) {
|
||||
hints.push("Try Emergency Mode, or rename the storage folder after backing it up.");
|
||||
return {
|
||||
category: "database",
|
||||
summary: "The local database may be corrupted.",
|
||||
hints,
|
||||
};
|
||||
}
|
||||
|
||||
if (combined.includes("!!! application crash detected !!!")) {
|
||||
hints.push("Open the saved crash report and meshchatx.log for the full diagnostic trace.");
|
||||
return {
|
||||
category: "python-crash",
|
||||
summary: "The Python backend crash recovery system captured a fatal error.",
|
||||
hints,
|
||||
};
|
||||
}
|
||||
|
||||
if (exitCode === 3221226505 || combined.includes("access violation") || combined.includes("segfault")) {
|
||||
hints.push(
|
||||
"This can happen on older GPUs or low-memory systems. Try creating a disable-gpu file in the data folder."
|
||||
);
|
||||
return {
|
||||
category: "native-crash",
|
||||
summary: "The backend process crashed natively (access violation).",
|
||||
hints,
|
||||
};
|
||||
}
|
||||
|
||||
if (exitCode != null && exitCode !== 0) {
|
||||
hints.push("Open the logs folder below and send meshchatx.log plus the crash report when asking for support.");
|
||||
return {
|
||||
category: "exit-code",
|
||||
summary: `The backend exited with code ${exitCode}.`,
|
||||
hints,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
category: "unknown",
|
||||
summary: "The backend stopped before the UI could connect.",
|
||||
hints: ["Review the saved crash report and meshchatx.log for details."],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} storageDir
|
||||
* @param {string} reticulumConfigDir
|
||||
* @returns {object}
|
||||
*/
|
||||
function getDiagnosticPaths(storageDir, reticulumConfigDir) {
|
||||
const logsDir = storageDir ? getLogsDir(storageDir) : null;
|
||||
return {
|
||||
storageDir: storageDir || null,
|
||||
reticulumConfigDir: reticulumConfigDir || null,
|
||||
logsDir,
|
||||
backendLogPath: storageDir ? getBackendLogPath(storageDir) : null,
|
||||
crashReportPath: storageDir ? getCrashReportPath(storageDir) : null,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
REPORT_FILENAME,
|
||||
clearCrashReport,
|
||||
diagnoseBackendCrash,
|
||||
getBackendLogPath,
|
||||
getCrashReportPath,
|
||||
getDiagnosticPaths,
|
||||
getLogsDir,
|
||||
loadCrashReport,
|
||||
normalizeCrash,
|
||||
parseMemoryLogLine,
|
||||
persistCrashReport,
|
||||
tailText,
|
||||
};
|
||||
|
|
@ -2,8 +2,16 @@ const path = require("node:path");
|
|||
const { spawn: defaultSpawn } = require("child_process");
|
||||
|
||||
const { verifyBackendIntegrity } = require("./backendIntegrity");
|
||||
const {
|
||||
clearCrashReport,
|
||||
getDiagnosticPaths,
|
||||
getLogsDir,
|
||||
loadCrashReport,
|
||||
persistCrashReport,
|
||||
} = require("./backendCrashReport");
|
||||
const { killOrphanBackendProcesses } = require("./backendProcessWin");
|
||||
|
||||
const LOG_LINE_CAP = 100;
|
||||
const LOG_LINE_CAP = 200;
|
||||
|
||||
function createInitialRuntimeState() {
|
||||
return {
|
||||
|
|
@ -34,6 +42,17 @@ function createBackendProcessManager(deps) {
|
|||
let resolvedExePath = null;
|
||||
let userProvidedArguments = [];
|
||||
|
||||
const storageDir = () => getDefaultStorageDir();
|
||||
const reticulumConfigDir = () => getDefaultReticulumConfigDir();
|
||||
|
||||
function hydratePersistedCrash() {
|
||||
const persisted = loadCrashReport(storageDir());
|
||||
if (persisted) {
|
||||
lastCrash = persisted;
|
||||
}
|
||||
}
|
||||
hydratePersistedCrash();
|
||||
|
||||
function isRunning() {
|
||||
return !!childProcess && childProcess.exitCode === null && childProcess.signalCode === null;
|
||||
}
|
||||
|
|
@ -63,6 +82,15 @@ function createBackendProcessManager(deps) {
|
|||
return lastCrash;
|
||||
}
|
||||
|
||||
function getStartupDiagnostics() {
|
||||
const paths = getDiagnosticPaths(storageDir(), reticulumConfigDir());
|
||||
return {
|
||||
runtime: getRuntimeState(),
|
||||
crash: lastCrash,
|
||||
paths,
|
||||
};
|
||||
}
|
||||
|
||||
function setUserProvidedArguments(args) {
|
||||
userProvidedArguments = Array.isArray(args) ? args : [];
|
||||
}
|
||||
|
|
@ -72,6 +100,35 @@ function createBackendProcessManager(deps) {
|
|||
return resolvedExePath;
|
||||
}
|
||||
|
||||
function recordCrash(code) {
|
||||
const logs = getJoinedLogs();
|
||||
lastCrash = {
|
||||
code,
|
||||
stdout: logs.stdout,
|
||||
stderr: logs.stderr,
|
||||
at: Date.now(),
|
||||
pid: runtimeState.pid,
|
||||
platform: process.platform,
|
||||
};
|
||||
try {
|
||||
persistCrashReport(storageDir(), lastCrash);
|
||||
} catch (error) {
|
||||
log(`Failed to persist backend crash report: ${error && error.message ? error.message : error}`);
|
||||
}
|
||||
}
|
||||
|
||||
function notifyStartupFailure(code) {
|
||||
const paths = getDiagnosticPaths(storageDir(), reticulumConfigDir());
|
||||
notifyRenderer("backend-startup-failed", {
|
||||
code,
|
||||
at: lastCrash?.at ?? Date.now(),
|
||||
stdout: lastCrash?.stdout || "",
|
||||
stderr: lastCrash?.stderr || "",
|
||||
paths,
|
||||
});
|
||||
notifyRenderer("backend-process-exited", { code, at: lastCrash?.at ?? Date.now() });
|
||||
}
|
||||
|
||||
function attachChildHandlers(proc) {
|
||||
logBuffers = { stdout: [], stderr: [] };
|
||||
|
||||
|
|
@ -105,18 +162,17 @@ function createBackendProcessManager(deps) {
|
|||
return;
|
||||
}
|
||||
|
||||
const logs = getJoinedLogs();
|
||||
lastCrash = {
|
||||
code,
|
||||
stdout: logs.stdout,
|
||||
stderr: logs.stderr,
|
||||
at: Date.now(),
|
||||
};
|
||||
recordCrash(code);
|
||||
|
||||
const page = getMainWindowPageKind();
|
||||
if (page === "loading") {
|
||||
notifyStartupFailure(code);
|
||||
return;
|
||||
}
|
||||
|
||||
notifyRenderer("backend-process-exited", { code, at: lastCrash.at });
|
||||
|
||||
const page = getMainWindowPageKind();
|
||||
if (page === "loading" || page === "app") {
|
||||
if (page === "app") {
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -128,6 +184,16 @@ function createBackendProcessManager(deps) {
|
|||
});
|
||||
}
|
||||
|
||||
function buildSpawnEnv() {
|
||||
const logsDir = getLogsDir(storageDir());
|
||||
return {
|
||||
...process.env,
|
||||
MESHCHAT_LOG_DIR: logsDir,
|
||||
MESHCHAT_STORAGE_DIR: storageDir(),
|
||||
MESHCHAT_RETICULUM_CONFIG_DIR: reticulumConfigDir(),
|
||||
};
|
||||
}
|
||||
|
||||
async function spawnBackend(exePath, integrityStatusRef) {
|
||||
if (!exePath) {
|
||||
throw new Error("Backend executable path is not set.");
|
||||
|
|
@ -136,6 +202,11 @@ function createBackendProcessManager(deps) {
|
|||
return { ok: true, alreadyRunning: true };
|
||||
}
|
||||
|
||||
const removed = killOrphanBackendProcesses(null);
|
||||
if (removed > 0) {
|
||||
log(`Removed ${removed} orphan backend process(es) before startup.`);
|
||||
}
|
||||
|
||||
resolvedExePath = exePath;
|
||||
const exeDir = path.dirname(exePath);
|
||||
integrityStatusRef.backend = verifyBackendIntegrity(exeDir);
|
||||
|
|
@ -154,13 +225,16 @@ function createBackendProcessManager(deps) {
|
|||
|
||||
const requiredArguments = ["--headless", "--port", "9337"];
|
||||
if (!userProvidedArguments.includes("--reticulum-config-dir")) {
|
||||
requiredArguments.push("--reticulum-config-dir", getDefaultReticulumConfigDir());
|
||||
requiredArguments.push("--reticulum-config-dir", reticulumConfigDir());
|
||||
}
|
||||
if (!userProvidedArguments.includes("--storage-dir")) {
|
||||
requiredArguments.push("--storage-dir", getDefaultStorageDir());
|
||||
requiredArguments.push("--storage-dir", storageDir());
|
||||
}
|
||||
|
||||
const proc = spawnFn(exePath, [...requiredArguments, ...userProvidedArguments]);
|
||||
const proc = spawnFn(exePath, [...requiredArguments, ...userProvidedArguments], {
|
||||
env: buildSpawnEnv(),
|
||||
windowsHide: true,
|
||||
});
|
||||
if (!proc || !proc.pid) {
|
||||
throw new Error("Failed to start backend process (no PID).");
|
||||
}
|
||||
|
|
@ -189,6 +263,18 @@ function createBackendProcessManager(deps) {
|
|||
if (childProcess.exitCode !== null || childProcess.signalCode !== null) {
|
||||
return;
|
||||
}
|
||||
if (process.platform === "win32") {
|
||||
try {
|
||||
const { execFileSync } = require("node:child_process");
|
||||
execFileSync("taskkill", ["/F", "/T", "/PID", String(childProcess.pid)], {
|
||||
stdio: "ignore",
|
||||
windowsHide: true,
|
||||
});
|
||||
} catch (error) {
|
||||
log(error);
|
||||
}
|
||||
return;
|
||||
}
|
||||
childProcess.kill(signal);
|
||||
}
|
||||
|
||||
|
|
@ -215,6 +301,13 @@ function createBackendProcessManager(deps) {
|
|||
return { ok: true };
|
||||
}
|
||||
|
||||
function markBackendHealthy() {
|
||||
runtimeState.lastExitCode = null;
|
||||
runtimeState.lastError = "";
|
||||
clearCrashReport(storageDir());
|
||||
lastCrash = null;
|
||||
}
|
||||
|
||||
return {
|
||||
createInitialRuntimeState,
|
||||
setUserProvidedArguments,
|
||||
|
|
@ -224,6 +317,8 @@ function createBackendProcessManager(deps) {
|
|||
openCrashReport,
|
||||
getRuntimeState,
|
||||
getLastCrash,
|
||||
getStartupDiagnostics,
|
||||
markBackendHealthy,
|
||||
getChildProcess,
|
||||
isRunning,
|
||||
killChild,
|
||||
|
|
|
|||
79
electron/backendProcessWin.js
Normal file
79
electron/backendProcessWin.js
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
"use strict";
|
||||
|
||||
const { execFileSync } = require("node:child_process");
|
||||
|
||||
const BACKEND_IMAGE = "ReticulumMeshChatX.exe";
|
||||
|
||||
/**
|
||||
* @param {number|null|undefined} ownPid
|
||||
* @returns {number[]}
|
||||
*/
|
||||
function listBackendPids(ownPid = null) {
|
||||
if (process.platform !== "win32") {
|
||||
return [];
|
||||
}
|
||||
try {
|
||||
const out = execFileSync("tasklist", ["/FI", `IMAGENAME eq ${BACKEND_IMAGE}`, "/FO", "CSV", "/NH"], {
|
||||
encoding: "utf8",
|
||||
windowsHide: true,
|
||||
});
|
||||
const pids = [];
|
||||
for (const line of out.split(/\r?\n/)) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.startsWith("INFO:")) {
|
||||
continue;
|
||||
}
|
||||
const match = trimmed.match(/"[^"]+","(\d+)"/);
|
||||
if (!match) {
|
||||
continue;
|
||||
}
|
||||
const pid = Number(match[1]);
|
||||
if (!Number.isFinite(pid) || pid <= 0) {
|
||||
continue;
|
||||
}
|
||||
if (ownPid != null && pid === ownPid) {
|
||||
continue;
|
||||
}
|
||||
pids.push(pid);
|
||||
}
|
||||
return pids;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {number[]} pids
|
||||
*/
|
||||
function killPids(pids) {
|
||||
if (process.platform !== "win32" || !Array.isArray(pids)) {
|
||||
return;
|
||||
}
|
||||
for (const pid of pids) {
|
||||
try {
|
||||
execFileSync("taskkill", ["/F", "/T", "/PID", String(pid)], {
|
||||
stdio: "ignore",
|
||||
windowsHide: true,
|
||||
});
|
||||
} catch {
|
||||
/* process may already be gone */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {number|null|undefined} ownPid
|
||||
* @returns {number}
|
||||
*/
|
||||
function killOrphanBackendProcesses(ownPid = null) {
|
||||
const pids = listBackendPids(ownPid);
|
||||
killPids(pids);
|
||||
return pids.length;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
BACKEND_IMAGE,
|
||||
killOrphanBackendProcesses,
|
||||
killPids,
|
||||
listBackendPids,
|
||||
};
|
||||
|
|
@ -78,6 +78,8 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<p id="saved-log-path" class="break-all text-[10px] text-slate-600 dark:text-zinc-400 px-1"></p>
|
||||
|
||||
<div class="space-y-3">
|
||||
<div class="flex flex-col sm:flex-row items-center justify-between gap-2 px-1">
|
||||
<h3
|
||||
|
|
@ -141,6 +143,19 @@
|
|||
// Get data from URL parameters
|
||||
const params = new URLSearchParams(window.location.search);
|
||||
document.getElementById("exit-code").innerText = params.get("code") || "Unknown";
|
||||
const savedLogPath = document.getElementById("saved-log-path");
|
||||
const logPath = params.get("logPath") || "";
|
||||
const crashReportPath = params.get("crashReportPath") || "";
|
||||
if (savedLogPath) {
|
||||
const parts = [];
|
||||
if (logPath) {
|
||||
parts.push(`meshchatx.log: ${logPath}`);
|
||||
}
|
||||
if (crashReportPath) {
|
||||
parts.push(`last-backend-crash.json: ${crashReportPath}`);
|
||||
}
|
||||
savedLogPath.textContent = parts.join(" | ");
|
||||
}
|
||||
|
||||
// Decoded from base64 to handle complex characters safely
|
||||
try {
|
||||
|
|
@ -161,7 +176,7 @@
|
|||
const stderr = document.getElementById("stderr").innerText;
|
||||
const exitCode = document.getElementById("exit-code").innerText;
|
||||
|
||||
const fullReport = `MeshChatX Crash Report\nExit Code: ${exitCode}\n\n--- STDOUT ---\n${stdout}\n\n--- STDERR ---\n${stderr}`;
|
||||
const fullReport = `MeshChatX Crash Report\nExit Code: ${exitCode}\nLog file: ${logPath}\nCrash report: ${crashReportPath}\n\n--- STDOUT ---\n${stdout}\n\n--- STDERR ---\n${stderr}`;
|
||||
|
||||
navigator.clipboard.writeText(fullReport).then(() => {
|
||||
const btn = event && event.currentTarget ? event.currentTarget : null;
|
||||
|
|
|
|||
|
|
@ -59,6 +59,53 @@
|
|||
id="connection-notice"
|
||||
class="mt-2 min-h-[2rem] text-center text-xs leading-relaxed text-amber-700 dark:text-amber-300"
|
||||
></p>
|
||||
<div
|
||||
id="startup-diagnostics"
|
||||
class="mt-3 hidden rounded-xl border border-red-200/80 bg-red-50/70 p-3 text-left dark:border-red-900/50 dark:bg-red-950/30"
|
||||
>
|
||||
<div
|
||||
id="startup-diagnosis-summary"
|
||||
class="text-xs font-medium text-red-800 dark:text-red-200"
|
||||
></div>
|
||||
<pre
|
||||
id="startup-diagnosis-log"
|
||||
class="mt-2 max-h-36 overflow-auto rounded-lg border border-red-200/60 bg-white/80 p-2 font-mono text-[10px] text-red-900 dark:border-red-900/40 dark:bg-zinc-950 dark:text-red-300"
|
||||
></pre>
|
||||
<p
|
||||
id="startup-log-path"
|
||||
class="mt-2 break-all text-[10px] text-red-700 dark:text-red-300"
|
||||
></p>
|
||||
<div class="mt-3 flex flex-wrap justify-center gap-2">
|
||||
<button
|
||||
id="btn-view-crash-report"
|
||||
type="button"
|
||||
class="rounded-lg bg-blue-600 px-3 py-1.5 text-[11px] font-semibold text-white hover:bg-blue-500"
|
||||
>
|
||||
View full report
|
||||
</button>
|
||||
<button
|
||||
id="btn-open-logs"
|
||||
type="button"
|
||||
class="rounded-lg border border-red-300 bg-white px-3 py-1.5 text-[11px] font-semibold text-red-800 hover:bg-red-100 dark:border-red-800 dark:bg-zinc-900 dark:text-red-200 dark:hover:bg-zinc-800"
|
||||
>
|
||||
Open logs folder
|
||||
</button>
|
||||
<button
|
||||
id="btn-restart-backend"
|
||||
type="button"
|
||||
class="rounded-lg border border-red-300 bg-white px-3 py-1.5 text-[11px] font-semibold text-red-800 hover:bg-red-100 dark:border-red-800 dark:bg-zinc-900 dark:text-red-200 dark:hover:bg-zinc-800"
|
||||
>
|
||||
Retry backend
|
||||
</button>
|
||||
<button
|
||||
id="btn-emergency-relaunch"
|
||||
type="button"
|
||||
class="rounded-lg border border-red-300 bg-white px-3 py-1.5 text-[11px] font-semibold text-red-800 hover:bg-red-100 dark:border-red-800 dark:bg-zinc-900 dark:text-red-200 dark:hover:bg-zinc-800"
|
||||
>
|
||||
Emergency mode
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<p class="mt-4 text-center text-[11px] text-slate-400 dark:text-zinc-600" id="app-version">
|
||||
v0.0.0
|
||||
</p>
|
||||
|
|
@ -72,6 +119,14 @@
|
|||
const statusLine = document.getElementById("status-line");
|
||||
const attemptHint = document.getElementById("attempt-hint");
|
||||
const connectionNotice = document.getElementById("connection-notice");
|
||||
const startupDiagnostics = document.getElementById("startup-diagnostics");
|
||||
const startupDiagnosisSummary = document.getElementById("startup-diagnosis-summary");
|
||||
const startupDiagnosisLog = document.getElementById("startup-diagnosis-log");
|
||||
const startupLogPath = document.getElementById("startup-log-path");
|
||||
const btnViewCrashReport = document.getElementById("btn-view-crash-report");
|
||||
const btnOpenLogs = document.getElementById("btn-open-logs");
|
||||
const btnRestartBackend = document.getElementById("btn-restart-backend");
|
||||
const btnEmergencyRelaunch = document.getElementById("btn-emergency-relaunch");
|
||||
|
||||
const API_HOST = "127.0.0.1";
|
||||
const API_PORT = "9337";
|
||||
|
|
@ -90,6 +145,18 @@
|
|||
|
||||
(async function bootstrap() {
|
||||
applyStartupErrorHint();
|
||||
wireStartupActions();
|
||||
if (window.electron && typeof window.electron.onBackendStartupFailed === "function") {
|
||||
window.electron.onBackendStartupFailed((payload) => {
|
||||
handleStartupFailure(payload);
|
||||
});
|
||||
}
|
||||
try {
|
||||
if (window.electron && typeof window.electron.backendStartupDiagnostics === "function") {
|
||||
cachedDiagnostics = await window.electron.backendStartupDiagnostics();
|
||||
cachedCrash = cachedDiagnostics?.crash || null;
|
||||
}
|
||||
} catch (e) {}
|
||||
try {
|
||||
if (window.electron && typeof window.electron.backendHttpOnly === "function") {
|
||||
const httpOnly = await window.electron.backendHttpOnly();
|
||||
|
|
@ -101,6 +168,105 @@
|
|||
check();
|
||||
})();
|
||||
|
||||
function wireStartupActions() {
|
||||
if (btnViewCrashReport) {
|
||||
btnViewCrashReport.addEventListener("click", async () => {
|
||||
if (!window.electron?.openBackendCrashReport) {
|
||||
return;
|
||||
}
|
||||
await window.electron.openBackendCrashReport();
|
||||
});
|
||||
}
|
||||
if (btnOpenLogs) {
|
||||
btnOpenLogs.addEventListener("click", async () => {
|
||||
const logsDir = cachedDiagnostics?.paths?.logsDir;
|
||||
if (!logsDir || !window.electron?.openPath) {
|
||||
return;
|
||||
}
|
||||
await window.electron.openPath(logsDir);
|
||||
});
|
||||
}
|
||||
if (btnRestartBackend) {
|
||||
btnRestartBackend.addEventListener("click", async () => {
|
||||
if (!window.electron?.restartBackend) {
|
||||
return;
|
||||
}
|
||||
startupFailed = false;
|
||||
if (startupDiagnostics) {
|
||||
startupDiagnostics.classList.add("hidden");
|
||||
}
|
||||
statusLine.textContent = "Retrying backend startup…";
|
||||
connectionNotice.textContent = "";
|
||||
const result = await window.electron.restartBackend();
|
||||
if (!result?.ok) {
|
||||
connectionNotice.textContent = result?.error || "Backend restart failed.";
|
||||
return;
|
||||
}
|
||||
attemptCount = 0;
|
||||
recentFailures.length = 0;
|
||||
scheduleCheck(250);
|
||||
});
|
||||
}
|
||||
if (btnEmergencyRelaunch) {
|
||||
btnEmergencyRelaunch.addEventListener("click", async () => {
|
||||
if (!window.electron?.relaunchEmergency) {
|
||||
return;
|
||||
}
|
||||
await window.electron.relaunchEmergency();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function scheduleCheck(delayMs) {
|
||||
if (pollTimer != null) {
|
||||
clearTimeout(pollTimer);
|
||||
}
|
||||
pollTimer = setTimeout(check, delayMs);
|
||||
}
|
||||
|
||||
function formatCrashExcerpt(crash) {
|
||||
if (!crash || typeof crash !== "object") {
|
||||
return "No backend output was captured.";
|
||||
}
|
||||
const stderr = (crash.stderr || "").trim();
|
||||
const stdout = (crash.stdout || "").trim();
|
||||
const chunk = stderr || stdout;
|
||||
if (!chunk) {
|
||||
return "No backend output was captured.";
|
||||
}
|
||||
if (chunk.length <= 1800) {
|
||||
return chunk;
|
||||
}
|
||||
return chunk.slice(-1800);
|
||||
}
|
||||
|
||||
function showStartupDiagnostics(issue) {
|
||||
if (!startupDiagnostics || !issue) {
|
||||
return;
|
||||
}
|
||||
startupDiagnostics.classList.remove("hidden");
|
||||
if (startupDiagnosisSummary) {
|
||||
const codeText = issue.exitCode != null ? `Exit code ${issue.exitCode}. ` : "";
|
||||
startupDiagnosisSummary.textContent = `${codeText}${issue.detail || issue.headline || ""}`;
|
||||
}
|
||||
if (startupDiagnosisLog) {
|
||||
startupDiagnosisLog.textContent = formatCrashExcerpt(issue.crash || cachedCrash);
|
||||
}
|
||||
if (startupLogPath) {
|
||||
startupLogPath.textContent = issue.logHint || "";
|
||||
}
|
||||
}
|
||||
|
||||
function handleStartupFailure(payload) {
|
||||
startupFailed = true;
|
||||
cachedCrash = payload || cachedCrash;
|
||||
if (pollTimer != null) {
|
||||
clearTimeout(pollTimer);
|
||||
pollTimer = null;
|
||||
}
|
||||
updateStartupNotice(true);
|
||||
}
|
||||
|
||||
function applyStartupErrorHint() {
|
||||
const startupError = startupParams.get("startup_error");
|
||||
if (startupError !== "backend_unreachable") {
|
||||
|
|
@ -151,6 +317,10 @@
|
|||
let attemptCount = 0;
|
||||
let runtimeProbeAttempt = 0;
|
||||
let cachedRuntimeState = null;
|
||||
let cachedDiagnostics = null;
|
||||
let cachedCrash = null;
|
||||
let startupFailed = false;
|
||||
let pollTimer = null;
|
||||
const recentFailures = [];
|
||||
|
||||
function parseStatusJson(text) {
|
||||
|
|
@ -229,6 +399,8 @@
|
|||
return helper.classifyConnectionIssue(recentFailures, cachedRuntimeState, {
|
||||
attemptCount: attemptCount,
|
||||
networkWarnAfterAttempts: NETWORK_WARNING_AFTER_ATTEMPTS,
|
||||
crash: cachedCrash,
|
||||
paths: cachedDiagnostics?.paths || null,
|
||||
});
|
||||
}
|
||||
return {
|
||||
|
|
@ -238,18 +410,36 @@
|
|||
};
|
||||
}
|
||||
|
||||
async function updateStartupNotice() {
|
||||
async function updateStartupNotice(forceDiagnostics) {
|
||||
if (startupFailed || forceDiagnostics) {
|
||||
await refreshBackendRuntimeStateIfNeeded();
|
||||
const issue = classifyConnectionIssue();
|
||||
statusLine.textContent = issue.headline;
|
||||
connectionNotice.textContent = issue.detail;
|
||||
showStartupDiagnostics(issue);
|
||||
return;
|
||||
}
|
||||
if (attemptCount < NOTICE_AFTER_ATTEMPTS) {
|
||||
connectionNotice.textContent = "";
|
||||
if (startupDiagnostics) {
|
||||
startupDiagnostics.classList.add("hidden");
|
||||
}
|
||||
return;
|
||||
}
|
||||
await refreshBackendRuntimeStateIfNeeded();
|
||||
const issue = classifyConnectionIssue();
|
||||
statusLine.textContent = issue.headline;
|
||||
connectionNotice.textContent = issue.detail;
|
||||
if (issue.reason === "backend-exited") {
|
||||
startupFailed = true;
|
||||
showStartupDiagnostics(issue);
|
||||
}
|
||||
}
|
||||
|
||||
async function check() {
|
||||
if (startupFailed) {
|
||||
return;
|
||||
}
|
||||
attemptCount += 1;
|
||||
if (attemptCount === 1) {
|
||||
attemptHint.textContent = "";
|
||||
|
|
@ -262,6 +452,11 @@
|
|||
for (const protocol of protocolOrder) {
|
||||
const result = await tryOnce(protocol);
|
||||
if (result.ok) {
|
||||
if (window.electron && typeof window.electron.markBackendHealthy === "function") {
|
||||
try {
|
||||
await window.electron.markBackendHealthy();
|
||||
} catch (e) {}
|
||||
}
|
||||
detectedProtocol = result.protocol;
|
||||
statusLine.textContent = "Opening the app…";
|
||||
attemptHint.textContent = "";
|
||||
|
|
@ -274,8 +469,8 @@
|
|||
rememberFailure(result.failure);
|
||||
}
|
||||
}
|
||||
await updateStartupNotice();
|
||||
setTimeout(check, 350);
|
||||
await updateStartupNotice(false);
|
||||
scheduleCheck(350);
|
||||
}
|
||||
|
||||
function onReady() {
|
||||
|
|
|
|||
|
|
@ -12,18 +12,187 @@
|
|||
return String(value).toLowerCase();
|
||||
}
|
||||
|
||||
function parseMemoryLogLine(text) {
|
||||
if (!text || typeof text !== "string") {
|
||||
return null;
|
||||
}
|
||||
for (const rawLine of text.split(/\r?\n/)) {
|
||||
const line = rawLine.trim();
|
||||
if (!line.startsWith("MESHCHAT_MEMORY:")) {
|
||||
continue;
|
||||
}
|
||||
const parsed = {};
|
||||
const payload = line.slice("MESHCHAT_MEMORY:".length).trim();
|
||||
for (const token of payload.split(/\s+/)) {
|
||||
const idx = token.indexOf("=");
|
||||
if (idx <= 0) {
|
||||
continue;
|
||||
}
|
||||
const key = token.slice(0, idx);
|
||||
const value = token.slice(idx + 1);
|
||||
if (key === "total_mb" || key === "available_mb" || key === "percent_used") {
|
||||
const num = Number(value);
|
||||
if (Number.isFinite(num)) {
|
||||
parsed[key] = num;
|
||||
}
|
||||
} else {
|
||||
parsed[key] = value;
|
||||
}
|
||||
}
|
||||
return Object.keys(parsed).length > 0 ? parsed : null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} stderr
|
||||
* @param {string} stdout
|
||||
* @param {number|null|undefined} exitCode
|
||||
* @returns {{ category: string, summary: string, hints: string[] }}
|
||||
*/
|
||||
function diagnoseBackendCrash(stderr, stdout, exitCode) {
|
||||
const combined = `${stdout || ""}\n${stderr || ""}`.toLowerCase();
|
||||
const rawCombined = `${stdout || ""}\n${stderr || ""}`;
|
||||
const hints = [];
|
||||
const memoryLine = parseMemoryLogLine(rawCombined);
|
||||
|
||||
if (memoryLine) {
|
||||
const available = Number(memoryLine.available_mb);
|
||||
const action = String(memoryLine.action || "");
|
||||
if (action === "abort" || (Number.isFinite(available) && available < 200)) {
|
||||
hints.push("Close other apps to free RAM, then relaunch.");
|
||||
hints.push("Try Emergency Mode if the device has 4 GB RAM or less.");
|
||||
return {
|
||||
category: "memory",
|
||||
summary: Number.isFinite(available)
|
||||
? `The backend reported critically low memory (${available.toFixed(0)} MB available).`
|
||||
: "The backend aborted startup due to critically low memory.",
|
||||
hints,
|
||||
};
|
||||
}
|
||||
if (action === "warn") {
|
||||
hints.push("Try Emergency Mode if the device has 4 GB RAM or less.");
|
||||
return {
|
||||
category: "memory",
|
||||
summary: Number.isFinite(available)
|
||||
? `The backend detected limited memory (${available.toFixed(0)} MB available).`
|
||||
: "The backend detected limited system memory during startup.",
|
||||
hints,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
combined.includes("memoryerror") ||
|
||||
combined.includes("out of memory") ||
|
||||
combined.includes("cannot allocate memory")
|
||||
) {
|
||||
hints.push("Close other apps to free RAM, then relaunch.");
|
||||
hints.push("Try Emergency Mode if the device has 4 GB RAM or less.");
|
||||
return {
|
||||
category: "memory",
|
||||
summary: "The backend likely ran out of memory during startup.",
|
||||
hints,
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
combined.includes("address already in use") ||
|
||||
combined.includes("eaddrinuse") ||
|
||||
combined.includes("only one usage of each socket")
|
||||
) {
|
||||
hints.push("End any ReticulumMeshChatX.exe process in Task Manager, then relaunch.");
|
||||
return {
|
||||
category: "port-conflict",
|
||||
summary: "Port 9337 is already in use.",
|
||||
hints,
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
combined.includes("eacces") ||
|
||||
combined.includes("eperm") ||
|
||||
combined.includes("permission denied") ||
|
||||
combined.includes("access is denied")
|
||||
) {
|
||||
hints.push("Ensure the MeshChatX data folder is writable.");
|
||||
return {
|
||||
category: "permission",
|
||||
summary: "The backend could not access a required file or folder.",
|
||||
hints,
|
||||
};
|
||||
}
|
||||
|
||||
if (combined.includes("!!! application crash detected !!!")) {
|
||||
hints.push("Open meshchatx.log for the full diagnostic trace from the crash recovery engine.");
|
||||
return {
|
||||
category: "python-crash",
|
||||
summary: "The backend crash recovery system reported a fatal Python error.",
|
||||
hints,
|
||||
};
|
||||
}
|
||||
|
||||
if (exitCode != null && exitCode !== 0) {
|
||||
hints.push("Open the logs folder below and review meshchatx.log and last-backend-crash.json.");
|
||||
return {
|
||||
category: "exit-code",
|
||||
summary: `The backend exited with code ${exitCode}.`,
|
||||
hints,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
category: "unknown",
|
||||
summary: "The backend stopped before the UI could connect.",
|
||||
hints: ["Review the saved crash report and meshchatx.log for details."],
|
||||
};
|
||||
}
|
||||
|
||||
function formatLogPathHint(paths) {
|
||||
if (!paths || typeof paths !== "object") {
|
||||
return "";
|
||||
}
|
||||
if (paths.backendLogPath) {
|
||||
return `Logs: ${paths.backendLogPath}`;
|
||||
}
|
||||
if (paths.logsDir) {
|
||||
return `Logs folder: ${paths.logsDir}`;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {unknown[]} failures
|
||||
* @param {object|null} runtimeState
|
||||
* @param {object} options
|
||||
* @returns {object}
|
||||
*/
|
||||
function classifyConnectionIssue(failures, runtimeState, options) {
|
||||
const entries = Array.isArray(failures) ? failures : [];
|
||||
const state = runtimeState && typeof runtimeState === "object" ? runtimeState : null;
|
||||
const opts = options && typeof options === "object" ? options : {};
|
||||
const attemptCount = Number(opts.attemptCount) || 0;
|
||||
const networkWarnAfterAttempts = Number(opts.networkWarnAfterAttempts) || 24;
|
||||
const crash = opts.crash && typeof opts.crash === "object" ? opts.crash : null;
|
||||
const paths = opts.paths && typeof opts.paths === "object" ? opts.paths : null;
|
||||
|
||||
if (state && state.running === false && state.lastExitCode != null) {
|
||||
const diagnosis = diagnoseBackendCrash(crash?.stderr || "", crash?.stdout || "", state.lastExitCode);
|
||||
const logHint = formatLogPathHint(paths);
|
||||
const detailParts = [diagnosis.summary, ...diagnosis.hints];
|
||||
if (logHint) {
|
||||
detailParts.push(logHint);
|
||||
}
|
||||
return {
|
||||
reason: "backend-exited",
|
||||
headline: "The backend process stopped unexpectedly.",
|
||||
detail: "Please restart MeshChatX. If this keeps happening, review crash logs.",
|
||||
detail: detailParts.join(" "),
|
||||
category: diagnosis.category,
|
||||
exitCode: state.lastExitCode,
|
||||
hints: diagnosis.hints,
|
||||
logHint,
|
||||
crash,
|
||||
paths,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -92,7 +261,9 @@
|
|||
}
|
||||
|
||||
return {
|
||||
classifyConnectionIssue: classifyConnectionIssue,
|
||||
classifyFetchError: classifyFetchError,
|
||||
classifyConnectionIssue,
|
||||
classifyFetchError,
|
||||
diagnoseBackendCrash,
|
||||
formatLogPathHint,
|
||||
};
|
||||
});
|
||||
|
|
|
|||
|
|
@ -181,6 +181,15 @@ ipcMain.handle("backend-runtime-state", () => {
|
|||
return getBackendManager().getRuntimeState();
|
||||
});
|
||||
|
||||
ipcMain.handle("backend-startup-diagnostics", () => {
|
||||
return getBackendManager().getStartupDiagnostics();
|
||||
});
|
||||
|
||||
ipcMain.handle("mark-backend-healthy", () => {
|
||||
getBackendManager().markBackendHealthy();
|
||||
return { ok: true };
|
||||
});
|
||||
|
||||
ipcMain.handle("restart-backend", async () => {
|
||||
return await getBackendManager().restartBackend(integrityStatus);
|
||||
});
|
||||
|
|
@ -479,6 +488,7 @@ async function loadBackendCrashPage(crash) {
|
|||
const stdoutBase64 = Buffer.from((crash && crash.stdout) || "").toString("base64");
|
||||
const stderrBase64 = Buffer.from((crash && crash.stderr) || "").toString("base64");
|
||||
const code = crash && crash.code != null ? String(crash.code) : "";
|
||||
const paths = getBackendManager().getStartupDiagnostics().paths;
|
||||
|
||||
if (!mainWindow || mainWindow.isDestroyed()) {
|
||||
await dialog.showMessageBox({
|
||||
|
|
@ -497,6 +507,8 @@ async function loadBackendCrashPage(crash) {
|
|||
code: code,
|
||||
stdout: stdoutBase64,
|
||||
stderr: stderrBase64,
|
||||
logPath: paths?.backendLogPath || "",
|
||||
crashReportPath: paths?.crashReportPath || "",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -99,6 +99,12 @@ contextBridge.exposeInMainWorld("electron", {
|
|||
backendRuntimeState: async function () {
|
||||
return await ipcRenderer.invoke("backend-runtime-state");
|
||||
},
|
||||
backendStartupDiagnostics: async function () {
|
||||
return await ipcRenderer.invoke("backend-startup-diagnostics");
|
||||
},
|
||||
markBackendHealthy: async function () {
|
||||
return await ipcRenderer.invoke("mark-backend-healthy");
|
||||
},
|
||||
restartBackend: async function () {
|
||||
return await ipcRenderer.invoke("restart-backend");
|
||||
},
|
||||
|
|
@ -111,4 +117,10 @@ contextBridge.exposeInMainWorld("electron", {
|
|||
}
|
||||
ipcRenderer.on("backend-process-exited", (_event, payload) => callback(payload));
|
||||
},
|
||||
onBackendStartupFailed: function (callback) {
|
||||
if (typeof callback !== "function") {
|
||||
return;
|
||||
}
|
||||
ipcRenderer.on("backend-startup-failed", (_event, payload) => callback(payload));
|
||||
},
|
||||
});
|
||||
|
|
|
|||
|
|
@ -142,7 +142,12 @@ from meshchatx.src.backend.nomadnet_utils import (
|
|||
)
|
||||
from meshchatx.src.backend.page_node_manager import PageNodeManager
|
||||
from meshchatx.src.backend.persistent_log_handler import PersistentLogHandler
|
||||
from meshchatx.src.backend.recovery import CrashRecovery, HealthMonitor
|
||||
from meshchatx.src.backend.recovery import (
|
||||
CrashRecovery,
|
||||
HealthMonitor,
|
||||
evaluate_startup_memory,
|
||||
format_memory_log_line,
|
||||
)
|
||||
from meshchatx.src.backend import reticulum_pathfinding
|
||||
from meshchatx.src.backend.rnprobe_handler import RNProbeHandler
|
||||
from meshchatx.src.backend.sideband_commands import SidebandCommands
|
||||
|
|
@ -19100,6 +19105,19 @@ def main():
|
|||
# init app (allow optional one-shot backup/restore before running)
|
||||
rns_log_cli = (args.rns_log_level or "").strip() or None
|
||||
|
||||
mem_check = evaluate_startup_memory(args.emergency)
|
||||
print(format_memory_log_line(mem_check), flush=True)
|
||||
if mem_check.get("message"):
|
||||
print(mem_check["message"], flush=True)
|
||||
if mem_check["action"] == "abort":
|
||||
print(
|
||||
"Startup aborted due to critically low memory. "
|
||||
"Free RAM or relaunch with --emergency.",
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
reticulum_meshchat = ReticulumMeshChat(
|
||||
identity,
|
||||
args.storage_dir,
|
||||
|
|
|
|||
|
|
@ -2,5 +2,16 @@
|
|||
|
||||
from .crash_recovery import CrashRecovery
|
||||
from .health_monitor import HealthMonitor
|
||||
from .memory_preflight import (
|
||||
evaluate_startup_memory,
|
||||
format_memory_log_line,
|
||||
parse_memory_log_line,
|
||||
)
|
||||
|
||||
__all__ = ["CrashRecovery", "HealthMonitor"]
|
||||
__all__ = [
|
||||
"CrashRecovery",
|
||||
"HealthMonitor",
|
||||
"evaluate_startup_memory",
|
||||
"format_memory_log_line",
|
||||
"parse_memory_log_line",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -379,6 +379,7 @@ class CrashRecovery:
|
|||
"reasoning": "Available system memory is extremely low, leading to allocation failures.",
|
||||
"suggestions": [
|
||||
"Close other memory-intensive applications.",
|
||||
"Relaunch in Emergency Mode (--emergency) to reduce startup memory use.",
|
||||
"Add more RAM or swap space to the system.",
|
||||
],
|
||||
},
|
||||
|
|
@ -518,12 +519,24 @@ class CrashRecovery:
|
|||
else:
|
||||
potential_causes["ASYNC_RACE"]["probability"] = 0.45
|
||||
|
||||
if exc_type is MemoryError or "memoryerror" in error_type:
|
||||
potential_causes["OOM"]["probability"] = 0.95
|
||||
potential_causes["OOM"]["reasoning"] += (
|
||||
" Python raised MemoryError during allocation."
|
||||
)
|
||||
|
||||
if symptoms["low_mem"]:
|
||||
# If we have a DB error and low memory, OOM is highly likely as the true cause
|
||||
if symptoms["sqlite_in_msg"]:
|
||||
potential_causes["OOM"]["probability"] = 0.85
|
||||
potential_causes["OOM"]["probability"] = max(
|
||||
potential_causes["OOM"]["probability"],
|
||||
0.85,
|
||||
)
|
||||
else:
|
||||
potential_causes["OOM"]["probability"] = 0.75
|
||||
potential_causes["OOM"]["probability"] = max(
|
||||
potential_causes["OOM"]["probability"],
|
||||
0.75,
|
||||
)
|
||||
|
||||
if symptoms["rns_config_missing"]:
|
||||
potential_causes["CONFIG_MISSING"]["probability"] = 0.99
|
||||
|
|
@ -739,7 +752,7 @@ class CrashRecovery:
|
|||
file.write(
|
||||
f"- Memory: {mem.percent}% used ({results['available_mem_mb']:.1f} MB available)\n",
|
||||
)
|
||||
if mem.percent > 95:
|
||||
if mem.percent > 95 or results["available_mem_mb"] < 400:
|
||||
results["low_memory"] = True
|
||||
file.write(" [CRITICAL] System memory is dangerously low!\n")
|
||||
|
||||
|
|
|
|||
106
meshchatx/src/backend/recovery/memory_preflight.py
Normal file
106
meshchatx/src/backend/recovery/memory_preflight.py
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
# SPDX-License-Identifier: 0BSD
|
||||
|
||||
"""Startup memory checks before heavy backend initialization."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
|
||||
import psutil
|
||||
|
||||
MEMORY_LOG_PREFIX = "MESHCHAT_MEMORY:"
|
||||
|
||||
_ABORT_AVAILABLE_MB = 150
|
||||
_CRITICAL_WARN_AVAILABLE_MB = 400
|
||||
_WARN_AVAILABLE_MB = 700
|
||||
_LOW_TOTAL_MB = 5120
|
||||
_CRITICAL_WARN_PERCENT = 90
|
||||
_WARN_PERCENT = 85
|
||||
|
||||
|
||||
def get_system_memory_snapshot() -> dict:
|
||||
"""Return host memory metrics in megabytes and percent used."""
|
||||
mem = psutil.virtual_memory()
|
||||
return {
|
||||
"total_mb": mem.total / (1024**2),
|
||||
"available_mb": mem.available / (1024**2),
|
||||
"percent_used": float(mem.percent),
|
||||
}
|
||||
|
||||
|
||||
def evaluate_startup_memory(emergency: bool = False) -> dict:
|
||||
"""Classify startup memory pressure and choose a guard action."""
|
||||
snapshot = get_system_memory_snapshot()
|
||||
available_mb = snapshot["available_mb"]
|
||||
total_mb = snapshot["total_mb"]
|
||||
percent_used = snapshot["percent_used"]
|
||||
|
||||
action = "ok"
|
||||
message = ""
|
||||
|
||||
if available_mb < _ABORT_AVAILABLE_MB:
|
||||
action = "abort"
|
||||
message = (
|
||||
"Available memory is critically low "
|
||||
f"({available_mb:.0f} MB). Close other apps or relaunch in Emergency Mode."
|
||||
)
|
||||
elif available_mb < _CRITICAL_WARN_AVAILABLE_MB or (
|
||||
total_mb <= _LOW_TOTAL_MB and percent_used >= _CRITICAL_WARN_PERCENT
|
||||
):
|
||||
action = "warn"
|
||||
message = (
|
||||
"Low system memory detected "
|
||||
f"({available_mb:.0f} MB available). Emergency Mode is strongly recommended."
|
||||
)
|
||||
elif available_mb < _WARN_AVAILABLE_MB or (
|
||||
total_mb <= _LOW_TOTAL_MB and percent_used >= _WARN_PERCENT
|
||||
):
|
||||
action = "warn"
|
||||
message = (
|
||||
"System memory is limited "
|
||||
f"({available_mb:.0f} MB available). Emergency Mode is recommended on 4 GB devices."
|
||||
)
|
||||
|
||||
return {
|
||||
**snapshot,
|
||||
"action": action,
|
||||
"message": message,
|
||||
"emergency_requested": emergency,
|
||||
"low_memory": action in ("abort", "warn"),
|
||||
}
|
||||
|
||||
|
||||
def format_memory_log_line(result: dict) -> str:
|
||||
"""Emit a single machine-readable startup memory line for Electron logs."""
|
||||
return (
|
||||
f"{MEMORY_LOG_PREFIX} total_mb={result['total_mb']:.1f} "
|
||||
f"available_mb={result['available_mb']:.1f} "
|
||||
f"percent_used={result['percent_used']:.1f} "
|
||||
f"action={result['action']} "
|
||||
f"emergency={str(result.get('emergency_requested', False)).lower()}"
|
||||
)
|
||||
|
||||
|
||||
def parse_memory_log_line(text: str) -> dict | None:
|
||||
"""Parse a MESHCHAT_MEMORY startup line from captured backend output."""
|
||||
if not text:
|
||||
return None
|
||||
for raw_line in text.splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line.startswith(MEMORY_LOG_PREFIX):
|
||||
continue
|
||||
payload = line[len(MEMORY_LOG_PREFIX) :].strip()
|
||||
parsed: dict[str, str | float | bool] = {}
|
||||
for token in payload.split():
|
||||
if "=" not in token:
|
||||
continue
|
||||
key, value = token.split("=", 1)
|
||||
if key in ("total_mb", "available_mb", "percent_used"):
|
||||
with contextlib.suppress(ValueError):
|
||||
parsed[key] = float(value)
|
||||
elif key == "emergency":
|
||||
parsed[key] = value.lower() in ("true", "1", "yes")
|
||||
else:
|
||||
parsed[key] = value
|
||||
return parsed if parsed else None
|
||||
return None
|
||||
|
|
@ -16,6 +16,10 @@ def resolve_log_dir():
|
|||
if env_dir:
|
||||
candidates.append(env_dir)
|
||||
|
||||
storage_dir = os.environ.get("MESHCHAT_STORAGE_DIR")
|
||||
if storage_dir:
|
||||
candidates.append(os.path.join(storage_dir, "logs"))
|
||||
|
||||
candidates.append("/config/logs")
|
||||
|
||||
if os.name == "nt":
|
||||
|
|
|
|||
|
|
@ -171,6 +171,16 @@ class TestCrashRecovery(unittest.TestCase):
|
|||
self.assertIsNotNone(oom_cause)
|
||||
self.assertEqual(oom_cause["probability"], 85)
|
||||
|
||||
def test_heuristic_analysis_memoryerror(self):
|
||||
causes = self.recovery._analyze_cause(
|
||||
MemoryError,
|
||||
MemoryError("unable to allocate array"),
|
||||
{"low_memory": False, "available_mem_mb": 1024},
|
||||
)
|
||||
oom_cause = next((c for c in causes if "OOM" in c["description"]), None)
|
||||
self.assertIsNotNone(oom_cause)
|
||||
self.assertEqual(oom_cause["probability"], 95)
|
||||
|
||||
def test_heuristic_analysis_rns_missing(self):
|
||||
"""Verify high confidence for missing RNS config."""
|
||||
exc_type = RuntimeError
|
||||
|
|
|
|||
82
tests/backend/test_memory_preflight.py
Normal file
82
tests/backend/test_memory_preflight.py
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
# SPDX-License-Identifier: 0BSD
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from meshchatx.src.backend.recovery.memory_preflight import (
|
||||
evaluate_startup_memory,
|
||||
format_memory_log_line,
|
||||
parse_memory_log_line,
|
||||
)
|
||||
|
||||
|
||||
def _mock_memory(total_mb, available_mb, percent):
|
||||
mem = MagicMock()
|
||||
mem.total = int(total_mb * 1024**2)
|
||||
mem.available = int(available_mb * 1024**2)
|
||||
mem.percent = percent
|
||||
return mem
|
||||
|
||||
|
||||
def test_evaluate_startup_memory_ok():
|
||||
with patch(
|
||||
"meshchatx.src.backend.recovery.memory_preflight.psutil.virtual_memory",
|
||||
return_value=_mock_memory(16384, 8192, 50),
|
||||
):
|
||||
result = evaluate_startup_memory(emergency=False)
|
||||
assert result["action"] == "ok"
|
||||
assert result["low_memory"] is False
|
||||
|
||||
|
||||
def test_evaluate_startup_memory_warn_on_low_total():
|
||||
with patch(
|
||||
"meshchatx.src.backend.recovery.memory_preflight.psutil.virtual_memory",
|
||||
return_value=_mock_memory(4096, 900, 78),
|
||||
):
|
||||
result = evaluate_startup_memory(emergency=False)
|
||||
assert result["action"] == "warn"
|
||||
assert result["low_memory"] is True
|
||||
|
||||
|
||||
def test_evaluate_startup_memory_critical_warn():
|
||||
with patch(
|
||||
"meshchatx.src.backend.recovery.memory_preflight.psutil.virtual_memory",
|
||||
return_value=_mock_memory(4096, 320, 92),
|
||||
):
|
||||
result = evaluate_startup_memory(emergency=False)
|
||||
assert result["action"] == "warn"
|
||||
assert "strongly recommended" in result["message"]
|
||||
|
||||
|
||||
def test_evaluate_startup_memory_abort():
|
||||
with patch(
|
||||
"meshchatx.src.backend.recovery.memory_preflight.psutil.virtual_memory",
|
||||
return_value=_mock_memory(4096, 120, 97),
|
||||
):
|
||||
result = evaluate_startup_memory(emergency=False)
|
||||
assert result["action"] == "abort"
|
||||
|
||||
|
||||
def test_evaluate_startup_memory_already_in_emergency():
|
||||
with patch(
|
||||
"meshchatx.src.backend.recovery.memory_preflight.psutil.virtual_memory",
|
||||
return_value=_mock_memory(4096, 320, 92),
|
||||
):
|
||||
result = evaluate_startup_memory(emergency=True)
|
||||
assert result["action"] == "warn"
|
||||
assert result["emergency_requested"] is True
|
||||
|
||||
|
||||
def test_format_and_parse_memory_log_line():
|
||||
result = {
|
||||
"total_mb": 4096.0,
|
||||
"available_mb": 512.5,
|
||||
"percent_used": 87.5,
|
||||
"action": "warn",
|
||||
"emergency_requested": False,
|
||||
}
|
||||
line = format_memory_log_line(result)
|
||||
assert line.startswith("MESHCHAT_MEMORY:")
|
||||
parsed = parse_memory_log_line(f"booting\n{line}\n")
|
||||
assert parsed["available_mb"] == 512.5
|
||||
assert parsed["action"] == "warn"
|
||||
assert parsed["emergency"] == "false"
|
||||
53
tests/electron/backendCrashReport.test.js
Normal file
53
tests/electron/backendCrashReport.test.js
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
clearCrashReport,
|
||||
diagnoseBackendCrash,
|
||||
getCrashReportPath,
|
||||
loadCrashReport,
|
||||
persistCrashReport,
|
||||
} from "../../electron/backendCrashReport.js";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
describe("electron/backendCrashReport", () => {
|
||||
it("persists and reloads crash reports under storage logs", () => {
|
||||
const storageDir = fs.mkdtempSync(path.join(os.tmpdir(), "meshchatx-crash-"));
|
||||
try {
|
||||
const saved = persistCrashReport(storageDir, {
|
||||
code: 17,
|
||||
stdout: "ok",
|
||||
stderr: "MemoryError: out of memory",
|
||||
at: Date.now(),
|
||||
});
|
||||
expect(saved).toBeTruthy();
|
||||
expect(fs.existsSync(getCrashReportPath(storageDir))).toBe(true);
|
||||
const loaded = loadCrashReport(storageDir);
|
||||
expect(loaded.code).toBe(17);
|
||||
expect(loaded.stderr).toContain("MemoryError");
|
||||
clearCrashReport(storageDir);
|
||||
expect(loadCrashReport(storageDir)).toBeNull();
|
||||
} finally {
|
||||
fs.rmSync(storageDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("diagnoseBackendCrash detects memory failures", () => {
|
||||
const diagnosis = diagnoseBackendCrash("MemoryError: allocation failed", "", 1);
|
||||
expect(diagnosis.category).toBe("memory");
|
||||
expect(diagnosis.summary).toContain("memory");
|
||||
});
|
||||
|
||||
it("diagnoseBackendCrash parses MESHCHAT_MEMORY startup lines", () => {
|
||||
const stdout =
|
||||
"MESHCHAT_MEMORY: total_mb=4096.0 available_mb=120.0 percent_used=97.0 action=abort emergency=false";
|
||||
const diagnosis = diagnoseBackendCrash("", stdout, 1);
|
||||
expect(diagnosis.category).toBe("memory");
|
||||
expect(diagnosis.summary).toContain("120");
|
||||
});
|
||||
|
||||
it("diagnoseBackendCrash detects port conflicts", () => {
|
||||
const diagnosis = diagnoseBackendCrash("", "OSError: [Errno 98] Address already in use", 1);
|
||||
expect(diagnosis.category).toBe("port-conflict");
|
||||
});
|
||||
});
|
||||
|
|
@ -49,6 +49,33 @@ describe("electron/backendProcess", () => {
|
|||
expect(manager.getRuntimeState().lastExitCode).toBe(255);
|
||||
});
|
||||
|
||||
it("notifies loading screen when backend exits during startup", async () => {
|
||||
const notifyRenderer = vi.fn();
|
||||
const showCrashPage = vi.fn();
|
||||
const manager = createBackendProcessManager({
|
||||
log: vi.fn(),
|
||||
getDefaultStorageDir: () => "/tmp/storage",
|
||||
getDefaultReticulumConfigDir: () => "/tmp/reticulum",
|
||||
getMainWindowPageKind: () => "loading",
|
||||
isQuiting: () => false,
|
||||
notifyRenderer,
|
||||
showCrashPage,
|
||||
spawn: spawnMock,
|
||||
});
|
||||
|
||||
manager.setUserProvidedArguments([]);
|
||||
await manager.spawnBackend("/tmp/ReticulumMeshChatX", { backend: { ok: true, issues: [] } });
|
||||
fakeProc.emit("exit", 9);
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
expect(notifyRenderer).toHaveBeenCalledWith(
|
||||
"backend-startup-failed",
|
||||
expect.objectContaining({ code: 9, paths: expect.any(Object) })
|
||||
);
|
||||
expect(showCrashPage).not.toHaveBeenCalled();
|
||||
expect(manager.getLastCrash()).toEqual(expect.objectContaining({ code: 9 }));
|
||||
});
|
||||
|
||||
it("opens the crash page when backend exits outside the main shell", async () => {
|
||||
const notifyRenderer = vi.fn();
|
||||
const showCrashPage = vi.fn();
|
||||
|
|
|
|||
|
|
@ -6,11 +6,38 @@ const notice = require("../../electron/loadingStatusNotice.js");
|
|||
|
||||
describe("electron/loadingStatusNotice", () => {
|
||||
it("classifyConnectionIssue prefers backend-exited when runtime not running", () => {
|
||||
const r = notice.classifyConnectionIssue([], {
|
||||
running: false,
|
||||
lastExitCode: 1,
|
||||
});
|
||||
const r = notice.classifyConnectionIssue(
|
||||
[],
|
||||
{
|
||||
running: false,
|
||||
lastExitCode: 1,
|
||||
},
|
||||
{
|
||||
crash: { stderr: "MemoryError: out of memory", stdout: "" },
|
||||
paths: { backendLogPath: "C:\\Users\\x\\.reticulum-meshchatx\\logs\\meshchatx.log" },
|
||||
}
|
||||
);
|
||||
expect(r.reason).toBe("backend-exited");
|
||||
expect(r.category).toBe("memory");
|
||||
expect(r.logHint).toContain("meshchatx.log");
|
||||
});
|
||||
|
||||
it("classifyConnectionIssue parses MESHCHAT_MEMORY startup lines", () => {
|
||||
const r = notice.classifyConnectionIssue(
|
||||
[],
|
||||
{
|
||||
running: false,
|
||||
lastExitCode: 1,
|
||||
},
|
||||
{
|
||||
crash: {
|
||||
stderr: "",
|
||||
stdout: "MESHCHAT_MEMORY: total_mb=4096.0 available_mb=120.0 percent_used=97.0 action=abort emergency=false",
|
||||
},
|
||||
}
|
||||
);
|
||||
expect(r.category).toBe("memory");
|
||||
expect(r.detail).toContain("120");
|
||||
});
|
||||
|
||||
it("classifyConnectionIssue flags loopback blocked", () => {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue