diff --git a/.dockerignore b/.dockerignore index 1bea410a..d218979d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -140,6 +140,10 @@ scripts/ci/ meshchatx/src/frontend/public/vendor/micron-parser-go/micron-parser-go.wasm meshchatx/src/frontend/public/vendor/micron-parser-go/wasm_exec.js +# Host-built visualiser WASM (Docker builds with Go from visualiser-wasm/) +meshchatx/src/frontend/public/vendor/visualiser-wasm/visualiser.wasm +meshchatx/src/frontend/public/vendor/visualiser-wasm/wasm_exec.js + .hypothesis .hypothesis/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 19408fce..297bf85e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ All notable changes to this project will be documented in this file. - CI benches use median-of-medians and quieter regression gates - Backend tests can run sharded in CI - Plugin strings live in plugin bundles, not main locale files +- Docker frontend build installs Go, builds visualiser WASM, and fails if WASM artifacts are missing ### Fixed @@ -37,10 +38,20 @@ All notable changes to this project will be documented in this file. - Startup check and disable unsupported interfaces - Nomad favourites: no more Unknown Node / lost custom sections - Relay Chat message dedupe. Network visualiser faster on large meshes -- Bots and RNSh work in frozen macOS/Windows builds (`--meshchatx-run-module`) +- Bots and RNSh work in frozen macOS/Windows builds - Sensitive config no longer mutable over WebSocket. Reticulum config repair on startup - Paper message URI encoding for non-ASCII title and content - Nightly releases and broader self-test / CI coverage +- LXMA contact import works with current RNS public-key loading and remembers the peer key before announce +- Android calls: overlay accept opens the phone tab so native audio attaches. Web-audio no longer permanently disabled after a bridge error +- Android Codec2: reliable libcodec2 preload, Gradle fails without Codec2 wheels or jniLibs, and unavailable Codec2 profiles are hidden +- Unknown meshchatx links return a clear error instead of falling through to LXMF +- NomadNet Micron copy no longer inserts a newline between every character +- Unread message count is a red pill on the Messages nav icon +- Notification bell removed from the header +- Unread badge stays circular and remains visible when the sidebar is collapsed +- Open conversations mark as read when a new message arrives without needing to reselect the thread +- Startup stage logs no longer print the same stage twice ## [4.7.2] - 2026-07-06 diff --git a/Dockerfile b/Dockerfile index 6918db77..25c303d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,8 @@ ARG PYTHON_HASH=sha256:dd4d2bd5b53d9b25a51da13addf2be586beebd5387e289e798e4083d9 # ---- STAGE 1: Frontend Build ---- FROM --platform=linux/amd64 ${NODE_IMAGE}@${NODE_HASH} AS build-frontend WORKDIR /src -RUN apk add --no-cache git python3 +# go is required to compile visualiser-wasm +RUN apk add --no-cache git python3 go COPY package.json pnpm-lock.yaml pnpm-workspace.yaml vite.config.js ./ COPY patches ./patches COPY scripts/fetch-micron-wasm.mjs scripts/fetch-micron-wasm.mjs @@ -27,11 +28,19 @@ COPY scripts/sync-meshchatx-docs.js scripts/sync-meshchatx-docs.js COPY scripts/pip_rns_remotes.py scripts/pip_rns_remotes.py COPY scripts/build/fetch_reticulum_manual.py scripts/build/fetch_reticulum_manual.py COPY docs ./docs +COPY visualiser-wasm ./visualiser-wasm COPY meshchatx/src/frontend ./meshchatx/src/frontend -RUN npm install -g pnpm@11.1.2 && \ +ENV GOCACHE=/tmp/go-cache +ENV GOTMPDIR=/tmp/go-tmp +RUN mkdir -p /tmp/go-cache /tmp/go-tmp && \ + npm install -g pnpm@11.1.2 && \ pnpm config set verify-store-integrity true && \ pnpm install --frozen-lockfile && \ - pnpm run build-frontend && \ + MESHCHATX_REQUIRE_VISUALISER_WASM=1 pnpm run build-frontend && \ + test -s meshchatx/src/frontend/public/vendor/visualiser-wasm/visualiser.wasm && \ + test -s meshchatx/src/frontend/public/vendor/visualiser-wasm/wasm_exec.js && \ + test -s meshchatx/src/frontend/public/vendor/micron-parser-go/micron-parser-go.wasm && \ + test -s meshchatx/src/frontend/public/vendor/micron-parser-go/wasm_exec.js && \ pnpm run build-docs # ---- STAGE 2: Python Builder ---- diff --git a/Dockerfile.hardened b/Dockerfile.hardened index e92e9c5e..1d8c706f 100644 --- a/Dockerfile.hardened +++ b/Dockerfile.hardened @@ -13,7 +13,8 @@ ARG PYTHON_RUNTIME_IMAGE=cgr.dev/chainguard/python:latest-dev FROM --platform=linux/amd64 ${NODE_IMAGE} AS build-frontend USER root WORKDIR /src -RUN apk add --no-cache git python3 +# go is required to compile visualiser-wasm (gitignored binary; not in image context) +RUN apk add --no-cache git python3 go COPY package.json pnpm-lock.yaml pnpm-workspace.yaml vite.config.js ./ COPY patches ./patches COPY scripts/fetch-micron-wasm.mjs scripts/fetch-micron-wasm.mjs @@ -24,11 +25,19 @@ COPY scripts/sync-meshchatx-docs.js scripts/sync-meshchatx-docs.js COPY scripts/pip_rns_remotes.py scripts/pip_rns_remotes.py COPY scripts/build/fetch_reticulum_manual.py scripts/build/fetch_reticulum_manual.py COPY docs ./docs +COPY visualiser-wasm ./visualiser-wasm COPY meshchatx/src/frontend ./meshchatx/src/frontend -RUN npm install -g pnpm@11.1.2 && \ +ENV GOCACHE=/tmp/go-cache +ENV GOTMPDIR=/tmp/go-tmp +RUN mkdir -p /tmp/go-cache /tmp/go-tmp && \ + npm install -g pnpm@11.1.2 && \ pnpm config set verify-store-integrity true && \ pnpm install --frozen-lockfile && \ - pnpm run build-frontend && \ + MESHCHATX_REQUIRE_VISUALISER_WASM=1 pnpm run build-frontend && \ + test -s meshchatx/src/frontend/public/vendor/visualiser-wasm/visualiser.wasm && \ + test -s meshchatx/src/frontend/public/vendor/visualiser-wasm/wasm_exec.js && \ + test -s meshchatx/src/frontend/public/vendor/micron-parser-go/micron-parser-go.wasm && \ + test -s meshchatx/src/frontend/public/vendor/micron-parser-go/wasm_exec.js && \ pnpm run build-docs FROM ${PYTHON_BUILD_IMAGE} AS builder diff --git a/android/app/build.gradle b/android/app/build.gradle index 815c7b87..567c97fb 100644 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -104,8 +104,20 @@ tasks.register("syncCodec2JniLibs", Exec) { "${projectDir}/src/main/jniLibs", abiArg ) - onlyIf { - vendorWheelDir.isDirectory() && vendorWheelDir.list()?.any { it.startsWith("chaquopy_libcodec2-") } + doFirst { + if (!vendorWheelDir.isDirectory()) { + throw new org.gradle.api.GradleException( + "Missing android/vendor directory at ${vendorWheelDir}. " + + "Run: bash scripts/build-android-wheels-local.sh" + ) + } + def wheels = vendorWheelDir.list()?.toList() ?: [] + if (!wheels.any { it.startsWith("chaquopy_libcodec2-") }) { + throw new org.gradle.api.GradleException( + "Missing chaquopy_libcodec2 wheels in ${vendorWheelDir}. " + + "Run: bash scripts/build-android-wheels-local.sh" + ) + } } } @@ -128,6 +140,25 @@ tasks.register("verifyVendorWheels") { ) } } + if (!wheels.any { it.startsWith("pycodec2-") && it.contains("android_24_${abiTag}") }) { + throw new org.gradle.api.GradleException( + "Missing pycodec2 wheel for ${abi} in ${vendorWheelDir}. " + + "Run: bash scripts/build-android-wheels-local.sh" + ) + } + if (!wheels.any { it.startsWith("chaquopy_libcodec2-") && it.contains("android_24_${abiTag}") }) { + throw new org.gradle.api.GradleException( + "Missing chaquopy_libcodec2 wheel for ${abi} in ${vendorWheelDir}. " + + "Run: bash scripts/build-android-wheels-local.sh" + ) + } + def jniLib = file("${projectDir}/src/main/jniLibs/${abi}/libcodec2.so") + if (!jniLib.isFile() || jniLib.length() < 100_000) { + throw new org.gradle.api.GradleException( + "Missing or tiny jniLibs/${abi}/libcodec2.so after sync. " + + "Expected a real Codec2 shared library for Android calls." + ) + } } } } @@ -142,8 +173,24 @@ tasks.register("repackAndroidPycodec2Wheels", Exec) { workingDir = repoRoot def pyExe = (System.getenv("MESHCHATX_REPACK_PYTHON") ?: "python3") commandLine(pyExe, "scripts/repack-android-pycodec2-wheels.py", "--vendor-dir", vendorWheelDir.absolutePath) - onlyIf { - vendorWheelDir.isDirectory() && vendorWheelDir.list()?.any { it.startsWith("pycodec2-") && it.endsWith(".whl") } + doFirst { + if (!vendorWheelDir.isDirectory()) { + throw new org.gradle.api.GradleException( + "Missing android/vendor directory at ${vendorWheelDir}" + ) + } + def wheels = vendorWheelDir.list()?.toList() ?: [] + if (!wheels.any { it.startsWith("pycodec2-") && it.endsWith(".whl") }) { + throw new org.gradle.api.GradleException( + "Missing pycodec2 wheels in ${vendorWheelDir}. " + + "Run: bash scripts/build-android-wheels-local.sh" + ) + } + if (!wheels.any { it.startsWith("chaquopy_libcodec2-") && it.endsWith(".whl") }) { + throw new org.gradle.api.GradleException( + "Missing chaquopy_libcodec2 wheels needed to repack pycodec2 in ${vendorWheelDir}" + ) + } } } @@ -151,6 +198,10 @@ tasks.named("syncCodec2JniLibs").configure { dependsOn(tasks.named("repackAndroidPycodec2Wheels")) } +tasks.named("verifyVendorWheels").configure { + dependsOn(tasks.named("syncCodec2JniLibs")) +} + tasks.register("fetchRepositoryBundledWheels", Exec) { workingDir = repoRoot def pyExe = (System.getenv("MESHCHATX_FETCH_PYTHON") ?: "python3") diff --git a/meshchatx.rsm b/meshchatx.rsm index 007ea419..1e273517 100644 Binary files a/meshchatx.rsm and b/meshchatx.rsm differ diff --git a/meshchatx/android_codec2.py b/meshchatx/android_codec2.py index debc3897..f6cee32a 100644 --- a/meshchatx/android_codec2.py +++ b/meshchatx/android_codec2.py @@ -23,7 +23,24 @@ def _is_chaquopy_android() -> bool: return True +def _cdll_load(path_or_name: str): + """Load a shared library with RTLD_GLOBAL when the platform supports it. + + pycodec2.so declares NEEDED libcodec2.so. Loading with RTLD_GLOBAL lets the + later dlopen of the extension resolve that dependency. + """ + mode = getattr(ctypes, "RTLD_GLOBAL", None) + if mode is None: + return ctypes.CDLL(path_or_name) + return ctypes.CDLL(path_or_name, mode=mode) + + def _libcodec2_candidates() -> list[Path]: + """Return candidate paths for libcodec2.so without importing pycodec2. + + ``import pycodec2`` loads the extension which already needs libcodec2.so. + Searching sys.path on disk avoids that chicken-and-egg failure. + """ candidates: list[Path] = [] seen: set[str] = set() @@ -34,17 +51,12 @@ def _libcodec2_candidates() -> list[Path]: seen.add(key) candidates.append(path) - try: - import pycodec2 - - add(Path(pycodec2.__file__).resolve().parent / "libcodec2.so") - except Exception: - pass - for entry in sys.path: if not entry: continue - add(Path(entry) / "chaquopy" / "lib" / "libcodec2.so") + root = Path(entry) + add(root / "pycodec2" / "libcodec2.so") + add(root / "chaquopy" / "lib" / "libcodec2.so") return candidates @@ -53,7 +65,7 @@ def ensure_codec2_native_library() -> bool: """Preload ``libcodec2.so`` so ``import pycodec2`` works on Android. Chaquopy installs ``chaquopy-libcodec2`` separately from ``pycodec2``. The - extension module only declares a NEEDED entry for ``libcodec2.so``; without + extension module only declares a NEEDED entry for ``libcodec2.so``. Without preloading or bundling the shared library next to ``pycodec2.so``, imports fail at runtime with ``dlopen`` errors. """ @@ -68,7 +80,7 @@ def ensure_codec2_native_library() -> bool: return True try: - ctypes.CDLL("libcodec2.so") + _cdll_load("libcodec2.so") return True except OSError: pass @@ -78,7 +90,7 @@ def ensure_codec2_native_library() -> bool: if not lib_path.is_file(): continue try: - ctypes.CDLL(str(lib_path)) + _cdll_load(str(lib_path)) logger.info("Loaded Codec2 native library from %s", lib_path) return True except OSError as exc: @@ -106,3 +118,10 @@ def probe_pycodec2() -> tuple[bool, str | None]: def codec2_preload_error() -> str | None: """Return the last preload failure message, if any.""" return _codec2_preload_error + + +def reset_codec2_preload_state_for_tests() -> None: + """Clear preload memoization (tests only).""" + global _codec2_preload_done, _codec2_preload_error + _codec2_preload_done = False + _codec2_preload_error = None diff --git a/meshchatx/meshchat.py b/meshchatx/meshchat.py index b53e73af..7db40ffc 100644 --- a/meshchatx/meshchat.py +++ b/meshchatx/meshchat.py @@ -1394,10 +1394,14 @@ class ReticulumMeshChat: ensure_safe_reticulum_runtime_flags(config_path) def _set_startup_stage(self, stage: str, error: str | None = None) -> None: + previous = getattr(self, "_startup_stage", None) self._startup_stage = stage if error is not None: self._startup_error = error - print(f"Startup stage: {stage}", flush=True) + # Same stage can be set from both the network-setup wrapper and + # setup_identity. Only log transitions to keep console noise down. + if previous != stage or error is not None: + print(f"Startup stage: {stage}", flush=True) def _mark_network_ready(self) -> None: self._network_ready = True @@ -1533,7 +1537,6 @@ class ReticulumMeshChat: self._set_startup_stage("failed", "No identity available for network setup") return try: - self._set_startup_stage("rns") self.setup_identity(identity) if self.config is not None and getattr(self, "session_secret_key", None): try: @@ -1603,7 +1606,6 @@ class ReticulumMeshChat: loglevel=rns_loglevel, ) _restore_rns_console_logging_after_reticulum_init(self) - self._set_startup_stage("identity") self.page_node_manager.load_nodes() self.page_node_manager.start_all() self.plugin_manager.set_app(self) @@ -10572,10 +10574,13 @@ class ReticulumMeshChat: int(profile_id), ) self.config.telephone_audio_profile_id.set(resolved) + requested = int(profile_id) return web.json_response( { "message": f"Switched to profile {resolved}", "profile_id": resolved, + "requested_profile_id": requested, + "remapped": requested != resolved, }, ) except Exception as e: @@ -10586,17 +10591,25 @@ class ReticulumMeshChat: async def telephone_codec2_status(request): from meshchatx import android_codec2 - available = await asyncio.to_thread( - self.telephone_manager.codec2_available, - ) - return web.json_response( - { + def _status(): + probe_ok, probe_error = android_codec2.probe_pycodec2() + lxst_ok = self.telephone_manager.codec2_available() + available = bool(probe_ok and lxst_ok) + return { "codec2_available": available, "preload_error": android_codec2.codec2_preload_error(), + "probe_error": None if probe_ok else probe_error, + "platform": ( + "android" + if android_codec2._is_chaquopy_android() + else "desktop" + ), "preferred_profile_id": self.telephone_manager.preferred_profile_id, "resolved_profile_id": self.telephone_manager.resolve_audio_profile_id(), - }, - ) + } + + payload = await asyncio.to_thread(_status) + return web.json_response(payload) # initiate a telephone call # initiate outgoing telephone call @@ -10664,22 +10677,37 @@ class ReticulumMeshChat: @routes.get("/api/v1/telephone/audio-profiles") async def telephone_audio_profiles(request): from LXST.Primitives.Telephony import Profiles + from meshchatx import android_codec2 - # get audio profiles - audio_profiles = [ - { - "id": available_profile, - "name": Profiles.profile_name(available_profile), + def _profiles(): + probe_ok, _probe_err = android_codec2.probe_pycodec2() + codec2_ok = bool(probe_ok and self.telephone_manager.codec2_available()) + codec2_ids = { + Profiles.BANDWIDTH_ULTRA_LOW, + Profiles.BANDWIDTH_VERY_LOW, + Profiles.BANDWIDTH_LOW, } - for available_profile in Profiles.available_profiles() - ] - - return web.json_response( - { - "default_audio_profile_id": Profiles.DEFAULT_PROFILE, + audio_profiles = [] + for profile_id in Profiles.available_profiles(): + entry = { + "id": profile_id, + "name": Profiles.profile_name(profile_id), + "available": True, + } + if profile_id in codec2_ids and not codec2_ok: + entry["available"] = False + entry["unavailable_reason"] = "codec2" + audio_profiles.append(entry) + return { + "default_audio_profile_id": self.telephone_manager.resolve_audio_profile_id( + Profiles.DEFAULT_PROFILE, + ), + "codec2_available": codec2_ok, "audio_profiles": audio_profiles, - }, - ) + } + + payload = await asyncio.to_thread(_profiles) + return web.json_response(payload) # voicemail status @routes.get("/api/v1/telephone/voicemail/status") @@ -19312,6 +19340,28 @@ class ReticulumMeshChat: ) return + # Known hosts (map/docs) are handled above. Relay and app + # deep links are frontend-routed. Anything else must not + # fall through to LXMF ingest. + AsyncUtils.run_async( + client.send_str( + json.dumps( + { + "type": "lxm.ingest_uri.result", + "status": "error", + "message": ( + f"Unknown or unsupported meshchatx link host " + f"'{_host or '(empty)'}'. " + "Supported hosts include map, docs, relay, and app." + ), + "ingest_type": "unknown_meshchatx", + "host": _host, + }, + ), + ), + ) + return + # LXMA contact sharing URI: # lxma://: if uri.lower().startswith("lxma://"): @@ -19337,18 +19387,13 @@ class ReticulumMeshChat: bytes.fromhex(destination_hash_hex) raw_bytes = bytes.fromhex(public_key_hex) - identity = RNS.Identity(create_keys=False) - loaded = False - for candidate in ( - raw_bytes, - raw_bytes[:32] if len(raw_bytes) > 32 else None, - ): - if not candidate: - continue - if identity.load_public_key(candidate): - loaded = True - break - if not loaded: + # RNS Identity.load_public_key docs say True/False but the + # implementation returns None on success. Prefer full 64-byte + # keys; truncated 32-byte material is not a valid RNS pubkey. + identity = self._identity_from_public_key_bytes(raw_bytes) + if identity is None and len(raw_bytes) > 32: + identity = self._identity_from_public_key_bytes(raw_bytes[:32]) + if identity is None: raise ValueError("Invalid LXMA public key") remote_identity_hash = identity.hash.hex() @@ -19369,6 +19414,20 @@ class ReticulumMeshChat: lxmf_address=destination_hash_hex, ) + # Persist pubkey so outbound LXMF works before any announce. + try: + RNS.Identity.remember( + None, + bytes.fromhex(destination_hash_hex), + identity.get_public_key(), + None, + ) + except Exception as remember_exc: + print( + f"LXMA remember failed for {destination_hash_hex}: " + f"{type(remember_exc).__name__}: {remember_exc!r}", + ) + AsyncUtils.run_async( client.send_str( json.dumps( @@ -20368,8 +20427,8 @@ class ReticulumMeshChat: if announce and announce.get("identity_public_key"): public_key = base64.b64decode(announce["identity_public_key"]) - identity = RNS.Identity(create_keys=False) - if identity.load_public_key(public_key): + identity = self._identity_from_public_key_bytes(public_key) + if identity is not None: return identity except Exception as e: @@ -20377,6 +20436,27 @@ class ReticulumMeshChat: return None + @staticmethod + def _identity_from_public_key_bytes(public_key: bytes) -> RNS.Identity | None: + """Load an RNS Identity from raw public-key bytes. + + ``Identity.load_public_key`` is documented as returning True/False, but + current RNS releases return ``None`` on both success and failure. Treat + a non-None ``identity.pub`` (and a computed hash) as success. + """ + if not public_key: + return None + identity = RNS.Identity(create_keys=False) + try: + identity.load_public_key(public_key) + except Exception: + return None + if getattr(identity, "pub", None) is None: + return None + if not getattr(identity, "hash", None): + return None + return identity + # convert an lxmf message to a dictionary, for sending over websocket # convert database announce to a dictionary @@ -20546,9 +20626,7 @@ class ReticulumMeshChat: if not identity and announce.get("identity_public_key"): # Try to load from public key if recall failed public_key = base64.b64decode(announce["identity_public_key"]) - identity = RNS.Identity(create_keys=False) - if not identity.load_public_key(public_key): - identity = None + identity = self._identity_from_public_key_bytes(public_key) if identity: try: diff --git a/meshchatx/src/backend/telephone_manager.py b/meshchatx/src/backend/telephone_manager.py index b1dfd813..4ab7739b 100644 --- a/meshchatx/src/backend/telephone_manager.py +++ b/meshchatx/src/backend/telephone_manager.py @@ -98,6 +98,15 @@ class TelephoneManager: @staticmethod def codec2_available() -> bool: """Return whether LXST can construct Codec2 codecs (pycodec2 + libcodec2).""" + try: + from meshchatx import android_codec2 + + if android_codec2._is_chaquopy_android(): + ok, _err = android_codec2.probe_pycodec2() + if not ok: + return False + except Exception: + pass try: from LXST.Codecs import Codec2 diff --git a/meshchatx/src/frontend/components/App.vue b/meshchatx/src/frontend/components/App.vue index bcf225ff..b31a6a56 100644 --- a/meshchatx/src/frontend/components/App.vue +++ b/meshchatx/src/frontend/components/App.vue @@ -91,7 +91,6 @@ />