From c2a183d688da793e20dfe6876d2c12add7b6ced2 Mon Sep 17 00:00:00 2001 From: Ivan Date: Fri, 10 Jul 2026 00:57:53 -0500 Subject: [PATCH] feat: map overlays, identity restore, settings, conversations, and i18n updates --- CHANGELOG.md | 31 +- README.md | 2 +- .../app/src/main/python/meshchat_wrapper.py | 40 +- docs/en/architecture.md | 4 +- docs/en/rns-link-api.md | 6 +- docs/en/tools.md | 2 +- .../src/backend/auto_propagation_manager.py | 19 +- meshchatx/src/backend/config_manager.py | 50 + .../plugins/mesh-observatory/frontend/main.js | 231 ----- .../plugins/mesh-observatory/locales/en.json | 23 - .../data/plugins/mesh-observatory/plugin.json | 41 - meshchatx/src/backend/database/__init__.py | 53 +- .../src/backend/database/map_overlays.py | 138 +++ meshchatx/src/backend/database/messages.py | 39 +- meshchatx/src/backend/database/provider.py | 23 +- meshchatx/src/backend/database/schema.py | 72 +- meshchatx/src/backend/docs_manager.py | 13 +- meshchatx/src/backend/identity_context.py | 20 + meshchatx/src/backend/identity_manager.py | 57 +- meshchatx/src/backend/lxmf_utils.py | 176 ++-- meshchatx/src/backend/map_geo_validator.py | 259 ++++++ meshchatx/src/backend/map_overlay_export.py | 282 ++++++ meshchatx/src/backend/map_overlay_manager.py | 861 ++++++++++++++++++ meshchatx/src/backend/map_overlay_sources.py | 271 ++++++ meshchatx/src/backend/memory_pressure.py | 9 +- meshchatx/src/backend/message_handler.py | 44 +- .../src/backend/repository_server_manager.py | 77 +- .../src/backend/reticulum_config_guard.py | 13 + meshchatx/src/backend/rncp_handler.py | 119 ++- meshchatx/src/backend/rngit_sparse_fetcher.py | 268 ++++++ meshchatx/src/backend/rnpath_trace_handler.py | 12 +- meshchatx/src/backend/self_check.py | 2 +- meshchatx/src/frontend/components/App.vue | 93 +- .../src/frontend/components/TutorialModal.vue | 221 ++++- .../frontend/components/about/AboutPage.vue | 6 +- .../components/forwarder/ForwarderPage.vue | 27 +- .../interfaces/AddInterfacePage.vue | 4 +- .../components/layout/AppShellBanners.vue | 47 +- .../src/frontend/components/map/MapPage.vue | 173 ++++ .../map/internal/MapRemoteOverlayPanel.vue | 295 ++++++ .../messages/ConversationDropDownMenu.vue | 68 +- .../messages/ConversationPeerHeader.vue | 6 +- .../messages/ConversationViewer.vue | 358 +++++--- .../messages/MessageReactionsOverlay.vue | 26 +- .../components/messages/MessagesPage.vue | 34 +- .../nomadnetwork/NomadNetworkPage.vue | 10 +- .../src/frontend/components/ping/PingPage.vue | 2 +- .../PropagationNodesPage.vue | 18 +- .../components/relay/RelayChatPage.vue | 2 + .../src/frontend/components/rncp/RNCPPage.vue | 23 +- .../components/rnprobe/RNProbePage.vue | 2 +- .../components/rnstatus/RNStatusPage.vue | 8 +- .../components/settings/IdentitiesPage.vue | 96 +- .../settings/PluginsSettingsSection.vue | 57 +- .../components/settings/SettingsPage.vue | 282 +++++- .../frontend/components/tools/RNPathPage.vue | 22 +- meshchatx/src/frontend/index.html | 18 +- meshchatx/src/frontend/js/GlobalState.js | 2 + meshchatx/src/frontend/js/Utils.js | 8 +- .../src/frontend/js/WebSocketConnection.js | 2 + meshchatx/src/frontend/js/lxmfReactions.js | 16 +- .../src/frontend/js/networkStartupWait.js | 24 +- .../registries/coreSettingsSectionKeywords.js | 9 + .../src/frontend/js/rnode/AndroidBridge.js | 7 + meshchatx/src/frontend/locales/de.json | 86 +- meshchatx/src/frontend/locales/en.json | 114 ++- meshchatx/src/frontend/locales/es.json | 86 +- meshchatx/src/frontend/locales/fi.json | 86 +- meshchatx/src/frontend/locales/fr.json | 86 +- meshchatx/src/frontend/locales/it.json | 86 +- meshchatx/src/frontend/locales/nl.json | 86 +- meshchatx/src/frontend/locales/ru.json | 86 +- meshchatx/src/frontend/locales/zh.json | 86 +- meshchatx/src/frontend/main.js | 50 +- .../public/meshchatx-docs/en/architecture.md | 4 +- .../public/meshchatx-docs/en/rns-link-api.md | 6 +- .../public/meshchatx-docs/en/tools.md | 2 +- meshchatx/src/frontend/style.css | 28 + scripts/sync-meshchatx-docs.js | 10 +- tests/backend/fixtures/http_api_routes.json | 60 ++ tests/backend/test_auto_propagation.py | 33 + tests/backend/test_call_codec2_regressions.py | 4 +- tests/backend/test_docs_manager.py | 25 + tests/backend/test_identity_restore.py | 75 ++ .../backend/test_identity_restore_http_api.py | 45 + tests/backend/test_map_geo_validator.py | 140 +++ tests/backend/test_map_overlay_api.py | 106 +++ tests/backend/test_map_overlay_export.py | 61 ++ tests/backend/test_map_overlay_manager.py | 468 ++++++++++ tests/backend/test_map_overlay_sources.py | 103 +++ tests/backend/test_memory_pressure.py | 22 +- .../backend/test_message_handler_extended.py | 8 + .../backend/test_message_sending_failures.py | 69 +- tests/backend/test_ping_api.py | 70 ++ .../backend/test_repository_server_manager.py | 31 + tests/backend/test_rncp_handler_extended.py | 51 ++ tests/backend/test_rngit_sparse_fetcher.py | 128 +++ tests/backend/test_rnpath_trace_handler.py | 28 + tests/backend/test_rns_link_plugin.py | 10 +- tests/backend/test_rns_startup_recovery.py | 160 ++++ .../test_sqlite_landlock_temp_store.py | 262 ++++++ tests/backend/test_sqlite_memory_pressure.py | 13 + tests/frontend/AddInterfaceOptions.test.js | 19 +- tests/frontend/AppPropagationSync.test.js | 12 +- tests/frontend/BootLoadSmoothness.test.js | 112 +++ .../ConversationDropDownMenu.shareApk.test.js | 109 +++ .../frontend/ConversationMobileChrome.test.js | 142 +++ tests/frontend/ConversationPeerHeader.test.js | 32 +- tests/frontend/ConversationViewer.test.js | 50 +- .../ConversationViewerReactions.test.js | 306 +++++++ tests/frontend/ForwarderPage.test.js | 11 +- tests/frontend/IdentitiesPage.test.js | 59 ++ tests/frontend/MapRemoteOverlayPanel.test.js | 96 ++ .../frontend/MessageReactionsOverlay.test.js | 81 ++ tests/frontend/MessageSendingFailures.test.js | 92 +- tests/frontend/MobilePopoutVisibility.test.js | 157 ++++ .../NotificationBellConversationSync.test.js | 2 +- tests/frontend/RNSHManagerPage.test.js | 4 +- tests/frontend/RNStatusPage.test.js | 16 + tests/frontend/TutorialModalMigration.test.js | 194 ++++ tests/frontend/Utils.test.js | 7 + tests/frontend/WebSocketConnection.test.js | 2 +- .../frontend/fixtures/settingsPageTestApi.js | 10 + tests/frontend/lxmfReactions.test.js | 40 + tests/frontend/networkStartupWait.test.js | 41 +- 125 files changed, 8786 insertions(+), 905 deletions(-) delete mode 100644 meshchatx/src/backend/data/plugins/mesh-observatory/frontend/main.js delete mode 100644 meshchatx/src/backend/data/plugins/mesh-observatory/locales/en.json delete mode 100644 meshchatx/src/backend/data/plugins/mesh-observatory/plugin.json create mode 100644 meshchatx/src/backend/database/map_overlays.py create mode 100644 meshchatx/src/backend/map_geo_validator.py create mode 100644 meshchatx/src/backend/map_overlay_export.py create mode 100644 meshchatx/src/backend/map_overlay_manager.py create mode 100644 meshchatx/src/backend/map_overlay_sources.py create mode 100644 meshchatx/src/backend/rngit_sparse_fetcher.py create mode 100644 meshchatx/src/frontend/components/map/internal/MapRemoteOverlayPanel.vue create mode 100644 tests/backend/test_map_geo_validator.py create mode 100644 tests/backend/test_map_overlay_api.py create mode 100644 tests/backend/test_map_overlay_export.py create mode 100644 tests/backend/test_map_overlay_manager.py create mode 100644 tests/backend/test_map_overlay_sources.py create mode 100644 tests/backend/test_ping_api.py create mode 100644 tests/backend/test_rngit_sparse_fetcher.py create mode 100644 tests/backend/test_rnpath_trace_handler.py create mode 100644 tests/backend/test_rns_startup_recovery.py create mode 100644 tests/backend/test_sqlite_landlock_temp_store.py create mode 100644 tests/frontend/BootLoadSmoothness.test.js create mode 100644 tests/frontend/ConversationDropDownMenu.shareApk.test.js create mode 100644 tests/frontend/ConversationMobileChrome.test.js create mode 100644 tests/frontend/ConversationViewerReactions.test.js create mode 100644 tests/frontend/MapRemoteOverlayPanel.test.js create mode 100644 tests/frontend/MessageReactionsOverlay.test.js create mode 100644 tests/frontend/MobilePopoutVisibility.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index d33c1100..2c899cc0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file. ### Added +- **Map / remote overlays**: Import KMZ/KML/GeoJSON overlays from NomadNet ``hash:/file/...`` links and sparse RNGit ``rns://hash/group/repo`` fetches (specific paths at branch/tag/commit). Managed overlay sources with backend cache, refresh/autorefresh, retries with backoff, path requests, atomic writes, SHA256 skip, re-export (GeoJSON/KML/KMZ), and Settings → Map limits/timeouts. APIs under ``/api/v1/map/overlays*``. - **Plugins / security**: MeshChatX plugin signing with RSG verification (`plugin_rsg.py`), canonical ZIP/dir payloads, trusted publishers with digest tamper detection, post-install integrity hashing, and heuristic security findings. Invalid signatures hard-block install. Preview/install expose signature and findings. Trusted publisher APIs under `/api/v1/plugins/trusted-publishers`. - **Plugins / WASM bundles**: Single-file `.wasm` install with embedded `meshchatx.plugin` / `meshchatx.files` / `meshchatx.signature` custom sections (`plugin_wasm_bundle.py`). Preview/install accept `.wasm` as well as ZIP. Signing CLI `scripts/sign-plugin.py` for dir/zip/wasm/py. - **Plugins / Python backend**: ZIP/WASM-unpacked `backend.type: "python"` runtime (`plugin_python_runtime.py`) with permission-checked host, hooks/invoke, and Python badge in settings. @@ -17,32 +18,37 @@ All notable changes to this project will be documented in this file. - **Plugins**: Contribution-point registries for sidebar navigation, tools catalog, command palette, settings sections, and typed WebSocket event dispatch. Core UI surfaces are data-driven instead of hardcoded per component. - **Plugins**: **Bundled i18n**. Plugins ship `locales/{locale}.json` in the package. The host loads labels from plugin assets so third-party plugins do not need changes to MeshChatX main locale files. - **Plugins**: Declarative UI slot vocabulary in `PluginSlotRenderer` / `PluginSlotNode` for sections, action button rows, badges, card lists, grid rows, and text variants on plugin tool pages. -- **Plugins**: Bundled **Mesh Observatory** example plugin with live announce feed, searchable path table with hop/interface/state columns, and announce-driven refresh. +- **Plugins**: Bundled **Bug Reports** plugin (`com.meshchatx.mcx-bugs`) with sender and collector modes over aspect ``mcx-bugs-v1``, selectable log redaction, and Tools/nav contribution. Host capabilities ``debugLog.read`` and ``bugReport.*`` plus checkbox/textarea plugin UI slots. - **Plugins**: Security guardrails in `plugin_guard.py` for ZIP size/magic validation, zip-slip protection, asset path normalization, WASM size/magic checks, invoke payload limits, and an error budget that auto-disables misbehaving plugins after repeated failures. - **Plugins**: `POST /api/v1/plugins/{id}/report-failure` for frontend worker crash reporting. Kill-switch broadcasts over WebSocket when a plugin is auto-disabled. - **Dependencies**: Added **wasmtime** for backend WASM plugin execution. - **Plugins**: `--disable-plugins` CLI flag and `MESHCHAT_DISABLE_PLUGINS` environment variable to disable the plugin system entirely at runtime. -- **RNS Link API**: Generic WebSocket Link transport (`rns.link.open` / `identify` / `request` / `send` / `close` plus `rns.link.event` broadcasts) so external apps can use MeshChatX as an RNS transport. Based on contributions from [attermann/api_extensions](https://github.com/attermann/MeshChatX/tree/api_extensions). Auth-gated when password auth is enabled; in-flight open/request tasks cancel on client disconnect. +- **RNS Link API**: Generic WebSocket Link transport (`rns.link.open` / `identify` / `request` / `send` / `close` plus `rns.link.event` broadcasts) so external apps can use MeshChatX as an RNS transport. Based on contributions from [attermann/api_extensions](https://github.com/attermann/MeshChatX/tree/api_extensions). Auth-gated when password auth is enabled. In-flight open/request tasks cancel on client disconnect. - **Plugins / RNS Link**: Manager capabilities `rnsLink.open`, `rnsLink.identify`, `rnsLink.request`, `rnsLink.send`, `rnsLink.close` and hook `rns.link.event` for embedding transport-node tools (for example microReticulum management) as MeshChatX plugins. - **Tests / RNS Link**: Unit, edge-case, Hypothesis fuzz/property, plugin capability, WS auth, smoke, behavior-contract, and self-check coverage for the generic Link API (`websocket_rns_link_good` probe). -- **Plugins / install consent**: ZIP install previews via `POST /api/v1/plugins/preview`, then a confirmation dialog listing requested permissions and scanned/declared external HTTP endpoints. Users can deny individual grants; runtime enforces declared+granted hooks/managers/storage/`network:fetch`. +- **Plugins / install consent**: ZIP install previews via `POST /api/v1/plugins/preview`, then a confirmation dialog listing requested permissions and scanned/declared external HTTP endpoints. Users can deny individual grants. Runtime enforces declared+granted hooks/managers/storage/`network:fetch`. - **Vendored LXMFy**: Refreshed `vendor/lxmfy` to upstream **1.6.5** (`d92cfe0`) with Landlock LSM sandbox for bot processes and external cogs, propagation-node init fix, cog permission fix, and dependency alignment with RNS 1.3.5+ / LXMF 1.0.1+. - **Mutation testing**: Backend uses **mutmut** (`task test:mutation:backend`). Frontend uses in-repo **MeshMut** (`task test:mutation:frontend`) with regex-based mutators and Vitest for pure JS modules. Optional `mutation.yml` workflow for manual or scheduled runs. ### Changed -- **CI benchmarks**: Suite runs **3 times** and reports **median of medians** with MAD/CV in the JSON extra field. A smart gate replaces flat ratio alerts: noise floor (0.5 ms), minimum absolute delta (1.5 ms), and adaptive ratio thresholds that widen for tiny/noisy baselines. The previous ``Trim Announces`` 2.57x false positive (0.177→0.455 ms) is ignored; that bench now re-seeds before each sample so it measures a real DELETE. ``actions/cache`` bumped to **v5** (Node 24). -- **Startup**: HTTP server binds before Reticulum/identity setup. RNS and identity context initialize on a background thread while Electron can open the UI shell; `/api/v1/status` reports `starting`/`ok` with `stage` and `network_ready`, and the Vue boot splash waits until the network stack is ready. CLI one-shots (`--self-check`, backup/restore, etc.) still initialize synchronously. Off-main-thread RNS construction skips ``signal.signal`` registration (Python restriction) and reinstalls SIGINT/SIGTERM handlers on the main loop once ready. -- **Relay Chat**: Collapsed sidebar add control is a plain plus (no dashed border). Available Rooms can collapse and the state is remembered in localStorage. Discover hub list spacing is tighter and the search placeholder shows the heard hub count. Hub create/settings support announce interval (slider + minutes input); hosted hub cards show human-readable uptime, label connected peers as users, and expose settings to rename or change/disable announces. Consecutive join/leave/connection system lines auto-collapse into a summary that can be expanded. Auto-rejoin after reconnect records ``You rejoined #room``; link drops/manual disconnect/reconnect also write ``Connection lost`` / ``Disconnected from hub`` / ``Reconnected to hub`` into joined room timelines. +- **CI benchmarks**: Suite runs **3 times** and reports **median of medians** with MAD/CV in the JSON extra field. A smart gate replaces flat ratio alerts: noise floor (0.5 ms), minimum absolute delta (1.5 ms), and adaptive ratio thresholds that widen for tiny/noisy baselines. The previous ``Trim Announces`` 2.57x false positive (0.177→0.455 ms) is ignored. That bench now re-seeds before each sample so it measures a real DELETE. ``actions/cache`` bumped to **v5** (Node 24). +- **Startup**: HTTP server binds before Reticulum/identity setup. RNS and identity context initialize on a background thread while Electron can open the UI shell. `/api/v1/status` reports `starting`/`ok` with `stage` and `network_ready`, and the Vue boot splash waits until the network stack is ready. CLI one-shots (`--self-check`, backup/restore, etc.) still initialize synchronously. Off-main-thread RNS construction skips ``signal.signal`` registration (Python restriction) and reinstalls SIGINT/SIGTERM handlers on the main loop once ready. +- **Relay Chat**: Collapsed sidebar add control is a plain plus (no dashed border). Available Rooms can collapse and the state is remembered in localStorage. Discover hub list spacing is tighter and the search placeholder shows the heard hub count. Hub create/settings support announce interval (slider + minutes input). Hosted hub cards show human-readable uptime, label connected peers as users, and expose settings to rename or change/disable announces. Consecutive join/leave/connection system lines auto-collapse into a summary that can be expanded. Auto-rejoin after reconnect records ``You rejoined #room``. Link drops/manual disconnect/reconnect also write ``Connection lost`` / ``Disconnected from hub`` / ``Reconnected to hub`` into joined room timelines. + +- **Dependencies**: pnpm overrides bump transitive **minimist** to **>=1.2.8** and **fast-uri** to **>=3.1.2** (prototype pollution / URI normalization advisories via electron-builder tooling). +- **Settings**: Settings section search keywords moved into `settingsSectionRegistry` for reuse by plugins and core sections. +- **App shell**: WebSocket handling in `App.vue` migrated to typed per-event handlers via `wsEventRegistry`. +- **Locales**: Main app locale files retain only **Settings → Plugins** UI strings. Per-plugin copy lives in each plugin bundle. ### Fixed - **Settings**: RPC key is hidden by default (star/bullet mask) and reveals on click/tap. Failed self-test rows expand to show the failure reason. Plugins moved to their own Settings tab. Notification sound enable toggle no longer crowds the description. Community Interfaces settings include a refresh control that fetches from ``directory.rns.recipes`` submitted + discovered online listings (updated default URLs). - **Android / startup**: Loading screen no longer shows attempt counters like ``(12/120)``. Copy uses short friendly phases, splash uses a full uncropped logo, and the adaptive launcher foreground is padded so corners are not clipped by the circular/squircle mask. -- **Android**: Nightly/APK boot no longer crashes with ``ModuleNotFoundError: No module named 'lxmfy'``. Gradle syncs vendored ``vendor/lxmfy/lxmfy`` into Chaquopy ``src/main/python/lxmfy`` (desktop already got it via setuptools; Android pip never installed it). +- **Android**: Nightly/APK boot no longer crashes with ``ModuleNotFoundError: No module named 'lxmfy'``. Gradle syncs vendored ``vendor/lxmfy/lxmfy`` into Chaquopy ``src/main/python/lxmfy`` (desktop already got it via setuptools, Android pip never installed it). - **Android**: Backend boot no longer exits with ``SystemExit: 1`` when ``fcntl.flock`` is unimplemented (common on Android). ``StorageLock`` falls back to a PID soft lock, and the Chaquopy wrapper clears a stale ``.meshchatx.lock`` before ``main()``. - **CI / Android**: Nightly and ``workflow_dispatch`` emulator smoke (``.github/workflows/android-emulator-smoke.yml``) builds an x86_64 debug APK, installs it on an AVD, launches ``MainActivity``, and requires on-device ``/api/v1/status`` to return ok (catches Chaquopy boot failures that ``assembleDebug`` alone misses). -- **Network visualiser**: Physics and canvas draw cost brought below upstream MeshChat for large meshes — disabled Barnes–Hut ``avoidOverlap``, matched upstream gravity, dropped per-edge arrows/dashes (direct vs multi-hop still distinguished by color/width), hide edges while zooming, larger adaptive build chunks with sync path for small graphs, debounced search rebuilds, cheaper LOD updates, and removed toolbar/legend backdrop-blur compositing over the canvas. +- **Network visualiser**: Physics and canvas draw cost brought below upstream MeshChat for large meshes - disabled Barnes-Hut ``avoidOverlap``, matched upstream gravity, dropped per-edge arrows/dashes (direct vs multi-hop still distinguished by color/width), hide edges while zooming, larger adaptive build chunks with sync path for small graphs, debounced search rebuilds, cheaper LOD updates, and removed toolbar/legend backdrop-blur compositing over the canvas. - **Relay Chat**: Message list no longer stacks or duplicates text. Keys prefer message ``seq``, websocket/history loads dedupe, and room loads merge live websocket arrivals instead of wiping them. - **Nomad Network / favourites**: Favourite names no longer become **Unknown Node** when a path or announce is missing. Resolution falls back to the stored favourite name (and announce cache), and unknown/localized placeholders no longer overwrite real names on re-add or bulk-add. - **Nomad Network / sections**: Moving favourites into custom named sections now persists across reload and identity switches (DB-backed). Layout reconciliation no longer wipes storage while favourites are still loading. @@ -54,7 +60,7 @@ All notable changes to this project will be documented in this file. - **CI / nightly**: Daily ``nightly-YYYY.MM.DD-`` tags from ``dev`` now explicitly ``workflow_dispatch`` ``build-release.yml`` after tagging so full release assets are produced. - **CI / nightly**: Release upload creates nightlies and previews as **drafts**, attaches all assets, then publishes as prereleases so immutable-release repos can still receive binaries. - **Plugins**: Plugin worker `postRequest` Promise wrapper, plugin locale loading at boot, cached UI on page open, and slot renderer recursion for nested column/list/row children. -- **Plugins**: Mesh Observatory layout with spaced action buttons, section cards, truncated interface names, and state badges instead of squashed single-line rows. +- **Plugins**: Removed the Mesh Observatory example plugin. - **RNode / Android**: Hardened `rnode_support` startup guards. Desktop TCP RNode no longer incorrectly requires pyserial. Desktop BLE now checks for bleak instead of pyserial. Whitespace-only Bluetooth ports classify correctly. Invalid `tcp:///` hosts are no longer backfilled. `RNodeIPInterface` entries get `tcp_host` backfill on Android. RNodeMulti sibling sub-interfaces with invalid TX power are detected and disabled. Txpower guard honors both `enabled` and `interface_enabled` keys. - **RNode / desktop**: Added **bleak** as a core dependency and a desktop startup guard that disables unsupported RNode interfaces before Reticulum starts, fixing backend crashes when RNode over BLE is configured on Windows without bleak installed ([#46](https://github.com/Quad4-Software/MeshChatX/issues/46)). - **CI / tests**: Dependency contract test for bleak, startup integration test for the desktop RNode guard, and cx_Freeze build verification that bleak is bundled. @@ -63,13 +69,6 @@ All notable changes to this project will be documented in this file. - **Settings**: Tabbed settings navigation with section-to-tab mapping, search across tabs, and `SettingsNav` component. - **Settings**: Plugin settings search no longer treats `index.mu` / `index.html` literals as missing i18n keys. -### Changed - -- **Dependencies**: pnpm overrides bump transitive **minimist** to **>=1.2.8** and **fast-uri** to **>=3.1.2** (prototype pollution / URI normalization advisories via electron-builder tooling). -- **Settings**: Settings section search keywords moved into `settingsSectionRegistry` for reuse by plugins and core sections. -- **App shell**: WebSocket handling in `App.vue` migrated to typed per-event handlers via `wsEventRegistry`. -- **Locales**: Main app locale files retain only **Settings → Plugins** UI strings. Per-plugin copy lives in each plugin bundle. - ### Tests - **Startup**: Deferred RNS/identity init covered by unit, middleware (503 vs status/auth/csrf), concurrent status reads, failure/idempotency edge cases, Hypothesis fuzz of status payloads, Vue ``networkStartupWait`` polls, and Electron ``loadingStatusProbe`` accept/reject rules for ``starting``/``ok``/``failed``. diff --git a/README.md b/README.md index 1f08bd0d..cce2b9be 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ rngit: `git clone rns://06a54b505bb67b25ef3f8097e8001edc/public/MeshChatX` - Uses LXST for calls - Integrates [RRC](https://rrc.kc1awv.net/0) - Expanded tools -- Map w/ MBTiles support +- Map w/ MBTiles support, remote KMZ/KML/GeoJSON overlays (NomadNet `/file/` and RNGit sparse fetch) - Panes and Tabs - Replaced Peewee ORM with raw SQL. - Replaced Axios with native fetch. diff --git a/android/app/src/main/python/meshchat_wrapper.py b/android/app/src/main/python/meshchat_wrapper.py index ee9fef11..901cc9b8 100644 --- a/android/app/src/main/python/meshchat_wrapper.py +++ b/android/app/src/main/python/meshchat_wrapper.py @@ -18,14 +18,33 @@ def _ensure_android_reticulum_config(reticulum_config_dir): if os.path.exists(config_path): with open(config_path, encoding="utf-8") as existing_file: content = existing_file.read() + changed = False if "share_instance = Yes" in content: content = content.replace("share_instance = Yes", "share_instance = No") + changed = True + if "panic_on_interface_error" not in content: + if "[reticulum]" in content: + content = content.replace( + "[reticulum]", + "[reticulum]\n panic_on_interface_error = No", + 1, + ) + else: + content = "[reticulum]\n panic_on_interface_error = No\n\n" + content + changed = True + if changed: with open(config_path, "w", encoding="utf-8") as config_file: config_file.write(content) return with open(config_path, "w", encoding="utf-8") as config_file: - config_file.write("[reticulum]\n share_instance = No\n\n[interfaces]\n") + config_file.write( + "[reticulum]\n" + " share_instance = No\n" + " panic_on_interface_error = No\n" + "\n" + "[interfaces]\n" + ) def _patch_asyncio_signal_handlers_for_android(): @@ -89,6 +108,19 @@ def _clear_stale_storage_lock(storage_dir): print(f"meshchat_wrapper: could not clear storage lock: {exc}") +def _patch_rns_panic_for_android(): + """Stop RNS.panic/os._exit from killing the whole Android process.""" + try: + from meshchatx.src.backend.rns_startup_recovery import ( + install_rns_panic_containment, + ) + + return install_rns_panic_containment() + except Exception as exc: + print(f"meshchat_wrapper: RNS panic containment skipped: {exc}") + return False + + def start_server(port=8000, app_files_dir=None): global _server_loop_active with _server_loop_lock: @@ -121,8 +153,12 @@ def start_server(port=8000, app_files_dir=None): signal.signal = _safe_signal asyncio_signal_patch = _patch_asyncio_signal_handlers_for_android() aiohttp_run_app_patch = _patch_aiohttp_run_app_for_android() + _patch_rns_panic_for_android() try: - from meshchatx.android_codec2 import ensure_codec2_native_library, probe_pycodec2 + from meshchatx.android_codec2 import ( + ensure_codec2_native_library, + probe_pycodec2, + ) ensure_codec2_native_library() ok, err = probe_pycodec2() diff --git a/docs/en/architecture.md b/docs/en/architecture.md index be4c7677..18aed096 100644 --- a/docs/en/architecture.md +++ b/docs/en/architecture.md @@ -10,6 +10,8 @@ MeshChatX is a heavily extended fork of Reticulum MeshChat. The goals below shap - Keep the Python backend and Vue frontend independently testable. - Run in constrained environments with predictable SQLite behaviour. +Mesh features should follow Reticulum’s post-IP design patterns (portable identity hashes, announces, store-and-forward, transport-agnostic APIs, scarce payloads). Agent and contributor gates live in `docs/agents/conventions/reticulum-zen.md` and `docs/agents/skills/reticulum-design-gates/SKILL.md`, derived from the [Zen of Reticulum](https://reticulum.network/manual/zen.html). + ## Process overview One Python process owns the web server, Reticulum stack, and all per-identity managers. The Vue frontend is static assets served from `meshchatx/public/` after a Vite build. @@ -138,7 +140,7 @@ Practical extension paths today: - Database schema changes through migrations - Generic RNS Link transport over WebSocket (`rns.link.*`) for external consoles and plugins (see **RNS Link API**) -Granted plugin manager capabilities include `destinationPath.read` and `rnsLink.open` / `identify` / `request` / `send` / `close`. Hooks include `announce.received` and `rns.link.event`. Storage (`storage:isolated`) and outbound HTTP (`network:fetch`) are also grantable; install preview scans plugin files for external URLs and stores the user-selected grant subset. +Granted plugin manager capabilities include `destinationPath.read`, `debugLog.read`, `bugReport.*`, and `rnsLink.open` / `identify` / `request` / `send` / `close`. Hooks include `announce.received` and `rns.link.event`. Storage (`storage:isolated`) and outbound HTTP (`network:fetch`) are also grantable; install preview scans plugin files for external URLs and stores the user-selected grant subset. When adding features, prefer identity-scoped state, explicit migrations, endpoint tests, and narrowly declared plugin permissions. diff --git a/docs/en/rns-link-api.md b/docs/en/rns-link-api.md index f9e6e3ee..64376caf 100644 --- a/docs/en/rns-link-api.md +++ b/docs/en/rns-link-api.md @@ -60,9 +60,9 @@ Example manifest fragment: ## Implementation -- `meshchatx/src/backend/rns_link_manager.py` — link cache, open/identify/request/send/close -- `meshchatx/meshchat.py` — WebSocket dispatch and per-client task tracking -- `meshchatx/src/backend/plugin_manager.py` — capability wrappers and hook fan-out +- `meshchatx/src/backend/rns_link_manager.py` - link cache, open/identify/request/send/close +- `meshchatx/meshchat.py` - WebSocket dispatch and per-client task tracking +- `meshchatx/src/backend/plugin_manager.py` - capability wrappers and hook fan-out ## Related diff --git a/docs/en/tools.md b/docs/en/tools.md index 783a50e3..e16c18a5 100644 --- a/docs/en/tools.md +++ b/docs/en/tools.md @@ -77,7 +77,7 @@ When `rrc_enabled` is on, you can run a local RRC hub from relay chat server set ## Plugins -Installed plugins can add rows to **Tools** and **Navigation** through contribution manifests. Example bundled plugin: **Mesh Observatory** (`com.meshchatx.mesh-observatory`) for live announce feeds and path tables. +Installed plugins can add rows to **Tools** and **Navigation** through contribution manifests. Example bundled plugin: **Bug Reports** (`com.meshchatx.mcx-bugs`) for sending redacted debug logs to an `mcx-bugs-v1` collector (or running a collector yourself). Plugins are capability-gated, not fully open-ended: they cannot rewrite core MeshChatX. Supported packaged runtimes are **frontend JS** (Worker), optional **backend WASM** (wasmtime), and optional **backend Python** (`backend.type: "python"`). Install sources include ZIP archives and single-file **WASM bundles** with embedded `plugin.json` / files / optional RSG signature. diff --git a/meshchatx/src/backend/auto_propagation_manager.py b/meshchatx/src/backend/auto_propagation_manager.py index f1c02c36..37055bf5 100644 --- a/meshchatx/src/backend/auto_propagation_manager.py +++ b/meshchatx/src/backend/auto_propagation_manager.py @@ -234,18 +234,7 @@ class AutoPropagationManager: ) return - # None of the candidates worked. If the previously-selected node is - # still unreachable, clear it rather than restoring a broken node. - if previous_hex: - try: - previous_dest = bytes.fromhex(previous_hex) - if RNS.Transport.has_path(previous_dest): - self.app.set_active_propagation_node( - previous_hex, context=self.context - ) - return - except Exception: - pass - self.app.remove_active_propagation_node(context=self.context) - else: - self.app.remove_active_propagation_node(context=self.context) + # None of the candidates worked (including the previous node if it was + # probed). Clear the active node rather than restoring a sync-broken one + # just because a transport path still exists. + self.app.remove_active_propagation_node(context=self.context) diff --git a/meshchatx/src/backend/config_manager.py b/meshchatx/src/backend/config_manager.py index 97e3d0f7..0043e37c 100644 --- a/meshchatx/src/backend/config_manager.py +++ b/meshchatx/src/backend/config_manager.py @@ -304,6 +304,56 @@ class ConfigManager: "map_nominatim_api_url", "https://nominatim.openstreetmap.org", ) + self.map_overlay_max_bytes = self.IntConfig( + self, + "map_overlay_max_bytes", + 8 * 1024 * 1024, + ) + self.map_overlay_max_features = self.IntConfig( + self, + "map_overlay_max_features", + 50_000, + ) + self.map_overlay_max_kmz_uncompressed_bytes = self.IntConfig( + self, + "map_overlay_max_kmz_uncompressed_bytes", + 16 * 1024 * 1024, + ) + self.map_overlay_max_sources = self.IntConfig( + self, + "map_overlay_max_sources", + 64, + ) + self.map_overlay_max_concurrent_jobs = self.IntConfig( + self, + "map_overlay_max_concurrent_jobs", + 2, + ) + self.map_overlay_path_timeout_seconds = self.IntConfig( + self, + "map_overlay_path_timeout_seconds", + 30, + ) + self.map_overlay_transfer_timeout_seconds = self.IntConfig( + self, + "map_overlay_transfer_timeout_seconds", + 120, + ) + self.map_overlay_job_timeout_seconds = self.IntConfig( + self, + "map_overlay_job_timeout_seconds", + 300, + ) + self.map_overlay_max_retries = self.IntConfig( + self, + "map_overlay_max_retries", + 3, + ) + self.map_overlay_retry_delay_seconds = self.IntConfig( + self, + "map_overlay_retry_delay_seconds", + 2, + ) # telemetry config self.telemetry_enabled = self.BoolConfig(self, "telemetry_enabled", False) diff --git a/meshchatx/src/backend/data/plugins/mesh-observatory/frontend/main.js b/meshchatx/src/backend/data/plugins/mesh-observatory/frontend/main.js deleted file mode 100644 index 0e8d6799..00000000 --- a/meshchatx/src/backend/data/plugins/mesh-observatory/frontend/main.js +++ /dev/null @@ -1,231 +0,0 @@ -const MAX_ANNOUNCES = 80; - -/** - * @param {{ t: (key: string) => string }} api - * @param {string} key - * @param {Record} [params] - */ -function formatLabel(api, key, params = {}) { - let text = api.t(key); - for (const [name, value] of Object.entries(params)) { - text = text.replace(`{${name}}`, String(value)); - } - return text; -} - -function shortHash(hash) { - if (!hash || hash.length < 12) { - return hash || "—"; - } - return `${hash.slice(0, 10)}…${hash.slice(-6)}`; -} - -function shortInterface(name) { - if (!name || typeof name !== "string") { - return "—"; - } - const match = name.match(/\[([^\]]+)\]/); - if (match) { - return match[1]; - } - if (name.length > 36) { - return `${name.slice(0, 18)}…${name.slice(-10)}`; - } - return name; -} - -function hopLabel(api, hops) { - if (hops == null) { - return formatLabel(api, "hops_unknown"); - } - if (hops === 1) { - return formatLabel(api, "hops_one"); - } - return formatLabel(api, "hops_many", { count: hops }); -} - -function stateNode(api, state) { - let label = formatLabel(api, "state_unknown"); - let variant = "muted"; - if (state === 1) { - label = formatLabel(api, "state_responsive"); - variant = "success"; - } else if (state === 2) { - label = formatLabel(api, "state_unresponsive"); - variant = "danger"; - } - return { type: "badge", label, variant }; -} - -/** - * @param {{ t: (key: string) => string, invoke: Function, setUi: Function, onAction: Function, onEvent: Function, onRefresh: Function, getInputValue: Function }} api - */ -export async function activate(api) { - /** @type {Array>} */ - let announces = []; - /** @type {{ paths: Array>, total: number, responsive: number, unresponsive: number }} */ - let pathData = { paths: [], total: 0, responsive: 0, unresponsive: 0 }; - - async function refreshPaths() { - const search = (api.getInputValue("path-search") || "").trim(); - pathData = await api.invoke("readPaths", { - search: search || undefined, - limit: 150, - }); - } - - function render() { - const announceFilter = (api.getInputValue("announce-filter") || "").trim().toLowerCase(); - const filteredAnnounces = announces.filter((entry) => { - if (!announceFilter) { - return true; - } - const haystack = - `${entry.aspect || ""} ${entry.destination_hash || ""} ${entry.app_data || ""}`.toLowerCase(); - return haystack.includes(announceFilter); - }); - - api.setUi({ - type: "column", - children: [ - { - type: "text", - variant: "title", - value: formatLabel(api, "title"), - }, - { - type: "text", - variant: "body", - value: formatLabel(api, "description"), - }, - { - type: "actions", - items: [ - { - type: "button", - id: "refresh", - label: formatLabel(api, "refresh"), - }, - ], - }, - { - type: "section", - title: formatLabel(api, "announces_section"), - description: formatLabel(api, "announce_stats", { - shown: Math.min(filteredAnnounces.length, 40), - total: announces.length, - }), - children: [ - { - type: "input", - id: "announce-filter", - label: formatLabel(api, "filter"), - placeholder: formatLabel(api, "filter_placeholder"), - }, - { - type: "actions", - items: [ - { - type: "button", - id: "clear-announces", - variant: "secondary", - label: formatLabel(api, "clear_feed"), - }, - ], - }, - { - type: "list", - variant: "cards", - emptyText: formatLabel(api, "no_announces"), - items: filteredAnnounces.slice(0, 40).map((entry) => ({ - type: "row", - variant: "announce-card", - children: [ - { type: "text", variant: "mono", value: entry.receivedAt || "—" }, - { type: "text", variant: "stat", value: entry.aspect || "—" }, - { type: "text", variant: "mono", value: shortHash(entry.destination_hash) }, - { - type: "text", - variant: "caption", - value: (entry.app_data || "").slice(0, 72) || "—", - }, - ], - })), - }, - ], - }, - { - type: "section", - title: formatLabel(api, "paths_section"), - description: formatLabel(api, "path_stats", { - total: pathData.total || 0, - responsive: pathData.responsive || 0, - unresponsive: pathData.unresponsive || 0, - }), - children: [ - { - type: "input", - id: "path-search", - label: formatLabel(api, "path_search"), - placeholder: formatLabel(api, "path_search_placeholder"), - }, - { - type: "list", - variant: "cards", - emptyText: formatLabel(api, "no_paths"), - items: (pathData.paths || []).map((entry) => ({ - type: "row", - variant: "card", - children: [ - { - type: "text", - variant: "mono", - value: shortHash(entry.destination_hash), - }, - { type: "text", variant: "stat", value: hopLabel(api, entry.hops) }, - { - type: "text", - variant: "caption", - value: shortInterface(entry.interface), - }, - stateNode(api, entry.state), - ], - })), - }, - ], - }, - ], - }); - } - - async function refresh() { - await refreshPaths(); - render(); - } - - api.onAction(async (actionId) => { - if (actionId === "refresh") { - await refresh(); - } else if (actionId === "clear-announces") { - announces = []; - render(); - } - }); - - api.onEvent("announce.received", async (payload) => { - announces.unshift({ - aspect: payload?.aspect || "", - destination_hash: payload?.destination_hash || "", - app_data: payload?.app_data || "", - receivedAt: new Date().toLocaleTimeString(), - }); - if (announces.length > MAX_ANNOUNCES) { - announces = announces.slice(0, MAX_ANNOUNCES); - } - await refreshPaths(); - render(); - }); - - api.onRefresh(refresh); - await refresh(); -} diff --git a/meshchatx/src/backend/data/plugins/mesh-observatory/locales/en.json b/meshchatx/src/backend/data/plugins/mesh-observatory/locales/en.json deleted file mode 100644 index ab5a8a09..00000000 --- a/meshchatx/src/backend/data/plugins/mesh-observatory/locales/en.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "nav": "Mesh Observatory", - "title": "Mesh Observatory", - "description": "Watch live announces and browse your Reticulum path table in one place.", - "announces_section": "Live announces", - "announce_stats": "Showing {shown} of {total} captured announces", - "filter": "Filter announces", - "filter_placeholder": "Aspect, hash, or app data", - "refresh": "Refresh paths", - "clear_feed": "Clear announce feed", - "no_announces": "No announces captured yet. Activity will appear here as the mesh announces.", - "paths_section": "Path table", - "path_stats": "{total} routes — {responsive} responsive, {unresponsive} unresponsive", - "path_search": "Search paths", - "path_search_placeholder": "Destination or via hash", - "no_paths": "No paths match your search.", - "hops_unknown": "Unknown hops", - "hops_one": "1 hop", - "hops_many": "{count} hops", - "state_responsive": "Responsive", - "state_unresponsive": "Unresponsive", - "state_unknown": "Unknown" -} diff --git a/meshchatx/src/backend/data/plugins/mesh-observatory/plugin.json b/meshchatx/src/backend/data/plugins/mesh-observatory/plugin.json deleted file mode 100644 index ddb63ca6..00000000 --- a/meshchatx/src/backend/data/plugins/mesh-observatory/plugin.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "id": "com.meshchatx.mesh-observatory", - "version": "1.0.0", - "apiVersion": 1, - "name": "Mesh Observatory", - "description": "Live announce feed and searchable path table for your mesh.", - "frontend": { - "entry": "frontend/main.js", - "type": "js" - }, - "i18n": { - "directory": "locales", - "defaultLocale": "en" - }, - "contributes": { - "navItems": [ - { - "id": "mesh-observatory", - "route": { "name": "plugin-mesh-observatory" }, - "icon": "chart-line", - "labelKey": "nav" - } - ], - "toolsPageEntries": [ - { - "name": "mesh-observatory", - "route": { "name": "plugin-mesh-observatory" }, - "icon": "chart-line", - "iconBg": "tool-card__icon bg-violet-50 text-violet-600 dark:bg-violet-900/30 dark:text-violet-200", - "titleKey": "title", - "descriptionKey": "description" - } - ] - }, - "permissions": { - "hooks": ["announce.received"], - "managers": ["destinationPath.read"], - "storage": "isolated", - "network": "none" - } -} diff --git a/meshchatx/src/backend/database/__init__.py b/meshchatx/src/backend/database/__init__.py index 858974ec..d86afc1f 100644 --- a/meshchatx/src/backend/database/__init__.py +++ b/meshchatx/src/backend/database/__init__.py @@ -16,6 +16,7 @@ from .crash_history import CrashHistoryDAO from .debug_logs import DebugLogsDAO from .gifs import UserGifsDAO from .map_drawings import MapDrawingsDAO +from .map_overlays import MapOverlaysDAO from .messages import MessageDAO from .misc import MiscDAO from .provider import DatabaseProvider @@ -81,6 +82,7 @@ class Database: self.notification_sounds = NotificationSoundDAO(self.provider) self.contacts = ContactsDAO(self.provider) self.map_drawings = MapDrawingsDAO(self.provider) + self.map_overlays = MapOverlaysDAO(self.provider) self.stickers = UserStickersDAO(self.provider) self.sticker_packs = UserStickerPacksDAO(self.provider) self.gifs = UserGifsDAO(self.provider) @@ -96,8 +98,28 @@ class Database: def execute_sql(self, query, params=None): return self.provider.execute(query, params) + def _ensure_sqlite_temp_dir(self): + """Prefer a storage-local temp dir so Landlock can open spill files.""" + try: + db_dir = self._identity_storage_dir() + except Exception: + return None + if not db_dir: + return None + temp_dir = os.path.join(db_dir, "sqlite-tmp") + try: + os.makedirs(temp_dir, exist_ok=True) + except OSError: + return None + # SQLite uses TMPDIR for PRAGMA temp_store=FILE spill files. + os.environ["TMPDIR"] = temp_dir + os.environ["TMP"] = temp_dir + os.environ["TEMP"] = temp_dir + return temp_dir + def _tune_sqlite_pragmas(self): try: + self.provider.prefer_temp_store_file = False self.execute_sql("PRAGMA journal_mode=WAL") self.execute_sql("PRAGMA synchronous=NORMAL") self.execute_sql("PRAGMA wal_autocheckpoint=1000") @@ -109,15 +131,36 @@ class Database: except Exception as exc: print(f"SQLite pragma setup failed: {exc}") - def apply_memory_pressure_pragmas(self, relax: bool) -> bool: - """Move SQLite temp/cache work toward disk when host RAM is low.""" + def apply_memory_pressure_pragmas( + self, + relax: bool, + *, + landlock_active: bool = False, + ) -> bool: + """Shrink SQLite cache under low RAM. + + FILE temp spills break complex conversation queries under Landlock + (``unable to open database file``), even when TMPDIR is inside the + allowed storage tree. Keep MEMORY temp while Landlock is active and + only reduce cache/mmap. Without Landlock, FILE temp is still used. + """ try: if relax: - self.execute_sql("PRAGMA temp_store=FILE") + self._ensure_sqlite_temp_dir() + use_file_temp = not landlock_active + self.provider.prefer_temp_store_file = use_file_temp + if use_file_temp: + self.execute_sql("PRAGMA temp_store=FILE") + else: + self.execute_sql("PRAGMA temp_store=MEMORY") + _log.info( + "Memory pressure under Landlock: keeping temp_store=MEMORY", + ) self.execute_sql("PRAGMA cache_size=-2000") # 2 MB self.execute_sql("PRAGMA mmap_size=0") self._sqlite_memory_relaxed = True else: + self.provider.prefer_temp_store_file = False self.execute_sql("PRAGMA temp_store=MEMORY") self.execute_sql("PRAGMA cache_size=-8000") self.execute_sql("PRAGMA mmap_size=67108864") @@ -217,7 +260,7 @@ class Database: def check_db_health_at_open(self, storage_path): """Run integrity and baseline checks after opening the database. - Returns human-readable issue strings; empty if healthy. + Returns human-readable issue strings. Empty if healthy. """ issues = [] try: @@ -266,7 +309,7 @@ class Database: def check_db_health_at_close(self, storage_path): """Run health checks before closing the database (for logging only). - Returns issue strings; empty if healthy. + Returns issue strings. Empty if healthy. """ issues = [] try: diff --git a/meshchatx/src/backend/database/map_overlays.py b/meshchatx/src/backend/database/map_overlays.py new file mode 100644 index 00000000..2037e82b --- /dev/null +++ b/meshchatx/src/backend/database/map_overlays.py @@ -0,0 +1,138 @@ +# SPDX-License-Identifier: 0BSD + +from datetime import UTC, datetime + +from .provider import DatabaseProvider + + +class MapOverlaysDAO: + def __init__(self, provider: DatabaseProvider): + self.provider = provider + + def count_for_identity(self, identity_hash: str) -> int: + row = self.provider.fetchone( + "SELECT COUNT(*) AS c FROM map_overlay_sources WHERE identity_hash = ?", + (identity_hash,), + ) + return int(row["c"]) if row else 0 + + def get_by_id(self, overlay_id: int): + return self.provider.fetchone( + "SELECT * FROM map_overlay_sources WHERE id = ?", + (overlay_id,), + ) + + def get_by_unique( + self, + identity_hash: str, + kind: str, + destination_hash: str, + path_or_repo_path: str, + ref: str, + ): + return self.provider.fetchone( + """ + SELECT * FROM map_overlay_sources + WHERE identity_hash = ? + AND kind = ? + AND destination_hash = ? + AND path_or_repo_path = ? + AND ref = ? + """, + (identity_hash, kind, destination_hash, path_or_repo_path, ref), + ) + + def list_for_identity(self, identity_hash: str): + return self.provider.fetchall( + """ + SELECT * FROM map_overlay_sources + WHERE identity_hash = ? + ORDER BY updated_at DESC, id DESC + """, + (identity_hash,), + ) + + def list_due_autorefresh(self, now_iso: str): + return self.provider.fetchall( + """ + SELECT * FROM map_overlay_sources + WHERE enabled = 1 + AND refresh_interval_seconds > 0 + AND status != 'fetching' + AND ( + next_refresh_at IS NULL + OR next_refresh_at <= ? + ) + ORDER BY (next_refresh_at IS NOT NULL), next_refresh_at ASC, id ASC + """, + (now_iso,), + ) + + def insert( + self, + identity_hash: str, + *, + kind: str, + destination_hash: str, + path_or_repo_path: str, + ref: str, + name: str, + group_name: str | None = None, + repository: str | None = None, + enabled: int = 1, + visible: int = 1, + refresh_interval_seconds: int = 0, + status: str = "pending", + ) -> int: + now = datetime.now(UTC) + cur = self.provider.execute( + """ + INSERT INTO map_overlay_sources ( + identity_hash, kind, destination_hash, path_or_repo_path, ref, + group_name, repository, name, enabled, visible, + refresh_interval_seconds, status, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + identity_hash, + kind, + destination_hash, + path_or_repo_path, + ref, + group_name, + repository, + name, + enabled, + visible, + refresh_interval_seconds, + status, + now, + now, + ), + ) + return int(cur.lastrowid) + + def update_fields(self, overlay_id: int, **fields) -> None: + if not fields: + return + fields = dict(fields) + fields["updated_at"] = datetime.now(UTC) + cols = ", ".join(f"{k} = ?" for k in fields) + values = list(fields.values()) + [overlay_id] + self.provider.execute( + f"UPDATE map_overlay_sources SET {cols} WHERE id = ?", + tuple(values), + ) + + def delete(self, overlay_id: int) -> None: + self.provider.execute( + "DELETE FROM map_overlay_sources WHERE id = ?", + (overlay_id,), + ) + + def delete_for_identity(self, identity_hash: str, overlay_id: int) -> bool: + row = self.get_by_id(overlay_id) + if not row or row["identity_hash"] != identity_hash: + return False + self.delete(overlay_id) + return True diff --git a/meshchatx/src/backend/database/messages.py b/meshchatx/src/backend/database/messages.py index 5f456cda..e3990f14 100644 --- a/meshchatx/src/backend/database/messages.py +++ b/meshchatx/src/backend/database/messages.py @@ -392,15 +392,50 @@ class MessageDAO: rows = self.provider.fetchall( "SELECT id, hash, peer_hash, source_hash, destination_hash, " - "is_incoming, title, content, fields, timestamp " + "is_incoming, title, " + "substr(COALESCE(content, ''), 1, 240) as content, " + "CASE WHEN length(COALESCE(fields, '')) > 16384 THEN NULL ELSE fields END as fields, " + "CASE WHEN fields IS NOT NULL AND fields != '' AND fields != '{}' " + "AND (instr(fields, '\"image\"') > 0 OR instr(fields, '\"0x05\"') > 0 " + "OR instr(fields, '\"audio\"') > 0 OR instr(fields, '\"0x06\"') > 0 " + "OR instr(fields, '\"file_attachments\"') > 0 OR instr(fields, '\"0x07\"') > 0) " + "THEN 1 ELSE 0 END as has_attachments, " + "CASE WHEN fields IS NOT NULL AND fields != '' AND fields != '{}' " + "AND (instr(fields, '\"reaction\"') > 0 OR instr(fields, '\"0x40\"') > 0) " + "THEN 1 ELSE 0 END as has_reaction, " + "CASE WHEN fields IS NOT NULL AND fields != '' AND fields != '{}' " + "AND (instr(fields, '\"image\"') > 0 OR instr(fields, '\"0x05\"') > 0) " + "THEN 1 ELSE 0 END as has_image, " + "CASE WHEN fields IS NOT NULL AND fields != '' AND fields != '{}' " + "AND (instr(fields, '\"audio\"') > 0 OR instr(fields, '\"0x06\"') > 0) " + "THEN 1 ELSE 0 END as has_audio, " + "CASE WHEN fields IS NOT NULL AND fields != '' AND fields != '{}' " + "AND (instr(fields, '\"file_attachments\"') > 0 OR instr(fields, '\"0x07\"') > 0) " + "THEN 1 ELSE 0 END as has_files, " + "timestamp " "FROM lxmf_messages WHERE peer_hash = ? AND is_incoming = 1 " "ORDER BY timestamp DESC LIMIT ?", (peer_hash, scan_limit), ) for row in rows: row_dict = dict(row) if not isinstance(row, dict) else row + fields = row_dict.get("fields") + if fields is None and ( + row_dict.get("has_attachments") + or row_dict.get("has_image") + or row_dict.get("has_audio") + or row_dict.get("has_files") + ): + # Huge attachment blob omitted from SELECT: still user-facing. + return row_dict + if row_dict.get("has_reaction") and not ( + (row_dict.get("content") and str(row_dict.get("content")).strip()) + or (row_dict.get("title") and str(row_dict.get("title")).strip()) + or row_dict.get("has_attachments") + ): + continue if is_user_facing_lxmf_payload( - row_dict.get("fields"), + fields, row_dict.get("content"), row_dict.get("title"), ): diff --git a/meshchatx/src/backend/database/provider.py b/meshchatx/src/backend/database/provider.py index 857bb406..c592c982 100644 --- a/meshchatx/src/backend/database/provider.py +++ b/meshchatx/src/backend/database/provider.py @@ -22,9 +22,13 @@ class DatabaseProvider: self._local = threading.local() self._all_locals.add(self._local) self._memory_connection = None + # Per-connection default. Worker threads opened via asyncio.to_thread + # never see Database._tune_sqlite_pragmas(), so this must be set here. + # FILE temp under Landlock often fails with "unable to open database file" + # when SQLite spills sort/hash work for large conversation queries. + self.prefer_temp_store_file = False - @staticmethod - def _configure_connection(connection): + def _configure_connection(self, connection): if connection is None: return try: @@ -35,6 +39,21 @@ class DatabaseProvider: connection.execute("PRAGMA journal_mode=WAL") except sqlite3.OperationalError: pass + try: + if self.prefer_temp_store_file: + connection.execute("PRAGMA temp_store=FILE") + connection.execute("PRAGMA cache_size=-2000") + connection.execute("PRAGMA mmap_size=0") + else: + connection.execute("PRAGMA temp_store=MEMORY") + connection.execute("PRAGMA cache_size=-8000") + connection.execute("PRAGMA mmap_size=67108864") + except sqlite3.OperationalError: + pass + try: + connection.execute("PRAGMA synchronous=NORMAL") + except sqlite3.OperationalError: + pass @classmethod def get_instance(cls, db_path=None): diff --git a/meshchatx/src/backend/database/schema.py b/meshchatx/src/backend/database/schema.py index 0ea4c91f..b0ecaa3c 100644 --- a/meshchatx/src/backend/database/schema.py +++ b/meshchatx/src/backend/database/schema.py @@ -19,7 +19,7 @@ def _validate_identifier(name: str, label: str = "identifier") -> str: class DatabaseSchema: - LATEST_VERSION = 49 + LATEST_VERSION = 50 def __init__(self, provider: DatabaseProvider): self.provider = provider @@ -441,6 +441,36 @@ class DatabaseSchema: updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ) """, + "map_overlay_sources": """ + CREATE TABLE IF NOT EXISTS map_overlay_sources ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + identity_hash TEXT NOT NULL, + kind TEXT NOT NULL, + destination_hash TEXT NOT NULL, + path_or_repo_path TEXT NOT NULL, + ref TEXT NOT NULL DEFAULT 'HEAD', + group_name TEXT, + repository TEXT, + name TEXT NOT NULL, + enabled INTEGER NOT NULL DEFAULT 1, + visible INTEGER NOT NULL DEFAULT 1, + refresh_interval_seconds INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'pending', + last_error TEXT, + last_fetched_at DATETIME, + next_refresh_at DATETIME, + content_sha256 TEXT, + resolved_ref TEXT, + format TEXT, + byte_size INTEGER, + cache_relpath TEXT, + job_id TEXT, + generation INTEGER NOT NULL DEFAULT 0, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, + UNIQUE(identity_hash, kind, destination_hash, path_or_repo_path, ref) + ) + """, "user_stickers": """ CREATE TABLE IF NOT EXISTS user_stickers ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -1318,3 +1348,43 @@ class DatabaseSchema: updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ) """) + + if current_version < 50: + self._safe_execute(""" + CREATE TABLE IF NOT EXISTS map_overlay_sources ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + identity_hash TEXT NOT NULL, + kind TEXT NOT NULL, + destination_hash TEXT NOT NULL, + path_or_repo_path TEXT NOT NULL, + ref TEXT NOT NULL DEFAULT 'HEAD', + group_name TEXT, + repository TEXT, + name TEXT NOT NULL, + enabled INTEGER NOT NULL DEFAULT 1, + visible INTEGER NOT NULL DEFAULT 1, + refresh_interval_seconds INTEGER NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'pending', + last_error TEXT, + last_fetched_at DATETIME, + next_refresh_at DATETIME, + content_sha256 TEXT, + resolved_ref TEXT, + format TEXT, + byte_size INTEGER, + cache_relpath TEXT, + job_id TEXT, + generation INTEGER NOT NULL DEFAULT 0, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, + UNIQUE(identity_hash, kind, destination_hash, path_or_repo_path, ref) + ) + """) + self._safe_execute( + "CREATE INDEX IF NOT EXISTS idx_map_overlay_sources_identity " + "ON map_overlay_sources(identity_hash)", + ) + self._safe_execute( + "CREATE INDEX IF NOT EXISTS idx_map_overlay_sources_refresh " + "ON map_overlay_sources(enabled, next_refresh_at)", + ) diff --git a/meshchatx/src/backend/docs_manager.py b/meshchatx/src/backend/docs_manager.py index 65436816..a2090a85 100644 --- a/meshchatx/src/backend/docs_manager.py +++ b/meshchatx/src/backend/docs_manager.py @@ -219,9 +219,18 @@ class DocsManager: logging.exception(f"Failed to populate MeshChatX docs: {e}") def _sync_docs_tree(self, src_docs, dest_dir): - """Copy manifest, markdown, and text files from src_docs into dest_dir.""" - for root, _, files in os.walk(src_docs): + """Copy manifest, markdown, and text files from src_docs into dest_dir. + + Skips ``agents/`` (contributor and automated-agent guidance, not + end-user documentation). + """ + for root, dirnames, files in os.walk(src_docs): rel_root = os.path.relpath(root, src_docs) + if rel_root == ".": + dirnames[:] = [d for d in dirnames if d != "agents"] + elif rel_root == "agents" or rel_root.startswith(f"agents{os.sep}"): + dirnames[:] = [] + continue target_root = ( dest_dir if rel_root == "." else os.path.join(dest_dir, rel_root) ) diff --git a/meshchatx/src/backend/identity_context.py b/meshchatx/src/backend/identity_context.py index 7ba13b66..66f3464e 100644 --- a/meshchatx/src/backend/identity_context.py +++ b/meshchatx/src/backend/identity_context.py @@ -24,6 +24,7 @@ from meshchatx.src.backend.integrity_manager import ( select_critical_integrity_issues, ) from meshchatx.src.backend.map_manager import MapManager +from meshchatx.src.backend.map_overlay_manager import MapOverlayManager from meshchatx.src.backend.meshchat_utils import create_lxmf_router from meshchatx.src.backend.message_handler import MessageHandler from meshchatx.src.backend.nomadnet_utils import NomadNetworkManager @@ -80,6 +81,7 @@ class IdentityContext: self.announce_manager = None self.archiver_manager = None self.map_manager = None + self.map_overlay_manager = None self.docs_manager = None self.repository_server_manager = None self.nomadnet_manager = None @@ -192,6 +194,18 @@ class IdentityContext: self.announce_manager = AnnounceManager(self.database, self.config) self.archiver_manager = ArchiverManager(self.database) self.map_manager = MapManager(self.config, self.app.storage_dir) + self.map_overlay_manager = MapOverlayManager( + self.config, + self.database, + self.storage_path, + reticulum_config_dir=getattr(self.app, "reticulum_config_dir", None), + identity=self.identity, + reticulum=getattr(self.app, "reticulum", None), + ) + try: + self.map_overlay_manager.start_scheduler() + except Exception: + pass self.docs_manager = DocsManager( self.config, self.app.get_public_path(), @@ -767,6 +781,12 @@ class IdentityContext: if self.archiver_manager: self.archiver_manager = None + if self.map_overlay_manager: + try: + self.map_overlay_manager.cleanup() + except Exception: + pass + self.map_overlay_manager = None if self.map_manager: self.map_manager = None diff --git a/meshchatx/src/backend/identity_manager.py b/meshchatx/src/backend/identity_manager.py index 4d406fdf..761fb642 100644 --- a/meshchatx/src/backend/identity_manager.py +++ b/meshchatx/src/backend/identity_manager.py @@ -185,20 +185,40 @@ class IdentityManager: new_provider.close_all() - # Save metadata - metadata = { - "display_name": display_name, - "icon_name": None, - "icon_foreground_colour": None, - "icon_background_colour": None, - } + # Preserve icon/address metadata when re-importing an existing identity. metadata_path = os.path.join(identity_dir, "metadata.json") + existing_metadata = {} + if os.path.exists(metadata_path): + with contextlib.suppress(Exception), open(metadata_path) as f: + loaded = json.load(f) + if isinstance(loaded, dict): + existing_metadata = loaded + + resolved_name = ( + (display_name or "").strip() + or existing_metadata.get("display_name") + or "Anonymous Peer" + ) + metadata = { + "display_name": resolved_name, + "icon_name": existing_metadata.get("icon_name"), + "icon_foreground_colour": existing_metadata.get( + "icon_foreground_colour", + ), + "icon_background_colour": existing_metadata.get( + "icon_background_colour", + ), + } + for key in ("lxmf_address", "lxst_address"): + if key in existing_metadata: + metadata[key] = existing_metadata[key] + with open(metadata_path, "w") as f: json.dump(metadata, f) return { "hash": identity_hash, - "display_name": display_name, + "display_name": resolved_name, } def update_metadata_cache(self, identity_hash: str, metadata: dict): @@ -229,11 +249,17 @@ class IdentityManager: return True return False + _MAX_IDENTITY_BYTES = 65536 + def restore_identity_from_bytes( self, identity_bytes: bytes, display_name: str | None = None, ) -> dict: + if not identity_bytes: + raise ValueError("Identity file is empty") + if len(identity_bytes) > self._MAX_IDENTITY_BYTES: + raise ValueError("Identity file is too large") try: # We use RNS.Identity.from_bytes to validate and get the hash identity = RNS.Identity.from_bytes(identity_bytes) @@ -242,6 +268,8 @@ class IdentityManager: name = (display_name or "").strip() or "Restored Identity" return self._save_new_identity(identity, name) + except ValueError: + raise except Exception as exc: raise ValueError(f"Failed to restore identity: {exc}") from exc @@ -250,11 +278,16 @@ class IdentityManager: base32_value: str, display_name: str | None = None, ) -> dict: + if base32_value is None: + raise ValueError("base32 value is required") + normalized = "".join(str(base32_value).split()) + if not normalized: + raise ValueError("base32 value is required") try: - identity_bytes = base64.b32decode(base32_value, casefold=True) - return self.restore_identity_from_bytes( - identity_bytes, display_name=display_name - ) + identity_bytes = base64.b32decode(normalized, casefold=True) except Exception as exc: msg = f"Invalid base32 identity: {exc}" raise ValueError(msg) from exc + return self.restore_identity_from_bytes( + identity_bytes, display_name=display_name + ) diff --git a/meshchatx/src/backend/lxmf_utils.py b/meshchatx/src/backend/lxmf_utils.py index 598e7f51..18260b30 100644 --- a/meshchatx/src/backend/lxmf_utils.py +++ b/meshchatx/src/backend/lxmf_utils.py @@ -7,7 +7,7 @@ import LXMF from meshchatx.src.backend.telemetry_utils import Telemeter -# MeshChatX app extensions (field 16); not used for LXMF-standard reactions. +# MeshChatX app extensions (field 16). not used for LXMF-standard reactions. LXMF_APP_EXTENSIONS_FIELD = 16 # LXMF reply / reaction field standards (see LXMF.py FIELD_REPLY_* / FIELD_REACTION) @@ -256,27 +256,34 @@ def _lxmf_sidebar_actor_label( ) +def _row_flag_true(row: dict, *keys) -> bool: + for key in keys: + value = row.get(key) + if value in (1, True, "1"): + return True + return False + + def lxmf_sidebar_preview_for_conversation_latest_row( row: dict, *, local_hash: str, peer_display_name: str, ) -> str: - """Single-line preview for conversation list APIs (reactions and some media have empty body).""" + """Single-line preview for conversation list APIs (reactions and some media have empty body). + + Conversation list rows may omit full ``fields`` (to avoid loading multi-MB + attachment blobs). In that case SQL-derived flags such as ``has_image`` / + ``has_reaction`` are used instead. + """ content = row.get("content") if content is not None and str(content).strip(): - return str(content) - - fields_raw = row.get("fields") - try: - if isinstance(fields_raw, str): - fields = json.loads(fields_raw) if fields_raw else {} - elif isinstance(fields_raw, dict): - fields = fields_raw - else: - fields = {} - except (json.JSONDecodeError, TypeError): - fields = {} + # List queries may already truncate content. keep previews bounded. + text = str(content) + stripped = text.strip() + if len(stripped) <= 240: + return text if len(text) <= 240 else stripped[:237] + "..." + return stripped[:237] + "..." actor = _lxmf_sidebar_actor_label( row, @@ -285,55 +292,88 @@ def lxmf_sidebar_preview_for_conversation_latest_row( ) incoming = bool(row.get("is_incoming")) - emoji = _reaction_emoji_from_parsed_lxmf_fields(fields) - if emoji: - return f"{actor} reacted {emoji}" + fields_raw = row.get("fields") + fields = {} + if fields_raw is not None: + try: + if isinstance(fields_raw, str): + # Never json.loads multi-MB attachment blobs for a sidebar line. + if len(fields_raw) > 16384: + fields = {} + else: + fields = json.loads(fields_raw) if fields_raw else {} + elif isinstance(fields_raw, dict): + fields = fields_raw + except (json.JSONDecodeError, TypeError): + fields = {} - telemetry = fields.get("telemetry") - if isinstance(telemetry, dict): - loc = telemetry.get("location") - if isinstance(loc, dict) and loc: + if fields: + emoji = _reaction_emoji_from_parsed_lxmf_fields(fields) + if emoji: + return f"{actor} reacted {emoji}" + + telemetry = fields.get("telemetry") + if isinstance(telemetry, dict): + loc = telemetry.get("location") + if isinstance(loc, dict) and loc: + if actor == "You": + return "You shared your location" + return f"{actor} shared their location" + + ts = fields.get("telemetry_stream") + if isinstance(ts, list) and len(ts) > 0: + return f"{actor} sent a telemetry stream" + + if isinstance(telemetry, dict) and len(telemetry) > 0: + return f"{actor} sent telemetry" + + commands = fields.get("commands") + if isinstance(commands, list): + for cmd in commands: + if isinstance(cmd, dict) and "0x01" in cmd: + if incoming: + return f"{actor} requested your location" + return f"{actor} sent a location request" + + image = fields.get("image") + if isinstance(image, dict) and image: if actor == "You": - return "You shared your location" - return f"{actor} shared their location" + return "You sent an image" + return f"{actor} sent an image" - ts = fields.get("telemetry_stream") - if isinstance(ts, list) and len(ts) > 0: - return f"{actor} sent a telemetry stream" + audio = fields.get("audio") + if isinstance(audio, dict) and audio: + if actor == "You": + return "You sent a voice note" + return f"{actor} sent a voice note" - if isinstance(telemetry, dict) and len(telemetry) > 0: - return f"{actor} sent telemetry" + file_attachments = fields.get("file_attachments") + if isinstance(file_attachments, list) and len(file_attachments) > 0: + n = len(file_attachments) + if n == 1: + if actor == "You": + return "You sent a file" + return f"{actor} sent a file" + if actor == "You": + return f"You sent {n} files" + return f"{actor} sent {n} files" - commands = fields.get("commands") - if isinstance(commands, list): - for cmd in commands: - if isinstance(cmd, dict) and "0x01" in cmd: - if incoming: - return f"{actor} requested your location" - return f"{actor} sent a location request" - - image = fields.get("image") - if isinstance(image, dict) and image: + if _row_flag_true(row, "has_reaction"): + return f"{actor} reacted" + if _row_flag_true(row, "has_image"): if actor == "You": return "You sent an image" return f"{actor} sent an image" - - audio = fields.get("audio") - if isinstance(audio, dict) and audio: + if _row_flag_true(row, "has_audio"): if actor == "You": return "You sent a voice note" return f"{actor} sent a voice note" - - file_attachments = fields.get("file_attachments") - if isinstance(file_attachments, list) and len(file_attachments) > 0: - n = len(file_attachments) - if n == 1: - if actor == "You": - return "You sent a file" - return f"{actor} sent a file" + if _row_flag_true(row, "has_files"): if actor == "You": - return f"You sent {n} files" - return f"{actor} sent {n} files" + return "You sent a file" + return f"{actor} sent a file" + if _row_flag_true(row, "has_telemetry"): + return f"{actor} sent telemetry" return str(content or "") @@ -811,7 +851,7 @@ def compute_lxmf_conversation_unread_from_latest_row(row, *, require_user_facing """Return whether the conversation row should appear as unread. Uses ``lxmf_conversation_read_state.last_read_at`` only. The latest message - must be incoming; outbound-only threads are not unread (matches + must be incoming. outbound-only threads are not unread (matches ``filter_unread`` in ``MessageHandler.get_conversations``). When ``require_user_facing`` is True, the row's latest message must also be @@ -823,16 +863,32 @@ def compute_lxmf_conversation_unread_from_latest_row(row, *, require_user_facing if not row.get("is_incoming"): return False - if require_user_facing and not is_user_facing_lxmf_payload( - row.get("fields"), - row.get("content"), - row.get("title"), - ): - return False + if require_user_facing: + if row.get("fields") is not None: + if not is_user_facing_lxmf_payload( + row.get("fields"), + row.get("content"), + row.get("title"), + ): + return False + elif _row_flag_true(row, "has_reaction") and not ( + (row.get("content") and str(row.get("content")).strip()) + or (row.get("title") and str(row.get("title")).strip()) + or _row_flag_true( + row, "has_image", "has_audio", "has_files", "has_attachments" + ) + ): + return False last_read_at_raw = row.get("last_read_at") if not last_read_at_raw: return True - last_read_at = datetime.fromisoformat(last_read_at_raw) + try: + last_read_at = datetime.fromisoformat(str(last_read_at_raw)) + except (TypeError, ValueError): + return True if last_read_at.tzinfo is None: last_read_at = last_read_at.replace(tzinfo=UTC) - return row["timestamp"] > last_read_at.timestamp() + try: + return float(row["timestamp"]) > last_read_at.timestamp() + except (TypeError, ValueError, KeyError): + return False diff --git a/meshchatx/src/backend/map_geo_validator.py b/meshchatx/src/backend/map_geo_validator.py new file mode 100644 index 00000000..369b2ec6 --- /dev/null +++ b/meshchatx/src/backend/map_geo_validator.py @@ -0,0 +1,259 @@ +# SPDX-License-Identifier: 0BSD + +"""Validate GeoJSON / KML / KMZ bytes for map overlay import.""" + +from __future__ import annotations + +import json +import zipfile +from dataclasses import dataclass +from io import BytesIO +from xml.etree import ElementTree as ET + +ZIP_LOCAL_HEADER = b"PK\x03\x04" +MAX_KMZ_ENTRIES = 512 +MAX_COMPRESSION_RATIO = 100.0 + + +class GeoValidationError(ValueError): + def __init__(self, code: str, message: str | None = None): + self.code = code + super().__init__(message or code) + + +@dataclass +class GeoValidationResult: + format: str + feature_count: int + byte_size: int + + +def _looks_like_html(text: str) -> bool: + head = text.lstrip()[:200].lower() + return head.startswith(" int: + if not isinstance(obj, dict): + raise GeoValidationError("invalid_geojson") + t = obj.get("type") + if t == "FeatureCollection": + feats = obj.get("features") + if not isinstance(feats, list): + raise GeoValidationError("invalid_geojson") + return len(feats) + if t == "Feature": + return 1 + if t in ( + "Point", + "MultiPoint", + "LineString", + "MultiLineString", + "Polygon", + "MultiPolygon", + "GeometryCollection", + ): + return 1 + raise GeoValidationError("invalid_geojson") + + +def _validate_coords_finite(obj, *, depth: int = 0) -> None: + if depth > 32: + raise GeoValidationError("geometry_too_deep") + if isinstance(obj, dict): + if "coordinates" in obj: + _walk_coords(obj["coordinates"], depth=0) + if "geometries" in obj and isinstance(obj["geometries"], list): + for g in obj["geometries"]: + _validate_coords_finite(g, depth=depth + 1) + if "geometry" in obj and obj["geometry"] is not None: + _validate_coords_finite(obj["geometry"], depth=depth + 1) + if "features" in obj and isinstance(obj["features"], list): + for f in obj["features"]: + _validate_coords_finite(f, depth=depth + 1) + + +def _walk_coords(node, *, depth: int) -> None: + if depth > 16: + raise GeoValidationError("geometry_too_deep") + if isinstance(node, (int, float)): + if node != node or node in (float("inf"), float("-inf")): + raise GeoValidationError("invalid_coordinates") + return + if isinstance(node, list): + if node and all(isinstance(x, (int, float)) for x in node): + if len(node) < 2: + raise GeoValidationError("invalid_coordinates") + lon, lat = float(node[0]), float(node[1]) + if lon != lon or lat != lat: + raise GeoValidationError("invalid_coordinates") + if lon < -180.0 or lon > 180.0 or lat < -90.0 or lat > 90.0: + raise GeoValidationError("coordinates_out_of_range") + return + for item in node: + _walk_coords(item, depth=depth + 1) + return + raise GeoValidationError("invalid_coordinates") + + +def validate_geojson_bytes( + data: bytes, + *, + max_bytes: int, + max_features: int, +) -> GeoValidationResult: + if len(data) > max_bytes: + raise GeoValidationError("file_too_large") + try: + text = data.decode("utf-8") + except UnicodeDecodeError as exc: + raise GeoValidationError("invalid_encoding") from exc + if _looks_like_html(text): + raise GeoValidationError("not_geo_content") + try: + obj = json.loads(text) + except json.JSONDecodeError as exc: + raise GeoValidationError("invalid_geojson") from exc + count = _count_geojson_features(obj) + if count > max_features: + raise GeoValidationError("too_many_features") + _validate_coords_finite(obj) + return GeoValidationResult( + format="geojson", feature_count=count, byte_size=len(data) + ) + + +def _strip_ns(tag: str) -> str: + if "}" in tag: + return tag.rsplit("}", 1)[-1] + return tag + + +def validate_kml_bytes( + data: bytes, + *, + max_bytes: int, + max_features: int, +) -> GeoValidationResult: + if len(data) > max_bytes: + raise GeoValidationError("file_too_large") + try: + text = data.decode("utf-8") + except UnicodeDecodeError as exc: + raise GeoValidationError("invalid_encoding") from exc + if _looks_like_html(text): + raise GeoValidationError("not_geo_content") + try: + root = ET.fromstring(text) + except ET.ParseError as exc: + raise GeoValidationError("invalid_kml") from exc + if _strip_ns(root.tag).lower() != "kml": + raise GeoValidationError("invalid_kml") + placemarks = [el for el in root.iter() if _strip_ns(el.tag).lower() == "placemark"] + count = len(placemarks) + if count > max_features: + raise GeoValidationError("too_many_features") + return GeoValidationResult(format="kml", feature_count=count, byte_size=len(data)) + + +def validate_kmz_bytes( + data: bytes, + *, + max_bytes: int, + max_uncompressed_bytes: int, + max_features: int, +) -> GeoValidationResult: + if len(data) > max_bytes: + raise GeoValidationError("file_too_large") + if not data.startswith(ZIP_LOCAL_HEADER): + raise GeoValidationError("invalid_kmz") + try: + zf = zipfile.ZipFile(BytesIO(data)) + except zipfile.BadZipFile as exc: + raise GeoValidationError("invalid_kmz") from exc + with zf: + infos = [i for i in zf.infolist() if not i.is_dir()] + if len(infos) > MAX_KMZ_ENTRIES: + raise GeoValidationError("kmz_too_many_entries") + total_uncomp = 0 + kml_name = None + for info in infos: + name = info.filename.replace("\\", "/") + if ".." in name.split("/"): + raise GeoValidationError("path_traversal") + total_uncomp += int(info.file_size) + if total_uncomp > max_uncompressed_bytes: + raise GeoValidationError("kmz_uncompressed_too_large") + if info.compress_size > 0: + ratio = float(info.file_size) / float(info.compress_size) + if ratio > MAX_COMPRESSION_RATIO and info.file_size > 1024 * 1024: + raise GeoValidationError("kmz_compression_ratio") + lower = name.lower() + if lower.endswith(".kml"): + if kml_name is None or lower.endswith("doc.kml") or lower == "doc.kml": + if lower == "doc.kml" or lower.endswith("/doc.kml"): + kml_name = name + elif kml_name is None: + kml_name = name + if not kml_name: + raise GeoValidationError("kmz_missing_kml") + kml_bytes = zf.read(kml_name) + kml_result = validate_kml_bytes( + kml_bytes, + max_bytes=max_uncompressed_bytes, + max_features=max_features, + ) + return GeoValidationResult( + format="kmz", + feature_count=kml_result.feature_count, + byte_size=len(data), + ) + + +def sniff_format(data: bytes, hinted: str | None = None) -> str: + if hinted in ("geojson", "kml", "kmz"): + return hinted + if data.startswith(ZIP_LOCAL_HEADER): + return "kmz" + sample = data[:256].lstrip() + if sample.startswith(b"{") or sample.startswith(b"["): + return "geojson" + lower = sample.lower() + if lower.startswith(b" GeoValidationResult: + if not data: + raise GeoValidationError("empty_file") + fmt = sniff_format(data, hinted_format) + if fmt == "geojson": + return validate_geojson_bytes( + data, + max_bytes=max_bytes, + max_features=max_features, + ) + if fmt == "kml": + return validate_kml_bytes( + data, + max_bytes=max_bytes, + max_features=max_features, + ) + if fmt == "kmz": + return validate_kmz_bytes( + data, + max_bytes=max_bytes, + max_uncompressed_bytes=max_kmz_uncompressed_bytes, + max_features=max_features, + ) + raise GeoValidationError("unknown_format") diff --git a/meshchatx/src/backend/map_overlay_export.py b/meshchatx/src/backend/map_overlay_export.py new file mode 100644 index 00000000..2d89ab31 --- /dev/null +++ b/meshchatx/src/backend/map_overlay_export.py @@ -0,0 +1,282 @@ +# SPDX-License-Identifier: 0BSD + +"""Export / transcode cached map overlays between GeoJSON, KML, and KMZ.""" + +from __future__ import annotations + +import json +import zipfile +from io import BytesIO +from xml.etree import ElementTree as ET +from xml.sax.saxutils import escape + +from meshchatx.src.backend.map_geo_validator import ( + GeoValidationError, + sniff_format, + validate_geo_bytes, +) + + +class OverlayExportError(ValueError): + def __init__(self, code: str, message: str | None = None): + self.code = code + super().__init__(message or code) + + +def _strip_ns(tag: str) -> str: + if "}" in tag: + return tag.rsplit("}", 1)[-1] + return tag + + +def _coords_to_kml(coords, geom_type: str) -> str: + if geom_type == "Point": + lon, lat = coords[0], coords[1] + alt = coords[2] if len(coords) > 2 else 0 + return f"{lon},{lat},{alt}" + if geom_type in ("LineString", "MultiPoint"): + parts = [] + for c in coords: + lon, lat = c[0], c[1] + alt = c[2] if len(c) > 2 else 0 + parts.append(f"{lon},{lat},{alt}") + return " ".join(parts) + if geom_type == "Polygon": + # outer ring only for simple export + ring = coords[0] if coords else [] + return _coords_to_kml(ring, "LineString") + return "" + + +def geojson_to_kml(data: bytes) -> bytes: + obj = json.loads(data.decode("utf-8")) + features = [] + if obj.get("type") == "FeatureCollection": + features = obj.get("features") or [] + elif obj.get("type") == "Feature": + features = [obj] + else: + features = [{"type": "Feature", "properties": {}, "geometry": obj}] + + parts = [ + '', + '', + ] + for feat in features: + if not isinstance(feat, dict): + continue + props = feat.get("properties") or {} + name = escape(str(props.get("name") or props.get("title") or "feature")) + geom = feat.get("geometry") or {} + gtype = geom.get("type") + coords = geom.get("coordinates") + if not gtype or coords is None: + continue + parts.append("") + parts.append(f"{name}") + if gtype == "Point": + parts.append( + f"{_coords_to_kml(coords, 'Point')}", + ) + elif gtype == "LineString": + parts.append( + f"{_coords_to_kml(coords, 'LineString')}", + ) + elif gtype == "Polygon": + parts.append( + "" + f"{_coords_to_kml(coords, 'Polygon')}" + "", + ) + else: + # Skip complex geometries in simple transcoder + parts.append("") + continue + parts.append("") + parts.append("") + return "\n".join(parts).encode("utf-8") + + +def kml_to_geojson(data: bytes) -> bytes: + root = ET.fromstring(data.decode("utf-8")) + features = [] + for pm in root.iter(): + if _strip_ns(pm.tag).lower() != "placemark": + continue + name = None + geom = None + for child in list(pm): + tag = _strip_ns(child.tag).lower() + if tag == "name": + name = (child.text or "").strip() + elif tag == "point": + coords_el = next( + ( + c + for c in child.iter() + if _strip_ns(c.tag).lower() == "coordinates" + ), + None, + ) + if coords_el is not None and coords_el.text: + parts = coords_el.text.strip().split(",") + if len(parts) >= 2: + geom = { + "type": "Point", + "coordinates": [float(parts[0]), float(parts[1])], + } + elif tag == "linestring": + coords_el = next( + ( + c + for c in child.iter() + if _strip_ns(c.tag).lower() == "coordinates" + ), + None, + ) + if coords_el is not None and coords_el.text: + line = [] + for token in coords_el.text.strip().split(): + bits = token.split(",") + if len(bits) >= 2: + line.append([float(bits[0]), float(bits[1])]) + if line: + geom = {"type": "LineString", "coordinates": line} + elif tag == "polygon": + coords_el = next( + ( + c + for c in child.iter() + if _strip_ns(c.tag).lower() == "coordinates" + ), + None, + ) + if coords_el is not None and coords_el.text: + ring = [] + for token in coords_el.text.strip().split(): + bits = token.split(",") + if len(bits) >= 2: + ring.append([float(bits[0]), float(bits[1])]) + if ring: + geom = {"type": "Polygon", "coordinates": [ring]} + if geom is None: + continue + features.append( + { + "type": "Feature", + "properties": {"name": name} if name else {}, + "geometry": geom, + }, + ) + return json.dumps( + {"type": "FeatureCollection", "features": features}, + separators=(",", ":"), + ).encode("utf-8") + + +def kmz_to_kml(data: bytes) -> bytes: + with zipfile.ZipFile(BytesIO(data)) as zf: + names = [n for n in zf.namelist() if not n.endswith("/")] + kml_name = None + for n in names: + lower = n.replace("\\", "/").lower() + if lower == "doc.kml" or lower.endswith("/doc.kml"): + kml_name = n + break + if kml_name is None: + for n in names: + if n.lower().endswith(".kml"): + kml_name = n + break + if kml_name is None: + raise OverlayExportError("kmz_missing_kml") + return zf.read(kml_name) + + +def kml_to_kmz(kml_bytes: bytes) -> bytes: + buf = BytesIO() + with zipfile.ZipFile(buf, "w", compression=zipfile.ZIP_DEFLATED) as zf: + zf.writestr("doc.kml", kml_bytes) + return buf.getvalue() + + +def merge_geojson_bytes(chunks: list[bytes]) -> bytes: + features = [] + for chunk in chunks: + obj = json.loads(chunk.decode("utf-8")) + if obj.get("type") == "FeatureCollection": + features.extend(obj.get("features") or []) + elif obj.get("type") == "Feature": + features.append(obj) + else: + features.append( + {"type": "Feature", "properties": {}, "geometry": obj}, + ) + return json.dumps( + {"type": "FeatureCollection", "features": features}, + separators=(",", ":"), + ).encode("utf-8") + + +def to_geojson(data: bytes, source_format: str | None = None) -> bytes: + fmt = source_format or sniff_format(data) + if fmt == "geojson": + return data + if fmt == "kml": + return kml_to_geojson(data) + if fmt == "kmz": + return kml_to_geojson(kmz_to_kml(data)) + raise OverlayExportError("unknown_format") + + +def from_geojson(geojson_bytes: bytes, target_format: str) -> bytes: + if target_format == "geojson": + return geojson_bytes + if target_format == "kml": + return geojson_to_kml(geojson_bytes) + if target_format == "kmz": + return kml_to_kmz(geojson_to_kml(geojson_bytes)) + raise OverlayExportError("unknown_format") + + +def convert_overlay_bytes( + data: bytes, + *, + source_format: str | None, + target_format: str, + max_bytes: int, + max_features: int, + max_kmz_uncompressed_bytes: int, +) -> bytes: + if target_format not in ("geojson", "kml", "kmz"): + raise OverlayExportError("invalid_export_format") + src = source_format or sniff_format(data) + if src == target_format: + out = data + else: + gj = to_geojson(data, src) + out = from_geojson(gj, target_format) + try: + validate_geo_bytes( + out, + hinted_format=target_format, + max_bytes=max_bytes, + max_features=max_features, + max_kmz_uncompressed_bytes=max_kmz_uncompressed_bytes, + ) + except GeoValidationError as exc: + raise OverlayExportError(exc.code) from exc + return out + + +CONTENT_TYPES = { + "geojson": "application/geo+json", + "kml": "application/vnd.google-earth.kml+xml", + "kmz": "application/vnd.google-earth.kmz", +} + +EXTENSIONS = { + "geojson": ".geojson", + "kml": ".kml", + "kmz": ".kmz", +} diff --git a/meshchatx/src/backend/map_overlay_manager.py b/meshchatx/src/backend/map_overlay_manager.py new file mode 100644 index 00000000..a6cff9a0 --- /dev/null +++ b/meshchatx/src/backend/map_overlay_manager.py @@ -0,0 +1,861 @@ +# SPDX-License-Identifier: 0BSD + +"""Manage remote map overlay sources: fetch, cache, refresh, export.""" + +from __future__ import annotations + +import asyncio +import hashlib +import logging +import os +import random +import re +import shutil +import uuid +from datetime import UTC, datetime, timedelta +from typing import Any + +from meshchatx.src.backend.map_geo_validator import ( + GeoValidationError, + validate_geo_bytes, +) +from meshchatx.src.backend.map_overlay_export import ( + CONTENT_TYPES, + EXTENSIONS, + OverlayExportError, + convert_overlay_bytes, + merge_geojson_bytes, + to_geojson, + from_geojson, +) +from meshchatx.src.backend.map_overlay_sources import ( + KIND_NOMADNET_FILE, + KIND_RNGIT_FILES, + OverlaySourceParseError, + OverlaySourceSpec, + guess_format_from_path, + parse_create_payload, +) +from meshchatx.src.backend.nomadnet_downloader import NomadnetFileDownloader +from meshchatx.src.backend.rngit_sparse_fetcher import ( + RngitFetchError, + RngitSparseFetcher, +) + +_log = logging.getLogger("meshchatx.map_overlays") + +_SAFE_NAME_RE = re.compile(r"[^A-Za-z0-9._-]+") + +CONFIG_CLAMPS = { + "map_overlay_max_bytes": (64 * 1024, 64 * 1024 * 1024), + "map_overlay_max_features": (100, 500_000), + "map_overlay_max_kmz_uncompressed_bytes": (256 * 1024, 128 * 1024 * 1024), + "map_overlay_max_sources": (1, 256), + "map_overlay_max_concurrent_jobs": (1, 8), + "map_overlay_path_timeout_seconds": (5, 300), + "map_overlay_transfer_timeout_seconds": (15, 600), + "map_overlay_job_timeout_seconds": (30, 1800), + "map_overlay_max_retries": (0, 10), + "map_overlay_retry_delay_seconds": (1, 120), +} + + +def clamp_overlay_config_value(key: str, value: int) -> int: + lo, hi = CONFIG_CLAMPS[key] + return max(lo, min(hi, int(value))) + + +def atomic_write_bytes(path: str, data: bytes) -> None: + parent = os.path.dirname(path) + os.makedirs(parent, exist_ok=True) + tmp = path + ".tmp" + with open(tmp, "wb") as f: + f.write(data) + f.flush() + os.fsync(f.fileno()) + os.replace(tmp, path) + + +def _safe_filename(name: str, ext: str) -> str: + base = _SAFE_NAME_RE.sub("_", (name or "overlay").strip())[:80] or "overlay" + if not ext.startswith("."): + ext = "." + ext + return base + ext + + +def _row_to_dict(row) -> dict[str, Any]: + if row is None: + return {} + if hasattr(row, "keys"): + return {k: row[k] for k in row.keys()} + return dict(row) + + +class MapOverlayManager: + def __init__( + self, + config, + database, + storage_dir: str, + *, + reticulum_config_dir: str | None = None, + identity=None, + reticulum=None, + file_downloader_factory=None, + rngit_fetcher_factory=None, + ): + self.config = config + self.database = database + self.storage_dir = storage_dir + self.reticulum_config_dir = reticulum_config_dir + self.identity = identity + self.reticulum = reticulum + self._file_downloader_factory = ( + file_downloader_factory or self._default_file_downloader_factory + ) + self._rngit_fetcher_factory = ( + rngit_fetcher_factory or self._default_rngit_fetcher_factory + ) + + self._jobs: dict[str, dict[str, Any]] = {} + self._source_locks: dict[int, asyncio.Lock] = {} + self._active_fetchers: dict[str, Any] = {} + self._job_semaphore: asyncio.Semaphore | None = None + self._scheduler_task: asyncio.Task | None = None + self._stopped = False + + def overlay_root(self) -> str: + path = os.path.join(self.storage_dir, "map_overlays") + os.makedirs(path, exist_ok=True) + return path + + def work_root(self) -> str: + path = os.path.join(self.overlay_root(), ".work") + os.makedirs(path, exist_ok=True) + return path + + def cache_path_for(self, identity_hash: str, overlay_id: int, fmt: str) -> str: + rel = os.path.join(identity_hash, f"{overlay_id}.{fmt}") + return os.path.join(self.overlay_root(), rel), rel + + def _cfg_int(self, key: str) -> int: + conf = getattr(self.config, key) + raw = conf.get() + return clamp_overlay_config_value(key, int(raw)) + + def limits(self) -> dict[str, int]: + return {k: self._cfg_int(k) for k in CONFIG_CLAMPS} + + def _source_lock(self, overlay_id: int) -> asyncio.Lock: + lock = self._source_locks.get(overlay_id) + if lock is None: + lock = asyncio.Lock() + self._source_locks[overlay_id] = lock + return lock + + def _default_file_downloader_factory(self, **kwargs): + return NomadnetFileDownloader(**kwargs) + + def _default_rngit_fetcher_factory(self, **kwargs): + return RngitSparseFetcher(**kwargs) + + def start_scheduler(self) -> None: + if self._scheduler_task is not None: + return + self._stopped = False + try: + loop = asyncio.get_running_loop() + except RuntimeError: + return + self._scheduler_task = loop.create_task(self._autorefresh_loop()) + + def stop_scheduler(self) -> None: + self._stopped = True + if self._scheduler_task is not None: + self._scheduler_task.cancel() + self._scheduler_task = None + for fetcher in list(self._active_fetchers.values()): + try: + fetcher.cancel() + except Exception: + pass + + async def _autorefresh_loop(self) -> None: + while not self._stopped: + try: + await self._tick_autorefresh() + except asyncio.CancelledError: + raise + except Exception: + _log.exception("map overlay autorefresh tick failed") + await asyncio.sleep(15) + + async def _tick_autorefresh(self) -> None: + now = datetime.now(UTC).isoformat() + rows = self.database.map_overlays.list_due_autorefresh(now) + for row in rows: + overlay_id = int(row["id"]) + identity_hash = row["identity_hash"] + if self._source_lock(overlay_id).locked(): + continue + try: + await self.refresh_overlay( + identity_hash, overlay_id, reason="autorefresh" + ) + except Exception: + _log.exception("autorefresh failed for overlay %s", overlay_id) + + def list_overlays(self, identity_hash: str) -> list[dict[str, Any]]: + rows = self.database.map_overlays.list_for_identity(identity_hash) + return [_row_to_dict(r) for r in rows] + + def get_overlay(self, identity_hash: str, overlay_id: int) -> dict[str, Any] | None: + row = self.database.map_overlays.get_by_id(overlay_id) + if not row or row["identity_hash"] != identity_hash: + return None + return _row_to_dict(row) + + def get_job(self, job_id: str) -> dict[str, Any] | None: + return self._jobs.get(job_id) + + async def create_overlays( + self, + identity_hash: str, + payload: dict[str, Any], + ) -> dict[str, Any]: + specs = parse_create_payload(payload) + max_sources = self._cfg_int("map_overlay_max_sources") + current = self.database.map_overlays.count_for_identity(identity_hash) + if current + len(specs) > max_sources: + raise OverlaySourceParseError("max_sources_exceeded") + + created_ids: list[int] = [] + for spec in specs: + existing = self.database.map_overlays.get_by_unique( + identity_hash, + spec.kind, + spec.destination_hash, + spec.path_or_repo_path, + spec.ref, + ) + if existing: + created_ids.append(int(existing["id"])) + continue + oid = self.database.map_overlays.insert( + identity_hash, + kind=spec.kind, + destination_hash=spec.destination_hash, + path_or_repo_path=spec.path_or_repo_path, + ref=spec.ref, + name=spec.name or "overlay", + group_name=spec.group_name, + repository=spec.repository, + refresh_interval_seconds=spec.refresh_interval_seconds, + status="pending", + ) + created_ids.append(oid) + + job_id = await self._start_job_for_ids(identity_hash, created_ids, specs) + overlays = [self.get_overlay(identity_hash, i) for i in created_ids] + return {"job_id": job_id, "overlays": overlays} + + async def refresh_overlay( + self, + identity_hash: str, + overlay_id: int, + *, + reason: str = "manual", + ) -> dict[str, Any]: + row = self.get_overlay(identity_hash, overlay_id) + if not row: + raise OverlaySourceParseError("not_found") + spec = OverlaySourceSpec( + kind=row["kind"], + destination_hash=row["destination_hash"], + path_or_repo_path=row["path_or_repo_path"], + ref=row.get("ref") or "HEAD", + group_name=row.get("group_name"), + repository=row.get("repository"), + name=row.get("name"), + paths=[row["path_or_repo_path"]], + refresh_interval_seconds=int(row.get("refresh_interval_seconds") or 0), + ) + job_id = await self._start_job_for_ids(identity_hash, [overlay_id], [spec]) + return { + "job_id": job_id, + "overlay": self.get_overlay(identity_hash, overlay_id), + "reason": reason, + } + + async def _start_job_for_ids( + self, + identity_hash: str, + overlay_ids: list[int], + specs: list[OverlaySourceSpec], + ) -> str: + job_id = uuid.uuid4().hex + generations: dict[int, int] = {} + for oid in overlay_ids: + row = self.database.map_overlays.get_by_id(oid) + gen = int(row["generation"] or 0) + 1 if row else 1 + generations[oid] = gen + self.database.map_overlays.update_fields( + oid, + status="fetching", + last_error=None, + job_id=job_id, + generation=gen, + ) + + self._jobs[job_id] = { + "job_id": job_id, + "identity_hash": identity_hash, + "overlay_ids": list(overlay_ids), + "status": "running", + "phase": "queued", + "progress": 0.0, + "error": None, + "created_at": datetime.now(UTC).isoformat(), + } + + async def runner(): + try: + await self._run_job( + job_id, identity_hash, overlay_ids, specs, generations + ) + except Exception as exc: + _log.exception("overlay job %s failed", job_id) + self._jobs[job_id]["status"] = "error" + self._jobs[job_id]["error"] = str(exc) + + try: + loop = asyncio.get_running_loop() + loop.create_task(runner()) + except RuntimeError: + await runner() + return job_id + + async def _run_job( + self, + job_id: str, + identity_hash: str, + overlay_ids: list[int], + specs: list[OverlaySourceSpec], + generations: dict[int, int], + ) -> None: + max_conc = self._cfg_int("map_overlay_max_concurrent_jobs") + if self._job_semaphore is None: + self._job_semaphore = asyncio.Semaphore(max_conc) + # Resize not supported mid-flight; new semaphore if limit changed and idle + async with self._job_semaphore: + job = self._jobs[job_id] + try: + kind = specs[0].kind if specs else None + if kind == KIND_NOMADNET_FILE: + await self._fetch_nomadnet_job( + job_id, + identity_hash, + overlay_ids[0], + specs[0], + generations[overlay_ids[0]], + ) + elif kind == KIND_RNGIT_FILES: + await self._fetch_rngit_job( + job_id, + identity_hash, + overlay_ids, + specs, + generations, + ) + else: + raise OverlaySourceParseError("unsupported_kind") + job["status"] = "success" + job["phase"] = "done" + job["progress"] = 1.0 + except asyncio.CancelledError: + job["status"] = "cancelled" + job["error"] = "cancelled" + raise + except ( + OverlaySourceParseError, + GeoValidationError, + RngitFetchError, + ) as exc: + code = getattr(exc, "code", str(exc)) + job["status"] = "error" + job["error"] = code + for oid in overlay_ids: + if not self._generation_current(oid, generations[oid]): + continue + self._mark_error(oid, code) + except Exception as exc: + job["status"] = "error" + job["error"] = str(exc) + for oid in overlay_ids: + if not self._generation_current(oid, generations[oid]): + continue + self._mark_error(oid, "fetch_failed") + + def _generation_current(self, overlay_id: int, generation: int) -> bool: + row = self.database.map_overlays.get_by_id(overlay_id) + return bool(row) and int(row["generation"] or 0) == generation + + def _mark_error(self, overlay_id: int, code: str) -> None: + row = self.database.map_overlays.get_by_id(overlay_id) + interval = int(row["refresh_interval_seconds"] or 0) if row else 0 + next_at = None + if interval > 0: + # backoff after failure: at least interval + next_at = (datetime.now(UTC) + timedelta(seconds=interval)).isoformat() + self.database.map_overlays.update_fields( + overlay_id, + status="error", + last_error=code, + next_refresh_at=next_at, + ) + + def _set_phase( + self, job_id: str, phase: str, progress: float | None = None + ) -> None: + job = self._jobs.get(job_id) + if not job: + return + job["phase"] = phase + if progress is not None: + job["progress"] = progress + + async def _fetch_nomadnet_job( + self, + job_id: str, + identity_hash: str, + overlay_id: int, + spec: OverlaySourceSpec, + generation: int, + ) -> None: + async with self._source_lock(overlay_id): + await self._fetch_with_retries( + job_id, + lambda: self._download_nomadnet_once( + job_id, identity_hash, overlay_id, spec, generation + ), + ) + + async def _fetch_rngit_job( + self, + job_id: str, + identity_hash: str, + overlay_ids: list[int], + specs: list[OverlaySourceSpec], + generations: dict[int, int], + ) -> None: + # Lock all sources in id order to avoid deadlocks + locks = [self._source_lock(oid) for oid in sorted(overlay_ids)] + for lock in locks: + await lock.acquire() + try: + await self._fetch_with_retries( + job_id, + lambda: self._download_rngit_once( + job_id, + identity_hash, + overlay_ids, + specs, + generations, + ), + ) + finally: + for lock in reversed(locks): + lock.release() + + async def _fetch_with_retries(self, job_id: str, attempt_fn) -> None: + max_retries = self._cfg_int("map_overlay_max_retries") + base_delay = self._cfg_int("map_overlay_retry_delay_seconds") + last_exc = None + for attempt in range(max_retries + 1): + try: + await attempt_fn() + return + except (GeoValidationError, OverlaySourceParseError) as exc: + # Do not retry validation / parse errors + raise exc + except RngitFetchError as exc: + if exc.code in ( + "cancelled", + "rngit_tools_unavailable", + "path_missing", + "path_traversal", + ): + raise + last_exc = exc + except Exception as exc: + last_exc = exc + if attempt >= max_retries: + break + delay = min(120.0, base_delay * (2**attempt)) + delay *= 0.5 + random.random() + self._set_phase(job_id, "retry_wait", progress=0.0) + await asyncio.sleep(delay) + if last_exc: + raise last_exc + raise RuntimeError("fetch_failed") + + async def _download_nomadnet_once( + self, + job_id: str, + identity_hash: str, + overlay_id: int, + spec: OverlaySourceSpec, + generation: int, + ) -> None: + path_timeout = self._cfg_int("map_overlay_path_timeout_seconds") + transfer_timeout = self._cfg_int("map_overlay_transfer_timeout_seconds") + job_timeout = self._cfg_int("map_overlay_job_timeout_seconds") + + loop = asyncio.get_running_loop() + done = asyncio.Event() + result: dict[str, Any] = {} + + def on_success(file_name: str, payload: bytes): + result["ok"] = True + result["name"] = file_name + result["payload"] = payload + loop.call_soon_threadsafe(done.set) + + def on_failure(reason: str): + result["ok"] = False + result["error"] = reason + loop.call_soon_threadsafe(done.set) + + def on_progress(p: float): + self._set_phase(job_id, "transferring", progress=float(p or 0)) + + def on_phase(phase: str): + self._set_phase(job_id, phase) + + downloader = self._file_downloader_factory( + destination_hash=bytes.fromhex(spec.destination_hash), + page_path=spec.path_or_repo_path, + on_file_download_success=on_success, + on_file_download_failure=on_failure, + on_progress_update=on_progress, + timeout=transfer_timeout, + on_phase=on_phase, + reticulum=self.reticulum, + ) + self._active_fetchers[job_id] = downloader + try: + await asyncio.wait_for( + downloader.download( + path_lookup_timeout=path_timeout, + link_establishment_timeout=path_timeout, + ), + timeout=job_timeout, + ) + await asyncio.wait_for(done.wait(), timeout=job_timeout) + except TimeoutError as exc: + downloader.cancel() + raise RngitFetchError("job_timeout") from exc + finally: + self._active_fetchers.pop(job_id, None) + + if not result.get("ok"): + raise RngitFetchError(str(result.get("error") or "request_failed")) + + payload = result["payload"] + if not isinstance(payload, (bytes, bytearray)): + raise GeoValidationError("invalid_response_body") + await self._commit_bytes( + job_id, + identity_hash, + overlay_id, + generation, + bytes(payload), + hinted_format=guess_format_from_path(spec.path_or_repo_path), + resolved_ref=None, + refresh_interval=spec.refresh_interval_seconds, + ) + + async def _download_rngit_once( + self, + job_id: str, + identity_hash: str, + overlay_ids: list[int], + specs: list[OverlaySourceSpec], + generations: dict[int, int], + ) -> None: + job_timeout = self._cfg_int("map_overlay_job_timeout_seconds") + first = specs[0] + paths = [s.path_or_repo_path for s in specs] + fetcher = self._rngit_fetcher_factory( + work_root=self.work_root(), + reticulum_config_dir=self.reticulum_config_dir, + ) + self._active_fetchers[job_id] = fetcher + + def on_phase(phase: str): + self._set_phase(job_id, phase) + + try: + result = await asyncio.wait_for( + fetcher.fetch( + destination_hash=first.destination_hash, + group=first.group_name, + repository=first.repository, + paths=paths, + ref=first.ref, + job_id=job_id, + timeout_seconds=job_timeout, + on_phase=on_phase, + ), + timeout=job_timeout + 5, + ) + finally: + self._active_fetchers.pop(job_id, None) + + for oid, spec in zip(overlay_ids, specs, strict=True): + if not self._generation_current(oid, generations[oid]): + continue + payload = result.files.get(spec.path_or_repo_path) + if payload is None: + self._mark_error(oid, "path_missing") + continue + await self._commit_bytes( + job_id, + identity_hash, + oid, + generations[oid], + payload, + hinted_format=guess_format_from_path(spec.path_or_repo_path), + resolved_ref=result.resolved_ref, + refresh_interval=spec.refresh_interval_seconds, + ) + + async def _commit_bytes( + self, + job_id: str, + identity_hash: str, + overlay_id: int, + generation: int, + payload: bytes, + *, + hinted_format: str | None, + resolved_ref: str | None, + refresh_interval: int, + ) -> None: + if not self._generation_current(overlay_id, generation): + return + limits = self.limits() + self._set_phase(job_id, "validating") + validated = validate_geo_bytes( + payload, + hinted_format=hinted_format, + max_bytes=limits["map_overlay_max_bytes"], + max_features=limits["map_overlay_max_features"], + max_kmz_uncompressed_bytes=limits["map_overlay_max_kmz_uncompressed_bytes"], + ) + digest = hashlib.sha256(payload).hexdigest() + row = self.database.map_overlays.get_by_id(overlay_id) + if row and row.get("content_sha256") == digest and row.get("cache_relpath"): + abs_existing = os.path.join(self.overlay_root(), row["cache_relpath"]) + if os.path.isfile(abs_existing): + now = datetime.now(UTC) + next_at = None + if refresh_interval > 0: + next_at = (now + timedelta(seconds=refresh_interval)).isoformat() + self.database.map_overlays.update_fields( + overlay_id, + status="ready", + last_error=None, + last_fetched_at=now.isoformat(), + next_refresh_at=next_at, + resolved_ref=resolved_ref or row.get("resolved_ref"), + format=validated.format, + byte_size=validated.byte_size, + ) + return + + abs_path, rel = self.cache_path_for(identity_hash, overlay_id, validated.format) + atomic_write_bytes(abs_path, payload) + now = datetime.now(UTC) + next_at = None + if refresh_interval > 0: + next_at = (now + timedelta(seconds=refresh_interval)).isoformat() + if not self._generation_current(overlay_id, generation): + return + self.database.map_overlays.update_fields( + overlay_id, + status="ready", + last_error=None, + last_fetched_at=now.isoformat(), + next_refresh_at=next_at, + content_sha256=digest, + resolved_ref=resolved_ref, + format=validated.format, + byte_size=validated.byte_size, + cache_relpath=rel, + ) + + def cancel_job(self, job_id: str) -> bool: + job = self._jobs.get(job_id) + if not job or job.get("status") not in ("running",): + return False + fetcher = self._active_fetchers.get(job_id) + if fetcher is not None: + try: + fetcher.cancel() + except Exception: + pass + job["status"] = "cancelled" + job["error"] = "cancelled" + for oid in job.get("overlay_ids") or []: + row = self.database.map_overlays.get_by_id(oid) + if row and row.get("job_id") == job_id and row.get("status") == "fetching": + self.database.map_overlays.update_fields( + oid, + status="error", + last_error="cancelled", + ) + return True + + def patch_overlay( + self, + identity_hash: str, + overlay_id: int, + data: dict[str, Any], + ) -> dict[str, Any]: + row = self.get_overlay(identity_hash, overlay_id) + if not row: + raise OverlaySourceParseError("not_found") + fields: dict[str, Any] = {} + if "name" in data and data["name"] is not None: + fields["name"] = str(data["name"]).strip()[:200] or row["name"] + if "enabled" in data: + fields["enabled"] = 1 if data["enabled"] else 0 + if "visible" in data: + fields["visible"] = 1 if data["visible"] else 0 + if "refresh_interval_seconds" in data: + try: + ri = int(data["refresh_interval_seconds"]) + except (TypeError, ValueError) as exc: + raise OverlaySourceParseError("invalid_refresh_interval") from exc + if ri < 0: + raise OverlaySourceParseError("invalid_refresh_interval") + if 0 < ri < 60: + ri = 60 + if ri > 86400: + ri = 86400 + fields["refresh_interval_seconds"] = ri + if ri > 0: + base = row.get("last_fetched_at") + if base: + fields["next_refresh_at"] = ( + datetime.fromisoformat(str(base).replace("Z", "+00:00")) + + timedelta(seconds=ri) + ).isoformat() + else: + fields["next_refresh_at"] = datetime.now(UTC).isoformat() + else: + fields["next_refresh_at"] = None + if "ref" in data and row["kind"] == KIND_RNGIT_FILES: + from meshchatx.src.backend.map_overlay_sources import normalize_ref + + fields["ref"] = normalize_ref(data["ref"]) + if fields: + self.database.map_overlays.update_fields(overlay_id, **fields) + return self.get_overlay(identity_hash, overlay_id) + + def delete_overlay(self, identity_hash: str, overlay_id: int) -> bool: + row = self.get_overlay(identity_hash, overlay_id) + if not row: + return False + rel = row.get("cache_relpath") + if rel: + abs_path = os.path.join(self.overlay_root(), rel) + try: + if os.path.isfile(abs_path): + os.remove(abs_path) + except OSError: + pass + return self.database.map_overlays.delete_for_identity(identity_hash, overlay_id) + + def read_cache_bytes( + self, identity_hash: str, overlay_id: int + ) -> tuple[bytes, str] | None: + row = self.get_overlay(identity_hash, overlay_id) + if not row or not row.get("cache_relpath") or not row.get("format"): + return None + abs_path = os.path.join(self.overlay_root(), row["cache_relpath"]) + if not os.path.isfile(abs_path): + return None + with open(abs_path, "rb") as f: + data = f.read() + return data, row["format"] + + def export_overlay( + self, + identity_hash: str, + overlay_id: int, + target_format: str, + ) -> tuple[bytes, str, str]: + cached = self.read_cache_bytes(identity_hash, overlay_id) + if not cached: + raise OverlayExportError("cache_missing") + data, src_fmt = cached + limits = self.limits() + out = convert_overlay_bytes( + data, + source_format=src_fmt, + target_format=target_format, + max_bytes=limits["map_overlay_max_bytes"], + max_features=limits["map_overlay_max_features"], + max_kmz_uncompressed_bytes=limits["map_overlay_max_kmz_uncompressed_bytes"], + ) + row = self.get_overlay(identity_hash, overlay_id) + filename = _safe_filename( + row.get("name") if row else "overlay", EXTENSIONS[target_format] + ) + return out, CONTENT_TYPES[target_format], filename + + def export_many( + self, + identity_hash: str, + overlay_ids: list[int], + target_format: str, + ) -> tuple[bytes, str, str]: + if not overlay_ids: + raise OverlayExportError("missing_ids") + if len(overlay_ids) == 1: + return self.export_overlay(identity_hash, overlay_ids[0], target_format) + geo_chunks: list[bytes] = [] + limits = self.limits() + for oid in overlay_ids: + cached = self.read_cache_bytes(identity_hash, oid) + if not cached: + raise OverlayExportError("cache_missing") + data, src_fmt = cached + geo_chunks.append(to_geojson(data, src_fmt)) + merged = merge_geojson_bytes(geo_chunks) + out = from_geojson(merged, target_format) + max_bytes = limits["map_overlay_max_bytes"] * max(1, len(overlay_ids)) + max_features = limits["map_overlay_max_features"] * max(1, len(overlay_ids)) + max_uncomp = limits["map_overlay_max_kmz_uncompressed_bytes"] * max( + 1, + len(overlay_ids), + ) + if len(out) > max_bytes: + raise OverlayExportError("file_too_large") + validate_geo_bytes( + out, + hinted_format=target_format, + max_bytes=max_bytes, + max_features=max_features, + max_kmz_uncompressed_bytes=max_uncomp, + ) + filename = _safe_filename("overlays", EXTENSIONS[target_format]) + return out, CONTENT_TYPES[target_format], filename + + def cleanup(self) -> None: + self.stop_scheduler() + work = os.path.join(self.overlay_root(), ".work") + if os.path.isdir(work): + shutil.rmtree(work, ignore_errors=True) diff --git a/meshchatx/src/backend/map_overlay_sources.py b/meshchatx/src/backend/map_overlay_sources.py new file mode 100644 index 00000000..e6dffefc --- /dev/null +++ b/meshchatx/src/backend/map_overlay_sources.py @@ -0,0 +1,271 @@ +# SPDX-License-Identifier: 0BSD + +"""Parse and validate NomadNet / RNGit map overlay source descriptors.""" + +from __future__ import annotations + +import os +import re +from dataclasses import dataclass, field +from typing import Any + +import RNS + +KIND_NOMADNET_FILE = "nomadnet_file" +KIND_RNGIT_FILES = "rngit_files" + +_HASH_HEX_LEN = RNS.Reticulum.TRUNCATED_HASHLENGTH // 4 +_HASH_RE = re.compile(rf"^[0-9a-fA-F]{{{_HASH_HEX_LEN}}}$") +_SLUG_RE = re.compile(r"^[A-Za-z0-9._-]+$") +_REF_RE = re.compile(r"^[A-Za-z0-9._/@+-]{1,256}$") +_COMMIT_LIKE_RE = re.compile(r"^[0-9a-fA-F]{7,40}$") + + +class OverlaySourceParseError(ValueError): + def __init__(self, code: str, message: str | None = None): + self.code = code + super().__init__(message or code) + + +@dataclass +class OverlaySourceSpec: + kind: str + destination_hash: str + path_or_repo_path: str + ref: str = "HEAD" + group_name: str | None = None + repository: str | None = None + name: str | None = None + paths: list[str] = field(default_factory=list) + refresh_interval_seconds: int = 0 + + def unique_key(self) -> tuple[str, str, str, str]: + return ( + self.kind, + self.destination_hash, + self.path_or_repo_path, + self.ref, + ) + + +def normalize_destination_hash_hex(value: str) -> str | None: + if not isinstance(value, str): + return None + raw = value.strip().lower().replace(":", "") + if not _HASH_RE.fullmatch(raw): + return None + try: + bytes.fromhex(raw) + except ValueError: + return None + return raw + + +def slug_segment(name: str) -> str | None: + if not isinstance(name, str): + return None + s = name.strip() + if not s or len(s) > 256 or not _SLUG_RE.fullmatch(s): + return None + return s + + +def normalize_ref(ref: str | None) -> str: + if ref is None or str(ref).strip() == "": + return "HEAD" + s = str(ref).strip() + if not _REF_RE.fullmatch(s): + raise OverlaySourceParseError("invalid_ref") + if ".." in s or s.startswith("-"): + raise OverlaySourceParseError("invalid_ref") + return s + + +def is_commit_like_ref(ref: str) -> bool: + return bool(_COMMIT_LIKE_RE.fullmatch(ref)) + + +def _safe_repo_relpath(path: str) -> str: + if not isinstance(path, str): + raise OverlaySourceParseError("invalid_path") + p = path.strip().replace("\\", "/") + if not p or p.startswith("/") or p.startswith("~"): + raise OverlaySourceParseError("invalid_path") + parts = [seg for seg in p.split("/") if seg and seg != "."] + if not parts or any(seg == ".." for seg in parts): + raise OverlaySourceParseError("path_traversal") + joined = "/".join(parts) + if len(joined) > 1024: + raise OverlaySourceParseError("path_too_long") + return joined + + +def _safe_nomadnet_file_path(path: str) -> str: + if not isinstance(path, str): + raise OverlaySourceParseError("invalid_path") + p = path.strip().replace("\\", "/") + if p.startswith("file/"): + p = "/" + p + if not p.startswith("/file/"): + raise OverlaySourceParseError("not_file_path") + rest = p[len("/file/") :] + if not rest or rest.endswith("/"): + raise OverlaySourceParseError("invalid_path") + parts = [seg for seg in rest.split("/") if seg and seg != "."] + if not parts or any(seg == ".." for seg in parts): + raise OverlaySourceParseError("path_traversal") + return "/file/" + "/".join(parts) + + +def _default_name_from_path(path: str) -> str: + base = os.path.basename(path.rstrip("/")) + return base or "overlay" + + +def parse_nomadnet_file_url(url: str) -> OverlaySourceSpec: + if not isinstance(url, str) or not url.strip(): + raise OverlaySourceParseError("empty_url") + raw = url.strip() + for prefix in ("nomadnet://", "nomadnetwork://"): + if raw.lower().startswith(prefix): + raw = raw[len(prefix) :] + break + if ":" in raw: + hash_part, path_part = raw.split(":", 1) + elif "/file/" in raw: + idx = raw.lower().find("/file/") + hash_part, path_part = raw[:idx], raw[idx:] + else: + raise OverlaySourceParseError("invalid_nomadnet_url") + dest = normalize_destination_hash_hex(hash_part) + if not dest: + raise OverlaySourceParseError("invalid_destination_hash") + file_path = _safe_nomadnet_file_path(path_part) + return OverlaySourceSpec( + kind=KIND_NOMADNET_FILE, + destination_hash=dest, + path_or_repo_path=file_path, + ref="HEAD", + name=_default_name_from_path(file_path), + paths=[file_path], + ) + + +def parse_rngit_repo_url(url: str) -> tuple[str, str, str]: + if not isinstance(url, str) or not url.strip(): + raise OverlaySourceParseError("empty_url") + raw = url.strip() + if raw.lower().startswith("rns://"): + raw = raw[6:] + parts = [p for p in raw.split("/") if p] + if len(parts) < 3: + raise OverlaySourceParseError("invalid_rngit_url") + dest = normalize_destination_hash_hex(parts[0]) + if not dest: + raise OverlaySourceParseError("invalid_destination_hash") + group = slug_segment(parts[1]) + repo = slug_segment(parts[2]) + if not group or not repo: + raise OverlaySourceParseError("invalid_repository") + return dest, group, repo + + +def _clamp_refresh_interval(value: Any) -> int: + try: + refresh_i = int(value) + except (TypeError, ValueError) as exc: + raise OverlaySourceParseError("invalid_refresh_interval") from exc + if refresh_i < 0: + raise OverlaySourceParseError("invalid_refresh_interval") + if 0 < refresh_i < 60: + return 60 + if refresh_i > 86400: + return 86400 + return refresh_i + + +def _looks_like_nomadnet(url: str) -> bool: + u = url.strip().lower() + return ( + u.startswith("nomadnet://") + or u.startswith("nomadnetwork://") + or ":/file/" in u + or (len(u) > _HASH_HEX_LEN and "/file/" in u) + ) + + +def parse_create_payload(data: dict[str, Any]) -> list[OverlaySourceSpec]: + """Parse POST /api/v1/map/overlays body into one or more source specs.""" + if not isinstance(data, dict): + raise OverlaySourceParseError("invalid_body") + + kind = (data.get("kind") or "").strip().lower() + url = str(data.get("url") or data.get("source") or "").strip() + if not url: + raise OverlaySourceParseError("empty_url") + + refresh_i = _clamp_refresh_interval(data.get("refresh_interval_seconds", 0)) + name_override = data.get("name") + if name_override is not None: + name_override = str(name_override).strip()[:200] or None + + use_nomadnet = kind in ("nomadnet", "nomadnet_file") or ( + kind in ("",) + and _looks_like_nomadnet(url) + and not url.lower().startswith("rns://") + ) + use_rngit = kind in ("rngit", "rngit_files") or url.lower().startswith("rns://") + + if use_nomadnet and not use_rngit: + spec = parse_nomadnet_file_url(url) + if name_override: + spec.name = name_override + spec.refresh_interval_seconds = refresh_i + return [spec] + + if use_rngit: + dest, group, repo = parse_rngit_repo_url(url) + ref = normalize_ref(data.get("ref")) + paths_raw = data.get("paths") or data.get("files") or [] + if isinstance(paths_raw, str): + paths_raw = [ + line.strip() for line in paths_raw.splitlines() if line.strip() + ] + if not isinstance(paths_raw, list) or not paths_raw: + raise OverlaySourceParseError("missing_paths") + if len(paths_raw) > 32: + raise OverlaySourceParseError("too_many_paths") + specs: list[OverlaySourceSpec] = [] + for raw_path in paths_raw: + rel = _safe_repo_relpath(str(raw_path)) + lower = rel.lower() + if not lower.endswith((".geojson", ".json", ".kml", ".kmz")): + raise OverlaySourceParseError("unsupported_extension") + nm = name_override or _default_name_from_path(rel) + specs.append( + OverlaySourceSpec( + kind=KIND_RNGIT_FILES, + destination_hash=dest, + path_or_repo_path=rel, + ref=ref, + group_name=group, + repository=repo, + name=nm, + paths=[rel], + refresh_interval_seconds=refresh_i, + ), + ) + return specs + + raise OverlaySourceParseError("unsupported_kind") + + +def guess_format_from_path(path: str) -> str | None: + lower = path.lower() + if lower.endswith(".kmz"): + return "kmz" + if lower.endswith(".kml"): + return "kml" + if lower.endswith(".geojson") or lower.endswith(".json"): + return "geojson" + return None diff --git a/meshchatx/src/backend/memory_pressure.py b/meshchatx/src/backend/memory_pressure.py index 46baf94c..709bc9e4 100644 --- a/meshchatx/src/backend/memory_pressure.py +++ b/meshchatx/src/backend/memory_pressure.py @@ -92,9 +92,16 @@ class MemoryPressureManager: db = getattr(self.app, "database", None) if self.app else None if db is not None and hasattr(db, "apply_memory_pressure_pragmas"): try: - db.apply_memory_pressure_pragmas(True) + landlock_active = bool( + getattr(self.app, "landlock_active", False), + ) + db.apply_memory_pressure_pragmas( + True, + landlock_active=landlock_active, + ) self._sqlite_relaxed = True stats["sqlite_relaxed"] = True + stats["sqlite_file_temp"] = not landlock_active except Exception as exc: _log.debug("SQLite pressure pragmas failed: %s", exc) _log.warning( diff --git a/meshchatx/src/backend/message_handler.py b/meshchatx/src/backend/message_handler.py index 10072df2..ace962f6 100644 --- a/meshchatx/src/backend/message_handler.py +++ b/meshchatx/src/backend/message_handler.py @@ -71,6 +71,33 @@ class MessageHandler: params = [like_term, like_term, like_term, limit] return self.db.provider.fetchall(query, params) + # Keep conversation-list payloads small. Full ``fields`` often embeds + # multi-MB base64 attachments and must never be loaded into the list API. + _CONVERSATION_CONTENT_PREVIEW_CHARS = 240 + _FIELDS_HAS_IMAGE_SQL = ( + "(m1.fields IS NOT NULL AND m1.fields != '' AND m1.fields != '{}' " + "AND (instr(m1.fields, '\"image\"') > 0 OR instr(m1.fields, '\"0x05\"') > 0))" + ) + _FIELDS_HAS_AUDIO_SQL = ( + "(m1.fields IS NOT NULL AND m1.fields != '' AND m1.fields != '{}' " + "AND (instr(m1.fields, '\"audio\"') > 0 OR instr(m1.fields, '\"0x06\"') > 0))" + ) + _FIELDS_HAS_FILES_SQL = ( + "(m1.fields IS NOT NULL AND m1.fields != '' AND m1.fields != '{}' " + "AND (instr(m1.fields, '\"file_attachments\"') > 0 " + "OR instr(m1.fields, '\"0x07\"') > 0))" + ) + _FIELDS_HAS_REACTION_SQL = ( + "(m1.fields IS NOT NULL AND m1.fields != '' AND m1.fields != '{}' " + "AND (instr(m1.fields, '\"reaction\"') > 0 OR instr(m1.fields, '\"0x40\"') > 0))" + ) + _FIELDS_HAS_TELEMETRY_SQL = ( + "(m1.fields IS NOT NULL AND m1.fields != '' AND m1.fields != '{}' " + "AND (instr(m1.fields, '\"telemetry\"') > 0 OR instr(m1.fields, '\"0x08\"') > 0 " + "OR instr(m1.fields, '\"telemetry_stream\"') > 0))" + ) + _FIELDS_HAS_ATTACHMENTS_SQL = f"({_FIELDS_HAS_IMAGE_SQL} OR {_FIELDS_HAS_AUDIO_SQL} OR {_FIELDS_HAS_FILES_SQL})" + def get_conversations( self, local_hash, @@ -82,13 +109,22 @@ class MessageHandler: limit=500, offset=0, ): - query = """ + preview_chars = self._CONVERSATION_CONTENT_PREVIEW_CHARS + query = f""" SELECT m1.id, m1.hash, m1.source_hash, m1.destination_hash, m1.peer_hash, m1.state, m1.progress, m1.is_incoming, - m1.title, m1.content, m1.fields, m1.timestamp, + m1.title, + substr(COALESCE(m1.content, ''), 1, {preview_chars}) as content, + m1.timestamp, m1.is_spam, m1.reply_to_hash, m1.created_at, m1.updated_at, + CASE WHEN {self._FIELDS_HAS_IMAGE_SQL} THEN 1 ELSE 0 END as has_image, + CASE WHEN {self._FIELDS_HAS_AUDIO_SQL} THEN 1 ELSE 0 END as has_audio, + CASE WHEN {self._FIELDS_HAS_FILES_SQL} THEN 1 ELSE 0 END as has_files, + CASE WHEN {self._FIELDS_HAS_REACTION_SQL} THEN 1 ELSE 0 END as has_reaction, + CASE WHEN {self._FIELDS_HAS_TELEMETRY_SQL} THEN 1 ELSE 0 END as has_telemetry, + CASE WHEN {self._FIELDS_HAS_ATTACHMENTS_SQL} THEN 1 ELSE 0 END as has_attachments, a.app_data as peer_app_data, c.display_name as custom_display_name, con.custom_image as contact_image, @@ -139,9 +175,7 @@ class MessageHandler: where_clauses.append("m1.state = 'failed'") if filter_has_attachments: - where_clauses.append( - "(m1.fields IS NOT NULL AND m1.fields != '{}' AND m1.fields != '')", - ) + where_clauses.append(self._FIELDS_HAS_ATTACHMENTS_SQL) if search: search = _strip_utf16_surrogates(search) or "" diff --git a/meshchatx/src/backend/repository_server_manager.py b/meshchatx/src/backend/repository_server_manager.py index 9754764c..4ce4fc67 100644 --- a/meshchatx/src/backend/repository_server_manager.py +++ b/meshchatx/src/backend/repository_server_manager.py @@ -432,6 +432,7 @@ class RepositoryServerManager: "completed": 0, "total": 0, } + self._refresh_lock = threading.Lock() os.makedirs(self.uploads_dir, exist_ok=True) os.makedirs(self.bundled_dir, exist_ok=True) self._seed_bundled_from_public() @@ -661,18 +662,32 @@ class RepositoryServerManager: } def refresh_bundled_wheels(self) -> dict[str, Any]: - """Download wheels into ``bundled_dir`` (PyPI JSON + ``urllib``).""" + """Download wheels into ``bundled_dir`` (PyPI JSON + ``urllib``). + + Downloads into a temporary directory first, then atomically replaces + the live bundled directory so a failed refresh cannot wipe existing + wheels. Concurrent refreshes are rejected. + """ + if not self._refresh_lock.acquire(blocking=False): + return { + "ok": False, + "downloaded": [], + "failed": {}, + "error": "refresh_already_running", + } + self._last_refresh_error = None self._last_refresh_ok = [] self._last_refresh_failed = {} dest = Path(self.bundled_dir) dest.mkdir(parents=True, exist_ok=True) - for old in dest.glob("*.whl"): - try: - old.unlink() - except OSError: - pass + staging = ( + dest.parent / f".bundled-refresh-{os.getpid()}-{threading.get_ident()}" + ) + if staging.exists(): + shutil.rmtree(staging, ignore_errors=True) + staging.mkdir(parents=True, exist_ok=True) total = len(bundled_pip_targets()) ok: list[str] = [] @@ -685,9 +700,42 @@ class RepositoryServerManager: running=True, current=pkg, completed=i, total=t ) - result = download_bundled_wheels_to_directory(dest, on_package=_on_pkg) + result = download_bundled_wheels_to_directory(staging, on_package=_on_pkg) ok = result["downloaded"] failed = result["failed"] + if ok: + backup = ( + dest.parent + / f".bundled-backup-{os.getpid()}-{threading.get_ident()}" + ) + if backup.exists(): + shutil.rmtree(backup, ignore_errors=True) + try: + os.replace(dest, backup) + except OSError: + shutil.move(str(dest), str(backup)) + try: + try: + os.replace(staging, dest) + except OSError: + shutil.move(str(staging), str(dest)) + except Exception: + # Restore previous wheels if the staged swap fails. + if not dest.exists() and backup.exists(): + try: + os.replace(backup, dest) + except OSError: + shutil.move(str(backup), str(dest)) + raise + else: + shutil.rmtree(backup, ignore_errors=True) + else: + shutil.rmtree(staging, ignore_errors=True) + except Exception as exc: + shutil.rmtree(staging, ignore_errors=True) + failed = {"refresh": str(exc)} + self._last_refresh_error = str(exc) + ok = [] finally: self._set_refresh_progress( running=False, @@ -695,16 +743,21 @@ class RepositoryServerManager: completed=total, total=total, ) + self._refresh_lock.release() self._last_refresh_ok = ok self._last_refresh_failed = failed - if not ok and failed: - self._last_refresh_error = "all_downloads_failed" - elif failed: - self._last_refresh_error = "partial_failure" + if self._last_refresh_error is None: + if not ok and failed: + self._last_refresh_error = "all_downloads_failed" + elif failed: + self._last_refresh_error = "partial_failure" - return { + payload = { "ok": bool(ok), "downloaded": ok, "failed": failed, } + if self._last_refresh_error and not ok: + payload["error"] = self._last_refresh_error + return payload diff --git a/meshchatx/src/backend/reticulum_config_guard.py b/meshchatx/src/backend/reticulum_config_guard.py index 7fff7ea8..85299b8d 100644 --- a/meshchatx/src/backend/reticulum_config_guard.py +++ b/meshchatx/src/backend/reticulum_config_guard.py @@ -81,3 +81,16 @@ def repair_unparseable_reticulum_config(config_path: str, *, write_default) -> b write_default(config_path) return True + + +def ensure_safe_reticulum_runtime_flags(config_path: str) -> bool: + """Force runtime flags that keep MeshChatX alive when interfaces fail. + + Currently forces ``panic_on_interface_error = No`` so RNS does not call + ``os._exit`` on interface faults. + """ + from meshchatx.src.backend.rns_startup_recovery import ( + ensure_panic_on_interface_error_disabled, + ) + + return ensure_panic_on_interface_error_disabled(config_path) diff --git a/meshchatx/src/backend/rncp_handler.py b/meshchatx/src/backend/rncp_handler.py index b9c2bbd8..0d05c78f 100644 --- a/meshchatx/src/backend/rncp_handler.py +++ b/meshchatx/src/backend/rncp_handler.py @@ -27,6 +27,7 @@ class RNCPHandler: self._listener_fetch_registered = False self._listener_fetch_allowed = False self.on_receive_completed = None + self._cancelled_transfers: set[str] = set() def _emit_receive_event(self, payload): if self.on_receive_completed: @@ -35,6 +36,30 @@ class RNCPHandler: except Exception: pass + def _default_fetch_save_dir(self) -> str: + path = os.path.join(self.storage_dir, "rncp", "downloads") + os.makedirs(path, exist_ok=True) + return path + + def cancel_transfer(self, transfer_id: str | None = None) -> dict: + """Mark one or all active transfers as cancelled.""" + if transfer_id: + self._cancelled_transfers.add(transfer_id) + transfer = self.active_transfers.get(transfer_id) + if transfer is not None: + transfer["status"] = "cancelled" + return {"cancelled": [transfer_id]} + ids = list(self.active_transfers.keys()) + for tid in ids: + self._cancelled_transfers.add(tid) + self.active_transfers[tid]["status"] = "cancelled" + return {"cancelled": ids} + + def _is_cancelled(self, transfer_id: str | None) -> bool: + if transfer_id and transfer_id in self._cancelled_transfers: + return True + return False + def teardown_receive_destination(self): if self.receive_destination is None: self.allowed_identity_hashes = [] @@ -351,6 +376,13 @@ class RNCPHandler: pass while resource.status < RNS.Resource.COMPLETE: + if self._is_cancelled(transfer_id): + with contextlib.suppress(Exception): + link.teardown() + if transfer_id in self.active_transfers: + self.active_transfers[transfer_id]["status"] = "cancelled" + msg = "Transfer cancelled" + raise InterruptedError(msg) await asyncio.sleep(0.1) if resource.status > RNS.Resource.COMPLETE: msg = "File was not accepted by destination" @@ -455,51 +487,54 @@ class RNCPHandler: resource_status = "started" saved_filename = None + save_error = None + effective_save_path = ( + os.path.abspath(os.path.expanduser(save_path)) + if isinstance(save_path, str) and save_path.strip() + else self._default_fetch_save_dir() + ) def fetch_resource_concluded(resource): - nonlocal resource_resolved, resource_status, saved_filename - if resource.status == RNS.Resource.COMPLETE: - if resource.metadata: - try: - filename = os.path.basename( - resource.metadata["name"].decode("utf-8"), - ) - if save_path: - save_dir = os.path.abspath(os.path.expanduser(save_path)) + nonlocal resource_resolved, resource_status, saved_filename, save_error + try: + if resource.status == RNS.Resource.COMPLETE: + if resource.metadata: + try: + filename = os.path.basename( + resource.metadata["name"].decode("utf-8"), + ) + save_dir = effective_save_path os.makedirs(save_dir, exist_ok=True) saved_filename = os.path.join(save_dir, filename) - else: - saved_filename = filename - counter = 0 - if allow_overwrite: - if os.path.isfile(saved_filename): - try: - os.unlink(saved_filename) - except OSError: - # Failed to delete existing file, which is fine, - # we'll just fall through to the naming loop - pass + counter = 0 + if allow_overwrite: + if os.path.isfile(saved_filename): + try: + os.unlink(saved_filename) + except OSError: + pass - while os.path.isfile(saved_filename): - counter += 1 - base, ext = os.path.splitext(filename) - saved_filename = os.path.join( - os.path.dirname(saved_filename) if save_path else ".", - f"{base}.{counter}{ext}", - ) + while os.path.isfile(saved_filename): + counter += 1 + base, ext = os.path.splitext(filename) + saved_filename = os.path.join( + save_dir, + f"{base}.{counter}{ext}", + ) - shutil.move(resource.data.name, saved_filename) - resource_status = "completed" - except Exception as e: + shutil.move(resource.data.name, saved_filename) + resource_status = "completed" + except Exception as e: + resource_status = "error" + save_error = str(e) + else: resource_status = "error" - raise e + save_error = "missing resource metadata" else: - resource_status = "error" - else: - resource_status = "failed" - - resource_resolved = True + resource_status = "failed" + finally: + resource_resolved = True link.set_resource_strategy(RNS.Link.ACCEPT_ALL) link.set_resource_started_callback(fetch_resource_started) @@ -532,6 +567,13 @@ class RNCPHandler: raise Exception(msg) while not resource_resolved: + if current_resource is not None and hasattr(current_resource, "hash"): + tid = getattr(current_resource, "hash", None) + if tid is not None and self._is_cancelled(tid.hex()): + with contextlib.suppress(Exception): + link.teardown() + msg = "Transfer cancelled" + raise InterruptedError(msg) await asyncio.sleep(0.1) if resource_status == "completed": @@ -541,7 +583,10 @@ class RNCPHandler: "file_path": saved_filename, } link.teardown() - msg = f"Transfer failed: {resource_status}" + if save_error: + msg = f"Transfer failed: {resource_status}: {save_error}" + else: + msg = f"Transfer failed: {resource_status}" raise Exception(msg) def get_transfer_status(self, transfer_id: str): diff --git a/meshchatx/src/backend/rngit_sparse_fetcher.py b/meshchatx/src/backend/rngit_sparse_fetcher.py new file mode 100644 index 00000000..d043f110 --- /dev/null +++ b/meshchatx/src/backend/rngit_sparse_fetcher.py @@ -0,0 +1,268 @@ +# SPDX-License-Identifier: 0BSD + +"""Sparse fetch of specific files from an RNGit ``rns://`` repository.""" + +from __future__ import annotations + +import asyncio +import os +import shutil +import signal +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +from meshchatx.src.backend.map_overlay_sources import is_commit_like_ref + + +class RngitFetchError(RuntimeError): + def __init__(self, code: str, message: str | None = None): + self.code = code + super().__init__(message or code) + + +@dataclass +class RngitFetchResult: + files: dict[str, bytes] + resolved_ref: str + + +def tools_available( + *, + which: Callable[[str], str | None] | None = None, +) -> tuple[bool, str | None]: + finder = which or shutil.which + if not finder("git"): + return False, "git_missing" + if not finder("git-remote-rns"): + return False, "git_remote_rns_missing" + return True, None + + +async def _run_git( + args: list[str], + *, + cwd: str | None, + env: dict[str, str], + timeout: float, + processes: list, +) -> tuple[int, bytes, bytes]: + proc = await asyncio.create_subprocess_exec( + *args, + cwd=cwd, + env=env, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + start_new_session=True, + ) + processes.append(proc) + try: + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout) + except TimeoutError as exc: + _kill_process_group(proc) + raise RngitFetchError("git_timeout") from exc + return proc.returncode or 0, stdout or b"", stderr or b"" + + +def _kill_process_group(proc) -> None: + try: + if proc.returncode is None: + os.killpg(proc.pid, signal.SIGTERM) + except Exception: + try: + proc.kill() + except Exception: + pass + + +class RngitSparseFetcher: + def __init__( + self, + *, + work_root: str, + reticulum_config_dir: str | None, + rngit_config_dir: str | None = None, + which: Callable[[str], str | None] | None = None, + ): + self.work_root = work_root + self.reticulum_config_dir = reticulum_config_dir + self.rngit_config_dir = rngit_config_dir + self._which = which or shutil.which + self._cancelled = False + self._processes: list = [] + + def cancel(self) -> None: + self._cancelled = True + for proc in list(self._processes): + _kill_process_group(proc) + + def _check_cancelled(self) -> None: + if self._cancelled: + raise RngitFetchError("cancelled") + + def _build_env(self) -> dict[str, str]: + env = os.environ.copy() + if self.reticulum_config_dir: + env["RNS_CONFIG"] = self.reticulum_config_dir + if self.rngit_config_dir: + env["RNGIT_CONFIG"] = self.rngit_config_dir + return env + + async def fetch( + self, + *, + destination_hash: str, + group: str, + repository: str, + paths: list[str], + ref: str = "HEAD", + job_id: str, + timeout_seconds: float = 300.0, + on_phase: Callable[[str], None] | None = None, + ) -> RngitFetchResult: + ok, missing = tools_available(which=self._which) + if not ok: + raise RngitFetchError("rngit_tools_unavailable", missing) + + if not paths: + raise RngitFetchError("missing_paths") + + workdir = os.path.join(self.work_root, job_id) + if os.path.exists(workdir): + shutil.rmtree(workdir, ignore_errors=True) + os.makedirs(workdir, exist_ok=True) + + env = self._build_env() + remote = f"rns://{destination_hash}/{group}/{repository}" + deadline_budget = float(timeout_seconds) + + def emit(phase: str) -> None: + if on_phase: + try: + on_phase(phase) + except Exception: + pass + + try: + self._check_cancelled() + emit("cloning") + code, _out, err = await _run_git( + [ + "git", + "clone", + "--filter=blob:none", + "--sparse", + "--no-checkout", + remote, + workdir, + ], + cwd=None, + env=env, + timeout=deadline_budget, + processes=self._processes, + ) + if code != 0: + raise RngitFetchError( + "git_clone_failed", + err.decode("utf-8", errors="replace")[:500], + ) + + self._check_cancelled() + emit("sparse_checkout") + code, _out, err = await _run_git( + ["git", "sparse-checkout", "set", "--no-cone", "--", *paths], + cwd=workdir, + env=env, + timeout=min(60.0, deadline_budget), + processes=self._processes, + ) + if code != 0: + raise RngitFetchError( + "sparse_checkout_failed", + err.decode("utf-8", errors="replace")[:500], + ) + + self._check_cancelled() + emit("fetching_ref") + fetch_ref = ref if ref != "HEAD" else "HEAD" + if is_commit_like_ref(ref): + code, _out, err = await _run_git( + ["git", "fetch", "--depth", "1", "origin", ref], + cwd=workdir, + env=env, + timeout=deadline_budget, + processes=self._processes, + ) + if code != 0: + raise RngitFetchError( + "git_fetch_failed", + err.decode("utf-8", errors="replace")[:500], + ) + checkout_target = "FETCH_HEAD" + else: + code, _out, err = await _run_git( + ["git", "fetch", "--depth", "1", "origin", fetch_ref], + cwd=workdir, + env=env, + timeout=deadline_budget, + processes=self._processes, + ) + if code != 0 and fetch_ref != "HEAD": + raise RngitFetchError( + "git_fetch_failed", + err.decode("utf-8", errors="replace")[:500], + ) + checkout_target = "FETCH_HEAD" if code == 0 else "HEAD" + + self._check_cancelled() + emit("checking_out") + code, _out, err = await _run_git( + ["git", "checkout", checkout_target, "--", *paths], + cwd=workdir, + env=env, + timeout=min(120.0, deadline_budget), + processes=self._processes, + ) + if code != 0: + # Fallback: checkout tree then ensure paths exist + code2, _out2, err2 = await _run_git( + ["git", "checkout", checkout_target], + cwd=workdir, + env=env, + timeout=min(120.0, deadline_budget), + processes=self._processes, + ) + if code2 != 0: + raise RngitFetchError( + "git_checkout_failed", + (err or err2).decode("utf-8", errors="replace")[:500], + ) + + code, out, err = await _run_git( + ["git", "rev-parse", "HEAD"], + cwd=workdir, + env=env, + timeout=30.0, + processes=self._processes, + ) + if code != 0: + raise RngitFetchError("rev_parse_failed") + resolved = out.decode("utf-8", errors="replace").strip() + + files: dict[str, bytes] = {} + root = Path(workdir) + for rel in paths: + abs_path = (root / rel).resolve() + try: + abs_path.relative_to(root.resolve()) + except ValueError as exc: + raise RngitFetchError("path_traversal") from exc + if not abs_path.is_file(): + raise RngitFetchError("path_missing", rel) + files[rel] = abs_path.read_bytes() + + emit("done") + return RngitFetchResult(files=files, resolved_ref=resolved) + finally: + shutil.rmtree(workdir, ignore_errors=True) + self._processes.clear() diff --git a/meshchatx/src/backend/rnpath_trace_handler.py b/meshchatx/src/backend/rnpath_trace_handler.py index 10955d0b..7d7405d9 100644 --- a/meshchatx/src/backend/rnpath_trace_handler.py +++ b/meshchatx/src/backend/rnpath_trace_handler.py @@ -62,7 +62,17 @@ class RNPathTraceHandler: path.append({"type": "local", "hash": local_hash, "name": "Local Node"}) - if hops == 1: + if hops == 0: + path.append( + { + "type": "destination", + "hash": destination_hash_str, + "hops": 0, + "interface": next_hop_interface, + "name": "Local destination", + }, + ) + elif hops == 1: # Direct path.append( { diff --git a/meshchatx/src/backend/self_check.py b/meshchatx/src/backend/self_check.py index 09212356..e63536ca 100644 --- a/meshchatx/src/backend/self_check.py +++ b/meshchatx/src/backend/self_check.py @@ -706,7 +706,7 @@ def check_plugins_runtime(app: Any) -> dict[str, str]: plugins_enabled = bool(getattr(app, "plugins_enabled", True)) if not plugins_enabled: return _status(True) - bundled_id = "com.meshchatx.mesh-observatory" + bundled_id = "com.meshchatx.mcx-bugs" if any(isinstance(item, dict) and item.get("id") == bundled_id for item in plugins): return _status(True) try: diff --git a/meshchatx/src/frontend/components/App.vue b/meshchatx/src/frontend/components/App.vue index 97159768..72191005 100644 --- a/meshchatx/src/frontend/components/App.vue +++ b/meshchatx/src/frontend/components/App.vue @@ -17,8 +17,15 @@ :view-backend-logs-label="$t('app.view_backend_logs')" :show-ws-reconnected="wsReconnectedBanner" :ws-reconnected-label="$t('app.backend_reconnected')" + :show-network-degraded="showNetworkDegradedBanner" + :network-degraded-label="networkDegradedBannerLabel" + :network-recovering="networkRecovering" + :recover-network-label="$t('app.recover_network')" + :open-interfaces-label="$t('app.open_interfaces')" @restart-backend="onRestartBackend" @view-backend-logs="onViewBackendCrashReport" + @recover-network="onRecoverNetwork" + @open-interfaces="onOpenInterfacesForRecovery" /> @@ -425,22 +432,24 @@
- +
@@ -676,6 +685,8 @@ export default { backendProcessExited: false, backendExitCode: null, backendRestarting: false, + networkRecovering: false, + userInitiatedPropagationSync: false, identitySwitchDedupeHash: null, identitySwitchDedupeAt: 0, @@ -705,6 +716,11 @@ export default { return listNavItems().filter((item) => this.isNavItemVisible(item)); }, isSyncingPropagationNode() { + // Only treat sync as "running" in the chrome when the user started it. + // Background auto-sync must not keep the header spinner forever. + if (!this.userInitiatedPropagationSync) { + return false; + } return [ "path_requested", "link_establishing", @@ -738,6 +754,16 @@ export default { typeof window.electron?.restartBackend === "function" ); }, + showNetworkDegradedBanner() { + return Boolean(GlobalState.networkDegraded) && this.$route?.name !== "auth"; + }, + networkDegradedBannerLabel() { + const detail = GlobalState.networkDegradedError; + if (detail) { + return `${this.$t("app.network_degraded")}: ${detail}`; + } + return this.$t("app.network_degraded"); + }, identitySidebarLabel() { const raw = this.displayName; const name = raw != null && String(raw).trim() !== "" ? String(raw).trim() : ""; @@ -1013,6 +1039,34 @@ export default { this.backendRestarting = false; } }, + onOpenInterfacesForRecovery() { + this.$router.push({ name: "interfaces" }); + }, + async onRecoverNetwork() { + if (this.networkRecovering) { + return; + } + this.networkRecovering = true; + try { + const response = await window.api.post("/api/v1/reticulum/recover", {}); + if (response.data?.status?.network_ready) { + GlobalState.networkDegraded = false; + GlobalState.networkDegradedError = null; + ToastUtils.success(response.data.message || this.$t("app.network_recovered")); + return; + } + const err = response.data?.error || response.data?.message || this.$t("app.network_recover_failed"); + GlobalState.networkDegradedError = err; + ToastUtils.error(err); + } catch (e) { + const err = + e.response?.data?.error || e.response?.data?.message || this.$t("app.network_recover_failed"); + GlobalState.networkDegradedError = err; + ToastUtils.error(err); + } finally { + this.networkRecovering = false; + } + }, async onViewBackendCrashReport() { if (!window.electron?.openBackendCrashReport) { return; @@ -1594,6 +1648,8 @@ export default { return; } + this.userInitiatedPropagationSync = true; + // request sync try { const preferredHash = this.config?.lxmf_preferred_propagation_node_destination_hash; @@ -1602,6 +1658,7 @@ export default { } await window.api.get("/api/v1/lxmf/propagation-node/sync"); } catch (e) { + this.userInitiatedPropagationSync = false; const errorMessage = e.response?.data?.message ?? this.$t("app.sync_error_generic"); ToastUtils.error(errorMessage); return; @@ -1626,6 +1683,7 @@ export default { this._propagationSyncPollTimer = null; } await this.stopSyncingPropagationNode(); + this.userInitiatedPropagationSync = false; ToastUtils.error( this.$t("app.sync_error", { status: this.propagationSyncStatusLabel("path_timeout"), @@ -1640,6 +1698,7 @@ export default { clearInterval(this._propagationSyncPollTimer); this._propagationSyncPollTimer = null; } + this.userInitiatedPropagationSync = false; ToastUtils.dismiss(propagationSyncToastKey); const status = this.propagationNodeStatus?.state; const messagesReceived = this.propagationNodeStatus?.messages_received ?? 0; @@ -1665,6 +1724,8 @@ export default { if (this.isSyncingPropagationNode) { ToastUtils.loading(this.propagationSyncLiveToastMessage(), 0, propagationSyncToastKey); this._propagationSyncPollTimer = setInterval(poll, 500); + } else { + this.userInitiatedPropagationSync = false; } await poll(); }, @@ -1697,6 +1758,7 @@ export default { } // Clear the polling guard flag this._isPropagationSyncPolling = false; + this.userInitiatedPropagationSync = false; ToastUtils.dismiss(propagationSyncToastKey); await this.updatePropagationNodeStatus(); }, @@ -1704,6 +1766,21 @@ export default { try { const response = await window.api.get("/api/v1/lxmf/propagation-node/status"); this.propagationNodeStatus = response.data.propagation_node_status; + const state = this.propagationNodeStatus?.state; + if ( + this.userInitiatedPropagationSync && + state && + ![ + "path_requested", + "link_establishing", + "link_established", + "request_sent", + "receiving", + "response_received", + ].includes(state) + ) { + this.userInitiatedPropagationSync = false; + } } catch { // do nothing on error } diff --git a/meshchatx/src/frontend/components/TutorialModal.vue b/meshchatx/src/frontend/components/TutorialModal.vue index 4c0d7e2a..5912d460 100644 --- a/meshchatx/src/frontend/components/TutorialModal.vue +++ b/meshchatx/src/frontend/components/TutorialModal.vue @@ -281,7 +281,7 @@ @@ -294,7 +294,7 @@ ? 'border-blue-500 bg-blue-500/5' : 'border-gray-200 dark:border-zinc-700 hover:border-blue-400' " - @click="identityMode = 'new'" + @click="setIdentityMode('new')" >
@@ -314,7 +314,7 @@ ? 'border-blue-500 bg-blue-500/5' : 'border-gray-200 dark:border-zinc-700 hover:border-blue-400' " - @click="identityMode = 'import'" + @click="setIdentityMode('import')" >
@@ -341,9 +341,13 @@ v-if="identityMode === 'import'" class="space-y-3 pt-2 border-t border-gray-200 dark:border-zinc-800" > +

+ {{ $t("tutorial.identity_import_key_only_hint") }} +

-

+

@@ -758,7 +770,10 @@ v-if="showFooterContinue" type="button" class="tutorial-action-btn tutorial-action-btn-primary" - :disabled="currentStep === 2 && identityImportInProgress" + :disabled=" + (currentStep === 2 && identityImportInProgress) || + (currentStep === 2 && identityMode === 'import' && !hasIdentityImportInput) + " @click="handlePrimaryAction" > {{ $t("tutorial.next") }} @@ -768,6 +783,7 @@ v-else type="button" class="tutorial-action-btn tutorial-action-btn-success" + :disabled="finishingTutorial" @click="finishTutorial" > {{ $t("tutorial.finish_setup") }} @@ -1050,7 +1066,7 @@ @@ -1063,7 +1079,7 @@ ? 'border-blue-500 bg-blue-500/5' : 'border-gray-200 dark:border-zinc-700 hover:border-blue-400' " - @click="identityMode = 'new'" + @click="setIdentityMode('new')" >
@@ -1083,7 +1099,7 @@ ? 'border-blue-500 bg-blue-500/5' : 'border-gray-200 dark:border-zinc-700 hover:border-blue-400' " - @click="identityMode = 'import'" + @click="setIdentityMode('import')" >
@@ -1112,9 +1128,13 @@ v-if="identityMode === 'import'" class="space-y-4 pt-3 border-t border-gray-200 dark:border-zinc-800" > +

+ {{ $t("tutorial.identity_import_key_only_hint") }} +

-

+

@@ -1569,7 +1597,10 @@ v-if="showFooterContinue" type="button" class="tutorial-action-btn tutorial-action-btn-primary" - :disabled="currentStep === 2 && identityImportInProgress" + :disabled=" + (currentStep === 2 && identityImportInProgress) || + (currentStep === 2 && identityMode === 'import' && !hasIdentityImportInput) + " @click="handlePrimaryAction" > {{ $t("tutorial.continue") }} @@ -1579,6 +1610,7 @@ v-else type="button" class="tutorial-action-btn tutorial-action-btn-success" + :disabled="finishingTutorial" @click="finishTutorial" > {{ $t("tutorial.finish_setup") }} @@ -1620,6 +1652,7 @@ export default { identityImportError: "", identityImportedHash: null, originalIdentityHash: null, + finishingTutorial: false, interfaceAddedViaTutorial: false, connectionMode: null, addingLocal: false, @@ -1652,7 +1685,7 @@ export default { return "Anonymous Peer"; }, hasIdentityImportInput() { - return Boolean(this.identityImportFile || this.identityImportBase32.trim()); + return Boolean(this.identityImportFile || this.normalizeBase32(this.identityImportBase32)); }, showFooterContinue() { if (this.currentStep === 3) { @@ -1687,6 +1720,23 @@ export default { this.identityImportInProgress = false; this.identityImportedHash = null; this.originalIdentityHash = null; + this.finishingTutorial = false; + }, + setIdentityMode(mode) { + this.identityMode = mode; + this.identityImportError = ""; + if (mode === "new") { + this.identityImportFile = null; + this.identityImportBase32 = ""; + this.identityImportedHash = null; + } + }, + normalizeBase32(value) { + return String(value || "").replace(/\s+/g, ""); + }, + onIdentityImportBase32Input() { + this.identityImportedHash = null; + this.identityImportError = ""; }, async loadIdentitySetupDefaults() { try { @@ -1705,8 +1755,18 @@ export default { }, onIdentityImportFileChange(event) { const files = event?.target?.files; - this.identityImportFile = files?.[0] || null; + const file = files?.[0] || null; + this.identityImportedHash = null; this.identityImportError = ""; + if (file && file.size === 0) { + this.identityImportFile = null; + this.identityImportError = this.$t("tutorial.identity_import_empty_file"); + } else if (file && file.size > 65536) { + this.identityImportFile = null; + this.identityImportError = this.$t("tutorial.identity_import_file_too_large"); + } else { + this.identityImportFile = file; + } if (event?.target) { event.target.value = ""; } @@ -1723,7 +1783,7 @@ export default { return response.data?.identity?.hash || null; }, async importIdentityFromBase32(base32, displayName) { - const payload = { base32 }; + const payload = { base32: this.normalizeBase32(base32) }; if (displayName) { payload.display_name = displayName; } @@ -1761,7 +1821,7 @@ export default { importedHash = await this.importIdentityFromFile(this.identityImportFile, trimmedName); this.identityImportFile = null; } else { - importedHash = await this.importIdentityFromBase32(this.identityImportBase32.trim(), trimmedName); + importedHash = await this.importIdentityFromBase32(this.identityImportBase32, trimmedName); this.identityImportBase32 = ""; } if (!importedHash) { @@ -1956,20 +2016,41 @@ export default { } }, gotoAddInterface() { - if (!this.isPage) { - this.visible = false; - } - if (this.$router) { - this.$router.push({ path: "/interfaces/add" }); - } + void this.closeWithPendingImportGuard().then((closed) => { + if (!closed) { + return; + } + if (this.$router) { + this.$router.push({ path: "/interfaces/add" }); + } + }); }, gotoRoute(routeName) { + void this.closeWithPendingImportGuard().then((closed) => { + if (!closed) { + return; + } + if (this.$router) { + this.$router.push({ name: routeName }); + } + }); + }, + async closeWithPendingImportGuard() { + if (this.identityImportedHash && this.identityImportedHash !== this.originalIdentityHash) { + const activate = await DialogUtils.confirm(this.$t("tutorial.identity_import_pending_activate")); + if (activate) { + const activated = await this.activateImportedIdentity(); + if (!activated) { + return false; + } + } else { + ToastUtils.warning(this.$t("tutorial.identity_import_pending_kept")); + } + } if (!this.isPage) { this.visible = false; } - if (this.$router) { - this.$router.push({ name: routeName }); - } + return true; }, async handlePrimaryAction() { if (this.currentStep === 2) { @@ -1997,10 +2078,22 @@ export default { this.currentStep--; }, async skipTutorial() { - if (await DialogUtils.confirm(this.$t("tutorial.skip_confirm"))) { - this.visible = false; - this.markSeen(); + if (!(await DialogUtils.confirm(this.$t("tutorial.skip_confirm")))) { + return; } + if (this.identityImportedHash && this.identityImportedHash !== this.originalIdentityHash) { + const activate = await DialogUtils.confirm(this.$t("tutorial.identity_import_pending_activate")); + if (activate) { + const activated = await this.activateImportedIdentity(); + if (!activated) { + return; + } + } else { + ToastUtils.warning(this.$t("tutorial.identity_import_pending_kept")); + } + } + this.visible = false; + this.markSeen(); }, async markSeen() { if (this.markingSeen) return; @@ -2013,35 +2106,71 @@ export default { this.markingSeen = false; } }, - async finishTutorial() { - if (GlobalState.hasPendingInterfaceChanges) { - const reloaded = await this.reloadReticulum(); - if (!reloaded) { - return; - } + async activateImportedIdentity() { + if (!this.identityImportedHash) { + return true; } - if (this.identityImportedHash && this.identityImportedHash !== this.originalIdentityHash) { - try { - await window.api.post("/api/v1/identities/switch", { - identity_hash: this.identityImportedHash, - }); - if (this.originalIdentityHash) { + if (this.identityImportedHash === this.originalIdentityHash) { + return true; + } + try { + const response = await window.api.post("/api/v1/identities/switch", { + identity_hash: this.identityImportedHash, + }); + if (this.originalIdentityHash) { + try { await window.api.delete(`/api/v1/identities/${this.originalIdentityHash}`); + } catch (deleteError) { + console.error("Failed to delete default identity after import:", deleteError); + ToastUtils.warning(this.$t("tutorial.identity_default_delete_failed")); } - } catch (e) { - ToastUtils.error(e.response?.data?.message || this.$t("tutorial.identity_switch_failed")); - return; } + if (response?.data?.hotswapped === false) { + ToastUtils.info(this.$t("identities.switch_scheduled")); + setTimeout(() => { + window.location.reload(); + }, 1500); + } + this.identityImportedHash = null; + return true; + } catch (e) { + ToastUtils.error(e.response?.data?.message || this.$t("tutorial.identity_switch_failed")); + return false; } - await this.markSeen(); - this.visible = false; - if (this.interfaceAddedViaTutorial) { - ToastUtils.success(this.$t("tutorial.ready_finished")); + }, + async finishTutorial() { + if (this.finishingTutorial) { + return; + } + this.finishingTutorial = true; + try { + if (GlobalState.hasPendingInterfaceChanges) { + const reloaded = await this.reloadReticulum(); + if (!reloaded) { + return; + } + } + if (this.identityImportedHash && this.identityImportedHash !== this.originalIdentityHash) { + const activated = await this.activateImportedIdentity(); + if (!activated) { + return; + } + } + await this.markSeen(); + this.visible = false; + if (this.interfaceAddedViaTutorial) { + ToastUtils.success(this.$t("tutorial.ready_finished")); + } + } finally { + this.finishingTutorial = false; } }, async onVisibleUpdate(val) { if (!val) { - // if closed by clicking away or programmatically, mark as seen + // Closing without finish still marks seen, but warn if import was pending. + if (this.identityImportedHash && this.identityImportedHash !== this.originalIdentityHash) { + ToastUtils.warning(this.$t("tutorial.identity_import_pending_kept")); + } this.markSeen(); } }, diff --git a/meshchatx/src/frontend/components/about/AboutPage.vue b/meshchatx/src/frontend/components/about/AboutPage.vue index c76d48e8..4bd067e8 100644 --- a/meshchatx/src/frontend/components/about/AboutPage.vue +++ b/meshchatx/src/frontend/components/about/AboutPage.vue @@ -554,7 +554,7 @@
- {{ $t("about.dep_lxmfy_subtitle") }} + LXMFy
v{{ (appInfo.dependencies && appInfo.dependencies.lxmfy) || "unknown" }} @@ -574,7 +574,7 @@
- {{ $t("about.dep_lxmf_subtitle") }} + LXMF
v{{ appInfo.lxmf_version }} @@ -594,7 +594,7 @@
- {{ $t("about.dep_rns_subtitle") }} + RNS
diff --git a/meshchatx/src/frontend/components/forwarder/ForwarderPage.vue b/meshchatx/src/frontend/components/forwarder/ForwarderPage.vue index e789c6d9..450637a9 100644 --- a/meshchatx/src/frontend/components/forwarder/ForwarderPage.vue +++ b/meshchatx/src/frontend/components/forwarder/ForwarderPage.vue @@ -149,6 +149,7 @@ import MaterialDesignIcon from "../MaterialDesignIcon.vue"; import WebSocketConnection from "../../js/WebSocketConnection"; import DialogUtils from "../../js/DialogUtils"; +import ToastUtils from "../../js/ToastUtils"; import ToolsPageHeader from "../tools/ToolsPageHeader.vue"; export default { @@ -195,33 +196,51 @@ export default { }, addRule() { if (!this.newRule.forward_to_hash) return; - WebSocketConnection.send( + const hash = String(this.newRule.forward_to_hash || "").trim(); + if (hash.length !== 32 || !/^[0-9a-fA-F]+$/.test(hash)) { + ToastUtils.warning(this.$t("forwarder.invalid_hash")); + return; + } + const sent = WebSocketConnection.send( JSON.stringify({ type: "lxmf.forwarding.rule.add", - rule: { ...this.newRule }, + rule: { ...this.newRule, forward_to_hash: hash }, }) ); + if (sent === false) { + ToastUtils.error(this.$t("forwarder.send_failed")); + return; + } this.newRule.name = ""; this.newRule.forward_to_hash = ""; this.newRule.source_filter_hash = ""; + ToastUtils.success(this.$t("forwarder.rule_added")); }, async deleteRule(id) { if (await DialogUtils.confirm(this.$t("forwarder.delete_confirm"))) { - WebSocketConnection.send( + const sent = WebSocketConnection.send( JSON.stringify({ type: "lxmf.forwarding.rule.delete", id: id, }) ); + if (sent === false) { + ToastUtils.error(this.$t("forwarder.send_failed")); + return; + } + ToastUtils.success(this.$t("forwarder.rule_deleted")); } }, toggleRule(id) { - WebSocketConnection.send( + const sent = WebSocketConnection.send( JSON.stringify({ type: "lxmf.forwarding.rule.toggle", id: id, }) ); + if (sent === false) { + ToastUtils.error(this.$t("forwarder.send_failed")); + } }, }, }; diff --git a/meshchatx/src/frontend/components/interfaces/AddInterfacePage.vue b/meshchatx/src/frontend/components/interfaces/AddInterfacePage.vue index 0eb14c39..3c6c6ae1 100644 --- a/meshchatx/src/frontend/components/interfaces/AddInterfacePage.vue +++ b/meshchatx/src/frontend/components/interfaces/AddInterfacePage.vue @@ -2122,9 +2122,7 @@ export default { return this.reticulumInstance.enable_transport === true; }, hasExistingI2PInterface() { - return Object.values(this.existingInterfaces || {}).some( - (iface) => iface && iface.type === "I2PInterface" - ); + return Object.values(this.existingInterfaces || {}).some((iface) => iface && iface.type === "I2PInterface"); }, canAddI2PInterface() { if (this.isEditingInterface && this.newInterfaceType === "I2PInterface") { diff --git a/meshchatx/src/frontend/components/layout/AppShellBanners.vue b/meshchatx/src/frontend/components/layout/AppShellBanners.vue index f4f15d7a..89651039 100644 --- a/meshchatx/src/frontend/components/layout/AppShellBanners.vue +++ b/meshchatx/src/frontend/components/layout/AppShellBanners.vue @@ -45,6 +45,31 @@ > {{ wsReconnectedLabel }}
+
+

{{ networkDegradedLabel }}

+
+ + +
+
@@ -95,7 +120,27 @@ export default { type: String, default: "", }, + showNetworkDegraded: { + type: Boolean, + default: false, + }, + networkDegradedLabel: { + type: String, + default: "", + }, + networkRecovering: { + type: Boolean, + default: false, + }, + recoverNetworkLabel: { + type: String, + default: "", + }, + openInterfacesLabel: { + type: String, + default: "", + }, }, - emits: ["restart-backend", "view-backend-logs"], + emits: ["restart-backend", "view-backend-logs", "recover-network", "open-interfaces"], }; diff --git a/meshchatx/src/frontend/components/map/MapPage.vue b/meshchatx/src/frontend/components/map/MapPage.vue index e5f839fd..9392e89c 100644 --- a/meshchatx/src/frontend/components/map/MapPage.vue +++ b/meshchatx/src/frontend/components/map/MapPage.vue @@ -571,6 +571,14 @@ @export-kmz="exportVectorKmz" /> + +
@@ -1166,6 +1174,7 @@ import MapExportConfigPanel from "./internal/MapExportConfigPanel.vue"; import MapExportProgressPanel from "./internal/MapExportProgressPanel.vue"; import MapLoadingOverlay from "./internal/MapLoadingOverlay.vue"; import MapVectorExchangePanel from "./internal/MapVectorExchangePanel.vue"; +import MapRemoteOverlayPanel from "./internal/MapRemoteOverlayPanel.vue"; import { buildMeshchatMapUri, buildWebHashMapUrl } from "../../js/mapLinkUtils.js"; import { readGeoJsonToFeatures, writeFeaturesToGeoJson } from "../../js/mapExchange/geoJsonCodec.js"; import { readKmlToFeatures, writeFeaturesToKml } from "../../js/mapExchange/kmlCodec.js"; @@ -1203,6 +1212,7 @@ export default { MapExportProgressPanel, MapLoadingOverlay, MapVectorExchangePanel, + MapRemoteOverlayPanel, }, props: { embedded: { @@ -1300,6 +1310,10 @@ export default { showTileConnectivityBanner: false, tileConnectivityBannerTimer: null, + // remote overlay layers (id -> { source, layer }) + remoteOverlayLayers: {}, + remoteOverlayLoadGeneration: 0, + // drawing tools draw: null, modify: null, @@ -1636,6 +1650,9 @@ export default { settingsEl.style.willChange = ""; } if (this.map) { + for (const id of Object.keys(this.remoteOverlayLayers || {})) { + this.removeRemoteOverlayLayer(id); + } const v = this.map.getView(); if (v && typeof v.un === "function") { v.un("change:rotation", this.syncMapNorthIndicatorFromViewRotation); @@ -4449,6 +4466,162 @@ export default { ToastUtils.error(this.$t("map.vector_import_failed")); }, + onRemoteOverlayError(err) { + console.error(err); + ToastUtils.error(this.$t("map.remote_overlays_error")); + }, + + async onRemoteOverlaysChanged(overlays) { + if (!this.map) { + return; + } + const gen = ++this.remoteOverlayLoadGeneration; + const list = Array.isArray(overlays) ? overlays : []; + const keep = new Set(list.map((o) => String(o.id))); + for (const id of Object.keys(this.remoteOverlayLayers)) { + if (!keep.has(id)) { + this.removeRemoteOverlayLayer(id); + } + } + for (const overlay of list) { + if (gen !== this.remoteOverlayLoadGeneration) { + return; + } + const id = String(overlay.id); + const visible = Boolean(overlay.visible); + if (overlay.status !== "ready" || !overlay.format) { + const existing = this.remoteOverlayLayers[id]; + if (existing?.layer) { + existing.layer.setVisible(false); + } + continue; + } + try { + await this.ensureRemoteOverlayLayer(overlay); + if (gen !== this.remoteOverlayLoadGeneration) { + return; + } + const entry = this.remoteOverlayLayers[id]; + if (entry?.layer) { + entry.layer.setVisible(visible); + } + } catch (e) { + console.error(e); + } + } + }, + + removeRemoteOverlayLayer(id) { + const entry = this.remoteOverlayLayers[id]; + if (!entry) { + return; + } + if (this.map && entry.layer) { + this.map.removeLayer(entry.layer); + } + delete this.remoteOverlayLayers[id]; + }, + + async ensureRemoteOverlayLayer(overlay) { + const id = String(overlay.id); + const contentRes = await fetch(`/api/v1/map/overlays/${overlay.id}/content`, { + credentials: "same-origin", + }); + if (!contentRes.ok) { + throw new Error(`overlay content ${contentRes.status}`); + } + let features = []; + const fmt = overlay.format; + if (fmt === "kmz") { + const buf = await contentRes.arrayBuffer(); + features = await readKmzToFeatures(buf, "EPSG:3857"); + } else { + const text = await contentRes.text(); + if (fmt === "kml") { + features = readKmlToFeatures(text, "EPSG:3857"); + } else { + features = readGeoJsonToFeatures(text, "EPSG:3857"); + } + } + for (const f of features) { + f.set("type", "remote_overlay"); + f.set("overlay_id", overlay.id); + } + let entry = this.remoteOverlayLayers[id]; + if (!entry) { + const source = new VectorSource(); + const layer = new VectorLayer({ + source, + zIndex: 45, + opacity: 0.95, + }); + this.map.addLayer(layer); + entry = { source, layer, sha: overlay.content_sha256 }; + this.remoteOverlayLayers[id] = entry; + } + entry.source.clear(); + entry.source.addFeatures(features); + entry.sha = overlay.content_sha256; + }, + + async onRemoteOverlayExport({ id, format }) { + try { + const res = await fetch(`/api/v1/map/overlays/${id}/export?format=${encodeURIComponent(format)}`, { + credentials: "same-origin", + }); + if (!res.ok) { + throw new Error(`export ${res.status}`); + } + const blob = await res.blob(); + const cd = res.headers.get("Content-Disposition") || ""; + const match = /filename="([^"]+)"/.exec(cd); + const name = match?.[1] || `overlay-${id}.${format}`; + this.downloadBlobFile(name, blob, blob.type || "application/octet-stream"); + ToastUtils.success(this.$t("map.remote_overlays_export_ok")); + } catch (e) { + console.error(e); + ToastUtils.error(this.$t("map.remote_overlays_export_failed")); + } + }, + + async onRemoteOverlayCopyToDrawings(overlay) { + if (!this.drawSource || !overlay?.id) { + return; + } + try { + const contentRes = await fetch(`/api/v1/map/overlays/${overlay.id}/content`, { + credentials: "same-origin", + }); + if (!contentRes.ok) { + throw new Error(`overlay content ${contentRes.status}`); + } + let features = []; + const fmt = overlay.format; + if (fmt === "kmz") { + const buf = await contentRes.arrayBuffer(); + features = await readKmzToFeatures(buf, "EPSG:3857"); + } else { + const text = await contentRes.text(); + if (fmt === "kml") { + features = readKmlToFeatures(text, "EPSG:3857"); + } else { + features = readGeoJsonToFeatures(text, "EPSG:3857"); + } + } + for (const f of features) { + f.set("type", "draw"); + f.unset("overlay_id"); + } + this.drawSource.addFeatures(features); + this.rebuildMeasurementOverlays(); + this.saveMapState(); + ToastUtils.success(this.$t("map.remote_overlays_copied")); + } catch (e) { + console.error(e); + ToastUtils.error(this.$t("map.remote_overlays_error")); + } + }, + onMapDragOver(ev) { if (ev.dataTransfer && ev.dataTransfer.types.includes("Files")) { this.isMapDropTarget = true; diff --git a/meshchatx/src/frontend/components/map/internal/MapRemoteOverlayPanel.vue b/meshchatx/src/frontend/components/map/internal/MapRemoteOverlayPanel.vue new file mode 100644 index 00000000..5eb6efae --- /dev/null +++ b/meshchatx/src/frontend/components/map/internal/MapRemoteOverlayPanel.vue @@ -0,0 +1,295 @@ + + +