diff --git a/Dockerfile b/Dockerfile index cd909dc0..817a4c21 100644 --- a/Dockerfile +++ b/Dockerfile @@ -67,6 +67,7 @@ COPY vendor ./vendor COPY scripts/docker-bake-lxst-filterlib-musl.py ./scripts/docker-bake-lxst-filterlib-musl.py COPY scripts/patch_lxst_pyogg_ogg_ctypes.py ./scripts/patch_lxst_pyogg_ogg_ctypes.py COPY scripts/patch_lxst_codec2_optional.py ./scripts/patch_lxst_codec2_optional.py +COPY meshchatx/src/backend/lxst_pyogg_ctypes_compat.py ./meshchatx/src/backend/lxst_pyogg_ctypes_compat.py # Third-party deps layer: stable across app-only updates when uv.lock is unchanged. # --inexact keeps setuptools/jaraco.context already in the venv (cffi bake needs them). RUN uv sync --no-group dev --no-install-project --inexact && \ diff --git a/Dockerfile.hardened b/Dockerfile.hardened index bc34cff5..c319f6cc 100644 --- a/Dockerfile.hardened +++ b/Dockerfile.hardened @@ -58,6 +58,7 @@ COPY logo ./logo COPY vendor ./vendor COPY scripts/patch_lxst_pyogg_ogg_ctypes.py ./scripts/patch_lxst_pyogg_ogg_ctypes.py COPY scripts/patch_lxst_codec2_optional.py ./scripts/patch_lxst_codec2_optional.py +COPY meshchatx/src/backend/lxst_pyogg_ctypes_compat.py ./meshchatx/src/backend/lxst_pyogg_ctypes_compat.py # Third-party deps layer: stable across app-only updates when uv.lock is unchanged. # --inexact keeps setuptools/jaraco.context already in the venv (cffi needs them). RUN uv sync --no-group dev --no-install-project --inexact && \ diff --git a/docs/en/installation.md b/docs/en/installation.md index cfe7cff3..d5f2b944 100644 --- a/docs/en/installation.md +++ b/docs/en/installation.md @@ -72,6 +72,8 @@ meshchatx --headless --host 127.0.0.1 The `meshchat` command is a compatibility alias for the same entry point. +On hosts where `libopus` is installed but `libogg` is not, LXST's vendored pyogg can raise `NameError: c_int_p` on import. MeshChatX applies a ctypes compatibility fix at startup (the same patch Docker runs after install). Optional telephony audio still needs the usual Opus/Ogg system libraries when you use those codecs. + ## Linux AppImage and packages **AppImage** diff --git a/meshchatx.rsm b/meshchatx.rsm index f30515c8..e5f7ea6a 100644 Binary files a/meshchatx.rsm and b/meshchatx.rsm differ diff --git a/meshchatx/__init__.py b/meshchatx/__init__.py index 703f9447..c28f1b82 100644 --- a/meshchatx/__init__.py +++ b/meshchatx/__init__.py @@ -4,3 +4,14 @@ # Synced from package.json via scripts/sync_version.js (also writes meshchatx/src/version.py). __version__ = "4.8.0" + +# LXST vendored pyogg can NameError on import when libopus is present but +# libogg is not. Apply before any meshchatx module imports LXST. +try: + from meshchatx.src.backend.lxst_pyogg_ctypes_compat import ( + ensure_lxst_pyogg_ctypes_compat, + ) + + ensure_lxst_pyogg_ctypes_compat() +except (ImportError, OSError, ValueError): + pass diff --git a/meshchatx/meshchat.py b/meshchatx/meshchat.py index a805affc..3fce57cc 100644 --- a/meshchatx/meshchat.py +++ b/meshchatx/meshchat.py @@ -42,9 +42,12 @@ from urllib.parse import urlparse import aiohttp import bcrypt import LXMF -import LXST import psutil import RNS + +# meshchatx/__init__ already ensures pyogg ctypes aliases. Import LXST after +# that package init so plain pip installs do not crash without the Docker patch. +import LXST from aiohttp import WSCloseCode, WSMessage, WSMsgType, web from aiohttp_session import get_session from aiohttp_session import setup as setup_session diff --git a/meshchatx/src/backend/lxst_pyogg_ctypes_compat.py b/meshchatx/src/backend/lxst_pyogg_ctypes_compat.py new file mode 100644 index 00000000..32b86a45 --- /dev/null +++ b/meshchatx/src/backend/lxst_pyogg_ctypes_compat.py @@ -0,0 +1,204 @@ +# SPDX-License-Identifier: 0BSD +"""Ensure LXST vendored pyogg exports ctypes aliases Opus needs. + +Upstream ogg.py defines c_int_p and related names only inside +if PYOGG_OGG_AVAIL. When libopus loads but libogg does not, opus.py +still references those names and raises NameError on import. + +Docker and Taskfile run a post-install disk patch. Plain pip install +does not, so MeshChatX applies the same fix before importing LXST. +""" + +from __future__ import annotations + +import importlib.abc +import importlib.metadata +import importlib.util +import sys +from ctypes import POINTER, c_char_p, c_float, c_int, c_ubyte +from pathlib import Path + +_MARKER = "# meshchatx-lxst-pyogg-ctypes-compat (do not remove; idempotency)\n" + +_LEGACY_MARKER = ( + "# meshchatx-lxst-pyogg-ctypes-pointer-aliases (do not remove; idempotency)\n" +) + +_LEGACY_BLOCK = ( + _LEGACY_MARKER + + "c_int_p = POINTER(c_int)\n" + + "c_float_p = POINTER(c_float)\n" + + "c_uchar_p = POINTER(c_ubyte)\n" + + "c_char_p_p = POINTER(c_char_p)\n" +) + +_INSERT = ( + _MARKER + + "c_int_p = POINTER(c_int)\n" + + "c_float_p = POINTER(c_float)\n" + + "c_uchar_p = POINTER(c_ubyte)\n" + + "c_char_p_p = POINTER(c_char_p)\n" + + "c_uchar = c_ubyte\n" +) + +_NEEDLE = ( + "from ctypes import c_int, c_int8, c_int16, c_int32, c_int64, c_uint, c_uint8, " + "c_uint16, c_uint32, c_uint64, c_float, c_long, c_ulong, c_char, c_char_p, " + "c_ubyte, c_longlong, c_ulonglong, c_size_t, c_void_p, c_double, POINTER, " + "pointer, cast\n" +) + +_OGG_MODULE = "LXST.Codecs.libs.pyogg.ogg" +_HOOK_INSTALLED = False + +_ALIAS_VALUES = { + "c_int_p": POINTER(c_int), + "c_float_p": POINTER(c_float), + "c_uchar_p": POINTER(c_ubyte), + "c_char_p_p": POINTER(c_char_p), + "c_uchar": c_ubyte, +} + + +def find_ogg_py() -> Path | None: + """Locate LXST bundled pyogg ogg.py without importing LXST.""" + try: + dist = importlib.metadata.distribution("lxst") + except importlib.metadata.PackageNotFoundError: + return None + for rel in dist.files or (): + parts = rel.parts + if len(parts) >= 2 and parts[-2] == "pyogg" and parts[-1] == "ogg.py": + return Path(dist.locate_file(rel)) + return None + + +def inject_ctypes_aliases(ogg_module) -> list[str]: + """Set missing ctypes aliases on a loaded ogg module. Returns names set.""" + added: list[str] = [] + for name, value in _ALIAS_VALUES.items(): + if not hasattr(ogg_module, name): + setattr(ogg_module, name, value) + added.append(name) + return added + + +def apply_disk_patch(ogg: Path | None = None) -> str: + """Patch ogg.py on disk. Returns status token.""" + target = ogg if ogg is not None else find_ogg_py() + if target is None: + return "missing" + text = target.read_text(encoding="utf-8") + if _MARKER in text: + return "already" + if _LEGACY_BLOCK in text: + target.write_text(text.replace(_LEGACY_BLOCK, _INSERT, 1), encoding="utf-8") + return "upgraded" + if _NEEDLE not in text: + return "unexpected" + target.write_text(text.replace(_NEEDLE, _NEEDLE + _INSERT, 1), encoding="utf-8") + return "patched" + + +class _OggCompatLoader(importlib.abc.Loader): + """Load ogg.py then inject module-level ctypes aliases.""" + + def __init__(self, origin: Path): + self.origin = origin + + def create_module(self, spec): + return None + + def exec_module(self, module): + source = self.origin.read_text(encoding="utf-8") + code = compile(source, str(self.origin), "exec") + exec(code, module.__dict__) + inject_ctypes_aliases(module) + + +class _OggCompatFinder(importlib.abc.MetaPathFinder): + """Intercept LXST pyogg ogg import when disk patch is unavailable.""" + + def find_spec(self, fullname, path, target=None): + if fullname != _OGG_MODULE: + return None + if fullname in sys.modules: + return None + ogg = find_ogg_py() + if ogg is None or not ogg.is_file(): + return None + try: + text = ogg.read_text(encoding="utf-8") + except OSError: + return None + if _MARKER in text: + return None + loader = _OggCompatLoader(ogg) + return importlib.util.spec_from_file_location( + fullname, + ogg, + loader=loader, + submodule_search_locations=None, + ) + + +def install_import_hook() -> bool: + """Install a one-shot meta path finder for read-only site-packages.""" + global _HOOK_INSTALLED + if _HOOK_INSTALLED: + return False + for finder in sys.meta_path: + if isinstance(finder, _OggCompatFinder): + _HOOK_INSTALLED = True + return False + sys.meta_path.insert(0, _OggCompatFinder()) + _HOOK_INSTALLED = True + return True + + +def ensure_lxst_pyogg_ctypes_compat() -> str: + """Disk-patch when possible and install an import hook as fallback. + + Safe to call repeatedly. Does not import LXST. + """ + status = "skipped" + try: + status = apply_disk_patch() + except OSError: + status = "readonly" + install_import_hook() + existing = sys.modules.get(_OGG_MODULE) + if existing is not None: + inject_ctypes_aliases(existing) + return status + + +def patch_cli() -> int: + """CLI used by scripts/patch_lxst_pyogg_ogg_ctypes.py.""" + try: + status = apply_disk_patch() + except OSError as exc: + print(f"patch_lxst_pyogg_ogg_ctypes: write failed ({exc})", file=sys.stderr) + return 1 + ogg = find_ogg_py() + if status == "missing": + print( + "patch_lxst_pyogg_ogg_ctypes: lxst not installed, nothing to do", + file=sys.stderr, + ) + return 0 + if status == "already": + print(f"patch_lxst_pyogg_ogg_ctypes: already applied ({ogg})") + return 0 + if status == "upgraded": + print(f"patch_lxst_pyogg_ogg_ctypes: upgraded legacy block ({ogg})") + return 0 + if status == "unexpected": + print( + "patch_lxst_pyogg_ogg_ctypes: unexpected ogg.py layout; " + f"manual check required ({ogg})", + file=sys.stderr, + ) + return 1 + print(f"patch_lxst_pyogg_ogg_ctypes: patched {ogg}") + return 0 diff --git a/scripts/patch_lxst_pyogg_ogg_ctypes.py b/scripts/patch_lxst_pyogg_ogg_ctypes.py index fd2af120..5d575f01 100644 --- a/scripts/patch_lxst_pyogg_ogg_ctypes.py +++ b/scripts/patch_lxst_pyogg_ogg_ctypes.py @@ -1,95 +1,46 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: 0BSD -"""Patch LXST bundled pyogg ogg.py for Python 3.14+. +"""Patch LXST bundled pyogg ogg.py for missing ctypes aliases. opus.py does from .ogg import * but needs extra ctypes names that -ogg.py never defined: POINTER aliases, and c_uchar (used as c_uchar*0 -for flexible array argtypes; same layout as c_ubyte). +ogg.py only defines when libogg loads. Idempotent: safe after every +pip install / poetry install / uv sync. -Idempotent: safe to run after every pip install / poetry install. +MeshChatX also applies this at package import for plain PyPI installs. +This script remains for Docker, CI, and packaging (including the Docker +deps layer before the project package is installed). """ from __future__ import annotations -import importlib.metadata -import sys +import importlib.util from pathlib import Path -_MARKER = "# meshchatx-lxst-pyogg-ctypes-compat (do not remove; idempotency)\n" -# First revision (older Docker/local installs): bump handled below. -_LEGACY_MARKER = ( - "# meshchatx-lxst-pyogg-ctypes-pointer-aliases (do not remove; idempotency)\n" -) - -_LEGACY_BLOCK = ( - _LEGACY_MARKER - + "c_int_p = POINTER(c_int)\n" - + "c_float_p = POINTER(c_float)\n" - + "c_uchar_p = POINTER(c_ubyte)\n" - + "c_char_p_p = POINTER(c_char_p)\n" -) - -_INSERT = ( - _MARKER - + "c_int_p = POINTER(c_int)\n" - + "c_float_p = POINTER(c_float)\n" - + "c_uchar_p = POINTER(c_ubyte)\n" - + "c_char_p_p = POINTER(c_char_p)\n" - + "c_uchar = c_ubyte\n" -) - -_NEEDLE = ( - "from ctypes import c_int, c_int8, c_int16, c_int32, c_int64, c_uint, c_uint8, " - "c_uint16, c_uint32, c_uint64, c_float, c_long, c_ulong, c_char, c_char_p, " - "c_ubyte, c_longlong, c_ulonglong, c_size_t, c_void_p, c_double, POINTER, " - "pointer, cast\n" -) - - -def _find_ogg_py() -> Path | None: +def _load_compat(): try: - dist = importlib.metadata.distribution("lxst") - except importlib.metadata.PackageNotFoundError: - return None - for rel in dist.files or (): - parts = rel.parts - if len(parts) >= 2 and parts[-2] == "pyogg" and parts[-1] == "ogg.py": - return Path(dist.locate_file(rel)) - return None + from meshchatx.src.backend.lxst_pyogg_ctypes_compat import patch_cli - -def main() -> int: - ogg = _find_ogg_py() - if ogg is None: - print( - "patch_lxst_pyogg_ogg_ctypes: lxst not installed, nothing to do", - file=sys.stderr, - ) - return 0 - text = ogg.read_text(encoding="utf-8") - - if _MARKER in text: - print(f"patch_lxst_pyogg_ogg_ctypes: already applied ({ogg})") - return 0 - - if _LEGACY_BLOCK in text: - ogg.write_text(text.replace(_LEGACY_BLOCK, _INSERT, 1), encoding="utf-8") - print(f"patch_lxst_pyogg_ogg_ctypes: upgraded legacy block ({ogg})") - return 0 - - if _NEEDLE not in text: - print( - "patch_lxst_pyogg_ogg_ctypes: unexpected ogg.py layout; " - f"manual check required ({ogg})", - file=sys.stderr, - ) - return 1 - - ogg.write_text(text.replace(_NEEDLE, _NEEDLE + _INSERT, 1), encoding="utf-8") - print(f"patch_lxst_pyogg_ogg_ctypes: patched {ogg}") - return 0 + return patch_cli + except ImportError: + pass + path = ( + Path(__file__).resolve().parents[1] + / "meshchatx" + / "src" + / "backend" + / "lxst_pyogg_ctypes_compat.py" + ) + spec = importlib.util.spec_from_file_location( + "lxst_pyogg_ctypes_compat", + path, + ) + if spec is None or spec.loader is None: + raise ImportError(f"cannot load compat module from {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.patch_cli if __name__ == "__main__": - raise SystemExit(main()) + raise SystemExit(_load_compat()()) diff --git a/tests/backend/test_lxst_pyogg_ctypes_compat.py b/tests/backend/test_lxst_pyogg_ctypes_compat.py new file mode 100644 index 00000000..168e9cd0 --- /dev/null +++ b/tests/backend/test_lxst_pyogg_ctypes_compat.py @@ -0,0 +1,125 @@ +# SPDX-License-Identifier: 0BSD +"""Tests for LXST pyogg ctypes alias compatibility.""" + +from __future__ import annotations + +import sys +import textwrap +from pathlib import Path + +import pytest + +from meshchatx.src.backend import lxst_pyogg_ctypes_compat as compat + + +_NEEDLE = compat._NEEDLE + + +def _ogg_source(*, with_marker: bool = False, with_legacy: bool = False) -> str: + body = _NEEDLE + if with_marker: + body += compat._INSERT + elif with_legacy: + body += compat._LEGACY_BLOCK + body += textwrap.dedent( + """ + libogg = None + PYOGG_OGG_AVAIL = False + """ + ) + return body + + +def test_apply_disk_patch_inserts_module_level_aliases(tmp_path: Path): + ogg = tmp_path / "ogg.py" + ogg.write_text(_ogg_source(), encoding="utf-8") + + assert compat.apply_disk_patch(ogg) == "patched" + text = ogg.read_text(encoding="utf-8") + assert compat._MARKER in text + assert "c_int_p = POINTER(c_int)" in text + assert "c_uchar = c_ubyte" in text + assert compat.apply_disk_patch(ogg) == "already" + + +def test_apply_disk_patch_upgrades_legacy_block(tmp_path: Path): + ogg = tmp_path / "ogg.py" + ogg.write_text(_ogg_source(with_legacy=True), encoding="utf-8") + + assert compat.apply_disk_patch(ogg) == "upgraded" + text = ogg.read_text(encoding="utf-8") + assert compat._MARKER in text + assert compat._LEGACY_MARKER not in text + assert "c_uchar = c_ubyte" in text + + +def test_apply_disk_patch_unexpected_layout(tmp_path: Path): + ogg = tmp_path / "ogg.py" + ogg.write_text("print('not pyogg')\n", encoding="utf-8") + assert compat.apply_disk_patch(ogg) == "unexpected" + + +def test_inject_ctypes_aliases_only_fills_missing(): + module = type(sys)("fake_ogg") + module.c_int_p = object() + added = compat.inject_ctypes_aliases(module) + assert "c_int_p" not in added + assert "c_float_p" in added + assert "c_uchar" in added + assert hasattr(module, "c_float_p") + assert hasattr(module, "c_uchar") + + +def test_import_hook_loader_injects_aliases(tmp_path: Path): + ogg = tmp_path / "ogg.py" + ogg.write_text(_ogg_source(), encoding="utf-8") + module = type(sys)("test_ogg_compat") + module.__file__ = str(ogg) + compat._OggCompatLoader(ogg).exec_module(module) + assert hasattr(module, "c_int_p") + assert hasattr(module, "c_uchar") + assert module.PYOGG_OGG_AVAIL is False + + +def test_finder_returns_spec_only_when_unpatched(tmp_path: Path, monkeypatch): + ogg = tmp_path / "ogg.py" + ogg.write_text(_ogg_source(), encoding="utf-8") + monkeypatch.setattr(compat, "find_ogg_py", lambda: ogg) + sys.modules.pop(compat._OGG_MODULE, None) + finder = compat._OggCompatFinder() + spec = finder.find_spec(compat._OGG_MODULE, None) + assert spec is not None + assert isinstance(spec.loader, compat._OggCompatLoader) + + ogg.write_text(_ogg_source(with_marker=True), encoding="utf-8") + assert finder.find_spec(compat._OGG_MODULE, None) is None + + +def test_ensure_prefers_disk_patch(tmp_path: Path, monkeypatch): + ogg = tmp_path / "ogg.py" + ogg.write_text(_ogg_source(), encoding="utf-8") + monkeypatch.setattr(compat, "find_ogg_py", lambda: ogg) + compat._HOOK_INSTALLED = False + sys.meta_path[:] = [ + f for f in sys.meta_path if not isinstance(f, compat._OggCompatFinder) + ] + + status = compat.ensure_lxst_pyogg_ctypes_compat() + assert status == "patched" + assert compat._MARKER in ogg.read_text(encoding="utf-8") + + +@pytest.mark.parametrize( + ("status", "expected_code"), + [ + ("missing", 0), + ("already", 0), + ("upgraded", 0), + ("patched", 0), + ("unexpected", 1), + ], +) +def test_patch_cli_exit_codes(monkeypatch, status, expected_code): + monkeypatch.setattr(compat, "apply_disk_patch", lambda: status) + monkeypatch.setattr(compat, "find_ogg_py", lambda: Path("/tmp/ogg.py")) + assert compat.patch_cli() == expected_code