diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 489cd71b..0fd43f08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -220,6 +220,9 @@ jobs: - name: Verify frozen backend import runtime run: bash scripts/ci/github-verify-frozen-runtime.sh build/exe + - name: Verify frozen pycodec2/libcodec2 layout + run: bash scripts/ci/github-verify-frozen-codec2.sh build/exe + - name: Verify frozen backend has no package bloat run: bash scripts/ci/verify-package-contents.sh frozen build/exe @@ -266,6 +269,8 @@ jobs: .artifacts/linux-build-check/build/exe bash scripts/ci/github-verify-frozen-runtime.sh \ .artifacts/linux-build-check/build/exe + bash scripts/ci/github-verify-frozen-codec2.sh \ + .artifacts/linux-build-check/build/exe bash scripts/ci/verify-package-contents.sh frozen \ .artifacts/linux-build-check/build/exe echo "Linux build artifact download + content validation passed." diff --git a/Taskfile.yml b/Taskfile.yml index 4f48ce5a..9f72c8a0 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -841,6 +841,11 @@ tasks: cmds: - bash scripts/ci/verify-package-contents-smoke.sh + verify:frozen:codec2: + desc: Fail if frozen pycodec2 cannot resolve libcodec2 + cmds: + - bash scripts/ci/github-verify-frozen-codec2.sh "{{.CLI_ARGS}}" + # --- Maintenance --- dist: diff --git a/cx_setup.py b/cx_setup.py index 05a0a035..3f67a1f0 100644 --- a/cx_setup.py +++ b/cx_setup.py @@ -61,6 +61,7 @@ packages = [ "RNS.Interfaces", "LXMF", "LXST", + "pycodec2", "lxmfy", "rns_filesync", "websockets", diff --git a/meshchatx.rsm b/meshchatx.rsm index 55c5017e..c7031905 100644 Binary files a/meshchatx.rsm and b/meshchatx.rsm differ diff --git a/meshchatx/src/backend/bake_frozen_pycodec2.py b/meshchatx/src/backend/bake_frozen_pycodec2.py new file mode 100644 index 00000000..b3b796f4 --- /dev/null +++ b/meshchatx/src/backend/bake_frozen_pycodec2.py @@ -0,0 +1,215 @@ +# SPDX-License-Identifier: 0BSD + +"""Bundle libcodec2 next to frozen pycodec2 so dyld/dlopen can find it. + +cx_Freeze rewrites Darwin load commands to @executable_path/lib/. +The published pycodec2 wheel ships libcodec2 under pycodec2/.dylibs/ with a +versioned id (libcodec2.1.2.dylib). The x64 sdist slice is copied as +libcodec2.dylib. If those layouts differ, unify-backend-plain-files.sh used +to delete both copies as arch-only Mach-O files, so the shipped .so still +named a dylib that was not in the app bundle. Import LXST then raised +ImportError at process start. +""" + +from __future__ import annotations + +import importlib.metadata +import shutil +import subprocess +import sys +from pathlib import Path + +_CANONICAL_DYLIB = "libcodec2.dylib" +_CANONICAL_SO = "libcodec2.so" +_CANONICAL_DLL = "libcodec2.dll" +_LOADER_PATH_DYLIB = "@loader_path/libcodec2.dylib" + + +def _pycodec2_dist_dir() -> Path | None: + """Return the installed pycodec2 package dir without importing the extension.""" + try: + dist = importlib.metadata.distribution("pycodec2") + except importlib.metadata.PackageNotFoundError: + return None + for rel in dist.files or (): + parts = rel.parts + if not parts or parts[0] != "pycodec2": + continue + located = Path(dist.locate_file(rel)) + if located.parent.name == "pycodec2": + return located.parent.resolve() + if located.name == "pycodec2" and located.is_dir(): + return located.resolve() + locate = Path(dist.locate_file("pycodec2")) + if locate.is_dir(): + return locate.resolve() + return None + + +def _is_codec2_lib(name: str) -> bool: + lower = name.lower() + if lower.startswith("libcodec2") or lower.startswith("codec2"): + return lower.endswith((".dylib", ".so", ".dll")) or ".so." in lower + return False + + +def _iter_codec2_libs(root: Path, *, recursive: bool = True) -> list[Path]: + found: list[Path] = [] + if not root.is_dir(): + return found + iterator = root.rglob("*") if recursive else root.iterdir() + for path in iterator: + if not path.is_file(): + continue + if _is_codec2_lib(path.name): + found.append(path) + return found + + +def _extension_module(pkg: Path) -> Path | None: + matches = sorted( + [ + path + for path in pkg.iterdir() + if path.is_file() + and path.name.startswith("pycodec2") + and path.suffix in {".so", ".pyd", ".dylib"} + ] + ) + return matches[0] if matches else None + + +def _copy_file(src: Path, dest: Path) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + if src.resolve() == dest.resolve(): + return + shutil.copy2(src, dest) + + +def _darwin_load_commands(ext_so: Path) -> list[str]: + try: + result = subprocess.run( + ["otool", "-L", str(ext_so)], + check=False, + capture_output=True, + text=True, + ) + except FileNotFoundError: + return [] + if result.returncode != 0: + return [] + refs: list[str] = [] + for line in result.stdout.splitlines()[1:]: + ref = line.strip().split(" ", 1)[0] + if "libcodec2" in ref or "/codec2" in ref: + refs.append(ref) + return refs + + +def _rewrite_darwin_extension(ext_so: Path, dylib: Path) -> None: + refs = _darwin_load_commands(ext_so) + for old_ref in refs: + if old_ref == _LOADER_PATH_DYLIB: + continue + subprocess.run( + ["install_name_tool", "-change", old_ref, _LOADER_PATH_DYLIB, str(ext_so)], + check=False, + ) + subprocess.run( + ["install_name_tool", "-id", _LOADER_PATH_DYLIB, str(dylib)], + check=False, + ) + subprocess.run( + ["codesign", "--force", "--sign", "-", str(ext_so), str(dylib)], + check=False, + capture_output=True, + ) + + +def _pick_source_lib(candidates: list[Path]) -> Path | None: + if not candidates: + return None + for path in candidates: + if path.name in {_CANONICAL_DYLIB, _CANONICAL_SO, _CANONICAL_DLL}: + return path + return candidates[0] + + +def bake_frozen_pycodec2(build_dir: Path) -> None: + """Copy libcodec2 into freeze-stable paths next to pycodec2 and under lib/.""" + lib_dir = build_dir / "lib" + pkg = lib_dir / "pycodec2" + if not pkg.is_dir(): + raise SystemExit(f"bake_frozen_pycodec2: missing {pkg}") + + ext_so = _extension_module(pkg) + if ext_so is None: + raise SystemExit(f"bake_frozen_pycodec2: no pycodec2 extension under {pkg}") + + candidates = _iter_codec2_libs(pkg) + _iter_codec2_libs(lib_dir, recursive=False) + dist_dir = _pycodec2_dist_dir() + if dist_dir is not None: + candidates.extend(_iter_codec2_libs(dist_dir)) + + unique: list[Path] = [] + seen: set[Path] = set() + for path in candidates: + resolved = path.resolve() + if resolved in seen: + continue + seen.add(resolved) + unique.append(path) + + source = _pick_source_lib(unique) + if source is None: + if sys.platform == "darwin": + raise SystemExit( + "bake_frozen_pycodec2: libcodec2 not found next to frozen pycodec2 " + "or in the build environment. The macOS app will crash on import LXST." + ) + print("bake_frozen_pycodec2: libcodec2 not bundled, skipping") + return + + if source.suffix == ".dll" or source.name.lower().endswith(".dll"): + canonical_name = _CANONICAL_DLL + elif source.suffix == ".dylib" or ".dylib" in source.name: + canonical_name = _CANONICAL_DYLIB + else: + canonical_name = _CANONICAL_SO + + dest_pkg = pkg / canonical_name + dest_lib = lib_dir / canonical_name + destinations = {dest_pkg, dest_lib} + if source.name != canonical_name: + destinations.add(lib_dir / source.name) + for ref in _darwin_load_commands(ext_so): + base = Path(ref).name + if _is_codec2_lib(base): + destinations.add(lib_dir / base) + destinations.add(pkg / base) + for dest in destinations: + _copy_file(source, dest) + + dylibs_dir = pkg / ".dylibs" + if dylibs_dir.is_dir(): + shutil.rmtree(dylibs_dir) + + if sys.platform == "darwin" and dest_pkg.suffix == ".dylib": + _rewrite_darwin_extension(ext_so, dest_pkg) + + print( + "bake_frozen_pycodec2: OK " + f"ext={ext_so.name} lib={canonical_name} src={source.name}" + ) + + +def main() -> None: + if len(sys.argv) != 2: + raise SystemExit( + "usage: python -m meshchatx.src.backend.bake_frozen_pycodec2 " + ) + bake_frozen_pycodec2(Path(sys.argv[1]).resolve()) + + +if __name__ == "__main__": + main() diff --git a/meshchatx/src/backend/frozen_freeze_probe.py b/meshchatx/src/backend/frozen_freeze_probe.py index b3ccc821..6151c2ce 100644 --- a/meshchatx/src/backend/frozen_freeze_probe.py +++ b/meshchatx/src/backend/frozen_freeze_probe.py @@ -40,11 +40,17 @@ def main() -> None: import email.policy # noqa: F401 import aiohttp # noqa: F401 + import LXST # noqa: F401 native = _lxst_filterlib_path() if native is None: raise SystemExit("frozen-freeze-probe: LXST filterlib native artifact missing") + if sys.platform == "darwin": + import pycodec2 + + pycodec2.Codec2(1600) + print("frozen-freeze-probe ok", flush=True) diff --git a/packaging/arch/PKGBUILD b/packaging/arch/PKGBUILD index 18775379..c4fb3984 100644 --- a/packaging/arch/PKGBUILD +++ b/packaging/arch/PKGBUILD @@ -36,6 +36,7 @@ prepare() { uv lock --check uv sync --group dev --active python scripts/patch_lxst_pyogg_ogg_ctypes.py + python scripts/patch_lxst_codec2_optional.py } build() { @@ -50,14 +51,13 @@ build() { package() { cd "$_pkgname" - + install -d "$pkgdir/usr/lib/$_pkgname" cp -a dist/linux-unpacked/* "$pkgdir/usr/lib/$_pkgname/" - + install -d "$pkgdir/usr/bin" ln -s "/usr/lib/$_pkgname/reticulum-meshchatx" "$pkgdir/usr/bin/$_pkgname" - + install -Dm644 "$srcdir/reticulum-meshchatx.desktop" "$pkgdir/usr/share/applications/reticulum-meshchatx.desktop" install -Dm644 logo/logo.png "$pkgdir/usr/share/icons/hicolor/512x512/apps/reticulum-meshchatx.png" } - diff --git a/scripts/build-backend.js b/scripts/build-backend.js index d55e3fba..b0b34081 100755 --- a/scripts/build-backend.js +++ b/scripts/build-backend.js @@ -342,6 +342,18 @@ try { env: env, }); failOnSpawnResult("LXST filterlib prune", bakeResult); + let codec2Args = [...bakeCmdParts.slice(1), "-m", "meshchatx.src.backend.bake_frozen_pycodec2", buildDir]; + let codec2SpawnCmd = bakeCmd; + if (rosettaX64) { + codec2SpawnCmd = "arch"; + codec2Args = ["-x86_64", bakeCmd, ...codec2Args]; + } + const codec2Result = spawnSync(codec2SpawnCmd, codec2Args, { + stdio: "inherit", + shell: false, + env: env, + }); + failOnSpawnResult("pycodec2 libcodec2 bake", codec2Result); if (!verifyBinaryArchitecture(buildDir, arch, targetName)) { process.exit(1); } diff --git a/scripts/build-macos-universal.sh b/scripts/build-macos-universal.sh index 1af5c9bf..fb7ffc03 100644 --- a/scripts/build-macos-universal.sh +++ b/scripts/build-macos-universal.sh @@ -60,4 +60,9 @@ bash scripts/thin-backend-mach-o.sh bash scripts/unify-backend-plain-files.sh +bash scripts/ci/github-verify-frozen-codec2.sh "$ROOT/build/exe/darwin-arm64" +bash scripts/ci/github-verify-frozen-codec2.sh "$ROOT/build/exe/darwin-x64" +bash scripts/ci/github-verify-frozen-runtime.sh "$ROOT/build/exe/darwin-arm64" +bash scripts/ci/github-verify-frozen-runtime.sh "$ROOT/build/exe/darwin-x64" + exec pnpm exec electron-builder --mac --universal --publish=never diff --git a/scripts/ci/github-build-macos.sh b/scripts/ci/github-build-macos.sh index 7968b20c..d412d89e 100755 --- a/scripts/ci/github-build-macos.sh +++ b/scripts/ci/github-build-macos.sh @@ -11,3 +11,5 @@ pnpm run dist:mac-universal bash scripts/ci/github-prune-electron-dist-staging.sh bash scripts/ci/github-verify-electron-dist.sh mac +bash scripts/ci/github-verify-frozen-codec2.sh build/exe/darwin-arm64 +bash scripts/ci/github-verify-frozen-codec2.sh build/exe/darwin-x64 diff --git a/scripts/ci/github-build-windows.sh b/scripts/ci/github-build-windows.sh index 5613979f..687be208 100755 --- a/scripts/ci/github-build-windows.sh +++ b/scripts/ci/github-build-windows.sh @@ -30,6 +30,7 @@ bash scripts/ci/github-verify-electron-dist.sh win if [[ -d build/exe ]]; then bash scripts/ci/github-verify-frozen-sandbox.sh build/exe bash scripts/ci/github-verify-frozen-runtime.sh build/exe + bash scripts/ci/github-verify-frozen-codec2.sh build/exe fi # Optional packaged smoke (manual / future CI job on a Windows runner): diff --git a/scripts/ci/github-install-deps.sh b/scripts/ci/github-install-deps.sh index 051d76e8..d74c478c 100755 --- a/scripts/ci/github-install-deps.sh +++ b/scripts/ci/github-install-deps.sh @@ -31,6 +31,7 @@ fi uv lock --check uv sync --group dev uv run python scripts/patch_lxst_pyogg_ogg_ctypes.py +uv run python scripts/patch_lxst_codec2_optional.py if [[ "$(uname -s)" == "Darwin" ]]; then uv run python -c " @@ -42,8 +43,7 @@ print('arm64 venv numpy', numpy.__version__, 'ok') # it to pycodec2/libcodec2.dylib so this tree's relative layout matches the # darwin-x64 slice built in scripts/ci/github-install-macos-x64-python-deps.sh, # which scripts/unify-backend-plain-files.sh requires to merge the two trees. - bash "$(dirname "$0")/macos-normalize-pycodec2-dylib.sh" "${ROOT}/.venv/bin/python" || - echo "github-install-deps: pycodec2 dylib normalization failed, continuing (unify-backend may drop it later)" >&2 + bash "$(dirname "$0")/macos-normalize-pycodec2-dylib.sh" "${ROOT}/.venv/bin/python" fi if [[ "$(uname -s)" == "Darwin" ]]; then diff --git a/scripts/ci/github-install-macos-x64-python-deps.sh b/scripts/ci/github-install-macos-x64-python-deps.sh index d6d672b6..f4733edc 100755 --- a/scripts/ci/github-install-macos-x64-python-deps.sh +++ b/scripts/ci/github-install-macos-x64-python-deps.sh @@ -145,7 +145,17 @@ uv pip uninstall --python "$_PY" Cython wheel # and matches the relative layout scripts/unify-backend-plain-files.sh expects when # reconciling this slice against the arm64 wheel's .dylibs/ bundle. if [[ -n "${_codec2:-}" ]]; then - _pycodec2_dir="$(arch -x86_64 "$_PY" -c 'import pathlib, pycodec2; print(pathlib.Path(pycodec2.__file__).resolve().parent)')" + _pycodec2_dir="$(arch -x86_64 "$_PY" -c ' +import importlib.metadata +from pathlib import Path +dist = importlib.metadata.distribution("pycodec2") +for rel in dist.files or (): + if rel.parts and rel.parts[0] == "pycodec2": + located = Path(dist.locate_file(rel)) + if located.parent.name == "pycodec2": + print(located.parent.resolve()) + break +')" for _lib in "${_codec2}/lib/libcodec2.dylib" "${_codec2}/lib/libcodec2.so"; do if [[ -f "$_lib" ]]; then cp -f "$_lib" "${_pycodec2_dir}/libcodec2.dylib" @@ -153,10 +163,10 @@ if [[ -n "${_codec2:-}" ]]; then fi done fi -arch -x86_64 bash "$(dirname "$0")/macos-normalize-pycodec2-dylib.sh" "$_PY" || - echo "github-install-macos-x64-python-deps: pycodec2 dylib normalization failed, continuing (unify-backend may drop it later)" >&2 +arch -x86_64 bash "$(dirname "$0")/macos-normalize-pycodec2-dylib.sh" "$_PY" arch -x86_64 "$_PY" scripts/patch_lxst_pyogg_ogg_ctypes.py +arch -x86_64 "$_PY" scripts/patch_lxst_codec2_optional.py arch -x86_64 "$_PY" -c " import importlib.metadata diff --git a/scripts/ci/github-verify-frozen-codec2.sh b/scripts/ci/github-verify-frozen-codec2.sh new file mode 100755 index 00000000..4874b897 --- /dev/null +++ b/scripts/ci/github-verify-frozen-codec2.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# Fail if frozen pycodec2 cannot resolve libcodec2. +# +# Usage: +# github-verify-frozen-codec2.sh [build/exe] +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +BUILD_EXE="${1:-${ROOT}/build/exe}" + +if [[ ! -d "${BUILD_EXE}" ]]; then + echo "frozen codec2 verify: cx_Freeze output not found at ${BUILD_EXE}" >&2 + exit 1 +fi + +if [[ ! -d "${BUILD_EXE}/lib" ]]; then + for sub in "${BUILD_EXE}"/*; do + if [[ -d "${sub}/lib" ]]; then + BUILD_EXE="${sub}" + break + fi + done +fi + +PKG="${BUILD_EXE}/lib/pycodec2" +if [[ ! -d "${PKG}" ]]; then + echo "frozen codec2 verify: missing ${PKG}" >&2 + exit 1 +fi + +shopt -s nullglob +exts=("${PKG}"/pycodec2*.so "${PKG}"/pycodec2*.pyd "${PKG}"/pycodec2*.dylib) +shopt -u nullglob +if [[ ${#exts[@]} -eq 0 ]]; then + echo "frozen codec2 verify: no pycodec2 extension under ${PKG}" >&2 + exit 1 +fi +EXT="${exts[0]}" + +shopt -s nullglob +candidates=( + "${PKG}"/libcodec2.dylib + "${PKG}"/libcodec2.so + "${PKG}"/libcodec2.dll + "${PKG}"/.dylibs/libcodec2*.dylib + "${BUILD_EXE}/lib"/libcodec2*.dylib + "${BUILD_EXE}/lib"/libcodec2.so* + "${BUILD_EXE}/lib"/libcodec2.dll +) +shopt -u nullglob + +libs=() +for _lib in "${candidates[@]}"; do + if [[ -f "${_lib}" ]]; then + libs+=("${_lib}") + fi +done + +if [[ "$(uname -s)" == "Darwin" && ${#libs[@]} -eq 0 ]]; then + echo "frozen codec2 verify: libcodec2 missing next to ${EXT}" >&2 + echo " expected lib/pycodec2/libcodec2.dylib or lib/libcodec2*.dylib" >&2 + exit 1 +fi + +resolve_dep() { + local dep="$1" + local so_dir + so_dir="$(cd "$(dirname "${EXT}")" && pwd)" + case "${dep}" in + @executable_path/*) + printf '%s\n' "${BUILD_EXE}/${dep#@executable_path/}" + ;; + @loader_path/*) + printf '%s\n' "${so_dir}/${dep#@loader_path/}" + ;; + @rpath/*) + printf '%s\n' "${BUILD_EXE}/lib/${dep#@rpath/}" + ;; + /*) + printf '%s\n' "${dep}" + ;; + *) + printf '%s\n' "${so_dir}/${dep}" + ;; + esac +} + +if [[ "$(uname -s)" == "Darwin" ]] && command -v otool >/dev/null 2>&1; then + while IFS= read -r dep; do + [[ -n "${dep}" ]] || continue + case "${dep}" in + /opt/homebrew/* | /usr/local/* | /opt/local/*) + echo "frozen codec2 verify: ${EXT} still links absolute ${dep}" >&2 + echo " rewrite to @loader_path/libcodec2.dylib before shipping" >&2 + exit 1 + ;; + esac + resolved="$(resolve_dep "${dep}")" + if [[ ! -f "${resolved}" ]]; then + echo "frozen codec2 verify: ${EXT} needs ${dep}" >&2 + echo " resolved missing path: ${resolved}" >&2 + exit 1 + fi + done < <(otool -L "${EXT}" | awk '/libcodec2/{print $1}') +fi + +if [[ ${#libs[@]} -gt 0 ]]; then + echo "frozen codec2 verify: OK (${EXT} + ${libs[0]})" +else + echo "frozen codec2 verify: OK (${EXT}, libcodec2 not bundled)" +fi diff --git a/scripts/ci/macos-normalize-pycodec2-dylib.sh b/scripts/ci/macos-normalize-pycodec2-dylib.sh index ffe1d952..1a4413eb 100755 --- a/scripts/ci/macos-normalize-pycodec2-dylib.sh +++ b/scripts/ci/macos-normalize-pycodec2-dylib.sh @@ -8,11 +8,9 @@ # Published macOS wheels bundle libcodec2 under pycodec2/.dylibs/libcodec2..dylib # (delocate convention). Building pycodec2 from sdist for architectures without a # published wheel (e.g. cp314 macOS x86_64) instead links against whatever Homebrew -# codec2 path was on LDFLAGS at build time. Without normalization the two cx_Freeze -# build trees (darwin-arm64 vs darwin-x64) have different relative file layouts for -# the same logical dependency, so scripts/unify-backend-plain-files.sh treats the -# bundled dylib as arch-only and drops it from BOTH trees, breaking Codec2 audio -# (voice messages) in the shipped universal macOS app. +# codec2 path was on LDFLAGS at build time. Without this layout both cx_Freeze +# slices disagree on the dylib path, and the shipped .so still names a file that +# is not in the app bundle. # # Usage: macos-normalize-pycodec2-dylib.sh set -euo pipefail @@ -24,7 +22,20 @@ if [[ "$(uname -s)" != "Darwin" ]]; then exit 0 fi -_site_dir="$("$PY" -c 'import pycodec2, pathlib; print(pathlib.Path(pycodec2.__file__).resolve().parent)' 2>/dev/null || true)" +_site_dir="$("$PY" -c ' +import importlib.metadata +from pathlib import Path +try: + dist = importlib.metadata.distribution("pycodec2") +except importlib.metadata.PackageNotFoundError: + raise SystemExit(0) +for rel in dist.files or (): + if rel.parts and rel.parts[0] == "pycodec2": + located = Path(dist.locate_file(rel)) + if located.parent.name == "pycodec2": + print(located.parent.resolve()) + break +')" if [[ -z "$_site_dir" || ! -d "$_site_dir" ]]; then echo "macos-normalize-pycodec2-dylib: pycodec2 not installed, skipping" >&2 exit 0 @@ -32,8 +43,8 @@ fi _ext_so="$(find "$_site_dir" -maxdepth 1 -name 'pycodec2*.so' -print -quit)" if [[ -z "$_ext_so" ]]; then - echo "macos-normalize-pycodec2-dylib: no pycodec2 extension module found under ${_site_dir}, skipping" >&2 - exit 0 + echo "macos-normalize-pycodec2-dylib: no pycodec2 extension module found under ${_site_dir}" >&2 + exit 1 fi _target="${_site_dir}/libcodec2.dylib" @@ -51,10 +62,21 @@ if [[ -z "$_src_dylib" ]]; then fi done fi +if [[ -z "$_src_dylib" ]] && command -v brew >/dev/null 2>&1; then + _prefix="$(brew --prefix codec2 2>/dev/null || true)" + if [[ -n "${_prefix}" ]]; then + for _candidate in "${_prefix}/lib"/libcodec2*.dylib "${_prefix}/lib"/libcodec2.dylib; do + if [[ -f "$_candidate" ]]; then + _src_dylib="$_candidate" + break + fi + done + fi +fi if [[ -z "$_src_dylib" ]]; then - echo "macos-normalize-pycodec2-dylib: no bundled libcodec2 found under ${_site_dir}, skipping" >&2 - exit 0 + echo "macos-normalize-pycodec2-dylib: no libcodec2 found for ${_site_dir}" >&2 + exit 1 fi if [[ "$_src_dylib" != "$_target" ]]; then @@ -67,11 +89,15 @@ if [[ -f "${_site_dir}/libcodec2.so" && "${_site_dir}/libcodec2.so" != "$_target rm -f "${_site_dir}/libcodec2.so" fi -_old_ref="$( (otool -L "$_ext_so" | awk '/libcodec2/{print $1; exit}') 2>/dev/null || true)" -if [[ -n "$_old_ref" && "$_old_ref" != "@loader_path/libcodec2.dylib" ]]; then +install_name_tool -id "@loader_path/libcodec2.dylib" "$_target" >/dev/null 2>&1 || true +while IFS= read -r _old_ref; do + [[ -n "$_old_ref" ]] || continue + if [[ "$_old_ref" == "@loader_path/libcodec2.dylib" ]]; then + continue + fi install_name_tool -change "$_old_ref" "@loader_path/libcodec2.dylib" "$_ext_so" - codesign --force --sign - "$_ext_so" >/dev/null 2>&1 || true echo "macos-normalize-pycodec2-dylib: rewrote ${_old_ref} -> @loader_path/libcodec2.dylib in $(basename "$_ext_so")" -fi +done < <(otool -L "$_ext_so" | awk '/libcodec2/{print $1}') +codesign --force --sign - "$_ext_so" "$_target" >/dev/null 2>&1 || true echo "macos-normalize-pycodec2-dylib: normalized $(basename "$_ext_so") + libcodec2.dylib under ${_site_dir}" diff --git a/scripts/pip-rns-deps.sh b/scripts/pip-rns-deps.sh index f9341379..c6353189 100755 --- a/scripts/pip-rns-deps.sh +++ b/scripts/pip-rns-deps.sh @@ -188,8 +188,10 @@ done if [[ "${DRY_RUN}" -eq 0 ]]; then if command -v uv >/dev/null 2>&1; then run_cmd uv run python scripts/patch_lxst_pyogg_ogg_ctypes.py + run_cmd uv run python scripts/patch_lxst_codec2_optional.py else run_cmd python3 scripts/patch_lxst_pyogg_ogg_ctypes.py + run_cmd python3 scripts/patch_lxst_codec2_optional.py fi fi diff --git a/scripts/unify-backend-plain-files.sh b/scripts/unify-backend-plain-files.sh index 364887f0..0ea195c1 100755 --- a/scripts/unify-backend-plain-files.sh +++ b/scripts/unify-backend-plain-files.sh @@ -25,6 +25,17 @@ synced=0 dropped=0 normalized=0 +required_native_rel() { + local base + base="$(basename "$1")" + case "$base" in + libcodec2* | pycodec2*.so | pycodec2*.dylib | pycodec2*.pyd) + return 0 + ;; + esac + return 1 +} + copy_missing() { local src_dir="$1" dst_dir="$2" label="$3" while IFS= read -r -d '' rel; do @@ -34,6 +45,14 @@ copy_missing() { local ft ft=$(file --brief --no-pad "$src_file" 2>/dev/null || true) if [[ "$ft" == Mach-O* ]]; then + if required_native_rel "$rel"; then + echo "unify-backend: ERROR: required native $rel exists only in $label" >&2 + echo " source reports: $ft" >&2 + echo " Both darwin-arm64 and darwin-x64 must ship libcodec2 next to pycodec2." >&2 + echo " Run scripts/ci/macos-normalize-pycodec2-dylib.sh on each venv and" >&2 + echo " meshchatx.src.backend.bake_frozen_pycodec2 after each cx_Freeze slice." >&2 + exit 1 + fi echo "unify-backend: dropping arch-only Mach-O for consistency: $rel" >&2 echo " ($label); source reports: $ft" >&2 echo " Hint: this native library/extension only exists in one arch's" >&2 diff --git a/tests/backend/test_bake_frozen_pycodec2.py b/tests/backend/test_bake_frozen_pycodec2.py new file mode 100644 index 00000000..22254083 --- /dev/null +++ b/tests/backend/test_bake_frozen_pycodec2.py @@ -0,0 +1,221 @@ +# SPDX-License-Identifier: 0BSD + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +import pytest + +from meshchatx.src.backend.bake_frozen_pycodec2 import ( + bake_frozen_pycodec2, + _is_codec2_lib, + _pick_source_lib, +) + +_VERIFY = Path("scripts/ci/github-verify-frozen-codec2.sh") + + +@pytest.fixture(autouse=True) +def _ignore_host_pycodec2(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "meshchatx.src.backend.bake_frozen_pycodec2._pycodec2_dist_dir", + lambda: None, + ) + + +def _frozen_pycodec2_tree(tmp_path: Path) -> Path: + pkg = tmp_path / "lib" / "pycodec2" + pkg.mkdir(parents=True) + (pkg / "pycodec2.cpython-314-darwin.so").write_bytes(b"so") + (pkg / "__init__.py").write_text("from .pycodec2 import *\n", encoding="utf-8") + return tmp_path + + +def test_is_codec2_lib_accepts_versioned_names() -> None: + assert _is_codec2_lib("libcodec2.1.2.dylib") is True + assert _is_codec2_lib("libcodec2.dylib") is True + assert _is_codec2_lib("libcodec2.so.1.2") is True + assert _is_codec2_lib("libcodec2.dll") is True + assert _is_codec2_lib("pycodec2.cpython-314-darwin.so") is False + assert _is_codec2_lib("libother.dylib") is False + + +def test_pick_source_lib_prefers_canonical_dylib_name(tmp_path: Path) -> None: + versioned = tmp_path / "libcodec2.1.2.dylib" + canonical = tmp_path / "libcodec2.dylib" + versioned.write_bytes(b"v") + canonical.write_bytes(b"c") + picked = _pick_source_lib([versioned, canonical]) + assert picked == canonical + + +def test_bake_copies_dylibs_layout_to_canonical_and_executable_path( + tmp_path: Path, +) -> None: + root = _frozen_pycodec2_tree(tmp_path) + pkg = root / "lib" / "pycodec2" + dylibs = pkg / ".dylibs" + dylibs.mkdir() + (dylibs / "libcodec2.1.2.dylib").write_bytes(b"codec2-bytes") + + bake_frozen_pycodec2(root) + + assert (pkg / "libcodec2.dylib").read_bytes() == b"codec2-bytes" + assert (root / "lib" / "libcodec2.dylib").read_bytes() == b"codec2-bytes" + assert (root / "lib" / "libcodec2.1.2.dylib").read_bytes() == b"codec2-bytes" + assert not dylibs.exists() + + +def test_bake_fails_when_libcodec2_missing_on_darwin( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + "meshchatx.src.backend.bake_frozen_pycodec2.sys.platform", "darwin" + ) + root = _frozen_pycodec2_tree(tmp_path) + with pytest.raises(SystemExit, match="libcodec2 not found"): + bake_frozen_pycodec2(root) + + +def test_bake_skips_missing_libcodec2_on_linux( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + "meshchatx.src.backend.bake_frozen_pycodec2.sys.platform", "linux" + ) + root = _frozen_pycodec2_tree(tmp_path) + bake_frozen_pycodec2(root) + assert not (root / "lib" / "pycodec2" / "libcodec2.dylib").exists() + + +def test_bake_fails_when_pycodec2_package_missing(tmp_path: Path) -> None: + (tmp_path / "lib").mkdir() + with pytest.raises(SystemExit, match="missing"): + bake_frozen_pycodec2(tmp_path) + + +def test_bake_fails_when_extension_missing(tmp_path: Path) -> None: + pkg = tmp_path / "lib" / "pycodec2" + pkg.mkdir(parents=True) + (pkg / "__init__.py").write_text("", encoding="utf-8") + with pytest.raises(SystemExit, match="no pycodec2 extension"): + bake_frozen_pycodec2(tmp_path) + + +def test_bake_copies_executable_path_basename_from_load_command( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + "meshchatx.src.backend.bake_frozen_pycodec2.sys.platform", "darwin" + ) + monkeypatch.setattr( + "meshchatx.src.backend.bake_frozen_pycodec2._darwin_load_commands", + lambda _ext: ["@executable_path/lib/libcodec2.1.2.dylib"], + ) + monkeypatch.setattr( + "meshchatx.src.backend.bake_frozen_pycodec2._rewrite_darwin_extension", + lambda *_args, **_kwargs: None, + ) + root = _frozen_pycodec2_tree(tmp_path) + (root / "lib" / "pycodec2" / "libcodec2.dylib").write_bytes(b"lib") + + bake_frozen_pycodec2(root) + + assert (root / "lib" / "libcodec2.1.2.dylib").read_bytes() == b"lib" + assert (root / "lib" / "libcodec2.dylib").read_bytes() == b"lib" + + +def test_bake_copies_from_build_env_when_freeze_tree_has_no_dylib( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root = _frozen_pycodec2_tree(tmp_path) + dist = tmp_path / "site" / "pycodec2" + dist.mkdir(parents=True) + (dist / "libcodec2.dylib").write_bytes(b"from-venv") + monkeypatch.setattr( + "meshchatx.src.backend.bake_frozen_pycodec2._pycodec2_dist_dir", + lambda: dist, + ) + + bake_frozen_pycodec2(root) + + assert (root / "lib" / "pycodec2" / "libcodec2.dylib").read_bytes() == b"from-venv" + assert (root / "lib" / "libcodec2.dylib").read_bytes() == b"from-venv" + + +def test_verify_frozen_codec2_script_rejects_missing_extension(tmp_path: Path) -> None: + (tmp_path / "lib" / "pycodec2").mkdir(parents=True) + result = subprocess.run( + ["bash", str(_VERIFY), str(tmp_path)], + check=False, + capture_output=True, + text=True, + ) + assert result.returncode != 0 + assert "no pycodec2 extension" in result.stderr + + +def test_verify_frozen_codec2_script_requires_dylib_on_darwin(tmp_path: Path) -> None: + root = _frozen_pycodec2_tree(tmp_path) + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + uname = bin_dir / "uname" + uname.write_text("#!/bin/sh\necho Darwin\n", encoding="utf-8") + uname.chmod(0o755) + env = os.environ.copy() + env["PATH"] = f"{bin_dir}{os.pathsep}{env.get('PATH', '')}" + result = subprocess.run( + ["bash", str(_VERIFY), str(root)], + check=False, + capture_output=True, + text=True, + env=env, + ) + assert result.returncode != 0 + assert "libcodec2 missing" in result.stderr + + +def test_verify_frozen_codec2_script_accepts_extension(tmp_path: Path) -> None: + root = _frozen_pycodec2_tree(tmp_path) + (root / "lib" / "pycodec2" / "libcodec2.dylib").write_bytes(b"lib") + result = subprocess.run( + ["bash", str(_VERIFY), str(root)], + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0 + assert "frozen codec2 verify: OK" in result.stdout + + +def test_ci_wires_codec2_freeze_guards() -> None: + deps = Path("scripts/ci/github-install-deps.sh").read_text(encoding="utf-8") + x64 = Path("scripts/ci/github-install-macos-x64-python-deps.sh").read_text( + encoding="utf-8" + ) + universal = Path("scripts/build-macos-universal.sh").read_text(encoding="utf-8") + backend_js = Path("scripts/build-backend.js").read_text(encoding="utf-8") + probe = Path("meshchatx/src/backend/frozen_freeze_probe.py").read_text( + encoding="utf-8" + ) + unify = Path("scripts/unify-backend-plain-files.sh").read_text(encoding="utf-8") + macos_ci = Path("scripts/ci/github-build-macos.sh").read_text(encoding="utf-8") + windows_ci = Path("scripts/ci/github-build-windows.sh").read_text(encoding="utf-8") + ci_yml = Path(".github/workflows/ci.yml").read_text(encoding="utf-8") + + assert "patch_lxst_codec2_optional.py" in deps + assert "unify-backend may drop it later" not in deps + assert "patch_lxst_codec2_optional.py" in x64 + assert "unify-backend may drop it later" not in x64 + assert "bake_frozen_pycodec2" in backend_js + assert "github-verify-frozen-codec2.sh" in universal + assert "github-verify-frozen-runtime.sh" in universal + assert "github-verify-frozen-codec2.sh" in macos_ci + assert "github-verify-frozen-codec2.sh" in windows_ci + assert "github-verify-frozen-codec2.sh" in ci_yml + assert "import LXST" in probe + assert "import pycodec2" in probe + assert "required native" in unify + assert "libcodec2" in unify