feat: improve Mach-O thinning script and runtime verification for Apple Silicon compatibility

scripts/ci/github-verify-frozen-runtime.sh

@@ -1,5 +1,8 @@
 #!/usr/bin/env bash
 # Run import probes inside the cx_Freeze binary (aiohttp/email/LXST natives).
+# On Apple Silicon, darwin-x64 trees must be exec'd with arch -x86_64. A
+# universal2 freeze stub otherwise starts as arm64 and dlopen fails on
+# x86_64 zlib.cpython-*-darwin.so.
 set -euo pipefail
 
 ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
@@ -51,5 +54,49 @@ if [[ ! -f "${BUILD_EXE}/lib/email/header.py" ]]; then
     fi
 fi
 
-"${EXE}" --meshchatx-run-module meshchatx.src.backend.frozen_freeze_probe
+run_prefix=()
+if [[ "$(uname -s)" == "Darwin" ]]; then
+    host_arch="$(uname -m)"
+    case "${host_arch}" in
+    x86_64 | amd64) host_arch=x86_64 ;;
+    arm64 | aarch64) host_arch=arm64 ;;
+    esac
+    want=""
+    case "${BUILD_EXE}" in
+    *darwin-x64*) want=x86_64 ;;
+    *darwin-arm64*) want=arm64 ;;
+    esac
+    exe_archs=""
+    if command -v lipo >/dev/null 2>&1; then
+        exe_archs="$(lipo -archs "${EXE}" 2>/dev/null || true)"
+    fi
+    if [[ -z "${want}" && -n "${exe_archs}" ]]; then
+        want="$(awk '{print $1}' <<<"${exe_archs}")"
+    fi
+    if [[ -n "${want}" && -n "${exe_archs}" ]] && ! grep -qw "${want}" <<<"${exe_archs}"; then
+        echo "frozen runtime verify: ${EXE} archs=${exe_archs} cannot run as ${want}" >&2
+        echo "  darwin-x64 must ship an x86_64 (or universal) ReticulumMeshChatX stub." >&2
+        echo "  thin-backend-mach-o.sh must lipo-thin that executable, not only .so/.dylib." >&2
+        exit 1
+    fi
+    if [[ -n "${want}" && "${want}" != "${host_arch}" ]]; then
+        if ! command -v arch >/dev/null 2>&1; then
+            echo "frozen runtime verify: need arch -${want} to run ${EXE} on ${host_arch}" >&2
+            exit 1
+        fi
+        if ! arch "-${want}" /usr/bin/true >/dev/null 2>&1; then
+            echo "frozen runtime verify: arch -${want} failed on host ${host_arch}" >&2
+            echo "  Install Rosetta 2 to probe the darwin-x64 freeze tree on Apple Silicon." >&2
+            exit 1
+        fi
+        run_prefix=(arch "-${want}")
+        echo "frozen runtime verify: using arch -${want} (host ${host_arch})"
+    fi
+fi
+
+if [[ ${#run_prefix[@]} -gt 0 ]]; then
+    "${run_prefix[@]}" "${EXE}" --meshchatx-run-module meshchatx.src.backend.frozen_freeze_probe
+else
+    "${EXE}" --meshchatx-run-module meshchatx.src.backend.frozen_freeze_probe
+fi
 echo "frozen runtime verify: OK (${EXE})"

scripts/thin-backend-mach-o.sh

@@ -1,11 +1,15 @@
 #!/usr/bin/env bash
+# Lipo-thin every Mach-O in each per-arch freeze tree, including the
+# ReticulumMeshChatX stub. CPython macOS installs are often universal2.
+# If only .so/.dylib files are thinned, Apple Silicon exec of the x64 tree
+# uses the arm64 slice of the stub and then fails to dlopen x86_64 zlib.
 
 set -euo pipefail
 
 ROOT="$(cd "$(dirname "$0")/.." && pwd)"
 
-ARM64_DIR="$ROOT/build/exe/darwin-arm64"
-X64_DIR="$ROOT/build/exe/darwin-x64"
+ARM64_DIR="${MESHCHATX_THIN_ARM64_DIR:-$ROOT/build/exe/darwin-arm64}"
+X64_DIR="${MESHCHATX_THIN_X64_DIR:-$ROOT/build/exe/darwin-x64}"
 
 if [[ ! -d "$ARM64_DIR" || ! -d "$X64_DIR" ]]; then
     echo "thin-backend: one or both backend dirs missing, skipping"
@@ -77,12 +81,15 @@ thin_tree() {
             cat "$tmp" >"$f"
             rm -f "$tmp"
             thinned=$((thinned + 1))
+            if command -v codesign >/dev/null 2>&1; then
+                codesign --force --sign - "$f" >/dev/null 2>&1 || true
+            fi
         else
             rm -f "$tmp"
             echo "thin-backend: WARNING: lipo -thin $want_arch failed on $f" >&2
             skipped=$((skipped + 1))
         fi
-    done < <(find "$tree" -type f \( -name "*.so" -o -name "*.dylib" -o -name "*.bundle" \) -print0)
+    done < <(find "$tree" -type f -print0)
     echo "thin-backend: ${tree#"$ROOT"/} -> $want_arch (thinned=$thinned, already-single=$already, skipped=$skipped)"
 }
 

tests/backend/test_frozen_runtime_arch.py

@@ -0,0 +1,195 @@
+# SPDX-License-Identifier: 0BSD
+
+from __future__ import annotations
+
+import os
+import stat
+import subprocess
+from pathlib import Path
+
+_THIN = Path("scripts/thin-backend-mach-o.sh")
+_VERIFY = Path("scripts/ci/github-verify-frozen-runtime.sh")
+
+
+def _write_exec(path: Path, body: str) -> None:
+    path.write_text(body, encoding="utf-8")
+    path.chmod(path.stat().st_mode | stat.S_IEXEC)
+
+
+def _fake_darwin_bin(tmp_path: Path, *, uname_m: str = "arm64") -> Path:
+    bin_dir = tmp_path / "bin"
+    bin_dir.mkdir()
+    _write_exec(
+        bin_dir / "uname",
+        "#!/bin/sh\n"
+        'if [ "$1" = "-s" ]; then echo Darwin; exit 0; fi\n'
+        f'if [ "$1" = "-m" ]; then echo {uname_m}; exit 0; fi\n'
+        "echo Darwin\n",
+    )
+    return bin_dir
+
+
+def _frozen_tree(root: Path) -> Path:
+    root.mkdir(parents=True, exist_ok=True)
+    _write_exec(
+        root / "ReticulumMeshChatX",
+        "#!/bin/sh\necho frozen-freeze-probe ok\nexit 0\n",
+    )
+    email = root / "lib" / "email"
+    email.mkdir(parents=True)
+    (email / "header.py").write_text("# header\n", encoding="utf-8")
+    return root
+
+
+def test_thin_backend_find_includes_all_mach_o() -> None:
+    text = _THIN.read_text(encoding="utf-8")
+    assert 'find "$tree" -type f -print0' in text
+    assert '-name "*.so"' not in text
+    assert "MESHCHATX_THIN_ARM64_DIR" in text
+    assert "ReticulumMeshChatX stub" in text
+
+
+def test_thin_backend_thins_executable_not_only_dylibs(tmp_path: Path) -> None:
+    arm = tmp_path / "arm"
+    x64 = tmp_path / "x64"
+    for tree in (arm, x64):
+        (tree / "lib").mkdir(parents=True)
+        (tree / "ReticulumMeshChatX").write_bytes(b"fat-exe")
+        (tree / "ReticulumMeshChatX").chmod(0o755)
+        (tree / "lib" / "zlib.cpython-314-darwin.so").write_bytes(b"fat-so")
+        (tree / "readme.txt").write_text("leave me", encoding="utf-8")
+
+    bin_dir = _fake_darwin_bin(tmp_path)
+    _write_exec(
+        bin_dir / "file",
+        "#!/bin/sh\n"
+        'f="$1"\n'
+        'while [ "$#" -gt 0 ]; do f="$1"; shift; done\n'
+        'base=$(basename "$f")\n'
+        'case "$base" in\n'
+        "ReticulumMeshChatX|zlib.cpython-314-darwin.so)\n"
+        '  echo "Mach-O universal binary with 2 architectures: x86_64 arm64"\n'
+        "  ;;\n"
+        '*) echo "ASCII text" ;;\n'
+        "esac\n",
+    )
+    _write_exec(
+        bin_dir / "lipo",
+        "#!/bin/sh\n"
+        'if [ "$1" = "-archs" ]; then echo "x86_64 arm64"; exit 0; fi\n'
+        'if [ "$1" = "-thin" ]; then\n'
+        '  arch="$2"\n'
+        '  out=""\n'
+        '  while [ "$#" -gt 0 ]; do\n'
+        '    if [ "$1" = "-output" ]; then out="$2"; shift 2; continue; fi\n'
+        "    shift\n"
+        "  done\n"
+        '  printf "thinned-%s\\n" "$arch" >"$out"\n'
+        "  exit 0\n"
+        "fi\n"
+        "exit 1\n",
+    )
+    env = os.environ.copy()
+    env["PATH"] = f"{bin_dir}{os.pathsep}{env.get('PATH', '')}"
+    env["MESHCHATX_THIN_ARM64_DIR"] = str(arm)
+    env["MESHCHATX_THIN_X64_DIR"] = str(x64)
+    result = subprocess.run(
+        ["bash", str(_THIN)],
+        check=False,
+        capture_output=True,
+        text=True,
+        env=env,
+    )
+    assert result.returncode == 0, result.stderr
+    assert (arm / "ReticulumMeshChatX").read_text(encoding="utf-8") == "thinned-arm64\n"
+    assert (x64 / "ReticulumMeshChatX").read_text(
+        encoding="utf-8"
+    ) == "thinned-x86_64\n"
+    assert (arm / "lib" / "zlib.cpython-314-darwin.so").read_text(
+        encoding="utf-8"
+    ) == "thinned-arm64\n"
+    assert (x64 / "lib" / "zlib.cpython-314-darwin.so").read_text(
+        encoding="utf-8"
+    ) == "thinned-x86_64\n"
+    assert (arm / "readme.txt").read_text(encoding="utf-8") == "leave me"
+
+
+def test_verify_frozen_runtime_uses_arch_x86_64_for_darwin_x64(
+    tmp_path: Path,
+) -> None:
+    root = _frozen_tree(tmp_path / "darwin-x64")
+    marker = tmp_path / "arch-used"
+    bin_dir = _fake_darwin_bin(tmp_path, uname_m="arm64")
+    _write_exec(
+        bin_dir / "lipo",
+        '#!/bin/sh\nif [ "$1" = "-archs" ]; then echo "x86_64"; exit 0; fi\nexit 1\n',
+    )
+    _write_exec(
+        bin_dir / "arch",
+        "#!/bin/sh\n"
+        f'echo "$1" >"{marker}"\n'
+        'if [ "$1" = "-x86_64" ]; then shift; exec "$@"; fi\n'
+        "exit 1\n",
+    )
+    env = os.environ.copy()
+    env["PATH"] = f"{bin_dir}{os.pathsep}{env.get('PATH', '')}"
+    result = subprocess.run(
+        ["bash", str(_VERIFY), str(root)],
+        check=False,
+        capture_output=True,
+        text=True,
+        env=env,
+    )
+    assert result.returncode == 0, result.stderr + result.stdout
+    assert marker.read_text(encoding="utf-8").strip() == "-x86_64"
+    assert "using arch -x86_64" in result.stdout
+    assert "frozen runtime verify: OK" in result.stdout
+
+
+def test_verify_frozen_runtime_runs_native_on_darwin_arm64(tmp_path: Path) -> None:
+    root = _frozen_tree(tmp_path / "darwin-arm64")
+    marker = tmp_path / "arch-used"
+    bin_dir = _fake_darwin_bin(tmp_path, uname_m="arm64")
+    _write_exec(
+        bin_dir / "lipo",
+        '#!/bin/sh\nif [ "$1" = "-archs" ]; then echo "arm64"; exit 0; fi\nexit 1\n',
+    )
+    _write_exec(
+        bin_dir / "arch",
+        f'#!/bin/sh\necho "$1" >"{marker}"\nexit 1\n',
+    )
+    env = os.environ.copy()
+    env["PATH"] = f"{bin_dir}{os.pathsep}{env.get('PATH', '')}"
+    result = subprocess.run(
+        ["bash", str(_VERIFY), str(root)],
+        check=False,
+        capture_output=True,
+        text=True,
+        env=env,
+    )
+    assert result.returncode == 0, result.stderr + result.stdout
+    assert not marker.exists()
+    assert "using arch" not in result.stdout
+    assert "frozen runtime verify: OK" in result.stdout
+
+
+def test_verify_frozen_runtime_rejects_arm64_only_stub_in_x64_tree(
+    tmp_path: Path,
+) -> None:
+    root = _frozen_tree(tmp_path / "darwin-x64")
+    bin_dir = _fake_darwin_bin(tmp_path, uname_m="arm64")
+    _write_exec(
+        bin_dir / "lipo",
+        '#!/bin/sh\nif [ "$1" = "-archs" ]; then echo "arm64"; exit 0; fi\nexit 1\n',
+    )
+    env = os.environ.copy()
+    env["PATH"] = f"{bin_dir}{os.pathsep}{env.get('PATH', '')}"
+    result = subprocess.run(
+        ["bash", str(_VERIFY), str(root)],
+        check=False,
+        capture_output=True,
+        text=True,
+        env=env,
+    )
+    assert result.returncode != 0
+    assert "cannot run as x86_64" in result.stderr