MeshChatX/scripts/unify-backend-plain-files.sh

#!/usr/bin/env bash
# Make the two per-arch cx_Freeze backend trees compatible with
# @electron/universal's merge requirements:
#
#   1. Every file must exist in BOTH trees (no unique-to-one-arch files).
#   2. Every non-Mach-O file must be byte-identical across trees.
#
# Python bytecode (.pyc inside library.zip) is architecture-independent.
# Only timestamps and zip metadata cause SHA differences.

set -euo pipefail

ROOT="$(cd "$(dirname "$0")/.." && pwd)"

ARM64_DIR="${MESHCHATX_UNIFY_ARM64_DIR:-$ROOT/build/exe/darwin-arm64}"
X64_DIR="${MESHCHATX_UNIFY_X64_DIR:-$ROOT/build/exe/darwin-x64}"

if [[ ! -d "$ARM64_DIR" || ! -d "$X64_DIR" ]]; then
    echo "unify-backend: one or both backend dirs missing, skipping"
    exit 0
fi

unified=0
synced=0
dropped=0
normalized=0

required_native_rel() {
    local base
    base="$(basename "$1")"
    case "$base" in
    libcodec2* | pycodec2*.so | pycodec2*.dylib | pycodec2*.pyd)
        return 0
        ;;
    esac
    return 1
}

# Intel NumPy wheels ship OpenBLAS (plus gfortran/quadmath/gcc_s). Apple
# Silicon NumPy uses Accelerate, so those dylibs are x64-only. Dropping them
# to equalize the two trees makes numpy._core._multiarray_umath fail to
# dlopen on the Intel slice. Mirror them into the other tree so
# @electron/universal sees the same paths. x64ArchFiles keeps the backend
# from being lipo'd, so the unused copy is never loaded.
mirror_native_rel() {
    local base
    base="$(basename "$1")"
    case "$base" in
    libscipy_openblas* | libgfortran* | libquadmath* | libgcc_s*)
        return 0
        ;;
    esac
    return 1
}

copy_missing() {
    local src_dir="$1" dst_dir="$2" label="$3"
    while IFS= read -r -d '' rel; do
        rel="${rel#./}"
        if [[ ! -f "$dst_dir/$rel" ]]; then
            local src_file="$src_dir/$rel"
            local ft
            ft=$(file --brief --no-pad "$src_file" 2>/dev/null || true)
            if [[ "$ft" == Mach-O* ]]; then
                if required_native_rel "$rel"; then
                    echo "unify-backend: ERROR: required native $rel exists only in $label" >&2
                    echo "  source reports: $ft" >&2
                    echo "  Both darwin-arm64 and darwin-x64 must ship libcodec2 next to pycodec2." >&2
                    echo "  Run scripts/ci/macos-normalize-pycodec2-dylib.sh on each venv and" >&2
                    echo "  meshchatx.src.backend.bake_frozen_pycodec2 after each cx_Freeze slice." >&2
                    exit 1
                fi
                if mirror_native_rel "$rel"; then
                    mkdir -p "$dst_dir/$(dirname "$rel")"
                    cp "$src_file" "$dst_dir/$rel"
                    echo "unify-backend: mirroring arch-only native $rel ($label)" >&2
                    echo "  source reports: $ft" >&2
                    synced=$((synced + 1))
                    continue
                fi
                echo "unify-backend: dropping arch-only Mach-O for consistency: $rel" >&2
                echo "  ($label); source reports: $ft" >&2
                echo "  Hint: this native library/extension only exists in one arch's" >&2
                echo "  cx_Freeze output. Check the corresponding per-OS dependency" >&2
                echo "  install script builds/bundles it for both darwin-arm64 and" >&2
                echo "  darwin-x64 (e.g. missing a Homebrew lib on one arch, or a" >&2
                echo "  package that only ships a wheel for one arch)." >&2
                rm -f "$src_file"
                dropped=$((dropped + 1))
                continue
            fi
            mkdir -p "$dst_dir/$(dirname "$rel")"
            cp "$src_file" "$dst_dir/$rel"
            echo "  synced ($label): $rel"
            synced=$((synced + 1))
        fi
    done < <(cd "$src_dir" && find . -type f -print0)
}

copy_missing "$ARM64_DIR" "$X64_DIR" "arm64 -> x64"
copy_missing "$X64_DIR" "$ARM64_DIR" "x64 -> arm64"

while IFS= read -r -d '' rel; do
    rel="${rel#./}"
    arm64_file="$ARM64_DIR/$rel"
    x64_file="$X64_DIR/$rel"

    [[ -f "$x64_file" ]] || continue

    if cmp -s "$arm64_file" "$x64_file"; then
        continue
    fi

    filetype=$(file --brief --no-pad "$arm64_file" 2>/dev/null || true)
    if [[ "$filetype" != Mach-O* ]]; then
        # .pyc is always portable CPython bytecode regardless of what else
        # lives in the same package directory: differences here come from
        # the embedded source mtime/size (PEP 552 header) or from hash-seed-
        # dependent constant ordering (e.g. frozenset literals), never from
        # the target architecture. Unlike a real native-extension conflict,
        # unifying it can't break the .so/.dylib siblings, which are handled
        # entirely separately below.
        if [[ "$rel" == *.pyc ]]; then
            cp "$arm64_file" "$x64_file"
            echo "  unified (.pyc, source-independent of arch): $rel"
            unified=$((unified + 1))
            continue
        fi
        # cx_Freeze bundles pure-Python modules into lib/library.zip. Native
        # extensions are always written to the filesystem separately (they
        # can't be dlopen'd from inside a zip), so this archive's *contents*
        # are pure CPython bytecode just like loose .pyc files. Its raw bytes
        # almost always differ across two independent builds (each entry's
        # own PEP 552 header + the zip's own per-entry timestamps), so only
        # trust a blind copy once we've confirmed both slices bundled the
        # same set of modules. A differing member list would mean the two
        # Python environments actually resolved different dependencies.
        if [[ "$(basename "$rel")" == "library.zip" ]]; then
            # Exclude *.dist-info provenance files that record *how* a package
            # was installed rather than anything about the package itself:
            # direct_url.json/INSTALLER only appear when pip/uv installs from
            # a local wheel path (true for our manually-built pycodec2 wheel),
            # and uv_build.json/uv_cache.json only appear when uv has to build
            # a package from sdist (true for cbor2/cryptography, which don't
            # publish a macOS x86_64 wheel for this Python version). None of
            # these are ever read by the running app, only by pip/uv itself.
            _dist_info_noise='\.dist-info/(direct_url\.json|INSTALLER|uv_build\.json|uv_cache\.json)$'
            arm64_members="$(mktemp)"
            x64_members="$(mktemp)"
            zipinfo -1 "$arm64_file" 2>/dev/null | grep -vE "$_dist_info_noise" | sort >"$arm64_members"
            zipinfo -1 "$x64_file" 2>/dev/null | grep -vE "$_dist_info_noise" | sort >"$x64_members"
            if cmp -s "$arm64_members" "$x64_members"; then
                cp "$arm64_file" "$x64_file"
                echo "  unified (library.zip, same bundled module set): $rel"
                unified=$((unified + 1))
                rm -f "$arm64_members" "$x64_members"
                continue
            fi
            echo "unify-backend: ERROR: $rel bundles a different set of modules on arm64 vs x64" >&2
            echo "  cx_Freeze's module finder resolved different transitive imports for" >&2
            echo "  .venv and .venv-x64. Check that both install the same locked" >&2
            echo "  dependency set (uv.lock) via scripts/ci/github-install-deps.sh and" >&2
            echo "  scripts/ci/github-install-macos-x64-python-deps.sh." >&2
            echo "  Only present on arm64 (missing from x64):" >&2
            comm -23 "$arm64_members" "$x64_members" | sed 's/^/    /' >&2
            echo "  Only present on x64 (missing from arm64):" >&2
            comm -13 "$arm64_members" "$x64_members" | sed 's/^/    /' >&2
            rm -f "$arm64_members" "$x64_members"
            exit 1
        fi
        pkg_dir="$(dirname "$rel")"
        if find "$ARM64_DIR/$pkg_dir" "$X64_DIR/$pkg_dir" -maxdepth 1 \
            \( -name '*.so' -o -name '*.dylib' -o -name '*.bundle' \) -print -quit 2>/dev/null | grep -q .; then
            echo "unify-backend: ERROR: byte mismatch in a native Python package: $rel" >&2
            echo "  arm64 and x64 cx_Freeze trees differ under ${pkg_dir}/, which also" >&2
            echo "  contains native extensions. Copying one arch's .py over the other's" >&2
            echo "  .so breaks imports (e.g. NumPy _add_newdoc_ufunc). Rebuild the x64" >&2
            echo "  slice with scripts/ci/github-install-macos-x64-python-deps.sh." >&2
            exit 1
        fi
        cp "$arm64_file" "$x64_file"
        echo "  unified: $rel"
        unified=$((unified + 1))
        continue
    fi

    x64_filetype=$(file --brief --no-pad "$x64_file" 2>/dev/null || true)
    if [[ "$x64_filetype" == Mach-O* ]] &&
       [[ "$x64_filetype" == *arm64* ]] &&
       [[ "$x64_filetype" != *universal* ]] &&
       [[ "$x64_filetype" != *x86_64* ]]; then
        cp "$arm64_file" "$x64_file"
        echo "unify-backend: normalized same-arch Mach-O (arm64 in both trees): $rel" >&2
        echo "  x64 copy replaced with arm64 tree's bytes; @electron/universal" >&2
        echo "  will use x64ArchFiles to skip lipo.  For a true universal2 binary," >&2
        echo "  ensure the x64 Python env compiles with CC=\"clang -arch x86_64\"." >&2
        normalized=$((normalized + 1))
    fi
done < <(cd "$ARM64_DIR" && find . -type f -print0)

total=$((unified + synced + normalized))
if [[ $dropped -gt 0 ]]; then
    echo "unify-backend: WARNING: dropped $dropped arch-only Mach-O binary/binaries for consistency (pure Python fallback active)"
fi
if [[ $normalized -gt 0 ]]; then
    echo "unify-backend: WARNING: normalized $normalized same-arch Mach-O file(s) (both trees had arm64; x64ArchFiles will skip lipo)"
fi
if [[ $total -gt 0 ]]; then
    echo "unify-backend: synced $synced missing file(s), unified $unified differing file(s), normalized $normalized same-arch file(s)"
else
    echo "unify-backend: all files already identical and present in both trees"
fi