diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index adda68d3c..2ac4de954 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -247,15 +247,14 @@ jobs: - os: ubuntu-24.04-arm target: aarch64-unknown-linux-gnu manylinux: 2_28 - # NOTE: `macos-15-intel` (x86_64-apple-darwin) is intentionally - # NOT in the matrix. `ort-sys 2.0.0-rc.12` (transitive via our - # ML compression backend) does not provide prebuilt ONNX Runtime - # binaries for `x86_64-apple-darwin`, and building ORT from - # source would add CMake + ~5 minutes per build. Apple Silicon - # macOS is fully covered below; Intel-mac users install the - # platform-independent sdist (also produced by this matrix). - # Tracked as a follow-up: switch to `ort-tract` or upstream a - # request for x86_64 macOS prebuilts. + # Intel macOS x86_64. `ort-sys 2.0.0-rc.12` does not ship + # prebuilt ONNX Runtime binaries for this triple, so + # `headroom-core/Cargo.toml` selects `ort-load-dynamic` (same + # as Windows) and `headroom/_ort.py` pins `ORT_DYLIB_PATH` to + # the pip `onnxruntime` dylib at import time. + - os: macos-15-intel + target: x86_64-apple-darwin + manylinux: "" - os: macos-14 target: aarch64-apple-darwin manylinux: "" @@ -267,8 +266,9 @@ jobs: # or crates.io). `manylinux` is meaningless on Windows (that tag # is Linux-only) — leave it empty. The ONNX Runtime story is # already handled: `crates/headroom-core/Cargo.toml` selects - # `ort-load-dynamic` under `cfg(windows)`, so the wheel does not - # bundle/link DirectML SDK libs and loads ORT at runtime instead. + # `ort-load-dynamic` under `cfg(windows)` and Intel macOS, so + # the wheel does not bundle/link platform ORT SDK libs and loads + # ORT at runtime instead. - os: windows-latest target: x86_64-pc-windows-msvc manylinux: "" @@ -507,11 +507,11 @@ jobs: # runners (PR #376) host the container. - { runner: ubuntu-24.04-arm, image: "quay.io/pypa/manylinux_2_28_aarch64", python: "3.11", wheel_target: aarch64-unknown-linux-gnu, wheel_artifact: wheels-ubuntu-24.04-arm-aarch64-unknown-linux-gnu, glibc_label: "2.28-floor" } - { runner: ubuntu-24.04-arm, image: "ubuntu:22.04", python: "3.12", wheel_target: aarch64-unknown-linux-gnu, wheel_artifact: wheels-ubuntu-24.04-arm-aarch64-unknown-linux-gnu, glibc_label: "2.35" } - # macOS arm64 — runs natively on the host runner; no - # container. Apple Silicon is the only macOS target we - # ship today (Intel macOS dropped in PR #371 — ort-sys - # has no x86_64-apple-darwin prebuilts). + # macOS arm64 — runs natively on the host runner; no container. - { runner: macos-14, image: "", python: "3.13", wheel_target: aarch64-apple-darwin, wheel_artifact: wheels-macos-14-aarch64-apple-darwin, glibc_label: "" } + # macOS x86_64 (Intel) — `ort-load-dynamic`; see build-wheels + # matrix comment for rationale. + - { runner: macos-15-intel, image: "", python: "3.12", wheel_target: x86_64-apple-darwin, wheel_artifact: wheels-macos-15-intel-x86_64-apple-darwin, glibc_label: "" } # Windows x86_64 — runs natively on the host runner; no # container. `image: ""` routes this row past the Linux docker # path; a dedicated PowerShell step below installs + imports the @@ -680,16 +680,21 @@ jobs: if: matrix.image == '' && runner.os == 'macOS' env: PYTHON_VERSION: ${{ matrix.python }} + WHEEL_TARGET: ${{ matrix.wheel_target }} run: | set -e py_tag=cp$(echo "$PYTHON_VERSION" | tr -d .) - # macOS wheels are tagged macosx_*_arm64 (Apple Silicon only). - whl=$(find dist -maxdepth 1 -type f -name "headroom_ai-*-${py_tag}-${py_tag}-macosx_*_arm64.whl" -print -quit) + case "$WHEEL_TARGET" in + aarch64-apple-darwin) mac_arch=arm64 ;; + x86_64-apple-darwin) mac_arch=x86_64 ;; + *) echo "ERROR: unknown macOS wheel target $WHEEL_TARGET" >&2; exit 1 ;; + esac + whl=$(find dist -maxdepth 1 -type f -name "headroom_ai-*-${py_tag}-${py_tag}-macosx_*_${mac_arch}.whl" -print -quit) if [ -z "$whl" ]; then # Stable-ABI (abi3) fallback — see the Linux path above. A # cp3-abi3 wheel installs on any CPython >= floor. py_minor=${PYTHON_VERSION#*.} - abi3=$(find dist -maxdepth 1 -type f -name "headroom_ai-*-abi3-macosx_*_arm64.whl" -print -quit) + abi3=$(find dist -maxdepth 1 -type f -name "headroom_ai-*-abi3-macosx_*_${mac_arch}.whl" -print -quit) if [ -n "$abi3" ]; then abi3_base=${abi3##*/} abi3_floor=${abi3_base#*-cp3} @@ -699,7 +704,7 @@ jobs: fi fi if [ -z "$whl" ]; then - echo "ERROR: no macOS wheel matching python=$PYTHON_VERSION" + echo "ERROR: no macOS wheel matching python=$PYTHON_VERSION arch=$mac_arch target=$WHEEL_TARGET" find dist -maxdepth 1 -type f -exec basename {} \; exit 1 fi diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 2c2857133..32cb1c146 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -75,14 +75,11 @@ jobs: - os: macos-14 target: aarch64-apple-darwin maturin-target: aarch64-apple-darwin - # macOS x86_64 (Intel) is NOT in this matrix. - # `fastembed` → `ort` → `ort-sys` does not publish prebuilt ONNX - # Runtime binaries for `x86_64-apple-darwin`; building from source - # in CI is a multi-hour cmake job. Apple Silicon has been the - # default macOS target since 2020 and is sufficient for the wheels - # we ship. If a customer needs Intel macOS, build from source - # locally (the toolchain works; only prebuilt distribution skips - # this target). + - os: macos-15-intel + target: x86_64-apple-darwin + maturin-target: x86_64-apple-darwin + # Intel macOS uses `ort-load-dynamic` (no prebuilt ORT from ort-sys); + # Apple Silicon bundles ORT via `ort-download-binaries-rustls-tls`. steps: - uses: actions/checkout@v6 - uses: actions/setup-python@v6 diff --git a/README.md b/README.md index 34de0b3dd..f46bd7a12 100644 --- a/README.md +++ b/README.md @@ -380,8 +380,8 @@ winget install Rustlang.Rustup && rustup default stable Restart your shell, then `pip install "headroom-ai[all]"`. A prebuilt wheel avoids the Rust build entirely where available: `pip install --only-binary headroom-ai headroom-ai`. Prebuilt wheels are published for Windows (`win_amd64`), Linux (`x86_64` / `aarch64`), and macOS -(Apple Silicon), so installs on those platforms never need a local Rust toolchain — the -Rust-first dance above is only for the platform-independent sdist fallback (e.g. Intel macOS). +(Apple Silicon and Intel), so installs on those platforms never need a local Rust toolchain — the +Rust-first dance above is only for the platform-independent sdist fallback when no wheel matches. Two runtime assets are fetched over TLS; if they are blocked, trust your corporate CA via `REQUESTS_CA_BUNDLE` / `SSL_CERT_FILE` / `CURL_CA_BUNDLE`: diff --git a/crates/headroom-core/Cargo.toml b/crates/headroom-core/Cargo.toml index a5705ba8a..360346e89 100644 --- a/crates/headroom-core/Cargo.toml +++ b/crates/headroom-core/Cargo.toml @@ -122,13 +122,25 @@ redis = { version = "0.27", optional = true, default-features = false } # cycling through the proxy crate. Tiny crate (no I/O, just types). http = "1" -[target.'cfg(not(windows))'.dependencies] +[target.'cfg(all(not(windows), not(all(target_os = "macos", target_arch = "x86_64"))))'.dependencies] fastembed = { version = "5", default-features = false, features = [ "hf-hub-rustls-tls", "ort-download-binaries-rustls-tls", "image-models", ] } +[target.'cfg(all(target_os = "macos", target_arch = "x86_64"))'.dependencies] +# `ort-sys 2.0.0-rc.12` does not ship prebuilt ONNX Runtime binaries for +# `x86_64-apple-darwin`. Building ORT from source in CI would add CMake and +# several minutes per wheel. Load the pip `onnxruntime` dylib at runtime +# instead (same approach as Windows below). `headroom/_ort.py` pins +# `ORT_DYLIB_PATH` before `headroom._core` imports. +fastembed = { version = "5", default-features = false, features = [ + "hf-hub-rustls-tls", + "ort-load-dynamic", + "image-models", +] } + [target.'cfg(windows)'.dependencies] # `ort-download-binaries-*` emits DirectML link libs on Windows (`DXCORE`, # `DXGI`, `D3D12`, `DirectML`). Users installing `headroom-ai[all]` from diff --git a/headroom/_ort.py b/headroom/_ort.py index 00a55d83c..09b6d84fd 100644 --- a/headroom/_ort.py +++ b/headroom/_ort.py @@ -1,31 +1,37 @@ -"""Pin the ONNX Runtime dylib for the Rust core on Windows. +"""Pin the ONNX Runtime dylib for the Rust core on dynamic-ORT platforms. Why this module exists ---------------------- -On Windows, ``headroom._core`` consumers of the ``ort`` crate (magika -content detection, fastembed embeddings) are built with -``ort-load-dynamic``: the native ``onnxruntime.dll`` is resolved at -*runtime*. Unless ``ORT_DYLIB_PATH`` is set, ort falls back to a bare -``LoadLibrary("onnxruntime.dll")`` and the Windows DLL search order -applies — and ``C:\\Windows\\System32`` wins. +On Windows and Intel macOS (``x86_64-apple-darwin``), ``headroom._core`` +consumers of the ``ort`` crate (magika content detection, fastembed +embeddings) are built with ``ort-load-dynamic``: the native ONNX Runtime +library is resolved at *runtime*. -Windows 11 24H2+ ships ``System32\\onnxruntime.dll`` as part of Windows -ML (observed: 1.17.2603 "os-germanium"). Initializing an ort 2.x -session against that OS build does not fail — it deadlocks -indefinitely at 0% CPU, which the tiered detection fallback cannot -catch (a hang is not an ``Err``). Reproduced and bracketed with -``scripts/diag_magika_windows.py``: the identical session inits in -~400ms when ``ORT_DYLIB_PATH`` points at the ``onnxruntime`` pip -package's DLL (which ``headroom-ai[proxy]`` already depends on). +Windows: unless ``ORT_DYLIB_PATH`` is set, ort falls back to a bare +``LoadLibrary("onnxruntime.dll")`` and the Windows DLL search order +applies — and ``C:\\Windows\\System32`` wins. Windows 11 24H2+ ships +``System32\\onnxruntime.dll`` as part of Windows ML (observed: +1.17.2603 "os-germanium"). Initializing an ort 2.x session against that +OS build does not fail — it deadlocks indefinitely at 0% CPU, which the +tiered detection fallback cannot catch (a hang is not an ``Err``). +Reproduced and bracketed with ``scripts/diag_magika_windows.py``: the +identical session inits in ~400ms when ``ORT_DYLIB_PATH`` points at the +``onnxruntime`` pip package's DLL (which ``headroom-ai[proxy]`` already +depends on). + +Intel macOS: ``ort-sys 2.0.0-rc.12`` does not ship prebuilt ONNX Runtime +binaries for ``x86_64-apple-darwin``, so the wheel/sdist build uses +``ort-load-dynamic`` and expects a pip-installed ``onnxruntime`` dylib +at runtime (same contract as Windows). The fix: before anything can import ``headroom._core``, resolve the -pip-installed ``onnxruntime\\capi\\onnxruntime.dll`` and export it via +pip-installed ``onnxruntime`` native library and export it via ``ORT_DYLIB_PATH``. ``headroom/__init__.py`` calls this hook, which guarantees ordering for every package-level consumer. Behavior contract ----------------- -- Windows-only; a no-op everywhere else. +- Active on Windows and Intel macOS only; a no-op elsewhere. - Respects a pre-set ``ORT_DYLIB_PATH`` (user override wins). - Locates the ``onnxruntime`` package via ``find_spec`` WITHOUT importing it (importing would load its native code; this hook must @@ -41,6 +47,7 @@ from __future__ import annotations import importlib.util import logging import os +import platform import sys from pathlib import Path @@ -54,11 +61,12 @@ _pinned: object = _UNSET def ensure_ort_dylib_pinned() -> str | None: - """Export ``ORT_DYLIB_PATH`` for the Rust core's ort runtime (Windows). + """Export ``ORT_DYLIB_PATH`` for the Rust core's ort runtime. Returns the effective dylib path (pinned now or already present in - the environment), or ``None`` when no pin applies (non-Windows, or - no ``onnxruntime`` package to point at). Idempotent and exception-free. + the environment), or ``None`` when no pin applies (platforms that + bundle ORT at build time, or no ``onnxruntime`` package to point at). + Idempotent and exception-free. """ global _pinned if _pinned is not _UNSET: @@ -67,8 +75,23 @@ def ensure_ort_dylib_pinned() -> str | None: return _pinned # type: ignore[return-value] +def _needs_ort_dylib_pin() -> bool: + if sys.platform.startswith("win"): + return True + return sys.platform == "darwin" and platform.machine() == "x86_64" + + +def _resolve_ort_native_library(capi_dir: Path) -> Path | None: + if sys.platform.startswith("win"): + candidate = capi_dir / "onnxruntime.dll" + return candidate if candidate.is_file() else None + + matches = sorted(capi_dir.glob("libonnxruntime*.dylib")) + return matches[0] if matches else None + + def _resolve_and_pin() -> str | None: - if not sys.platform.startswith("win"): + if not _needs_ort_dylib_pin(): return None try: @@ -80,26 +103,26 @@ def _resolve_and_pin() -> str | None: spec = importlib.util.find_spec("onnxruntime") if spec is None or not spec.origin: logger.debug( - "onnxruntime package not found; %s left unset. The Rust ML detection " - "may pick up the Windows ML System32 onnxruntime.dll, which is known " - "to deadlock ort init on Windows 11 24H2+ (it then degrades to non-ML " - "tiers via HEADROOM_MAGIKA_INIT_TIMEOUT_SECS). Install onnxruntime or " - "set %s explicitly.", + "onnxruntime package not found; %s left unset. Rust ML detection " + "needs a pip-installed onnxruntime on this platform (install " + "headroom-ai[proxy] or set %s explicitly).", _ENV_VAR, _ENV_VAR, ) return None - dll = Path(spec.origin).parent / "capi" / "onnxruntime.dll" - if not dll.is_file(): + native = _resolve_ort_native_library(Path(spec.origin).parent / "capi") + if native is None: logger.debug( - "onnxruntime package found but %s is missing; %s left unset", dll, _ENV_VAR + "onnxruntime package found but no native library under %s; %s left unset", + Path(spec.origin).parent / "capi", + _ENV_VAR, ) return None - os.environ[_ENV_VAR] = str(dll) - logger.info("Pinned %s to bundled ONNX Runtime: %s", _ENV_VAR, dll) - return str(dll) + os.environ[_ENV_VAR] = str(native) + logger.info("Pinned %s to bundled ONNX Runtime: %s", _ENV_VAR, native) + return str(native) except Exception as exc: # never break `import headroom` over an accelerator pin logger.debug("ort dylib pin skipped: %s: %s", type(exc).__name__, exc) return None diff --git a/tests/test_release_workflows.py b/tests/test_release_workflows.py index b27acbc7e..07a53077d 100644 --- a/tests/test_release_workflows.py +++ b/tests/test_release_workflows.py @@ -200,24 +200,25 @@ def test_fastembed_uses_rustls_features() -> None: def test_fastembed_uses_dynamic_ort_on_windows() -> None: - """Windows sdist builds must not link Pyke's DirectML ORT binaries. + """Windows and Intel macOS sdist builds must not link Pyke's ORT binaries. - `ort-download-binaries-*` emits DXCORE/DXGI/D3D12/DirectML link libs on - Windows. Those SDK libs are not present on many Python build hosts, so the - Windows target must use ORT dynamic loading instead. + `ort-download-binaries-*` emits platform SDK link libs (DirectML on + Windows; unavailable prebuilts on `x86_64-apple-darwin`). Those targets + must use ORT dynamic loading instead. """ cargo = (ROOT / "crates" / "headroom-core" / "Cargo.toml").read_text(encoding="utf-8") - assert "[target.'cfg(windows)'.dependencies]" in cargo - windows_section = cargo.split("[target.'cfg(windows)'.dependencies]", 1)[1].split( - "\n[", - 1, - )[0] - windows_dependency_lines = "\n".join( - line for line in windows_section.splitlines() if not line.lstrip().startswith("#") - ) - assert '"ort-load-dynamic"' in windows_section - assert "ort-download-binaries" not in windows_dependency_lines + for section_marker in ( + "[target.'cfg(windows)'.dependencies]", + '[target.\'cfg(all(target_os = "macos", target_arch = "x86_64"))\'.dependencies]', + ): + assert section_marker in cargo, f"missing Cargo target section: {section_marker}" + section = cargo.split(section_marker, 1)[1].split("\n[", 1)[0] + dependency_lines = "\n".join( + line for line in section.splitlines() if not line.lstrip().startswith("#") + ) + assert '"ort-load-dynamic"' in section + assert "ort-download-binaries" not in dependency_lines def test_dockerfiles_no_longer_install_openssl_devel() -> None: @@ -289,16 +290,13 @@ def test_release_yml_does_not_install_openssl_or_perl_for_wheels() -> None: ) -def test_build_wheels_matrix_excludes_intel_macos() -> None: - """`ort-sys 2.0.0-rc.12` (transitive via the ML compression backend) +def test_build_wheels_matrix_includes_intel_macos_with_dynamic_ort() -> None: + """Intel macOS wheels use `ort-load-dynamic` because `ort-sys 2.0.0-rc.12` has no prebuilt ONNX Runtime binaries for `x86_64-apple-darwin`. - Building ORT from source would add CMake + ~5 minutes per build. - Apple Silicon macOS is fully covered; Intel-mac users install from - the platform-independent sdist this matrix also produces. We assert against the actual matrix entry shape (`target: ` - on a non-comment line) so explanatory comments mentioning the - excluded triple don't false-positive. + on a non-comment line) so explanatory comments mentioning other + triples don't false-positive. """ content = (ROOT / ".github" / "workflows" / "release.yml").read_text(encoding="utf-8") @@ -319,14 +317,10 @@ def test_build_wheels_matrix_excludes_intel_macos() -> None: assert "aarch64-apple-darwin" in matrix_targets, "Apple Silicon must stay in the matrix" assert "x86_64-unknown-linux-gnu" in matrix_targets assert "aarch64-unknown-linux-gnu" in matrix_targets - - # Intel macOS must NOT be a matrix entry — re-add only after switching - # off ort-sys (e.g., to ort-tract) or adding a CMake-from-source step. - assert "x86_64-apple-darwin" not in matrix_targets, ( - f"x86_64-apple-darwin must not be a wheel-matrix target; got {matrix_targets}" + assert "x86_64-apple-darwin" in matrix_targets, ( + f"x86_64-apple-darwin must be a wheel-matrix target; got {matrix_targets}" ) - # The runner OS itself shouldn't appear as a configured `os:` either. matrix_os: list[str] = [] for raw in body.splitlines(): stripped = raw.lstrip() @@ -334,7 +328,27 @@ def test_build_wheels_matrix_excludes_intel_macos() -> None: continue if stripped.startswith("os:"): matrix_os.append(stripped.split(":", 1)[1].strip()) - assert "macos-15-intel" not in matrix_os + elif stripped.startswith("- os:"): + matrix_os.append(stripped.split(":", 1)[1].strip()) + assert "macos-15-intel" in matrix_os + + +def test_smoke_import_macos_selects_wheel_arch_from_target() -> None: + """The macOS smoke-import step must pick the wheel tag from the matrix + target (arm64 for Apple Silicon, x86_64 for Intel) instead of + hardcoding `_arm64` for every macOS row.""" + content = (ROOT / ".github" / "workflows" / "release.yml").read_text(encoding="utf-8") + + step_start = content.index("- name: Smoke-import wheel on macOS host") + step_end = content.index("- name: Smoke-import wheel on Windows host", step_start) + macos_block = content[step_start:step_end] + + assert "WHEEL_TARGET: ${{ matrix.wheel_target }}" in macos_block + assert "aarch64-apple-darwin) mac_arch=arm64" in macos_block + assert "x86_64-apple-darwin) mac_arch=x86_64" in macos_block + assert "macosx_*_${mac_arch}.whl" in macos_block + assert "headroom_ai-*-${py_tag}-${py_tag}-macosx_*_arm64.whl" not in macos_block + assert "headroom_ai-*-abi3-macosx_*_arm64.whl" not in macos_block def test_aarch64_wheel_uses_native_arm64_runner() -> None: diff --git a/tests/test_transforms/test_ort_dylib.py b/tests/test_transforms/test_ort_dylib.py index 8930a07ab..74101565e 100644 --- a/tests/test_transforms/test_ort_dylib.py +++ b/tests/test_transforms/test_ort_dylib.py @@ -1,9 +1,9 @@ -"""Tests for headroom._ort — the Windows ORT_DYLIB_PATH auto-pin. +"""Tests for headroom._ort — the ORT_DYLIB_PATH auto-pin. -The resolver guards the Rust core against the Windows DLL search picking -up the Windows ML System32 onnxruntime.dll (deadlocks ort session init on -Win11 24H2+, see headroom/_ort.py). The platform gate is monkeypatched so -the full logic runs on any CI OS. +The resolver guards the Rust core on platforms that use `ort-load-dynamic` +(Windows and Intel macOS). On Windows it avoids the System32 onnxruntime.dll +deadlock (Win11 24H2+, see headroom/_ort.py). Platform gates are +monkeypatched so the full logic runs on any CI OS. """ from __future__ import annotations @@ -27,6 +27,11 @@ def _force_windows(monkeypatch): monkeypatch.setattr(sys, "platform", "win32") +def _force_intel_macos(monkeypatch): + monkeypatch.setattr(sys, "platform", "darwin") + monkeypatch.setattr(_ort.platform, "machine", lambda: "x86_64") + + def _fake_spec_for(monkeypatch, package_dir): """Make find_spec('onnxruntime') resolve to a fake package directory.""" spec = SimpleNamespace(origin=str(package_dir / "__init__.py")) @@ -37,11 +42,16 @@ def _fake_spec_for(monkeypatch, package_dir): ) -def test_noop_on_non_windows(monkeypatch): +def test_noop_on_non_dynamic_platforms(monkeypatch): monkeypatch.setattr(sys, "platform", "linux") assert _ort.ensure_ort_dylib_pinned() is None assert "ORT_DYLIB_PATH" not in _ort.os.environ + monkeypatch.setattr(sys, "platform", "darwin") + monkeypatch.setattr(_ort.platform, "machine", lambda: "arm64") + assert _ort.ensure_ort_dylib_pinned() is None + assert "ORT_DYLIB_PATH" not in _ort.os.environ + def test_respects_existing_env(monkeypatch): _force_windows(monkeypatch) @@ -63,6 +73,19 @@ def test_pins_to_package_capi_dll(monkeypatch, tmp_path): assert _ort.os.environ["ORT_DYLIB_PATH"] == str(dll) +def test_pins_to_package_capi_dylib_on_intel_macos(monkeypatch, tmp_path): + _force_intel_macos(monkeypatch) + pkg = tmp_path / "onnxruntime" + capi = pkg / "capi" + capi.mkdir(parents=True) + dylib = capi / "libonnxruntime.1.23.2.dylib" + dylib.write_bytes(b"not really a dylib") + _fake_spec_for(monkeypatch, pkg) + + assert _ort.ensure_ort_dylib_pinned() == str(dylib) + assert _ort.os.environ["ORT_DYLIB_PATH"] == str(dylib) + + def test_idempotent_after_first_resolution(monkeypatch, tmp_path): _force_windows(monkeypatch) pkg = tmp_path / "onnxruntime"