mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-08-27 14:17:10 -04:00
## Description Rust fastembed enables ORT C API 24, but the Python dependency allowed ONNX Runtime 1.23.2. Entering ort's initializer with that library deadlocks permanently instead of returning an error. Align dependency resolution where compatible wheels exist and preflight native detection where they do not. Closes #2960 ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [ ] Performance improvement - [ ] Code refactoring (no functional changes) ## Changes Made - Require ONNX Runtime 1.24+ for Python 3.11+ in the proxy and voice extras. - Keep the available pre-1.24 runtime on Python 3.10 for Python ONNX consumers. - Refuse to auto-pin an incompatible runtime into the Rust extension. - Bypass native detection immediately when API 24 is unavailable, preserving Python fallback without a five-second watchdog delay or stuck native thread. - Add dependency, pinning, override, and router regression coverage. ## Testing - [x] Unit tests pass (`pytest`) - [x] Linting passes (`ruff check .`) - [ ] Type checking passes (`mypy headroom`) - [x] New tests added for new functionality - [x] Manual testing performed ### Test Output ```text $ uv run pytest -q tests/test_transforms/test_ort_dylib.py tests/test_onnx_dependency_contract.py tests/test_onnx_runtime.py tests/test_transforms/test_content_router.py 88 passed in 9.31s $ uv run ruff check headroom/_ort.py headroom/transforms/content_router.py tests/test_transforms/test_ort_dylib.py tests/test_onnx_dependency_contract.py All checks passed! ``` ## Real Behavior Proof - Environment: macOS arm64; Python 3.13.14 and uv-managed Python 3.10.20. - Exact command / steps: run the issue's direct `headroom._core.detect_content_type` call in a subprocess with a 12-second timeout on Python 3.13; run `_detect_content` on Python 3.10 after resolving the proxy extra. - Observed result: Python 3.13 resolves ORT 1.26.0 and native detection returns `json_array`; Python 3.10 resolves ORT 1.23.2, leaves `ORT_DYLIB_PATH` unset, reports compatibility false, and immediately returns the Python `json_array` fallback. - Not tested: Linux-specific shared-object execution locally; CI's existing Linux Rust job already preflights ORT 1.24+ and exercises native tests. ## Runtime Rollout Safety - Rollout-managed feature(s): Native Rust content detection. - Minimum rollout channel: Stable/default; this is a deadlock prevention guard. - Stable/default behavior changed: Python 3.11+ installs a compatible ORT; Python 3.10 skips incompatible native detection. - Kill switch / disable path: `HEADROOM_DETECT_BACKEND=python` remains available; an explicit `ORT_DYLIB_PATH` remains an operator override. - Unsafe override required: No. - Qualification impact: Native detection stays enabled only with API-24-compatible ORT. - Rollback path: Revert this PR, which restores the old watchdog-only degradation. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [x] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [x] I did **not** edit `CHANGELOG.md` — it is generated by release-please from my Conventional Commit PR title (a CI guard enforces this) ## Screenshots (if applicable) Not applicable. ## Additional Notes The large lockfile diff is dependency resolution: Python 3.10 keeps ORT 1.23.2 while 3.11+ resolves 1.26.0. The functional Python change is intentionally small and keeps explicit `ORT_DYLIB_PATH` overrides working.
159 lines
5.4 KiB
Python
159 lines
5.4 KiB
Python
"""Tests for headroom._ort -- the ORT_DYLIB_PATH auto-pin.
|
|
|
|
The resolver points the Rust core's ort-load-dynamic runtime at the pip
|
|
onnxruntime package's shared library on every platform: on Windows it
|
|
guards against the DLL search picking up the Windows ML System32
|
|
onnxruntime.dll, and on Linux/macOS it avoids static ORT import-time
|
|
CPU feature faults on older x86_64 CPUs (#1278). The platform is
|
|
monkeypatched so every branch runs on any CI OS.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
import headroom._ort as _ort
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _fresh_resolver(monkeypatch):
|
|
"""Reset the module-level cache and scrub the env before every test."""
|
|
monkeypatch.setattr(_ort, "_pinned", _ort._UNSET)
|
|
monkeypatch.setattr(_ort, "_pinned_from_override", False)
|
|
monkeypatch.setattr(_ort, "_installed_ort_version", lambda: (1, 24))
|
|
monkeypatch.delenv("ORT_DYLIB_PATH", raising=False)
|
|
|
|
|
|
def _force_windows(monkeypatch):
|
|
monkeypatch.setattr(sys, "platform", "win32")
|
|
|
|
|
|
def _fake_spec_for(monkeypatch, package_dir):
|
|
"""Make find_spec('onnxruntime') resolve to a fake package directory."""
|
|
spec = SimpleNamespace(origin=str(package_dir / "__init__.py"))
|
|
monkeypatch.setattr(
|
|
_ort.importlib.util,
|
|
"find_spec",
|
|
lambda name: spec if name == "onnxruntime" else None,
|
|
)
|
|
|
|
|
|
def test_pins_versioned_so_on_linux(monkeypatch, tmp_path):
|
|
monkeypatch.setattr(sys, "platform", "linux")
|
|
pkg = tmp_path / "onnxruntime"
|
|
capi = pkg / "capi"
|
|
capi.mkdir(parents=True)
|
|
so = capi / "libonnxruntime.so.1.22.0"
|
|
so.write_bytes(b"not really a shared object")
|
|
_fake_spec_for(monkeypatch, pkg)
|
|
|
|
assert _ort.ensure_ort_dylib_pinned() == str(so)
|
|
assert _ort.os.environ["ORT_DYLIB_PATH"] == str(so)
|
|
|
|
|
|
def test_pins_dylib_on_macos(monkeypatch, tmp_path):
|
|
monkeypatch.setattr(sys, "platform", "darwin")
|
|
pkg = tmp_path / "onnxruntime"
|
|
capi = pkg / "capi"
|
|
capi.mkdir(parents=True)
|
|
dylib = capi / "libonnxruntime.1.23.2.dylib"
|
|
dylib.write_bytes(b"not really a dylib")
|
|
_fake_spec_for(monkeypatch, pkg)
|
|
|
|
assert _ort.ensure_ort_dylib_pinned() == str(dylib)
|
|
assert _ort.os.environ["ORT_DYLIB_PATH"] == str(dylib)
|
|
|
|
|
|
def test_respects_existing_env(monkeypatch):
|
|
_force_windows(monkeypatch)
|
|
monkeypatch.setenv("ORT_DYLIB_PATH", r"C:\custom\onnxruntime.dll")
|
|
assert _ort.ensure_ort_dylib_pinned() == r"C:\custom\onnxruntime.dll"
|
|
assert _ort.os.environ["ORT_DYLIB_PATH"] == r"C:\custom\onnxruntime.dll"
|
|
assert _ort.rust_ort_runtime_compatible()
|
|
|
|
|
|
def test_incompatible_package_is_not_pinned(monkeypatch, tmp_path, caplog):
|
|
monkeypatch.setattr(_ort, "_installed_ort_version", lambda: (1, 23))
|
|
pkg = tmp_path / "onnxruntime"
|
|
capi = pkg / "capi"
|
|
capi.mkdir(parents=True)
|
|
(capi / "libonnxruntime.so.1.23.2").write_bytes(b"old")
|
|
_fake_spec_for(monkeypatch, pkg)
|
|
|
|
assert _ort.ensure_ort_dylib_pinned() is None
|
|
assert not _ort.rust_ort_runtime_compatible()
|
|
assert "older C API" in caplog.text
|
|
|
|
|
|
def test_content_router_bypasses_native_detector_for_incompatible_ort(monkeypatch, caplog):
|
|
import headroom.transforms.content_router as router
|
|
|
|
monkeypatch.setenv("HEADROOM_DETECT_BACKEND", "rust")
|
|
monkeypatch.setattr(_ort, "rust_ort_runtime_compatible", lambda: False)
|
|
monkeypatch.setattr(router, "_detect_native_unhealthy", False)
|
|
|
|
result = router._detect_content('{"safe": true}')
|
|
|
|
assert result.content_type.value.startswith("json")
|
|
assert router._detect_native_unhealthy is True
|
|
assert "requires ONNX Runtime 1.24+" in caplog.text
|
|
|
|
|
|
def test_pins_to_package_capi_dll(monkeypatch, tmp_path):
|
|
_force_windows(monkeypatch)
|
|
pkg = tmp_path / "onnxruntime"
|
|
capi = pkg / "capi"
|
|
capi.mkdir(parents=True)
|
|
dll = capi / "onnxruntime.dll"
|
|
dll.write_bytes(b"not really a dll")
|
|
_fake_spec_for(monkeypatch, pkg)
|
|
|
|
assert _ort.ensure_ort_dylib_pinned() == str(dll)
|
|
assert _ort.os.environ["ORT_DYLIB_PATH"] == str(dll)
|
|
|
|
|
|
def test_idempotent_after_first_resolution(monkeypatch, tmp_path):
|
|
_force_windows(monkeypatch)
|
|
pkg = tmp_path / "onnxruntime"
|
|
(pkg / "capi").mkdir(parents=True)
|
|
(pkg / "capi" / "onnxruntime.dll").write_bytes(b"x")
|
|
_fake_spec_for(monkeypatch, pkg)
|
|
|
|
first = _ort.ensure_ort_dylib_pinned()
|
|
# Resolution must not re-run: break find_spec and call again.
|
|
monkeypatch.setattr(
|
|
_ort.importlib.util,
|
|
"find_spec",
|
|
lambda name: pytest.fail("resolution ran twice"),
|
|
)
|
|
assert _ort.ensure_ort_dylib_pinned() == first
|
|
|
|
|
|
def test_no_pin_when_package_missing(monkeypatch):
|
|
_force_windows(monkeypatch)
|
|
monkeypatch.setattr(_ort.importlib.util, "find_spec", lambda name: None)
|
|
assert _ort.ensure_ort_dylib_pinned() is None
|
|
assert "ORT_DYLIB_PATH" not in _ort.os.environ
|
|
|
|
|
|
def test_no_pin_when_native_library_absent(monkeypatch, tmp_path):
|
|
_force_windows(monkeypatch)
|
|
pkg = tmp_path / "onnxruntime"
|
|
pkg.mkdir() # package exists, but no capi/onnxruntime.dll inside
|
|
_fake_spec_for(monkeypatch, pkg)
|
|
assert _ort.ensure_ort_dylib_pinned() is None
|
|
assert "ORT_DYLIB_PATH" not in _ort.os.environ
|
|
|
|
|
|
def test_never_raises(monkeypatch):
|
|
_force_windows(monkeypatch)
|
|
|
|
def boom(name):
|
|
raise RuntimeError("synthetic find_spec failure")
|
|
|
|
monkeypatch.setattr(_ort.importlib.util, "find_spec", boom)
|
|
assert _ort.ensure_ort_dylib_pinned() is None
|
|
assert "ORT_DYLIB_PATH" not in _ort.os.environ
|