mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-08-27 14:17:10 -04:00
Production incident (Finding #2 of HEADROOM_PROXY_LOG_FINDINGS_2026_05_03.md): on this customer's deployment the Rust extension `headroom._core` was never installed into the runtime Docker image. Diff compression failed 54 times in a single day; "Optimization failed: ModuleNotFoundError" hit 379 times. The failure rate climbed every day and reached ~223/day on 2026-05-03 — effectively 100% of requests on the Rust path. Every Rust PR we'd merged (MessageScorer, ICM, DiffCompressor, etc.) was providing zero customer value because the module wasn't loadable at all. Root cause: the Dockerfile builder stage installed Python deps and the in-tree `headroom-ai` package but never ran `maturin build` for the `headroom-py` crate, so the runtime image shipped without `_core.so`. The Python proxy continued to start because the extension's absence is caught and routed through Python-only fallbacks that either silently no-op or raise per-request. This change makes that mode impossible by default: * `headroom.proxy.server._check_rust_core()` runs as the first step of the FastAPI lifespan. If the import fails it prints a structured diagnostic, logs `event=rust_core_missing`, and calls `sys.exit(78)` (sysexits.h `EX_CONFIG`). Process supervisors (systemd / k8s / docker) treat this as a deliberate config error and stop restart loops. * `HEADROOM_REQUIRE_RUST_CORE=false` is the explicit opt-out for Python-only `pip install -e .` developer flows; lifespan logs `event=rust_core_disabled` and continues. Any other value (including unset) keeps the fail-loud default. * `/health` now surfaces `rust_core: "loaded" | "disabled" | "missing"` (plus `rust_core_error` when non-loaded) so operators can alert on the degraded state rather than discovering it via a customer ticket. * `scripts/build_rust_extension.sh` is the single dev-time path: build → install → import-verify with the same `hello()` marker the lifespan checks. Failures are loud at every step. * `Makefile` exposes the script as `make verify-rust-core`. * `Dockerfile` now installs `rustup` + `maturin`, builds the wheel from `crates/headroom-py`, force-installs it into site-packages, and runs the same `hello()` import-verify in the build image so a broken build fails the docker-build, not the next runtime restart. Tests: * `tests/test_rust_core_smoke.py` pins all four contracts: - `_core.hello()` returns `"headroom-core"` - missing extension + default env → `SystemExit(78)` - missing extension + opt-out env → lifespan starts, `/health` returns `rust_core: "disabled"` with the underlying error - present extension + default env → `("loaded", None)` Per-finding-#2: ~/Desktop/HEADROOM_PROXY_LOG_FINDINGS_2026_05_03.md.
140 lines
5.3 KiB
Docker
140 lines
5.3 KiB
Docker
ARG PYTHON_VERSION=3.11
|
|
ARG UV_VERSION=0.6.17
|
|
# Pinned 2026-04-15. Update via Dependabot or: docker pull python:3.11-slim
|
|
ARG PYTHON_DIGEST=sha256:233de06753d30d120b1a3ce359d8d3be8bda78524cd8f520c99883bfe33964cf
|
|
# Pinned 2026-04-15. Update via Dependabot or: docker pull gcr.io/distroless/python3-debian13
|
|
ARG DISTROLESS_DIGEST=sha256:ed3a4beb46f8f8baac068743ba1b1f95ea3f793422129cf6dd23967f779b6018
|
|
ARG DISTROLESS_IMAGE=gcr.io/distroless/python3-debian13
|
|
ARG PYTHON_SITE_PACKAGES=/usr/local/lib/python${PYTHON_VERSION}/site-packages
|
|
|
|
# ---- Build stage: compile native extensions, build wheel ----
|
|
FROM python:${PYTHON_VERSION}-slim@${PYTHON_DIGEST} AS builder
|
|
|
|
ARG UV_VERSION
|
|
|
|
# build-essential / g++ for any C extension wheels uv may need to build
|
|
# from source. curl + ca-certificates are required by the rustup
|
|
# bootstrap below. Hotfix-A0 (Finding #2) added the rust toolchain so the
|
|
# image actually carries `headroom._core`; previously the runtime image
|
|
# shipped without the Rust extension and every compressed request fell
|
|
# back to a Python-only path or no-op.
|
|
RUN apt-get update && \
|
|
apt-get install -y --no-install-recommends \
|
|
build-essential \
|
|
g++ \
|
|
curl \
|
|
ca-certificates \
|
|
&& rm -rf /var/lib/apt/lists/*
|
|
|
|
RUN python -m pip install --no-cache-dir uv==${UV_VERSION}
|
|
|
|
# Rust toolchain for the headroom._core extension build. Pinned via
|
|
# rust-toolchain.toml at the repo root so this matches what local devs
|
|
# build with. Installed as root before WORKDIR change so the env
|
|
# additions stick for every subsequent RUN.
|
|
ENV CARGO_HOME=/usr/local/cargo \
|
|
RUSTUP_HOME=/usr/local/rustup \
|
|
PATH=/usr/local/cargo/bin:${PATH}
|
|
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \
|
|
| sh -s -- -y --no-modify-path --profile minimal --default-toolchain stable
|
|
|
|
WORKDIR /build
|
|
|
|
# Layer 1: install deps only (cached unless pyproject.toml/uv.lock change)
|
|
COPY pyproject.toml uv.lock README.md ./
|
|
# Stub package so uv can resolve the local extras without full source
|
|
RUN mkdir -p headroom && touch headroom/__init__.py
|
|
ARG HEADROOM_EXTRAS=proxy,code
|
|
RUN --mount=type=cache,target=/root/.cache/uv \
|
|
uv pip install --system ".[${HEADROOM_EXTRAS}]"
|
|
|
|
# Layer 2: copy real source, reinstall only headroom-ai (no deps)
|
|
COPY headroom/ headroom/
|
|
RUN --mount=type=cache,target=/root/.cache/uv \
|
|
uv pip install --system --no-deps --reinstall-package headroom-ai .
|
|
|
|
# Layer 3 (Hotfix-A0): build and install the Rust extension wheel. uv
|
|
# already installed `maturin` as a transitive of the [proxy]/[code]
|
|
# extras; if it didn't, install it explicitly here so the build never
|
|
# silently skips.
|
|
COPY crates/ crates/
|
|
COPY Cargo.toml Cargo.lock rust-toolchain.toml ./
|
|
RUN --mount=type=cache,target=/root/.cache/uv \
|
|
--mount=type=cache,target=/root/.cargo/registry \
|
|
--mount=type=cache,target=/build/target \
|
|
uv pip install --system maturin \
|
|
&& maturin build --release -m crates/headroom-py/Cargo.toml --out /build/wheels \
|
|
&& uv pip install --system --force-reinstall --no-deps /build/wheels/headroom_core_py-*.whl
|
|
|
|
# Layer 4 (Hotfix-A0): verify the extension actually loads end-to-end
|
|
# inside the build image. If this fails, the runtime image would fail
|
|
# its lifespan smoke test on every restart — better to break the build
|
|
# loudly here than ship a broken image.
|
|
RUN python -c "from headroom._core import hello; \
|
|
marker = hello(); \
|
|
assert marker == 'headroom-core', f'expected headroom-core, got {marker!r}'; \
|
|
print(f'build-stage rust core verify OK: {marker}')"
|
|
|
|
# ---- Runtime stage (python-slim): supports root/nonroot via build arg ----
|
|
FROM python:${PYTHON_VERSION}-slim@${PYTHON_DIGEST} AS runtime-slim-base
|
|
|
|
ARG RUNTIME_USER=nonroot
|
|
ARG PYTHON_SITE_PACKAGES
|
|
|
|
RUN apt-get update && \
|
|
apt-get install -y --no-install-recommends curl && \
|
|
rm -rf /var/lib/apt/lists/*
|
|
|
|
COPY --from=builder ${PYTHON_SITE_PACKAGES} ${PYTHON_SITE_PACKAGES}
|
|
COPY --from=builder /usr/local/bin/headroom /usr/local/bin/headroom
|
|
|
|
RUN mkdir -p /home/nonroot /data && \
|
|
if [ "$RUNTIME_USER" = "nonroot" ]; then \
|
|
groupadd --gid 1000 nonroot && \
|
|
useradd --uid 1000 --gid nonroot --create-home nonroot && \
|
|
mkdir -p /home/nonroot/.headroom && \
|
|
chown -R nonroot:nonroot /data /home/nonroot; \
|
|
else \
|
|
mkdir -p /root/.headroom; \
|
|
fi
|
|
|
|
USER ${RUNTIME_USER}
|
|
WORKDIR /home/nonroot
|
|
|
|
ENV HEADROOM_HOST=0.0.0.0 \
|
|
PYTHONUNBUFFERED=1 \
|
|
PYTHONDONTWRITEBYTECODE=1
|
|
|
|
EXPOSE 8787
|
|
|
|
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \
|
|
CMD ["curl", "--fail", "--silent", "http://127.0.0.1:8787/readyz"]
|
|
|
|
ENTRYPOINT ["headroom", "proxy"]
|
|
CMD ["--host", "0.0.0.0", "--port", "8787"]
|
|
|
|
FROM ${DISTROLESS_IMAGE}@${DISTROLESS_DIGEST} AS runtime-slim
|
|
|
|
ARG RUNTIME_USER=nonroot
|
|
ARG PYTHON_SITE_PACKAGES
|
|
|
|
COPY --from=builder ${PYTHON_SITE_PACKAGES} ${PYTHON_SITE_PACKAGES}
|
|
|
|
USER ${RUNTIME_USER}
|
|
WORKDIR /app
|
|
|
|
ENV HEADROOM_HOST=0.0.0.0 \
|
|
PYTHONUNBUFFERED=1 \
|
|
PYTHONDONTWRITEBYTECODE=1 \
|
|
PYTHONPATH=${PYTHON_SITE_PACKAGES}
|
|
|
|
EXPOSE 8787
|
|
|
|
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \
|
|
CMD ["python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8787/readyz', timeout=5)"]
|
|
|
|
ENTRYPOINT ["python3", "-m", "headroom.cli", "proxy"]
|
|
CMD ["--host", "0.0.0.0", "--port", "8787"]
|
|
|
|
# Default published image remains python-slim runtime
|
|
FROM runtime-slim-base AS runtime
|