diff --git a/.cargo/audit.toml b/.cargo/audit.toml new file mode 100644 index 000000000..e8a321e36 --- /dev/null +++ b/.cargo/audit.toml @@ -0,0 +1,23 @@ +# cargo-audit configuration for the Rust workspace. +# +# Path matters: cargo-audit reads `.cargo/audit.toml`, not a root-level +# `audit.toml`. A file at the repo root is silently ignored. +# +# The `audit` job in .github/workflows/rust.yml is a BLOCKING gate. It runs on +# every PR touching Rust and nightly on the schedule (the `rust-changes` job +# reports `rust=true` for `schedule`/`workflow_dispatch`, so a newly-disclosed +# advisory surfaces without anyone touching Rust code). +# +# It was `continue-on-error: true` until the change that added this file, which meant it reported findings +# nobody saw: RUSTSEC-2026-0258 (h2, unbounded empty DATA frames) sat in a green +# run. Anything ignored here has to be listed explicitly, with a reason. + +[advisories] +ignore = [ + # `paste` is unmaintained — an advisory of project status, not a + # vulnerability; there is no patched version to move to. It is transitive + # and unavoidable at our layer: tokenizers -> paste and rav1e -> paste, + # both reached via fastembed. Re-evaluate when tokenizers moves to + # `pastey` (the maintained drop-in fork). + "RUSTSEC-2024-0436", +] diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index f2743b3d8..da2b968e9 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -5,14 +5,14 @@ }, "metadata": { "description": "Headroom marketplace for Claude Code and GitHub Copilot CLI plugins.", - "version": "0.35.0" + "version": "0.36.0" }, "plugins": [ { "name": "headroom", "source": "./plugins/headroom-agent-hooks", "description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.", - "version": "0.35.0", + "version": "0.36.0", "author": { "name": "Headroom Contributors", "url": "https://github.com/chopratejas/headroom" diff --git a/.env.example b/.env.example index 0cab283d1..23c25635f 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,14 @@ -# Copy this file to .env and fill in real values before running in production. -# IMPORTANT: Change NEO4J_AUTH before deploying — default credentials are insecure. +# Copy this file to .env and fill in real values before running. +# docker-compose.yml requires these — it will refuse to start with defaults. + +# Neo4j credentials for the graph memory backend (format: user/password). NEO4J_AUTH=neo4j/CHANGEME +# Password only, for library / non-Docker use of the Neo4j memory backend. +NEO4J_PASSWORD=CHANGEME + +# Proxy token — gates the data plane whenever the proxy is not loopback-only. +# Generate: openssl rand -hex 32 +HEADROOM_PROXY_TOKEN=CHANGEME + +# Optional: set to 0.0.0.0 to expose the proxy on the network (requires a token). +# HEADROOM_BIND_ADDR=127.0.0.1 diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.json index f2743b3d8..da2b968e9 100644 --- a/.github/plugin/marketplace.json +++ b/.github/plugin/marketplace.json @@ -5,14 +5,14 @@ }, "metadata": { "description": "Headroom marketplace for Claude Code and GitHub Copilot CLI plugins.", - "version": "0.35.0" + "version": "0.36.0" }, "plugins": [ { "name": "headroom", "source": "./plugins/headroom-agent-hooks", "description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.", - "version": "0.35.0", + "version": "0.36.0", "author": { "name": "Headroom Contributors", "url": "https://github.com/chopratejas/headroom" diff --git a/.github/workflows/release-metadata-sync.yml b/.github/workflows/release-metadata-sync.yml index ef775cf3b..2526b9770 100644 --- a/.github/workflows/release-metadata-sync.yml +++ b/.github/workflows/release-metadata-sync.yml @@ -50,15 +50,26 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: + # Prefer a short-lived, repo-scoped GitHub App installation token over a + # personal PAT. Gated on the repo variable so an unconfigured app simply + # falls through to the existing chain instead of breaking the release. + - name: Mint installation token + id: app-token + if: ${{ vars.RELEASE_APP_ID != '' }} + continue-on-error: true + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ vars.RELEASE_APP_ID }} + private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} + - uses: actions/checkout@v7 with: ref: ${{ github.ref_name }} - # PAT (not GITHUB_TOKEN) for the same reason release-please.yml uses one: - # a push made with GITHUB_TOKEN does not trigger workflows, so the release - # PR's checks would never re-run against the synced commit and would stay - # red. Falls back to GITHUB_TOKEN, where the sync still lands and a manual - # re-run of the PR's checks picks it up. - token: ${{ secrets.RELEASE_PLEASE_TOKEN || secrets.GITHUB_TOKEN }} + # Do NOT persist the credential into .git/config. The next step runs + # scripts/version-sync.py *from the checked-out branch*, and this job + # triggers on a push to the unprotected glob release-please--branches--**. + # A persisted token would be readable by that script. + persist-credentials: false - uses: actions/setup-python@v6 with: @@ -72,6 +83,14 @@ jobs: run: python scripts/verify-versions.py - name: Commit and push if anything changed + env: + # An app installation token if one was minted, else the existing + # chain. A PAT (not GITHUB_TOKEN) is still preferred here for the same + # reason release-please.yml wants one: a push made with GITHUB_TOKEN + # does not trigger workflows, so the release PR's checks would never + # re-run against the synced commit and would stay red. Supplied only + # to this step, after the branch-supplied script has already run. + SYNC_TOKEN: ${{ steps.app-token.outputs.token || secrets.RELEASE_PLEASE_TOKEN || secrets.GITHUB_TOKEN }} run: | if git diff --quiet; then echo "Already in sync — nothing to commit." @@ -81,7 +100,12 @@ jobs: git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git add -A git commit -m "chore: sync generated version metadata" + # Push via an explicit remote URL because the checkout no longer + # persists credentials. Passed on stdin-free env expansion so the + # token is not written to the command line or into .git/config. # This push re-triggers this workflow. version-sync.py is idempotent, so # the next run finds no diff and exits above without pushing — the loop # terminates after one no-op run. - git push origin HEAD:"${GITHUB_REF_NAME}" + git push \ + "https://x-access-token:${SYNC_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" \ + HEAD:"${GITHUB_REF_NAME}" diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index ce458bfd8..c40893be2 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -42,16 +42,31 @@ jobs: release-please: runs-on: ubuntu-latest steps: + # Prefer a short-lived, repo-scoped GitHub App installation token. A + # personal PAT carries the maintainer's whole account — with a classic + # `repo` scope that reaches every other repository they can access — and + # this credential can tag past branch protection and reaches PyPI, npm and + # GHCR through the `release: published` publishes. An installation token is + # scoped to this repository and expires in an hour. Gated on the repo + # variable so an unconfigured app falls through instead of blocking a + # release. See #2955. + - name: Mint installation token + id: app-token + if: ${{ vars.RELEASE_APP_ID != '' }} + continue-on-error: true + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ vars.RELEASE_APP_ID }} + private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} + - uses: googleapis/release-please-action@v5 with: - # PAT (not GITHUB_TOKEN): a release/tag created by GITHUB_TOKEN does - # NOT emit events that trigger other workflows, so release.yml - # (PyPI/npm) and docker.yml — which fire on `release: published` — - # never ran, and releases had to be cut by hand. A PAT is treated as a - # real user, so the release it creates DOES trigger those publishes; it - # also lets the bot tag past branch/tag protection. Falls back to - # GITHUB_TOKEN when the secret is unset (the release PR still opens; it - # just won't trigger the downstream publishes). - token: ${{ secrets.RELEASE_PLEASE_TOKEN || secrets.GITHUB_TOKEN }} + # Neither an app token nor a PAT is GITHUB_TOKEN, and that matters: a + # release/tag created by GITHUB_TOKEN does NOT emit events that trigger + # other workflows, so release.yml (PyPI/npm) and docker.yml — which fire + # on `release: published` — never ran, and releases had to be cut by + # hand. Falls back to GITHUB_TOKEN when nothing else is set (the release + # PR still opens; it just won't trigger the downstream publishes). + token: ${{ steps.app-token.outputs.token || secrets.RELEASE_PLEASE_TOKEN || secrets.GITHUB_TOKEN }} config-file: .release-please-config.json manifest-file: .release-please-manifest.json diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index e14ef49d1..ce9d173ea 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -224,8 +224,12 @@ jobs: uses: taiki-e/install-action@v2 with: tool: cargo-audit,cargo-deny - - name: cargo audit (soft-fail) - continue-on-error: true + # Blocking. Soft-failing this made it useless: RUSTSEC-2026-0258 (h2, + # unbounded empty DATA frames -> unbounded memory or a panic) was + # reported by this job for as long as it existed and never turned a run + # red, so nobody acted on it. Accepted advisories go in audit.toml with + # a written reason rather than being swallowed wholesale here. + - name: cargo audit run: cargo audit - name: cargo deny check licenses continue-on-error: true diff --git a/.github/workflows/tools-hash-refresh.yml b/.github/workflows/tools-hash-refresh.yml new file mode 100644 index 000000000..d7bb22ee9 --- /dev/null +++ b/.github/workflows/tools-hash-refresh.yml @@ -0,0 +1,29 @@ +name: tools-hash-refresh + +# Enforce that headroom/tools.json SHA-256 pins match the published assets for +# the currently pinned tool versions. Fails if a version was bumped without +# refreshing pins (run scripts/refresh_tool_hashes.py locally). See WEB-03. + +on: + pull_request: + paths: + - "headroom/tools.json" + - "scripts/refresh_tool_hashes.py" + - ".github/workflows/tools-hash-refresh.yml" + schedule: + - cron: "0 6 * * 1" # Mondays 06:00 UTC + workflow_dispatch: {} + +permissions: + contents: read + +jobs: + verify-pins: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Verify tool SHA-256 pins + run: python scripts/refresh_tool_hashes.py --check diff --git a/.gitignore b/.gitignore index 4aae0af7a..8a477f307 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ scripts/* !scripts/audit_wheel_glibc_symbols.py !scripts/replay_codex_ws_load.py !scripts/export_kompress_v2_onnx.py +!scripts/refresh_tool_hashes.py !scripts/record_kompress_fixtures.py !scripts/record_code_compressor_fixtures.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f14940ae4..f362b3d0d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,7 +33,7 @@ repos: # unconditionally, so installing hooks is not required for enforcement. args: [--assume-in-merge] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.22 + rev: v0.16.2 hooks: - id: ruff args: [--fix] diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 3a39fd8cf..93c546c8d 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "0.35.0" + ".": "0.36.0" } diff --git a/.releasemetadata b/.releasemetadata index 017607cea..010cb2f47 100644 --- a/.releasemetadata +++ b/.releasemetadata @@ -1,10 +1,10 @@ { - "version": "0.35.0", + "version": "0.36.0", "packages": { - "pypi": "0.35.0", - "npm-sdk": "0.35.0", - "npm-openclaw": "0.35.0", - "npm-opencode": "0.35.0", - "agent-hooks-plugin": "0.35.0" + "pypi": "0.36.0", + "npm-sdk": "0.36.0", + "npm-openclaw": "0.36.0", + "npm-opencode": "0.36.0", + "agent-hooks-plugin": "0.36.0" } } diff --git a/CHANGELOG.md b/CHANGELOG.md index 89fc5d9d0..bdac98a30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -284,6 +284,95 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **code:** fix two `CodeAwareCompressor` AST-reassembly bugs: an exported JS/TS function or class (`export function foo() {`) produced a duplicated `export export` keyword and invalid syntax, because line-based node slicing (used to preserve indentation) pulled in the preceding `export` sibling's text on top of the `export_statement` handler's own prefix reconstruction. Separately, in every supported language, a doc comment immediately above a top-level function, class, or type was detached from its declaration during extraction and re-emitted in a cluster at the end of the compressed output instead of staying attached to what it documents. - * **proxy:** Buffered upstream responses containing a `server_tool_use` (or any other unrecognized Anthropic content block) no longer turn a fully-generated response into an HTTP 502. `StreamingMixin._response_to_sse` raised `ValueError` on unknown block types after the entire upstream generation had already been buffered, so a slow-but-successful response failed and the client retried the whole multi-minute request. Unknown blocks are now emitted verbatim in `content_block_start` (following the existing redacted_thinking` pattern), so `server_tool_use`, `server_tool_result`, `mcp_tool_use`, and future block types round-trip ([#1806](https://github.com/headroomlabs-ai/headroom/issues/1806)). +## [0.36.0](https://github.com/headroomlabs-ai/headroom/compare/v0.35.0...v0.36.0) (2026-08-20) + + +### Features + +* add deterministic runtime rollout controls ([#1490](https://github.com/headroomlabs-ai/headroom/issues/1490)) ([3077ac8](https://github.com/headroomlabs-ai/headroom/commit/3077ac81e8ef3ddefebbe308ea37a4e9bb2100e6)) +* **proxy:** let extensions report cost savings and their own latency ([#3051](https://github.com/headroomlabs-ai/headroom/issues/3051)) ([f9807fd](https://github.com/headroomlabs-ai/headroom/commit/f9807fd69e220f43068ec168515ae886dd36166f)) +* **proxy:** unify savings attribution across stats, perf, metrics, and dashboard ([1b0b0b8](https://github.com/headroomlabs-ai/headroom/commit/1b0b0b89a4bf751c8bd592890aef9c3b339e8e37)), closes [#2976](https://github.com/headroomlabs-ai/headroom/issues/2976) +* **wrap/claude:** make the --1m fallback model configurable via HEADROOM_1M_MODEL ([#2983](https://github.com/headroomlabs-ai/headroom/issues/2983)) ([2a84725](https://github.com/headroomlabs-ai/headroom/commit/2a8472525d3a027c95dc38a10c4b6707b482cabc)) + + +### Bug Fixes + +* **anthropic:** honor the [1m] 1M-context tier, and price it correctly ([#3073](https://github.com/headroomlabs-ai/headroom/issues/3073)) ([6d2254d](https://github.com/headroomlabs-ai/headroom/commit/6d2254dfb5eb97f92249e0ee7aa04b2697adfa69)) +* **ccr:** make --no-ccr disable server-side response handling too ([#3101](https://github.com/headroomlabs-ai/headroom/issues/3101)) ([131b119](https://github.com/headroomlabs-ai/headroom/commit/131b119c053e66fe825dabb3c242f6dc5c6049d7)), closes [#3082](https://github.com/headroomlabs-ai/headroom/issues/3082) +* **ccr:** make StreamingCCRHandler work on OpenAI streams ([#3069](https://github.com/headroomlabs-ai/headroom/issues/3069)) ([7ef736f](https://github.com/headroomlabs-ai/headroom/commit/7ef736fb1a8852a3dee52a362043c47084628a2a)) +* **ccr:** only buffer a stream when a marker is actually redeemable ([#3092](https://github.com/headroomlabs-ai/headroom/issues/3092)) ([c502087](https://github.com/headroomlabs-ai/headroom/commit/c502087db702e9b6aa1d1736086cf4a69a7775e6)) +* **ccr:** re-inject headroom_retrieve when history references it on the sessionless path ([942af56](https://github.com/headroomlabs-ai/headroom/commit/942af56f11cbd8466e25ae189c65ba56a9ddd602)) +* **ccr:** relay a successful upstream turn when post-processing fails ([#3094](https://github.com/headroomlabs-ai/headroom/issues/3094)) ([0ec73fa](https://github.com/headroomlabs-ai/headroom/commit/0ec73faa2805502a5c13eab7e4f086f8ae2e175e)) +* **ccr:** send Accept: application/json on a buffered stream:false turn ([#3102](https://github.com/headroomlabs-ai/headroom/issues/3102)) ([139c7cb](https://github.com/headroomlabs-ai/headroom/commit/139c7cbdde6a68ae3ade24341a79e5ba659c2cf3)), closes [#3078](https://github.com/headroomlabs-ai/headroom/issues/3078) +* **ccr:** verify a scanned marker's hash before advertising it ([#2908](https://github.com/headroomlabs-ai/headroom/issues/2908)) ([41dab2d](https://github.com/headroomlabs-ai/headroom/commit/41dab2d09925658b96fed492d534346ce1930f4c)) +* **ci:** prevent native detector from hanging test shards ([#2996](https://github.com/headroomlabs-ai/headroom/issues/2996)) ([a708c05](https://github.com/headroomlabs-ai/headroom/commit/a708c0571eecfb53eaab6b787b7a6ace9b21c162)) +* **ci:** scope the release credential and stop persisting it to disk ([#3062](https://github.com/headroomlabs-ai/headroom/issues/3062)) ([ac8646a](https://github.com/headroomlabs-ai/headroom/commit/ac8646aa3c6323c3c0b7051e09831f779859af6f)) +* **ci:** unjam release and Docker publishing ([#2958](https://github.com/headroomlabs-ai/headroom/issues/2958)) ([e269afb](https://github.com/headroomlabs-ai/headroom/commit/e269afb935f298a833a189acfb8573e908b3b60b)) +* **claude:** reject conflicting auth before proxy startup ([#2993](https://github.com/headroomlabs-ai/headroom/issues/2993)) ([2d88e31](https://github.com/headroomlabs-ai/headroom/commit/2d88e31a404e2be6c1c428deb2a387599eb820ba)) +* **cli/install:** resolve the deployment profile instead of dead-ending on default ([#2832](https://github.com/headroomlabs-ai/headroom/issues/2832)) ([8252619](https://github.com/headroomlabs-ai/headroom/commit/82526191a103a8d0e079d170e47631b3c2bcb0d9)) +* **cli:** stop the macOS malloc re-exec replacing an embedder's process ([#3064](https://github.com/headroomlabs-ai/headroom/issues/3064)) ([96c25f5](https://github.com/headroomlabs-ai/headroom/commit/96c25f518154536cf15f4e0b2d3fed80de6e67f6)) +* **copilot:** route VS Code inline completions to Copilot, not OpenAI ([#3077](https://github.com/headroomlabs-ai/headroom/issues/3077)) ([204e751](https://github.com/headroomlabs-ai/headroom/commit/204e751d2f01b0e987e9c05edec21664bb2df279)) +* **copilot:** send VS Code inline completions to the host that serves them ([#3112](https://github.com/headroomlabs-ai/headroom/issues/3112)) ([b77d612](https://github.com/headroomlabs-ai/headroom/commit/b77d61291399976985f12adcd6014aba2f0275cf)) +* **deps:** bump datasets past PYSEC-2026-3716 ([#3136](https://github.com/headroomlabs-ai/headroom/issues/3136)) ([df6ff6b](https://github.com/headroomlabs-ai/headroom/commit/df6ff6bd5b47837c1247cf4eb8ac151ebd799aa5)) +* **deps:** clear the two Rust advisories and make cargo audit blocking ([#3121](https://github.com/headroomlabs-ai/headroom/issues/3121)) ([93c474e](https://github.com/headroomlabs-ai/headroom/commit/93c474e84b2eeee147c274f3d75f48e5ea42d0d5)) +* **deps:** raise the GitPython floor to 3.1.58 to clear 9 open advisories ([#3120](https://github.com/headroomlabs-ai/headroom/issues/3120)) ([8156d4d](https://github.com/headroomlabs-ai/headroom/commit/8156d4dc3a376476513ef6f78104ff81d08967ac)) +* **docker:** publish compose ports on loopback only ([#3061](https://github.com/headroomlabs-ai/headroom/issues/3061)) ([481e0b8](https://github.com/headroomlabs-ai/headroom/commit/481e0b83d5393419b27b17d95767104c7c1bda26)) +* **docker:** ship Bedrock auth and current registry ([#2982](https://github.com/headroomlabs-ai/headroom/issues/2982)) ([eafdf11](https://github.com/headroomlabs-ai/headroom/commit/eafdf11a2cea44aabc51ce59bbc031e0aaee9640)) +* **doctor:** surface that Claude Desktop agent sessions bypass the proxy ([#2987](https://github.com/headroomlabs-ai/headroom/issues/2987)) ([be5b26d](https://github.com/headroomlabs-ai/headroom/commit/be5b26d807be81d83594c9144a8520f6f0f1b273)) +* **install:** consolidate Windows fallback and cleanup safety ([#2980](https://github.com/headroomlabs-ai/headroom/issues/2980)) ([ddd2a25](https://github.com/headroomlabs-ai/headroom/commit/ddd2a259ecce4e57202a68a74a2c1adcb879679b)) +* **install:** honor HEADROOM_PORT in install apply and deploy ([#3085](https://github.com/headroomlabs-ai/headroom/issues/3085)) ([58f28dc](https://github.com/headroomlabs-ai/headroom/commit/58f28dc7a6b6ce5bbf0f88524bd78cbe3f3ffa4b)) +* **install:** stop the PowerShell installer leaking temp dirs into the real user PATH ([#2985](https://github.com/headroomlabs-ai/headroom/issues/2985)) ([ddd9f76](https://github.com/headroomlabs-ai/headroom/commit/ddd9f76729d5662201b84bd0a51281cd3ac64ad3)) +* **learn:** include stdout in CLI failure messages, not just stderr ([#3080](https://github.com/headroomlabs-ai/headroom/issues/3080)) ([c5563d3](https://github.com/headroomlabs-ai/headroom/commit/c5563d3a7dd8b7f88767cf503f1b1696917e36ee)) +* **mcp:** restore SDK v1 compatibility cap ([#2978](https://github.com/headroomlabs-ai/headroom/issues/2978)) ([6077e5a](https://github.com/headroomlabs-ai/headroom/commit/6077e5a149ee6548edaff033f2cdffffce6ea0cf)) +* **memory:** sanitize entity_refs to prevent dict-shaped entries crashing search ([#2951](https://github.com/headroomlabs-ai/headroom/issues/2951)) ([2d1e96b](https://github.com/headroomlabs-ai/headroom/commit/2d1e96b85c61cc7aab821750f549f24d54cbb6f5)) +* **onnx:** enforce Rust API-24 runtime compatibility ([#2979](https://github.com/headroomlabs-ai/headroom/issues/2979)) ([a3fe5cb](https://github.com/headroomlabs-ai/headroom/commit/a3fe5cb65bed625e2a6cb415821bd0798754ce08)) +* **openclaw-plugin:** circuit breaker + per-request timeout for proxy resilience ([#639](https://github.com/headroomlabs-ai/headroom/issues/639)) ([6576ef6](https://github.com/headroomlabs-ai/headroom/commit/6576ef639cbb7be8bc5e6c25134956803d18f8d8)) +* **opencode:** send x-headroom-project header on all proxied requests ([#2868](https://github.com/headroomlabs-ai/headroom/issues/2868)) ([eeb038b](https://github.com/headroomlabs-ai/headroom/commit/eeb038bc0c28fc8078986db0849bfcff6743c158)) +* **policy:** price net-cost mutations with the 1h cache-write tier ([#2780](https://github.com/headroomlabs-ai/headroom/issues/2780)) ([ef7e07e](https://github.com/headroomlabs-ai/headroom/commit/ef7e07e0f5d6510ab96b5abb1698b1b681b5f9bf)) +* **providers:** don't crash on a non-object HEADROOM_MODEL_LIMITS / models.json ([#3089](https://github.com/headroomlabs-ai/headroom/issues/3089)) ([3ed8f76](https://github.com/headroomlabs-ai/headroom/commit/3ed8f7601935cb08eebfd34007e97675903180a5)) +* **proxy/anthropic:** don't buffer a CCR stream when passthrough discards the stream flip ([#2953](https://github.com/headroomlabs-ai/headroom/issues/2953)) ([f1c34d3](https://github.com/headroomlabs-ai/headroom/commit/f1c34d336cf35db341153c1c65e8c15219398340)) +* **proxy/anthropic:** don't replay recorded prefix over live history ([#3026](https://github.com/headroomlabs-ai/headroom/issues/3026)) ([#3052](https://github.com/headroomlabs-ai/headroom/issues/3052)) ([c16be9b](https://github.com/headroomlabs-ai/headroom/commit/c16be9bbbec4aec6d4b35e482c166daef8afa72c)) +* **proxy/anthropic:** repair headroom_retrieve history references the tools array cannot support ([#2876](https://github.com/headroomlabs-ai/headroom/issues/2876)) ([7de3573](https://github.com/headroomlabs-ai/headroom/commit/7de35739c61bed385dd078aee1b36865938c486d)) +* **proxy/anthropic:** stop answering a non-streaming turn with an event stream ([#3142](https://github.com/headroomlabs-ai/headroom/issues/3142)) ([0e26fb8](https://github.com/headroomlabs-ai/headroom/commit/0e26fb80de600795e96435473486c4a7c79c6eaa)) +* **proxy/cache:** strip cache_control from messages in the semantic cache key ([#3086](https://github.com/headroomlabs-ai/headroom/issues/3086)) ([2cae0f8](https://github.com/headroomlabs-ai/headroom/commit/2cae0f8eaf627f6b743deb215f7c19c499c26bcd)) +* **proxy/gemini:** guard CCR continuation usage against present-null counts ([#3035](https://github.com/headroomlabs-ai/headroom/issues/3035)) ([a01897c](https://github.com/headroomlabs-ai/headroom/commit/a01897c791f4bb6471defafd560d29d491eb2df8)) +* **proxy/openai:** propagate provider usage on the Responses WS->HTTP fallback ([#2988](https://github.com/headroomlabs-ai/headroom/issues/2988)) ([536c949](https://github.com/headroomlabs-ai/headroom/commit/536c949a692f4855719d71d612abc4968040286b)) +* **proxy:** adapt 200 SSE upstream replies on buffered /v1/responses instead of 502 ([#2622](https://github.com/headroomlabs-ai/headroom/issues/2622)) ([d76fce0](https://github.com/headroomlabs-ai/headroom/commit/d76fce04a39b3f206e38a02e012d50b2c728f7ca)) +* **proxy:** align signed-thinking wire accounting ([#3015](https://github.com/headroomlabs-ai/headroom/issues/3015)) ([b3f4436](https://github.com/headroomlabs-ai/headroom/commit/b3f443636d279d4bad845a8ef2bddb7ca50e9bc6)) +* **proxy:** complete stateless Responses and buffered CCR lifecycle ([#2997](https://github.com/headroomlabs-ai/headroom/issues/2997)) ([8a1d38b](https://github.com/headroomlabs-ai/headroom/commit/8a1d38bc5da87b49a530df22090c3a156d2d0cd6)) +* **proxy:** guard feedback endpoints and add CSRF checks to loopback writes ([#3060](https://github.com/headroomlabs-ai/headroom/issues/3060)) ([a6ab359](https://github.com/headroomlabs-ai/headroom/commit/a6ab359a5d8d67a85f734131b55dbcef768a821a)) +* **proxy:** keep prefixed core tools resident ([#3046](https://github.com/headroomlabs-ai/headroom/issues/3046)) ([2f4d001](https://github.com/headroomlabs-ai/headroom/commit/2f4d001c9ffd7f856c8dab3e31a8240a1c676f04)) +* **proxy:** preserve Codex WebSocket model attribution ([#3029](https://github.com/headroomlabs-ai/headroom/issues/3029)) ([a06a51e](https://github.com/headroomlabs-ai/headroom/commit/a06a51eca63f88271dfa77f2ee6bf3c8da6b24e4)) +* **proxy:** relocate stray system-role messages to the top-level system param ([#765](https://github.com/headroomlabs-ai/headroom/issues/765)) ([#1357](https://github.com/headroomlabs-ai/headroom/issues/1357)) ([9fde127](https://github.com/headroomlabs-ai/headroom/commit/9fde12753416a6102535235b822e44afebf76e9e)) +* **proxy:** restore the buffered-CCR heartbeat behind a grace window ([#3091](https://github.com/headroomlabs-ai/headroom/issues/3091)) ([a29d201](https://github.com/headroomlabs-ai/headroom/commit/a29d2015e5eaf72730a4155f0307cbfac1ea1c9b)) +* **proxy:** scope the signed-thinking lock to blocks that actually changed ([#3124](https://github.com/headroomlabs-ai/headroom/issues/3124)) ([17522fb](https://github.com/headroomlabs-ai/headroom/commit/17522fb0a1013c012e8123b1e713dbb2f3e770d9)) +* **proxy:** stop a lone surrogate turning a thinking body into a 500 ([#3134](https://github.com/headroomlabs-ai/headroom/issues/3134)) ([284ff31](https://github.com/headroomlabs-ai/headroom/commit/284ff31947ec9eac1de0e2dc1cf5de4933c29a50)) +* **proxy:** stop cached responses replaying the producing turn's wire framing ([#3024](https://github.com/headroomlabs-ai/headroom/issues/3024)) ([9d37059](https://github.com/headroomlabs-ai/headroom/commit/9d370592b022d01e6bc44a88649a611507794776)) +* **proxy:** stop operator secrets following a client-chosen upstream ([#3122](https://github.com/headroomlabs-ai/headroom/issues/3122)) ([05f5ef4](https://github.com/headroomlabs-ai/headroom/commit/05f5ef47cbc8b31a60458553d6bf240896a47e16)) +* **proxy:** tune macOS libmalloc and trim allocator pages so long-lived RSS stays bounded ([#2879](https://github.com/headroomlabs-ai/headroom/issues/2879)) ([6d87825](https://github.com/headroomlabs-ai/headroom/commit/6d87825f62e47bc65eeae05fbb8a131d545fe5a2)) +* **reporting:** show net vs gross savings, real skip thresholds, and the effective profile ([#3123](https://github.com/headroomlabs-ai/headroom/issues/3123)) ([250ede2](https://github.com/headroomlabs-ai/headroom/commit/250ede2f7f4752c0ab08831013fad3f753f4a578)) +* tool_search_tool_regex deferred and falsely resolved on direct-Anthropic path ([#2971](https://github.com/headroomlabs-ai/headroom/issues/2971)) ([8ea87e7](https://github.com/headroomlabs-ai/headroom/commit/8ea87e7804abfbb55beaf869e50dcb66deab975a)) +* **vscode:** persist compatible Claude modes and route Copilot CAPI ([#2986](https://github.com/headroomlabs-ai/headroom/issues/2986)) ([1aa701a](https://github.com/headroomlabs-ai/headroom/commit/1aa701adaa1ff792dd0e701f498d8d0326655670)) +* **wrap:** set xAI upstream for grok-build proxy ([#2772](https://github.com/headroomlabs-ai/headroom/issues/2772)) ([c831081](https://github.com/headroomlabs-ai/headroom/commit/c8310819a4221b0d120436786fc499a24c8e55f1)) +* **wrap:** stop the Serena pre-index stalling the launch path for 300s ([#2945](https://github.com/headroomlabs-ai/headroom/issues/2945)) ([6147883](https://github.com/headroomlabs-ai/headroom/commit/6147883d5e3a92cc7b890e6c05dce4391090c7e4)) +* **wrap:** verify proxy deps before mutating Codex config ([#1628](https://github.com/headroomlabs-ai/headroom/issues/1628)) ([b7f342c](https://github.com/headroomlabs-ai/headroom/commit/b7f342c153a3e6e43a9d3df006bcd4dd69842d00)) + + +### Performance Improvements + +* **perf:** skip rotated logs outside the requested window ([#3081](https://github.com/headroomlabs-ai/headroom/issues/3081)) ([6c9f41e](https://github.com/headroomlabs-ai/headroom/commit/6c9f41e08c47f2bfc440c5a4c6ac8a357ad5ada0)) + + +### Dependencies + +* bump axum from 0.7.9 to 0.8.9 ([#2966](https://github.com/headroomlabs-ai/headroom/issues/2966)) ([5731be7](https://github.com/headroomlabs-ai/headroom/commit/5731be7e68f57292aed40d76e770657a88f78c13)) +* bump criterion from 0.5.1 to 0.8.2 ([#2965](https://github.com/headroomlabs-ai/headroom/issues/2965)) ([b30f339](https://github.com/headroomlabs-ai/headroom/commit/b30f339d694abcd8dada76a34a1d69e30390bfc2)) +* bump ruff from 0.15.22 to 0.16.2 in the pip-minor-patch group across 1 directory ([#2962](https://github.com/headroomlabs-ai/headroom/issues/2962)) ([ff17961](https://github.com/headroomlabs-ai/headroom/commit/ff17961cd76a7cea1cff0a9dcfb7338929f37c5a)) +* bump sha2 from 0.10.9 to 0.11.0 ([#2288](https://github.com/headroomlabs-ai/headroom/issues/2288)) ([322425c](https://github.com/headroomlabs-ai/headroom/commit/322425c43bffde1ed0b64fecf3cf5951565dd82b)) +* bump the cargo-minor-patch group across 1 directory with 4 updates ([#2964](https://github.com/headroomlabs-ai/headroom/issues/2964)) ([888a9f4](https://github.com/headroomlabs-ai/headroom/commit/888a9f4e147cf1f87244977fac81d5e9613352d7)) +* bump tokio-tungstenite from 0.24.0 to 0.30.0 ([#2967](https://github.com/headroomlabs-ai/headroom/issues/2967)) ([bbe9013](https://github.com/headroomlabs-ai/headroom/commit/bbe901319d49a3d70caf7b37da2c29f7d7996e07)) +* update mcp requirement from <2.0.0,>=1.28.1 to >=1.28.1,<3.0.0 ([#2963](https://github.com/headroomlabs-ai/headroom/issues/2963)) ([d6fb536](https://github.com/headroomlabs-ai/headroom/commit/d6fb5365f67b9b7f90c7c55caead16ca6b41c586)) + ## [0.35.0](https://github.com/headroomlabs-ai/headroom/compare/v0.34.0...v0.35.0) (2026-08-12) diff --git a/Cargo.lock b/Cargo.lock index 2da32bedf..fe0ddb991 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -194,9 +194,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.92" +version = "0.1.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" dependencies = [ "proc-macro2", "quote", @@ -260,9 +260,9 @@ dependencies = [ [[package]] name = "aws-config" -version = "1.10.1" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b180a3c8b55960db3426d8964b8745e652466a1a49fe1a2eda828046d30b5e4" +checksum = "701418aa459dac33e50a0f8e818e5662a16bc018a6ac7423659b70f3799d67a8" dependencies = [ "aws-credential-types", "aws-runtime", @@ -325,9 +325,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.9.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9007227e10b5fed2f3e0a2beff489211e2b5604c400b7a9d5d81ca9d64c24bb" +checksum = "a6b50a43f3ccdf331521c6d6c68b7cc9668b6e09d439ebda9569df5722324d76" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -350,9 +350,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.105.0" +version = "1.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ffd0fbe7873cb548a7aa60f9573c268fff94155397fd4f14dc9f1ecaaab8516" +checksum = "b53416d16c278234845392e38d93bd4481d2f09daa0f005a2277f0aa91f59c22" dependencies = [ "arc-swap", "aws-credential-types", @@ -376,9 +376,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.107.0" +version = "1.106.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "175763eb222a46377df7aa257a3bca980ab3e96703fefc8f4d0b8da6ad2e254c" +checksum = "cc9b706c3305ed0285d5b1b696c747aa34950f830fb03e3e6c76890f99b9f188" dependencies = [ "arc-swap", "aws-credential-types", @@ -402,9 +402,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.110.0" +version = "1.109.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b14781dfbff48984017d57167b6ea0b6471c6920ec52b44a2677c7feb3c13" +checksum = "32d214cdfa5bbe17f117e76a7643fadf32a5234fb597322ef8b1fb4b2f17dbbd" dependencies = [ "arc-swap", "aws-credential-types", @@ -444,7 +444,7 @@ dependencies = [ "http 0.2.12", "http 1.5.0", "percent-encoding", - "sha2 0.11.0", + "sha2", "time", "tracing", ] @@ -540,9 +540,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.12.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07505b34e8f4b3591a4fa69e9792b52289b95488dbbc68c3c0075b7bedb245e1" +checksum = "bea94a9ff8464016338c851e24b472d7131c388c88898a502e781815b2ee6045" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -656,15 +656,15 @@ dependencies = [ [[package]] name = "axum" -version = "0.7.9" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ - "async-trait", "axum-core", "axum-macros", "base64 0.22.1", "bytes", + "form_urlencoded", "futures-util", "http 1.5.0", "http-body 1.0.1", @@ -677,15 +677,14 @@ dependencies = [ "mime", "percent-encoding", "pin-project-lite", - "rustversion", - "serde", + "serde_core", "serde_json", "serde_path_to_error", "serde_urlencoded", "sha1 0.10.6", "sync_wrapper", "tokio", - "tokio-tungstenite 0.24.0", + "tokio-tungstenite 0.29.0", "tower", "tower-layer", "tower-service", @@ -694,19 +693,17 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.4.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" dependencies = [ - "async-trait", "bytes", - "futures-util", + "futures-core", "http 1.5.0", "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", - "rustversion", "sync_wrapper", "tower-layer", "tower-service", @@ -715,9 +712,9 @@ dependencies = [ [[package]] name = "axum-macros" -version = "0.4.2" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d123550fa8d071b7255cb0cc04dc302baa6c8c4a79f55701552684d8399bce" +checksum = "7aa268c23bfbbd2c4363b9cd302a4f504fb2a9dfe7e3451d66f35dd392e20aca" dependencies = [ "proc-macro2", "quote", @@ -894,9 +891,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.2" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" +checksum = "9066c49992464636f92905fa096ec58baaa4d57ec19a5c096c68d3e25ef3d136" dependencies = [ "find-msvc-tools", "jobserver", @@ -1212,9 +1209,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] @@ -1808,9 +1805,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.15" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27" dependencies = [ "atomic-waker", "bytes", @@ -1899,7 +1896,7 @@ dependencies = [ "rusqlite", "serde", "serde_json", - "sha2 0.10.9", + "sha2", "tempfile", "thiserror 2.0.20", "tiktoken-rs", @@ -1962,7 +1959,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "sha2 0.10.9", + "sha2", "thiserror 2.0.20", "tokio", "tokio-stream", @@ -2564,9 +2561,9 @@ dependencies = [ [[package]] name = "libsqlite3-sys" -version = "0.38.2" +version = "0.38.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" +checksum = "f6c19a05435c21ac299d71b6a9c13db3e3f47c520517d58990a462a1397a61db" dependencies = [ "cc", "pkg-config", @@ -2669,9 +2666,9 @@ dependencies = [ [[package]] name = "matchit" -version = "0.7.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" [[package]] name = "matrixmultiply" @@ -3214,7 +3211,7 @@ dependencies = [ "bitflags", "num-traits", "rand 0.9.4", - "rand_chacha 0.9.0", + "rand_chacha", "rand_xorshift", "regex-syntax", "rusty-fork", @@ -3393,24 +3390,13 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "rand" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" -dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - [[package]] name = "rand" version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ - "rand_chacha 0.9.0", + "rand_chacha", "rand_core 0.9.5", ] @@ -3425,16 +3411,6 @@ dependencies = [ "rand_core 0.10.1", ] -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", -] - [[package]] name = "rand_chacha" version = "0.9.0" @@ -3445,15 +3421,6 @@ dependencies = [ "rand_core 0.9.5", ] -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.17", -] - [[package]] name = "rand_core" version = "0.9.5" @@ -3506,7 +3473,7 @@ dependencies = [ "paste", "profiling", "rand 0.9.4", - "rand_chacha 0.9.0", + "rand_chacha", "simd_helpers", "thiserror 2.0.20", "v_frame", @@ -3710,9 +3677,9 @@ dependencies = [ [[package]] name = "rusqlite" -version = "0.40.2" +version = "0.40.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" +checksum = "11438310b19e3109b6446c33d1ed5e889428cf2e278407bc7896bc4aaea43323" dependencies = [ "bitflags", "fallible-iterator", @@ -3995,17 +3962,6 @@ dependencies = [ "digest 0.11.3", ] -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest 0.10.7", -] - [[package]] name = "sha2" version = "0.11.0" @@ -4433,14 +4389,14 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.24.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" +checksum = "8f72a05e828585856dacd553fba484c242c46e391fb0e58917c942ee9202915c" dependencies = [ "futures-util", "log", "tokio", - "tungstenite 0.24.0", + "tungstenite 0.29.0", ] [[package]] @@ -4770,20 +4726,18 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tungstenite" -version = "0.24.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" +checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8" dependencies = [ - "byteorder", "bytes", "data-encoding", "http 1.5.0", "httparse", "log", - "rand 0.8.6", + "rand 0.9.4", "sha1 0.10.6", - "thiserror 1.0.69", - "utf-8", + "thiserror 2.0.20", ] [[package]] @@ -4922,12 +4876,6 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" -[[package]] -name = "utf-8" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" - [[package]] name = "utf8-zero" version = "0.8.1" diff --git a/Cargo.toml b/Cargo.toml index a9a66b0b5..c57cf2361 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,7 +61,7 @@ tracing = { version = "0.1", features = ["log"] } anyhow = "1" clap = { version = "4", features = ["derive"] } tokio = { version = "1", features = ["macros", "rt-multi-thread", "signal"] } -axum = "0.7" +axum = "0.8" tower = "0.5" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } pyo3 = { version = "0.29", features = ["abi3-py310"] } diff --git a/crates/headroom-core/Cargo.toml b/crates/headroom-core/Cargo.toml index 1c0dd8b13..6f81cdd1e 100644 --- a/crates/headroom-core/Cargo.toml +++ b/crates/headroom-core/Cargo.toml @@ -31,7 +31,7 @@ hf-hub = { version = "0.5", default-features = false, features = ["ureq", "rustl md-5 = "0.10" # `sha2` for `_hash_field_name` in smart_crusher (SHA256 truncated to 16 # hex chars). Python uses `hashlib.sha256` so we need byte-exact parity. -sha2 = "0.10" +sha2 = "0.11" # `dashmap` for the CCR storage backend. Concurrent HashMap with sharded # locking — distinct keys hashed to different shards never contend, so # multi-worker proxy load doesn't queue on a single Mutex. Lock-free diff --git a/crates/headroom-core/src/compression_policy.rs b/crates/headroom-core/src/compression_policy.rs index 4d9d786c6..145a9dd57 100644 --- a/crates/headroom-core/src/compression_policy.rs +++ b/crates/headroom-core/src/compression_policy.rs @@ -135,11 +135,24 @@ pub(crate) const MAX_LOSSY_RATIO_SUBSCRIPTION: f32 = 0.25; /// net-cost mutation formula (#856). pub const CACHE_WRITE_MULTIPLIER: f32 = 1.25; +/// Anthropic prompt-cache write multiplier for the 1-hour TTL tier. +pub const CACHE_WRITE_MULTIPLIER_1H: f32 = 2.0; + /// Anthropic prompt-cache read multiplier: a `cache_read` token costs /// 0.1× a plain input token. Input to the net-cost mutation formula /// (#856). pub const CACHE_READ_MULTIPLIER: f32 = 0.1; +/// Return the cache-write multiplier for a prompt-cache TTL tier. +/// +/// Invalid, missing, and non-positive values retain the 5-minute default. +pub fn cache_write_multiplier_for_ttl(ttl_seconds: Option) -> f32 { + match ttl_seconds { + Some(ttl) if ttl.is_finite() && ttl >= 3_600.0 => CACHE_WRITE_MULTIPLIER_1H, + _ => CACHE_WRITE_MULTIPLIER, + } +} + /// Per-auth-mode policy that downstream compression stages consult. /// /// `Copy` because the struct is small POD (two `bool`s + a `u32` + an @@ -269,7 +282,26 @@ impl CompressionPolicy { expected_reads: f32, p_alive: f32, ) -> f32 { - let w = CACHE_WRITE_MULTIPLIER; + self.net_mutation_gain_with_write_multiplier( + delta_t, + suffix_tokens, + expected_reads, + p_alive, + None, + ) + } + + /// Variant of [`Self::net_mutation_gain`] with an explicit cache-write + /// multiplier. `None` uses the 5-minute default. + pub fn net_mutation_gain_with_write_multiplier( + &self, + delta_t: u32, + suffix_tokens: u32, + expected_reads: f32, + p_alive: f32, + write_multiplier: Option, + ) -> f32 { + let w = write_multiplier.unwrap_or(CACHE_WRITE_MULTIPLIER); let r = CACHE_READ_MULTIPLIER; // f32::max ignores NaN (returns the other operand), so NaN reads // land on 0.0; clamp would propagate NaN, so guard alive explicitly. @@ -299,7 +331,32 @@ impl CompressionPolicy { expected_reads: f32, p_alive: f32, ) -> bool { - self.net_mutation_gain(delta_t, suffix_tokens, expected_reads, p_alive) > 0.0 + self.should_mutate_deep_with_write_multiplier( + delta_t, + suffix_tokens, + expected_reads, + p_alive, + None, + ) + } + + /// Variant of [`Self::should_mutate_deep`] with an explicit cache-write + /// multiplier. `None` uses the 5-minute default. + pub fn should_mutate_deep_with_write_multiplier( + &self, + delta_t: u32, + suffix_tokens: u32, + expected_reads: f32, + p_alive: f32, + write_multiplier: Option, + ) -> bool { + self.net_mutation_gain_with_write_multiplier( + delta_t, + suffix_tokens, + expected_reads, + p_alive, + write_multiplier, + ) > 0.0 } /// Remaining-read count at which a warm-cache (P_alive = 1) @@ -315,10 +372,21 @@ impl CompressionPolicy { /// session lasts N more turns"). Returns 0 when `delta_t` is 0 /// (no savings — callers gate on `delta_t > 0`). pub fn break_even_reads(&self, delta_t: u32, suffix_tokens: u32) -> f32 { + self.break_even_reads_with_write_multiplier(delta_t, suffix_tokens, None) + } + + /// Variant of [`Self::break_even_reads`] with an explicit cache-write + /// multiplier. `None` uses the 5-minute default. + pub fn break_even_reads_with_write_multiplier( + &self, + delta_t: u32, + suffix_tokens: u32, + write_multiplier: Option, + ) -> f32 { if delta_t == 0 { return 0.0; } - let w = CACHE_WRITE_MULTIPLIER; + let w = write_multiplier.unwrap_or(CACHE_WRITE_MULTIPLIER); let r = CACHE_READ_MULTIPLIER; ((w - r) / r) * ((suffix_tokens as f32) / (delta_t as f32)) } @@ -447,6 +515,29 @@ mod tests { assert!(p.should_mutate_deep(50_000, 10_000, 3.0, 1.0)); } + #[test] + fn net_gain_big_shave_shallow_suffix_is_loss_at_1h_tier() { + let p = CompressionPolicy::for_mode(AuthMode::Payg); + let default_gain = p.net_mutation_gain(50_000, 10_000, 3.0, 1.0); + assert!(default_gain > 0.0, "default gain = {default_gain}"); + + let gain = p.net_mutation_gain_with_write_multiplier( + 50_000, + 10_000, + 3.0, + 1.0, + Some(CACHE_WRITE_MULTIPLIER_1H), + ); + assert!((gain - (-4_000.0)).abs() < 1.0, "gain = {gain}"); + assert!(!p.should_mutate_deep_with_write_multiplier( + 50_000, + 10_000, + 3.0, + 1.0, + Some(CACHE_WRITE_MULTIPLIER_1H), + )); + } + #[test] fn net_gain_no_suffix_edit_profitable_with_reads_remaining() { // S = 0: nothing cached after the edit is invalidated. Warm-case diff --git a/crates/headroom-core/src/rollout.rs b/crates/headroom-core/src/rollout.rs index fc9d4fb92..1ff027413 100644 --- a/crates/headroom-core/src/rollout.rs +++ b/crates/headroom-core/src/rollout.rs @@ -323,7 +323,12 @@ pub fn feature_names() -> BTreeSet<&'static str> { fn digest_value(value: &Value) -> String { let canonical = serde_json::to_vec(value).expect("rollout provenance is serializable"); - format!("sha256:{:x}", Sha256::digest(canonical)) + let digest = Sha256::digest(canonical); + let mut hex = String::with_capacity(digest.len() * 2); + for byte in digest { + hex.push_str(&format!("{byte:02x}")); + } + format!("sha256:{hex}") } #[cfg(test)] diff --git a/crates/headroom-core/src/transforms/smart_crusher/hashing.rs b/crates/headroom-core/src/transforms/smart_crusher/hashing.rs index 2648639c5..437278bf1 100644 --- a/crates/headroom-core/src/transforms/smart_crusher/hashing.rs +++ b/crates/headroom-core/src/transforms/smart_crusher/hashing.rs @@ -28,7 +28,10 @@ pub fn hash_field_name(field_name: &str) -> String { let digest = hasher.finalize(); // Truncate to first 8 hex chars (4 bytes of digest). MUST match // Python's `[:8]` — see module-level note above. - let hex = format!("{:x}", digest); + let mut hex = String::with_capacity(digest.len() * 2); + for byte in digest { + hex.push_str(&format!("{byte:02x}")); + } hex[..8].to_string() } diff --git a/crates/headroom-proxy/Cargo.toml b/crates/headroom-proxy/Cargo.toml index f5feb7fc2..7345ca1db 100644 --- a/crates/headroom-proxy/Cargo.toml +++ b/crates/headroom-proxy/Cargo.toml @@ -77,7 +77,7 @@ prometheus = { version = "=0.14.0", default-features = false } # `aws-smithy-runtime-api`); promoted here to a direct, normal-build # dependency so the drift detector compiles outside `cfg(test)`. Also # used by PR-E4 for `prompt_cache_key` derivation. -sha2 = "0.10" +sha2 = "0.11" # PR-E6: bounded session-scoped cache of structural hashes. The # detector evicts the oldest session at 1000 entries — we never want # unbounded memory growth from a flood of unique session keys. `lru` @@ -113,7 +113,7 @@ tokio-stream = "0.1" # way to gate "the proxy did not perturb the request" because JSON # value-equality misses whitespace, key order, and Unicode escape # differences that all bust the prompt cache. -sha2 = "0.10" +sha2 = "0.11" # PR-C1: property tests for the byte-level SSE parser. The parser # must never panic on arbitrary input bytes (TCP can hand us anything, # including malformed UTF-8 split mid-codepoint or fuzz-generated diff --git a/crates/headroom-proxy/src/proxy.rs b/crates/headroom-proxy/src/proxy.rs index 1005fb03f..0389e14c2 100644 --- a/crates/headroom-proxy/src/proxy.rs +++ b/crates/headroom-proxy/src/proxy.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use std::time::Instant; use axum::body::{to_bytes, Body}; -use axum::extract::{ConnectInfo, DefaultBodyLimit, State, WebSocketUpgrade}; +use axum::extract::{ConnectInfo, DefaultBodyLimit, FromRequestParts, State, WebSocketUpgrade}; use axum::http::{HeaderMap, HeaderName, Request, Response, StatusCode, Uri}; use axum::response::IntoResponse; use axum::routing::{any, get, post}; @@ -189,14 +189,14 @@ pub fn build_app(state: AppState) -> Router { // publisher endpoints look like // `POST /v1beta1/projects/{p}/locations/{l}/publishers/anthropic/models/{m}:rawPredict` // (and `:streamRawPredict`). The trailing `:` is awkward - // in axum's `:param` syntax, so we capture the entire trailing - // segment as `:model_action` and split on the last `:` inside + // in axum's `{param}` syntax, so we capture the entire trailing + // segment as `{model_action}` and split on the last `:` inside // the dispatcher. Both verbs share the same axum route shape // — matchit can't distinguish two patterns that overlap on the // literal parameter. The verb dispatch lives in // [`crate::vertex::handle_vertex_predict_dispatch`]. .route( - "/v1beta1/projects/:project/locations/:location/publishers/anthropic/models/:model_action", + "/v1beta1/projects/{project}/locations/{location}/publishers/anthropic/models/{model_action}", post(crate::vertex::handle_vertex_predict_dispatch), ); @@ -219,11 +219,11 @@ pub fn build_app(state: AppState) -> Router { // Bedrock handlers identically. let bedrock_router: Router = Router::new() .route( - "/model/:model_id/invoke", + "/model/{model_id}/invoke", post(crate::bedrock::invoke::handle_invoke), ) .route( - "/model/:model_id/converse", + "/model/{model_id}/converse", post(crate::bedrock::invoke::handle_invoke), ) // PR-D2/PR-D5: streaming counterparts. Bedrock's protocol is @@ -235,11 +235,11 @@ pub fn build_app(state: AppState) -> Router { // processing pipeline, so both route to the same handler. // See `bedrock::invoke_streaming`. .route( - "/model/:model_id/invoke-with-response-stream", + "/model/{model_id}/invoke-with-response-stream", post(crate::bedrock::invoke_streaming::handle_invoke_streaming), ) .route( - "/model/:model_id/converse-stream", + "/model/{model_id}/converse-stream", post(crate::bedrock::invoke_streaming::handle_invoke_streaming), ) .route_layer(axum::middleware::from_fn( @@ -281,18 +281,18 @@ pub fn build_app(state: AppState) -> Router { post(crate::handlers::conversations::handle_conversations_create), ) .route( - "/v1/conversations/:conversation_id", + "/v1/conversations/{conversation_id}", get(crate::handlers::conversations::handle_conversations_get) .post(crate::handlers::conversations::handle_conversations_update) .delete(crate::handlers::conversations::handle_conversations_delete), ) .route( - "/v1/conversations/:conversation_id/items", + "/v1/conversations/{conversation_id}/items", post(crate::handlers::conversations::handle_conversations_items_create) .get(crate::handlers::conversations::handle_conversations_items_list), ) .route( - "/v1/conversations/:conversation_id/items/:item_id", + "/v1/conversations/{conversation_id}/items/{item_id}", get(crate::handlers::conversations::handle_conversations_item_get) .delete(crate::handlers::conversations::handle_conversations_item_delete), ); @@ -315,17 +315,22 @@ pub fn build_app(state: AppState) -> Router { async fn catch_all( State(state): State, ConnectInfo(client_addr): ConnectInfo, - ws: Option, req: Request, ) -> Response { - if is_websocket_upgrade(req.headers()) { - if let Some(ws) = ws { + let (mut parts, body) = req.into_parts(); + if is_websocket_upgrade(&parts.headers) { + // axum 0.8 requires optional extractors to opt in explicitly, and + // WebSocketUpgrade intentionally does not. Extract it only after the + // upgrade headers have identified this as a WebSocket request. + if let Ok(ws) = WebSocketUpgrade::from_request_parts(&mut parts, &state).await { + let req = Request::from_parts(parts, body); return ws_handler(ws, state, client_addr, req).await; } // Header says websocket but axum didn't extract it (likely missing // Sec-WebSocket-Key) — fall through to HTTP forwarding which will // surface the upstream error. } + let req = Request::from_parts(parts, body); forward_http(state, client_addr, req) .await .unwrap_or_else(|e| e.into_response()) diff --git a/crates/headroom-proxy/src/websocket.rs b/crates/headroom-proxy/src/websocket.rs index 509cd6bfd..3a1be2e4f 100644 --- a/crates/headroom-proxy/src/websocket.rs +++ b/crates/headroom-proxy/src/websocket.rs @@ -232,10 +232,10 @@ fn ax_to_tg(m: AxMsg) -> Option { fn tg_to_ax(m: TgMsg) -> Option { Some(match m { - TgMsg::Text(t) => AxMsg::Text(t.as_str().to_string()), - TgMsg::Binary(b) => AxMsg::Binary(b.to_vec()), - TgMsg::Ping(p) => AxMsg::Ping(p.to_vec()), - TgMsg::Pong(p) => AxMsg::Pong(p.to_vec()), + TgMsg::Text(t) => AxMsg::Text(t.as_str().to_string().into()), + TgMsg::Binary(b) => AxMsg::Binary(b.to_vec().into()), + TgMsg::Ping(p) => AxMsg::Ping(p.to_vec().into()), + TgMsg::Pong(p) => AxMsg::Pong(p.to_vec().into()), TgMsg::Close(Some(cf)) => AxMsg::Close(Some(CloseFrame { code: cf.code.into(), reason: cf.reason.to_string().into(), diff --git a/crates/headroom-proxy/tests/integration_bedrock_authmode.rs b/crates/headroom-proxy/tests/integration_bedrock_authmode.rs index 81fe30085..6fa056889 100644 --- a/crates/headroom-proxy/tests/integration_bedrock_authmode.rs +++ b/crates/headroom-proxy/tests/integration_bedrock_authmode.rs @@ -104,7 +104,7 @@ async fn bedrock_classified_as_oauth() { auth_mode.as_str().to_string() } let app = Router::new() - .route("/model/:model_id/invoke", post(probe)) + .route("/model/{model_id}/invoke", post(probe)) .route_layer(axum::middleware::from_fn(classify_and_attach_auth_mode)); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); diff --git a/docker-compose.yml b/docker-compose.yml index 204aecb81..c4af64da9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,10 +13,23 @@ # on its own (`docker run -p 8787:8787 ghcr.io/headroomlabs-ai/headroom`); the two # database services below are only required for the memory/relevance features. # -# Ports exposed on the host: +# Ports published on the host — all bound to 127.0.0.1 (this machine only): # 8787 proxy (OpenAI-compatible endpoint) # 6333 Qdrant REST 6334 Qdrant gRPC # 7474 Neo4j Browser 7687 Neo4j Bolt +# +# None of these three services authenticates inbound callers by default: the +# proxy's /v1/* data plane is open unless HEADROOM_PROXY_TOKEN is set, Qdrant +# has no API key, and Neo4j falls back to a published dev password. Publishing +# them on 0.0.0.0 therefore hands any peer on your network a relay through the +# proxy plus direct read/write on the embeddings and graph derived from your +# prompts. They are bound to loopback so that `docker compose up -d` is safe on +# a shared or untrusted network. +# +# To reach the proxy from another machine, publish it deliberately AND require +# a token — never one without the other: +# HEADROOM_PROXY_TOKEN=$(openssl rand -hex 32) # put this in .env +# ports: ["8787:8787"] # override in a compose override file # ============================================================================= services: @@ -31,6 +44,11 @@ services: command: ["--host", "0.0.0.0"] environment: - HEADROOM_HOST=0.0.0.0 + # The proxy binds 0.0.0.0 *inside* the container (required for Docker port + # forwarding); it is confined to host loopback by the published port below. + # A proxy token is required so the data plane is never open if you widen the + # bind. Generate one with: openssl rand -hex 32 + - HEADROOM_PROXY_TOKEN=${HEADROOM_PROXY_TOKEN:?set HEADROOM_PROXY_TOKEN (see .env.example; e.g. openssl rand -hex 32)} - HOME=/home/nonroot # Keep all Headroom read/write state on the named volume below. - HEADROOM_WORKSPACE_DIR=/home/nonroot/.headroom @@ -38,8 +56,14 @@ services: # if you want to use a custom OpenAI-compatible API endpoint, # uncomment and set the following line with the desired URL # - OPENAI_TARGET_API_URL=https://api.x.ai + # Required before publishing this port beyond loopback: without it the + # /v1/* data plane accepts unauthenticated callers. + # - HEADROOM_PROXY_TOKEN=${HEADROOM_PROXY_TOKEN} ports: - - "8787:8787" + # Loopback-only. The container still listens on 0.0.0.0 (above) so the + # other compose services can reach it by name; this line controls only + # which host interfaces the port is published on. + - "127.0.0.1:8787:8787" volumes: - headroom_workspace:/home/nonroot/.headroom # Readiness probe: the orchestrator polls /readyz so dependents and @@ -62,8 +86,10 @@ services: qdrant: image: qdrant/qdrant:v1.17.1 ports: - - "6333:6333" # REST API - - "6334:6334" # gRPC + # Loopback-only: Qdrant runs unauthenticated here and holds embeddings + # derived from your prompts. + - "127.0.0.1:6333:6333" # REST API + - "127.0.0.1:6334:6334" # gRPC # Named volume keeps the vector index across container restarts/recreates. volumes: - qdrant_data:/qdrant/storage @@ -75,20 +101,19 @@ services: neo4j: image: neo4j:5.26 ports: - - "7474:7474" # HTTP (Browser) - - "7687:7687" # Bolt + # Loopback-only to keep the graph store off the network. + - "127.0.0.1:7474:7474" # HTTP (Browser) + - "127.0.0.1:7687:7687" # Bolt # Named volume persists the graph data across container restarts/recreates. volumes: - neo4j_data:/data environment: - # Credentials come from .env (NEO4J_AUTH=user/password). The default here - # is for LOCAL DEV ONLY — override it before exposing Neo4j anywhere. - - NEO4J_AUTH=${NEO4J_AUTH:-neo4j/devpassword} + # No default credential — must be supplied (see .env.example). + - NEO4J_AUTH=${NEO4J_AUTH:?set NEO4J_AUTH, e.g. neo4j/} # APOC: Neo4j's standard procedure library, needed by Headroom's queries. - NEO4J_PLUGINS=["apoc"] - - NEO4J_apoc_export_file_enabled=true - - NEO4J_apoc_import_file_enabled=true - - NEO4J_apoc_import_file_use__neo4j__config=true + # APOC file import/export stays disabled (its Neo4j default) — it grants + # filesystem read/write via stored procedures. Do not enable unless required. # Named volumes — managed by Docker, survive `docker compose down` (use # `docker compose down -v` to delete the stored data as well). diff --git a/docs/content/docs/configuration.mdx b/docs/content/docs/configuration.mdx index c8a4fca82..fbcc6b297 100644 --- a/docs/content/docs/configuration.mdx +++ b/docs/content/docs/configuration.mdx @@ -151,6 +151,30 @@ curl http://127.0.0.1:8787/v1/messages \ When `HEADROOM_STRIP_INTERNAL_HEADERS` is `enabled` (the default), the proxy reads this header for routing and then strips it before forwarding upstream. +#### Configured secret headers are not sent to arbitrary upstreams + +`ANTHROPIC_TARGET_API_HEADERS` / `OPENAI_TARGET_API_HEADERS` hold operator +secrets. Because `x-headroom-base-url` is chosen by the *client*, those headers +are only attached when the destination is one the operator designated: + +- a host in the configured provider targets (`ANTHROPIC_TARGET_API_URL`, + `OPENAI_TARGET_API_URL`, and the Gemini/Vertex/Cloud Code equivalents), or +- a host listed in `HEADROOM_UPSTREAM_ALLOWED_HOSTS` (comma-separated). + +A request to any other upstream is **still proxied** — it just does not carry +your configured headers, and the proxy logs +`upstream_extra_headers_withheld host=` once per host. If you route to a +gateway via this header and need your configured headers to reach it, add its +host to `HEADROOM_UPSTREAM_ALLOWED_HOSTS`: + +```bash +export HEADROOM_UPSTREAM_ALLOWED_HOSTS="gateway.internal,api.example-gateway.ai" +``` + +Matching is on the parsed hostname and is exact — no wildcards — so +`api.anthropic.com.evil.example` and `https://api.anthropic.com@evil.example` +do not match `api.anthropic.com`. + ## SmartCrusher Configuration Fine-tune JSON compression behavior: @@ -210,6 +234,24 @@ response = client.chat.completions.create( The `RollingWindowConfig`, `IntelligentContextConfig`, and `ScoringWeights` classes are no longer part of Headroom. Context management now happens automatically inside the pipeline (live-zone-only compression). +### Claude 1M context window (`headroom wrap claude --1m`) + +`headroom wrap claude --1m` opts a Claude Code session into Anthropic's 1M-token context window by selecting a `[1m]`-suffixed model id, which makes Claude Code send the `context-1m` beta header. The model that `--1m` targets is resolved in this order: + +1. an explicit `--model` / `ANTHROPIC_MODEL` value (used as-is, with a `[1m]` suffix appended when missing), +2. otherwise `HEADROOM_1M_MODEL`, when set, +3. otherwise the built-in default (currently `claude-opus-5`). + +Set `HEADROOM_1M_MODEL` to point `--1m` at a specific model without pinning `ANTHROPIC_MODEL` globally, so the default can follow a new Opus generation without a code change: + +```bash +# Route --1m at a specific model for this shell / session +export HEADROOM_1M_MODEL=claude-opus-5 +headroom wrap claude --1m +``` + +`HEADROOM_1M_MODEL` is a fallback only: an explicit `--model` or `ANTHROPIC_MODEL` always wins. The value may be given with or without the `[1m]` suffix; both `claude-opus-5` and `claude-opus-5[1m]` are accepted, and the suffix is added when absent. + ## Pipeline Extensions Use a `headroom.pipeline_extension` entry point when you need to normalize or annotate requests before they leave Headroom. The `PRE_SEND` stage is the right place for provider-specific request cleanup, such as turning `content: null` into `content: ""` for upstreams that reject OpenAI-spec tool-call messages. @@ -317,6 +359,7 @@ headroom proxy --learn --min-evidence 3 | `HEADROOM_DEDUPE` | Whole-conversation verbatim cross-turn dedup in the router (cache-safe, information-preserving via retrieval markers). Superseded-read drop + lossless folds run without it; this adds verbatim dedup. | `off` | | `HEADROOM_CACHE_TTL_LEARN` | Append per-turn cache-outcome observations (provider, model, idle, hit/miss) to `cache_ttl_observations.jsonl` for the offline `headroom-cache-ttl` learner. Observation-only (no request-behavior change); respects `HEADROOM_STATELESS`; the log is size-bounded. | `off` | | `HEADROOM_KOMPRESS_ENDPOINT` / `HEADROOM_KOMPRESS_ENDPOINT_TOKEN` | Offload ML compression (Kompress) to a remote endpoint instead of the local ONNX model — used by reasoning compaction and the router when set. | -- | +| `HEADROOM_1M_MODEL` | Fallback model that `headroom wrap claude --1m` targets when neither `--model` nor `ANTHROPIC_MODEL` is set. Accepts the id with or without the `[1m]` suffix (added when absent); an explicit `--model` / `ANTHROPIC_MODEL` always wins. See [Claude 1M context window](#claude-1m-context-window-headroom-wrap-claude---1m). | `claude-opus-5` | For provider-only proxying, prefer `HEADROOM_HTTP_PROXY` over process-wide variables such as `HTTP_PROXY`, `HTTPS_PROXY`, `ALL_PROXY`, or `NO_PROXY`. HTTPX reads those global variables, but Headroom also passes them through to tool executions. diff --git a/docs/content/docs/pipeline-extensions.mdx b/docs/content/docs/pipeline-extensions.mdx index a525404ee..640099c06 100644 --- a/docs/content/docs/pipeline-extensions.mdx +++ b/docs/content/docs/pipeline-extensions.mdx @@ -78,6 +78,8 @@ curl http://localhost:8787/v1/chat/completions \ Internal `x-headroom-*` headers (including this one) are stripped before the request is forwarded upstream by default — see `HEADROOM_STRIP_INTERNAL_HEADERS` in [Configuration](/docs/configuration). +Because this header is client-driven, operator-configured secret headers (`OPENAI_TARGET_API_HEADERS` / `ANTHROPIC_TARGET_API_HEADERS`) are only attached when the resolved upstream host is one you designated — a configured provider target, or a host in `HEADROOM_UPSTREAM_ALLOWED_HOSTS`. Other upstreams are still routed to, just without those headers. See [Configuration](/docs/configuration) for details. + ## Per-request model routing with `request.state.headroom_route` `x-headroom-base-url` is client-driven and points at one OpenAI-compatible base. When the choice of model belongs to an extension instead of the caller — a router that picks a cheaper model per turn, say — publish it on the request state and Headroom serves that one request from a backend that speaks the target provider: diff --git a/docs/content/docs/proxy.mdx b/docs/content/docs/proxy.mdx index 35bb7a77f..3390a086d 100644 --- a/docs/content/docs/proxy.mdx +++ b/docs/content/docs/proxy.mdx @@ -284,6 +284,7 @@ Rewrite the upstream model per request — for example, send small, tool-free ca | `--telemetry` / `HEADROOM_TELEMETRY` | off | **Local-only** usage stats for your own `/stats`, `/metrics`, and dashboard. Nothing leaves the machine. | | `--log-file` / `HEADROOM_LOG_FILE` | none | JSONL request/response log. | | `--log-messages` | `false` | Include full message bodies in the log (may contain sensitive data). | +| `HEADROOM_LOG_LEVEL` | `warning` | uvicorn's log level (`critical`, `error`, `warning`, `info`, `debug`, `trace`). Raise to `info` for the per-request access log when diagnosing a deployed proxy. An unrecognized value warns and falls back to `warning`. | | `HEADROOM_OTEL_METRICS_ENABLED` | `false` | Export OpenTelemetry metrics (`HEADROOM_OTEL_METRICS_ENDPOINT`, …). See [OTLP export](/docs/metrics#opentelemetry-otlp-export). | | `HEADROOM_LANGFUSE_ENABLED` | `false` | Emit Langfuse traces (`LANGFUSE_PUBLIC_KEY` / `LANGFUSE_SECRET_KEY`). | diff --git a/docs/content/docs/vscode-copilot.mdx b/docs/content/docs/vscode-copilot.mdx index a552a946d..003224905 100644 --- a/docs/content/docs/vscode-copilot.mdx +++ b/docs/content/docs/vscode-copilot.mdx @@ -45,8 +45,8 @@ The command: 1. validates Copilot subscription access and resolves the account API endpoint; 2. starts Headroom on `127.0.0.1:8787` with the short-lived upstream token; 3. adds a marker-owned block to VS Code user settings containing - `github.copilot.advanced.debug.overrideProxyUrl` and - `github.copilot.advanced.debug.overrideAuthType`; + `github.copilot.advanced.debug.overrideProxyUrl` (inline completions) and + `github.copilot.advanced.debug.overrideCapiUrl` (chat); 4. keeps running until `Ctrl+C` so the local proxy is available to VS Code. Continue using Copilot's normal model picker. The request body—and therefore the diff --git a/docs/metrics-technical-guide.md b/docs/metrics-technical-guide.md new file mode 100644 index 000000000..63dbe691e --- /dev/null +++ b/docs/metrics-technical-guide.md @@ -0,0 +1,223 @@ +# Headroom Metrics — Dashboard Guide + +What each metric shows, so you can build panels against it. + +**Two endpoints.** Both are on the proxy (default `:8787`). + +| Surface | How to get it | Use it for | +|---|---|---| +| **Prometheus** — `GET /metrics` | Always on, no config | Everything below. Start here. | +| **OpenTelemetry** — OTLP/HTTP | `HEADROOM_OTEL_METRICS_ENABLED=1` + `pip install "headroom-ai[proxy,otel]"` | Same data, dotted names, plus per-tenant labels | + +Names differ between them: Prometheus uses `headroom_tokens_saved_total` (**milliseconds** for timings), OTel uses `headroom.proxy.tokens.saved` (**seconds**). Both are listed below. + +--- + +## The savings panel — start here + +**`headroom.proxy.tokens.saved`** is the headline number. It already combines compression + tool-schema deferral — no need to add anything to it. + +| Metric | What it shows | +|---|---| +| **`headroom.proxy.tokens.saved`** *(OTel)* | **Total input tokens Headroom kept out of the request.** Compression + tool savings, combined. This is your hero number. | +| `headroom.proxy.savings.usd{source}` *(OTel)* | **Dollars saved**, split by layer: `compression`, `tool_schema`, `output_shaping`, `provider_cache`. Sum for the total. | +| `headroom_persistent_savings_tokens_saved_total` | Same tokens-saved number, but **survives proxy restarts**. Use for "lifetime saved" tiles. | +| `headroom_persistent_savings_compression_savings_usd_total` | **Lifetime dollars saved**, durable across restarts. | +| `headroom_tokens_input_total` | Input tokens actually sent upstream (post-compression). The denominator for a reduction %. | +| `headroom_tokens_output_total` | Output tokens returned by the provider. | + +```promql +# Hero tile: tokens saved per second +rate(headroom_tokens_saved_total[5m]) + + sum(rate(headroom_savings_attributed_tokens_total{source="tool_search",realized="true"}[5m])) + +# Context reduction % +100 * rate(headroom_tokens_saved_total[5m]) + / clamp_min(rate(headroom_tokens_input_total[5m]) + rate(headroom_tokens_saved_total[5m]), 1) + +# Lifetime tiles (survive restart) +headroom_persistent_savings_tokens_saved_total +headroom_persistent_savings_compression_savings_usd_total +``` + +> **One catch on the Prometheus side.** `headroom_tokens_saved_total` is compression **only** — it leaves out tool-schema deferral. The OTel `headroom.proxy.tokens.saved` includes both. That's why the query above adds the `tool_search` term back in. On tool-heavy workloads the gap is large. + +--- + +## Latency panel + +All Prometheus timings are in **milliseconds**, exposed as `_sum` / `_count` / `_min` / `_max`. Build means with `rate(sum)/rate(count)`. + +| Metric | What it shows | +|---|---| +| **`headroom_overhead_ms_*`** | **Latency Headroom itself adds.** Handler entry → end of compression. Excludes the LLM call. This is the "what does this cost us" number. | +| `headroom_latency_ms_*` | Total request duration, including the provider. | +| `headroom_ttfb_ms_*` | Time to first byte from upstream. Streaming requests only. | +| `headroom_stage_timing_ms_*{path,stage}` | Where time went inside the handler — `compression_first_stage`, `upstream_connect`, `memory_context`, etc. | +| `headroom_transform_timing_ms_*{transform}` | Time per compression transform. Use to find a slow transform. | + +```promql +# Headroom's added overhead, mean ms +rate(headroom_overhead_ms_sum[5m]) / rate(headroom_overhead_ms_count[5m]) + +# End-to-end, mean ms +rate(headroom_latency_ms_sum[5m]) / rate(headroom_latency_ms_count[5m]) + +# Slowest stages +topk(5, rate(headroom_stage_timing_ms_sum[5m]) / rate(headroom_stage_timing_ms_count[5m])) +``` + +> **No percentiles are available.** There are no histogram buckets on `/metrics`, and the OTel histograms ship with default buckets that put every request into one bucket, so `histogram_quantile()` returns nonsense. **Means work fine.** For real p95/p99 today, use the `headroom perf` CLI. +> +> Also: divide each `_sum` by **its own** `_count`. Overhead and TTFB are only sampled when > 0, so their counts are smaller than the latency count. + +--- + +## Cache panel + +| Metric | What it shows | +|---|---| +| `headroom_provider_cache_hit_requests_total{provider}` | Requests that read from the provider's prompt cache. | +| `headroom_provider_cache_requests_total{provider}` | Requests with any cache activity. **The correct denominator for hit rate.** | +| `headroom_cache_read_tokens_total{provider}` | Tokens served from cache (the discounted ones). | +| `headroom_cache_write_tokens_total{provider}` | Tokens written into cache (these carry a premium). | +| `headroom_cache_write_ttl_tokens_total{provider,ttl}` | Cache writes split by TTL — `5m` vs `1h`. | +| `headroom_uncached_input_tokens_total{provider}` | Input tokens that missed cache entirely. | +| `headroom_cache_bust_total` | Requests where compression broke a cached prefix. **Should stay near zero.** | +| `headroom_cache_miss_attribution_total{provider,reason}` | Why a cached prefix missed — `ttl_expiry`, `prefix_change`, `unknown`. | + +```promql +# Cache hit rate by provider +sum by (provider) (rate(headroom_provider_cache_hit_requests_total[5m])) + / sum by (provider) (rate(headroom_provider_cache_requests_total[5m])) + +# Compression breaking cache — alert if this rises +rate(headroom_cache_bust_total[5m]) +``` + +> **Don't use `headroom_requests_cached_total` as a hit rate.** It mixes the provider's prompt cache with Headroom's own response cache into one boolean, so it measures neither. + +--- + +## Traffic & health panel + +| Metric | What it shows | +|---|---| +| `headroom_requests_total` | Requests handled. Unlabelled. | +| `headroom_requests_by_provider{provider}` | Traffic split by provider — `anthropic`, `openai`, `gemini`, `bedrock`… | +| `headroom_requests_by_model{model}` | Traffic split by model. Capped at 1024 distinct; overflow lands in `model="other"`. | +| `headroom_requests_failed_total` | Upstream 5xx errors. | +| `headroom_requests_rate_limited_total` | Requests **Headroom** rejected via its own rate limiter (not upstream 429s). | +| `headroom_compression_failed_total{reason}` | Compression failures — `timeout` or `error`. Fails open, so traffic keeps flowing but savings quietly stop. **Worth an alert.** | +| `headroom_compression_quarantine_total{event}` | Compression disabled after repeated timeouts — `activated`, `skipped`, `released`. | +| `headroom_inbound_requests_active` | In-flight requests, gauge. Counts all HTTP including `/metrics`. | +| `headroom_active_ws_sessions` | Live Codex WebSocket sessions, gauge. | + +```promql +# Failure rate +rate(headroom_requests_failed_total[5m]) + / clamp_min(rate(headroom_requests_total[5m]) + rate(headroom_requests_failed_total[5m]), 1) + +# Savings silently stopped +sum by (reason) (rate(headroom_compression_failed_total[5m])) + +# Traffic mix +sum by (provider) (rate(headroom_requests_by_provider[5m])) +``` + +--- + +## Anthropic subscription panel + +Only if you're on an Anthropic OAuth/subscription plan. OTel only, gauges, no labels. + +| Metric | What it shows | +|---|---| +| `headroom.subscription.5h_utilization_pct` | How much of the 5-hour rate-limit window is used (0–100). | +| `headroom.subscription.7d_utilization_pct` | Same for the 7-day window. | +| `headroom.subscription.5h_seconds_to_reset` | Seconds until the 5-hour window resets. | +| `headroom.subscription.7d_seconds_to_reset` | Seconds until the 7-day window resets. | +| `headroom.subscription.overage_usd` | Extra-usage credits consumed, in dollars. | + +--- + +## Attribution — where savings came from + +| Metric | What it shows | +|---|---| +| `headroom_savings_attributed_tokens_total{source,realized}` | Tokens saved, broken out by named source. `source="tool_search"` is tool-schema deferral. | +| `headroom_savings_attributed_usd_total{source,realized}` | Dollars saved by source. **Gauge, can go negative** — don't `rate()` it. | +| `headroom_savings_attribution_events_total{source,realized}` | How often each source contributed. | +| `headroom_waste_signal_tokens_total{signal}` | Wasteful patterns *detected* in the input — `json_bloat`, `base64`, `repetition`, `reread`… This is diagnosis, **not savings**. | + +These rows *explain* the headline total — they are never added to it. + +--- + +## Compression internals + +| Metric | What it shows | +|---|---| +| `headroom.compression.tokens.input` *(OTel)* | Tokens going into the compression pipeline. | +| `headroom.compression.tokens.output` *(OTel)* | Tokens coming out. | +| `headroom.compression.tokens.saved` *(OTel)* | The difference. Pipeline-level view of compression only. | +| `headroom.compression.runs` *(OTel)* | Pipeline executions. Note: **per pipeline run, not per request.** | +| `headroom.compression.pipeline.duration` *(OTel, seconds)* | How long the pipeline took. | +| `headroom.compression.transforms{transform}` *(OTel)* | Which transforms fired. **High cardinality — drop or aggregate at the collector.** | + +--- + +## Five things that will break a dashboard + +1. **Only savings counters survive a restart.** 55 of 60 Prometheus families reset to zero when the proxy restarts. Only `headroom_persistent_savings_*` is durable, and it needs `HEADROOM_WORKSPACE_DIR` on a persistent volume — otherwise it resets on every deploy. + +2. **No percentiles anywhere.** Use means. See the latency section. + +3. **`headroom_latency_ms` measures differently for streaming.** On streaming requests the timer starts *after* compression, so end-to-end is `latency + overhead`. On non-streaming it's just `latency`. Don't mix both in one panel. + +4. **A 5xx erases its own savings.** Requests that fail upstream are dropped from every savings and token counter. During a provider incident, savings rates look artificially clean while throughput falls. + +5. **`/metrics` needs auth if you set a proxy token.** With `HEADROOM_PROXY_TOKEN` set, any non-loopback scraper must send `Authorization: Bearer `. Loopback is always exempt. + +--- + +## Metrics the docs mention that don't exist + +If panels came back empty, this is probably why. These names appear in the published docs but not in the code: + +`headroom_compression_ratio` · `headroom_latency_seconds` (and `_bucket`) · `headroom_cache_hits_total` · `headroom_cache_misses_total` · `headroom_cost_usd_total` · the `mode="optimize"` label on `headroom_requests_total` + +The shipped `examples/grafana/headroom-dashboard.json` also filters every panel on `pool` and `hook` labels that no metric emits — the dropdowns will be permanently empty. Its metric names are otherwise correct. + +--- + +## Setup reference + +```bash +# Prometheus — nothing to do, GET /metrics is always on + +# OpenTelemetry +pip install "headroom-ai[proxy,otel]" +export HEADROOM_OTEL_METRICS_ENABLED=1 +export HEADROOM_OTEL_METRICS_ENDPOINT=https://otel.corp.example/v1/metrics +export HEADROOM_OTEL_METRICS_HEADERS="authorization=Bearer XXX" +export HEADROOM_OTEL_RESOURCE_ATTRIBUTES="service.instance.id=$HOSTNAME" +``` + +| Variable | Default | Notes | +|---|---|---| +| `HEADROOM_OTEL_METRICS_ENABLED` | `0` | Master switch | +| `HEADROOM_OTEL_METRICS_EXPORTER` | `otlp_http` | Or `console`. No gRPC exporter exists. | +| `HEADROOM_OTEL_METRICS_ENDPOINT` | unset | Passed verbatim — `/v1/metrics` is **not** appended | +| `HEADROOM_OTEL_METRICS_HEADERS` | unset | `k=v,k2=v2` | +| `HEADROOM_OTEL_METRICS_EXPORT_INTERVAL_MS` | `10000` | | +| `HEADROOM_OTEL_SERVICE_NAME` | `headroom-proxy` | | +| `HEADROOM_OTEL_RESOURCE_ATTRIBUTES` | unset | **Set `service.instance.id` here** — Headroom doesn't, and replicas will collide | + +Verify with `curl -s localhost:8787/stats | jq .otel`. + +**Multi-tenant labels:** `register_otel_metric_attribute_provider()` adds request-scoped attributes (tenant, team, cost centre) to every OTel datapoint. Max 16 attributes, 256 chars each. + +**Air-gapped deployments:** `HEADROOM_OFFLINE=1` disables all outbound traffic — the anonymous usage beacon (which is **on by default**), the update check, and model downloads. + +--- diff --git a/e2e/wrap/run.py b/e2e/wrap/run.py index c28a8b1a9..1a6b613c7 100644 --- a/e2e/wrap/run.py +++ b/e2e/wrap/run.py @@ -796,8 +796,9 @@ def verify_vscode_wrap(base_env: dict[str, str], project_dir: Path) -> None: "VS Code wrap should route Copilot Chat generation through Headroom", ) assert_true( - '"github.copilot.advanced.debug.overrideAuthType": "token"' in configured, - "VS Code wrap should configure token auth", + "overrideAuthType" not in configured, + "VS Code wrap must not write overrideAuthType: no such setting exists in " + "the modern Copilot Chat extension, so VS Code flags it as unknown (#3076)", ) assert_true( "synthetic-e2e-token" not in configured, "Settings must not contain credentials" diff --git a/headroom/agent_savings.py b/headroom/agent_savings.py index 9861bf8ea..a201dbb44 100644 --- a/headroom/agent_savings.py +++ b/headroom/agent_savings.py @@ -207,27 +207,43 @@ _PROFILES: dict[str, AgentSavingsProfile] = { def get_agent_savings_profile(name: str | None = None) -> AgentSavingsProfile: """Return a named agent savings profile. - An unrecognized name falls back to the ``balanced`` profile with a warning - instead of raising. The savings profile is a soft config knob, but it is - resolved during proxy startup (``proxy_pipeline_kwargs`` -> ``create_app``), - so raising here takes the whole proxy down before it can open its port. That - happens on desktop/runtime version skew: a newer client requests a profile - (e.g. ``coding``) that an older pinned or fallback runtime predates. Degrade - to ``balanced`` rather than leaving the user with no proxy at all. + An unrecognized name degrades with a warning instead of raising. The savings + profile is a soft config knob, but it is resolved during proxy startup + (``proxy_pipeline_kwargs`` -> ``create_app``), so raising here takes the + whole proxy down before it can open its port. That happens on desktop/runtime + version skew: a newer client requests a profile (e.g. ``coding``) that an + older pinned or fallback runtime predates. Degrade rather than leaving the + user with no proxy at all. + + **Where it degrades to matters.** This used to land on ``balanced`` + unconditionally, which is a drastically different posture from the + out-of-box default: cache->token mode, cross-turn dedup off, tool-search + off, user messages uncompressed, the message floor 25x higher (250 vs 10) + and the block floor 20x higher (500 vs 25). A single typo in + ``HEADROOM_SAVINGS_PROFILE`` therefore silently reconfigured the whole + proxy, and the only trace was one WARNING at startup that operators read + past. Prefer :data:`DEFAULT_PROFILE` — the documented out-of-box posture and + the same thing an unset variable resolves to, so a typo now costs nothing. + ``balanced`` remains the last resort for the genuine version-skew case, + where an older runtime has no ``DEFAULT_PROFILE`` entry to fall back to. """ key = (name or DEFAULT_PROFILE).strip().lower() profile = _PROFILES.get(key) if profile is not None: return profile + fallback_name = DEFAULT_PROFILE if DEFAULT_PROFILE in _PROFILES else FALLBACK_PROFILE valid = ", ".join(sorted(_PROFILES)) logger.warning( - "unknown savings profile %r; falling back to %r (known: %s)", + "unknown savings profile %r; falling back to %r (known: %s). " + "Set HEADROOM_SAVINGS_PROFILE to one of the known names, or unset it to " + "get %r explicitly.", name, - FALLBACK_PROFILE, + fallback_name, valid, + DEFAULT_PROFILE, ) - return _PROFILES[FALLBACK_PROFILE] + return _PROFILES[fallback_name] def apply_agent_savings_env_defaults( @@ -300,6 +316,28 @@ def proxy_pipeline_kwargs(config: object) -> dict[str, object]: # unset → Kompress decides / ambient default applies). if profile.target_ratio is not None: kwargs["target_ratio"] = profile.target_ratio + # Block-compression char floor. Every OTHER router pipeline kwarg in this + # function travels on the config object; this one alone was populated + # only from ``HEADROOM_MIN_CHARS_FOR_BLOCK`` (read below), so a proxy + # whose config carries ``savings_profile="coding"`` but whose process env + # was never seeded applied every sibling coding knob while this floor + # silently stayed at ``ContentRouterConfig.min_chars_for_block_compression`` + # (500) instead of the profile's 25 — a 20x gap on the gate that governs + # tool_result blocks, the dominant content type in agent traffic. + # + # NOTE: this does not make the profile fully config-deliverable. The + # profile's ``cross_turn_dedup`` / ``tool_search`` / ``lossless_then_lossy`` + # / ``protect_reads`` / ``code_aware`` / ``effort_router`` / ``lossless`` + # fields are still env-only, but by a different mechanism: their consumers + # read ``os.environ`` directly (ContentRouter.__init__ for HEADROOM_DEDUPE, + # the Anthropic handler for HEADROOM_TOOL_SEARCH) and never pass through + # this function at all. Those remain seed-dependent and are the reason a + # profile can still be half-applied; fixing them means threading each + # consumer, which is a larger change than this one. + # + # The env read below still wins, since it is an explicit operator override. + if profile.min_chars_for_block is not None: + kwargs["min_chars_for_block_compression"] = profile.min_chars_for_block if getattr(config, "compress_user_messages", False): kwargs["compress_user_messages"] = True diff --git a/headroom/binaries.py b/headroom/binaries.py index 229bd817e..200257871 100644 --- a/headroom/binaries.py +++ b/headroom/binaries.py @@ -226,6 +226,8 @@ def _mirror_url(url: str) -> str: mirror = os.environ.get("HEADROOM_BINARIES_MIRROR") if not mirror: return url + if not mirror.startswith("https://"): + raise BinaryFetchError(f"HEADROOM_BINARIES_MIRROR must use https:// (got {mirror!r})") # Only substitute the github.com host so that paths remain intact. for prefix in ("https://github.com", "https://objects.githubusercontent.com"): if url.startswith(prefix): @@ -245,6 +247,8 @@ def _download(url: str, dest: Path, *, progress: bool = True) -> None: if not _is_writable_dir(dest.parent): raise OSError(f"binary cache directory is not writable: {dest.parent}") final_url = _mirror_url(url) + if not final_url.startswith("https://"): + raise BinaryFetchError(f"refusing non-https download URL: {final_url!r}") req = urllib.request.Request(final_url, headers={"User-Agent": "headroom-binaries/1"}) attempts = 3 for attempt in range(1, attempts + 1): @@ -307,10 +311,16 @@ def _sha256_file(path: Path) -> str: def _verify_sha256(path: Path, expected: str | None) -> None: + if os.environ.get("HEADROOM_BINARIES_ALLOW_UNVERIFIED"): + logger.warning( + "skipping sha256 verification for %s (HEADROOM_BINARIES_ALLOW_UNVERIFIED=1)", + path.name, + ) + return if not expected: - # Upstream release not SHA-pinned in registry. HTTPS + the GitHub CDN - # is the only integrity check. Log at INFO so verbose runs can see - # this state; `doctor` surfaces the same fact via `sha_pinned=False`. + # No pin in the registry (e.g. an off-registry version override). All + # shipped assets ARE pinned — enforced by the tools-hash-refresh CI gate — + # so a missing pin means an off-registry fetch; fall back to HTTPS trust. logger.info("binary %s downloaded without sha256 pin (HTTPS trust only)", path.name) return got = _sha256_file(path) @@ -319,6 +329,40 @@ def _verify_sha256(path: Path, expected: str | None) -> None: raise Sha256Mismatch(f"sha256 mismatch for {path.name}: expected {expected}, got {got}") +def sha256_for_url(url: str) -> str | None: + """Return the registry's pinned sha256 for a download URL, if present.""" + for tool in _registry().get("tools", {}).values(): + for asset in tool.get("assets", {}).values(): + if asset.get("url") == url: + pin = asset.get("sha256") + return pin if isinstance(pin, str) else None + return None + + +def verify_download_bytes(data: bytes, *, url: str, name: str) -> None: + """Fail-closed integrity check for an in-memory downloaded archive. + + Used by installers (rtk, lean-ctx, codebase-memory-mcp) that download and + extract on their own instead of going through the fetch path above. Verifies + the bytes against the tools.json pin for ``url`` and refuses an unpinned URL + unless HEADROOM_BINARIES_ALLOW_UNVERIFIED=1. + """ + if os.environ.get("HEADROOM_BINARIES_ALLOW_UNVERIFIED"): + logger.warning( + "skipping sha256 verification for %s (HEADROOM_BINARIES_ALLOW_UNVERIFIED=1)", name + ) + return + expected = sha256_for_url(url) + if not expected: + # Off-registry URL (e.g. a version override); shipped assets are all + # pinned via the CI gate, so fall back to HTTPS trust here. + logger.info("%s downloaded without sha256 pin (HTTPS trust only)", name) + return + got = hashlib.sha256(data).hexdigest() + if got.lower() != expected.lower(): + raise Sha256Mismatch(f"sha256 mismatch for {name}: expected {expected}, got {got}") + + # ---------- Archive extraction ------------------------------------------- # diff --git a/headroom/cache/prefix_tracker.py b/headroom/cache/prefix_tracker.py index 8157e9e9a..200adddb8 100644 --- a/headroom/cache/prefix_tracker.py +++ b/headroom/cache/prefix_tracker.py @@ -449,7 +449,7 @@ def overlay_cached_prefix( previous_original_messages: list[dict[str, Any]] | None, previous_forwarded_messages: list[dict[str, Any]] | None, ) -> list[dict[str, Any]]: - """Replay the previously-forwarded (cached, compressed) prefix byte-identical. + """Replay a positional, non-inflating cached prefix when it is safe. Provider-agnostic cache-safety guard for the freeze path. When a message is "frozen", the compression pipeline may emit the agent's ORIGINAL bytes for @@ -467,14 +467,23 @@ def overlay_cached_prefix( ``optimized_messages`` unchanged (accept a possible bust rather than forward wrong content). - This makes freezing byte-identical in BOTH proxy modes, so the only remaining - difference between them is how large a mutable (still-compressible) tail each - leaves — not whether the frozen prefix busts the cache. + The optimized and current-original lists must be positionally aligned, and + compact UTF-8 JSON for the replayed result must not exceed the optimized + candidate. These bounds prefer a cache miss to corrupting or inflating a + client's live history. """ prev_orig = previous_original_messages prev_fwd = previous_forwarded_messages if not prev_orig or not prev_fwd: return optimized_messages + if len(optimized_messages) != len(current_original_messages): + logger.debug( + "overlay: optimized/current-original length mismatch (optimized=%d, current=%d) " + "— skipping positional cached-prefix replay", + len(optimized_messages), + len(current_original_messages), + ) + return optimized_messages n = len(prev_orig) # Positional 1:1 correspondence between prev_orig[i] and prev_fwd[i] holds # only when last turn forwarded exactly one message per original (the @@ -534,11 +543,21 @@ def overlay_cached_prefix( len(current_content) - split, message_index, ) - return ( + replayed = ( list(prev_fwd[:message_index]) + [merged] + list(optimized_messages[message_index + 1 :]) ) + replayed_bytes = _compact_json_bytes(replayed) + optimized_bytes = _compact_json_bytes(optimized_messages) + if ( + replayed_bytes is None + or optimized_bytes is None + or len(replayed_bytes) > len(optimized_bytes) + ): + logger.debug("overlay: block replay inflated compact JSON — skipping") + return optimized_messages + return replayed # Append-only guard on CONTENT ONLY, message-by-message. Replay the # previously-forwarded (cached, compressed) bytes for the longest LEADING # run of messages that is byte-for-byte (content-canonical) identical to @@ -562,7 +581,7 @@ def overlay_cached_prefix( # current_original[k] canonicalize-equals prev_orig[k], and prev_fwd[k] # positionally corresponds to prev_orig[k] (guaranteed by the count check # above), so no wrong bytes are ever forwarded. - limit = min(n, len(current_original_messages), len(optimized_messages)) + limit = min(n, len(current_original_messages)) k = 0 while k < limit and _canonicalize_for_prefix_compare( current_original_messages[k] @@ -584,7 +603,30 @@ def overlay_cached_prefix( ) # Replay the cached (compressed) prefix byte-identical up to the first # divergence; keep this turn's freshly-produced output for the rest. - return list(prev_fwd[:k]) + list(optimized_messages[k:]) + replayed = list(prev_fwd[:k]) + list(optimized_messages[k:]) + replayed_bytes = _compact_json_bytes(replayed) + optimized_bytes = _compact_json_bytes(optimized_messages) + if ( + replayed_bytes is None + or optimized_bytes is None + or len(replayed_bytes) > len(optimized_bytes) + ): + logger.debug("overlay: replay inflated compact JSON — skipping cached-prefix replay") + return optimized_messages + return replayed + + +def _compact_json_bytes(value: Any) -> bytes | None: + """Return compact JSON bytes, or ``None`` when sizing cannot be proved.""" + try: + return json.dumps( + value, + separators=(",", ":"), + ensure_ascii=False, + default=str, + ).encode("utf-8") + except (TypeError, ValueError, OverflowError, UnicodeError): + return None _STABLE_BOUNDARY_ENV = "HEADROOM_STABLE_BOUNDARY_BREAKPOINT" diff --git a/headroom/ccr/response_handler.py b/headroom/ccr/response_handler.py index 7cdea7a49..a4acfeb3d 100644 --- a/headroom/ccr/response_handler.py +++ b/headroom/ccr/response_handler.py @@ -571,11 +571,23 @@ class StreamingCCRBuffer: chunks: list[bytes] = field(default_factory=list) detected_ccr: bool = False complete_response: dict[str, Any] | None = None + provider: str = "anthropic" - # Patterns to detect tool_use in stream + # Wire markers for the start of a tool call. Anthropic streams + # `"type":"tool_use"` content blocks; OpenAI-compatible streams carry a + # `"tool_calls"` array inside `choices[].delta` and never emit the + # Anthropic marker, so scanning only for the latter meant CCR was never + # detected on an OpenAI stream. _tool_use_start: bytes = b'"type":"tool_use"' + _openai_tool_use_start: bytes = b'"tool_calls"' _ccr_tool_pattern: bytes = f'"{CCR_TOOL_NAME}"'.encode() + def _tool_call_marker(self) -> bytes: + """The provider's on-the-wire marker for the start of a tool call.""" + if self.provider == "anthropic": + return self._tool_use_start + return self._openai_tool_use_start + def add_chunk(self, chunk: bytes) -> bool: """Add a chunk and check for CCR tool calls. @@ -587,7 +599,7 @@ class StreamingCCRBuffer: # Quick check: does accumulated content contain CCR tool? accumulated = b"".join(self.chunks) - if self._tool_use_start in accumulated and self._ccr_tool_pattern in accumulated: + if self._tool_call_marker() in accumulated and self._ccr_tool_pattern in accumulated: self.detected_ccr = True return True @@ -622,7 +634,7 @@ class StreamingCCRHandler: ) -> None: self.response_handler = response_handler self.provider = provider - self.buffer = StreamingCCRBuffer() + self.buffer = StreamingCCRBuffer(provider=provider) async def process_stream( self, @@ -648,8 +660,12 @@ class StreamingCCRHandler: Response chunks (possibly from continuation response). """ # Phase 1: Initial detection - # Buffer chunks until we can determine if there's a CCR call - detection_complete = False + # Buffer chunks until we can determine if there's a CCR call. + # + # The end-of-stream marker is provider-specific. Anthropic signals the + # terminal state with `stop_reason` in `message_delta`; OpenAI-compatible + # streams have no such field and terminate with the `[DONE]` sentinel. + end_marker = b'"stop_reason"' if self.provider == "anthropic" else b"data: [DONE]" async for chunk in stream_iterator: self.buffer.add_chunk(chunk) @@ -660,9 +676,7 @@ class StreamingCCRHandler: accumulated = self.buffer.get_accumulated() # Look for stream end markers - if b'"stop_reason"' in accumulated: - detection_complete = True - + if end_marker in accumulated: if self.buffer.detected_ccr: # CCR detected - need to handle break @@ -679,13 +693,15 @@ class StreamingCCRHandler: yield buffered_chunk self.buffer.clear() - # Continue streaming rest of response - if not detection_complete and not self.buffer.detected_ccr: - async for chunk in stream_iterator: - if self.buffer.detected_ccr: - self.buffer.add_chunk(chunk) - else: - yield chunk + # The end marker is not guaranteed to arrive: upstream can truncate, a + # provider can omit the sentinel, or the stream can be a shape this + # detector does not recognise. Anything still buffered once the source + # iterator is exhausted is real response data the client has never + # seen, so flush it instead of dropping it. + if not self.buffer.detected_ccr and self.buffer.chunks: + for buffered_chunk in self.buffer.chunks: + yield buffered_chunk + self.buffer.clear() # Phase 2: Handle CCR if detected if self.buffer.detected_ccr: @@ -903,13 +919,38 @@ class StreamingCCRHandler: } tool_calls_map: dict[int, dict[str, Any]] = {} + finish_reason: str | None = None + envelope: dict[str, Any] = {} + usage: Any = None for event in events: - choices = event.get("choices", []) - if not choices: + # Carry the chunk envelope through. Dropping it left the + # reconstructed body without `id`, `model`, `created` or `usage`, + # which downstream middleware reads for routing and metering. + for key in ("id", "created", "model", "system_fingerprint"): + value = event.get(key) + if value is not None: + envelope[key] = value + if event.get("usage") is not None: + usage = event["usage"] + + choices = event.get("choices") + if not isinstance(choices, list) or not choices: + continue + choice = choices[0] + if not isinstance(choice, dict): continue - delta = choices[0].get("delta", {}) + # `finish_reason` is null on every chunk but the last, so keep the + # most recent non-null value rather than the first one seen. + if choice.get("finish_reason") is not None: + finish_reason = choice["finish_reason"] + + # Some OpenAI-compatible providers send `"delta": null` on the + # terminal chunk instead of an empty object. + delta = choice.get("delta") + if not isinstance(delta, dict): + delta = {} if "content" in delta and delta["content"]: message["content"] = (message.get("content") or "") + delta["content"] @@ -944,14 +985,94 @@ class StreamingCCRHandler: tc["function"]["arguments"] += fn["arguments"] message["tool_calls"] = [tool_calls_map[i] for i in sorted(tool_calls_map.keys())] - if not message["tool_calls"]: + has_tool_calls = bool(message["tool_calls"]) + if not has_tool_calls: del message["tool_calls"] if not message["content"]: message["content"] = None - return { - "choices": [{"message": message, "finish_reason": "stop"}], + # OpenAI requires `finish_reason: "tool_calls"` whenever the message + # carries tool calls. This was hardcoded to "stop", which tells any + # client that drives its agent loop off `finish_reason` that the turn + # is over, so the reconstructed tool calls were never executed. + if has_tool_calls: + finish_reason = "tool_calls" + elif finish_reason is None: + finish_reason = "stop" + + response: dict[str, Any] = { + "object": "chat.completion", + **envelope, + "choices": [{"index": 0, "message": message, "finish_reason": finish_reason}], } + if usage is not None: + response["usage"] = usage + return response + + def _openai_response_to_chunks(self, response: dict[str, Any]) -> list[bytes]: + """Split a non-streaming ``chat.completion`` body into SSE chunk frames. + + A streaming client reads ``choices[].delta``, not ``choices[].message``. + Serialising the reconstructed non-streaming body into a single SSE frame + produced a stream in which both the text and the tool calls were + invisible to the client. + """ + choices = response.get("choices") + choice = choices[0] if isinstance(choices, list) and choices else {} + if not isinstance(choice, dict): + choice = {} + message = choice.get("message") + if not isinstance(message, dict): + message = {} + finish_reason = choice.get("finish_reason") or "stop" + + base: dict[str, Any] = {"object": "chat.completion.chunk"} + for key in ("id", "created", "model", "system_fingerprint"): + if response.get(key) is not None: + base[key] = response[key] + + def frame(delta: dict[str, Any], reason: str | None) -> bytes: + payload = { + **base, + "choices": [{"index": 0, "delta": delta, "finish_reason": reason}], + } + return f"data: {json.dumps(payload)}\n\n".encode() + + frames = [frame({"role": message.get("role") or "assistant"}, None)] + + content = message.get("content") + if content: + frames.append(frame({"content": content}, None)) + + tool_calls = message.get("tool_calls") + if isinstance(tool_calls, list): + for index, tool_call in enumerate(tool_calls): + if not isinstance(tool_call, dict): + continue + function = tool_call.get("function") + if not isinstance(function, dict): + function = {} + frames.append( + frame( + { + "tool_calls": [ + { + "index": index, + "id": tool_call.get("id", ""), + "type": tool_call.get("type", "function"), + "function": { + "name": function.get("name", ""), + "arguments": function.get("arguments", ""), + }, + } + ] + }, + None, + ) + ) + + frames.append(frame({}, finish_reason)) + return frames async def _response_to_sse( self, @@ -968,6 +1089,7 @@ class StreamingCCRHandler: for chunk in StreamingMixin()._response_to_sse(response, "anthropic"): yield chunk else: - # OpenAI SSE format - yield f"data: {json.dumps(response)}\n\n".encode() + # OpenAI SSE format: `chat.completion.chunk` frames, then [DONE]. + for chunk in self._openai_response_to_chunks(response): + yield chunk yield b"data: [DONE]\n\n" diff --git a/headroom/cli/doctor.py b/headroom/cli/doctor.py index a45a0cbf1..cd5d4839a 100644 --- a/headroom/cli/doctor.py +++ b/headroom/cli/doctor.py @@ -14,6 +14,7 @@ from __future__ import annotations import json import os import re +import sys from collections.abc import Callable, Mapping from dataclasses import asdict, dataclass from datetime import datetime @@ -224,6 +225,51 @@ def check_claude_auth_conflict( ) +def claude_desktop_config_dir() -> Path: + """Return Claude Desktop's per-user config directory for this platform. + + Claude Desktop (``com.anthropic.claudefordesktop``) stores its config here, + distinct from Claude Code CLI's ``~/.claude``. Directory existence is used as + a proxy for "Desktop is installed / has been run" (#2925). + """ + home = Path.home() + if sys.platform == "darwin": + return home / "Library" / "Application Support" / "Claude" + if os.name == "nt": + appdata = os.environ.get("APPDATA") + base = Path(appdata) if appdata else home / "AppData" / "Roaming" + return base / "Claude" + xdg = os.environ.get("XDG_CONFIG_HOME") + base = Path(xdg) if xdg else home / ".config" + return base / "Claude" + + +def check_claude_desktop(config_dir: Path) -> CheckResult | None: + """Surface that Claude Desktop agent sessions bypass the proxy (#2925 / #869). + + Claude Desktop unconditionally overwrites ``ANTHROPIC_BASE_URL`` when it + spawns agent sessions, so a correctly-wrapped ``~/.claude/settings.json`` + (which the ``claude`` check verifies for the terminal CLI) does not route + Desktop traffic. Without this, ``doctor`` passes on the settings value alone + and never hints that Desktop sessions are unrouted. + + Reported as its own per-surface row -- like ``wrap_marker`` and ``shell env`` + -- and only when Desktop is detected, so it never contradicts a genuinely + routed CLI. Returns ``None`` when Desktop is absent (no row). + """ + if not config_dir.exists(): + return None + return CheckResult( + name="claude desktop", + status=WARN, + summary="agent sessions bypass the proxy (Desktop overwrites ANTHROPIC_BASE_URL)", + hint=( + "Desktop routing is not supported yet (see #869); use the terminal " + "Claude Code CLI for proxy-routed sessions." + ), + ) + + def check_claude_remote_control_gate( settings_path: Path, environ: Mapping[str, str], @@ -630,6 +676,9 @@ def doctor(port: int, emit_json: bool) -> None: ) if remote_control_gate_check is not None: checks.append(remote_control_gate_check) + desktop_check = check_claude_desktop(claude_desktop_config_dir()) + if desktop_check is not None: + checks.append(desktop_check) deployments = check_deployments(list_manifests()) if deployments is not None: checks.append(deployments) diff --git a/headroom/cli/install.py b/headroom/cli/install.py index 50e897ad4..a81a7d941 100644 --- a/headroom/cli/install.py +++ b/headroom/cli/install.py @@ -505,9 +505,10 @@ def _echo_installed(manifest: DeploymentManifest, *, prefix: str = "Installed pe "--port", "-p", default=8787, + envvar="HEADROOM_PORT", type=click.IntRange(1, 65535), show_default=True, - help="Persistent proxy port.", + help="Persistent proxy port (env: HEADROOM_PORT).", ) @click.option( "--backend", @@ -682,7 +683,13 @@ def install_apply( @main.command("deploy") @click.option("--profile", default="default", show_default=True, help="Deployment profile name.") @click.option( - "--port", "-p", default=8787, type=int, show_default=True, help="Persistent proxy port." + "--port", + "-p", + default=8787, + envvar="HEADROOM_PORT", + type=int, + show_default=True, + help="Persistent proxy port (env: HEADROOM_PORT).", ) @click.option( "--backend", diff --git a/headroom/cli/proxy.py b/headroom/cli/proxy.py index 73e2aee8a..e802bd0d6 100644 --- a/headroom/cli/proxy.py +++ b/headroom/cli/proxy.py @@ -5,6 +5,7 @@ import os import sys import warnings from importlib import import_module +from pathlib import Path from typing import Any, Literal, cast import click @@ -113,6 +114,55 @@ def _get_env_bool_optional(name: str) -> bool | None: return _get_env_bool(name, False) +# libmalloc reads these before main() runs, so they cannot be set from inside +# the current process — the proxy re-execs itself once to apply them. Without +# them, freed pages from large concurrent request bodies stay resident +# (``vmmap`` shows whole "MALLOC_LARGE (empty)" regions) and long-lived proxy +# RSS only ratchets upward (#2820). Vars the operator already set are left +# untouched; HEADROOM_MALLOC_TUNING=0 disables the re-exec entirely. +_MALLOC_TUNING = { + "MallocAggressiveMadvise": "1", # madvise freed pages back to the OS eagerly + "MallocLargeCache": "0", # no death-row cache for freed large allocations +} + + +def _process_is_headroom_cli_entrypoint() -> bool: + """Is this process the Headroom CLI itself, rather than an embedder? + + ``_reexec_with_malloc_tuning`` rebuilds the command line as + ``python -m headroom.cli ``. That is only a faithful + reconstruction when the process really was started as the Headroom CLI. If + something else invoked the ``proxy`` command in-process — pytest's + ``CliRunner``, an embedding application, ``runpy`` — then ``argv[1:]`` + belongs to *that* program, and ``os.execv`` would replace it with a Headroom + process parsing arguments that were never meant for us. + """ + argv0 = Path(sys.argv[0] or "") + if argv0.name in {"headroom", "headroom.exe"}: + return True + # `python -m headroom.cli` sets argv[0] to .../headroom/cli/__main__.py. + return argv0.parts[-3:] == ("headroom", "cli", "__main__.py") + + +def _reexec_with_malloc_tuning() -> None: + if sys.platform != "darwin": + return + if not _get_env_bool("HEADROOM_MALLOC_TUNING", True): + return + if os.environ.get("_HEADROOM_MALLOC_TUNED") == "1": + return + if not _process_is_headroom_cli_entrypoint(): + return + missing = {k: v for k, v in _MALLOC_TUNING.items() if k not in os.environ} + # Set the loop guard before the re-exec so the replacement process (which + # inherits this environment) skips this path instead of re-execing forever. + os.environ["_HEADROOM_MALLOC_TUNED"] = "1" + if not missing: + return + os.environ.update(missing) + os.execv(sys.executable, [sys.executable, "-m", "headroom.cli", *sys.argv[1:]]) + + def _get_env_int_optional(name: str) -> int | None: val = os.environ.get(name) if val is None or val == "": @@ -1065,6 +1115,7 @@ def proxy( Usage with OpenAI-compatible clients: OPENAI_BASE_URL=http://localhost:8787/v1 your-app """ + _reexec_with_malloc_tuning() ensure_proxy_dependencies() # Import here to avoid slow startup @@ -1261,6 +1312,10 @@ def proxy( rate_limit_requests_per_minute=rpm if rpm is not None else 60, rate_limit_tokens_per_minute=tpm if tpm is not None else 100_000, compress_user_messages=_get_env_bool("HEADROOM_COMPRESS_USER_MESSAGES", False), + periodic_malloc_trim_enabled=_get_env_bool( + "HEADROOM_MALLOC_TRIM", sys.platform == "darwin" + ), + malloc_trim_interval_seconds=_get_env_int("HEADROOM_MALLOC_TRIM_INTERVAL_SECONDS", 60), min_tokens_to_crush=_get_env_int("HEADROOM_MIN_TOKENS", 500), max_items_after_crush=_get_env_int("HEADROOM_MAX_ITEMS", 50), exclude_tools=_parse_exclude_tools(None) or None, @@ -1275,12 +1330,22 @@ def proxy( protect_recent=_get_env_int_optional("HEADROOM_PROTECT_RECENT"), protect_analysis_context=_get_env_bool_optional("HEADROOM_PROTECT_ANALYSIS_CONTEXT"), accuracy_guard=os.environ.get("HEADROOM_ACCURACY_GUARD") or None, - # CCR opt-out: --no-ccr disables both halves at once (markers in content - # AND the injected retrieve tool). Markers without a tool — or a tool - # without markers — are useless, so it is a single switch. Default keeps - # CCR fully on. + # CCR opt-out: --no-ccr disables every half at once — markers in + # content, the injected retrieve tool, AND server-side response + # handling. Markers without a tool, or a tool without markers, are + # useless, so it is a single switch. Default keeps CCR fully on. + # + # Response handling has to be part of it. The buffered stream:false + # path keys off ``headroom_retrieve`` being present in the *request's* + # tools, and a client can advertise that tool on its own — the bundled + # OpenCode plugin registers it unconditionally. So with response + # handling left on, `--no-ccr` silently kept flipping streaming turns + # to buffered whenever history still held a redeemable marker, and the + # documented escape hatch for the CCR buffered-stream bugs did nothing + # for exactly the clients told to use it (#3082). ccr_inject_tool=not no_ccr, ccr_inject_marker=not no_ccr, + ccr_handle_responses=not no_ccr, ccr_resolve_markers_inline=ccr_inline_resolve, lossless=lossless, ccr_proactive_expansion=not no_ccr_proactive_expansion, diff --git a/headroom/cli/wrap.py b/headroom/cli/wrap.py index 09c6642bb..03be77c4c 100644 --- a/headroom/cli/wrap.py +++ b/headroom/cli/wrap.py @@ -136,7 +136,12 @@ from headroom.providers.copilot import ( validate_configuration as _validate_copilot_configuration, ) from headroom.providers.cursor import render_setup_lines as _render_cursor_setup_lines -from headroom.providers.grok import build_launch_env as _build_grok_launch_env +from headroom.providers.grok import ( + DEFAULT_API_URL as _GROK_DEFAULT_API_URL, +) +from headroom.providers.grok import ( + build_launch_env as _build_grok_launch_env, +) from headroom.providers.grok_build import render_setup_lines as _render_grok_build_setup_lines from headroom.providers.grok_build.config import ( inject_grok_provider_config, @@ -316,9 +321,13 @@ _AGENT_SAVINGS_WRAP_AGENTS = {"claude", "codex", "cursor", "grok", "grok_build"} # so `--1m` forces the suffix via ANTHROPIC_MODEL on the launched process. _ANTHROPIC_MODEL_ENV = "ANTHROPIC_MODEL" _CONTEXT_1M_SUFFIX = "[1m]" -# Only used when no model is otherwise selected (no ANTHROPIC_MODEL set). The -# current default Opus; the suffix logic preserves any model the user did set. -_DEFAULT_1M_MODEL = "claude-opus-4-8" +_1M_MODEL_ENV = "HEADROOM_1M_MODEL" +# Fallback model for `--1m` when nothing else selects one (no ANTHROPIC_MODEL, +# no explicit --model). Overridable via HEADROOM_1M_MODEL so it can track new +# Opus releases without a code change and without pinning ANTHROPIC_MODEL +# globally (which would also change non-`--1m` sessions and override Claude +# Code's /model picker). #2937. +_DEFAULT_1M_MODEL = "claude-opus-5" _OPENCLAUDE_INSTRUCTIONS_FILE = "CONVENTIONS.md" @@ -326,11 +335,12 @@ def _resolve_1m_model(current: str | None) -> str: """Return the model id that makes Claude Code request the 1M window (#1158). Preserves a model the user already selected via ``ANTHROPIC_MODEL`` (only - appending the ``[1m]`` suffix when missing); falls back to the default Opus - when none is set. Idempotent — a value already ending in ``[1m]`` is - returned unchanged. + appending the ``[1m]`` suffix when missing). When none is set it falls back + to ``HEADROOM_1M_MODEL`` if defined, else the built-in default Opus (#2937). + Idempotent — a value already ending in ``[1m]`` is returned unchanged. """ - base = (current or "").strip() or _DEFAULT_1M_MODEL + fallback = (os.environ.get(_1M_MODEL_ENV) or "").strip() or _DEFAULT_1M_MODEL + base = (current or "").strip() or fallback return base if base.endswith(_CONTEXT_1M_SUFFIX) else f"{base}{_CONTEXT_1M_SUFFIX}" @@ -5533,9 +5543,8 @@ def vscode_copilot( f' "github.copilot.advanced.debug.overrideProxyUrl": "{vscode_proxy_url(actual_port, _project_name_from_cwd())}",' ) click.echo( - f' "github.copilot.advanced.debug.overrideCapiUrl": "{vscode_proxy_url(actual_port, _project_name_from_cwd())}",' + f' "github.copilot.advanced.debug.overrideCapiUrl": "{vscode_proxy_url(actual_port, _project_name_from_cwd())}"' ) - click.echo(' "github.copilot.advanced.debug.overrideAuthType": "token"') _run_proxy_only_watcher( agent_label="VS CODE COPILOT", @@ -6417,7 +6426,7 @@ def grok( backend=backend, anyllm_provider=anyllm_provider, region=region, - openai_api_url="https://api.x.ai", + openai_api_url=_GROK_DEFAULT_API_URL, ) @@ -6507,9 +6516,9 @@ def grok_build( \b Grok Build reads model endpoints from ``~/.grok/config.toml``. This - command starts the proxy, optionally sets up the selected CLI context - tool, injects a Headroom-managed ``[model.grok-build]`` override, and - prints next steps. + command starts the proxy (upstream ``https://api.x.ai``, same as + ``wrap grok``), injects a Headroom-managed ``[model.grok-build]`` + override, and prints next steps. \b Example: @@ -6536,6 +6545,8 @@ def grok_build( for line in _render_grok_build_setup_lines(actual_port, project=project): click.echo(line) + # Client hop is local proxy via config.toml; upstream must be xAI (not + # the OpenAI default). Omitting this caused 401s with Grok auth headers. _run_proxy_only_watcher( agent_label="grok-build", port=port, @@ -6544,6 +6555,7 @@ def grok_build( memory=memory, agent_type="grok_build", print_setup_lines=_print_grok_build_setup, + openai_api_url=_GROK_DEFAULT_API_URL, ) diff --git a/headroom/copilot_auth.py b/headroom/copilot_auth.py index 8861238bf..6ac8cd596 100644 --- a/headroom/copilot_auth.py +++ b/headroom/copilot_auth.py @@ -28,6 +28,17 @@ from headroom.copilot_macos_keychain import read_copilot_oauth_token as read_mac logger = logging.getLogger(__name__) DEFAULT_API_URL = "https://api.githubcopilot.com" +# Copilot serves *chat* from the CAPI host above and *inline completions* from a +# separate proxy host. GitHub's own client library keeps them apart: +# +# _getCAPIUrl(t) -> t?.endpoints.api || "https://api.githubcopilot.com" +# _getProxyUrl(t) -> t?.endpoints.proxy || DEFAULT_PROXY_BASE_URL +# DEFAULT_PROXY_BASE_URL = "https://copilot-proxy.githubusercontent.com" +# +# and builds completions as `${proxyBaseURL}/v1/engines//completions` +# (@vscode/copilot-api 0.5.2). Sending that path to the CAPI host is the wrong +# surface, so the completions default has to be its own constant (#3076). +DEFAULT_COMPLETIONS_PROXY_URL = "https://copilot-proxy.githubusercontent.com" DEFAULT_TOKEN_EXCHANGE_URL = "https://api.github.com/copilot_internal/v2/token" DEFAULT_USER_INFO_URL = "https://api.github.com/copilot_internal/user" DEFAULT_GITHUB_HOST = "github.com" @@ -245,6 +256,120 @@ def _configured_api_url() -> str: return DEFAULT_API_URL +def copilot_api_url() -> str: + """Return the configured Copilot API base URL without any network calls. + + Resolves ``GITHUB_COPILOT_API_URL``, then the configured enterprise domain, + then ``api.githubcopilot.com``. Unlike :func:`resolve_copilot_api_url` this + performs no token exchange, so it is safe to call while routing a request. + """ + + return _configured_api_url() + + +# GitHub's token exchange advertises the host that serves inline completions +# under ``endpoints.proxy``, alongside the ``endpoints.api`` chat host. It is +# recorded here when observed so completions routing uses GitHub's own answer +# instead of an assumption about which host serves that endpoint (#3076). +_observed_completions_base_url: str | None = None + + +def _remember_completions_endpoint(payload: Any) -> None: + """Record the completions host advertised by a token-exchange payload.""" + + global _observed_completions_base_url + endpoints = payload.get("endpoints") if isinstance(payload, dict) else None + proxy_url = endpoints.get("proxy") if isinstance(endpoints, dict) else None + if isinstance(proxy_url, str) and proxy_url.strip(): + _observed_completions_base_url = proxy_url.strip().rstrip("/") + + +def reset_observed_completions_endpoint() -> None: + """Forget the advertised completions host (test isolation).""" + + global _observed_completions_base_url + _observed_completions_base_url = None + + +def _url_host(value: str) -> str: + """Hostname for a URL, tolerating a scheme-less value. + + Mirrors the normalization :func:`is_copilot_api_url` performs, so a host + configured without "https://" is not silently treated as a different host. + """ + + parsed = urlparse(value) + netloc_or_path = parsed.netloc.lower() or parsed.path.lower() + return (parsed.hostname or netloc_or_path.split("/", 1)[0]).lower() + + +def is_copilot_completions_host(url: str | None) -> bool: + """Return True when *url* already points at a Copilot inline-completions host. + + Distinct from :func:`is_copilot_api_url`, which matches the CAPI (chat) + surface. A CAPI host is *not* a completions host, so the two must not be + conflated when deciding whether a completions request is already addressed + correctly. + """ + + if not url: + return False + # Compare hosts, never whole strings: this is asked both about a bare base + # URL (routing) and about a fully-built URL with the path appended (auth). + # A string compare answers True for the first and False for the second, so + # an operator override would route correctly and then be forwarded with no + # credentials at all. + host = _url_host(url) + if not host: + return False + override = os.environ.get("GITHUB_COPILOT_PROXY_URL", "").strip() + if override and host == _url_host(override): + return True + if host == "copilot-proxy.githubusercontent.com": + return True + # Per-SKU hosts GitHub hands out via `endpoints.proxy`, e.g. + # proxy.individual.githubcopilot.com / proxy.business… / proxy.enterprise…. + return host.startswith("proxy.") and host.endswith(".githubcopilot.com") + + +def copilot_completions_base_url() -> str: + """Return the base URL serving Copilot's inline-completions endpoint. + + Resolution order, most authoritative first: + + 1. ``GITHUB_COPILOT_PROXY_URL`` — an explicit operator override, so a + network that fronts Copilot behind its own gateway (or a GitHub change + to this endpoint) is a config edit rather than a code change. + 2. ``endpoints.proxy`` from the last Copilot token exchange — GitHub + telling us directly where completions go. + 3. ``copilot-proxy.githubusercontent.com`` — GitHub's own documented + default for this endpoint (see ``DEFAULT_COMPLETIONS_PROXY_URL``). + 4. For an enterprise or otherwise custom Copilot deployment, that + deployment's own host. Falling back to the public GitHub host there would + send an enterprise tenant's keystrokes outside their deployment, which is + worse than failing to resolve. + + Note what step 4 must *not* capture: a configured API URL that is itself a + public Copilot host. ``headroom wrap vscode`` sets ``GITHUB_COPILOT_API_URL`` + to the resolved subscription URL (e.g. ``api.business.githubcopilot.com``), + which is the chat surface — returning it here would put the completions path + straight back on the host that answers it with 404. Only a host outside + ``*.githubcopilot.com`` indicates a deployment whose traffic has to stay put. + + Never performs I/O; step 2 only reads what a previous exchange recorded. + """ + + override = os.environ.get("GITHUB_COPILOT_PROXY_URL", "").strip() + if override: + return override.rstrip("/") + if _observed_completions_base_url: + return _observed_completions_base_url + configured = _configured_api_url_override() + if configured and not _is_public_copilot_api_host(_url_host(configured)): + return configured + return DEFAULT_COMPLETIONS_PROXY_URL + + def _github_oauth_domain(domain: str | None = None) -> str: raw = (domain or DEFAULT_GITHUB_HOST).strip() if not raw: @@ -993,6 +1118,25 @@ def is_copilot_api_url(url: str | None) -> bool: return _is_public_copilot_api_host(hostname) or _is_ghe_copilot_api_host(hostname) +def is_copilot_upstream_url(url: str | None) -> bool: + """Return True for any Copilot-served upstream: chat (CAPI) or completions. + + Copilot has two surfaces on two different hosts, and code that asks "is this + request going to Copilot?" means the union. :func:`is_copilot_api_url` alone + answers only for chat, so the completions host looked like a stranger: + ``apply_copilot_api_auth`` attached no credentials to it (401) and + ``build_copilot_upstream_url`` skipped ``mark_request_routed_to_copilot``, + which mislabels the provider in telemetry. + + Deliberately *not* folded into :func:`is_copilot_api_url`, which also gates + validation of the ``endpoints.api`` value from a token exchange and the + Responses-API preference check — neither of which should treat a completions + host as a chat host (#3076). + """ + + return is_copilot_api_url(url) or is_copilot_completions_host(url) + + def _is_public_copilot_api_host(host: str) -> bool: """Return True for GitHub-hosted Copilot API domains.""" @@ -1056,12 +1200,35 @@ def reset_request_routed_to_copilot() -> None: _request_routed_to_copilot.set(False) +def is_copilot_completions_path(path: str) -> bool: + """Return True for Copilot's inline-completions ("ghost text") endpoint. + + The Copilot editor extensions send code completions to + ``/v1/engines//completions`` on whatever host + ``github.copilot.advanced.debug.overrideProxyUrl`` names — so when that + setting points at Headroom, this is the path that arrives. + + The shape identifies GitHub Copilot on its own. OpenAI's Engines API was + removed years ago and no other provider Headroom fronts serves it, so a + request on this path is Copilot's and can never be answered by the default + OpenAI target (#3076). + """ + + normalized = (path if path.startswith("/") else f"/{path}").rstrip("/") + prefix = "/v1/engines/" + suffix = "/completions" + if not normalized.startswith(prefix) or not normalized.endswith(suffix): + return False + engine = normalized[len(prefix) : -len(suffix)] + return bool(engine) and "/" not in engine + + def build_copilot_upstream_url(base_url: str, path: str) -> str: """Build an upstream URL, normalizing GitHub Copilot's non-/v1 path layout.""" normalized_base = base_url.rstrip("/") normalized_path = path if path.startswith("/") else f"/{path}" - if is_copilot_api_url(normalized_base): + if is_copilot_upstream_url(normalized_base): # Single routing chokepoint for every Copilot surface (OpenAI # chat/responses and Anthropic messages all build their upstream URL # here), so mark the request for provider relabeling downstream. @@ -1071,7 +1238,17 @@ def build_copilot_upstream_url(base_url: str, path: str) -> str: # Anthropic surface for Claude models IS ``/v1/messages`` (with the # ``/v1``); stripping it forwarded ``/messages`` and Copilot returned 404 # for claude-* models (#2409). Keep ``/v1`` for the messages endpoint. - if normalized_path.startswith("/v1/") and not normalized_path.startswith("/v1/messages"): + # + # Inline completions are the same story: the Copilot extension itself + # builds ``/v1/engines//completions``, so the path that reaches + # us is already the exact path Copilot serves. Stripping ``/v1`` there + # rewrites a Copilot-native path into one that 404s (#3076). The rule + # this encodes: strip only for clients speaking generic-OpenAI at + # Copilot, never for Copilot's own paths. + keep_v1 = normalized_path.startswith("/v1/messages") or is_copilot_completions_path( + normalized_path + ) + if normalized_path.startswith("/v1/") and not keep_v1: normalized_path = normalized_path[3:] else: reset_request_routed_to_copilot() @@ -1207,7 +1384,12 @@ class CopilotTokenProvider: try: with urllib_request.urlopen(request, timeout=10.0) as response: payload = json.loads(response.read().decode("utf-8")) - return payload if isinstance(payload, dict) else {} + if not isinstance(payload, dict): + return {} + # Every exchange funnels through here, so this is the one place + # that sees GitHub's advertised completions host (#3076). + _remember_completions_endpoint(payload) + return payload except urllib_error.HTTPError as exc: body = exc.read().decode("utf-8", errors="replace") raise RuntimeError( @@ -1314,7 +1496,11 @@ def _is_managed_copilot_seeded_bearer(token: str) -> bool: async def apply_copilot_api_auth(headers: dict[str, str], *, url: str) -> dict[str, str]: """Apply Copilot auth headers for GitHub Copilot API requests.""" resolved = dict(headers) - if not is_copilot_api_url(url): + # Both Copilot surfaces need credentials. Gating on the chat host alone left + # inline completions unauthenticated: the request reached + # copilot-proxy.githubusercontent.com with no Authorization header, and that + # host answers 401 (#3076). + if not is_copilot_upstream_url(url): return resolved for name, value in _copilot_chat_header_defaults().items(): diff --git a/headroom/graph/installer.py b/headroom/graph/installer.py index cb3b8f59a..ef0b7b21a 100644 --- a/headroom/graph/installer.py +++ b/headroom/graph/installer.py @@ -77,13 +77,17 @@ def download_cbm(version: str | None = None) -> Path: except Exception as e: raise RuntimeError(f"Failed to download codebase-memory-mcp from {url}: {e}") from e + from headroom.binaries import verify_download_bytes + + verify_download_bytes(data, url=url, name="codebase-memory-mcp") + # Extract binary from tar.gz try: with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as tar: for member in tar.getmembers(): if member.name.endswith(CBM_BIN_NAME) or member.name == CBM_BIN_NAME: member.name = target_path.name - tar.extract(member, CBM_BIN_DIR) + tar.extract(member, CBM_BIN_DIR, filter="data") break else: raise RuntimeError("codebase-memory-mcp binary not found in archive") diff --git a/headroom/install/planner.py b/headroom/install/planner.py index 290cff063..518688abf 100644 --- a/headroom/install/planner.py +++ b/headroom/install/planner.py @@ -9,6 +9,7 @@ from collections.abc import Iterable import click from headroom import paths as _paths +from headroom.providers.grok.runtime import DEFAULT_API_URL as _GROK_DEFAULT_API_URL from headroom.providers.install_registry import build_install_target_envs from headroom.rollout import RolloutChannel @@ -177,6 +178,19 @@ def build_manifest( base_env["HEADROOM_TELEMETRY"] = "on" if telemetry_enabled else "off" if memory_enabled: base_env["HEADROOM_MEMORY_ENABLED"] = "1" + # Grok / Grok Build need proxy upstream = xAI. Only auto-set when no other + # OpenAI-compatible tools share this proxy (those may need api.openai.com / + # Copilot). Explicit OPENAI_TARGET_API_URL in extra_env still wins below. + _openai_native = { + ToolTarget.CODEX.value, + ToolTarget.COPILOT.value, + ToolTarget.AIDER.value, + ToolTarget.OPENCODE.value, + } + _grok_targets = {ToolTarget.GROK.value, ToolTarget.GROK_BUILD.value} + target_set = set(resolved_targets) + if target_set & _grok_targets and not (target_set & _openai_native): + base_env.setdefault("OPENAI_TARGET_API_URL", _GROK_DEFAULT_API_URL) # Applied last so explicit --env overrides win over the auto-derived # defaults above (e.g. a custom HEADROOM_WORKSPACE_DIR). if extra_env: @@ -241,6 +255,9 @@ def build_manifest( proxy_args.extend(["--protect-tool-results", protect_tool_results]) if bedrock_profile: proxy_args.extend(["--bedrock-profile", bedrock_profile]) + openai_target = base_env.get("OPENAI_TARGET_API_URL") + if openai_target: + proxy_args.extend(["--openai-api-url", openai_target]) container_name = f"headroom-{normalized_profile}" return DeploymentManifest( diff --git a/headroom/learn/analyzer.py b/headroom/learn/analyzer.py index b939f7cf7..670b5c2b6 100644 --- a/headroom/learn/analyzer.py +++ b/headroom/learn/analyzer.py @@ -539,6 +539,40 @@ def _strip_fenced_json(raw: str) -> dict: return result +def _failure_detail( + stderr: str | None, stdout: str | None, *, result_text: str | None = None +) -> str: + """Build the operator-facing reason for a non-zero CLI exit. + + stderr alone is not enough. `claude -p --output-format stream-json` writes + *nothing* to stderr and reports API failures only in its final ``result`` + event on stdout, so a stderr-only message renders as a bare + ``failed (exit 1):`` with no reason at all -- the user (and we) cannot tell a + usage limit from an unreachable proxy from an expired login. + + Both streams are included when both have content, and stdout is tailed rather + than headed because CLI backends emit the error last (a streaming backend's + whole event log precedes it). + + Args: + stderr: Captured stderr, if any. + stdout: Captured stdout, if any. + result_text: Pre-extracted reason (claude-cli's final ``result`` field), + used in place of the raw stdout tail when available. + + Returns: + A non-empty snippet, or ``"(no output captured)"`` when both streams were + empty, so the message is never a dangling colon. + """ + parts: list[str] = [] + if stderr and stderr.strip(): + parts.append(stderr.strip()[:_MAX_SNIPPET_LEN]) + tail = result_text if result_text and result_text.strip() else stdout + if tail and tail.strip(): + parts.append(tail.strip()[-_MAX_SNIPPET_LEN:]) + return "\n".join(parts) if parts else "(no output captured)" + + def _call_cli_llm(digest: str, model: str) -> dict: """Call a locally installed CLI tool as the LLM backend. @@ -611,10 +645,8 @@ def _call_cli_llm(digest: str, model: str) -> dict: ) from None if result.returncode != 0: - stderr_snippet = (result.stderr or "")[:_MAX_SNIPPET_LEN] - raise RuntimeError( - f"`{' '.join(cmd)}` failed (exit {result.returncode}):\n{stderr_snippet}" - ) + detail = _failure_detail(result.stderr, result.stdout) + raise RuntimeError(f"`{' '.join(cmd)}` failed (exit {result.returncode}):\n{detail}") # Log stderr warnings even on success (auth refreshes, deprecation notices). if result.stderr and result.stderr.strip(): @@ -757,8 +789,14 @@ def _call_claude_cli_streaming( proc.wait() if proc.returncode != 0: - stderr_blob = "".join(stderr_lines)[:_MAX_SNIPPET_LEN] - raise RuntimeError(f"`{' '.join(cmd)}` failed (exit {proc.returncode}):\n{stderr_blob}") + # `final_result` is preferred over the raw stdout tail: claude emits a + # final `result` event even when the run fails, and its `result` field is + # the human-readable reason ("API Error: ...", "Not logged in", usage + # limits). + detail = _failure_detail( + "".join(stderr_lines), "".join(stdout_lines), result_text=final_result + ) + raise RuntimeError(f"`{' '.join(cmd)}` failed (exit {proc.returncode}):\n{detail}") stderr_blob = "".join(stderr_lines) if stderr_blob.strip(): diff --git a/headroom/memory/backends/direct_mem0.py b/headroom/memory/backends/direct_mem0.py index 20c454620..03d7394dc 100644 --- a/headroom/memory/backends/direct_mem0.py +++ b/headroom/memory/backends/direct_mem0.py @@ -51,6 +51,7 @@ import asyncio import hashlib import inspect import logging +import os import uuid from dataclasses import dataclass, field from datetime import datetime, timezone @@ -100,7 +101,7 @@ class Mem0Config: # Neo4j settings neo4j_uri: str = "neo4j://localhost:7687" neo4j_user: str = "neo4j" - neo4j_password: str = "password" + neo4j_password: str = field(default_factory=lambda: os.environ.get("NEO4J_PASSWORD", "")) # Qdrant settings (defaults resolve from HEADROOM_QDRANT_* env vars) qdrant_url: str | None = field(default_factory=qdrant_env.qdrant_env_url) diff --git a/headroom/memory/backends/mem0.py b/headroom/memory/backends/mem0.py index 7aefce0f6..c1c904c67 100644 --- a/headroom/memory/backends/mem0.py +++ b/headroom/memory/backends/mem0.py @@ -12,6 +12,7 @@ Supports both local mode (embedded services) and cloud mode (Mem0 API). from __future__ import annotations import asyncio +import os import uuid from dataclasses import dataclass, field from datetime import datetime, timezone @@ -57,7 +58,7 @@ class Mem0Config: # Local mode settings - Neo4j and Qdrant config neo4j_uri: str = "neo4j://localhost:7687" neo4j_user: str = "neo4j" - neo4j_password: str = "password" + neo4j_password: str = field(default_factory=lambda: os.environ.get("NEO4J_PASSWORD", "")) # Qdrant settings (defaults resolve from HEADROOM_QDRANT_* env vars) qdrant_url: str | None = field(default_factory=qdrant_env.qdrant_env_url) qdrant_host: str = field(default_factory=qdrant_env.qdrant_env_host) diff --git a/headroom/memory/easy.py b/headroom/memory/easy.py index 2ad109f9e..2497de3c3 100644 --- a/headroom/memory/easy.py +++ b/headroom/memory/easy.py @@ -33,6 +33,7 @@ Backends: from __future__ import annotations +import os from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, Any @@ -115,7 +116,7 @@ class Memory: qdrant_api_key: str | None = None, neo4j_uri: str = "neo4j://localhost:7687", neo4j_user: str = "neo4j", - neo4j_password: str = "password", + neo4j_password: str | None = None, ) -> None: from headroom.memory import qdrant_env @@ -145,7 +146,9 @@ class Memory: ) self._neo4j_uri = neo4j_uri self._neo4j_user = neo4j_user - self._neo4j_password = neo4j_password + self._neo4j_password = ( + neo4j_password if neo4j_password is not None else os.environ.get("NEO4J_PASSWORD", "") + ) async def _ensure_initialized(self) -> None: """Initialize the backend on first use.""" diff --git a/headroom/perf/analyzer.py b/headroom/perf/analyzer.py index b296e59f6..01a81f072 100644 --- a/headroom/perf/analyzer.py +++ b/headroom/perf/analyzer.py @@ -15,6 +15,7 @@ import os import re from dataclasses import asdict, dataclass, field from datetime import datetime, timedelta +from pathlib import Path from headroom import paths as _paths from headroom.pricing.litellm_pricing import resolve_litellm_model @@ -156,6 +157,12 @@ class PerfRecord: tokens_before: int = 0 tokens_after: int = 0 tokens_saved: int = 0 + # Tokens the forwarded request GREW by (PERF ``tok_inflated``). Both + # endpoints are clamped — ``tok_saved`` at zero and ``tok_inflated`` at zero + # — so a turn that left the proxy bigger reports ``tok_saved=0`` and hides + # its growth in a field nothing downstream read. Carrying it here is what + # lets the report state net alongside gross instead of implying they agree. + tokens_inflated: int = 0 tool_saved: int = 0 cache_read: int = 0 cache_write: int = 0 @@ -167,6 +174,11 @@ class PerfRecord: ttfb_ms: float = 0.0 stages: dict[str, float] = field(default_factory=dict) savings_breakdown: list[dict[str, object]] = field(default_factory=list) + # True when the proxy answered from its own response cache and never + # contacted the upstream. Such a turn has all-zero token counters and no + # upstream stage timings, so without this flag it reads as a turn that + # did nothing (#3019). Absent from pre-#3019 logs, hence the default. + from_response_cache: bool = False @dataclass @@ -213,6 +225,10 @@ class PerfReport: transform_records: list[TransformRecord] = field(default_factory=list) toin_records: list[ToinRecord] = field(default_factory=list) log_files_read: int = 0 + # Rotated files skipped unopened because they were last written before the + # requested window. Reported so coverage stays honest: `log_files_read` on + # its own would silently understate how much log exists on disk. + log_files_skipped: int = 0 total_lines_parsed: int = 0 # Window covered by the report. `requested_hours` is what the caller # asked for; `oldest_kept_ts` / `newest_kept_ts` are the actual @@ -297,7 +313,31 @@ def parse_log_files(last_n_hours: float = 168.0) -> PerfReport: report.newest_kept_ts = ts_str # Collect log files: proxy.log, proxy.log.1, proxy.log.2, ... - log_files = sorted(log_dir.glob("proxy.log*"), key=lambda p: p.stat().st_mtime) + # + # A rotated file last written before the cutoff cannot contain a record + # inside the window, so skip it without opening it. Without this the cost + # of a windowed query is O(total log history) rather than O(window): + # `/stats` recomputes throughput over the last hour on a 10s cache TTL, so + # a dashboard polling it re-read and re-regexed every byte of every + # rotated log, forever, for an answer that lives in the tail of the newest + # file. Measured on a developer machine with six rotations (54 MB). + # + # mtime is the safe discriminator: the logs are append-only, so a file + # untouched since before the cutoff has no line written after it. Files + # are stat'd once and the value reused for the sort. + cutoff_epoch = cutoff.timestamp() if cutoff is not None else None + dated_files: list[tuple[float, Path]] = [] + for path in log_dir.glob("proxy.log*"): + try: + mtime = path.stat().st_mtime + except OSError: + # Rotated away between glob and stat — nothing to read. + continue + if cutoff_epoch is not None and mtime < cutoff_epoch: + report.log_files_skipped += 1 + continue + dated_files.append((mtime, path)) + log_files = [path for _, path in sorted(dated_files, key=lambda pair: pair[0])] for log_file in log_files: report.log_files_read += 1 @@ -363,6 +403,7 @@ def parse_log_files(last_n_hours: float = 168.0) -> PerfReport: tokens_before=int(kv.get("tok_before", 0)), tokens_after=int(kv.get("tok_after", 0)), tokens_saved=int(kv.get("tok_saved", 0)), + tokens_inflated=int(kv.get("tok_inflated", 0)), tool_saved=int(kv.get("tool_saved", 0)), savings_breakdown=_decode_perf_savings(kv.get("savings", "none")), cache_read=int(kv.get("cache_read", 0)), @@ -373,6 +414,7 @@ def parse_log_files(last_n_hours: float = 168.0) -> PerfReport: total_ms=float(kv.get("total_ms", 0)), tokens_out=int(kv.get("tok_out", 0)), ttfb_ms=float(kv.get("ttfb_ms", 0)), + from_response_cache=kv.get("cached", "0") == "1", stages=stages_by_rid.get(m.group("rid"), {}), ) ) @@ -511,6 +553,19 @@ def format_report(report: PerfReport) -> str: # include tool bytes), so it used to render as a rival "Tool saved" line — which # read as a side metric and hid the win on tool-heavy turns where tok_saved=0. lines.append(f"Tokens saved: {total_headline_saved:,} ({headline_pct:.1f}% reduction)") + # Gross vs net. ``tok_saved`` is clamped at zero per request, so turns + # where Headroom made the body BIGGER (CCR proactive expansion, memory + # injection) contribute nothing negative to the headline — their growth + # lands in ``tok_inflated`` instead, which nothing here used to read. + # Printing "321,239,562 -> 313,274,727" directly above "8,455,763 saved" + # implies the two reconcile; they differ by exactly the inflation. Show + # it whenever it is non-zero so the arithmetic closes on the page. + total_inflated = sum(r.tokens_inflated for r in records) + if total_inflated > 0: + lines.append( + f" · inflated {total_inflated:,} " + f"(net message reduction {total_before - total_after:,})" + ) if total_tool_saved > 0: lines.append(f" · messages {max(0, total_saved):,}") lines.append(f" · tool schemas {total_tool_saved:,}") @@ -663,6 +718,31 @@ def format_report(report: PerfReport) -> str: lines.append( f" {name}: {avg_pct:.1f}% avg reduction, {len(recs)} uses, {total_s:,} saved" ) + # This table is built ONLY from "Transform NAME: B -> A tokens (saved N)" + # lines, which just one engine emits (transforms/pipeline.py). The + # OpenAI-Responses engine (transforms/compression_units.py + + # compression_batches.py) applies the same strategies and contains no + # logging calls at all, so none of its work appears above. On real + # traffic that hid ~7M of ~8.5M message-token savings — the table read + # "content_router: 189,783 saved" against a PERF total 44x larger, which + # invites exactly the wrong conclusion about which compressors work. + # + # State the divergence, NOT a coverage ratio. The two totals are + # different populations and neither strictly contains the other: the + # Transform lines carry no request_id, fire once per pipeline STAGE (so + # several can describe one request), and are emitted before the forwarder + # decides anything — a mutation later discarded by the signed-thinking + # byte-lock still logs its "saved" here while the request's PERF line + # correctly reports 0. So "table covers X of Y" would be a false subset + # claim in both directions; report the two sums and let the reader judge. + table_total = sum(r.tokens_saved for r in report.transform_records) + perf_total = sum(r.tokens_saved for r in report.perf_records) + if table_total != perf_total: + lines.append( + f" ! stage-level total {table_total:,} != PERF message total {perf_total:,} " + "— this table sees only engines that emit a Transform line, counts " + "per stage, and does not check whether the mutation shipped" + ) lines.append("") # Router routing breakdown @@ -682,11 +762,24 @@ def format_report(report: PerfReport) -> str: f" Excluded: {total_excluded} ({total_excluded / total_all * 100:.0f}%) — Read/Glob outputs" ) lines.append( - f" Skipped: {total_skipped} ({total_skipped / total_all * 100:.0f}%) — <50 words" + f" Skipped: {total_skipped} ({total_skipped / total_all * 100:.0f}%) — below size floor" ) lines.append( f" Unchanged: {total_unchanged} ({total_unchanged / total_all * 100:.0f}%) — ratio too high" ) + # These four buckets are NOT the router's full outcome space — the + # `[router] route_counts=` line carries 17 keys, and the ones omitted + # here (cache_hit, system_msg, error_protected, already_compressed, + # …) are individually larger than "Excluded". Percentages taken over + # this subset therefore overstate every share: on real traffic the + # "skipped" bucket read 77% here against 49.5% of actual terminal + # fates, which reads as a mis-set threshold rather than a narrow + # denominator. Say what the denominator is instead of implying it is + # everything. + lines.append( + f" (shares are of these 4 buckets only, n={total_all}; " + "see `[router] route_counts=` for the full outcome space)" + ) if total_excluded > total_compressed * 3: lines.append(" ! Excluded tools dominate — consider compressing stale Read outputs") lines.append("") @@ -765,6 +858,10 @@ PERF_RECORD_FIELDS = [ "ttfb_ms", "stages", "savings_breakdown", + # Appended last so every existing CSV column keeps its position; a reader + # that indexes by name is unaffected either way. + "from_response_cache", + "tokens_inflated", ] diff --git a/headroom/pricing/litellm_pricing.py b/headroom/pricing/litellm_pricing.py index 9a4252689..0b97e4e58 100644 --- a/headroom/pricing/litellm_pricing.py +++ b/headroom/pricing/litellm_pricing.py @@ -282,6 +282,47 @@ def estimate_cost( return input_cost + output_cost +def estimate_cost_from_tokens( + model: str, + input_tokens: int = 0, + output_tokens: int = 0, + cached_tokens: int = 0, +) -> float | None: + """Cost for one request from token counts, using LiteLLM's own cost model. + + Prefer this over :func:`estimate_cost` whenever a request may carry cached + tokens or exceed a model's long-context threshold. Flat per-1M rates cannot + express either: cache reads bill at their own rate, and on Anthropic's + Sonnet 4 / 4.5 family a prompt over 200K re-prices the *whole* request -- + input, output and cache alike. ``litellm.cost_per_token`` applies both. + + ``input_tokens`` is the TOTAL prompt, ``cached_tokens`` included. LiteLLM + subtracts the cached portion itself and tests the long-context threshold + against the total, so passing a cache-exclusive count would both + double-discount the cached tokens and understate the threshold. + + Returns ``None`` when LiteLLM is unavailable (the dependency is gated + ``python_version < '3.14'``) or doesn't know the model -- the caller's cue + to fall back to its own table. + """ + if not LITELLM_AVAILABLE: + return None + candidate = next((c for c in pricing_lookup_candidates(model) if c in litellm.model_cost), None) + if candidate is None: + return None + try: + prompt_cost, completion_cost = litellm.cost_per_token( + model=candidate, + prompt_tokens=input_tokens, + completion_tokens=output_tokens, + cache_read_input_tokens=cached_tokens, + ) + except Exception as exc: # pragma: no cover - depends on litellm internals + logger.debug("litellm.cost_per_token failed for %s: %s", candidate, exc) + return None + return float(prompt_cost) + float(completion_cost) + + def list_available_models() -> list[str]: """List all models with pricing info in LiteLLM's database. diff --git a/headroom/providers/anthropic.py b/headroom/providers/anthropic.py index b16251621..fb7278ab4 100644 --- a/headroom/providers/anthropic.py +++ b/headroom/providers/anthropic.py @@ -24,6 +24,7 @@ import warnings from typing import Any, cast from headroom import paths as _paths +from headroom.pricing.litellm_pricing import estimate_cost_from_tokens from headroom.tokenizers.base import ( TokenCountCache, coerce_countable_text, @@ -67,6 +68,21 @@ def sanitize_anthropic_model_id(model: str) -> str: return _DANGLING_ANSI_STYLE_SUFFIX_RE.sub("", cleaned) +# `[1m]` is not only an ANSI artifact: Claude Code appends it to a model id to +# request the 1M context tier, and only then sends the `context-1m` beta header +# (#1158). Upstream rejects the suffix, so `sanitize_anthropic_model_id()` must +# keep stripping it before forwarding (#2027) — but the tier it encodes has to +# be read off the id *before* that happens, or a 1M request gets budgeted as if +# it were the base model's window. +_CONTEXT_1M_SUFFIX_RE = re.compile(r"(?:\[1m\])+$") +CONTEXT_1M_TOKENS = 1_000_000 + + +def has_context_1m_suffix(model: str) -> bool: + """Return True if ``model`` carries Claude Code's ``[1m]`` 1M-tier marker.""" + return bool(_CONTEXT_1M_SUFFIX_RE.search(_ANSI_ESCAPE_RE.sub("", str(model)).strip())) + + def sanitize_anthropic_model_metadata(value: Any) -> Any: """Strip model-id styling artifacts from Anthropic model metadata payloads.""" if isinstance(value, list): @@ -154,6 +170,40 @@ ANTHROPIC_PRICING: dict[str, dict[str, float]] = { "claude-3-haiku-20240307": {"input": 0.25, "output": 1.25, "cached_input": 0.03}, } +# Anthropic's long-context premium. On models that reach 1M over a 200K base, +# a prompt above 200K re-prices the *entire* request -- input, output and cache +# alike -- rather than only the tokens past the threshold. Multipliers are +# derived from LiteLLM's `*_above_200k_tokens` fields ($3->$6 in, $15->$22.50 +# out, $0.30->$0.60 cache read). +# +# Only the Sonnet 4 / 4.5 family is tiered: Opus, and Sonnet 4.6 onward, are +# flat-rated across their whole window. This is the same population that needs +# the `[1m]` suffix to reach 1M at all, so a session that fills the window this +# unlocks is billed at these rates. +_LONG_CONTEXT_THRESHOLD = 200_000 +_LONG_CONTEXT_PREMIUM: dict[str, float] = {"input": 2.0, "output": 1.5, "cached_input": 2.0} +_LONG_CONTEXT_TIERED_MODELS = ( + "claude-sonnet-4-5", + "claude-sonnet-4-20250514", + "claude-4-sonnet-20250514", +) + + +def _apply_long_context_premium( + model: str, pricing: dict[str, float], input_tokens: int +) -> dict[str, float]: + """Return ``pricing`` scaled by the long-context premium where it applies. + + Used only on the manual fallback path; the LiteLLM path already applies the + published above-threshold rates itself. + """ + if input_tokens <= _LONG_CONTEXT_THRESHOLD: + return pricing + if not any(model.startswith(tiered) for tiered in _LONG_CONTEXT_TIERED_MODELS): + return pricing + return {key: rate * _LONG_CONTEXT_PREMIUM.get(key, 1.0) for key, rate in pricing.items()} + + # Default limits for pattern-based inference # Used when a model isn't in the explicit list but matches a known pattern _PATTERN_DEFAULTS = { @@ -226,6 +276,11 @@ def _load_custom_model_config() -> dict[str, Any]: # Try to parse as JSON string loaded = json.loads(env_config) + if not isinstance(loaded, dict): + raise ValueError( + f"HEADROOM_MODEL_LIMITS must be a JSON object, got {type(loaded).__name__}" + ) + # Check for anthropic-specific config, fall back to root level anthropic_config = loaded.get("anthropic", loaded) if "context_limits" in anthropic_config: @@ -234,7 +289,10 @@ def _load_custom_model_config() -> dict[str, Any]: config["pricing"].update(anthropic_config["pricing"]) logger.debug(f"Loaded custom model config from HEADROOM_MODEL_LIMITS: {loaded}") - except (json.JSONDecodeError, OSError) as e: + except (ValueError, OSError) as e: + # ValueError covers json.JSONDecodeError (a subclass) and the + # non-object guard above, so a malformed value warns and falls back + # to defaults instead of crashing provider init. logger.warning(f"Failed to load HEADROOM_MODEL_LIMITS: {e}") # Check config file. Prefer the canonical config-dir location, then fall @@ -249,6 +307,9 @@ def _load_custom_model_config() -> dict[str, Any]: with open(config_file, encoding="utf-8") as f: loaded = json.load(f) + if not isinstance(loaded, dict): + raise ValueError(f"{config_file} must contain a JSON object") + # Only load anthropic-specific config anthropic_config = loaded.get("anthropic", loaded) if "context_limits" in anthropic_config: @@ -262,7 +323,7 @@ def _load_custom_model_config() -> dict[str, Any]: config["pricing"][model] = pricing logger.debug(f"Loaded custom model config from {config_file}") - except (json.JSONDecodeError, OSError) as e: + except (ValueError, OSError) as e: logger.warning(f"Failed to load {config_file}: {e}") return config @@ -605,8 +666,16 @@ class AnthropicProvider(Provider): 6. Pattern-based inference (opus/sonnet/haiku) 7. Default fallback (200K for any Claude model) + A ``[1m]`` suffix raises the result to at least 1M: the caller asked for + the 1M tier and Claude Code sent the `context-1m` beta header, so the + real upstream window is 1M even when the base model's default is 200K. + Never raises an exception - uses sensible defaults for unknown models. """ + if has_context_1m_suffix(model): + # Recursion terminates: the sanitized id has no `[1m]` left. + base = self.get_context_limit(sanitize_anthropic_model_id(model)) + return max(base, CONTEXT_1M_TOKENS) model = sanitize_anthropic_model_id(model) # Check explicit and loaded limits if model in self._context_limits: @@ -685,58 +754,38 @@ class AnthropicProvider(Provider): """Estimate cost for a request. Tries LiteLLM first for up-to-date pricing, falls back to manual pricing. + Both paths apply Anthropic's long-context premium: on the Sonnet 4 / 4.5 + family a prompt over 200K re-prices the whole request (see + ``_LONG_CONTEXT_PREMIUM``). """ model = sanitize_anthropic_model_id(model) - # Try LiteLLM first for cost estimation - litellm, litellm_get_model_info = _get_litellm_clients() - if litellm is not None: - try: - cost = litellm.completion_cost( - model=model, - prompt="", - completion="", - prompt_tokens=input_tokens - cached_tokens, - completion_tokens=output_tokens, - ) - # Add cached token cost if applicable - if cached_tokens > 0: - try: - # Get cached input pricing from LiteLLM model info - info = ( - litellm_get_model_info(model) - if litellm_get_model_info is not None - else None - ) - if info and "input_cost_per_token" in info: - # LiteLLM typically applies 90% discount for cached tokens - cached_cost = cached_tokens * info["input_cost_per_token"] * 0.1 - cost += cached_cost - except Exception: - # Fall back to manual cached pricing - pricing = self._get_pricing(model) - if pricing: - cached_cost = (cached_tokens / 1_000_000) * pricing.get( - "cached_input", pricing["input"] - ) - cost += cached_cost - return cost # type: ignore[no-any-return] - except Exception as e: - logger.debug(f"LiteLLM cost estimation failed for {model}: {e}") + # LiteLLM knows per-model cache and long-context rates, so let it price + # the whole request rather than rebuilding the rate card here. + cost = estimate_cost_from_tokens( + model, + input_tokens=input_tokens, + output_tokens=output_tokens, + cached_tokens=cached_tokens, + ) + if cost is not None: + return cost # Fall back to manual pricing pricing = self._get_pricing(model) if not pricing: return None + rates = _apply_long_context_premium(model, pricing, input_tokens) + # Calculate cost non_cached_input = input_tokens - cached_tokens cost = ( - (non_cached_input / 1_000_000) * pricing["input"] - + (cached_tokens / 1_000_000) * pricing.get("cached_input", pricing["input"]) - + (output_tokens / 1_000_000) * pricing["output"] + (non_cached_input / 1_000_000) * rates["input"] + + (cached_tokens / 1_000_000) * rates.get("cached_input", rates["input"]) + + (output_tokens / 1_000_000) * rates["output"] ) - return cost # type: ignore[no-any-return] + return cost def _get_pricing(self, model: str) -> dict[str, float] | None: """Get pricing for a model with fallback logic.""" diff --git a/headroom/providers/codex/live.py b/headroom/providers/codex/live.py index f59baecb1..034d5c6a2 100644 --- a/headroom/providers/codex/live.py +++ b/headroom/providers/codex/live.py @@ -128,9 +128,13 @@ async def handle_codex_live_websocket( ) forwarded_headers = await apply_copilot_api_auth(forwarded_headers, url=upstream_url) config = getattr(proxy, "config", None) + # `openai_base_url` comes from the resolved provider target, not from a + # request header, so there is no per-request override to gate on here. forwarded_headers = merge_extra_headers( forwarded_headers, getattr(config, "openai_extra_headers", None), + upstream_url=None, + config=config, ) if not any(key.lower() == "authorization" for key in forwarded_headers): if os.environ.get("OPENAI_API_KEY", "").strip(): diff --git a/headroom/providers/cohere.py b/headroom/providers/cohere.py index 2b205a0a2..d557cc511 100644 --- a/headroom/providers/cohere.py +++ b/headroom/providers/cohere.py @@ -21,6 +21,7 @@ import warnings from datetime import date from typing import Any +from headroom.pricing.litellm_pricing import estimate_cost_from_tokens from headroom.tokenizers import EstimatingTokenCounter from .base import Provider, TokenCounter @@ -326,18 +327,13 @@ class CohereProvider(Provider): # Try LiteLLM first if LITELLM_AVAILABLE: for model_variant in [f"cohere/{model}", model]: - try: - cost = litellm.completion_cost( - model=model_variant, - prompt="", - completion="", - prompt_tokens=input_tokens, - completion_tokens=output_tokens, - ) - if cost is not None: - return float(cost) - except Exception: - pass + cost = estimate_cost_from_tokens( + model_variant, + input_tokens=input_tokens, + output_tokens=output_tokens, + ) + if cost is not None: + return float(cost) # Fallback to built-in pricing model_lower = model.lower() diff --git a/headroom/providers/copilot/vscode.py b/headroom/providers/copilot/vscode.py index 354a4efe5..28fa8418b 100644 --- a/headroom/providers/copilot/vscode.py +++ b/headroom/providers/copilot/vscode.py @@ -18,6 +18,15 @@ _MARKER_START = "// --- Headroom Copilot proxy ---" _MARKER_END = "// --- end Headroom Copilot proxy ---" _PROXY_KEY = "github.copilot.advanced.debug.overrideProxyUrl" _CAPI_KEY = "github.copilot.advanced.debug.overrideCapiUrl" +# Written by Headroom until #3076: it no longer exists. The modern Copilot Chat +# extension — the only one left after `GitHub.copilot` was deprecated in early +# 2026 — defines no `authType` setting in either its own configuration +# (`advanced.authPermissions`, `advanced.authProvider`, +# `advanced.debug.overrideCapiUrl`, `advanced.debug.overrideProxyUrl`, +# `advanced.debug.use*Fetcher`) or in the completions code merged into it. Still +# recognised below so a stale hand-written copy is detected, but never emitted: +# VS Code flags unknown keys, and shipping one that does nothing invited the +# conclusion that the override mechanism had stopped working. _AUTH_KEY = "github.copilot.advanced.debug.overrideAuthType" @@ -119,8 +128,7 @@ def _managed_block(proxy_url: str, *, owns_preceding_comma: bool, line_sep: str) return ( f"\t{marker}{line_sep}" f"\t{json.dumps(_PROXY_KEY)}: {json.dumps(proxy_url)},{line_sep}" - f"\t{json.dumps(_CAPI_KEY)}: {json.dumps(proxy_url)},{line_sep}" - f'\t{json.dumps(_AUTH_KEY)}: "token"{line_sep}' + f"\t{json.dumps(_CAPI_KEY)}: {json.dumps(proxy_url)}{line_sep}" f"\t{_MARKER_END}" ) diff --git a/headroom/providers/google.py b/headroom/providers/google.py index 8612d5502..4dc919df3 100644 --- a/headroom/providers/google.py +++ b/headroom/providers/google.py @@ -26,6 +26,7 @@ from datetime import date from typing import Any from headroom.models.registry import ModelRegistry +from headroom.pricing.litellm_pricing import estimate_cost_from_tokens from headroom.tokenizers import EstimatingTokenCounter from .base import Provider, TokenCounter @@ -346,18 +347,13 @@ class GoogleProvider(Provider): model_lower, # gemini-1.5-pro ] for variant in model_variants: - try: - cost = litellm.completion_cost( - model=variant, - prompt="", - completion="", - prompt_tokens=input_tokens, - completion_tokens=output_tokens, - ) - if cost is not None: - return cost - except Exception: - continue + cost = estimate_cost_from_tokens( + variant, + input_tokens=input_tokens, + output_tokens=output_tokens, + ) + if cost is not None: + return cost # Fallback to hardcoded pricing input_price, output_price = None, None diff --git a/headroom/providers/grok_build/runtime.py b/headroom/providers/grok_build/runtime.py index f8bc7f123..26ca11e4e 100644 --- a/headroom/providers/grok_build/runtime.py +++ b/headroom/providers/grok_build/runtime.py @@ -4,6 +4,7 @@ from __future__ import annotations from dataclasses import dataclass +from headroom.providers.grok.runtime import DEFAULT_API_URL from headroom.proxy.project_context import with_project_prefix @@ -41,6 +42,8 @@ def render_setup_lines(port: int, project: str | None = None) -> list[str]: " [model.grok-build]", f' base_url = "{target.base_url}"', "", + f" Proxy upstream (OpenAI-compatible): {DEFAULT_API_URL}", + "", " Start Grok Build in this project directory:", " grok", "", diff --git a/headroom/providers/litellm.py b/headroom/providers/litellm.py index 95d13311a..e02f3f683 100644 --- a/headroom/providers/litellm.py +++ b/headroom/providers/litellm.py @@ -25,6 +25,7 @@ import logging import os from typing import Any +from headroom.pricing.litellm_pricing import estimate_cost_from_tokens from headroom.tokenizers import EstimatingTokenCounter from .base import Provider, TokenCounter @@ -240,19 +241,13 @@ class LiteLLMProvider(Provider): Returns: Estimated cost in USD, or None if pricing unknown. """ - try: - # LiteLLM's cost calculation - cost = litellm.completion_cost( - model=model, - prompt="", # We're using token counts directly - completion="", - prompt_tokens=input_tokens, - completion_tokens=output_tokens, - ) - return cost - except Exception as e: - logger.debug(f"LiteLLM cost estimation failed for {model}: {e}") - return None + # LiteLLM's cost calculation, from token counts directly. + return estimate_cost_from_tokens( + model, + input_tokens=input_tokens, + output_tokens=output_tokens, + cached_tokens=cached_tokens, + ) @classmethod def list_supported_providers(cls) -> list[str]: diff --git a/headroom/providers/openai.py b/headroom/providers/openai.py index 891b13559..b0e2b9afc 100644 --- a/headroom/providers/openai.py +++ b/headroom/providers/openai.py @@ -16,6 +16,7 @@ from functools import lru_cache from typing import Any, cast from headroom import paths as _paths +from headroom.pricing.litellm_pricing import estimate_cost_from_tokens from headroom.tokenizers.base import coerce_countable_text, count_content_blocks from .base import Provider, TokenCounter @@ -199,6 +200,11 @@ def _load_custom_model_config() -> dict[str, Any]: # Try to parse as JSON string loaded = json.loads(env_config) + if not isinstance(loaded, dict): + raise ValueError( + f"HEADROOM_MODEL_LIMITS must be a JSON object, got {type(loaded).__name__}" + ) + openai_config = loaded.get("openai", loaded) if "context_limits" in openai_config: config["context_limits"].update(openai_config["context_limits"]) @@ -208,7 +214,10 @@ def _load_custom_model_config() -> dict[str, Any]: config["encodings"].update(openai_config["encodings"]) logger.debug("Loaded custom OpenAI model config from HEADROOM_MODEL_LIMITS") - except (json.JSONDecodeError, OSError) as e: + except (ValueError, OSError) as e: + # ValueError covers json.JSONDecodeError (a subclass) and the + # non-object guard above, so a malformed value warns and falls back + # to defaults instead of crashing provider init. logger.warning(f"Failed to load HEADROOM_MODEL_LIMITS: {e}") # Check config file. Prefer the canonical config-dir location, then fall @@ -223,6 +232,9 @@ def _load_custom_model_config() -> dict[str, Any]: with open(config_file, encoding="utf-8") as f: loaded = json.load(f) + if not isinstance(loaded, dict): + raise ValueError(f"{config_file} must contain a JSON object") + openai_config = loaded.get("openai", {}) if "context_limits" in openai_config: for model, limit in openai_config["context_limits"].items(): @@ -238,7 +250,7 @@ def _load_custom_model_config() -> dict[str, Any]: config["encodings"][model] = encoding logger.debug(f"Loaded custom OpenAI model config from {config_file}") - except (json.JSONDecodeError, OSError) as e: + except (ValueError, OSError) as e: logger.warning(f"Failed to load {config_file}: {e}") return config @@ -637,20 +649,16 @@ class OpenAIProvider(Provider): Returns: Estimated cost in USD, or None if pricing unknown. """ - # Try LiteLLM first (most up-to-date pricing) - litellm = _get_litellm_module() - if litellm is not None: - try: - # LiteLLM uses per-token pricing, returns total cost - cost = litellm.completion_cost( - model=model, - prompt_tokens=input_tokens, - completion_tokens=output_tokens, - ) - if cost is not None and cost > 0: - return float(cost) - except Exception: - pass # Fall through to manual pricing + # Try LiteLLM first (most up-to-date pricing, and it knows each model's + # real cached-input rate rather than the manual path's flat estimate) + cost = estimate_cost_from_tokens( + model, + input_tokens=input_tokens, + output_tokens=output_tokens, + cached_tokens=cached_tokens, + ) + if cost is not None and cost > 0: + return float(cost) # Fall back to hardcoded pricing return self._estimate_cost_manual(input_tokens, output_tokens, model, cached_tokens) diff --git a/headroom/providers/opencode/_dist/entry.opencode.js b/headroom/providers/opencode/_dist/entry.opencode.js index f2fb1fcbb..fcc5943bb 100644 --- a/headroom/providers/opencode/_dist/entry.opencode.js +++ b/headroom/providers/opencode/_dist/entry.opencode.js @@ -12487,6 +12487,7 @@ var childProcess = nodeRequire("node:child_process"); var fs = nodeRequire("node:fs"); var BASE_URL_HEADER = "x-headroom-base-url"; var ORIGINAL_PATH_HEADER = "x-headroom-original-path"; +var PROJECT_HEADER = "x-headroom-project"; var PROXY_ENV = "HEADROOM_OPENCODE_TRANSPORT_PROXY_URL"; var STATE_KEY = /* @__PURE__ */ Symbol.for("headroom.opencode.transport"); function getState() { @@ -12635,7 +12636,7 @@ function requestUrl(input) { } return new URL(String(input)); } -function mergeFetchHeaders(input, init, upstream, originalPath = void 0) { +function mergeFetchHeaders(input, init, upstream, originalPath = void 0, project = void 0) { const headers = new Headers(input instanceof Request ? input.headers : void 0); if (init?.headers) { new Headers(init.headers).forEach((value, key) => headers.set(key, value)); @@ -12647,9 +12648,12 @@ function mergeFetchHeaders(input, init, upstream, originalPath = void 0) { if (originalPath) { headers.set(ORIGINAL_PATH_HEADER, originalPath); } + if (project) { + headers.set(PROJECT_HEADER, project); + } return headers; } -function withRoutedFetchInput(input, init, proxy) { +function withRoutedFetchInput(input, init, proxy, project) { const upstream = requestUrl(input); if (!shouldRoute(upstream, proxy)) { return [input, init]; @@ -12657,7 +12661,7 @@ function withRoutedFetchInput(input, init, proxy) { const { url: nextUrl, originalPath } = routedUrlForOpenCode(upstream, proxy); const nextInit = { ...init, - headers: mergeFetchHeaders(input, init, upstream, originalPath) + headers: mergeFetchHeaders(input, init, upstream, originalPath, project) }; if (input instanceof Request) { return [new Request(nextUrl, input), nextInit]; @@ -12703,12 +12707,15 @@ function urlFromRequestOptions(options) { return void 0; } } -function headersForNodeRequest(options, upstream, originalPath) { +function headersForNodeRequest(options, upstream, originalPath, project) { const headers = new Headers(options.headers); headers.set(BASE_URL_HEADER, upstream.origin); if (originalPath) { headers.set(ORIGINAL_PATH_HEADER, originalPath); } + if (project) { + headers.set(PROJECT_HEADER, project); + } headers.delete("host"); const result = {}; headers.forEach((value, key) => { @@ -12716,7 +12723,7 @@ function headersForNodeRequest(options, upstream, originalPath) { }); return result; } -function routedNodeOptions(parts, proxy) { +function routedNodeOptions(parts, proxy, project) { if (!parts.url || !shouldRoute(parts.url, proxy)) { return void 0; } @@ -12747,7 +12754,7 @@ function routedNodeOptions(parts, proxy) { hostname: nextUrl.hostname, port: nextUrl.port || void 0, path: `${nextUrl.pathname}${nextUrl.search}`, - headers: headersForNodeRequest(parts.options, parts.url, originalPath) + headers: headersForNodeRequest(parts.options, parts.url, originalPath, project) }; } function wrapRequest(originalHttpRequest, originalHttpsRequest, originalRequest) { @@ -12758,7 +12765,7 @@ function wrapRequest(originalHttpRequest, originalHttpsRequest, originalRequest) } const proxy = normalizeProxyUrl(state.proxyUrl); const parts = splitNodeArgs(args); - const nextOptions = routedNodeOptions(parts, proxy); + const nextOptions = routedNodeOptions(parts, proxy, state.project); if (!nextOptions) { return Reflect.apply(originalRequest, this, args); } @@ -12794,6 +12801,7 @@ function installHeadroomTransport(options) { if (existing) { existing.refs += 1; existing.proxyUrl = options.proxyUrl; + existing.project = options.project; existing.debug = Boolean(options.debug); installProcessEnv(options.proxyUrl); return () => uninstallHeadroomTransport(); @@ -12801,6 +12809,7 @@ function installHeadroomTransport(options) { const state = { refs: 1, proxyUrl: options.proxyUrl, + project: options.project, debug: Boolean(options.debug), originalFetch: globalThis.fetch, originalHttpRequest: http.request, @@ -12821,7 +12830,7 @@ function installHeadroomTransport(options) { return state.originalFetch(...args); } const proxy = normalizeProxyUrl(current.proxyUrl); - const [nextInput, nextInit] = withRoutedFetchInput(args[0], args[1], proxy); + const [nextInput, nextInit] = withRoutedFetchInput(args[0], args[1], proxy, current.project); return state.originalFetch(nextInput, nextInit); }; http.request = wrapRequest(state.originalHttpRequest, state.originalHttpsRequest, state.originalHttpRequest); @@ -12871,9 +12880,11 @@ function resolveProxyUrl(options) { var HeadroomPlugin = async (input, options = {}) => { const pluginOptions = options; const proxyUrl = resolveProxyUrl(pluginOptions); + const project = pluginOptions.project ?? input.project?.id ?? input.directory; const retrieveTool = createHeadroomRetrieveTool({ proxyBaseUrl: proxyUrl }); const uninstallTransport = installHeadroomTransport({ proxyUrl, + project, debug: pluginOptions.debug }); return { @@ -12894,7 +12905,7 @@ var HeadroomPlugin = async (input, options = {}) => { "shell.env": async (_input, output) => { output.env.HEADROOM_ACTIVE = "1"; output.env.HEADROOM_PROXY_URL = proxyUrl; - output.env.HEADROOM_PROJECT = pluginOptions.project ?? input.project.id ?? input.directory; + output.env.HEADROOM_PROJECT = project; if (pluginOptions.backend) { output.env.HEADROOM_BACKEND = pluginOptions.backend; } diff --git a/headroom/providers/opencode/hook-shim/handler.js b/headroom/providers/opencode/hook-shim/handler.js index 1dbfba992..e4e0d7d12 100644 --- a/headroom/providers/opencode/hook-shim/handler.js +++ b/headroom/providers/opencode/hook-shim/handler.js @@ -8,6 +8,7 @@ var childProcess = nodeRequire("node:child_process"); var fs = nodeRequire("node:fs"); var BASE_URL_HEADER = "x-headroom-base-url"; var ORIGINAL_PATH_HEADER = "x-headroom-original-path"; +var PROJECT_HEADER = "x-headroom-project"; var PROXY_ENV = "HEADROOM_OPENCODE_TRANSPORT_PROXY_URL"; var STATE_KEY = /* @__PURE__ */ Symbol.for("headroom.opencode.transport"); function getState() { @@ -156,7 +157,7 @@ function requestUrl(input) { } return new URL(String(input)); } -function mergeFetchHeaders(input, init, upstream, originalPath = void 0) { +function mergeFetchHeaders(input, init, upstream, originalPath = void 0, project = void 0) { const headers = new Headers(input instanceof Request ? input.headers : void 0); if (init?.headers) { new Headers(init.headers).forEach((value, key) => headers.set(key, value)); @@ -168,9 +169,12 @@ function mergeFetchHeaders(input, init, upstream, originalPath = void 0) { if (originalPath) { headers.set(ORIGINAL_PATH_HEADER, originalPath); } + if (project) { + headers.set(PROJECT_HEADER, project); + } return headers; } -function withRoutedFetchInput(input, init, proxy) { +function withRoutedFetchInput(input, init, proxy, project) { const upstream = requestUrl(input); if (!shouldRoute(upstream, proxy)) { return [input, init]; @@ -178,7 +182,7 @@ function withRoutedFetchInput(input, init, proxy) { const { url: nextUrl, originalPath } = routedUrlForOpenCode(upstream, proxy); const nextInit = { ...init, - headers: mergeFetchHeaders(input, init, upstream, originalPath) + headers: mergeFetchHeaders(input, init, upstream, originalPath, project) }; if (input instanceof Request) { return [new Request(nextUrl, input), nextInit]; @@ -224,12 +228,15 @@ function urlFromRequestOptions(options) { return void 0; } } -function headersForNodeRequest(options, upstream, originalPath) { +function headersForNodeRequest(options, upstream, originalPath, project) { const headers = new Headers(options.headers); headers.set(BASE_URL_HEADER, upstream.origin); if (originalPath) { headers.set(ORIGINAL_PATH_HEADER, originalPath); } + if (project) { + headers.set(PROJECT_HEADER, project); + } headers.delete("host"); const result = {}; headers.forEach((value, key) => { @@ -237,7 +244,7 @@ function headersForNodeRequest(options, upstream, originalPath) { }); return result; } -function routedNodeOptions(parts, proxy) { +function routedNodeOptions(parts, proxy, project) { if (!parts.url || !shouldRoute(parts.url, proxy)) { return void 0; } @@ -268,7 +275,7 @@ function routedNodeOptions(parts, proxy) { hostname: nextUrl.hostname, port: nextUrl.port || void 0, path: `${nextUrl.pathname}${nextUrl.search}`, - headers: headersForNodeRequest(parts.options, parts.url, originalPath) + headers: headersForNodeRequest(parts.options, parts.url, originalPath, project) }; } function wrapRequest(originalHttpRequest, originalHttpsRequest, originalRequest) { @@ -279,7 +286,7 @@ function wrapRequest(originalHttpRequest, originalHttpsRequest, originalRequest) } const proxy = normalizeProxyUrl(state.proxyUrl); const parts = splitNodeArgs(args); - const nextOptions = routedNodeOptions(parts, proxy); + const nextOptions = routedNodeOptions(parts, proxy, state.project); if (!nextOptions) { return Reflect.apply(originalRequest, this, args); } @@ -315,6 +322,7 @@ function installHeadroomTransport(options) { if (existing) { existing.refs += 1; existing.proxyUrl = options.proxyUrl; + existing.project = options.project; existing.debug = Boolean(options.debug); installProcessEnv(options.proxyUrl); return () => uninstallHeadroomTransport(); @@ -322,6 +330,7 @@ function installHeadroomTransport(options) { const state = { refs: 1, proxyUrl: options.proxyUrl, + project: options.project, debug: Boolean(options.debug), originalFetch: globalThis.fetch, originalHttpRequest: http.request, @@ -342,7 +351,7 @@ function installHeadroomTransport(options) { return state.originalFetch(...args); } const proxy = normalizeProxyUrl(current.proxyUrl); - const [nextInput, nextInit] = withRoutedFetchInput(args[0], args[1], proxy); + const [nextInput, nextInit] = withRoutedFetchInput(args[0], args[1], proxy, current.project); return state.originalFetch(nextInput, nextInit); }; http.request = wrapRequest(state.originalHttpRequest, state.originalHttpsRequest, state.originalHttpRequest); diff --git a/headroom/providers/proxy_routes.py b/headroom/providers/proxy_routes.py index e77838bb0..8800a0809 100644 --- a/headroom/providers/proxy_routes.py +++ b/headroom/providers/proxy_routes.py @@ -6,7 +6,7 @@ from __future__ import annotations import logging from typing import Any -from fastapi import FastAPI, Request, WebSocket +from fastapi import FastAPI, HTTPException, Request, WebSocket from fastapi.responses import Response from headroom.providers.cloudcode import normalize_cloudcode_passthrough_path @@ -67,6 +67,7 @@ from headroom.proxy.passthrough import ( custom_base_passthrough_telemetry as _custom_base_passthrough_telemetry, ) from headroom.proxy.request_scope import normalize_request_path +from headroom.proxy.upstream_guard import is_safe_upstream_url logger = logging.getLogger("headroom.proxy.routes") @@ -266,6 +267,9 @@ def register_provider_routes(app: FastAPI, proxy: Any) -> None: # OpenAI-compatible and generic passthrough routes. custom_base = request.headers.get("x-headroom-base-url", "").strip() if custom_base: + if not is_safe_upstream_url(custom_base): + logger.warning("rejecting unsafe x-headroom-base-url: %r", custom_base) + raise HTTPException(status_code=400, detail="Rejected unsafe upstream base URL") return await proxy.handle_anthropic_messages( request, upstream_base_url=custom_base.rstrip("/") ) @@ -506,6 +510,9 @@ def register_provider_routes(app: FastAPI, proxy: Any) -> None: async def passthrough(request: Request, path: str): custom_base = request.headers.get("x-headroom-base-url") if custom_base: + if not is_safe_upstream_url(custom_base): + logger.warning("rejecting unsafe x-headroom-base-url: %r", custom_base) + raise HTTPException(status_code=400, detail="Rejected unsafe upstream base URL") base_url = custom_base.rstrip("/") endpoint_name, provider_name = _custom_base_passthrough_telemetry( request.method, @@ -530,5 +537,7 @@ def register_provider_routes(app: FastAPI, proxy: Any) -> None: return await proxy.handle_passthrough( request, - _select_passthrough_base_url(proxy, dict(request.headers)), + # The path matters here: this is where unrouted paths land, and + # Copilot's inline completions are one of them (#3076). + _select_passthrough_base_url(proxy, dict(request.headers), request.url.path), ) diff --git a/headroom/providers/proxy_targets.py b/headroom/providers/proxy_targets.py index 11dfd596f..86b2c5447 100644 --- a/headroom/providers/proxy_targets.py +++ b/headroom/providers/proxy_targets.py @@ -5,6 +5,11 @@ from __future__ import annotations from collections.abc import Mapping from typing import Any, cast +from headroom.copilot_auth import ( + copilot_completions_base_url, + is_copilot_completions_host, + is_copilot_completions_path, +) from headroom.providers.codex import resolve_codex_routing from headroom.providers.codex.endpoints import CHATGPT_BACKEND_API_URL from headroom.providers.vertex import vertex_target_for_location as _vertex_target_for_location @@ -29,7 +34,9 @@ def vertex_target_for_location(proxy: Any, location: str) -> str: return _vertex_target_for_location(api_target(proxy, "vertex"), location) -def select_passthrough_base_url(proxy: Any, headers: Mapping[str, str]) -> str: +def select_passthrough_base_url( + proxy: Any, headers: Mapping[str, str], path: str | None = None +) -> str: """Resolve the upstream base URL for catch-all proxy passthrough requests.""" routing = resolve_codex_routing(headers) if routing.is_chatgpt_auth: @@ -41,4 +48,39 @@ def select_passthrough_base_url(proxy: Any, headers: Mapping[str, str]) -> str: if azure_base: return azure_base.rstrip("/") provider_name = proxy.provider_runtime.model_metadata_provider(headers) - return api_target(proxy, provider_name) + target = api_target(proxy, provider_name) + if ( + path is not None + and provider_name == "openai" + and is_copilot_completions_path(path) + and not is_copilot_completions_host(target) + ): + # Copilot's inline completions arrive here because + # `/v1/engines//completions` matches no built-in route. Nothing + # above this line looks at the path, so the request fell through to the + # OpenAI target and Headroom forwarded editor keystrokes to + # api.openai.com — a host that has not served the Engines API for years, + # and one many corporate networks block outright (#3076). + # + # Only Copilot emits this path, so sending it to Copilot is unambiguous. + # `copilot_completions_base_url()` does no I/O: it prefers an operator + # override, then the completions host GitHub advertised in the last + # token exchange, then the Copilot API URL — so the destination is + # GitHub's own answer where we have it rather than a hardcoded guess, + # and GHE deployments keep their host. + # + # The guard is on the *completions* host, not "any Copilot host". A CAPI + # host is not a completions host: `headroom wrap vscode` points the + # OpenAI target at the resolved subscription URL, which is the chat + # surface (`GITHUB_COPILOT_API_URL`), and leaving that alone sent + # `/v1/engines/.../completions` to a host that does not serve it. An + # already-correct completions host — an operator override or a per-SKU + # `endpoints.proxy` value — is still left untouched. + # + # Scoped to the OpenAI fall-through, which is the branch that is wrong + # for this path. Every other branch above reflects a deliberate choice + # of upstream by the caller's own auth headers, and the Copilot editor + # extension sends none of them — so a request that took one of those + # branches is not Copilot's and keeps the upstream it asked for. + return copilot_completions_base_url() + return target diff --git a/headroom/proxy/body_forwarding.py b/headroom/proxy/body_forwarding.py index 4648fce6c..deb07fe20 100644 --- a/headroom/proxy/body_forwarding.py +++ b/headroom/proxy/body_forwarding.py @@ -56,8 +56,30 @@ def get_python_forwarder_mode() -> PythonForwarderMode: def serialize_body_canonical(body: dict[str, Any]) -> bytes: - """Re-serialize a request body deterministically with cache-stable formatting.""" - return json.dumps(body, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + """Re-serialize a request body deterministically with cache-stable formatting. + + ``ensure_ascii=False`` keeps the bytes compact and cache-stable, but it also + means a lone surrogate anywhere in the body raises here. That is reachable + input, not a hypothetical: ``"\\ud800"`` is valid JSON, ``json.loads`` + accepts it happily, and a tool result carrying truncated UTF-16 or sliced + binary produces one. Both forwarders resolve outbound bytes *outside* their + connection-retry loop, so the exception escapes as a 500 with no retry. + + #3124 made that newly load-bearing: mutated thinking-bearing bodies used to + return the client's bytes verbatim and never reached this function at all, + so the largest, most tool-result-heavy population in Claude Code traffic now + depends on it not raising. + + The escaped form is the right degradation -- it encodes the identical parsed + values, so upstream reconstructs exactly the same request, and every mutation + still reaches the wire (important: the caller's ``stream`` flip rides on + these bytes). Only the byte-level encoding differs, costing one cache miss on + a request that would otherwise have failed outright. + """ + try: + return json.dumps(body, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + except UnicodeEncodeError: + return json.dumps(body, separators=(",", ":"), ensure_ascii=True).encode("utf-8") def has_signed_thinking_blocks(body: dict[str, Any]) -> bool: @@ -81,6 +103,172 @@ def has_signed_thinking_blocks(body: dict[str, Any]) -> bool: return False +#: A body carrying a ``thinking`` or ``redacted_thinking`` block contains this +#: substring, because it appears in the block's own ``type`` value. Scanning for +#: it is orders of magnitude cheaper than parsing, and request bodies here reach +#: 9.3 MB in real agent traffic -- so this prescreen keeps the common case (no +#: thinking anywhere, ~2 of every 3 requests) from paying a full JSON parse it +#: cannot learn anything from. +#: +#: A false positive costs one parse we would have done anyway. A false negative +#: is not strictly impossible -- a ``type`` value whose ASCII letters are written +#: as JSON unicode escapes still parses to ``thinking`` while containing no +#: literal match, and that is legal JSON no standard encoder emits +#: (``json.dumps`` does not escape ASCII even under +#: ``ensure_ascii=True``, and neither does any client we forward for) -- and its +#: consequences are asymmetric in our favour: when the mutated body still holds +#: the block, ``has_signed_thinking_blocks(body)`` sees it on the parsed dict and +#: we lock anyway. The single reachable gap needs an escaped ``type`` in the +#: original AND a transform that removed the block from the body, which is the +#: rare #3015 shape crossed with an encoder nobody uses. Documented rather than +#: defended, because closing it means re-parsing every large body to catch a case +#: no observed client can produce. +_THINKING_SUBSTRING = b"thinking" + + +def _parse_original_body(original_body_bytes: bytes | None) -> dict[str, Any] | None: + """Parse the client's body once, or ``None`` when it cannot be used. + + Every caller here needs the same parsed document, and a 9.3 MB body parsed + twice per decision (and twice again in the handler's earlier probe) is real + latency on a stage that already has a 30s timeout whose expiry quarantines + compression process-wide. Parse in one place and pass the result down. + + ``None`` means "cannot prove anything from the original", which every caller + must treat as the conservative answer. + """ + if original_body_bytes is None: + return None + if _THINKING_SUBSTRING not in original_body_bytes: + return None + try: + original = json.loads(original_body_bytes) + except (json.JSONDecodeError, UnicodeDecodeError, ValueError, MemoryError, RecursionError): + return None + return original if isinstance(original, dict) else None + + +def _original_body_has_signed_thinking_blocks(original_body_bytes: bytes | None) -> bool: + original = _parse_original_body(original_body_bytes) + return original is not None and has_signed_thinking_blocks(original) + + +#: Allow canonical re-serialization on a thinking-bearing request when every +#: thinking block is provably unchanged. **Default ON**; set this to ``0`` (or +#: ``false``/``no``/``off``) to restore the previous blanket lock. +#: +#: The lock this relaxes was added by #2254 after real upstream 400s +#: ("`thinking` blocks ... cannot be modified"). +#: +#: What the signature actually covers is now measured, not assumed -- +#: ``tests/test_thinking_signature_scope_live.py`` pins it against the live API +#: on sonnet-4-5, opus-4-5, sonnet-4-6, sonnet-5 and opus-5, with identical +#: results on all five. Anthropic accepts a replayed turn whose signed thinking +#: block is intact while we compress a ``tool_result``, rewrite sibling +#: ``text``/``tool_use`` blocks *inside the same assistant message*, rewrite +#: top-level ``system``/``tools``, or re-serialize the whole body with reordered +#: keys. It rejects exactly one thing: a forged ``signature`` ("Invalid +#: `signature` in `thinking` block"), which is the negative control proving the +#: endpoint validates signatures on this shape at all -- without it every +#: acceptance above would be vacuous. +#: +#: So the seal binds the block, not the request, and #2254's stated cause (a +#: plain canonical re-encode) is disproven directly: variant F changes the bytes +#: and is accepted. The 400s were real but were never traced to their true +#: trigger. This relaxation stays narrower than the evidence permits anyway -- +#: it forwards edits ONLY when every thinking block is byte-identical to the +#: client's -- so the measurements above are headroom, not the safety margin. +#: +#: If Anthropic ever changes this, that live test fails loudly, and setting the +#: env var to ``0`` is a single-variable, no-deploy rollback to the blanket lock. +THINKING_PRESERVING_MUTATIONS_ENV = "HEADROOM_THINKING_PRESERVING_MUTATIONS" + + +def thinking_preserving_mutations_enabled() -> bool: + """Whether to forward edits that provably left every thinking block intact. + + Defaults to enabled. Only an explicit falsey value restores the blanket lock, + so an unset or unparseable variable keeps the documented default rather than + silently reverting behaviour. + """ + raw = os.environ.get(THINKING_PRESERVING_MUTATIONS_ENV) + if raw is None: + return True + return raw.strip().lower() not in ("0", "false", "no", "off") + + +def thinking_block_fingerprint(body: Any) -> list[tuple[int, int, str]]: + """Positional, order-sensitive fingerprint of every thinking block. + + Each entry is ``(message_index, block_index, canonical_json_of_block)``, so + the comparison catches a block whose text or ``signature`` changed, one that + was added, removed, reordered, or moved between messages. Keys are sorted so + a dict rebuilt in a different order is not mistaken for an edit -- the wire + contract is over parsed values, not key order. + """ + out: list[tuple[int, int, str]] = [] + messages = body.get("messages") if isinstance(body, dict) else None + if not isinstance(messages, list): + return out + for message_index, message in enumerate(messages): + if not isinstance(message, dict): + continue + content = message.get("content") + if not isinstance(content, list): + continue + for block_index, block in enumerate(content): + if isinstance(block, dict) and block.get("type") in { + "thinking", + "redacted_thinking", + }: + out.append( + ( + message_index, + block_index, + json.dumps(block, sort_keys=True, ensure_ascii=False), + ) + ) + return out + + +def thinking_blocks_survived_mutation( + body: dict[str, Any], + original_body_bytes: bytes | None, + original: dict[str, Any] | None = None, +) -> bool: + """True when every thinking block is byte-equal to the one the client sent. + + This is the whole point of the relaxation. Anthropic signs the thinking + block, not the request: the signature covers that block's own content, so + edits to a ``tool_result`` twenty turns back, or to the top-level ``tools`` + and ``system`` fields (which are not even inside ``messages`` and therefore + cannot be covered by any per-block signature), leave every seal intact. + Treating the presence of a sealed block as a reason to freeze the entire + body is a category error -- it protects bytes the signature says nothing + about, and on Claude Code traffic that is nearly the whole request. + + That scope claim is measured against the live API, not inferred -- see + ``tests/test_thinking_signature_scope_live.py``, which also pins the one + thing Anthropic does reject (a forged ``signature``). + + Conservative by construction: any parse failure, or any detectable + difference at all, returns False and the caller keeps today's passthrough. + + Pass ``original`` when the caller has already parsed the client body, so a + multi-megabyte document is not re-parsed once per predicate. + """ + if original is None: + original = _parse_original_body(original_body_bytes) + if original is None: + return False + try: + return thinking_block_fingerprint(body) == thinking_block_fingerprint(original) + except (TypeError, ValueError, RecursionError): + # An unserializable block (or pathological nesting) means we cannot prove + # the blocks are untouched, so we must not claim they are. + return False + + class BodyMutationTracker: """Records whether a request body was mutated and why.""" @@ -123,7 +311,31 @@ def select_outbound_body( upstream instead of silently claiming the edit landed. """ mode = forwarder_mode if forwarder_mode is not None else get_python_forwarder_mode() - if original_body_bytes is not None and has_signed_thinking_blocks(body): + # Parse the client body at most once for the whole decision (bodies here + # reach 9.3 MB in real traffic; this used to cost two full parses). + original_parsed = _parse_original_body(original_body_bytes) + if original_body_bytes is not None and ( + has_signed_thinking_blocks(body) + or (original_parsed is not None and has_signed_thinking_blocks(original_parsed)) + ): + # The lock is about the SEAL, not the box. When every thinking block is + # provably identical to the one the client sent, no signature can have + # been invalidated, so the remaining edits (compressed tool_result text, + # compacted tool schemas, tool-search deferral) are safe to forward. + # Without this, one thinking block anywhere in history froze the entire + # request for the rest of the session -- on real Claude Code traffic that + # discarded every computed compression on 34% of requests. + if thinking_preserving_mutations_enabled() and thinking_blocks_survived_mutation( + body, original_body_bytes, original=original_parsed + ): + if mode == "legacy_json_kwarg": + content = json.dumps(body, separators=(", ", ": "), ensure_ascii=True).encode( + "utf-8" + ) + return OutboundBody(content=content, source="legacy") + if body_mutated: + return OutboundBody(content=serialize_body_canonical(body), source="canonical") + return OutboundBody(content=original_body_bytes, source="passthrough") return OutboundBody( content=original_body_bytes, source="passthrough", @@ -177,7 +389,28 @@ def outbound_body_is_client_bytes( send the client's bytes verbatim. Asking before acting is cheaper than discovering it from a reply in the wrong wire format. - Mirrors the first branch of :func:`select_outbound_body`; the forwarder mode - is deliberately not consulted because that branch overrides it too. + Mirrors the first branch of :func:`select_outbound_body`, INCLUDING the + thinking-preserving relaxation. These two must agree exactly: the caller + flips ``stream`` to False to buy a buffered reply, and that flip only lands + if the mutated body actually reaches the wire. If this said "locked" while + ``select_outbound_body`` shipped canonical bytes, we would ask upstream for a + stream and then parse the reply as buffered JSON -- the 200-with-empty-body + failure from #2952, in reverse. + + The forwarder mode is deliberately not consulted: the lock branch overrides + it, and the relaxation only ever returns bytes derived from ``body``, so + either way the caller's edits are on the wire. """ - return original_body_bytes is not None and has_signed_thinking_blocks(body) + if original_body_bytes is None: + return False + original_parsed = _parse_original_body(original_body_bytes) + if not ( + has_signed_thinking_blocks(body) + or (original_parsed is not None and has_signed_thinking_blocks(original_parsed)) + ): + return False + if thinking_preserving_mutations_enabled() and thinking_blocks_survived_mutation( + body, original_body_bytes, original=original_parsed + ): + return False + return True diff --git a/headroom/proxy/buffered_ccr_response.py b/headroom/proxy/buffered_ccr_response.py new file mode 100644 index 000000000..cd0bc0b9a --- /dev/null +++ b/headroom/proxy/buffered_ccr_response.py @@ -0,0 +1,312 @@ +"""The ASGI wrapper for a buffered-CCR turn, shared by both provider handlers. + +Server-side CCR retrieval needs the whole upstream reply in hand before it can +answer, so a ``stream: true`` turn is flipped to ``stream: false`` upstream and +resynthesized as SSE on the way out. That leaves a window — the entire +generation — where the proxy is holding a request open with nothing to say yet, +and two constraints pull in opposite directions across it: + +* **Status fidelity.** Committing ``200 text/event-stream`` before the outcome + is known destroys it. Any reply that then fails to become SSE reaches the + client as a 200 with no ``message_start`` ("API returned an empty or + malformed response (HTTP 200)"), and the real status goes with it, so + client-side 429/5xx backoff never fires. This is what #2997 fixed by never + committing early. + +* **Liveness.** Sending nothing at all for the whole wait trips the client's + *stream-idle* watchdog. That timer is separate from the total-request budget + (``x-stainless-timeout``), and it is the one #2465 was about. #2479 fixed it + with a keepalive preamble, which #2997 removed as collateral — putting #2465's + condition back (#3079). + +Neither property is worth trading for the other, so this keeps both: + +1. For ``grace_seconds`` nothing is sent, and anything resolving inside that + window is handed to the client untouched — real status, real headers. Fast + failures (a 4xx, or a 429/529 that resolves once ``_retry_request`` has + honored ``Retry-After``) land here. +2. Past the window the response is committed as SSE and a heartbeat starts, so + a first byte always precedes any client idle watchdog. +3. A failure arriving *after* the commit can no longer carry an HTTP status, so + it is translated into the provider's own typed SSE error instead of a generic + one. A rate limit still reads as a rate limit, and client backoff still + fires — the property that made early commits harmful in the first place. + +Set ``grace_seconds`` to 0 or less to disable the heartbeat entirely and always +wait for full fidelity. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import Any + +logger = logging.getLogger("headroom.proxy") + +DEFAULT_BUFFERED_CCR_GRACE_SECONDS = 5.0 +"""Seconds to hold out for full status fidelity before committing to SSE. + +Comfortably under any client stream-idle watchdog, while still covering the +fast failures whose status matters most. +""" + +_HEARTBEAT_INTERVAL_SECONDS = 0.25 + + +@dataclass(frozen=True) +class BufferedCCRErrorFormat: + """The provider-specific error shapes this wrapper has to emit. + + Anthropic and OpenAI disagree on both the JSON envelope and the SSE framing, + and the difference is pure formatting — the decision logic above is shared. + """ + + provider: str + #: Build the pre-commit JSON body for a 502. + json_body: Callable[[str], bytes] + #: Build a post-commit SSE error event from an error type and message. + sse_event: Callable[[str, str], bytes] + #: Map an upstream HTTP status onto this provider's error type string. + error_type_for_status: Callable[[int], str] + #: The keepalive frame to emit while waiting, once committed. + heartbeat: bytes + + +def _anthropic_error_type(status: int) -> str: + # The wire types Anthropic documents; clients switch their retry behaviour + # on these, so a 429 must not arrive labelled `api_error`. + return { + 400: "invalid_request_error", + 401: "authentication_error", + 403: "permission_error", + 404: "not_found_error", + 413: "request_too_large", + 429: "rate_limit_error", + 529: "overloaded_error", + }.get(status, "api_error") + + +def _openai_error_type(status: int) -> str: + return { + 400: "invalid_request_error", + 401: "authentication_error", + 403: "permission_error", + 404: "not_found_error", + 413: "invalid_request_error", + 429: "rate_limit_error", + }.get(status, "server_error") + + +ANTHROPIC_ERROR_FORMAT = BufferedCCRErrorFormat( + provider="anthropic", + json_body=lambda message: json.dumps( + {"type": "error", "error": {"type": "api_error", "message": message}} + ).encode(), + sse_event=lambda error_type, message: ( + "event: error\ndata: " + + json.dumps({"type": "error", "error": {"type": error_type, "message": message}}) + + "\n\n" + ).encode(), + error_type_for_status=_anthropic_error_type, + # Anthropic's stream carries a real `ping` event type. + heartbeat=b'event: ping\ndata: {"type":"ping"}\n\n', +) + +OPENAI_ERROR_FORMAT = BufferedCCRErrorFormat( + provider="openai", + json_body=lambda message: json.dumps( + {"error": {"message": message, "type": "server_error", "code": "proxy_error"}} + ).encode(), + sse_event=lambda error_type, message: ( + "data: " + json.dumps({"error": {"message": message, "type": error_type}}) + "\n\n" + ).encode(), + error_type_for_status=_openai_error_type, + # OpenAI has no ping event; an SSE comment keeps the socket warm without + # putting a frame the client would try to parse on the wire. + heartbeat=b": ping\n\n", +) + +_GENERIC_FAILURE_MESSAGE = "An error occurred while processing your request. Please try again." + + +def _upstream_message(body: bytes | None, fallback: str) -> str: + """Prefer the upstream's own error text over a synthesized one. + + A committed stream cannot carry the upstream status, so the message is the + only place its detail survives. Falls back whenever the body is not a + recognizable error envelope. + """ + + if not body: + return fallback + try: + parsed = json.loads(body) + except (ValueError, TypeError): + return fallback + if not isinstance(parsed, dict): + return fallback + error = parsed.get("error") + if isinstance(error, dict): + message = error.get("message") + if isinstance(message, str) and message.strip(): + return message.strip() + message = parsed.get("message") + if isinstance(message, str) and message.strip(): + return message.strip() + return fallback + + +async def _send_committed_failure( + send: Callable[[dict[str, Any]], Awaitable[None]], + fmt: BufferedCCRErrorFormat, + *, + status: int, + body: bytes | None, +) -> None: + """Emit a typed SSE error for a failure that arrived after the commit.""" + + error_type = fmt.error_type_for_status(status) + message = _upstream_message(body, _GENERIC_FAILURE_MESSAGE) + await send( + { + "type": "http.response.body", + "body": fmt.sse_event(error_type, message), + "more_body": False, + } + ) + + +async def _forward_after_commit( + result: Any, + send: Callable[[dict[str, Any]], Awaitable[None]], + fmt: BufferedCCRErrorFormat, + *, + request_id: str, +) -> None: + """Relay a resolved result once SSE headers are already on the wire.""" + + status = int(getattr(result, "status_code", 200) or 200) + body_iterator = getattr(result, "body_iterator", None) + + if body_iterator is not None: + # Streaming results are already SSE — both the success path and the + # 502 CCR-failure paths, whose bodies are error events. Forwarding the + # bytes keeps whatever detail they carry. + async for chunk in body_iterator: + await send({"type": "http.response.body", "body": chunk, "more_body": True}) + await send({"type": "http.response.body", "body": b"", "more_body": False}) + return + + # A non-streaming result here is an upstream reply that never became SSE. + # Its status cannot reach the client anymore, so preserve its meaning in + # the error type instead of degrading to a generic failure (#3079). + body = getattr(result, "body", None) + if status != 200: + logger.warning( + f"[{request_id}] CCR: buffered upstream returned {status} after the response " + "was committed as SSE; relaying it as a typed stream error" + ) + await _send_committed_failure(send, fmt, status=status, body=body) + + +def buffered_ccr_asgi_call( + *, + operation: asyncio.Task, + fmt: BufferedCCRErrorFormat, + grace_seconds: float, + record_failed: Callable[..., Awaitable[None]], + request_id: str, +) -> Callable[[Any, Any, Any], Awaitable[None]]: + """Build the ``__call__`` body for a buffered-CCR ASGI response. + + Returned rather than subclassed so both handlers can keep their existing + ``Response`` shells and differ only in the error format they pass in. + """ + + async def __call__(scope: Any, receive: Any, send: Any) -> None: # noqa: ANN401 + loop = asyncio.get_running_loop() + committed = False + deadline = loop.time() + grace_seconds + heartbeat_enabled = grace_seconds > 0 + try: + while True: + if heartbeat_enabled: + timeout = ( + _HEARTBEAT_INTERVAL_SECONDS + if committed + else max(0.0, deadline - loop.time()) + ) + else: + timeout = None + done, _pending = await asyncio.wait({operation}, timeout=timeout) + + if done: + try: + result = operation.result() + except asyncio.CancelledError: + raise + except Exception as exc: + await record_failed(provider=fmt.provider) + logger.error(f"[{request_id}] Request failed: {type(exc).__name__}: {exc}") + if committed: + await _send_committed_failure(send, fmt, status=502, body=None) + return + await send( + { + "type": "http.response.start", + "status": 502, + "headers": [(b"content-type", b"application/json")], + } + ) + await send( + { + "type": "http.response.body", + "body": fmt.json_body(_GENERIC_FAILURE_MESSAGE), + "more_body": False, + } + ) + return + + if not committed: + # Nothing is on the wire yet, so the result keeps its + # own status and headers. This is the fidelity #2997 + # was protecting. + await result(scope, receive, send) + return + + await _forward_after_commit(result, send, fmt, request_id=request_id) + return + + if not committed: + # The grace window expired without an outcome. Commit now so + # a first byte beats the client's stream-idle watchdog. + await send( + { + "type": "http.response.start", + "status": 200, + "headers": [(b"content-type", b"text/event-stream")], + } + ) + committed = True + logger.debug( + f"[{request_id}] CCR: buffered turn exceeded the {grace_seconds}s " + "grace window; committing SSE and starting the heartbeat" + ) + await send({"type": "http.response.body", "body": fmt.heartbeat, "more_body": True}) + except asyncio.CancelledError: + raise + finally: + if not operation.done(): + operation.cancel() + try: + await operation + except asyncio.CancelledError: + pass + except Exception: + pass + + return __call__ diff --git a/headroom/proxy/extensions.py b/headroom/proxy/extensions.py index 9ead7a6cd..183028f71 100644 --- a/headroom/proxy/extensions.py +++ b/headroom/proxy/extensions.py @@ -18,6 +18,41 @@ Each ``install`` callable is invoked with the FastAPI ``app`` and the OSS makes no assumptions about what extensions do. The interface is deliberately minimal; extensions own the complexity behind it. +Reporting what an extension saved, and what it cost +--------------------------------------------------- + +An extension that changes the bill should say so, or the operator sees a +different total with nothing to attribute it to. Two calls, both taking the +ASGI ``scope`` so they work from middleware — which runs outside the request +handler and has no other way in:: + + from headroom.proxy.savings_attribution import ( + record_scope_savings, record_scope_timing, + ) + + record_scope_savings(scope, "my_extension", tokens=1200, usd=0.004) + record_scope_timing(scope, "my_extension", elapsed_ms) + +``record_scope_savings`` takes ``tokens``, ``usd``, or both, so an extension +that saves money WITHOUT saving tokens — routing a request to a cheaper model, +say — can report a real number instead of a token count nobody saved. Pass +``realized=False`` for a projection rather than a measured amount; the two are +kept apart everywhere they surface. Savings land on ``/stats`` under +``savings.by_source``, on the dashboard as their own card, and in Prometheus as +``headroom_savings_attributed_usd_total{source=...}``. **Attribution only** — +these rows explain the headline total, they are never added to it. + +``record_scope_timing`` is the other half of the trade: an extension's own +latency, which is otherwise invisible because ``overhead_ms`` is measured +inside the handler that the extension wraps. It lands in ``/stats`` under +``pipeline_timing``, in the dashboard's Performance panel, and in +``headroom_transform_timing_ms_*``, namespaced ``ext:`` so it can never +collide with a built-in transform. + +Both are bounded (32 sources, 16 stages), never raise, and never change a +response — telemetry from a plugin must not be able to break the request it is +describing. + **Extensions are opt-in.** Discovery enumerates every registered extension, but ``install_all`` only invokes those explicitly enabled by the operator. This protects users from silent behavior changes when a package they didn't diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py index 79045fa84..9b0c9d502 100644 --- a/headroom/proxy/handlers/anthropic.py +++ b/headroom/proxy/handlers/anthropic.py @@ -14,6 +14,7 @@ import time import uuid from datetime import datetime from typing import TYPE_CHECKING, Any +from urllib.parse import urlsplit from headroom.proxy.stage_timer import StageTimer, emit_stage_timings_log @@ -33,19 +34,47 @@ from headroom.proxy.auth_mode import ( classify_client, supports_mid_turn_coalescing, ) +from headroom.proxy.buffered_ccr_response import ( + ANTHROPIC_ERROR_FORMAT, + DEFAULT_BUFFERED_CCR_GRACE_SECONDS, + buffered_ccr_asgi_call, +) from headroom.proxy.compression_decision import CompressionDecision from headroom.proxy.forwarded_headers import resolve_client_ip from headroom.proxy.handlers._debug_dump import _debug_dump_mode, _redact_debug_value -from headroom.proxy.helpers import extract_tags, relocate_system_messages_to_top_level +from headroom.proxy.helpers import ( + extract_tags, + relocate_system_messages_to_top_level, + sanitize_forwarded_response_headers, +) +from headroom.proxy.identity import resolve_memory_identity from headroom.proxy.image_isolation import run_image_compression_isolated from headroom.proxy.memory_decision import MemoryDecision from headroom.proxy.memory_query import MemoryQuery from headroom.proxy.model_router import estimate_input_tokens +from headroom.proxy.nonstream_sse_policy import should_recover_sse_reply from headroom.proxy.outcome import RequestOutcome logger = logging.getLogger("headroom.proxy") +def _is_googleapis_endpoint(value: object) -> bool: + """Return whether *value* targets Google APIs by parsed hostname. + + A substring check would also trust attacker-controlled hosts such as + ``googleapis.com.example.test``. URL parsing plus a label-boundary suffix + check accepts Google API subdomains without widening the route gate. + """ + raw = str(value).strip() + if not raw: + return False + try: + hostname = (urlsplit(raw).hostname or "").rstrip(".").lower() + except ValueError: + return False + return hostname == "googleapis.com" or hostname.endswith(".googleapis.com") + + class _AnthropicTurnHookUsage: """Usage from hook-triggered Anthropic calls the main response omits. @@ -199,6 +228,66 @@ def _looks_like_sse_response(response: httpx.Response) -> bool: class AnthropicHandlerMixin: """Mixin providing Anthropic API handler methods for HeadroomProxy.""" + def _adapt_event_stream_to_json( + self, + response: httpx.Response, + request_id: str, + ) -> httpx.Response: + """Rebuild an SSE reply as the JSON a non-streaming caller asked for. + + A caller that sent ``stream: false`` cannot parse ``text/event-stream``, + so relaying it verbatim loses a turn the upstream already charged for + (#3130). Reconstruction is strict: a truncated stream, or one carrying + an ``error`` event, becomes an explicit 502 rather than a successful + HTTP 200 whose message is silently short. + """ + headers = { + k: v + for k, v in sanitize_forwarded_response_headers( + response.headers, + "content-type", + ).items() + if not k.lower().startswith("cf-") + } + + parsed = None + try: + parsed = self._parse_sse_to_response( + response.content.decode("utf-8", "replace"), + "anthropic", + require_complete=True, + ) + except Exception as exc: # pragma: no cover - defensive + logger.warning(f"[{request_id}] SSE->JSON reconstruction raised: {exc}") + + if parsed is None: + logger.error( + f"[{request_id}] Upstream answered a non-streaming request with an " + f"event stream that could not be faithfully reconstructed " + f"(body_bytes={len(response.content)}); returning 502 rather than " + f"a wire format the client cannot parse" + ) + return httpx.Response( + 502, + json={ + "type": "error", + "error": { + "type": "upstream_protocol_error", + "message": ( + "Upstream answered a non-streaming request with an " + "incomplete event stream." + ), + }, + }, + headers=headers, + ) + + logger.info( + f"[{request_id}] Upstream answered a non-streaming request with an " + f"event stream; adapted {len(response.content)} bytes of SSE to JSON" + ) + return httpx.Response(200, json=parsed, headers=headers) + async def _count_tokens_offloaded(self, model, messages): # noqa: ANN001, ANN201 from headroom.proxy.token_counting import count_tokens_offloaded @@ -243,7 +332,7 @@ class AnthropicHandlerMixin: ctx = _CtxFor( headers=dict(request.headers), system_prompt=_extract_sys_prompt(body), - base_user_id=request.headers.get("x-headroom-user-id", ""), + base_user_id=resolve_memory_identity(request, default=""), project_root_override=None, ) ident = ProjectResolver().resolve(ctx) @@ -291,6 +380,72 @@ class AnthropicHandlerMixin: return True return False + def _can_salvage_buffered_upstream(self, resp_json: Any) -> bool: + """May this upstream response be relayed when post-processing failed? + + Only when the client can actually consume it. The buffered path exists + because a ``headroom_retrieve`` call has to be resolved server-side, so + a response still carrying one is exactly the case the handler already + fails closed on: relaying it would hand the client a tool call naming an + endpoint it is not expected to reach, and a marker nobody expanded. + + Anything else — an ordinary answer, a client tool call, a turn whose + retrieval already resolved — is a complete provider turn and is safer in + the client's hands than a synthesized error (#3088). + """ + if not isinstance(resp_json, dict): + return False + handler = getattr(self, "ccr_response_handler", None) + if handler is None: + return True + try: + from headroom.ccr.response_handler import RESIDUAL_CCR_ERROR + + if handler.residual_ccr_status(resp_json, "anthropic") == RESIDUAL_CCR_ERROR: + return False + return not handler.has_ccr_tool_calls(resp_json, "anthropic") + except Exception: # pragma: no cover - defensive + logger.debug("CCR: salvage check failed; not salvaging", exc_info=True) + return False + + @staticmethod + def _outgoing_body_has_redeemable_marker(body: Any) -> bool: + """Does the body about to be sent carry a marker retrieval could expand? + + ``headroom_retrieve`` exists only to expand a ``<>`` marker, so + a request carrying none cannot benefit from the buffered path (#3071). + + Ownership is verified rather than shape-matched: the marker shape is not + unique to Headroom, and adopting another context tool's hash would send + the model to an endpoint that is guaranteed to miss (#2836). A hash that + survives ``verify_ownership`` is redeemable right now. + + Errors are swallowed deliberately and answered ``True``. This gates a + wire-format decision, and the safe direction on an unexpected message + shape is the long-standing buffered behavior, not a silent change. + """ + if not isinstance(body, dict): + return True + messages = body.get("messages") + if not isinstance(messages, list) or not messages: + return False + try: + from headroom.ccr.tool_injection import CCRToolInjector + + probe = CCRToolInjector( + provider="anthropic", + inject_tool=False, + inject_system_instructions=False, + ) + probe.scan_for_markers(messages) + if not probe.detected_hashes: + return False + probe.verify_ownership() + return bool(probe.detected_hashes) + except Exception: # pragma: no cover - defensive + logger.debug("CCR: marker probe failed; keeping the buffered path", exc_info=True) + return True + @staticmethod def _extract_anthropic_cache_ttl_metrics(usage: dict[str, Any] | None) -> tuple[int, int]: """Extract observed Anthropic cache-write TTL bucket usage. @@ -995,7 +1150,15 @@ class AnthropicHandlerMixin: _pre_strip_count = sum(1 for k in headers if k.lower().startswith("x-headroom-")) headers = _strip_internal_headers(headers) - headers = merge_extra_headers(headers, self.config.anthropic_extra_headers) + # `upstream_base_url` is the per-request `x-headroom-base-url` + # override when the client sent one. These headers are secrets, so + # they only travel to a host the operator designated. + headers = merge_extra_headers( + headers, + self.config.anthropic_extra_headers, + upstream_url=upstream_base_url, + config=self.config, + ) log_outbound_headers( forwarder="anthropic_messages", stripped_count=_pre_strip_count @@ -1061,10 +1224,7 @@ class AnthropicHandlerMixin: memory_user_id: str | None = None memory_request_ctx = None if self.memory_handler: - memory_user_id = request.headers.get( - "x-headroom-user-id", - os.environ.get("USER", os.environ.get("USERNAME", "default")), - ) + memory_user_id = resolve_memory_identity(request) # Per-project memory routing (GH #462). Build the context # once here so save / search / inject all resolve against # the same workspace. Tier order: explicit project-id / @@ -1176,15 +1336,32 @@ class AnthropicHandlerMixin: ) ) - # Remove compression headers from cached response - response_headers = dict(cached.response_headers) - response_headers.pop("content-encoding", None) - response_headers.pop("content-length", None) - # Drop the stored content-type too. Starlette lets an - # explicit header win over ``media_type``, so keeping the - # producing request's type would let a cache entry hand this - # caller a wire format it never asked for (#2952). - response_headers.pop("content-type", None) + # Strip the stored response's wire-framing headers. The + # entry carries whatever the *producing* upstream sent, + # and replaying that framing over a different connection + # breaks the body: a stale ``transfer-encoding: chunked`` + # makes the client parse plain JSON as chunked frames and + # read nothing out of an HTTP 200 (#3019). ``content-type`` + # goes too, because Starlette lets an explicit header win + # over ``media_type`` and the producing request's type + # would hand this caller a wire format it never asked + # for (#2952). + response_headers = sanitize_forwarded_response_headers( + cached.response_headers, + "content-type", + ) + + # A cache hit answers the client without touching the + # upstream, so it emits no outbound_request line and no + # upstream stage timings. Without this log a served-from- + # cache turn is indistinguishable from a turn that died + # silently, which is exactly how #3019 stayed invisible. + logger.info( + f"[{request_id}] RESPONSE-CACHE-HIT: model={model} " + f"bytes={len(cached.response_body)} " + f"age_s={(datetime.now() - cached.created_at).total_seconds():.0f} " + f"hits={cached.hit_count}" + ) # Unit 4: release the pre-upstream semaphore on cache # hit — no upstream call will happen. @@ -1343,24 +1520,20 @@ class AnthropicHandlerMixin: # lossless whole-prefix recompaction instead of the byte-identical splice # (the splice preserves a dead cache) and skips the overlay replay. Both are # deterministic → the recompacted prefix re-caches byte-stable on warm turns. + from headroom.transforms.cold_prefix import ( + anthropic_cache_ttl_seconds, + is_cold_prefix, + ) + + # Resolve the authoritative request-level prompt-cache tier once. + # The same value drives cold-prefix handling and net-cost pricing. + _cc_ttl = anthropic_cache_ttl_seconds(model, original_client_messages, system_prompt) _cold_recompact_active = False if os.environ.get("HEADROOM_COLD_RECOMPACT", "").strip().lower() in ( "1", "true", "yes", ): - from headroom.transforms.cold_prefix import ( - anthropic_cache_ttl_seconds, - is_cold_prefix, - ) - - # Read CC's ACTUAL prompt-cache TTL (request cache_control.ttl + the - # DISABLE_/ENABLE_/FORCE_PROMPT_CACHING_* env controls) instead of the - # static 300s guess — a wrong TTL is exactly what busts a warm cache. - # None ⇒ caching is OFF (no cache to bust) ⇒ recompact every turn. - _cc_ttl = anthropic_cache_ttl_seconds( - model, original_client_messages, system_prompt - ) _cold_recompact_active = _cc_ttl is None or is_cold_prefix( prefix_tracker, ttl_seconds=_cc_ttl ) @@ -1559,6 +1732,7 @@ class AnthropicHandlerMixin: biases=biases, request_id=request_id, compression_policy=compression_policy, + cache_ttl_seconds=_cc_ttl, **proxy_pipeline_kwargs(self.config), ), lambda bg_result: comp_cache.update_from_result( @@ -1603,6 +1777,7 @@ class AnthropicHandlerMixin: biases=biases, request_id=request_id, compression_policy=compression_policy, + cache_ttl_seconds=_cc_ttl, skip_kompress=True, **proxy_pipeline_kwargs(self.config), ), @@ -1653,6 +1828,7 @@ class AnthropicHandlerMixin: biases=biases, request_id=request_id, compression_policy=compression_policy, + cache_ttl_seconds=_cc_ttl, **proxy_pipeline_kwargs(self.config), ), timeout=COMPRESSION_TIMEOUT_SECONDS, @@ -1694,6 +1870,7 @@ class AnthropicHandlerMixin: biases=biases, request_id=request_id, compression_policy=compression_policy, + cache_ttl_seconds=_cc_ttl, **proxy_pipeline_kwargs(self.config), ), timeout=COMPRESSION_TIMEOUT_SECONDS, @@ -1826,6 +2003,7 @@ class AnthropicHandlerMixin: biases=biases, request_id=request_id, compression_policy=compression_policy, + cache_ttl_seconds=_cc_ttl, **proxy_pipeline_kwargs(self.config), ), timeout=COMPRESSION_TIMEOUT_SECONDS, @@ -1892,22 +2070,35 @@ class AnthropicHandlerMixin: overlay_cached_prefix, ) + _overlay_replayed = False # On a confirmed-cold turn we deliberately do NOT replay the previously # forwarded prefix: the cache is dead (nothing to keep byte-identical for) # and the replay would clobber the whole-prefix recompaction we just did. - if _cold_recompact_active: - _overlay_replayed = False + if _decision.should_compress and not _skip_compression_for_backpressure: + if _cold_recompact_active: + _overlay_replayed = False + else: + _ov = overlay_cached_prefix( + optimized_messages, + original_client_messages, + previous_original_messages, + previous_forwarded_messages, + ) + _overlay_replayed = _ov != optimized_messages + if _overlay_replayed: + optimized_messages = _ov + optimized_tokens = tokenizer.count_messages(optimized_messages) else: - _ov = overlay_cached_prefix( - optimized_messages, - original_client_messages, - previous_original_messages, - previous_forwarded_messages, + replay_skip_reason = ( + "pre_upstream_backpressure" + if _skip_compression_for_backpressure + else _decision.passthrough_reason + ) + logger.debug( + "[%s] Cached-prefix replay skipped: reason=%s", + request_id, + replay_skip_reason, ) - _overlay_replayed = _ov != optimized_messages - if _overlay_replayed: - optimized_messages = _ov - optimized_tokens = tokenizer.count_messages(optimized_messages) # Own cache_control placement: the client moves the breakpoint each # turn and the overlay replays past markers, so they accumulate ~1/turn @@ -2990,7 +3181,19 @@ class AnthropicHandlerMixin: # the top-level 'system' parameter ..."), so relocate it back to the # top-level ``system`` parameter as the last step before forwarding. relocated_messages, relocated_system, system_relocated = ( - relocate_system_messages_to_top_level(body["messages"], body.get("system")) + relocate_system_messages_to_top_level( + body["messages"], + body.get("system"), + ( + str(model) + if ( + not upstream_base_url + or getattr(self, "anthropic_backend", None) is not None + or _is_googleapis_endpoint(upstream_base_url) + ) + else None + ), + ) ) if system_relocated: body["messages"] = relocated_messages @@ -3131,7 +3334,12 @@ class AnthropicHandlerMixin: # Track metrics total_latency = (time.time() - start_time) * 1000 usage = backend_response.body.get("usage", {}) - output_tokens = usage.get("output_tokens", 0) + # A backend may report these counters as JSON null (key + # present, value null), for which ``.get(key, 0)`` returns + # ``None`` rather than the default. Coerce with ``or 0`` so + # the arithmetic below (and ``RequestOutcome``) never sees + # ``None`` — matching the direct-Anthropic path. + output_tokens = int(usage.get("output_tokens", 0) or 0) _backend_name = request_backend.name if request_backend else "anthropic" # Eligible-only denominator for the active @@ -3150,8 +3358,8 @@ class AnthropicHandlerMixin: except Exception: attempted_input_tokens = original_tokens - cr_tokens = usage.get("cache_read_input_tokens", 0) - cw_tokens = usage.get("cache_creation_input_tokens", 0) + cr_tokens = int(usage.get("cache_read_input_tokens", 0) or 0) + cw_tokens = int(usage.get("cache_creation_input_tokens", 0) or 0) cw_5m_tokens, cw_1h_tokens = self._extract_anthropic_cache_ttl_metrics( usage ) @@ -3311,13 +3519,40 @@ class AnthropicHandlerMixin: body=body, original_body_bytes=original_body_bytes, ) - wants_buffered_stream_ccr = bool( + # ``headroom_retrieve`` stays resident for the session lifetime so + # the tools array is byte-stable and the prompt cache survives, so + # its mere presence is a poor reason to buffer. Once a session went + # sticky, *every* later streaming turn took the buffered path, and + # buffering replaces incremental delivery with one write at the + # end: time-to-last-byte is roughly unchanged, but time-to-first- + # byte becomes the entire generation. In the traffic reported in + # #3071 that was 8s on average and up to 100s, on 234 requests in + # a single day. + # + # The tool can only expand a marker that is in the outgoing body + # and redeemable now, so a turn carrying none cannot benefit from + # server-side retrieval and should keep streaming. This reads + # ``body`` rather than the earlier scan of ``optimized_messages`` + # because memory hooks, pre-send extensions and the CCR/tool-search + # repairs can all replace the message list after that scan; the + # only list that matters is the one about to go on the wire. + retrieve_tool_is_offered = ( stream and ccr_response_handler_enabled and self._has_headroom_retrieve_tool( tools if tools is not None else body.get("tools") ) ) + buffered_retrieval_can_help = ( + retrieve_tool_is_offered and self._outgoing_body_has_redeemable_marker(body) + ) + wants_buffered_stream_ccr = bool(buffered_retrieval_can_help) + if retrieve_tool_is_offered and not buffered_retrieval_can_help: + logger.info( + f"[{request_id}] CCR: headroom_retrieve is resident but this " + "request carries no redeemable marker, so server-side " + "retrieval cannot fire; keeping the streaming path (#3071)" + ) buffered_stream_ccr = ( wants_buffered_stream_ccr and not outbound_locked_to_client_bytes ) @@ -3333,6 +3568,9 @@ class AnthropicHandlerMixin: body_mutation_tracker.mark_mutated( "ccr_streaming_retrieve_buffered_non_stream" ) + # The ``Accept`` rewrite this flip used to do lives at the + # buffered boundary below, which every non-streaming + # request reaches — this one and the client's own (#3130). logger.info( f"[{request_id}] CCR: stream:true request has " "headroom_retrieve available; using buffered stream:false " @@ -3371,6 +3609,60 @@ class AnthropicHandlerMixin: tools = _ttl_tools body_mutation_tracker.mark_mutated("cache_control_ttl_order") + # Signed thinking locks the request to the client's original + # bytes. Once all mutation sites have run, make every downstream + # observer use that same wire body and neutralize savings from + # edits that will not be sent (#2990). This covers PERF, /stats, + # durable savings, response headers, pipeline events, and the + # prefix tracker rather than fixing only one reporting surface. + # + # RECOMPUTE rather than reusing the probe from before the CCR + # branch. That probe answers "will my stream flip survive?" and + # has to be asked early; this asks "what are we actually about to + # bill for?" and must be asked last. The two diverge now that the + # predicate tests thinking-block CONTENT and not merely presence: + # `enforce_cache_control_ttl_order` immediately above rewrites + # `body["messages"]`, so a marker moved onto or off a message + # holding a thinking block changes the answer after the early + # probe was taken. Evaluating the same predicate on the same + # final `body` that `select_outbound_body` will see is what keeps + # the accounting and the wire in agreement. + final_locked_to_client_bytes = outbound_body_is_client_bytes( + body=body, + original_body_bytes=original_body_bytes, + ) + if final_locked_to_client_bytes and body_mutation_tracker.mutated: + discarded_reasons = body_mutation_tracker.reasons + try: + wire_body = json.loads(original_body_bytes or b"") + except (json.JSONDecodeError, UnicodeDecodeError, ValueError, RecursionError): + wire_body = None + if not isinstance(wire_body, dict): + raise ValueError( + "signed-thinking passthrough could not reconstruct its client wire body" + ) + + from headroom.proxy.savings_attribution import SAVINGS_ATTRIBUTION_TAG + from headroom.proxy.tool_schema_savings_policy import ( + TOOL_SCHEMA_SAVINGS_TAGS, + ) + + attribution = tags.get(SAVINGS_ATTRIBUTION_TAG) + if isinstance(attribution, list): + attribution.clear() + for savings_tag in TOOL_SCHEMA_SAVINGS_TAGS: + tags.pop(savings_tag, None) + tags.pop("tool_search_deferred_tools", None) + tags["wire_mutations_discarded"] = len(discarded_reasons) + tags["wire_mutation_reasons"] = ",".join(discarded_reasons) + + body = wire_body + optimized_messages = body.get("messages", []) + tools = body.get("tools") + optimized_tokens = original_tokens + tokens_saved = 0 + transforms_applied = [] + log_cache_breakpoints( request_id=request_id, inbound=inbound_breakpoints, @@ -3437,6 +3729,32 @@ class AnthropicHandlerMixin: session_key=session_key, ) else: + # Whatever set it — the client's own ``stream: false`` or + # the CCR flip above — this branch sends a non-streaming + # request, so the client's ``Accept: text/event-stream`` no + # longer describes what is being asked for. Forwarding it + # unchanged puts a self-contradicting request on the wire: + # "answer as JSON" in the body, "I only accept SSE" in the + # headers. + # + # Anthropic tolerates the contradiction; stricter + # Anthropic-compatible gateways do not — GitHub Copilot's + # answers a generic ``api_error`` (#3078). On a + # client-originated non-stream turn — Claude Code's retry + # after a failed stream — an SSE answer to a JSON request + # is the empty/malformed HTTP 200 of #3130. + # + # Mutated in place: ``headers`` is captured by the + # closures defined below, and rebinding it here would + # leave them holding the old mapping. + if body.get("stream", False) is False: + for _accept_key in [k for k in headers if k.lower() == "accept"]: + headers.pop(_accept_key, None) + headers["accept"] = "application/json" + + # Populated once the upstream answers 200 with parseable + # JSON, so the guard below can fall back to it (#3088). + _salvageable_upstream: dict[str, Any] = {} async def _buffered_ccr_operation(): async with stage_timer.measure("upstream_connect"): @@ -3604,10 +3922,43 @@ class AnthropicHandlerMixin: f"[{request_id}] Failed to write debug dump: {dump_err}" ) + # A non-streaming request answered with an event + # stream (#3130). The turn is complete and already + # paid for — it is just wearing the wrong wire + # format — so adapt it to the JSON this caller asked + # for *here*, ahead of everything that reads the + # body: CCR retrieval, memory, turn hooks, usage and + # cost accounting, prefix tracking, the response + # cache, marker resolution and the security scan. + # Adapting at the final return instead would leave + # every one of those looking at an unparseable body. + # + # ``stream`` is what the *client* asked for, not what + # went upstream: a buffered CCR turn deliberately + # requests JSON on behalf of a streaming client and + # re-emits SSE further down, and must keep doing so. + if should_recover_sse_reply( + client_requested_stream=bool(stream), + status_code=response.status_code, + content_type=response.headers.get("content-type"), + body_is_event_stream=_looks_like_sse_response(response), + ): + response = self._adapt_event_stream_to_json(response, request_id) + # Parse response for CCR handling resp_json = None try: resp_json = response.json() + if buffered_stream_ccr and response.status_code == 200 and resp_json: + # Remember the upstream's own answer before any + # post-processing touches it. Everything from here + # to the SSE resynthesis — retrieval, memory tool + # calls, turn hooks, usage accounting, caching — is + # work layered on top of a turn the provider has + # already produced and billed. If any of it raises + # unexpectedly, this is what the client should get + # instead of a synthesized error (#3088). + _salvageable_upstream["resp_json"] = resp_json except (json.JSONDecodeError, ValueError) as e: # DEBUG is right for the buffered non-stream path, where # an unparseable body is just "no CCR handling". On the @@ -3684,10 +4035,16 @@ class AnthropicHandlerMixin: body_mutated=True, ) ) + # A continuation is a non-streaming call, so it + # needs a matching Accept for the same reason the + # buffered flip above does (#3078). ccr_outbound_headers = { - **continuation_headers, - "content-type": "application/json", + k: v + for k, v in continuation_headers.items() + if k.lower() not in ("accept", "content-type") } + ccr_outbound_headers["content-type"] = "application/json" + ccr_outbound_headers["accept"] = "application/json" log_outbound_request( forwarder="anthropic_ccr_continuation", method="POST", @@ -3991,7 +4348,24 @@ class AnthropicHandlerMixin: # the key: the cache key has no ``stream`` component, so # a buffered request would be answered with a stream it # cannot read (#2952). - if self.cache and response.status_code == 200 and resp_json is not None: + # + # ``not stream`` mirrors the read gate at the cache + # lookup above. ``stream`` still holds the *client's* + # original flag here — the buffered-CCR conversion + # flips ``body["stream"]``, never this variable — so a + # turn the client asked to stream is the one case that + # can reach this store site with a buffered body. That + # body was shaped by a forced ``stream: false`` flip + # plus CCR tool injection, and the key cannot tell it + # apart from an ordinary non-streaming reply, so + # storing it lets a later caller be answered with a + # response built for a request it never made (#3019). + if ( + self.cache + and not stream + and response.status_code == 200 + and resp_json is not None + ): await self.cache.set( cache_lookup_messages, model, @@ -4083,12 +4457,23 @@ class AnthropicHandlerMixin: ) ) - # Remove compression headers since httpx already decompressed the response - response_headers = dict(response.headers) - response_headers.pop("content-encoding", None) - response_headers.pop( - "content-length", None - ) # Length changed after decompression + # Framing headers describe how the *upstream* framed + # its body, not what this response is: httpx already + # decompressed it, Starlette recomputes the length, + # and uvicorn owns the connection. Replaying a stale + # ``transfer-encoding: chunked`` over a fixed-length + # body is what made an HTTP 200 read as empty in + # #3019. ``cf-*`` is CDN provenance the caller has no + # use for, and the header set clients cite as + # evidence of an intermediary mangling a reply + # (#3130). + response_headers = { + k: v + for k, v in sanitize_forwarded_response_headers( + response.headers + ).items() + if not k.lower().startswith("cf-") + } # Inject Headroom compression metrics (for SaaS metering) response_headers["x-headroom-tokens-before"] = str(original_tokens) @@ -4306,59 +4691,81 @@ class AnthropicHandlerMixin: headers=response_headers, ) + async def _buffered_ccr_operation_salvaging(): + """Never trade a successful upstream turn for a synthesized error. + + Everything the buffered path does after the provider answers + — server-side retrieval, memory tool calls, turn hooks, usage + accounting, caching, SSE resynthesis — is post-processing on + a turn that already succeeded and was already billed. When a + step raised unexpectedly the whole turn surfaced as a generic + ``api_error``, so the client lost a complete 69KB answer the + provider had produced (#3088), and the cause was unlogged. + + Relay the upstream's own answer instead. Two things are + deliberately preserved: the exception is logged with a + traceback so the real defect stays diagnosable rather than + being papered over, and a response the client cannot safely + consume is never salvaged — see ``_can_salvage``. + """ + try: + return await _buffered_ccr_operation() + except asyncio.CancelledError: + raise + except Exception: + salvaged = _salvageable_upstream.get("resp_json") + if salvaged is None or not self._can_salvage_buffered_upstream(salvaged): + raise + logger.error( + f"[{request_id}] CCR: buffered post-processing failed after a " + "successful upstream turn; relaying the upstream response " + "instead of failing the request (#3088)", + exc_info=True, + ) + try: + events = self._response_to_sse(salvaged, "anthropic") + except Exception: + logger.error( + f"[{request_id}] CCR: could not resynthesize the salvaged " + "upstream response; failing the request", + exc_info=True, + ) + raise + + async def _salvaged_sse(): + for event in events: + yield event + + return StreamingResponse( + _salvaged_sse(), + media_type="text/event-stream", + ) + if buffered_stream_ccr: - operation = asyncio.create_task(_buffered_ccr_operation()) - record_failed = self.metrics.record_failed + operation = asyncio.create_task(_buffered_ccr_operation_salvaging()) + + # Holds out for the real status, then keeps the stream alive + # once waiting silently would risk the client's idle + # watchdog. Both halves live in one shared place so the + # OpenAI twin cannot drift from it (#3079). + _buffered_call = buffered_ccr_asgi_call( + operation=operation, + fmt=ANTHROPIC_ERROR_FORMAT, + grace_seconds=getattr( + self.config, + "buffered_ccr_grace_seconds", + DEFAULT_BUFFERED_CCR_GRACE_SECONDS, + ), + record_failed=self.metrics.record_failed, + request_id=request_id, + ) class _BufferedCCRResponse(Response): async def __call__(self, scope, receive, send): # noqa: ANN001 - # Send nothing until the buffered operation resolves. - # The previous keepalive preamble committed - # `200 text/event-stream` after 1s, i.e. before the - # outcome was known: any upstream reply that then - # failed to become SSE (non-200, unparseable body) - # reached the client as a 200 whose body carried no - # `message_start`, which Claude Code reports as "API - # returned an empty or malformed response (HTTP 200) — - # check for a proxy or gateway intercepting the - # request". The real status was lost with it, so - # client-side 429/5xx backoff never fired. Clients - # budget minutes for a turn (Claude Code sends - # `x-stainless-timeout: 600`), so waiting is free. - try: - result = await operation - except Exception as e: - await record_failed(provider=provider_name) - logger.error( - f"[{request_id}] Request failed: {type(e).__name__}: {e}" - ) - await send( - { - "type": "http.response.start", - "status": 502, - "headers": [(b"content-type", b"application/json")], - } - ) - await send( - { - "type": "http.response.body", - "body": json.dumps( - { - "type": "error", - "error": { - "type": "api_error", - "message": "An error occurred while processing your request. Please try again.", - }, - } - ).encode(), - "more_body": False, - } - ) - return - await result(scope, receive, send) + await _buffered_call(scope, receive, send) return _BufferedCCRResponse(media_type="text/event-stream") - return await _buffered_ccr_operation() + return await _buffered_ccr_operation_salvaging() except HTTPException: # FastAPI HTTPException carries its own status code, headers, # and client-facing message (e.g. 429 with Retry-After, 413 for @@ -4492,7 +4899,13 @@ class AnthropicHandlerMixin: _pre_strip_count = sum(1 for k in headers if k.lower().startswith("x-headroom-")) headers = _strip_internal_headers(headers) - headers = merge_extra_headers(headers, self.config.anthropic_extra_headers) + # Always the configured Anthropic target; no per-request override. + headers = merge_extra_headers( + headers, + self.config.anthropic_extra_headers, + upstream_url=None, + config=self.config, + ) log_outbound_headers( forwarder="anthropic_batch", stripped_count=_pre_strip_count, @@ -4506,6 +4919,8 @@ class AnthropicHandlerMixin: compressed_requests = [] pipeline_timing: dict[str, float] = {} + from headroom.transforms.cold_prefix import anthropic_cache_ttl_seconds + # Apply compression to each request in the batch for batch_req in requests_list: custom_id = batch_req.get("custom_id", "") @@ -4515,6 +4930,9 @@ class AnthropicHandlerMixin: messages = params.get("messages", []) original_messages = copy.deepcopy(messages) model = params.get("model", "unknown") + cache_ttl_seconds = anthropic_cache_ttl_seconds( + model, original_messages, params.get("system") + ) if not messages or not self.config.optimize: # No messages or optimization disabled - pass through unchanged @@ -4551,7 +4969,7 @@ class AnthropicHandlerMixin: # blocks every other request for the duration; a timeout # here is caught below and passes the item through. result = await self._run_compression_in_executor( - lambda messages=messages, model=model, context_limit=context_limit, frozen_message_count=frozen_message_count: ( + lambda messages=messages, model=model, context_limit=context_limit, frozen_message_count=frozen_message_count, cache_ttl_seconds=cache_ttl_seconds: ( self.anthropic_pipeline.apply( messages=messages, model=model, @@ -4559,6 +4977,7 @@ class AnthropicHandlerMixin: context=extract_user_query(messages), frozen_message_count=frozen_message_count, request_id=request_id, + cache_ttl_seconds=cache_ttl_seconds, **proxy_pipeline_kwargs(self.config), ) ), @@ -4776,7 +5195,13 @@ class AnthropicHandlerMixin: _pre_strip_count = sum(1 for k in headers if k.lower().startswith("x-headroom-")) headers = _strip_internal_headers(headers) - headers = merge_extra_headers(headers, self.config.anthropic_extra_headers) + # Always the configured Anthropic target; no per-request override. + headers = merge_extra_headers( + headers, + self.config.anthropic_extra_headers, + upstream_url=None, + config=self.config, + ) log_outbound_headers( forwarder="anthropic_batch_passthrough", stripped_count=_pre_strip_count, @@ -4912,7 +5337,13 @@ class AnthropicHandlerMixin: _pre_strip_count = sum(1 for k in headers if k.lower().startswith("x-headroom-")) headers = _strip_internal_headers(headers) - headers = merge_extra_headers(headers, self.config.anthropic_extra_headers) + # Always the configured Anthropic target; no per-request override. + headers = merge_extra_headers( + headers, + self.config.anthropic_extra_headers, + upstream_url=None, + config=self.config, + ) log_outbound_headers( forwarder="anthropic_batch_results", stripped_count=_pre_strip_count, diff --git a/headroom/proxy/handlers/gemini.py b/headroom/proxy/handlers/gemini.py index 3311fe69c..79463c579 100644 --- a/headroom/proxy/handlers/gemini.py +++ b/headroom/proxy/handlers/gemini.py @@ -8,7 +8,6 @@ from __future__ import annotations import asyncio import json import logging -import os import time from typing import TYPE_CHECKING, Any @@ -21,6 +20,7 @@ from headroom.copilot_auth import build_copilot_upstream_url from headroom.proxy.auth_mode import classify_client from headroom.proxy.compression_decision import CompressionDecision from headroom.proxy.helpers import COMPRESSION_TIMEOUT_SECONDS, extract_tags +from headroom.proxy.identity import resolve_memory_identity from headroom.proxy.outcome import RequestOutcome from headroom.proxy.token_counting import gemini_output_tokens @@ -316,6 +316,13 @@ class GeminiHandlerMixin: headers.pop("host", None) headers.pop("content-length", None) tags = extract_tags(headers) + # Anthropic and OpenAI bind here; Gemini did not, so anything an ASGI + # extension recorded into the request scope was dropped on the floor + # for Gemini traffic only — silently, because an empty ledger and an + # unbound one look identical at the outcome funnel. + from headroom.proxy.savings_attribution import bind_scope + + bind_scope(tags, request.scope) client = classify_client(headers) # PR-A5 (P5-49): strip internal x-headroom-* from upstream-bound # headers AFTER `_extract_tags` reads them. Memory user-id reads @@ -335,10 +342,7 @@ class GeminiHandlerMixin: memory_user_id: str | None = None memory_request_ctx = None if self.memory_handler: - memory_user_id = request.headers.get( - "x-headroom-user-id", - os.environ.get("USER", os.environ.get("USERNAME", "default")), - ) + memory_user_id = resolve_memory_identity(request) # Per-project memory routing (GH #462). Gemini's # ``systemInstruction`` field carries the system prompt; # ``extract_system_prompt`` doesn't know that shape, so we @@ -869,9 +873,22 @@ class GeminiHandlerMixin: resp_json = final_resp_json response_content = json.dumps(resp_json).encode() usage = resp_json.get("usageMetadata", {}) - total_input_tokens = usage.get("promptTokenCount", total_input_tokens) - output_tokens = usage.get("candidatesTokenCount", output_tokens) - cache_read_tokens = usage.get("cachedContentTokenCount", cache_read_tokens) + # A CCR continuation response can carry a present-null count + # (e.g. a safety-blocked continuation turn), where + # ``.get(key, prior)`` returns None rather than the prior + # value, and the ``max(0, prompt - cache_read)`` / + # ``total_input_tokens > 0`` arithmetic below would then raise + # TypeError and the outer handler would mask a successful 200 + # as a synthetic 502. Guard with ``_usage_int`` (keeping the + # pre-continuation count as the fallback), mirroring the two + # sibling extraction sites above. + total_input_tokens = _usage_int( + usage.get("promptTokenCount"), total_input_tokens + ) + output_tokens = _usage_int(usage.get("candidatesTokenCount"), output_tokens) + cache_read_tokens = _usage_int( + usage.get("cachedContentTokenCount"), cache_read_tokens + ) uncached_input_tokens = max(0, total_input_tokens - cache_read_tokens) @@ -1027,6 +1044,9 @@ class GeminiHandlerMixin: headers.pop("content-length", None) headers.pop("accept-encoding", None) tags = extract_tags(headers) + from headroom.proxy.savings_attribution import bind_scope + + bind_scope(tags, request.scope) # Note: streaming handlers delegate to _stream_response, which # does its own classify_client. No need to compute here. is_antigravity = self._is_cloudcode_antigravity_request(body, headers) @@ -1180,6 +1200,9 @@ class GeminiHandlerMixin: headers.pop("host", None) headers.pop("content-length", None) tags = extract_tags(headers) + from headroom.proxy.savings_attribution import bind_scope + + bind_scope(tags, request.scope) # Streaming variant — delegates to _stream_response which # classifies the client itself from headers. # PR-A5 (P5-49): strip internal x-headroom-* before forwarding upstream. @@ -1328,6 +1351,9 @@ class GeminiHandlerMixin: # outcome. Extract here so apply_to_tags below has a dict to # mutate and the outcome at end-of-call inherits the tag. tags = extract_tags(request.headers) + from headroom.proxy.savings_attribution import bind_scope + + bind_scope(tags, request.scope) _decision = CompressionDecision.decide( headers=request.headers, config=self.config, diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py index c011b4984..a6823a075 100644 --- a/headroom/proxy/handlers/openai.py +++ b/headroom/proxy/handlers/openai.py @@ -28,9 +28,12 @@ from headroom.proxy.helpers import ( _headroom_bypass_enabled, extract_tags, jitter_delay_ms, + sanitize_forwarded_response_headers, ) +from headroom.proxy.identity import resolve_memory_identity from headroom.proxy.loopback_guard import is_loopback_host from headroom.proxy.stage_timer import StageTimer, emit_stage_timings_log +from headroom.proxy.upstream_guard import is_safe_upstream_url from headroom.proxy.ws_headers import WS_HOP_BY_HOP_HEADERS from headroom.proxy.ws_session_registry import ( TerminationCause, @@ -70,6 +73,11 @@ from headroom.proxy.auth_mode import ( classify_client, should_stamp_codex_client, ) +from headroom.proxy.buffered_ccr_response import ( + DEFAULT_BUFFERED_CCR_GRACE_SECONDS, + OPENAI_ERROR_FORMAT, + buffered_ccr_asgi_call, +) from headroom.proxy.compression_decision import CompressionDecision from headroom.proxy.cost import header_safe_transforms from headroom.proxy.handlers._debug_dump import _debug_dump_mode, _redact_debug_value @@ -327,10 +335,10 @@ def _sanitize_forwarded_response_headers( headers: httpx.Headers | dict[str, str], *extra_names: str, ) -> dict[str, str]: - cleaned = dict(headers) - for name in ("content-encoding", "content-length", "server", *extra_names): - cleaned.pop(name, None) - return cleaned + # Thin alias kept for the many call sites in this module; the policy + # (and the list of framing headers) lives in one place so the Anthropic + # handler strips exactly the same set — see #3019. + return sanitize_forwarded_response_headers(headers, *extra_names) def _resolve_openai_handler_path( @@ -376,6 +384,13 @@ def _resolve_openai_upstream_base(request_headers: dict[str, str]) -> str | None if urlparse(normalized).scheme not in {"http", "https"}: return None + if not is_safe_upstream_url(normalized): + # Client-supplied upstream resolves to a private/loopback/link-local or + # cloud-metadata address (SSRF). Ignore the override and fall back to the + # configured upstream; set HEADROOM_ALLOWED_BASE_URLS to permit specific + # internal endpoints. + logger.warning("ignoring unsafe x-headroom-base-url override: %r", raw_base_url) + return None return normalized @@ -3119,7 +3134,14 @@ class OpenAIHandlerMixin: _pre_strip_count_chat = sum(1 for k in headers if k.lower().startswith("x-headroom-")) headers = _strip_internal_headers(headers) - headers = merge_extra_headers(headers, self.config.openai_extra_headers) + # `custom_upstream_base_url` is the per-request `x-headroom-base-url` + # override resolved above. Secrets only go to designated hosts. + headers = merge_extra_headers( + headers, + self.config.openai_extra_headers, + upstream_url=custom_upstream_base_url, + config=self.config, + ) log_outbound_headers( forwarder="openai_chat_completions", stripped_count=_pre_strip_count_chat, @@ -3147,10 +3169,7 @@ class OpenAIHandlerMixin: memory_user_id: str | None = None memory_request_ctx = None if self.memory_handler: - memory_user_id = request.headers.get( - "x-headroom-user-id", - os.environ.get("USER", os.environ.get("USERNAME", "default")), - ) + memory_user_id = resolve_memory_identity(request) # Per-project memory routing (GH #462). Built once per request # so every save/search/inject resolves to the same workspace. from headroom.memory.storage_router import ( @@ -3273,10 +3292,34 @@ class OpenAIHandlerMixin: ) ) - # Remove compression headers from cached response - response_headers = _sanitize_forwarded_response_headers(cached.response_headers) + # Strip the stored response's wire-framing headers, and its + # content-type: the entry carries whatever the *producing* + # upstream sent, and replaying that framing over a different + # connection breaks the body — a stale + # ``transfer-encoding: chunked`` makes the client parse plain + # JSON as chunked frames and read nothing out of an HTTP 200 + # (#3019, same reasoning as #2952 on the Anthropic twin). + response_headers = _sanitize_forwarded_response_headers( + cached.response_headers, + "content-type", + ) - return Response(content=cached.response_body, headers=response_headers) + # A cache hit answers without touching the upstream, so it + # emits no outbound_request line and no upstream stage + # timings. Log it, or a served-from-cache turn looks exactly + # like a turn that died silently (#3019). + logger.info( + f"[{request_id}] RESPONSE-CACHE-HIT: model={model} " + f"bytes={len(cached.response_body)} " + f"age_s={(datetime.now() - cached.created_at).total_seconds():.0f} " + f"hits={cached.hit_count}" + ) + + return Response( + content=cached.response_body, + headers=response_headers, + media_type="application/json", + ) # Token counting (offloaded off the event loop — GH #1701) tokenizer, original_tokens = await self._count_tokens_offloaded(model, messages) @@ -4810,7 +4853,16 @@ class OpenAIHandlerMixin: # Cache response under the SAME key it was looked up by: # cache_lookup_messages is the raw pre-mutation snapshot, not # the live (hooked) `messages` (#327). - if self.cache and response.status_code == 200: + # + # ``not stream`` mirrors the read gate at the cache lookup + # above. It is currently redundant here — a streaming chat + # request returns via ``_stream_response`` well before this + # point — but the Anthropic handler had the same shape until a + # buffered-CCR branch started falling through to its store + # site, which let a response built for a stream:true request + # answer a later non-streaming caller (#3019). Stating the + # invariant keeps that from being reintroduced silently. + if self.cache and not stream and response.status_code == 200: await self.cache.set( cache_lookup_messages, model, @@ -5078,7 +5130,15 @@ class OpenAIHandlerMixin: _pre_strip_count_resp = sum(1 for k in headers if k.lower().startswith("x-headroom-")) headers = _strip_internal_headers(headers) - headers = merge_extra_headers(headers, self.config.openai_extra_headers) + # This handler also honors `x-headroom-base-url` (resolved further + # below); resolve it here too so the secret headers are gated on the + # real destination rather than merged before it is known. + headers = merge_extra_headers( + headers, + self.config.openai_extra_headers, + upstream_url=_resolve_openai_upstream_base(request.headers), + config=self.config, + ) # Mirror the WS handler: never forward Codex's client-only lite header # upstream. OpenAI rejects newer Codex models when it leaks, and the HTTP # POST path (unlike the WS path) otherwise forwards request headers verbatim. @@ -5143,10 +5203,7 @@ class OpenAIHandlerMixin: memory_user_id: str | None = None memory_request_ctx = None if self.memory_handler: - memory_user_id = request.headers.get( - "x-headroom-user-id", - os.environ.get("USER", os.environ.get("USERNAME", "default")), - ) + memory_user_id = resolve_memory_identity(request) from headroom.memory.storage_router import ( RequestContext as _MemRequestContext, ) @@ -5577,6 +5634,12 @@ class OpenAIHandlerMixin: if body.get("stream") is not False: body["stream"] = False body_mutation_tracker.mark_mutated("ccr_streaming_retrieve_buffered_non_stream") + # Same contradiction as the Anthropic path: the body now asks for a + # non-streaming reply while the client's Accept still says SSE. This + # handler serves GitHub Copilot (see apply_copilot_api_auth below), + # whose gateway is one of the strict ones (#3078). + _accept_key = next((k for k in headers if k.lower() == "accept"), "accept") + headers[_accept_key] = "application/json" logger.info( f"[{request_id}] CCR: stream:true /v1/responses request has " "headroom_retrieve available; using buffered stream:false " @@ -6173,48 +6236,24 @@ class OpenAIHandlerMixin: if buffered_stream_ccr: operation = asyncio.create_task(_buffered_ccr_operation()) - record_failed = self.metrics.record_failed + + # Same wrapper as the Anthropic twin; only the error wire format + # differs. See headroom/proxy/buffered_ccr_response.py (#3079). + _buffered_call = buffered_ccr_asgi_call( + operation=operation, + fmt=OPENAI_ERROR_FORMAT, + grace_seconds=getattr( + self.config, + "buffered_ccr_grace_seconds", + DEFAULT_BUFFERED_CCR_GRACE_SECONDS, + ), + record_failed=self.metrics.record_failed, + request_id=request_id, + ) class _BufferedCCRResponse(Response): async def __call__(self, scope, receive, send): # noqa: ANN001 - # Send nothing until the buffered operation resolves — - # see the AnthropicHandler twin for the full rationale. - # Committing `200 text/event-stream` on a keepalive timer, - # before the outcome is known, turns every non-200 or - # unparseable upstream reply into a 200 with no usable - # body and discards the status the client needs to back - # off on. - try: - result = await operation - except Exception as e: - await record_failed(provider="openai") - logger.error( - f"[{request_id}] OpenAI responses request failed: {type(e).__name__}: {e}" - ) - await send( - { - "type": "http.response.start", - "status": 502, - "headers": [(b"content-type", b"application/json")], - } - ) - await send( - { - "type": "http.response.body", - "body": json.dumps( - { - "error": { - "message": "An error occurred while processing your request. Please try again.", - "type": "server_error", - "code": "proxy_error", - } - } - ).encode(), - "more_body": False, - } - ) - return - await result(scope, receive, send) + await _buffered_call(scope, receive, send) return _BufferedCCRResponse(media_type="text/event-stream") return await _buffered_ccr_operation() @@ -6582,8 +6621,13 @@ class OpenAIHandlerMixin: upstream: Any = None from headroom.proxy.helpers import merge_extra_headers + # The WS upstream is derived from config (chatgpt.com backend or + # OPENAI_API_URL), never from a request header. upstream_headers = merge_extra_headers( - upstream_headers, self.config.openai_extra_headers + upstream_headers, + self.config.openai_extra_headers, + upstream_url=None, + config=self.config, ) for ws_attempt in range(ws_connect_attempts): @@ -6938,12 +6982,7 @@ class OpenAIHandlerMixin: nonlocal memory_user_id, memory_request_ctx memory_user_id_candidate = ( - ws_headers.get( - "x-headroom-user-id", - os.environ.get("USER", os.environ.get("USERNAME", "default")), - ) - if self.memory_handler - else None + resolve_memory_identity(websocket) if self.memory_handler else None ) memory_decision = MemoryDecision.decide( headers=ws_headers, @@ -7160,7 +7199,6 @@ class OpenAIHandlerMixin: if isinstance(first_response_body, dict) else None ) - # Hot-fix follow-up to PR #406 — inline Rust compression on the # WS first frame before forwarding upstream. PR #406 enabled # the same call for HTTP /v1/responses; PR-C5's "WS-side @@ -8000,6 +8038,7 @@ class OpenAIHandlerMixin: response_output_items.clear() response_started_ms: float | None = None + completed_response_model = "unknown" async def _record_ws_response_metrics() -> None: """Record one completed Responses turn on long-lived WS sessions.""" @@ -8056,7 +8095,7 @@ class OpenAIHandlerMixin: ): return - model_for_metrics = str(body.get("model") or "unknown") + model_for_metrics = completed_response_model latency_ms = ( (time.perf_counter() * 1000.0 - response_started_ms) if response_started_ms is not None @@ -8209,6 +8248,13 @@ class OpenAIHandlerMixin: upstream_frame_index, ws_last_upstream_frame_type, ) + response = event.get("response") + completed_response_model = ( + str(response.get("model") or "unknown") + if isinstance(response, dict) + else "unknown" + ) + if event_type == "response.created": response_started_ms = time.perf_counter() * 1000.0 ( @@ -8561,9 +8607,23 @@ class OpenAIHandlerMixin: f"[{request_id}] WS upstream failed ({_ws_detail}), " f"falling back to HTTP POST streaming" ) - await self._ws_http_fallback( + ( + fb_input_tokens, + fb_output_tokens, + fb_cache_read_tokens, + fb_cache_write_tokens, + fb_uncached_tokens, + ) = await self._ws_http_fallback( websocket, body, first_msg_raw, upstream_headers, request_id ) + # Fold the fallback's provider usage into the session totals so + # the WS session-end outcome records the authoritative wire-token + # count instead of 0 (#2957). + ws_input_tokens_total += fb_input_tokens + ws_output_tokens_total += fb_output_tokens + ws_cache_read_tokens_total += fb_cache_read_tokens + ws_cache_write_tokens_total += fb_cache_write_tokens + ws_uncached_input_tokens_total += fb_uncached_tokens # ── WS session-end metric + RequestLog ────────────────── # @@ -8581,11 +8641,7 @@ class OpenAIHandlerMixin: ) if not isinstance(ws_inner_for_telemetry, dict): ws_inner_for_telemetry = {} - model_name = ( - ws_inner_for_telemetry.get("model") - or (body.get("model") if isinstance(body, dict) else None) - or "unknown" - ) + model_name = str(current_response_template.get("model") or "unknown") _final_auth_mode = classify_auth_mode(ws_headers) residual_input_tokens = max(0, ws_input_tokens_total - ws_recorded_input_tokens_total) residual_output_tokens = max( @@ -8813,14 +8869,31 @@ class OpenAIHandlerMixin: first_msg_raw: str, upstream_headers: dict[str, str], request_id: str, - ) -> None: + ) -> tuple[int, int, int, int, int]: """Fall back to HTTP POST streaming when upstream WS fails. Converts the WS ``response.create`` message to an HTTP POST to ``/v1/responses?stream=true``, reads SSE events, and relays each ``data:`` line as a WS text message to the client. This makes Codex work immediately instead of exhausting its WS retry budget. + + Returns ``(input, output, cache_read, cache_write, uncached)`` provider + usage parsed from the ``response.completed`` SSE event. The caller folds + it into the session totals so the WS session-end outcome uses the + authoritative wire-token count; otherwise a fallback recorded + ``input_tokens=0`` and savings percentages blew past 100 (#2957). """ + fallback_usage = [0, 0, 0, 0, 0] + + def _accumulate_usage(data_str: str) -> None: + try: + event = json.loads(data_str) + except (json.JSONDecodeError, TypeError): + return + if isinstance(event, dict) and event.get("type") == "response.completed": + for i, value in enumerate(_extract_responses_usage(event)): + fallback_usage[i] += value + # Route to correct endpoint based on auth mode is_chatgpt_fallback = has_chatgpt_account_header(upstream_headers) if is_chatgpt_fallback: @@ -8926,7 +8999,7 @@ class OpenAIHandlerMixin: }, } await websocket.send_text(json.dumps(error_event)) - return + return tuple(fallback_usage) # type: ignore[return-value] # Refresh Codex /stats from the fallback response # headers. We can't forward them onto the client 101 @@ -8952,10 +9025,11 @@ class OpenAIHandlerMixin: data = line[6:] if data == "[DONE]": continue + _accumulate_usage(data) try: await websocket.send_text(data) except Exception: - return + return tuple(fallback_usage) # type: ignore[return-value] elif line.startswith("event: "): # SSE event type — skip, the data line contains the type continue @@ -8964,9 +9038,10 @@ class OpenAIHandlerMixin: for line in buffer.strip().splitlines(): line = line.strip() if line.startswith("data: ") and line[6:] != "[DONE]": + _accumulate_usage(line[6:]) with contextlib.suppress(Exception): await websocket.send_text(line[6:]) - return + return tuple(fallback_usage) # type: ignore[return-value] except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as http_err: if http_attempt >= retry_attempts - 1: raise @@ -8997,6 +9072,7 @@ class OpenAIHandlerMixin: finally: with contextlib.suppress(Exception): await websocket.close() + return tuple(fallback_usage) # type: ignore[return-value] def _derived_compress_pipeline(self, key: str, **overrides: Any) -> Any: """Cached ``/v1/compress`` pipeline derived from the live OpenAI router. diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.py index 07d4f70be..5f86d48a4 100644 --- a/headroom/proxy/handlers/streaming.py +++ b/headroom/proxy/handlers/streaming.py @@ -350,7 +350,13 @@ class StreamingMixin: return usage_found if usage_found else None - def _parse_sse_to_response(self, sse_data: str, provider: str) -> dict[str, Any] | None: + def _parse_sse_to_response( + self, + sse_data: str, + provider: str, + *, + require_complete: bool = False, + ) -> dict[str, Any] | None: """Parse SSE data to reconstruct the API response JSON. Args: @@ -358,6 +364,10 @@ class StreamingMixin: from a complete-events bytes buffer (see ``parse_sse_events_from_byte_buffer``). provider: Provider type for parsing. + require_complete: Reject anything short of a whole, replayable + message — see the strictness note below. Off by default so + streaming callers keep the lenient reconstruction they were + written against. Returns: Reconstructed response dict or None if parsing fails. @@ -366,11 +376,32 @@ class StreamingMixin: ``text_delta``, ``input_json_delta``, ``thinking_delta``, ``signature_delta``, ``citations_delta``. Also preserves ``redacted_thinking.data`` and accumulates citations as a list. + + Permissive mode answers with whatever blocks it managed to + accumulate. That is right for a streaming caller salvaging a + partial stream, and wrong for #3130, where the reconstruction is + handed to the client *as* the turn: a truncated stream would become + a successful — and silently short — message, and an ``error`` event + would vanish behind an HTTP 200. ``require_complete`` demands + ``message_start``, a terminal ``message_stop``, every opened block + closed, no ``error`` event, and no delta type this reconstructor + cannot replay; anything else returns None so the caller can fail + loudly instead. """ if provider != "anthropic": return None # Only implemented for Anthropic + # Event framing is CRLF in some intermediaries (and mixed after a + # retry through one). Normalize before the line split so a + # correctly framed stream is never read as zero events. + sse_data = sse_data.replace("\r\n", "\n").replace("\r", "\n") + response: dict[str, Any] = {"content": [], "usage": {}} + saw_message_start = False + saw_message_stop = False + saw_error = False + saw_unreplayable_delta = False + open_block_indices: set[int] = set() # Track blocks by their `index` field so out-of-order events # don't corrupt the reconstruction. The current block pointer # remains for backward-compat with code that walks this dict @@ -392,9 +423,9 @@ class StreamingMixin: appended_block_keys: set[int] = set() for line in sse_data.split("\n"): - if not line.startswith("data: "): + if not line.startswith("data:"): continue - data_str = line[6:].strip() + data_str = line[5:].strip() if not data_str or data_str == "[DONE]": continue @@ -406,8 +437,12 @@ class StreamingMixin: event_type = data.get("type", "") if event_type == "message_start": + saw_message_start = True msg = data.get("message", {}) response["id"] = msg.get("id") + response["type"] = msg.get("type", "message") + if "stop_sequence" in msg: + response["stop_sequence"] = msg["stop_sequence"] response["model"] = msg.get("model") response["role"] = msg.get("role", "assistant") response["stop_reason"] = msg.get("stop_reason") @@ -454,6 +489,7 @@ class StreamingMixin: if _k != "type": current_block[_k] = _v blocks_by_index[block_index] = current_block + open_block_indices.add(block_index) elif event_type == "content_block_delta": # Resolve the target block by index (preferred) or fall @@ -490,6 +526,10 @@ class StreamingMixin: citation = delta.get("citation") if citation is not None: citations.append(citation) + else: + # A delta this reconstructor has no rule for: the + # accumulated block is missing whatever it carried. + saw_unreplayable_delta = True elif event_type == "content_block_stop": idx = data.get("index") @@ -523,6 +563,8 @@ class StreamingMixin: if block_key not in appended_block_keys: response["content"].append(target) appended_block_keys.add(block_key) + if idx is not None: + open_block_indices.discard(idx) current_block = None elif event_type == "message_delta": @@ -531,9 +573,38 @@ class StreamingMixin: response["stop_reason"] = delta["stop_reason"] if "stop_details" in delta: response["stop_details"] = delta["stop_details"] + if "stop_sequence" in delta: + response["stop_sequence"] = delta["stop_sequence"] if data.get("usage"): response["usage"].update(data["usage"]) + elif event_type == "message_stop": + saw_message_stop = True + + elif event_type == "error": + # An in-band failure. Permissive callers keep salvaging + # what arrived before it; a strict caller must not dress + # the remains up as a successful turn. + saw_error = True + + if require_complete: + if ( + not saw_message_start + or not saw_message_stop + or saw_error + or saw_unreplayable_delta + or open_block_indices + ): + return None + # ``index`` is a response-delta field. Anthropic rejects it on + # the next request ("content.0.text.index: Extra inputs are not + # permitted"), so it must not survive into a body the client + # will persist and echo back. + for block in response["content"]: + if isinstance(block, dict): + block.pop("index", None) + return response + return response if response.get("content") else None def _response_to_sse(self, response: dict[str, Any], provider: str) -> list[bytes]: @@ -1121,18 +1192,20 @@ class StreamingMixin: # bytes once before entering the connection-retry loop. When a # transform mutated the body we re-serialize canonically; otherwise # we forward the original client bytes verbatim. - from headroom.proxy.body_forwarding import prepare_outbound_body_bytes + from headroom.proxy.body_forwarding import select_outbound_body from headroom.proxy.helpers import ( capture_codex_wire_debug, codex_wire_debug_enabled, log_outbound_request, ) - outbound_bytes, outbound_source = prepare_outbound_body_bytes( + outbound = select_outbound_body( body=body, original_body_bytes=original_body_bytes, body_mutated=body_mutated, + mutation_reasons=list(mutation_reasons or []), ) + outbound_bytes, outbound_source = outbound.content, outbound.source outbound_headers = {**headers, "content-type": "application/json"} log_outbound_request( forwarder="streaming", @@ -1143,6 +1216,7 @@ class StreamingMixin: mutation_reasons=list(mutation_reasons or []), request_id=request_id, source=outbound_source, + dropped_mutation_reasons=outbound.dropped_mutation_reasons, ) _codex_wire_debug = ( codex_wire_debug_enabled() and provider == "openai" and "/responses" in url @@ -1266,11 +1340,22 @@ class StreamingMixin: if upstream_response is None: raise last_connect_error or RuntimeError("upstream connection did not start") # Retries exhausted (or a transport failure escaped the loop): emit a - # clean SSE error instead of letting an h2 StreamReset bubble up as a - # 502. Covers ConnectError/timeouts and Local/RemoteProtocolError. (#1639) + # clean SSE error instead of letting an h2 StreamReset bubble up as an + # unhandled 502. Covers ConnectError/timeouts and Local/RemoteProtocol- + # Error. (#1639) + # + # The status MUST stay non-2xx. No body byte has been forwarded yet, so + # the status line is still ours to set, and a 200 here is + # indistinguishable — to every Anthropic/OpenAI SDK — from a successful + # stream that produced no events: the client reports "empty or malformed + # response (HTTP 200)" and cannot retry, because 200 is not retryable. + # 502 keeps the structured SSE body for clients that read it while + # letting SDK retry logic treat a transient upstream transport failure + # as what it is. except httpx.TransportError as e: error_msg = str(e) or repr(e) logger.error(f"[{request_id}] Connection error to upstream API: {error_msg}") + self.metrics.record_upstream_connection_error(provider) async def _error_gen(): error_event = { @@ -1283,7 +1368,11 @@ class StreamingMixin: yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode() self._cleanup_mid_turn_stream(session_key) - return StreamingResponse(_error_gen(), media_type="text/event-stream") + return StreamingResponse( + _error_gen(), + status_code=502, + media_type="text/event-stream", + ) # Capture Codex rate-limit window data from the upstream response # headers, for *every* status. Codex (gpt-5.x) almost always streams, so diff --git a/headroom/proxy/helpers.py b/headroom/proxy/helpers.py index d98b67be3..40efb6571 100644 --- a/headroom/proxy/helpers.py +++ b/headroom/proxy/helpers.py @@ -317,6 +317,40 @@ def _headroom_bypass_enabled(headers: Any) -> bool: return bypass or passthrough +# Response headers that describe how the *upstream* framed its body on the +# wire, not what the payload means. Every one of them is invalid to replay: +# Starlette recomputes content-length, and uvicorn owns the connection +# framing. Forwarding a stale ``transfer-encoding: chunked`` onto a +# fixed-length body is the worst of them — RFC 9112 §6.1 makes +# Transfer-Encoding override Content-Length, so the client tries to parse a +# plain JSON body as chunked frames, finds no valid chunk-size line, and +# reads an empty body out of an HTTP 200 (#3019). +FRAMING_RESPONSE_HEADERS: tuple[str, ...] = ( + "content-encoding", + "content-length", + "transfer-encoding", + "connection", + "keep-alive", + "server", +) + + +def sanitize_forwarded_response_headers( + headers: Any, + *extra_names: str, +) -> dict[str, str]: + """Drop wire-framing headers before replaying an upstream response. + + Pass any additional header names to strip as ``extra_names`` (for + example ``"content-type"`` when the caller sets its own media type). + + Matching is case-insensitive, but the casing of the headers that + survive is left untouched. + """ + drop = {name.lower() for name in (*FRAMING_RESPONSE_HEADERS, *extra_names)} + return {key: value for key, value in dict(headers).items() if key.lower() not in drop} + + def log_outbound_request( *, forwarder: str, @@ -871,16 +905,15 @@ def _system_message_to_blocks(message: dict[str, Any]) -> list[Any]: def relocate_system_messages_to_top_level( messages: list[dict[str, Any]], system: Any, + model: str | None = None, ) -> tuple[list[dict[str, Any]], Any, bool]: - """Move any ``role="system"`` entries out of ``messages`` into ``system``. + """Relocate only system messages invalid for the selected Anthropic model. - Anthropic's Messages API rejects a ``system`` role inside ``messages`` with - HTTP 400 ("messages.0: use the top-level 'system' parameter for the initial - system prompt"). Internal transforms / pipeline extensions can leave a stray - system message in the list (e.g. a relocated harness system block during - compression). This is the Anthropic forwarder's last line of defense: it - guarantees the forwarded body never violates the wire contract, regardless - of which transform introduced the entry. + Supported models accept mid-conversation system sections after a user turn + (or an assistant server-tool result) when followed by an assistant turn or + placed at the end. Hoisting those changes semantics and invalidates the + cached prefix. The initial/invalid forms are still moved to the top-level + field as the issue-765 last-line wire-contract guard. The relocated content is appended after any existing top-level ``system`` so wire order (system prompt, then conversation) is preserved and no content @@ -890,9 +923,58 @@ def relocate_system_messages_to_top_level( message is present the inputs pass through unchanged (``changed=False``) so the common path is untouched. """ - system_indices = { - i for i, m in enumerate(messages) if isinstance(m, dict) and m.get("role") == _ROLE_SYSTEM - } + model_id = str(model or "").lower() + supports_mid_conversation = any( + family in model_id + for family in ( + "claude-fable-5", + "claude-mythos-5", + "claude-opus-4-8", + "claude-opus-5", + "claude-sonnet-5", + ) + ) + + def _assistant_ends_in_server_tool_result(message: object) -> bool: + if not isinstance(message, dict) or message.get("role") != "assistant": + return False + content = message.get("content") + if not isinstance(content, list) or not content: + return False + final = content[-1] + if not isinstance(final, dict): + return False + block_type = str(final.get("type") or "") + return block_type == "server_tool_use" or block_type.endswith("_tool_result") + + system_indices: set[int] = set() + index = 0 + while index < len(messages): + message = messages[index] + if not isinstance(message, dict) or message.get("role") != _ROLE_SYSTEM: + index += 1 + continue + + section_start = index + while ( + index + 1 < len(messages) + and isinstance(messages[index + 1], dict) + and messages[index + 1].get("role") == _ROLE_SYSTEM + ): + index += 1 + section_end = index + + previous = messages[section_start - 1] if section_start > 0 else None + following = messages[section_end + 1] if section_end + 1 < len(messages) else None + valid_previous = ( + isinstance(previous, dict) and previous.get("role") == "user" + ) or _assistant_ends_in_server_tool_result(previous) + valid_following = following is None or ( + isinstance(following, dict) and following.get("role") == "assistant" + ) + if not (supports_mid_conversation and valid_previous and valid_following): + system_indices.update(range(section_start, section_end + 1)) + index += 1 if not system_indices: return messages, system, False @@ -1520,15 +1602,40 @@ def _strip_internal_headers(headers: dict[str, str]) -> dict[str, str]: return strip_internal_headers(headers, mode=get_strip_internal_headers_mode()) -def merge_extra_headers(headers: dict[str, str], extra: dict[str, str] | None) -> dict[str, str]: +def merge_extra_headers( + headers: dict[str, str], + extra: dict[str, str] | None, + *, + upstream_url: str | None, + config: Any = None, +) -> dict[str, str]: """Merge configured extra headers into ``headers``, overriding same-named keys. ``extra`` comes from ``ProxyConfig.anthropic_extra_headers``/``openai_extra_headers`` (settings-panel/CLI-configured, for gateways that need one extra header alongside the client's own auth). Returns ``headers`` unchanged (no copy) when nothing is configured. + + ``upstream_url`` is where these headers are about to be sent, and it is + **required** rather than optional on purpose. These values are secrets, and + several handlers accept a per-request upstream from the ``x-headroom-base-url`` + request header; merging before the destination was known is what let a client + redirect the operator's gateway key to a host of its choosing. Making the + destination part of the signature means a new forwarder cannot merge a secret + without saying where it goes, so this cannot silently regress. + + Pass ``None`` when the caller is going to its configured target with no + per-request override. Anything else is checked against + ``upstream_trust.is_trusted_upstream``; an undesignated host still gets its + request proxied, just without these headers. """ if not extra: return headers + if upstream_url is not None: + from headroom.proxy.upstream_trust import is_trusted_upstream, warn_untrusted_once + + if not is_trusted_upstream(upstream_url, config): + warn_untrusted_once(upstream_url) + return headers # HTTP header names are case-insensitive: drop any existing key that # case-insensitively collides with a configured extra so the extra wins. # A plain {**headers, **extra} would emit both casings upstream. @@ -2811,6 +2918,13 @@ _TOOL_SEARCH_DEFAULT_NAME = "tool_search_tool_regex" _TOOL_SEARCH_MIN_TOOLS = 12 +def _tool_search_resident_key(name: Any) -> str: + """Normalize a client tool name for resident-tool membership checks.""" + # Oh My Pi prefixes every built-in with ``_``. Strip only leading namespace + # markers so internal separators such as ``mcp__server__read`` stay intact. + return str(name or "").lower().lstrip("_") + + def anthropic_first_party_tool_search_supported(api_base_url: str | None) -> bool: """Return whether Anthropic server-side tool search is valid for this upstream.""" from headroom.providers.claude.runtime import is_custom_anthropic_base_url @@ -2872,17 +2986,17 @@ def inject_tool_search_deferral( last_resident_real: dict[str, Any] | None = None resident_has_cache_control = False - # Clients disagree on casing for the same tool: Claude Code sends ``Bash`` / - # ``ToolSearch`` where opencode sends ``bash``. Compare case-insensitively so - # the exemption applies to both — an exact match silently deferred *every* - # tool for PascalCase clients, including their own tool-search tool. - core_lower = {name.lower() for name in core_tools} + # Clients disagree on casing and leading namespace markers for the same tool: + # Claude Code sends ``Bash``, opencode sends ``bash``, and Oh My Pi sends + # ``_bash``. Normalize both the configured names and each candidate so the + # exemption applies consistently across clients. + core_keys = {_tool_search_resident_key(name) for name in core_tools} for tool in tools: if ( not isinstance(tool, dict) or tool.get("type") - or str(tool.get("name") or "").lower() in core_lower + or _tool_search_resident_key(tool.get("name")) in core_keys ): # Non-dict, server/typed tools (web_search, computer, …), and core # tools stay resident and unchanged. @@ -3259,10 +3373,10 @@ def inject_tool_search_deferral_openai( out: list[Any] = [{"type": _OPENAI_TOOL_SEARCH_TYPE}] deferred = 0 - # Case-insensitive for the same reason as the Anthropic path above: the - # resident-name sets are lowercase, clients are not required to be. - resident_lower = {name.lower() for name in core_tools} | { - name.lower() for name in _OPENAI_TOOL_SEARCH_RESIDENT_NAMES + # Normalize for the same reason as the Anthropic path above: clients may use + # different casing or a leading namespace marker for the same resident tool. + resident_keys = {_tool_search_resident_key(name) for name in core_tools} | { + _tool_search_resident_key(name) for name in _OPENAI_TOOL_SEARCH_RESIDENT_NAMES } for tool in tools: if not isinstance(tool, dict): @@ -3273,7 +3387,7 @@ def inject_tool_search_deferral_openai( # trained to search namespaces / MCP servers). Everything else — core # coding tools and other hosted tools — stays resident. deferrable = ( - ttype == "function" and str(tool.get("name") or "").lower() not in resident_lower + ttype == "function" and _tool_search_resident_key(tool.get("name")) not in resident_keys ) or ttype == "mcp" if deferrable and not tool.get("defer_loading"): new_tool = dict(tool) diff --git a/headroom/proxy/identity.py b/headroom/proxy/identity.py new file mode 100644 index 000000000..18b9bed42 --- /dev/null +++ b/headroom/proxy/identity.py @@ -0,0 +1,92 @@ +"""Memory partition identity resolution (WEB-02). + +``x-headroom-user-id`` is a partition *hint*, not an authenticated identity. In +the OSS proxy it is honored only for loopback callers (the single-user local +model). For other callers the identity is bound to the proxy token (or the +server's OS user) so a network client cannot select another user's memory. + +Multi-tenant deployments (e.g. headroom-managed) replace the default with an +authenticated resolver via :func:`set_identity_resolver`, typically from a +``headroom.proxy_extension`` install hook. This keeps real per-tenant identity — +the enterprise differentiator — out of the OSS proxy while giving it a clean, +secure single-user default. +""" + +from __future__ import annotations + +import hashlib +import os +from typing import Any, Protocol + +from headroom.proxy.loopback_guard import is_loopback_host + +USER_ID_HEADER = "x-headroom-user-id" + + +class IdentityResolver(Protocol): + def __call__(self, request: Any, *, default: str) -> str: ... + + +_resolver: IdentityResolver | None = None + + +def set_identity_resolver(resolver: IdentityResolver | None) -> None: + """Install (or clear) a custom identity resolver — the enterprise hook.""" + global _resolver + _resolver = resolver + + +def _default_os_user() -> str: + return os.environ.get("USER", os.environ.get("USERNAME", "default")) + + +def _client_host(request: Any) -> str | None: + client = getattr(request, "client", None) + host = getattr(client, "host", None) if client is not None else None + return host if isinstance(host, str) else None + + +def _token_identity() -> str | None: + token = os.environ.get("HEADROOM_PROXY_TOKEN") + if not token: + return None + return "tok_" + hashlib.sha256(token.encode()).hexdigest()[:16] + + +def resolve_memory_identity(request: Any, *, default: str | None = None) -> str: + """Resolve the memory partition id for a request. + + A registered custom resolver wins. Otherwise the header is honored only for + loopback callers; every other caller is bound to the proxy token (or the OS + user), so it can never address another user's partition. + """ + fallback = default if default is not None else _default_os_user() + + if _resolver is not None: + return _resolver(request, default=fallback) + + header_value: str | None + try: + header_value = request.headers.get(USER_ID_HEADER) + except Exception: + header_value = None + if header_value is not None: + header_value = header_value.strip() or None + + host = _client_host(request) + # Missing peer metadata is not evidence of loopback. Fail closed so unusual + # ASGI transports or incomplete request doubles cannot opt into header trust. + is_local = host is not None and is_loopback_host(host) + + if header_value is not None: + if is_local: + return header_value + # A caller-supplied value cannot authenticate its own authority to select + # that partition. Remote multi-tenant selection belongs in the custom + # resolver hook, where it can be bound to authenticated caller context. + + if not is_local: + token_id = _token_identity() + if token_id is not None: + return token_id + return fallback diff --git a/headroom/proxy/malloc_trim.py b/headroom/proxy/malloc_trim.py new file mode 100644 index 000000000..d7cd06eff --- /dev/null +++ b/headroom/proxy/malloc_trim.py @@ -0,0 +1,135 @@ +"""Return freed-but-retained allocator pages to the OS on long-lived proxies. + +Large concurrent Anthropic bodies (0.5-1 MB of JSON parsed, deep-copied and +re-serialized per in-flight request) drive libmalloc and pymalloc to a +high-water mark that is never returned to the OS: after a burst the malloc +zones keep entire regions resident but empty (``vmmap`` lists them as +``MALLOC_LARGE (empty)`` / ``MALLOC_SMALL (empty)``), so process RSS only +ratchets upward. Over a multi-day proxy lifetime under Claude Code traffic +this reaches double-digit GB and starves the host. + +Neither runtime returns these pages on its own. macOS exposes +``malloc_zone_pressure_relief(NULL, 0)`` to purge every zone's free pages; +glibc has ``malloc_trim(0)``. ``trim()`` calls that entry point directly: it is +a C call that releases the GIL and reclaims whatever is already on the +allocator's free lists. It deliberately does not run a Python ``gc.collect()`` +-- a full cyclic collection holds the GIL, and this periodic task runs off the +event-loop thread precisely so it cannot stall request handling; freeing cyclic +garbage is left to CPython's own automatic collection. +""" + +from __future__ import annotations + +import asyncio +import ctypes +import logging +import sys +import time + +logger = logging.getLogger(__name__) + +# Interval bounds for the periodic trim task. A non-positive interval would make +# ``asyncio.sleep`` return immediately and spin a continuous collect/trim loop, +# so anything below the minimum falls back to the default. +_DEFAULT_TRIM_INTERVAL_SECONDS = 60 +_MIN_TRIM_INTERVAL_SECONDS = 1 + +# Lazily resolved (platform_tag, foreign_function | None). ``None`` function +# means the platform has no supported trim call and trim() is a no-op. +_relief: tuple[str, object | None] | None = None + + +def _resolve() -> tuple[str, object | None]: + global _relief + if _relief is not None: + return _relief + try: + libc = ctypes.CDLL(None) + if sys.platform == "darwin": + fn = libc.malloc_zone_pressure_relief + fn.argtypes = [ctypes.c_void_p, ctypes.c_size_t] + fn.restype = ctypes.c_size_t + _relief = ("darwin", fn) + else: + fn = libc.malloc_trim + fn.argtypes = [ctypes.c_size_t] + fn.restype = ctypes.c_int + _relief = ("glibc", fn) + except (OSError, AttributeError): + _relief = ("unsupported", None) + return _relief + + +def trim() -> int: + """Return allocator free pages to the OS. + + Calls the platform's allocator pressure-relief entry point + (``malloc_zone_pressure_relief`` on macOS, ``malloc_trim`` on glibc). This + is a C call that releases the GIL for its duration and reclaims pages + already on the allocator's free lists. It deliberately does *not* run a + Python ``gc.collect()`` (a full cyclic collection holds the GIL); cyclic + garbage is left to CPython's automatic collection, so this off-thread + periodic task never holds the GIL for a full-heap traversal. + + Returns the number of bytes freed on macOS (glibc's ``malloc_trim`` + reports only success, so 0 is returned there and on unsupported + platforms). + """ + kind, fn = _resolve() + if fn is None: + return 0 + if kind == "darwin": + return int(fn(None, 0)) # type: ignore[operator] + fn(0) # type: ignore[operator] + return 0 + + +async def trim_periodically(interval_seconds: int = 60) -> None: + """Background task that periodically returns allocator free pages to the OS. + + Runs in every worker process (allocator state is per-process). The trim is + the platform's allocator pressure-relief C call + (``malloc_zone_pressure_relief``/``malloc_trim``), dispatched via + ``asyncio.to_thread`` so it runs off the event-loop thread. Because it is a + C call that releases the GIL and runs no Python ``gc.collect()``, it holds + the GIL only as briefly as the to_thread hand-off, so a slow purge on a + large heap does not stall request handling. The task exits immediately on + platforms with no supported trim call, so it is a true no-op there. + + Args: + interval_seconds: How often to trim (default: 60 seconds). A value below + ``_MIN_TRIM_INTERVAL_SECONDS`` (which would busy-loop) falls back to + the default. + """ + _, fn = _resolve() + if fn is None: + # No supported allocator-trim call on this platform (Windows, musl, ...); + # do not spin a wakeup task that can only ever no-op. + logger.debug("MallocTrim: no supported trim on %s; task disabled", sys.platform) + return + + if interval_seconds < _MIN_TRIM_INTERVAL_SECONDS: + logger.warning( + "MallocTrim: interval %ss is below the %ds minimum; using default %ds", + interval_seconds, + _MIN_TRIM_INTERVAL_SECONDS, + _DEFAULT_TRIM_INTERVAL_SECONDS, + ) + interval_seconds = _DEFAULT_TRIM_INTERVAL_SECONDS + + while True: + await asyncio.sleep(interval_seconds) + try: + start = time.perf_counter() + # Off the event-loop thread: the C-level purge can pause for a while + # on a large heap, and that pause must not stall proxy traffic. + freed = await asyncio.to_thread(trim) + elapsed_ms = (time.perf_counter() - start) * 1000 + log = logger.info if freed >= (16 << 20) else logger.debug + log( + "MallocTrim: returned %.1f MB to OS in %.0f ms", + freed / 1048576, + elapsed_ms, + ) + except Exception as e: + logger.debug("MallocTrim failed: %s", e) diff --git a/headroom/proxy/memory_handler.py b/headroom/proxy/memory_handler.py index 66d8b6db4..10917396d 100644 --- a/headroom/proxy/memory_handler.py +++ b/headroom/proxy/memory_handler.py @@ -154,7 +154,7 @@ class MemoryConfig: qdrant_api_key: str | None = field(default_factory=qdrant_env.qdrant_env_api_key) neo4j_uri: str = "neo4j://localhost:7687" neo4j_user: str = "neo4j" - neo4j_password: str = "password" + neo4j_password: str = field(default_factory=lambda: os.environ.get("NEO4J_PASSWORD", "")) # Memory Bridge (bidirectional markdown <-> Headroom sync) bridge_enabled: bool = False bridge_md_paths: list[str] = field(default_factory=list) diff --git a/headroom/proxy/models.py b/headroom/proxy/models.py index 92c6ef366..6b85689c3 100644 --- a/headroom/proxy/models.py +++ b/headroom/proxy/models.py @@ -7,12 +7,14 @@ Extracted from server.py to keep the codebase maintainable. from __future__ import annotations import logging +import sys from dataclasses import InitVar, dataclass, field from datetime import datetime from typing import Any, Literal from headroom.memory import qdrant_env from headroom.providers.registry import ProviderApiOverrides +from headroom.proxy.buffered_ccr_response import DEFAULT_BUFFERED_CCR_GRACE_SECONDS from headroom.proxy.model_router import ModelRouterConfig from headroom.rollout import RolloutSnapshot, resolve_rollout @@ -369,6 +371,12 @@ class ProxyConfig: # Anthropic buffered reads can legitimately run longer than the generic # proxy request cap. Keep the generic timeout unchanged elsewhere. anthropic_buffered_request_timeout_seconds: int = 600 + # How long a buffered-CCR turn holds out for full status fidelity before it + # commits to SSE and starts a keepalive. Under the window, failures keep + # their real HTTP status; past it, the client gets a first byte before its + # stream-idle watchdog fires. 0 or less disables the keepalive entirely. + # See headroom/proxy/buffered_ccr_response.py (#3079). + buffered_ccr_grace_seconds: float = DEFAULT_BUFFERED_CCR_GRACE_SECONDS # Connection pool max_connections: int = 500 @@ -439,6 +447,17 @@ class ProxyConfig: # Env: HEADROOM_PERIODIC_TOIN_STATS=0. periodic_toin_stats_enabled: bool = True + # Periodic allocator trim. Long-lived proxies processing large concurrent + # request bodies ratchet RSS through freed-but-retained allocator pages; + # this returns them to the OS (malloc_zone_pressure_relief on macOS, + # malloc_trim on glibc). Default-on only on macOS, where the retained-page + # ratchet is the documented failure (#2820); an opt-in elsewhere via + # HEADROOM_MALLOC_TRIM=1 so glibc deployments do not silently take on a + # once-a-minute allocator purge they did not ask for. Envs: + # HEADROOM_MALLOC_TRIM=0/1, HEADROOM_MALLOC_TRIM_INTERVAL_SECONDS. + periodic_malloc_trim_enabled: bool = field(default_factory=lambda: sys.platform == "darwin") + malloc_trim_interval_seconds: int = 60 + # Stateless mode — disable all filesystem writes for read-only / container deployments stateless: bool = False diff --git a/headroom/proxy/nonstream_sse_policy.py b/headroom/proxy/nonstream_sse_policy.py new file mode 100644 index 000000000..9b0bba09a --- /dev/null +++ b/headroom/proxy/nonstream_sse_policy.py @@ -0,0 +1,129 @@ +"""Wire-format contract policy for the buffered (non-streaming) reply path. + +The problem +----------- + +The buffered Anthropic path returns the upstream reply with its headers +copied wholesale:: + + response_headers = dict(response.headers) + response_headers.pop("content-encoding", None) + response_headers.pop("content-length", None) + ... + return Response(content=..., status_code=..., headers=response_headers) + +``content-type`` rides along untouched. When the upstream answers a +``stream``-less request with ``text/event-stream``, that body reaches a +caller that asked for JSON, as a ``200`` it cannot parse. Clients report +it as an empty or malformed response and the turn is lost — the reply is +*present and complete*, just wearing the wrong wire format. + +The buffered-stream (CCR) path already refuses this shape, logging the +offending ``content-type`` and returning ``upstream_protocol_error`` +(#2952). The plain non-streaming path never got the same treatment: an +unparseable body there was assumed to mean "no CCR handling", so it was +logged at DEBUG and passed through. + +The contract +------------ + +A caller that did not set ``stream: true`` must never receive an +event-stream body. Headroom owns both ends of that boundary, so it can +enforce it rather than let the mismatch reach the client. + +Behaviour matrix +---------------- + +============================ ============== ========= ==================== +Client asked for streaming? Upstream C-T Status Result +============================ ============== ========= ==================== +yes any any untouched +no application/… any untouched +no text/event-… != 200 untouched (real error) +no text/event-… 200 recover, else refuse +============================ ============== ========= ==================== + +Recovery reuses ``StreamingMixin._parse_sse_to_response``, the same +reconstruction the streaming path already runs for usage accounting, so +this adds no new parsing surface. Recovering in place — rather than +returning early — keeps the rest of the buffered path (CCR, turn hooks, +security scan, usage accounting) operating on a normal reply. + +Non-200 is deliberately excluded: an error status is already actionable +by the client, and passing it through unchanged preserves the upstream's +own error payload. + +Public API +---------- + +* :func:`is_event_stream` — media-type test, parameter- and case-tolerant. +* :func:`should_recover_sse_reply` — the gate above, as one predicate. + +Header correction is *not* here — see the note beside the public functions. + +Constraints (per project memory) +-------------------------------- + +* pure: no I/O, no logging, no config — the handler owns those. +* no regexes: media-type parsing is a single ``split``. +* no silent fallbacks: the caller refuses loudly when recovery fails. +""" + +from __future__ import annotations + +SSE_MEDIA_TYPE = "text/event-stream" +JSON_MEDIA_TYPE = "application/json" + + +def media_type(content_type: str | None) -> str: + """Return the bare media type, lower-cased, with parameters dropped. + + ``"text/event-stream; charset=utf-8"`` and ``"Text/Event-Stream"`` both + yield ``"text/event-stream"``. Returns ``""`` for a missing header. + """ + if not content_type: + return "" + return content_type.split(";", 1)[0].strip().lower() + + +def is_event_stream(content_type: str | None) -> bool: + """True when ``content_type`` denotes an SSE body.""" + return media_type(content_type) == SSE_MEDIA_TYPE + + +def should_recover_sse_reply( + *, + client_requested_stream: bool, + status_code: int, + content_type: str | None, + body_is_event_stream: bool = False, +) -> bool: + """True when a buffered reply violates the caller's non-streaming contract. + + See the behaviour matrix in the module docstring. The three negative + arms are all deliberate: a streaming caller *wants* SSE, a JSON + content-type is already correct, and a non-200 carries an upstream + error the client should see verbatim. + + ``body_is_event_stream`` covers the reply that *is* an event stream while + saying otherwise — a mislabeled or absent ``content-type``. Trusting the + declared type alone would let exactly the same unparseable body through, + so the caller sniffs the payload and passes the answer in. It stays a + parameter rather than an import because this module is pure: the sniff + needs the response object, and the handler owns that. + """ + if client_requested_stream: + return False + if status_code != 200: + return False + return is_event_stream(content_type) or body_is_event_stream + + +# Header correction deliberately lives in +# ``helpers.sanitize_forwarded_response_headers`` rather than here. It already +# owns ``FRAMING_RESPONSE_HEADERS`` and strips ``connection``, ``keep-alive`` +# and ``server`` alongside the content-* family — a second copy of that list +# would drift, and the ones this module would have missed are load-bearing: +# leaving ``transfer-encoding`` on a rebuilt body is what produced an empty +# HTTP 200 in #3019, and ``server: cloudflare`` is one of the headers the +# client cites as evidence of an intermediary. diff --git a/headroom/proxy/outcome.py b/headroom/proxy/outcome.py index f98c88753..bc594f81e 100644 --- a/headroom/proxy/outcome.py +++ b/headroom/proxy/outcome.py @@ -399,7 +399,12 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None: from headroom.proxy.cost import _summarize_transforms from headroom.proxy.models import RequestLog from headroom.proxy.project_context import get_current_project - from headroom.proxy.savings_attribution import encode, from_tags, public_tags + from headroom.proxy.savings_attribution import ( + encode, + from_tags, + public_tags, + timings_from_tags, + ) from headroom.telemetry.session import record_outcome # GitHub Copilot: requests routed to the Copilot API travel on the OpenAI or @@ -467,6 +472,20 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None: tool_search_saved = tool_schema_saved_from_tags(outcome.tags or {}) savings_breakdown = from_tags(outcome.tags) + # Stage timings contributed from OUTSIDE the handler, folded in here rather + # than in each handler so every provider picks them up from one place. + # + # The handler's own timings win a name collision, which cannot happen while + # extension stages carry the ``ext:`` prefix but is the safe way round if + # that ever changes: a plugin must not be able to overwrite a measurement + # the pipeline made of itself. + extension_timing = timings_from_tags(outcome.tags) + pipeline_timing = ( + {**extension_timing, **(outcome.pipeline_timing or {})} + if extension_timing + else outcome.pipeline_timing + ) + # Billed input volume. Prefer the provider's own count where it reported one # — that is what the invoice charges for, and it is the number cache math is # already expressed in. Falls back to our local ``optimized_tokens`` when the @@ -488,7 +507,7 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None: cached=outcome.cache_hit, overhead_ms=outcome.overhead_ms, ttfb_ms=outcome.ttfb_ms, - pipeline_timing=outcome.pipeline_timing, + pipeline_timing=pipeline_timing, waste_signals=outcome.waste_signals, cache_read_tokens=outcome.cache_read_tokens, cache_write_tokens=outcome.cache_write_tokens, @@ -567,6 +586,15 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None: # line unchanged, and gives ``headroom perf --client X`` # parsers a clean key to filter on. client_part = f" client={outcome.client}" if outcome.client else "" + # ``cached=1`` marks a turn answered from Headroom's own response cache. + # Such a turn never contacts the upstream, so it has no outbound_request + # line, no upstream stage timings, and all-zero token counters — which + # made it indistinguishable in the logs from a turn that died silently + # (#3019). Appended only on a hit, so every other PERF line is unchanged + # and existing parsers keep working (``_parse_kv`` reads trailing + # key=value pairs after ``transforms=`` the same way it reads + # ``client=``). + cached_part = " cached=1" if outcome.from_response_cache else "" # Tool-schema DEFERRAL savings can't move tok_before/after (those count messages # only), so a tool-heavy turn shows tok_saved=0 while genuinely saving thousands of # tool-definition tokens. `tool_saved` carries that component and `total_saved` is @@ -592,4 +620,5 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None: f"savings={encoded_savings} " f"transforms={_summarize_transforms(list(outcome.transforms_applied))}" f"{client_part}" + f"{cached_part}" ) diff --git a/headroom/proxy/prometheus_metrics.py b/headroom/proxy/prometheus_metrics.py index 3586c7a7b..503a7ffe2 100644 --- a/headroom/proxy/prometheus_metrics.py +++ b/headroom/proxy/prometheus_metrics.py @@ -152,6 +152,14 @@ class PrometheusMetrics: # whether the compression budget is too tight vs. a real bug. self.compression_failed_by_reason: dict[str, int] = defaultdict(int) + # Upstream transport failures on the streaming path, keyed by provider. + # Raised when every connect retry is exhausted and the proxy synthesizes + # its own error response instead of forwarding an upstream status. That + # path emits no upstream status code to attribute the failure to, so + # without this counter it is invisible in metrics and survives only as a + # log line. + self.upstream_connection_errors_by_provider: dict[str, int] = defaultdict(int) + # Kompress size-gate outcomes, keyed by outcome ("within", # "exceeded"). The gate routes oversized blocks away from ML # compression (ContentRouter._kompress_max_tokens, #1171). This @@ -355,6 +363,7 @@ class PrometheusMetrics: self.savings_by_source.clear() with self._obs_counter_lock: self.compression_failed_by_reason.clear() + self.upstream_connection_errors_by_provider.clear() self.kompress_size_gate_by_outcome.clear() self.compression_quarantine_by_event.clear() @@ -568,6 +577,18 @@ class PrometheusMetrics: with self._obs_counter_lock: self.compression_failed_by_reason[reason or "error"] += 1 + def record_upstream_connection_error(self, provider: str) -> None: + """Record one exhausted-retries upstream transport failure. + + Called from the streaming handler's ``httpx.TransportError`` fallback + (handlers/streaming.py), where the proxy synthesizes its own 502 + because no upstream response ever arrived. Guarded by + ``_obs_counter_lock`` for the same reason as + ``record_compression_failed``. + """ + with self._obs_counter_lock: + self.upstream_connection_errors_by_provider[provider or "unknown"] += 1 + def record_kompress_size_gate(self, outcome: str) -> None: """Record one kompress size-gate decision, bucketed by ``outcome``. @@ -1404,9 +1425,23 @@ class PrometheusMetrics: # then format outside it (see _obs_counter_lock). with self._obs_counter_lock: compression_failed = dict(self.compression_failed_by_reason) + upstream_conn_errors = dict(self.upstream_connection_errors_by_provider) kompress_size_gate = dict(self.kompress_size_gate_by_outcome) compression_quarantine = dict(self.compression_quarantine_by_event) + if upstream_conn_errors: + lines.extend( + [ + "# HELP headroom_upstream_connection_errors_total Exhausted-retries upstream transport failures by provider; the proxy answered 502 itself because no upstream response arrived", + "# TYPE headroom_upstream_connection_errors_total counter", + ] + ) + for prov, count in upstream_conn_errors.items(): + lines.append( + f'headroom_upstream_connection_errors_total{{provider="{_escape_label_value(prov)}"}} {count}' + ) + lines.append("") + if compression_failed: lines.extend( [ diff --git a/headroom/proxy/savings_attribution.py b/headroom/proxy/savings_attribution.py index db7f517e9..2ea8c3fb4 100644 --- a/headroom/proxy/savings_attribution.py +++ b/headroom/proxy/savings_attribution.py @@ -4,6 +4,7 @@ from __future__ import annotations import base64 import json +import math import re from collections.abc import MutableMapping from typing import Any @@ -13,6 +14,43 @@ _NAME_RE = re.compile(r"[^a-z0-9_.-]+") MAX_SOURCES = 32 _SCOPE_KEY = "headroom_savings_attribution" +# Per-request stage timings contributed from outside the handler, merged into +# ``RequestOutcome.pipeline_timing`` at the outcome funnel. +# +# An ASGI middleware wraps the handler, so every millisecond it spends lands in +# the client's latency while ``overhead_ms`` -- measured inside the handler -- +# stays flat. An extension that halves the bill and adds 200ms per request is a +# trade the operator has to be able to see both halves of, and until now only +# one half reached the dashboard. +STAGE_TIMING_TAG = "_headroom_stage_timing" +_TIMING_SCOPE_KEY = "headroom_stage_timing" + +# Stage names are extension-supplied, so they are capped like every other +# client-influenced label in this proxy (see MAX_DISTINCT_MODELS). +MAX_STAGES = 16 + +# Namespace, so an extension can never shadow a built-in transform's timing -- +# ``deep_copy`` reported by a plugin and ``deep_copy`` reported by the pipeline +# must not accumulate into the same series. +STAGE_PREFIX = "ext:" + +# NON-FINITE VALUES POISON EVERY CONSUMER DOWNSTREAM, and they do it long after +# the call that introduced them. Starlette's JSONResponse encodes with +# ``allow_nan=False``, so a single ``inf`` reaching ``/stats`` raises +# ``ValueError: Out of range float values are not JSON compliant`` -- and the +# value sits in the process-wide metrics totals, so the endpoint stays broken +# until restart. Prometheus is no better: Python renders ``inf``, the exposition +# format wants ``+Inf``, and the scrape fails to parse. +# +# The request itself still returns 200 throughout, which is the worst shape a +# bug can have: the extension looks healthy while the operator's dashboard and +# scrape are dead. +# +# One hour bounds a single stage inside one request -- unreachable in practice, +# and it makes overflow-to-infinity on accumulation structurally impossible +# (16 stages x 1h is nowhere near the float ceiling). +MAX_STAGE_MS = 3_600_000.0 + def _source_name(value: object) -> str: name = _NAME_RE.sub("_", str(value or "other").strip().lower()).strip("_.-") @@ -29,7 +67,12 @@ def _ledger(tags: MutableMapping[str, Any]) -> list[dict[str, Any]]: def bind_scope(tags: MutableMapping[str, Any], scope: MutableMapping[str, Any]) -> None: - """Share one ledger between ASGI middleware and the request handler.""" + """Share the savings and timing ledgers between ASGI middleware and the handler. + + Both are bound together because an extension that reports one usually + reports the other, and a handler that binds only savings would drop the + timings silently -- which is the failure this call is here to prevent. + """ state = scope.setdefault("state", {}) ledger = state.get(_SCOPE_KEY) if not isinstance(ledger, list): @@ -37,6 +80,12 @@ def bind_scope(tags: MutableMapping[str, Any], scope: MutableMapping[str, Any]) state[_SCOPE_KEY] = ledger tags[SAVINGS_ATTRIBUTION_TAG] = ledger + timings = state.get(_TIMING_SCOPE_KEY) + if not isinstance(timings, dict): + timings = {} + state[_TIMING_SCOPE_KEY] = timings + tags[STAGE_TIMING_TAG] = timings + def record_scope_savings(scope: MutableMapping[str, Any], source: object, **values: Any) -> None: state = scope.setdefault("state", {}) @@ -47,6 +96,63 @@ def record_scope_savings(scope: MutableMapping[str, Any], source: object, **valu record_savings({SAVINGS_ATTRIBUTION_TAG: ledger}, source, **values) +def record_scope_timing(scope: MutableMapping[str, Any], stage: object, ms: float) -> None: + """Attribute milliseconds spent outside the handler to a named stage. + + Additive within one request, so a middleware that works in two passes + (before and after ``call_next``) reports each and gets their sum. Never + raises and never changes a response: a plugin's telemetry must not be able + to break the request it is describing. + """ + try: + elapsed = float(ms) + except (TypeError, ValueError): + return + # Non-positive is either a clock artifact or nothing happening; either way + # it is not a measurement, and averaging it in would drag the mean toward + # zero exactly where the stage is cheapest to ignore. Non-finite and + # absurdly large are not measurements either, and they break consumers + # rather than merely skewing them -- see MAX_STAGE_MS. + if not math.isfinite(elapsed) or not 0.0 < elapsed <= MAX_STAGE_MS: + return + + state = scope.setdefault("state", {}) + timings = state.get(_TIMING_SCOPE_KEY) + if not isinstance(timings, dict): + timings = {} + state[_TIMING_SCOPE_KEY] = timings + + name = STAGE_PREFIX + _source_name(stage) + if name not in timings and len(timings) >= MAX_STAGES: + return + timings[name] = round(float(timings.get(name, 0.0)) + elapsed, 4) + + +def timings_from_tags(tags: MutableMapping[str, Any] | None) -> dict[str, float]: + """Extension stage timings carried on the request's tags, if any.""" + raw = (tags or {}).get(STAGE_TIMING_TAG) + if not isinstance(raw, dict): + return {} + out: dict[str, float] = {} + for name, value in list(raw.items())[:MAX_STAGES]: + try: + elapsed = float(value) + except (TypeError, ValueError): + continue + # Re-checked rather than trusted: the ledger is a plain dict reachable + # through ``tags``, so a handler can be handed one this module never + # wrote. The guarantee has to hold at the read, not only at the write. + # + # Finiteness ONLY. ``MAX_STAGE_MS`` bounds a single sample at the write, + # where it prevents overflow; applying it here would test it against an + # ACCUMULATED total and silently discard a stage that legitimately ran + # for longer across many samples -- throwing away real data to guard + # against a value this path cannot produce. + if math.isfinite(elapsed) and elapsed > 0.0: + out[str(name)] = elapsed + return out + + def record_savings( tags: MutableMapping[str, Any], source: object, @@ -61,12 +167,25 @@ def record_savings( ledger = _ledger(tags) if len(ledger) >= MAX_SOURCES: return + # Same hazard as MAX_STAGE_MS, on the amounts rather than the durations: + # ``usd=inf`` reaches ``/stats`` and raises out of the JSON encoder, and + # ``int(inf)`` raises OverflowError right here, inside the handler, on a + # request that would otherwise have succeeded. Neither is a saving, so + # neither is recorded -- the alternative is a plugin's arithmetic bug + # taking down an endpoint it has nothing to do with. + try: + amount = float(usd or 0.0) + count = int(tokens or 0) + except (TypeError, ValueError, OverflowError): + return + if not math.isfinite(amount): + return item: dict[str, Any] = { "source": _source_name(source), "realized": bool(realized), "estimated": bool(estimated), - "tokens": max(0, int(tokens or 0)), - "usd": round(float(usd or 0.0), 12), + "tokens": max(0, count), + "usd": round(amount, 12), } if details: item["details"] = { @@ -84,8 +203,17 @@ def from_tags(tags: MutableMapping[str, Any] | None) -> list[dict[str, Any]]: return [dict(item) for item in raw[:MAX_SOURCES] if isinstance(item, dict)] +_INTERNAL_TAGS = frozenset({SAVINGS_ATTRIBUTION_TAG, STAGE_TIMING_TAG}) + + def public_tags(tags: MutableMapping[str, Any] | None) -> dict[str, Any]: - return {key: value for key, value in (tags or {}).items() if key != SAVINGS_ATTRIBUTION_TAG} + """Tags minus the internal ledgers, which are structures rather than labels. + + They are carried on ``tags`` because that is the one dict that reaches the + outcome funnel from every handler; letting them through to ``RequestLog`` + would put a list and a dict into a string-keyed label store. + """ + return {key: value for key, value in (tags or {}).items() if key not in _INTERNAL_TAGS} def encode(items: list[dict[str, Any]]) -> str: diff --git a/headroom/proxy/savings_tracker.py b/headroom/proxy/savings_tracker.py index 4c10cac6b..57d55e747 100644 --- a/headroom/proxy/savings_tracker.py +++ b/headroom/proxy/savings_tracker.py @@ -768,11 +768,31 @@ class SavingsTracker: delta_output_tokens_saved = max(_coerce_int(output_tokens_saved), 0) delta_cache_read_tokens = _coerce_int(cache_read_tokens) priced = estimated_savings_usd - delta_savings_usd = ( - max(_coerce_float(priced.get("compression")), 0.0) - if priced is not None - else _estimate_compression_savings_usd(model, delta_tokens_saved) - ) + # ``estimate_request_savings_usd`` prices FOUR buckets, but this method + # only ever read three — ``tool_schema`` was computed and dropped on the + # floor. Tool-schema deferral is a quarter of the token headline on real + # traffic (2.7M of 11.2M), and ``tokens_saved`` here is the bare + # message-level figure (the caller folds deferral in separately for the + # ledger, see prometheus_metrics.record_request), so the dollars were + # simply missing rather than counted elsewhere. That is why "Cost saved" + # read materially below the token-savings percent on the same traffic. + # Add it to the compression bucket, matching how the PERF headline and + # perf/analyzer fold deferral into one number. + if priced is not None: + # Two DISJOINT buckets: the caller passes bare message savings as + # ``compression_tokens_saved`` and deferral separately as + # ``tool_schema_tokens_saved`` (see prometheus_metrics.record_request), + # so summing them is additive, not double counting. Written as a + # statement rather than folded into the ternary below — a money path + # should not depend on the reader knowing that ``a + b if c else d`` + # groups as ``(a + b) if c else d``. + delta_savings_usd = max(_coerce_float(priced.get("compression")), 0.0) + max( + _coerce_float(priced.get("tool_schema")), 0.0 + ) + else: + # No priced breakdown available: only message savings are known here, + # so this path stays message-only exactly as before. + delta_savings_usd = _estimate_compression_savings_usd(model, delta_tokens_saved) delta_output_savings_usd = ( max(_coerce_float(priced.get("output_shaping")), 0.0) if priced is not None diff --git a/headroom/proxy/semantic_cache_key.py b/headroom/proxy/semantic_cache_key.py index cbef6dcd6..267eddb36 100644 --- a/headroom/proxy/semantic_cache_key.py +++ b/headroom/proxy/semantic_cache_key.py @@ -21,11 +21,20 @@ def compute_semantic_cache_key( model: str, **key_fields: Any, ) -> str: - """Compute the proxy semantic-cache key from generation-shaping inputs.""" + """Compute the proxy semantic-cache key from generation-shaping inputs. + + ``cache_control`` is stripped from ``messages`` as well as the shaping + fields: it is a prompt-caching directive for the upstream provider that + never changes the generated completion, so a moved breakpoint must not + fragment the key. Messages are the primary key component and, on the + Anthropic path, the most common place a client (e.g. Claude Code) moves a + breakpoint between turns, so leaving them un-stripped defeated the strip for + the field that matters most. + """ normalized = json.dumps( { "model": model, - "messages": messages, + "messages": strip_cache_control(messages), **{k: strip_cache_control(v) for k, v in key_fields.items()}, }, sort_keys=True, diff --git a/headroom/proxy/semantic_cache_key_policy.py b/headroom/proxy/semantic_cache_key_policy.py index 1fef49a7c..3b7d3625d 100644 --- a/headroom/proxy/semantic_cache_key_policy.py +++ b/headroom/proxy/semantic_cache_key_policy.py @@ -21,11 +21,20 @@ def compute_semantic_cache_key( model: str, **key_fields: Any, ) -> str: - """Compute a stable cache key from request content and shaping fields.""" + """Compute a stable cache key from request content and shaping fields. + + ``cache_control`` is stripped from ``messages`` as well as the shaping + fields: it is a prompt-caching directive for the upstream provider that + never changes the generated completion, so a moved breakpoint must not + fragment the key. Messages are the primary key component and, on the + Anthropic path, the most common place a client (e.g. Claude Code) moves a + breakpoint between turns, so leaving them un-stripped defeated the strip for + the field that matters most. + """ normalized = json.dumps( { "model": model, - "messages": messages, + "messages": strip_cache_control(messages), **{k: strip_cache_control(v) for k, v in key_fields.items()}, }, sort_keys=True, diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py index 4dccdda5c..ebf1c3bb5 100644 --- a/headroom/proxy/server.py +++ b/headroom/proxy/server.py @@ -66,7 +66,7 @@ except ImportError: sys.path.insert(0, str(Path(__file__).parent.parent.parent)) from headroom._version import __version__ -from headroom.agent_savings import proxy_pipeline_kwargs +from headroom.agent_savings import DEFAULT_PROFILE, proxy_pipeline_kwargs from headroom.cache.compression_feedback import get_compression_feedback from headroom.cache.compression_store import format_retrieval_miss_detail, get_compression_store from headroom.ccr import ( @@ -114,6 +114,7 @@ from headroom.proxy.audit import is_auditable_path, record_admin_action from headroom.proxy.auth_mode import should_stamp_codex_client from headroom.proxy.background_compression import BackgroundCompressor from headroom.proxy.budget_basis_policy import resolve_estimated_basis_policy +from headroom.proxy.buffered_ccr_response import DEFAULT_BUFFERED_CCR_GRACE_SECONDS from headroom.proxy.cors import cors_origin_regex_for_config, cors_origins_for_config # ============================================================================= @@ -145,6 +146,7 @@ from headroom.proxy.helpers import ( ) from headroom.proxy.loop_callback_failure_policy import is_known_websocket_callback_failure from headroom.proxy.loopback_guard import is_loopback_host +from headroom.proxy.malloc_trim import trim_periodically from headroom.proxy.memory_handler import MemoryConfig, MemoryHandler # Data models (extracted to headroom/proxy/models.py for maintainability) @@ -1736,6 +1738,38 @@ class HeadroomProxy( if self.config.mode == PROXY_MODE_CACHE: logger.info(" Prefix freeze: strict (all prior turns immutable)") logger.info(" Mutations: latest turn only") + # Effective compression posture, resolved (not merely requested). The + # savings profile reaches the router through two paths — the config + # object and the seeded process env — and `setdefault` semantics mean a + # stale HEADROOM_* value silently overrides the profile that names it. + # A deployment can therefore log `savings_profile=coding` while actually + # running balanced's thresholds, and the only prior evidence was a + # single easily-missed WARNING. Print what is actually in force so + # "which profile am I really running?" is answerable from the banner. + try: + _eff = proxy_pipeline_kwargs(self.config) + # Read cross-turn dedup off the CONSTRUCTED router, not off the env. + # ContentRouter resolves it as `config.enable_cross_turn_dedup OR + # $HEADROOM_DEDUPE`, so reporting the env alone would be a guess that + # happens to be right only while nothing sets the config field. A + # banner line exists to be trusted; it must read what was resolved. + _dedupe: object = "unknown" + for _t in getattr(self.anthropic_pipeline, "transforms", []): + if isinstance(_t, ContentRouter): + _dedupe = bool(getattr(_t, "_cross_turn_dedup_enabled", False)) + break + logger.info( + "Savings profile: %s (effective: min_tokens=%s min_chars_block=%s " + "compress_user=%s dedupe=%s tool_search=%s)", + self.config.savings_profile or DEFAULT_PROFILE, + _eff.get("min_tokens_to_compress", "default"), + _eff.get("min_chars_for_block_compression", "default(500)"), + _eff.get("compress_user_messages", False), + _dedupe, + os.environ.get("HEADROOM_TOOL_SEARCH", "1") in ("1", "true", "yes", "on", "auto"), + ) + except Exception: # never let a banner line block startup + logger.debug("effective savings-profile banner skipped", exc_info=True) logger.info(f"Caching: {'ENABLED' if self.config.cache_enabled else 'DISABLED'}") logger.info(f"Rate Limiting: {'ENABLED' if self.config.rate_limit_enabled else 'DISABLED'}") logger.info( @@ -2438,6 +2472,35 @@ def _normalized_http_origin(value: str) -> tuple[str, str, int] | None: return scheme, parsed.hostname.lower(), port +#: Feedback-pattern keys built verbatim from agent query text. They are useful +#: in-process for compression decisions but must never reach an HTTP response — +#: same privacy contract the TOIN endpoints were brought under in #2926/#2927. +_FEEDBACK_QUERY_TEXT_KEYS = ("common_queries", "queried_fields") + + +def _feedback_stats_without_query_text(stats: dict[str, Any]) -> dict[str, Any]: + """Return ``stats`` with per-tool query text stripped from ``tool_patterns``. + + Copies only the levels it edits; the aggregate counters are shared with the + caller's dict, which is fine because they are scalars. + """ + + patterns = stats.get("tool_patterns") + if not isinstance(patterns, dict): + return stats + + scrubbed: dict[str, Any] = {} + for name, pattern in patterns.items(): + if isinstance(pattern, dict): + scrubbed[name] = { + key: value for key, value in pattern.items() if key not in _FEEDBACK_QUERY_TEXT_KEYS + } + else: + scrubbed[name] = pattern + + return {**stats, "tool_patterns": scrubbed} + + _is_known_websocket_callback_failure = is_known_websocket_callback_failure @@ -2602,6 +2665,7 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI: app.state.ready = False app.state.startup_error = None app.state.periodic_toin_stats_task = None + app.state.periodic_malloc_trim_task = None try: try: @@ -2612,6 +2676,12 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI: app.state.periodic_toin_stats_task = asyncio.create_task( _log_toin_stats_periodically() ) + # Per-worker on purpose: allocator state is per-process, so + # every worker must trim its own zones (no beacon-owner gate). + if config.periodic_malloc_trim_enabled: + app.state.periodic_malloc_trim_task = asyncio.create_task( + trim_periodically(config.malloc_trim_interval_seconds) + ) if proxy.usage_reporter: await proxy.usage_reporter.start(proxy) if proxy.traffic_learner: @@ -2671,6 +2741,16 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI: ) app.state.periodic_toin_stats_task = None + periodic_malloc_trim_task = app.state.periodic_malloc_trim_task + if periodic_malloc_trim_task is not None: + periodic_malloc_trim_task.cancel() + await _timed( + asyncio.gather(periodic_malloc_trim_task, return_exceptions=True), + label="periodic_malloc_trim.stop", + timeout=3.0, + ) + app.state.periodic_malloc_trim_task = None + if _cc_reconciler is not None: await _timed(_cc_reconciler.stop(), label="cc_reconciler.stop", timeout=3.0) if _beacon_is_owner[0]: @@ -3481,7 +3561,10 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI: payload["runtime"] = _runtime_payload() return JSONResponse(status_code=200, content=payload) - @app.post("/admin/runtime-env", dependencies=[Depends(_require_loopback)]) + @app.post( + "/admin/runtime-env", + dependencies=[Depends(_require_loopback), Depends(_require_same_origin)], + ) async def admin_runtime_env(request: Request): """Hot-reload live env knobs (the output-shaper family, the ast-grep read threshold) without restarting the proxy. @@ -4422,7 +4505,10 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI: payload["persistence"] = {**persistence, "error": None} return payload - @app.post("/stats/reset", dependencies=[Depends(_require_loopback)]) + @app.post( + "/stats/reset", + dependencies=[Depends(_require_loopback), Depends(_require_same_origin)], + ) async def stats_reset(): """Reset in-memory proxy stats for local test/debug isolation.""" await proxy.metrics.reset_runtime() @@ -4559,7 +4645,10 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI: report = tracker.get_report() return report.to_dict() - @app.post("/cache/clear", dependencies=[Depends(_require_loopback)]) + @app.post( + "/cache/clear", + dependencies=[Depends(_require_loopback), Depends(_require_same_origin)], + ) async def clear_cache(): """Clear the response cache. @@ -4575,7 +4664,10 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI: return {"status": "cache disabled"} # CCR (Compress-Cache-Retrieve) endpoints - @app.post("/v1/retrieve", dependencies=[Depends(_require_loopback)]) + @app.post( + "/v1/retrieve", + dependencies=[Depends(_require_loopback), Depends(_require_same_origin)], + ) async def ccr_retrieve(request: Request): """Retrieve original content from CCR compression cache. @@ -4644,21 +4736,25 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI: ], } - @app.get("/v1/feedback") + @app.get("/v1/feedback", dependencies=[Depends(_require_loopback)]) async def ccr_feedback(): """Get CCR feedback loop statistics and learned patterns. This endpoint exposes the feedback loop's learned patterns for monitoring and debugging. It shows: - Per-tool retrieval rates (high = compress less aggressively) - - Common search queries per tool - - Queried fields (suggest what to preserve) + - Aggregate compression/retrieval counters per tool Use this to understand how well compression is working and whether the feedback loop is adjusting appropriately. + + Loopback-guarded and query-text free for the same reason as the + telemetry and TOIN endpoints (#2926/#2927): ``common_queries`` and + ``queried_fields`` are built verbatim from agent search queries, so + they stay out of the response even on the guarded path. """ feedback = get_compression_feedback() - stats = feedback.get_stats() + stats = _feedback_stats_without_query_text(feedback.get_stats()) return { "feedback": stats, "hints_example": { @@ -4677,12 +4773,15 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI: }, } - @app.get("/v1/feedback/{tool_name}") + @app.get("/v1/feedback/{tool_name}", dependencies=[Depends(_require_loopback)]) async def ccr_feedback_for_tool(tool_name: str): """Get compression hints for a specific tool. Returns feedback-based hints that would be used for compressing this tool's output. + + Loopback-guarded, and the pattern block excludes ``common_queries`` + and ``queried_fields`` — both are raw agent query text (#2926/#2927). """ feedback = get_compression_feedback() hints = feedback.get_compression_hints(tool_name) @@ -4705,8 +4804,6 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI: "retrieval_rate": patterns.retrieval_rate if patterns else 0.0, "full_retrieval_rate": patterns.full_retrieval_rate if patterns else 0.0, "search_rate": patterns.search_rate if patterns else 0.0, - "common_queries": list(patterns.common_queries.keys())[:10] if patterns else [], - "queried_fields": list(patterns.queried_fields.keys())[:10] if patterns else [], } if patterns else None, @@ -4752,7 +4849,10 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI: telemetry = get_telemetry_collector() return telemetry.export_stats() - @app.post("/v1/telemetry/import", dependencies=[Depends(_require_loopback)]) + @app.post( + "/v1/telemetry/import", + dependencies=[Depends(_require_loopback), Depends(_require_same_origin)], + ) async def telemetry_import(request: Request): """Import telemetry data from another source. @@ -5134,6 +5234,10 @@ def _proxy_config_from_env() -> ProxyConfig: 600, min_value=1, ), + buffered_ccr_grace_seconds=_get_env_float( + "HEADROOM_BUFFERED_CCR_GRACE_SECONDS", + DEFAULT_BUFFERED_CCR_GRACE_SECONDS, + ), vertex_api_url=os.environ.get("VERTEX_TARGET_API_URL"), backend=_get_env_str("HEADROOM_BACKEND", "anthropic"), bedrock_region=_get_env_str("HEADROOM_BEDROCK_REGION", "us-west-2"), @@ -5153,6 +5257,10 @@ def _proxy_config_from_env() -> ProxyConfig: http2=_get_env_bool("HEADROOM_HTTP2", True), http_proxy=os.environ.get("HEADROOM_HTTP_PROXY") or None, periodic_toin_stats_enabled=_get_env_bool("HEADROOM_PERIODIC_TOIN_STATS", True), + periodic_malloc_trim_enabled=_get_env_bool( + "HEADROOM_MALLOC_TRIM", sys.platform == "darwin" + ), + malloc_trim_interval_seconds=_get_env_int("HEADROOM_MALLOC_TRIM_INTERVAL_SECONDS", 60), proxy_token=os.environ.get("HEADROOM_PROXY_TOKEN") or None, offline=_get_env_bool("HEADROOM_OFFLINE", False), # Default mode is CACHE (Headroom's coding posture): delta-only compression @@ -5370,11 +5478,18 @@ def run_server( else: app_target = create_app(config) + # "warning" keeps the default terminal quiet (uvicorn's access log is one line + # per request). It was previously hardcoded, which left deployed proxies with + # no way to turn request logging on: operators diagnosing a production + # incident could not see uvicorn's view of the traffic at all, with no env var + # and no CLI flag to change it. Overridable now; the default is unchanged. + uvicorn_log_level = _resolve_uvicorn_log_level() + uvicorn.run( app_target, host=config.host, port=config.port, - log_level="warning", + log_level=uvicorn_log_level, workers=workers if workers > 1 else None, # None = single process (default) limit_concurrency=limit_concurrency, # Defense-in-depth: the loopback guard for /debug/* endpoints trusts @@ -5441,6 +5556,30 @@ def _get_env_str(name: str, default: str) -> str: return os.environ.get(name, default) +# uvicorn rejects anything outside this set with a KeyError during startup, so an +# operator typo in HEADROOM_LOG_LEVEL must not be able to stop the proxy booting. +_UVICORN_LOG_LEVELS = frozenset({"critical", "error", "warning", "info", "debug", "trace"}) +_UVICORN_LOG_LEVEL_DEFAULT = "warning" + + +def _resolve_uvicorn_log_level() -> str: + """Resolve uvicorn's log level from ``HEADROOM_LOG_LEVEL``. + + Falls back to the previous hardcoded default on an unset or unrecognized + value, warning loudly rather than failing the boot. + """ + raw = _get_env_str("HEADROOM_LOG_LEVEL", _UVICORN_LOG_LEVEL_DEFAULT).strip().lower() + if raw in _UVICORN_LOG_LEVELS: + return raw + logger.warning( + "Ignoring unrecognized HEADROOM_LOG_LEVEL=%r; using %r. Valid values: %s", + raw, + _UVICORN_LOG_LEVEL_DEFAULT, + ", ".join(sorted(_UVICORN_LOG_LEVELS)), + ) + return _UVICORN_LOG_LEVEL_DEFAULT + + def _parse_exclude_tools(cli_excludes: str | None) -> set[str]: """Parse extra never-compress tool names from CLI args and env var. diff --git a/headroom/proxy/upstream_guard.py b/headroom/proxy/upstream_guard.py new file mode 100644 index 000000000..783b6e9ab --- /dev/null +++ b/headroom/proxy/upstream_guard.py @@ -0,0 +1,110 @@ +"""SSRF guard for client-supplied upstream base URLs (WEB-01). + +Clients may redirect the proxy's upstream via the ``x-headroom-base-url`` header +(BYOK / custom OpenAI-compatible endpoints). Without validation this lets a +caller turn the proxy into a confused deputy — reaching cloud-metadata +(``169.254.169.254``) or internal RFC1918 hosts the caller cannot reach directly. + +Policy: + * Default: reject destinations that resolve to private, loopback, link-local, + or otherwise non-public addresses. Public hosts (api.openai.com, api.x.ai, + Azure, ...) are allowed so ordinary BYOK keeps working. + * When ``HEADROOM_ALLOWED_BASE_URLS`` is set (comma-separated hosts or URLs), + bare hosts permit every safe scheme/port for that host, while URLs permit + only their exact normalized origin. Because that is an explicit operator + choice, allowlisted destinations may point at internal/on-prem endpoints. + +This module intentionally depends only on the standard library so it is safe to +import from any handler without risking an import cycle. +""" + +from __future__ import annotations + +import ipaddress +import os +import socket +from urllib.parse import urlparse + +ALLOWED_BASE_URLS_ENV = "HEADROOM_ALLOWED_BASE_URLS" + +_SAFE_SCHEMES = {"http", "https", "ws", "wss"} + + +def _allowlisted_destinations() -> tuple[set[str], set[tuple[str, str, int]]] | None: + raw = os.environ.get(ALLOWED_BASE_URLS_ENV) + if not raw or not raw.strip(): + return None + hosts: set[str] = set() + origins: set[tuple[str, str, int]] = set() + for item in raw.split(","): + item = item.strip() + if not item: + continue + if "://" not in item: + parsed = urlparse(f"//{item}") + if parsed.hostname: + hosts.add(parsed.hostname.lower()) + continue + parsed = urlparse(item) + if parsed.scheme.lower() not in _SAFE_SCHEMES or not parsed.hostname: + continue + try: + port = parsed.port + except ValueError: + continue + if port is None: + port = 443 if parsed.scheme.lower() in {"https", "wss"} else 80 + origins.add((parsed.scheme.lower(), parsed.hostname.lower(), port)) + return hosts, origins + + +def _is_internal_address(ip: str) -> bool: + try: + addr = ipaddress.ip_address(ip) + except ValueError: + return True # unparseable (e.g. scoped link-local) -> treat as unsafe + return ( + addr.is_private + or addr.is_loopback + or addr.is_link_local + or addr.is_reserved + or addr.is_multicast + or addr.is_unspecified + ) + + +def is_safe_upstream_url(url: str) -> bool: + """Return True if ``url`` is a safe client-chosen upstream destination. + + In allowlist mode only allowlisted hosts pass. Otherwise the host is + resolved and rejected if any resolved address is internal/metadata, which + also catches DNS names that point at private space. + """ + parsed = urlparse((url or "").strip()) + if parsed.scheme.lower() not in _SAFE_SCHEMES: + return False + host = parsed.hostname + if not host: + return False + + allow = _allowlisted_destinations() + if allow is not None: + hosts, origins = allow + if host.lower() in hosts: + return True + try: + port = parsed.port + except ValueError: + return False + if port is None: + port = 443 if parsed.scheme.lower() in {"https", "wss"} else 80 + return (parsed.scheme.lower(), host.lower(), port) in origins + + try: + infos = socket.getaddrinfo(host, None, proto=socket.IPPROTO_TCP) + except OSError: + # Resolution and connection are separate operations, so allowing a DNS + # miss here would fail open if the name resolves on the later lookup. + # Operators can explicitly allowlist split-horizon/internal endpoints. + return False + return all(not _is_internal_address(str(info[4][0])) for info in infos) diff --git a/headroom/proxy/upstream_trust.py b/headroom/proxy/upstream_trust.py new file mode 100644 index 000000000..e68ea11b7 --- /dev/null +++ b/headroom/proxy/upstream_trust.py @@ -0,0 +1,150 @@ +"""Which upstreams are allowed to receive the operator's *own* credentials. + +``x-headroom-base-url`` lets a client pick the upstream for a single request, so +OpenAI-compatible gateways (LiteLLM, Azure, self-hosted vLLM) route through the +dedicated handlers instead of the generic passthrough. That is a deliberate +feature and this module does not take it away. + +What it takes away is the credential that used to ride along. ``*_extra_headers`` +is operator-configured, marked ``secret=True`` in the settings store, and its own +help text suggests an API key as the example value. It was merged into the +upstream-bound header set *before* the destination was resolved, so a request +carrying ``X-Headroom-Base-Url: https://attacker.example`` reached the attacker's +host with the operator's gateway key attached — one request, no user interaction, +from anything able to talk to the proxy port. + +The rule here is the one ``copilot_auth.is_copilot_upstream_url`` already applies +to Headroom's own Copilot token, generalized: **a secret only travels to a host +the operator designated.** Designated means one of + +* a host in the resolved provider API targets (``ANTHROPIC_TARGET_API_URL``, + ``OPENAI_TARGET_API_URL``, and the Gemini/Vertex/Cloud Code equivalents), or +* a host listed in ``HEADROOM_UPSTREAM_ALLOWED_HOSTS`` (comma-separated). + +Anything else still gets proxied — the request is not blocked — it just does not +get the operator's headers. + +Matching is on the parsed hostname, never the URL string: comparing whole strings +lets ``https://api.anthropic.com@evil.example`` and ``https://api.anthropic.com.evil.example`` +through, and a base URL matches while base+path does not. Exact hostname equality +only; no wildcards, because a suffix rule that forgets the label boundary is the +usual way this class of check fails open. +""" + +from __future__ import annotations + +import logging +import os +from typing import Any +from urllib.parse import urlparse + +logger = logging.getLogger("headroom.proxy") + +ALLOWED_HOSTS_ENV = "HEADROOM_UPSTREAM_ALLOWED_HOSTS" + +#: Config attributes holding an operator-designated upstream. +_API_URL_ATTRS = ( + "anthropic_api_url", + "openai_api_url", + "gemini_api_url", + "cloudcode_api_url", + "vertex_api_url", + "bedrock_api_url", +) + +# Hosts that are always operator-designated: they are what the provider targets +# resolve to when nothing is overridden, so omitting them would refuse the +# headers on a completely default install. +_DEFAULT_HOSTS = frozenset( + { + "api.anthropic.com", + "api.openai.com", + } +) + +# Warn once per destination rather than once per request; a client looping on a +# rejected host would otherwise flood the log. +_warned_hosts: set[str] = set() + + +def url_host(value: str | None) -> str | None: + """Return the lowercase hostname for ``value``, tolerating a missing scheme. + + ``urlparse("api.example.com/v1").hostname`` is ``None`` — the whole value is + read as a path — so a scheme-less configured URL would otherwise contribute + nothing to the trusted set and silently widen or narrow the check. + """ + + if not value: + return None + candidate = value.strip() + if not candidate: + return None + parsed = urlparse(candidate) + if not parsed.hostname and "//" not in candidate: + parsed = urlparse(f"//{candidate}") + host = parsed.hostname + return host.lower() if host else None + + +def _env_allowed_hosts() -> set[str]: + raw = os.environ.get(ALLOWED_HOSTS_ENV, "") + hosts: set[str] = set() + for entry in raw.split(","): + # Accept a bare host or a full URL, so operators can paste either. + host = url_host(entry) if entry.strip() else None + if host: + hosts.add(host) + return hosts + + +def trusted_upstream_hosts(config: Any = None) -> frozenset[str]: + """Hosts permitted to receive operator-configured secret headers.""" + + hosts = set(_DEFAULT_HOSTS) + for attr in _API_URL_ATTRS: + host = url_host(getattr(config, attr, None)) + if host: + hosts.add(host) + hosts |= _env_allowed_hosts() + return frozenset(hosts) + + +def is_trusted_upstream(url: str | None, config: Any = None) -> bool: + """True when ``url`` is a destination the operator designated. + + ``None``/empty means "no per-request override" — the handler is going to the + configured target — so it is trusted. + """ + + if not url: + return True + host = url_host(url) + if not host: + # Unparseable destination: refuse rather than guess. + return False + return host in trusted_upstream_hosts(config) + + +def warn_untrusted_once(url: str | None, *, request_id: str | None = None) -> None: + """Log the refusal once per host, with the remedy in the message.""" + + host = url_host(url) or "" + if host in _warned_hosts: + return + _warned_hosts.add(host) + prefix = f"[{request_id}] " if request_id else "" + logger.warning( + "%supstream_extra_headers_withheld host=%s reason=not_operator_designated. " + "The configured extra headers are secret and were NOT sent to this host. " + "If this upstream is legitimate, add it to %s (comma-separated hosts).", + prefix, + host, + ALLOWED_HOSTS_ENV, + ) + + +def reset_warning_state() -> None: + """Test hook: clear the once-per-host warning memo.""" + + _warned_hosts.clear() diff --git a/headroom/testing/README.md b/headroom/testing/README.md index 7b9ee7f99..37c398669 100644 --- a/headroom/testing/README.md +++ b/headroom/testing/README.md @@ -33,7 +33,11 @@ assert report.passed suite = ( Headroom.Suite("phase-1") .Add(Headroom.WithOpenAI().named("openai-cache").WithCompression(mode="cache")) - .Add(Headroom.WithBedrock(region="us-east-1").named("bedrock-token").WithCompression(mode="token")) + .Add( + Headroom.WithBedrock(region="us-east-1") + .named("bedrock-token") + .WithCompression(mode="token") + ) ) suite.write_manifest_bundle("headroom-testing-bundle.json", provider="openai", port_start=19000) diff --git a/headroom/tools.json b/headroom/tools.json index 1c7c0e90c..de5affb36 100644 --- a/headroom/tools.json +++ b/headroom/tools.json @@ -1,5 +1,5 @@ { - "_comment": "Registry of externally fetched CLI tool binaries. Bump versions and SHA256s via the weekly tools-version-check CI job (see .github/workflows/). sha256=null means HTTPS-trust-only (initial bootstrap); the CI job fills real SHAs per release.", + "_comment": "Registry of externally fetched CLI tool binaries. SHA-256 pins are enforced before execution (headroom/binaries.py). After bumping a version, regenerate pins with scripts/refresh_tool_hashes.py; the tools-hash-refresh CI workflow verifies they match the published assets.", "tools": { "difft": { "version": "0.64.0", @@ -11,39 +11,39 @@ "linux-x86_64-gnu": { "url": "https://github.com/Wilfred/difftastic/releases/download/0.64.0/difft-x86_64-unknown-linux-gnu.tar.gz", "member": "difft", - "sha256": null + "sha256": "9ec3aa9f784a54c8099d1af71b6bc18d3ad12ce50055565ffa2979aff93c5732" }, "linux-x86_64-musl": { "url": "https://github.com/Wilfred/difftastic/releases/download/0.64.0/difft-x86_64-unknown-linux-gnu.tar.gz", "member": "difft", - "sha256": null, + "sha256": "9ec3aa9f784a54c8099d1af71b6bc18d3ad12ce50055565ffa2979aff93c5732", "_comment": "same asset as linux-x86_64-gnu; upstream has no dedicated musl build" }, "linux-aarch64-gnu": { "url": "https://github.com/Wilfred/difftastic/releases/download/0.64.0/difft-aarch64-unknown-linux-gnu.tar.gz", "member": "difft", - "sha256": null + "sha256": "f657b0cc1baba3b5bcde4fdfbafbc58f2131476ed53d01e4880898d34a1cb124" }, "linux-aarch64-musl": { "url": "https://github.com/Wilfred/difftastic/releases/download/0.64.0/difft-aarch64-unknown-linux-gnu.tar.gz", "member": "difft", - "sha256": null, + "sha256": "f657b0cc1baba3b5bcde4fdfbafbc58f2131476ed53d01e4880898d34a1cb124", "_comment": "same asset as linux-aarch64-gnu" }, "darwin-x86_64": { "url": "https://github.com/Wilfred/difftastic/releases/download/0.64.0/difft-x86_64-apple-darwin.tar.gz", "member": "difft", - "sha256": null + "sha256": "d6b7f1c4a66495400f1f6e224efdaf82aa66e8930fcb257bc9357d69d52dd69d" }, "darwin-aarch64": { "url": "https://github.com/Wilfred/difftastic/releases/download/0.64.0/difft-aarch64-apple-darwin.tar.gz", "member": "difft", - "sha256": null + "sha256": "e1e59ff3cf6c0837c94ddd4910d43351ec14f8c169abcfa51e63136d14c9e528" }, "windows-x86_64": { "url": "https://github.com/Wilfred/difftastic/releases/download/0.64.0/difft-x86_64-pc-windows-msvc.zip", "member": "difft.exe", - "sha256": null + "sha256": "1ac0113adf9ade9417ee133bed38efe3085b7ccbb32ab620bdc36c4a9b365276" } } }, @@ -52,42 +52,42 @@ "binary": "scc", "source": "boyter/scc", "homepage": "https://github.com/boyter/scc", - "_comment": "scc is a statically-linked Go binary, so the same asset is used for both libc variants. The duplicated gnu/musl entries are intentional — they document musl support for `headroom tools doctor`.", + "_comment": "scc is a statically-linked Go binary, so the same asset is used for both libc variants. The duplicated gnu/musl entries are intentional \u2014 they document musl support for `headroom tools doctor`.", "assets": { "linux-x86_64-gnu": { "url": "https://github.com/boyter/scc/releases/download/v3.5.0/scc_Linux_x86_64.tar.gz", "member": "scc", - "sha256": null + "sha256": "6c31f4d0cf3b7a8c5ca910fa4e451949434798f6541ec5dea4b83f4973e13772" }, "linux-x86_64-musl": { "url": "https://github.com/boyter/scc/releases/download/v3.5.0/scc_Linux_x86_64.tar.gz", "member": "scc", - "sha256": null + "sha256": "6c31f4d0cf3b7a8c5ca910fa4e451949434798f6541ec5dea4b83f4973e13772" }, "linux-aarch64-gnu": { "url": "https://github.com/boyter/scc/releases/download/v3.5.0/scc_Linux_arm64.tar.gz", "member": "scc", - "sha256": null + "sha256": "64446b1ca954aa1ac34984bbb4f098f46e6c69c84d64a7d096275ea6e50461eb" }, "linux-aarch64-musl": { "url": "https://github.com/boyter/scc/releases/download/v3.5.0/scc_Linux_arm64.tar.gz", "member": "scc", - "sha256": null + "sha256": "64446b1ca954aa1ac34984bbb4f098f46e6c69c84d64a7d096275ea6e50461eb" }, "darwin-x86_64": { "url": "https://github.com/boyter/scc/releases/download/v3.5.0/scc_Darwin_x86_64.tar.gz", "member": "scc", - "sha256": null + "sha256": "33fee1db983a6d22297d5f4e41bca25438c8a09d5c7c558acbfb5f9cd4deb19b" }, "darwin-aarch64": { "url": "https://github.com/boyter/scc/releases/download/v3.5.0/scc_Darwin_arm64.tar.gz", "member": "scc", - "sha256": null + "sha256": "0e967d77ad564fa77bc5ed157d4e04bc6b56c5619edef317e5c99b15c9fca9d2" }, "windows-x86_64": { "url": "https://github.com/boyter/scc/releases/download/v3.5.0/scc_Windows_x86_64.zip", "member": "scc.exe", - "sha256": null + "sha256": "9ce15936440f1680bd133905c9f6c99a5d258bcea6f73fc372c3f8b4b888200f" } } }, @@ -98,6 +98,34 @@ "homepage": "https://ast-grep.github.io/", "_comment": "Installed via the ast-grep-cli PyPI wheel; we never fetch from GitHub for this tool. Listed here so `headroom tools doctor` can report it.", "assets": {} + }, + "codebase-memory-mcp": { + "version": "v0.8.1", + "binary": "codebase-memory-mcp", + "source": "DeusData/codebase-memory-mcp", + "_comment": "No Windows build published for v0.8.1.", + "assets": { + "darwin-arm64": { + "url": "https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.8.1/codebase-memory-mcp-darwin-arm64.tar.gz", + "member": "codebase-memory-mcp", + "sha256": "fbd047509852021b5446a11141bcb0a3d1dcaebf6e5112460960f29f052c1c58" + }, + "darwin-amd64": { + "url": "https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.8.1/codebase-memory-mcp-darwin-amd64.tar.gz", + "member": "codebase-memory-mcp", + "sha256": "fb62da3016ea12b948351208759b5c083fb1446cf6e78d6db8b7cd28fe86fd54" + }, + "linux-arm64": { + "url": "https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.8.1/codebase-memory-mcp-linux-arm64.tar.gz", + "member": "codebase-memory-mcp", + "sha256": "d2f842d1365da5c35d9c5796f57a821c9745267350994346735e1e6e04d46091" + }, + "linux-amd64": { + "url": "https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.8.1/codebase-memory-mcp-linux-amd64.tar.gz", + "member": "codebase-memory-mcp", + "sha256": "dbd3b92ea870ef240b63059f26bda15015f76ef9978931bebc3a0f9d09470973" + } + } } } } diff --git a/headroom/transforms/compression_policy.py b/headroom/transforms/compression_policy.py index faf54fa60..241db44d8 100644 --- a/headroom/transforms/compression_policy.py +++ b/headroom/transforms/compression_policy.py @@ -58,21 +58,35 @@ _MAX_LOSSY_RATIO_SUBSCRIPTION: float = 0.25 #: Anthropic prompt-cache write multiplier: a ``cache_creation`` token #: costs 1.25x a plain input token (5-minute TTL tier). Input to the #: net-cost mutation formula (#856). Mirrors the Rust ``pub const``. -#: ponytail: hardcoded to the 5m tier. A client on Anthropic's 1h cache -#: (ENABLE_PROMPT_CACHING_1H / cache_control.ttl="1h", which Headroom -#: preserves) writes at 2.0x, so its mutations are gated with a ~40% -#: under-stated write penalty. Harmless while the net-cost gate stays -#: default-off (HEADROOM_NET_COST_POLICY); thread the TTL from -#: cold_prefix.anthropic_cache_ttl_seconds through ContentRouter -> -#: net_mutation_gain if that gate is ever turned on. CACHE_WRITE_MULTIPLIER: float = 1.25 +#: Anthropic prompt-cache write multiplier for the 1-hour TTL tier. +CACHE_WRITE_MULTIPLIER_1H: float = 2.0 + #: Anthropic prompt-cache read multiplier: a ``cache_read`` token costs #: 0.1x a plain input token. Input to the net-cost mutation formula #: (#856). Mirrors the Rust ``pub const``. CACHE_READ_MULTIPLIER: float = 0.1 +def cache_write_multiplier_for_ttl(ttl_seconds: float | int | None) -> float: + """Return the cache-write multiplier for a prompt-cache TTL tier. + + The net-cost gate prefers an authoritative request-level TTL and falls + back to its environment setting when no request TTL is available. + Invalid and non-positive values retain the 5-minute default. + """ + if ttl_seconds is None: + return CACHE_WRITE_MULTIPLIER + try: + ttl = float(ttl_seconds) + except (TypeError, ValueError): + return CACHE_WRITE_MULTIPLIER + if not math.isfinite(ttl) or ttl <= 0.0: + return CACHE_WRITE_MULTIPLIER + return CACHE_WRITE_MULTIPLIER_1H if ttl >= 3600.0 else CACHE_WRITE_MULTIPLIER + + @dataclass(frozen=True, slots=True) class CompressionPolicy: """Per-auth-mode policy that downstream compression stages consult. @@ -136,6 +150,8 @@ class CompressionPolicy: suffix_tokens: int, expected_reads: float, p_alive: float, + *, + write_multiplier: float | None = None, ) -> float: """Net gain (in plain-input-token cost units) of a mutation that removes ``delta_t`` tokens from a message whose cached suffix is @@ -156,7 +172,7 @@ class CompressionPolicy: ``>= 0`` (NaN → 0), ``p_alive`` to ``[0, 1]`` (NaN → 1, the conservative full-penalty assumption — same as Rust). """ - w = CACHE_WRITE_MULTIPLIER + w = CACHE_WRITE_MULTIPLIER if write_multiplier is None else write_multiplier r = CACHE_READ_MULTIPLIER dt = max(0, delta_t) suffix = max(0, suffix_tokens) @@ -172,12 +188,29 @@ class CompressionPolicy: suffix_tokens: int, expected_reads: float, p_alive: float, + *, + write_multiplier: float | None = None, ) -> bool: """Decision form of :meth:`net_mutation_gain`: mutate iff the gain is strictly positive.""" - return self.net_mutation_gain(delta_t, suffix_tokens, expected_reads, p_alive) > 0.0 + return ( + self.net_mutation_gain( + delta_t, + suffix_tokens, + expected_reads, + p_alive, + write_multiplier=write_multiplier, + ) + > 0.0 + ) - def break_even_reads(self, delta_t: int, suffix_tokens: int) -> float: + def break_even_reads( + self, + delta_t: int, + suffix_tokens: int, + *, + write_multiplier: float | None = None, + ) -> float: """Remaining-read count at which a warm-cache (``p_alive=1``) mutation breaks even:: @@ -192,7 +225,7 @@ class CompressionPolicy: """ if delta_t <= 0: return 0.0 - w = CACHE_WRITE_MULTIPLIER + w = CACHE_WRITE_MULTIPLIER if write_multiplier is None else write_multiplier r = CACHE_READ_MULTIPLIER return ((w - r) / r) * (float(max(0, suffix_tokens)) / float(delta_t)) diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py index 0ba64cb17..a77446149 100644 --- a/headroom/transforms/content_router.py +++ b/headroom/transforms/content_router.py @@ -65,6 +65,7 @@ from ..tokenizers.base import count_content_blocks from ..tokenizers.estimator import EstimatingTokenCounter from . import mixed_content as _mixed_content from .base import Transform +from .compression_policy import cache_write_multiplier_for_ttl from .compressor_registry import ( CompressInput, CompressorDescriptor, @@ -1856,7 +1857,11 @@ class ContentRouter(Transform): ).strip().lower() in ("1", "true", "yes", "on") self._text_crusher: Any = None # Cross-turn dedup: config field OR env HEADROOM_DEDUPE (robust to how the - # config was built). Effective only in lossless mode (guarded in apply()). + # config was built). Runs in BOTH modes — the call site in ``apply()`` has + # no lossless guard, and ``_cross_turn_dedup_messages`` documents working + # against lossless folds and CCR-recoverable forms alike. (This comment + # previously claimed "lossless mode only", which reads as "inert in your + # config" to anyone auditing why dedup never fired.) self._cross_turn_dedup_enabled: bool = ( self.config.enable_cross_turn_dedup or os.environ.get("HEADROOM_DEDUPE", "").strip().lower() in ("1", "true", "yes", "on") @@ -4529,6 +4534,7 @@ class ContentRouter(Transform): transforms_applied: list[str], batch_state: dict[str, int | None] | None = None, p_alive_override: float | None = None, + write_multiplier: float | None = None, ) -> bool: """Break-even gate for one candidate mutation (#856 P2, flag-gated). @@ -4613,7 +4619,15 @@ class ContentRouter(Transform): p_alive = _p_alive except ValueError: logger.warning("HEADROOM_NET_COST_P_ALIVE malformed; using 1.0") - gain = float(policy.net_mutation_gain(delta_t, suffix, reads, p_alive)) + gain = float( + policy.net_mutation_gain( + delta_t, + suffix, + reads, + p_alive, + write_multiplier=write_multiplier, + ) + ) allowed = gain > 0.0 logger.info( "NetCostPolicy slot=%d delta_t=%d suffix=%d reads=%.1f p_alive=%.2f " @@ -4976,7 +4990,22 @@ class ContentRouter(Transform): # env-constant behaviour. Derived once here (not per slot) — idle is a # per-request property, like frozen_message_count. netcost_p_alive_override: float | None = None + netcost_write_multiplier: float | None = None if netcost_enabled: + # Prefer the authoritative per-request prompt-cache TTL when the + # caller has one; retain the env setting for other providers and + # legacy callers. + request_ttl = kwargs.get("cache_ttl_seconds") + if request_ttl is None: + netcost_ttl = _net_cost_cache_ttl_seconds() + else: + try: + netcost_ttl = float(request_ttl) + except (TypeError, ValueError): + netcost_ttl = _net_cost_cache_ttl_seconds() + if not math.isfinite(netcost_ttl) or netcost_ttl <= 0.0: + netcost_ttl = _net_cost_cache_ttl_seconds() + netcost_write_multiplier = cache_write_multiplier_for_ttl(netcost_ttl) netcost_suffix_tokens = [0] * (num_messages + 1) for j in range(num_messages - 1, -1, -1): netcost_suffix_tokens[j] = netcost_suffix_tokens[j + 1] + _netcost_message_tokens( @@ -4989,8 +5018,7 @@ class ContentRouter(Transform): except (TypeError, ValueError): idle_f = None if idle_f is not None and math.isfinite(idle_f) and idle_f >= 0.0: - ttl = _net_cost_cache_ttl_seconds() - netcost_p_alive_override = max(0.0, 1.0 - idle_f / ttl) + netcost_p_alive_override = max(0.0, 1.0 - idle_f / netcost_ttl) # Tasks: list of (slot_index, content, context, bias, content_key) _PendingTask = tuple[int, str, str, float, int, bool] @@ -5294,6 +5322,7 @@ class ContentRouter(Transform): transforms_applied=transforms_applied, batch_state=netcost_batch_state, p_alive_override=netcost_p_alive_override, + write_multiplier=netcost_write_multiplier, ): # Net-cost gate: mutation would cost more in cache # invalidation than it saves — leave untouched. @@ -5471,6 +5500,7 @@ class ContentRouter(Transform): transforms_applied=transforms_applied, batch_state=netcost_batch_state, p_alive_override=netcost_p_alive_override, + write_multiplier=netcost_write_multiplier, ): result_slots[slot_idx] = message continue @@ -5526,7 +5556,16 @@ class ContentRouter(Transform): if route_counts["user_msg"]: parts.append(f"{route_counts['user_msg']} skipped (user)") if route_counts["small"]: - parts.append(f"{route_counts['small']} skipped (<50 words)") + # Report the thresholds actually in force, not a literal. This line + # used to read "skipped (<50 words)" unconditionally: wrong number + # (the message gate is `min_tokens`, which profiles set anywhere from + # 10 to 250), wrong unit (tokens and characters, never words), and it + # merged two different gates under one label. Operators read it as + # evidence of a mis-set threshold and tuned the wrong knob. + parts.append( + f"{route_counts['small']} skipped " + f"(<{min_tokens} tok msg / <{min_chars_for_block_compression} chars block)" + ) if route_counts["recent_code"]: parts.append(f"{route_counts['recent_code']} protected (recent code)") if route_counts["analysis_ctx"]: diff --git a/plugins/headroom-agent-hooks/.claude-plugin/plugin.json b/plugins/headroom-agent-hooks/.claude-plugin/plugin.json index 4257625a4..0a2ec2d15 100644 --- a/plugins/headroom-agent-hooks/.claude-plugin/plugin.json +++ b/plugins/headroom-agent-hooks/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "headroom", - "version": "0.35.0", + "version": "0.36.0", "description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.", "author": { "name": "Headroom Contributors", diff --git a/plugins/headroom-agent-hooks/.github/plugin/plugin.json b/plugins/headroom-agent-hooks/.github/plugin/plugin.json index 974363f1d..84de5902b 100644 --- a/plugins/headroom-agent-hooks/.github/plugin/plugin.json +++ b/plugins/headroom-agent-hooks/.github/plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "headroom", - "version": "0.35.0", + "version": "0.36.0", "description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.", "author": { "name": "Headroom Contributors", diff --git a/plugins/openclaw/package.json b/plugins/openclaw/package.json index 77f26d5c0..5b41597d8 100644 --- a/plugins/openclaw/package.json +++ b/plugins/openclaw/package.json @@ -1,6 +1,6 @@ { "name": "headroom-openclaw", - "version": "0.35.0", + "version": "0.36.0", "description": "Headroom context compression plugin for OpenClaw — 70-90% token savings with zero LLM calls", "type": "module", "main": "./dist/index.js", diff --git a/plugins/opencode/package.json b/plugins/opencode/package.json index 355f9f589..19f18744b 100644 --- a/plugins/opencode/package.json +++ b/plugins/opencode/package.json @@ -1,6 +1,6 @@ { "name": "headroom-opencode", - "version": "0.35.0", + "version": "0.36.0", "description": "Headroom proxy integration plugin for OpenCode - routes LLM traffic through the Headroom proxy for token compression", "type": "module", "main": "./dist/index.js", diff --git a/plugins/opencode/src/plugin.ts b/plugins/opencode/src/plugin.ts index cd3813614..e9d62fa3f 100644 --- a/plugins/opencode/src/plugin.ts +++ b/plugins/opencode/src/plugin.ts @@ -28,9 +28,14 @@ function resolveProxyUrl(options?: HeadroomOpenCodePluginOptions): string { export const HeadroomPlugin: Plugin = async (input, options = {}) => { const pluginOptions = options as HeadroomOpenCodePluginOptions; const proxyUrl = resolveProxyUrl(pluginOptions); + const project = + pluginOptions.project ?? + (input.project as { id?: string } | undefined)?.id ?? + input.directory; const retrieveTool = createHeadroomRetrieveTool({ proxyBaseUrl: proxyUrl }); const uninstallTransport = installHeadroomTransport({ proxyUrl, + project, debug: pluginOptions.debug, }); @@ -54,10 +59,7 @@ export const HeadroomPlugin: Plugin = async (input, options = {}) => { "shell.env": async (_input, output) => { output.env.HEADROOM_ACTIVE = "1"; output.env.HEADROOM_PROXY_URL = proxyUrl; - output.env.HEADROOM_PROJECT = - pluginOptions.project ?? - (input.project as { id?: string }).id ?? - input.directory; + output.env.HEADROOM_PROJECT = project; if (pluginOptions.backend) { output.env.HEADROOM_BACKEND = pluginOptions.backend; } diff --git a/plugins/opencode/src/transport.test.ts b/plugins/opencode/src/transport.test.ts index 39edd98de..f6b2cfff4 100644 --- a/plugins/opencode/src/transport.test.ts +++ b/plugins/opencode/src/transport.test.ts @@ -386,6 +386,58 @@ describe("Headroom OpenCode transport", () => { } }); + it("sends x-headroom-project header on routed fetch calls when project is set", async () => { + const originalFetch = globalThis.fetch; + const fetchMock = vi.fn(async (..._args: FetchCall) => new Response("ok")); + globalThis.fetch = fetchMock as unknown as typeof fetch; + + installHeadroomTransport({ proxyUrl: "http://127.0.0.1:8787/v1", project: "my-project" }); + + await fetch("https://api.anthropic.com/v1/messages", { method: "POST" }); + + const headers = new Headers(fetchMock.mock.calls[0][1]?.headers); + expect(headers.get("x-headroom-project")).toBe("my-project"); + + globalThis.fetch = originalFetch; + }); + + it("omits x-headroom-project header when project is not set", async () => { + const originalFetch = globalThis.fetch; + const fetchMock = vi.fn(async (..._args: FetchCall) => new Response("ok")); + globalThis.fetch = fetchMock as unknown as typeof fetch; + + installHeadroomTransport({ proxyUrl: "http://127.0.0.1:8787/v1" }); + + await fetch("https://api.anthropic.com/v1/messages", { method: "POST" }); + + const headers = new Headers(fetchMock.mock.calls[0][1]?.headers); + expect(headers.get("x-headroom-project")).toBeNull(); + + globalThis.fetch = originalFetch; + }); + + it("sends x-headroom-project header on routed Node https.request calls when project is set", async () => { + const proxy = await proxyServer(); + installHeadroomTransport({ proxyUrl: proxy.url, project: "my-project" }); + + await new Promise((resolve, reject) => { + const req = https.request( + "https://api.anthropic.com/v1/messages", + { method: "POST" }, + (res) => { + res.resume(); + res.on("end", resolve); + }, + ); + req.on("error", reject); + req.end("{}"); + }); + + expect(proxy.seen[0].headers["x-headroom-project"]).toBe("my-project"); + + await proxy.close(); + }); + it("restores patched transports only after the final disposer", () => { const originalFetch = globalThis.fetch; const originalHttpRequest = http.request; diff --git a/plugins/opencode/src/transport.ts b/plugins/opencode/src/transport.ts index b15d5f67d..b4f768eec 100644 --- a/plugins/opencode/src/transport.ts +++ b/plugins/opencode/src/transport.ts @@ -9,6 +9,7 @@ const fs = nodeRequire("node:fs") as typeof import("node:fs"); const BASE_URL_HEADER = "x-headroom-base-url"; const ORIGINAL_PATH_HEADER = "x-headroom-original-path"; +const PROJECT_HEADER = "x-headroom-project"; const PROXY_ENV = "HEADROOM_OPENCODE_TRANSPORT_PROXY_URL"; const STATE_KEY = Symbol.for("headroom.opencode.transport"); @@ -25,12 +26,14 @@ type ChildFork = typeof childProcess.fork; interface InstallOptions { proxyUrl: string; + project?: string; debug?: boolean; } interface TransportState { refs: number; proxyUrl: string; + project: string | undefined; debug: boolean; originalFetch: typeof fetch; originalHttpRequest: HttpRequest; @@ -233,6 +236,7 @@ function mergeFetchHeaders( init: RequestInit | undefined, upstream: URL | undefined, originalPath: string | undefined = undefined, + project: string | undefined = undefined, ): Headers { const headers = new Headers(input instanceof Request ? input.headers : undefined); if (init?.headers) { @@ -245,10 +249,13 @@ function mergeFetchHeaders( if (originalPath) { headers.set(ORIGINAL_PATH_HEADER, originalPath); } + if (project) { + headers.set(PROJECT_HEADER, project); + } return headers; } -function withRoutedFetchInput(input: RequestInfo | URL, init: RequestInit | undefined, proxy: URL): FetchArgs { +function withRoutedFetchInput(input: RequestInfo | URL, init: RequestInit | undefined, proxy: URL, project: string | undefined): FetchArgs { const upstream = requestUrl(input); if (!shouldRoute(upstream, proxy)) { return [input, init]; @@ -257,7 +264,7 @@ function withRoutedFetchInput(input: RequestInfo | URL, init: RequestInit | unde const { url: nextUrl, originalPath } = routedUrlForOpenCode(upstream, proxy); const nextInit = { ...init, - headers: mergeFetchHeaders(input, init, upstream, originalPath), + headers: mergeFetchHeaders(input, init, upstream, originalPath, project), }; if (input instanceof Request) { @@ -314,12 +321,16 @@ function headersForNodeRequest( options: Record, upstream: URL, originalPath: string | undefined, + project: string | undefined, ): Record { const headers = new Headers(options.headers as HeadersInit | undefined); headers.set(BASE_URL_HEADER, upstream.origin); if (originalPath) { headers.set(ORIGINAL_PATH_HEADER, originalPath); } + if (project) { + headers.set(PROJECT_HEADER, project); + } headers.delete("host"); const result: Record = {}; @@ -329,7 +340,7 @@ function headersForNodeRequest( return result; } -function routedNodeOptions(parts: NodeRequestParts, proxy: URL): Record | undefined { +function routedNodeOptions(parts: NodeRequestParts, proxy: URL, project: string | undefined): Record | undefined { if (!parts.url || !shouldRoute(parts.url, proxy)) { return undefined; } @@ -362,7 +373,7 @@ function routedNodeOptions(parts: NodeRequestParts, proxy: URL): Record void { if (existing) { existing.refs += 1; existing.proxyUrl = options.proxyUrl; + existing.project = options.project; existing.debug = Boolean(options.debug); installProcessEnv(options.proxyUrl); return () => uninstallHeadroomTransport(); @@ -428,6 +440,7 @@ export function installHeadroomTransport(options: InstallOptions): () => void { const state: TransportState = { refs: 1, proxyUrl: options.proxyUrl, + project: options.project, debug: Boolean(options.debug), originalFetch: globalThis.fetch, originalHttpRequest: http.request, @@ -449,7 +462,7 @@ export function installHeadroomTransport(options: InstallOptions): () => void { return state.originalFetch(...args); } const proxy = normalizeProxyUrl(current.proxyUrl); - const [nextInput, nextInit] = withRoutedFetchInput(args[0], args[1], proxy); + const [nextInput, nextInit] = withRoutedFetchInput(args[0], args[1], proxy, current.project); return state.originalFetch(nextInput, nextInit); }; diff --git a/pyproject.toml b/pyproject.toml index e36dca4ae..f92a2f1a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "headroom-ai" -version = "0.35.0" +version = "0.36.0" description = "The Context Optimization Layer for LLM Applications - Cut costs by 50-90%" readme = "README.md" license = "Apache-2.0" @@ -246,12 +246,12 @@ voice = [ # Voice training (includes voice deps + training extras) voice-train = [ "headroom-ai[voice]", - "datasets>=2.14.0", + "datasets>=5.0.1", "accelerate>=0.20.0", ] # Evaluation framework evals = [ - "datasets>=2.14.0", + "datasets>=5.0.1", "sentence-transformers>=2.2.0,<6.0; sys_platform != 'darwin' or platform_machine != 'x86_64'", "numpy>=1.24.0", "scikit-learn>=1.3.0", @@ -277,7 +277,7 @@ dev = [ "pytest>=7.0.0", "pytest-cov>=4.0.0", "pytest-asyncio>=0.21.0", - "ruff==0.15.22", + "ruff==0.16.2", "mypy>=1.0.0", "pre-commit>=3.0.0", "openai>=1.0.0", @@ -360,8 +360,11 @@ constraint-dependencies = [ "pygments>=2.20.0", # GHSA-4xgf-cpjx-pc3j (Medium) — transitive via mcp; fix at 2.14.2 "pydantic-settings>=2.14.2", - # GHSA-mv93-w799-cj2w + 4 others (High) — transitive via agno; fix at 3.1.50 - "gitpython>=3.1.50", + # GHSA-hmq2-w58f-27jc, GHSA-jm78-9fvv-mhgr, GHSA-wvpp-8hx9-p66j (High), + # GHSA-hh9p-6wh2-4mfc (Medium) + earlier ones — transitive via agno. + # 3.1.58 clears every GitPython advisory published to date; the previous + # 3.1.50 floor resolved to 3.1.54, which nine open advisories still cover. + "gitpython>=3.1.58", # GHSA-f4xh-w4cj-qxq8 (High) — transitive via langchain-core; fix at 0.8.18 "langsmith>=0.9.0", # CVE-2026-49825 (High, XSS) — transitive via lxml[html-clean]; fix at 0.4.5 diff --git a/scripts/install.ps1 b/scripts/install.ps1 index b7444fb2a..9926690de 100644 --- a/scripts/install.ps1 +++ b/scripts/install.ps1 @@ -22,14 +22,38 @@ function Require-Command { function Ensure-PathEntry { param([string]$PathEntry) - $currentPath = [Environment]::GetEnvironmentVariable('Path', 'User') + # Persist to the User PATH by default. The 'User' scope lives in + # HKCU\Environment and is NOT redirected by a HOME/USERPROFILE override, so a + # caller that must not mutate the real persistent PATH (the installer test + # suite, which runs this against a throwaway fake home) sets + # HEADROOM_INSTALL_PATH_SCOPE=Process to keep the update ephemeral instead of + # leaking the temp shim dir into the developer's actual user PATH (#2970). + # + # Only those two persistence modes are supported. The value is handed to + # .NET's EnvironmentVariableTarget, whose 'Machine' member would rewrite the + # SYSTEM-wide PATH if this variable were inherited by an elevated installer, + # and a typo would otherwise fail late with an opaque enum-conversion error. + # Normalize case-insensitively and allow-list 'User'/'Process', failing early + # and clearly for 'Machine' or anything else. + $scope = 'User' + if ($env:HEADROOM_INSTALL_PATH_SCOPE) { + switch ($env:HEADROOM_INSTALL_PATH_SCOPE.Trim().ToLowerInvariant()) { + 'user' { $scope = 'User' } + 'process' { $scope = 'Process' } + default { + throw "HEADROOM_INSTALL_PATH_SCOPE must be 'User' or 'Process' (got '$($env:HEADROOM_INSTALL_PATH_SCOPE)'); 'Machine' and other targets are not supported." + } + } + } + + $currentPath = [Environment]::GetEnvironmentVariable('Path', $scope) $parts = @() if ($currentPath) { $parts = $currentPath -split ';' | Where-Object { $_ } } if ($parts -notcontains $PathEntry) { $newPath = @($PathEntry) + $parts - [Environment]::SetEnvironmentVariable('Path', ($newPath -join ';'), 'User') + [Environment]::SetEnvironmentVariable('Path', ($newPath -join ';'), $scope) } } diff --git a/scripts/pr-governance.py b/scripts/pr-governance.py index 8c5165e39..313be118f 100644 --- a/scripts/pr-governance.py +++ b/scripts/pr-governance.py @@ -41,6 +41,28 @@ ROLLOUT_FIELDS = ( "Rollback path", ) +# Conventional-commit types accepted by .commitlintrc.json. Keep the two in +# sync: commitlint gates the *commits* on a PR, but the repo squash-merges, so +# it is the PR *title* that becomes the subject line on main. +COMMIT_TYPES = ( + "build", + "chore", + "ci", + "deps", + "docs", + "feat", + "fix", + "parity", + "perf", + "refactor", + "revert", + "style", + "test", +) + +# type(optional-scope)!: subject +TITLE_RE = re.compile(rf"^(?:{'|'.join(COMMIT_TYPES)})(?:\([^)]+\))?!?: .+") + SECTION_RE = re.compile(r"^##\s+(.+?)\s*$", re.MULTILINE) CHECKBOX_RE = re.compile(r"^- \[(?P[ xX])\] (?P