diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 6f96ec33d..f2743b3d8 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -5,14 +5,14 @@ }, "metadata": { "description": "Headroom marketplace for Claude Code and GitHub Copilot CLI plugins.", - "version": "0.34.0" + "version": "0.35.0" }, "plugins": [ { "name": "headroom", "source": "./plugins/headroom-agent-hooks", "description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.", - "version": "0.34.0", + "version": "0.35.0", "author": { "name": "Headroom Contributors", "url": "https://github.com/chopratejas/headroom" diff --git a/.commitlintrc.json b/.commitlintrc.json index d42e91489..4bc3e4519 100644 --- a/.commitlintrc.json +++ b/.commitlintrc.json @@ -1,7 +1,7 @@ { "extends": ["@commitlint/config-conventional"], "rules": { - "body-max-line-length": [2, "always", 200], + "body-max-line-length": [0], "footer-leading-blank": [0], "subject-case": [0], "type-enum": [ @@ -12,6 +12,7 @@ "chore", "ci", "docs", + "deps", "feat", "fix", "parity", @@ -23,4 +24,4 @@ ] ] } -} \ No newline at end of file +} diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 2014d8dc8..3dd7ca601 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -40,6 +40,16 @@ Closes # - Observed result: - Not tested: +## Runtime Rollout Safety + +- Rollout-managed feature(s): +- Minimum rollout channel: +- Stable/default behavior changed: +- Kill switch / disable path: +- Unsafe override required: +- Qualification impact: +- Rollback path: + ## Review Readiness - [ ] I have performed a self-review diff --git a/.github/act/pr-governance-valid.json b/.github/act/pr-governance-valid.json index 8d1b07d16..a38b4b005 100644 --- a/.github/act/pr-governance-valid.json +++ b/.github/act/pr-governance-valid.json @@ -1,19 +1,13 @@ -{ - "action": "ready_for_review", - "number": 42, - "pull_request": { - "number": 42, - "draft": false, - "title": "feat: add PR governance", - "body": "## Description\n\nAdd a required PR governance check and commit-msg enforcement.\n\nCloses #123\n\n## Type of Change\n\n- [x] New feature (non-breaking change that adds functionality)\n\n## Changes Made\n\n- Added workflow validation for PR template completeness.\n- Added a commit-msg hook that runs commitlint locally.\n\n## Testing\n\n- [x] Unit tests pass (`pytest`)\n- [x] Manual testing performed\n\n### Test Output\n\n```text\npytest scripts/tests/test_pr_governance.py -q\n```\n\n## Real Behavior Proof\n\n- Environment: Ubuntu runner, Python 3.12\n- Exact command / steps: Opened a PR with an incomplete template, then fixed the body.\n- Observed result: The governance check failed until the template and readiness boxes were complete.\n- Not tested: Repository-level automatic Copilot rulesets.\n\n## Review Readiness\n\n- [x] I have performed a self-review\n- [x] This PR is ready for human review\n", - "user": { - "login": "octocat" - }, - "base": { - "sha": "dff6a199" - } - }, - "repository": { - "full_name": "JerrettDavis/headroom" - } -} +{ + "action": "ready_for_review", + "number": 42, + "pull_request": { + "number": 42, + "draft": false, + "title": "feat: add PR governance", + "body": "## Description\n\nAdd a required PR governance check and commit-msg enforcement.\n\n## Type of Change\n\n- [x] New feature (non-breaking change that adds functionality)\n\n## Changes Made\n\n- Added workflow validation for PR template completeness.\n\n## Testing\n\n- [x] Unit tests pass (`pytest`)\n\n### Test Output\n\n```text\npytest scripts/tests/test_pr_governance.py -q\n```\n\n## Real Behavior Proof\n\n- Environment: Ubuntu runner, Python 3.12\n- Exact command / steps: Opened a PR and ran governance.\n- Observed result: The check passed with complete facts.\n- Not tested: Repository settings.\n\n## Runtime Rollout Safety\n\n- Rollout-managed feature(s): None.\n- Minimum rollout channel: Stable.\n- Stable/default behavior changed: No.\n- Kill switch / disable path: Not applicable.\n- Unsafe override required: No.\n- Qualification impact: None.\n- Rollback path: Revert the workflow and script changes.\n\n## Review Readiness\n\n- [x] I have performed a self-review\n- [x] This PR is ready for human review\n", + "user": {"login": "octocat"}, + "base": {"sha": "dff6a199"} + }, + "repository": {"full_name": "JerrettDavis/headroom"} +} diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.json index 6f96ec33d..f2743b3d8 100644 --- a/.github/plugin/marketplace.json +++ b/.github/plugin/marketplace.json @@ -5,14 +5,14 @@ }, "metadata": { "description": "Headroom marketplace for Claude Code and GitHub Copilot CLI plugins.", - "version": "0.34.0" + "version": "0.35.0" }, "plugins": [ { "name": "headroom", "source": "./plugins/headroom-agent-hooks", "description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.", - "version": "0.34.0", + "version": "0.35.0", "author": { "name": "Headroom Contributors", "url": "https://github.com/chopratejas/headroom" diff --git a/.github/pr-images/issue-2552-windows-fallback-verification.png b/.github/pr-images/issue-2552-windows-fallback-verification.png new file mode 100644 index 000000000..91a1dc27e Binary files /dev/null and b/.github/pr-images/issue-2552-windows-fallback-verification.png differ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 49acfaf42..431860278 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -561,7 +561,7 @@ jobs: - name: Install test dependencies run: | python -m pip install --upgrade pip - pip install pytest + pip install pytest 'opentelemetry-api>=1.24.0' - name: Run native installer wrapper tests run: pytest tests/test_install/test_native_installers.py -q @@ -579,7 +579,7 @@ jobs: run: | brew install bash python -m pip install --upgrade pip - python -m pip install --retries 10 --timeout 60 pytest + python -m pip install --retries 10 --timeout 60 pytest 'opentelemetry-api>=1.24.0' - name: Run native installer wrapper tests run: | BASH_PREFIX="$(brew --prefix bash)" diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index accab994a..a3d06f76c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -204,10 +204,10 @@ jobs: - name: Upload digest marker uses: actions/upload-artifact@v7 with: - # Variant + arch in the artifact name so the manifest job can - # download with `pattern: digests--*` to gather all - # arches for one variant. `root` substitutes the empty-string - # variant since GHA artifact names can't end in a hyphen. + # Variant + arch uniquely identify the marker. The manifest job + # downloads both architecture artifacts by exact name; a glob such + # as `digests-code-*` would also match code-nonroot/code-slim. + # `root` substitutes the empty-string variant. name: digests-${{ matrix.variant.name || 'root' }}-${{ matrix.arch.name }} path: ${{ runner.temp }}/digests/* if-no-files-found: error @@ -220,6 +220,7 @@ jobs: # tags, and that manifest is what users pull by `:tag`. docker-manifest: needs: docker-build + if: ${{ always() }} runs-on: ubuntu-24.04 timeout-minutes: 20 strategy: @@ -272,12 +273,17 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Download per-arch digests for this variant + - name: Download amd64 digest for this variant uses: actions/download-artifact@v8 with: - pattern: digests-${{ matrix.variant.name || 'root' }}-* + name: digests-${{ matrix.variant.name || 'root' }}-amd64 + path: ${{ runner.temp }}/digests + + - name: Download arm64 digest for this variant + uses: actions/download-artifact@v8 + with: + name: digests-${{ matrix.variant.name || 'root' }}-arm64 path: ${{ runner.temp }}/digests - merge-multiple: true # Same tag rules as the pre-fan-out workflow — preserve every # tag flavor (semver, ref, sha-prefixed, version-suffixed, @@ -313,6 +319,11 @@ jobs: echo "ERROR: no digests downloaded for variant '${{ matrix.variant.name || 'root' }}'" >&2 exit 1 fi + digest_count="$(find "${DIGEST_DIR}" -maxdepth 1 -type f | wc -l)" + if [ "${digest_count}" -ne 2 ]; then + echo "ERROR: expected both architecture digests for variant '${{ matrix.variant.name || 'root' }}', found ${digest_count}" >&2 + exit 1 + fi digest_refs=() for f in "${DIGEST_DIR}"/*; do digest="$(basename "$f")" @@ -382,53 +393,13 @@ jobs: sleep "$sleep_for" done - promote-latest: - # Re-push the :latest tag pointing at the root variant *after* every - # variant manifest job has finished, so GHCR's package version - # listing (sorted by created_at) shows the root image with :latest - # at the top instead of whichever variant happened to finish last. - needs: docker-manifest - runs-on: ubuntu-24.04 - timeout-minutes: 10 - steps: - - name: Normalize image name - id: image-name - run: | - image_name="$(printf '%s' '${{ github.repository }}' | tr '[:upper:]' '[:lower:]')" - printf 'image_name=%s\n' "$image_name" >> "$GITHUB_OUTPUT" - - - name: Determine image version - id: version - env: - MANUAL_VERSION: ${{ inputs.version || github.event.inputs.version }} - RELEASE_TAG: ${{ github.event.release.tag_name }} - run: | - version="${MANUAL_VERSION#v}" - if [ -z "$version" ] && [ -n "$RELEASE_TAG" ]; then - version="${RELEASE_TAG#v}" - fi - printf 'version=%s\n' "$version" >> "$GITHUB_OUTPUT" - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 - - - name: Log in to GHCR - uses: docker/login-action@v4 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Re-tag root image as :latest - if: steps.version.outputs.version != '' + if: steps.manifest.outputs.index_digest != '' && matrix.variant.name == '' && steps.version.outputs.version != '' env: IMAGE: ${{ env.REGISTRY }}/${{ steps.image-name.outputs.image_name }} VERSION: ${{ steps.version.outputs.version }} run: | - # Add a unique annotation so the resulting image index manifest gets - # a new digest, which makes GHCR record a fresh package version with - # current timestamp (otherwise the existing root manifest is reused - # and stays where it was in the version listing). + # Add a unique annotation so GHCR records a fresh root package version. promoted_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" docker buildx imagetools create \ --annotation "index:io.headroom.promoted-at=${promoted_at}" \ diff --git a/.github/workflows/opencode-plugin.yml b/.github/workflows/opencode-plugin.yml index 4fd8103ab..4e81e6597 100644 --- a/.github/workflows/opencode-plugin.yml +++ b/.github/workflows/opencode-plugin.yml @@ -11,12 +11,14 @@ on: paths: - "plugins/opencode/**" - "headroom/providers/opencode/_dist/**" + - "headroom/providers/opencode/hook-shim/**" - ".github/workflows/opencode-plugin.yml" push: branches: [main] paths: - "plugins/opencode/**" - "headroom/providers/opencode/_dist/**" + - "headroom/providers/opencode/hook-shim/**" - ".github/workflows/opencode-plugin.yml" permissions: @@ -51,3 +53,6 @@ jobs: cmp dist-standalone/entry.opencode.js \ ../../headroom/providers/opencode/_dist/entry.opencode.js \ || { echo "::error::headroom/providers/opencode/_dist/entry.opencode.js is stale - run 'npm run build:standalone' in plugins/opencode and commit the result"; exit 1; } + cmp dist-standalone/hook-shim/handler.js \ + ../../headroom/providers/opencode/hook-shim/handler.js \ + || { echo "::error::headroom/providers/opencode/hook-shim/handler.js is stale - run 'npm run build:standalone' in plugins/opencode and commit the result"; exit 1; } diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 81b90ebbc..1e3f19bf6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,6 +12,7 @@ env: NPM_REGISTRY_URL: https://registry.npmjs.org NPM_SDK_PACKAGE: headroom-ai NPM_OPENCLAW_PACKAGE: headroom-openclaw + NPM_OPENCODE_PACKAGE: headroom-opencode # GitHub Package Registry GITHUB_PACKAGES_REGISTRY_URL: https://npm.pkg.github.com @@ -852,10 +853,30 @@ jobs: npm publish --access public continue-on-error: true - - name: npm publish notice - if: steps.npm-sdk-publish.outcome == 'failure' || steps.npm-openclaw-publish.outcome == 'failure' + - name: Publish ${{ env.NPM_OPENCODE_PACKAGE }} to npmjs.org + id: npm-opencode-publish + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} run: | - echo "::notice::One or more npm publishes failed. Set NPM_SKIP=true in repo Variables to skip both npm publishes if tokens are not configured." + version="${{ needs.detect-version.outputs.npm_version }}" + cd plugins/opencode + npm ci + npm run build + npm version "$version" --no-git-tag-version --allow-same-version + HEADROOM_NPM_VERSION="$version" node <<'EOF' + const fs = require("fs"); + const pkg = JSON.parse(fs.readFileSync("package.json", "utf8")); + pkg.dependencies = pkg.dependencies || {}; + pkg.dependencies["headroom-ai"] = `^${process.env.HEADROOM_NPM_VERSION}`; + fs.writeFileSync("package.json", `${JSON.stringify(pkg, null, 2)}\n`); + EOF + npm publish --access public + continue-on-error: true + + - name: npm publish notice + if: steps.npm-sdk-publish.outcome == 'failure' || steps.npm-openclaw-publish.outcome == 'failure' || steps.npm-opencode-publish.outcome == 'failure' + run: | + echo "::notice::One or more npm publishes failed. Set NPM_SKIP=true in repo Variables to skip npm publishes if tokens are not configured." publish-github-packages: needs: [detect-version, build] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 402975fd6..08ee4e215 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,7 +27,7 @@ repos: # unconditionally, so installing hooks is not required for enforcement. args: [--assume-in-merge] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.17 + rev: v0.15.22 hooks: - id: ruff args: [--fix] diff --git a/.release-please-config.json b/.release-please-config.json index dadabab91..3da8e177d 100644 --- a/.release-please-config.json +++ b/.release-please-config.json @@ -7,8 +7,8 @@ "bump-patch-for-minor-pre-major": false, "draft": false, "prerelease": false, - "separate-pull-requests": false, - "pull-request-title-pattern": "chore: release ${version}", + "separate-pull-requests": true, + "pull-request-title-pattern": "chore: release${component} ${version}", "packages": { ".": { "package-name": "headroom-ai", @@ -23,6 +23,11 @@ "type": "json", "path": "plugins/openclaw/package.json", "jsonpath": "$.version" + }, + { + "type": "json", + "path": "plugins/opencode/package.json", + "jsonpath": "$.version" } ] } diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 8878352c7..3a39fd8cf 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "0.34.0" + ".": "0.35.0" } diff --git a/.releasemetadata b/.releasemetadata index 25ef9aea4..017607cea 100644 --- a/.releasemetadata +++ b/.releasemetadata @@ -1,9 +1,10 @@ { - "version": "0.34.0", + "version": "0.35.0", "packages": { - "pypi": "0.34.0", - "npm-sdk": "0.34.0", - "npm-openclaw": "0.34.0", - "agent-hooks-plugin": "0.34.0" + "pypi": "0.35.0", + "npm-sdk": "0.35.0", + "npm-openclaw": "0.35.0", + "npm-opencode": "0.35.0", + "agent-hooks-plugin": "0.35.0" } } diff --git a/CHANGELOG.md b/CHANGELOG.md index 807f16593..89fc5d9d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -284,6 +284,137 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **code:** fix two `CodeAwareCompressor` AST-reassembly bugs: an exported JS/TS function or class (`export function foo() {`) produced a duplicated `export export` keyword and invalid syntax, because line-based node slicing (used to preserve indentation) pulled in the preceding `export` sibling's text on top of the `export_statement` handler's own prefix reconstruction. Separately, in every supported language, a doc comment immediately above a top-level function, class, or type was detached from its declaration during extraction and re-emitted in a cluster at the end of the compressed output instead of staying attached to what it documents. - * **proxy:** Buffered upstream responses containing a `server_tool_use` (or any other unrecognized Anthropic content block) no longer turn a fully-generated response into an HTTP 502. `StreamingMixin._response_to_sse` raised `ValueError` on unknown block types after the entire upstream generation had already been buffered, so a slow-but-successful response failed and the client retried the whole multi-minute request. Unknown blocks are now emitted verbatim in `content_block_start` (following the existing redacted_thinking` pattern), so `server_tool_use`, `server_tool_result`, `mcp_tool_use`, and future block types round-trip ([#1806](https://github.com/headroomlabs-ai/headroom/issues/1806)). +## [0.35.0](https://github.com/headroomlabs-ai/headroom/compare/v0.34.0...v0.35.0) (2026-08-12) + + +### Features + +* **beacon:** allowlist the routing summary key ([#2818](https://github.com/headroomlabs-ai/headroom/issues/2818)) ([7940c05](https://github.com/headroomlabs-ai/headroom/commit/7940c05ebf4486c6b9d00984067ae33cedf4dddb)) +* **beacon:** hourly R2 compaction, per-strategy savings, and a stack that reports ([#2853](https://github.com/headroomlabs-ai/headroom/issues/2853)) ([e0870ef](https://github.com/headroomlabs-ai/headroom/commit/e0870ef931e5ea6cc6cb52551f5d80cd9e3dc715)) +* **cli,pricing:** add CLI extension seam and prompt-cache TTL pricing ([#2802](https://github.com/headroomlabs-ai/headroom/issues/2802)) ([6ec3e34](https://github.com/headroomlabs-ai/headroom/commit/6ec3e3478abf058fe1460f91342bcdadf54a1ba8)) + + +### Bug Fixes + +* **anthropic:** strip first-party tool search on custom upstreams ([#2539](https://github.com/headroomlabs-ai/headroom/issues/2539)) ([7f6950b](https://github.com/headroomlabs-ai/headroom/commit/7f6950be34e29304deae0fa5138b852491b092fe)) +* **backends/anyllm:** convert Anthropic tools and tool_choice to OpenAI shape ([0d6866b](https://github.com/headroomlabs-ai/headroom/commit/0d6866b91a3777475abd58cd8b63a10cd0621e7f)) +* **backends/anyllm:** stream tool_use blocks and map finish_reason on the streaming path ([e4904e2](https://github.com/headroomlabs-ai/headroom/commit/e4904e23a6ba6f5cff2488332946481172446922)) +* **backends/litellm:** None-guard core token counts in OpenAI usage block ([#2324](https://github.com/headroomlabs-ai/headroom/issues/2324)) ([12f9f58](https://github.com/headroomlabs-ai/headroom/commit/12f9f58cb3dcfc67af1238424d404d8dd9bad1dd)) +* **beacon:** report all-layers savings, not context-compression only ([#2796](https://github.com/headroomlabs-ai/headroom/issues/2796)) ([e9a24f3](https://github.com/headroomlabs-ai/headroom/commit/e9a24f3ec1ffd278b0b3ca547a90942c40c99ec8)) +* **beacon:** split session failures by status code ([#2815](https://github.com/headroomlabs-ai/headroom/issues/2815)) ([2954e37](https://github.com/headroomlabs-ai/headroom/commit/2954e37048f8dcffe16e1c37b8f71afb0094a0a2)) +* **cache:** bound compression cache bookkeeping ([0ae948c](https://github.com/headroomlabs-ai/headroom/commit/0ae948c1510735df39317bf0861f8a8750cdbf9d)) +* **cache:** enforce Anthropic's 1h-before-5m cache_control ordering before forwarding ([#2941](https://github.com/headroomlabs-ai/headroom/issues/2941)) ([3752458](https://github.com/headroomlabs-ai/headroom/commit/3752458022f736c779f7b5a6c2d6d2ef0bc89f72)) +* **cache:** mirror client cache_control positions instead of single-marker consolidation ([def3d76](https://github.com/headroomlabs-ai/headroom/commit/def3d76e5ab4665e609b51bfba54dd6d25116925)) +* **cache:** stabilize Anthropic block-growing lineages ([#2917](https://github.com/headroomlabs-ai/headroom/issues/2917)) ([1a04c95](https://github.com/headroomlabs-ai/headroom/commit/1a04c957f53ef25ab1209166f425a7876913c4d3)) +* **ccr:** avoid injecting tool on chat streaming ([d0c1f5b](https://github.com/headroomlabs-ai/headroom/commit/d0c1f5b8ad68c7a44ed3aaa0fe40e3a656950123)) +* **ccr:** preserve exact SQLite TTL boundary ([#2669](https://github.com/headroomlabs-ai/headroom/issues/2669)) ([d0a86d4](https://github.com/headroomlabs-ai/headroom/commit/d0a86d409fab377f9c642d1f3680b6ece7f97b8a)) +* **ccr:** report embedded hashes from compress endpoint ([#717](https://github.com/headroomlabs-ai/headroom/issues/717)) ([685ebe4](https://github.com/headroomlabs-ai/headroom/commit/685ebe457d727922ba4057515556a2d2aac0f616)) +* **ccr:** resolve <<ccr:...>> markers inline when no retrieve-tool path exists ([#2512](https://github.com/headroomlabs-ai/headroom/issues/2512)) ([ce8ce83](https://github.com/headroomlabs-ai/headroom/commit/ce8ce8313f8cebf060392a62f9adaab18c0df386)) +* **ccr:** tolerate null/malformed OpenAI data in response handling ([#2467](https://github.com/headroomlabs-ai/headroom/issues/2467)) ([e583e08](https://github.com/headroomlabs-ai/headroom/commit/e583e082d8dee942229ac6211c742f9c9448a905)) +* **ci:** publish latest from the root Docker manifest ([#2252](https://github.com/headroomlabs-ai/headroom/issues/2252)) ([5568d73](https://github.com/headroomlabs-ai/headroom/commit/5568d738afb5e080d8df56e64500026996cbf025)) +* **claude:** stop forcing tool search on Foundry ([#2477](https://github.com/headroomlabs-ai/headroom/issues/2477)) ([7981396](https://github.com/headroomlabs-ai/headroom/commit/798139608c0fb5118eb3a7a183b8b2abe92341f1)) +* **cli/update:** let install ownership win over bare /.dockerenv so venv installs self-update ([#2830](https://github.com/headroomlabs-ai/headroom/issues/2830)) ([7092b53](https://github.com/headroomlabs-ai/headroom/commit/7092b53c466bf5dbda8a1cda88403d1a4b16deb1)) +* **codex:** route alpha search through the Codex backend ([#2538](https://github.com/headroomlabs-ai/headroom/issues/2538)) ([a540eb2](https://github.com/headroomlabs-ai/headroom/commit/a540eb2c61b1a47e5ab8b07ea4a80fee780b6514)) +* **content-router:** protect custom-tag blocks before mixed-content section split ([d7bc1e2](https://github.com/headroomlabs-ai/headroom/commit/d7bc1e275f411788abffa2d007db14aa17fd31c5)) +* **deps:** bump h2 to 4.4.1 for CVE-2026-71554 ([#2839](https://github.com/headroomlabs-ai/headroom/issues/2839)) ([564e0a8](https://github.com/headroomlabs-ai/headroom/commit/564e0a8d0fe440dff21a6c405c88e05698b3059f)) +* **deps:** enforce audited transitive dependency floors ([#2791](https://github.com/headroomlabs-ai/headroom/issues/2791)) ([64e2039](https://github.com/headroomlabs-ai/headroom/commit/64e203931b9810e5a010f063d26d154419016f86)) +* **doctor:** flag `ollama launch claude` proxy bypass instead of misdirecting ([#2566](https://github.com/headroomlabs-ai/headroom/issues/2566)) ([7f24d69](https://github.com/headroomlabs-ai/headroom/commit/7f24d695eea00b9bb3265fbaa6629acf0c2ff181)) +* emit SSE ping before message_start on Bedrock streaming path (issue [#902](https://github.com/headroomlabs-ai/headroom/issues/902)) ([#1080](https://github.com/headroomlabs-ai/headroom/issues/1080)) ([4dab254](https://github.com/headroomlabs-ai/headroom/commit/4dab254d52914c39ffe13071848604e1771b1bd1)) +* **gemini:** resolve native CCR retrieval calls ([#2253](https://github.com/headroomlabs-ai/headroom/issues/2253)) ([2483f57](https://github.com/headroomlabs-ai/headroom/commit/2483f570025763cd9183a93749ea8cf38f1aeb85)) +* **health:** label kompress as degraded/optional when not yet loaded ([#2865](https://github.com/headroomlabs-ai/headroom/issues/2865)) ([8949371](https://github.com/headroomlabs-ai/headroom/commit/89493714d2cffdc1f81a8f417ea09891453d7009)) +* **image:** decouple routing types from trained_router so importing the compressor doesn't import torch ([#2513](https://github.com/headroomlabs-ai/headroom/issues/2513)) ([#2537](https://github.com/headroomlabs-ai/headroom/issues/2537)) ([d7cf981](https://github.com/headroomlabs-ai/headroom/commit/d7cf981093cf505192a3736dadd0254a120830a1)) +* **install/windows:** register persistent-task from S4U hidden XML ([#2453](https://github.com/headroomlabs-ai/headroom/issues/2453)) ([#2459](https://github.com/headroomlabs-ai/headroom/issues/2459)) ([1edaeb8](https://github.com/headroomlabs-ai/headroom/commit/1edaeb8b76f6b872a6c810d404c944caf1a594b2)) +* **install:** don't crash the PowerShell installer when $PROFILE is unset ([#2469](https://github.com/headroomlabs-ai/headroom/issues/2469)) ([fc5c4e2](https://github.com/headroomlabs-ai/headroom/commit/fc5c4e239ce32f2b90a6777772a01bdf49c66cb6)) +* **install:** trust Docker bridge for dashboard metadata ([e044139](https://github.com/headroomlabs-ai/headroom/commit/e044139001680fd5198147bf373df6f00db32cc7)) +* **install:** use --userns=keep-id under Podman so bind-mount writes don't fail ([#2846](https://github.com/headroomlabs-ai/headroom/issues/2846)) ([3488f8d](https://github.com/headroomlabs-ai/headroom/commit/3488f8d4b5fae4eab157e0c4031ccf712bcbcc0d)) +* **learn/gemini:** stop double-counting session tokens ([#2230](https://github.com/headroomlabs-ai/headroom/issues/2230)) ([29d8a5e](https://github.com/headroomlabs-ai/headroom/commit/29d8a5e563cf16dbd3a53a1571f4f352e61e1b33)) +* **learn/grok:** detect a Windows absolute project path ([#2283](https://github.com/headroomlabs-ai/headroom/issues/2283)) ([e240df2](https://github.com/headroomlabs-ai/headroom/commit/e240df2b698e601324b85956bd93cb304f6030ab)) +* **learn:** stop classifying a successful exit code 0 as an error ([#2289](https://github.com/headroomlabs-ai/headroom/issues/2289)) ([a24fe7d](https://github.com/headroomlabs-ai/headroom/commit/a24fe7dcbfe5ab30d0cef631c936e2245c12d123)) +* **litellm:** add async_post_call_success_hook to HeadroomCallback ([#1322](https://github.com/headroomlabs-ai/headroom/issues/1322)) ([3107994](https://github.com/headroomlabs-ai/headroom/commit/3107994aed5fd42e713d3c26f3f08121a62b980e)) +* **litellm:** don't forward a caller key the target cannot accept ([#2883](https://github.com/headroomlabs-ai/headroom/issues/2883)) ([2f2950a](https://github.com/headroomlabs-ai/headroom/commit/2f2950a626cebf851aac29255e7188fbb1639f5a)) +* **memory:** bound the TrafficLearner pending-pattern accumulator (memory leak) ([#2579](https://github.com/headroomlabs-ai/headroom/issues/2579)) ([1f5feff](https://github.com/headroomlabs-ai/headroom/commit/1f5fefffd3e82c73bddd928cfd53334031e807bc)) +* **memory:** close DirectMem0 resources ([6596182](https://github.com/headroomlabs-ai/headroom/commit/65961827cf5e90d7b4e7026feb89aac000a73ea3)) +* **memory:** close MCP backend on shutdown ([4bd8ecd](https://github.com/headroomlabs-ai/headroom/commit/4bd8ecd1e31475365801791d35630f66f7393553)) +* **memory:** don't crash inline memory extraction on a non-object <memory> block ([#2470](https://github.com/headroomlabs-ai/headroom/issues/2470)) ([e00c6ff](https://github.com/headroomlabs-ai/headroom/commit/e00c6ff81ce2003e04042b8f2d1bd6aa3c6e885c)) +* **memory:** keep vector metadata in sync ([#2295](https://github.com/headroomlabs-ai/headroom/issues/2295)) ([c471800](https://github.com/headroomlabs-ai/headroom/commit/c471800e8ee22986c308464b02a85da5575f34cc)) +* **memory:** make explicit-project and user store keys collision-resistant ([#2231](https://github.com/headroomlabs-ai/headroom/issues/2231)) ([f840d5f](https://github.com/headroomlabs-ai/headroom/commit/f840d5f2fe938432e542c3f71f2218eeecd06b05)) +* **memory:** skip <system-reminder> blocks when building the retrieval query ([#2195](https://github.com/headroomlabs-ai/headroom/issues/2195)) ([#2541](https://github.com/headroomlabs-ai/headroom/issues/2541)) ([4e5a67a](https://github.com/headroomlabs-ai/headroom/commit/4e5a67a342be4be659b62c7863a9e72422605788)) +* **memory:** sync FTS5 and vector indexes on CLI delete/edit/prune/purge ([fd4628d](https://github.com/headroomlabs-ai/headroom/commit/fd4628d82156c65d4fa22df9513315790a6cd2fb)) +* **oauth2:** make repository lint checks pass ([c85abf7](https://github.com/headroomlabs-ai/headroom/commit/c85abf7a87920012e01f0a677f6fbd98c4b08de0)) +* **observability:** aggregate tool savings in OTEL ([#2936](https://github.com/headroomlabs-ai/headroom/issues/2936)) ([941c25d](https://github.com/headroomlabs-ai/headroom/commit/941c25d31e6c6e0b436c307cbe212771ff76b45f)) +* **onnx:** stop ONNX thread pools from spinning idle cores ([#2495](https://github.com/headroomlabs-ai/headroom/issues/2495)) ([#2540](https://github.com/headroomlabs-ai/headroom/issues/2540)) ([5c561bd](https://github.com/headroomlabs-ai/headroom/commit/5c561bd913ea60fad2c3c53f4b65e679e7d248d0)) +* **openai:** skip Responses tool-search deferral for clients that cannot execute it ([#2696](https://github.com/headroomlabs-ai/headroom/issues/2696)) ([54ea28d](https://github.com/headroomlabs-ai/headroom/commit/54ea28d9839a0dcfa4dd0cf4210a4421f03beeff)) +* **opencode:** ship the transport hook-shim so wheel installs route Node child traffic ([702dbc5](https://github.com/headroomlabs-ai/headroom/commit/702dbc5902ff184a7c20178958a811beb9c78fa3)) +* **providers/anthropic:** don't crash token estimation on null tool_calls ([#2472](https://github.com/headroomlabs-ai/headroom/issues/2472)) ([08466f3](https://github.com/headroomlabs-ai/headroom/commit/08466f3cae4dbb2647dc6f249fe42c4e840600c5)) +* **providers/openai:** bound tiktoken vocab loads with the guarded loader ([#2554](https://github.com/headroomlabs-ai/headroom/issues/2554)) ([0805e8e](https://github.com/headroomlabs-ai/headroom/commit/0805e8e410543d75c7ddd3b83dde5eda3bc13144)) +* **proxy/anthropic:** inject headroom_retrieve whenever a CCR marker is present, not only for new markers ([#2848](https://github.com/headroomlabs-ai/headroom/issues/2848)) ([3808f60](https://github.com/headroomlabs-ai/headroom/commit/3808f60ca61e84faf3ea8f8e003a6e6c8e9af4da)) +* **proxy/anthropic:** None-guard usage token counts on the direct buffered path ([#2434](https://github.com/headroomlabs-ai/headroom/issues/2434)) ([2b5ee7c](https://github.com/headroomlabs-ai/headroom/commit/2b5ee7cde809ca37f6998d9679b1eb2133ab50ca)) +* **proxy/anthropic:** run tool-search history repair after turn hooks ([c6f9948](https://github.com/headroomlabs-ai/headroom/commit/c6f99482e1bea024db6014a70c8e6da419543957)) +* **proxy/batch:** don't crash an OpenAI batch on a valid-JSON non-object line ([#2316](https://github.com/headroomlabs-ai/headroom/issues/2316)) ([1f2c681](https://github.com/headroomlabs-ai/headroom/commit/1f2c681c0b48150a569277d3ebd5e95709dc7c39)) +* **proxy/bedrock:** report uncached input tokens from backend usage, not the live-zone count ([#2318](https://github.com/headroomlabs-ai/headroom/issues/2318)) ([c19e412](https://github.com/headroomlabs-ai/headroom/commit/c19e412b3356d80dece001887d4ff48b6fd5150b)) +* **proxy/gemini:** keep streaming-parity baseline so eligible_pct can't exceed 100 ([#2824](https://github.com/headroomlabs-ai/headroom/issues/2824)) ([b97c7c6](https://github.com/headroomlabs-ai/headroom/commit/b97c7c6e99eac84df49c7a7e5f21dedb298716fe)) +* **proxy/metrics:** cap client-supplied model label cardinality ([#2480](https://github.com/headroomlabs-ai/headroom/issues/2480)) ([e24a7e6](https://github.com/headroomlabs-ai/headroom/commit/e24a7e66b95fa908c4ea6fd079809ece7692e6b2)) +* **proxy/metrics:** escape label values in the Prometheus export ([#2463](https://github.com/headroomlabs-ai/headroom/issues/2463)) ([6a53861](https://github.com/headroomlabs-ai/headroom/commit/6a53861063c3839e698bbec7194517bdfd851c38)) +* **proxy/openai:** don't crash the Responses memory tool loops on null arguments ([#2273](https://github.com/headroomlabs-ai/headroom/issues/2273)) ([a30db2c](https://github.com/headroomlabs-ai/headroom/commit/a30db2cae49b4ef03ebbd404ec1fc6c4f5f2404d)) +* **proxy/openai:** feed Codex WS traffic into the traffic learner ([#2334](https://github.com/headroomlabs-ai/headroom/issues/2334)) ([f669149](https://github.com/headroomlabs-ai/headroom/commit/f6691497692869b7067438597421ff12aace6bf4)) +* **proxy/openai:** run response hooks on Responses, and bill their re-drives ([#2872](https://github.com/headroomlabs-ai/headroom/issues/2872)) ([675d13f](https://github.com/headroomlabs-ai/headroom/commit/675d13f08d42455c8fa17bda878c1a11b905cee4)) +* **proxy:** allow settings routes for trusted gateway/dashboard clients ([#2491](https://github.com/headroomlabs-ai/headroom/issues/2491)) ([a5b0a8f](https://github.com/headroomlabs-ai/headroom/commit/a5b0a8f4cc54d68afcf371a422b3a4a9635b7e7f)) +* **proxy:** cache litellm model resolution to stop repeated Provider List spam ([99f07e7](https://github.com/headroomlabs-ai/headroom/commit/99f07e7bbdded9dadc70e35ee6ab025279d1aa22)) +* **proxy:** cancel periodic TOIN task on shutdown ([739fdef](https://github.com/headroomlabs-ai/headroom/commit/739fdef423fa8cbc82537481c875d4570b0ecad4)) +* **proxy:** close the upstream stream when a streaming body is never consumed ([0951663](https://github.com/headroomlabs-ai/headroom/commit/09516635621caccf7e3db4f537eb49ea49b8a453)) +* **proxy:** compress cache-mode cold starts and tag prefix-mismatch passthrough ([#2365](https://github.com/headroomlabs-ai/headroom/issues/2365)) ([aaeba0a](https://github.com/headroomlabs-ai/headroom/commit/aaeba0a319f12b98cad3bfcf1cf991b694b946bf)) +* **proxy:** emit request log timestamps in UTC ([620028f](https://github.com/headroomlabs-ai/headroom/commit/620028fa18843622d3e454bd40fb91a93e607dbf)) +* **proxy:** enable tool search by default and repair poisoned transcripts ([#2807](https://github.com/headroomlabs-ai/headroom/issues/2807)) ([0237cbf](https://github.com/headroomlabs-ai/headroom/commit/0237cbffbbc456ad8a7398005602d76881862d99)) +* **proxy:** gate mid-turn message coalescing to Claude Code clients ([#1643](https://github.com/headroomlabs-ai/headroom/issues/1643)) ([a4bd2e6](https://github.com/headroomlabs-ai/headroom/commit/a4bd2e62a5bb73f15b3b12e979c69e2b555bee10)) +* **proxy:** give each Codex /v1/responses WS turn a unique request_id ([#2164](https://github.com/headroomlabs-ai/headroom/issues/2164)) ([d02df10](https://github.com/headroomlabs-ai/headroom/commit/d02df1075894b414d60626aca2bbcadd7a3577a0)) +* **proxy:** graceful shutdown and reliable Ctrl+C exit ([#621](https://github.com/headroomlabs-ai/headroom/issues/621)) ([17cdb18](https://github.com/headroomlabs-ai/headroom/commit/17cdb185bc79d8cfec104e781a7e555af3ef11e1)) +* **proxy:** guard telemetry and TOIN endpoints ([cde1513](https://github.com/headroomlabs-ai/headroom/commit/cde1513c91b6c6c240869bc5660f4b8966197bbc)) +* **proxy:** include tool_search_deferral savings in the savings ledger ([12149f7](https://github.com/headroomlabs-ai/headroom/commit/12149f74466c08b69be8d5fe751425be63c2fda4)) +* **proxy:** pass through cross-region prefixed Bedrock model IDs directly ([#2330](https://github.com/headroomlabs-ai/headroom/issues/2330)) ([64cb46e](https://github.com/headroomlabs-ai/headroom/commit/64cb46e24bf7b223ea71b14b6f5e86e78fa7ac45)) +* **proxy:** port session-sticky beta headers to the Rust proxy ([#2381](https://github.com/headroomlabs-ai/headroom/issues/2381)) ([f6398a6](https://github.com/headroomlabs-ai/headroom/commit/f6398a64768a095b722a5fb0b2445c7953dee1c6)) +* **proxy:** preserve merged session and quarantine contracts ([#2943](https://github.com/headroomlabs-ai/headroom/issues/2943)) ([039cd24](https://github.com/headroomlabs-ai/headroom/commit/039cd2431aaec7d59fefaf7e97aeda1fd7ab3afa)) +* **proxy:** preserve signed Anthropic thinking blocks on outbound re-serialize ([#2254](https://github.com/headroomlabs-ai/headroom/issues/2254)) ([dc163bc](https://github.com/headroomlabs-ai/headroom/commit/dc163bcd1cba4cd8898f23286eb1365fcf6e0356)) +* **proxy:** stop discarding compressed Codex WS later-frame payloads ([#2823](https://github.com/headroomlabs-ai/headroom/issues/2823)) ([4ec416d](https://github.com/headroomlabs-ai/headroom/commit/4ec416df8899036544e679f561f1cf921f3da0dd)) +* **proxy:** time-cap the compression timeout-debt quarantine ([#2360](https://github.com/headroomlabs-ai/headroom/issues/2360)) ([#2412](https://github.com/headroomlabs-ai/headroom/issues/2412)) ([c5a08d2](https://github.com/headroomlabs-ai/headroom/commit/c5a08d22e05a7dd2b929f3cca76ee3fb42f122db)) +* **proxy:** unwrap Hermes tool_call bridge in tool name map ([#2717](https://github.com/headroomlabs-ai/headroom/issues/2717)) ([a97b824](https://github.com/headroomlabs-ai/headroom/commit/a97b82413bdc86655c064417ed4628ff4d9d7c9d)) +* publish headroom-opencode in release workflow ([#2372](https://github.com/headroomlabs-ai/headroom/issues/2372)) ([7859154](https://github.com/headroomlabs-ai/headroom/commit/78591545ceb8303fdf9b93cd5ff02b626df97d2b)) +* **settings:** accept documented HEADROOM_* env names as settings keys ([#2833](https://github.com/headroomlabs-ai/headroom/issues/2833)) ([de9e052](https://github.com/headroomlabs-ai/headroom/commit/de9e0523dad47b700062464adecd60f82547f332)) +* **subscription:** dedup transcript usage by message id ([#2340](https://github.com/headroomlabs-ai/headroom/issues/2340) token inflation) ([#2408](https://github.com/headroomlabs-ai/headroom/issues/2408)) ([74275b7](https://github.com/headroomlabs-ai/headroom/commit/74275b7c3e2b39be5198f9efa35057a5e026e665)) +* **toin:** bound private query and pattern retention ([8cd1380](https://github.com/headroomlabs-ai/headroom/commit/8cd138039edbfc295080ec474325d527fb3aedf3)) +* **tokenizer:** coerce non-string tool_call fields before counting ([#2801](https://github.com/headroomlabs-ai/headroom/issues/2801)) ([b6f9877](https://github.com/headroomlabs-ai/headroom/commit/b6f9877c78b3fa3b1d705426bd27d74be77f4fa0)) +* **tokenizer:** price CJK in the Rust fixed-ratio estimator (Python parity) ([#2260](https://github.com/headroomlabs-ai/headroom/issues/2260)) ([6840153](https://github.com/headroomlabs-ai/headroom/commit/6840153473caa0d61e982215e16a8cf54b0b6cc7)) +* **transforms/adaptive-sizer:** honor max_k on small-input fast path ([#2319](https://github.com/headroomlabs-ai/headroom/issues/2319)) ([8a90523](https://github.com/headroomlabs-ai/headroom/commit/8a905232091d993fac9e19a59bc449f201d4cdf3)) +* **transforms/smart_crusher:** don't crash on a tool call with a null function ([#2232](https://github.com/headroomlabs-ai/headroom/issues/2232)) ([3bb02f8](https://github.com/headroomlabs-ai/headroom/commit/3bb02f8f75f12cf8258a5b1c2a7fbdc190f9d074)) +* Vertex model pricing shows $0.00 for versioned model names and vertex:anthropic provider ([#2517](https://github.com/headroomlabs-ai/headroom/issues/2517)) ([eb5b5e4](https://github.com/headroomlabs-ai/headroom/commit/eb5b5e41988f5c27d29ae8ae3e5fe74e56493b8c)) +* **wrap/claude:** keep --1m effective when an explicit --model is passed through ([c093bf1](https://github.com/headroomlabs-ai/headroom/commit/c093bf11eb5f356f71367ebb7b56ae3c2b434a12)) +* **wrap/opencode:** verify the opencode binary before mutating config ([ae38486](https://github.com/headroomlabs-ai/headroom/commit/ae384862a4950cec057103e9daf75e74107640df)) +* **wrap/serena:** install Serena from the serena-agent PyPI wheel, not the git source ([d7b25ae](https://github.com/headroomlabs-ai/headroom/commit/d7b25ae3bb3364cde4931509ecb65e32085e5b09)) +* **wrap:** honor Copilot OAuth wire-api override and model default ([#2387](https://github.com/headroomlabs-ai/headroom/issues/2387)) ([1db6d88](https://github.com/headroomlabs-ai/headroom/commit/1db6d88ab4ea25654b8277358902b7df700db6b4)) +* **wrap:** serialize shared proxy startup ([#2946](https://github.com/headroomlabs-ai/headroom/issues/2946)) ([e540d64](https://github.com/headroomlabs-ai/headroom/commit/e540d64febf27f2e7997d3a1a1d89478cc1ef658)) +* **wrap:** stop the launch cwd from shadowing the installed package in the proxy subprocess ([#2843](https://github.com/headroomlabs-ai/headroom/issues/2843)) ([c49be26](https://github.com/headroomlabs-ai/headroom/commit/c49be269a18446779cd8a048caaa7f0ba3a3b48b)) + + +### Performance Improvements + +* cut hot-path latency 27% (token-count memo, startup preloads, JSON scan memo) ([#2838](https://github.com/headroomlabs-ai/headroom/issues/2838)) ([53af90d](https://github.com/headroomlabs-ai/headroom/commit/53af90d68c723f644a5a41dd273a606117109866)) +* **proxy:** bound upstream calls and hot-path costs ([#2852](https://github.com/headroomlabs-ai/headroom/issues/2852)) ([f624d3a](https://github.com/headroomlabs-ai/headroom/commit/f624d3a00ac271db7947443ddeb0c8bc2e93d3eb)) +* **subscription:** skip transcripts older than the window in compute_window_tokens ([#2861](https://github.com/headroomlabs-ai/headroom/issues/2861)) ([91d6bf3](https://github.com/headroomlabs-ai/headroom/commit/91d6bf33cde777b541375fb182d4479fdd78f81b)) + + +### Dependencies + +* bump brace-expansion from 5.0.7 to 5.0.9 in /docs ([#2751](https://github.com/headroomlabs-ai/headroom/issues/2751)) ([56ee57b](https://github.com/headroomlabs-ai/headroom/commit/56ee57be98bf109f0a46de522724ef169a4bc51c)) +* bump bytesize from 1.3.3 to 2.4.2 ([#2286](https://github.com/headroomlabs-ai/headroom/issues/2286)) ([6448545](https://github.com/headroomlabs-ai/headroom/commit/6448545a7f5a1dee88bce6f0830bdbfd1c99c617)) +* bump hf-hub from 0.4.3 to 0.5.0 ([#2285](https://github.com/headroomlabs-ai/headroom/issues/2285)) ([4925bf6](https://github.com/headroomlabs-ai/headroom/commit/4925bf6a829735977bab5000b469c3edb19c75b1)) +* bump next from 16.2.10 to 16.3.0 in /docs ([#2750](https://github.com/headroomlabs-ai/headroom/issues/2750)) ([0fd0b99](https://github.com/headroomlabs-ai/headroom/commit/0fd0b996a4b58a166491b145f4d3885c21b27cc0)) +* bump postcss from 8.5.19 to 8.5.25 in /plugins/openclaw ([#2749](https://github.com/headroomlabs-ai/headroom/issues/2749)) ([cd60ee9](https://github.com/headroomlabs-ai/headroom/commit/cd60ee9ae886b32ba5da3203e35bb6b088031fd3)) +* bump postcss from 8.5.19 to 8.5.25 in /plugins/opencode ([#2748](https://github.com/headroomlabs-ai/headroom/issues/2748)) ([ff4e016](https://github.com/headroomlabs-ai/headroom/commit/ff4e0167bbccbd4ae51bf23ddec144e61c94cd68)) +* bump postcss from 8.5.19 to 8.5.25 in /sdk/typescript ([#2747](https://github.com/headroomlabs-ai/headroom/issues/2747)) ([267c2bd](https://github.com/headroomlabs-ai/headroom/commit/267c2bdcb56e132b2dd9c065dab3498dbf730ca3)) +* bump postcss from 8.5.19 to 8.5.26 in /docs ([#2881](https://github.com/headroomlabs-ai/headroom/issues/2881)) ([e6e5826](https://github.com/headroomlabs-ai/headroom/commit/e6e5826423a0a700a8c544ce2c8cbcdef694160e)) +* bump ruff from 0.15.17 to 0.15.22 in the pip-minor-patch group ([#2501](https://github.com/headroomlabs-ai/headroom/issues/2501)) ([ecf130d](https://github.com/headroomlabs-ai/headroom/commit/ecf130d3ac6fb864098cb93fafd2621ae3ac7e12)) +* bump rusqlite from 0.32.1 to 0.40.1 ([#2287](https://github.com/headroomlabs-ai/headroom/issues/2287)) ([522faa1](https://github.com/headroomlabs-ai/headroom/commit/522faa1a59aa94e4adfd4a4afe0202d1126e187d)) +* bump the cargo-minor-patch group across 1 directory with 22 updates ([#2916](https://github.com/headroomlabs-ai/headroom/issues/2916)) ([148d860](https://github.com/headroomlabs-ai/headroom/commit/148d8605e2087f3c8d6a3fa4b8d248ad2da5858f)) + ## [0.34.0](https://github.com/headroomlabs-ai/headroom/compare/v0.33.0...v0.34.0) (2026-08-05) diff --git a/Cargo.lock b/Cargo.lock index 5959d3596..2da32bedf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,9 +24,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" dependencies = [ "memchr", ] @@ -49,6 +49,15 @@ dependencies = [ "equator", ] +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -122,9 +131,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.103" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "arbitrary" @@ -149,7 +158,7 @@ checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -185,13 +194,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.89" +version = "0.1.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] @@ -221,7 +230,7 @@ dependencies = [ "num-traits", "pastey", "rayon", - "thiserror 2.0.18", + "thiserror 2.0.20", "v_frame", "y4m", ] @@ -251,9 +260,9 @@ dependencies = [ [[package]] name = "aws-config" -version = "1.9.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47712fde1909402600ccfbb26e47d482d2e58bb9e9e603d9f17e67cc435a6319" +checksum = "1b180a3c8b55960db3426d8964b8745e652466a1a49fe1a2eda828046d30b5e4" dependencies = [ "aws-credential-types", "aws-runtime", @@ -271,8 +280,8 @@ dependencies = [ "bytes", "fastrand", "hex", - "http 1.4.2", - "sha1", + "http 1.5.0", + "sha1 0.10.6", "time", "tokio", "tracing", @@ -316,9 +325,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.8.1" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7816e98ee912159f45d307e5ee6bfea4a335a55aee15f7f3e32f81a6f3000f1d" +checksum = "c9007227e10b5fed2f3e0a2beff489211e2b5604c400b7a9d5d81ca9d64c24bb" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -331,7 +340,7 @@ dependencies = [ "bytes", "bytes-utils", "fastrand", - "http 1.4.2", + "http 1.5.0", "http-body 1.0.1", "percent-encoding", "pin-project-lite", @@ -341,9 +350,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.103.0" +version = "1.105.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0469f435f645ad2162cfb463b15bde37115966ee3acf2d87fb4871ee309b8401" +checksum = "6ffd0fbe7873cb548a7aa60f9573c268fff94155397fd4f14dc9f1ecaaab8516" dependencies = [ "arc-swap", "aws-credential-types", @@ -360,16 +369,16 @@ dependencies = [ "bytes", "fastrand", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-ssooidc" -version = "1.105.0" +version = "1.107.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "085faefb253f770655e162b9304321e62a1e71adf7f019ee1f4454228a377b3a" +checksum = "175763eb222a46377df7aa257a3bca980ab3e96703fefc8f4d0b8da6ad2e254c" dependencies = [ "arc-swap", "aws-credential-types", @@ -386,16 +395,16 @@ dependencies = [ "bytes", "fastrand", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-sts" -version = "1.108.0" +version = "1.110.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c72b08911d8128dd360fe1b22a9fec0fa8b552dde8ec828dcf20ef5ec974e9f" +checksum = "dd8b14781dfbff48984017d57167b6ea0b6471c6920ec52b44a2677c7feb3c13" dependencies = [ "arc-swap", "aws-credential-types", @@ -413,7 +422,7 @@ dependencies = [ "aws-types", "fastrand", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "regex-lite", "tracing", ] @@ -433,7 +442,7 @@ dependencies = [ "hex", "hmac", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "percent-encoding", "sha2 0.11.0", "time", @@ -463,7 +472,7 @@ dependencies = [ "bytes-utils", "futures-core", "futures-util", - "http 1.4.2", + "http 1.5.0", "http-body 1.0.1", "http-body-util", "percent-encoding", @@ -482,7 +491,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "h2", - "http 1.4.2", + "http 1.5.0", "hyper", "hyper-rustls", "hyper-util", @@ -518,19 +527,22 @@ dependencies = [ [[package]] name = "aws-smithy-query" -version = "0.61.1" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd22a6ba36e3f113cb8d5b3d1fe0ed31c76ee608ef63322d753bb8d2c9479e77" +checksum = "512346c7212ab7436df2d77a16d976a468ae44a418835511d2a69269810aaf62" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", + "aws-smithy-xml", "urlencoding", ] [[package]] name = "aws-smithy-runtime" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bea94a9ff8464016338c851e24b472d7131c388c88898a502e781815b2ee6045" +checksum = "07505b34e8f4b3591a4fa69e9792b52289b95488dbbc68c3c0075b7bedb245e1" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -542,7 +554,7 @@ dependencies = [ "bytes", "fastrand", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "http-body 0.4.6", "http-body 1.0.1", "http-body-util", @@ -554,16 +566,16 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.13.0" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ed1ebe6e0a95ea84570225f5a8208dec4b8f77e61a9b0d6f51773fcb4612f0" +checksum = "3b98f2e1fd67ec06618f9c291e5e495a468e60519e44c9c1979cd0521f3affdb" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api-macros", "aws-smithy-types", "bytes", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "pin-project-lite", "tokio", "tracing", @@ -578,7 +590,7 @@ checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -589,7 +601,7 @@ checksum = "7d56e0a4e53127a632224e43633b0fe045fa9e1e3cfc68b9830f1115e103f910" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", - "http 1.4.2", + "http 1.5.0", ] [[package]] @@ -602,7 +614,7 @@ dependencies = [ "bytes", "bytes-utils", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "http-body 0.4.6", "http-body 1.0.1", "http-body-util", @@ -617,9 +629,9 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.61.1" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea3f68eec3607f02acd24067969ce2abc6ba16aa7d5ce59ca450ed2fb5f78957" +checksum = "ce84f71c72fee2cbbadde6e7d082f5fb466e3a84733855295fa7aafd1b31b7d8" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-schema", @@ -629,9 +641,9 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e957a6c6dbce82b7a91f44231c09273159703769f447cbe85e854dfe9cf67f86" +checksum = "eec1cd5469f328c782dc3e33d4153cf118a54e33cbb3356d60d16f89883e1f94" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -654,7 +666,7 @@ dependencies = [ "base64 0.22.1", "bytes", "futures-util", - "http 1.4.2", + "http 1.5.0", "http-body 1.0.1", "http-body-util", "hyper", @@ -670,10 +682,10 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sha1", + "sha1 0.10.6", "sync_wrapper", "tokio", - "tokio-tungstenite", + "tokio-tungstenite 0.24.0", "tower", "tower-layer", "tower-service", @@ -689,7 +701,7 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 1.4.2", + "http 1.5.0", "http-body 1.0.1", "http-body-util", "mime", @@ -709,7 +721,7 @@ checksum = "57d123550fa8d071b7255cb0cc04dc302baa6c8c4a79f55701552684d8399bce" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -772,9 +784,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.8.5" +version = "1.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +checksum = "76ae7bad254120e9e4c63bafc385310756f90c484eac0e36b8317cf09cb92a77" dependencies = [ "arrayref", "arrayvec", @@ -861,9 +873,9 @@ dependencies = [ [[package]] name = "bytesize" -version = "1.3.3" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e93abca9e28e0a1b9877922aacb20576e05d4679ffa78c3d6dc22a26a216659" +checksum = "7354288c522e7e980fafd2075d63d1285794c3a6a16cdd492f189ea406e5f18b" [[package]] name = "cast" @@ -882,9 +894,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.67" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -904,6 +916,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "chrono" version = "0.4.45" @@ -947,9 +970,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.2" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd059f9da4f5c36b3787f65d38ccaab1cc315f07b01f89abc8359ee6a8205011" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" dependencies = [ "clap_builder", "clap_derive", @@ -957,9 +980,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.2" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" dependencies = [ "anstream", "anstyle", @@ -969,14 +992,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] @@ -1037,19 +1060,6 @@ dependencies = [ "static_assertions", ] -[[package]] -name = "console" -version = "0.15.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" -dependencies = [ - "encode_unicode", - "libc", - "once_cell", - "unicode-width", - "windows-sys 0.59.0", -] - [[package]] name = "console" version = "0.16.3" @@ -1157,25 +1167,24 @@ dependencies = [ [[package]] name = "criterion" -version = "0.5.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" dependencies = [ + "alloca", "anes", "cast", "ciborium", "clap", "criterion-plot", - "is-terminal", - "itertools 0.10.5", + "itertools 0.13.0", "num-traits", - "once_cell", "oorandom", + "page_size", "plotters", "rayon", "regex", "serde", - "serde_derive", "serde_json", "tinytemplate", "walkdir", @@ -1183,12 +1192,12 @@ dependencies = [ [[package]] name = "criterion-plot" -version = "0.5.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" dependencies = [ "cast", - "itertools 0.10.5", + "itertools 0.13.0", ] [[package]] @@ -1271,7 +1280,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.118", ] [[package]] @@ -1282,7 +1291,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1356,7 +1365,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1366,7 +1375,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn", + "syn 2.0.118", ] [[package]] @@ -1420,7 +1429,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1476,7 +1485,7 @@ checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1549,7 +1558,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3c8600c9ec79b51d60c19911fe14eac04fe9c2895e87d2a3e80e2213d645a32" dependencies = [ "anyhow", - "hf-hub 0.5.0", + "hf-hub", "image", "ndarray", "ort", @@ -1582,9 +1591,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "flate2" @@ -1625,9 +1634,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" dependencies = [ "futures-channel", "futures-core", @@ -1640,9 +1649,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" dependencies = [ "futures-core", "futures-sink", @@ -1650,15 +1659,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" [[package]] name = "futures-executor" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" dependencies = [ "futures-core", "futures-task", @@ -1667,38 +1676,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" [[package]] name = "futures-macro" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] name = "futures-sink" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" dependencies = [ "futures-channel", "futures-core", @@ -1721,7 +1730,7 @@ dependencies = [ "base64 0.22.1", "bytes", "chrono", - "http 1.4.2", + "http 1.5.0", "http-body-util", "hyper", "hyper-rustls", @@ -1731,7 +1740,7 @@ dependencies = [ "rustls-pki-types", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tracing", "tracing-futures", @@ -1784,6 +1793,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", ] [[package]] @@ -1807,7 +1817,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.4.2", + "http 1.5.0", "indexmap", "slab", "tokio", @@ -1831,9 +1841,6 @@ name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -dependencies = [ - "ahash", -] [[package]] name = "hashbrown" @@ -1861,11 +1868,11 @@ dependencies = [ [[package]] name = "hashlink" -version = "0.9.1" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +checksum = "32069d97bb81e38fa67eab65e3393bf804bb85969f2bc06bf13f64aef5aba248" dependencies = [ - "hashbrown 0.14.5", + "hashbrown 0.17.1", ] [[package]] @@ -1879,8 +1886,8 @@ dependencies = [ "dashmap", "fastembed", "flate2", - "hf-hub 0.4.3", - "http 1.4.2", + "hf-hub", + "http 1.5.0", "icu_segmenter", "magika", "md-5", @@ -1894,7 +1901,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "tempfile", - "thiserror 2.0.18", + "thiserror 2.0.20", "tiktoken-rs", "tokenizers", "toml", @@ -1920,7 +1927,7 @@ dependencies = [ "headroom-core", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -1942,7 +1949,7 @@ dependencies = [ "gcp_auth", "headroom-core", "headroom-simulators", - "http 1.4.2", + "http 1.5.0", "http-body-util", "humantime", "hyper", @@ -1956,10 +1963,10 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.9", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tokio-stream", - "tokio-tungstenite", + "tokio-tungstenite 0.30.0", "tokio-util", "tower", "tower-http 0.7.0", @@ -1989,12 +1996,12 @@ dependencies = [ "bytes", "clap", "crc32fast", - "http 1.4.2", + "http 1.5.0", "http-body-util", "reqwest", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tower", "tracing", @@ -2019,26 +2026,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -[[package]] -name = "hf-hub" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "629d8f3bbeda9d148036d6b0de0a3ab947abd08ce90626327fc3547a49d59d97" -dependencies = [ - "dirs", - "http 1.4.2", - "indicatif 0.17.11", - "libc", - "log", - "rand 0.9.4", - "reqwest", - "serde", - "serde_json", - "thiserror 2.0.18", - "ureq 2.12.1", - "windows-sys 0.60.2", -] - [[package]] name = "hf-hub" version = "0.5.0" @@ -2046,16 +2033,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aef3982638978efa195ff11b305f51f1f22f4f0a6cabee7af79b383ebee6a213" dependencies = [ "dirs", - "http 1.4.2", - "indicatif 0.18.4", + "http 1.5.0", + "indicatif", "libc", "log", "rand 0.9.4", "reqwest", "serde", "serde_json", - "thiserror 2.0.18", - "ureq 3.3.0", + "thiserror 2.0.20", + "ureq", "windows-sys 0.61.2", ] @@ -2081,9 +2068,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", "itoa", @@ -2107,7 +2094,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.4.2", + "http 1.5.0", ] [[package]] @@ -2118,7 +2105,7 @@ checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" dependencies = [ "bytes", "futures-core", - "http 1.4.2", + "http 1.5.0", "http-body 1.0.1", "pin-project-lite", ] @@ -2152,16 +2139,16 @@ dependencies = [ [[package]] name = "hyper" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" dependencies = [ "atomic-waker", "bytes", "futures-channel", "futures-core", "h2", - "http 1.4.2", + "http 1.5.0", "http-body 1.0.1", "httparse", "httpdate", @@ -2178,7 +2165,7 @@ version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ - "http 1.4.2", + "http 1.5.0", "hyper", "hyper-util", "rustls", @@ -2199,7 +2186,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.4.2", + "http 1.5.0", "http-body 1.0.1", "hyper", "ipnet", @@ -2441,26 +2428,13 @@ dependencies = [ "hashbrown 0.17.1", ] -[[package]] -name = "indicatif" -version = "0.17.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" -dependencies = [ - "console 0.15.11", - "number_prefix", - "portable-atomic", - "unicode-width", - "web-time", -] - [[package]] name = "indicatif" version = "0.18.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25470f23803092da7d239834776d653104d551bc4d7eacaf31e6837854b8e9eb" dependencies = [ - "console 0.16.3", + "console", "portable-atomic", "unicode-width", "unit-prefix", @@ -2475,7 +2449,7 @@ checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2484,32 +2458,12 @@ version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" -[[package]] -name = "is-terminal" -version = "0.4.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys 0.61.2", -] - [[package]] name = "is_terminal_polyfill" version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.13.0" @@ -2610,9 +2564,9 @@ dependencies = [ [[package]] name = "libsqlite3-sys" -version = "0.30.1" +version = "0.38.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" dependencies = [ "cc", "pkg-config", @@ -2663,9 +2617,9 @@ dependencies = [ [[package]] name = "lru" -version = "0.18.1" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b6180140927ee907000b0aa540091f6ea512ead4447c92b8fc35bc72788a5a6" +checksum = "5d2f2f9b4ba7e6b24d95e7e899329d35be83bcded72c8540cdd5368932d1d90a" dependencies = [ "hashbrown 0.17.1", ] @@ -2807,7 +2761,7 @@ checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2917,7 +2871,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2959,12 +2913,6 @@ dependencies = [ "libc", ] -[[package]] -name = "number_prefix" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" - [[package]] name = "once_cell" version = "1.21.4" @@ -3042,6 +2990,16 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -3100,7 +3058,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3228,7 +3186,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4488a4a36b9a4ba6b9334a32a39971f77c1436ec82c38707bce707699cc3bbcb" dependencies = [ "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3242,7 +3200,7 @@ dependencies = [ "lazy_static", "memchr", "parking_lot", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -3272,9 +3230,9 @@ checksum = "e0c5ccf5294c6ccd63a74f1565028353830a9c2f5eb0c682c355c471726a6e3f" [[package]] name = "pyo3" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd274650b21d4bfc26a0a47587962c1edb425f69287324355cd040c3ea66071c" +checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b" dependencies = [ "libc", "once_cell", @@ -3286,18 +3244,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5e2a7d2f0d013342f295c048ad19237add5154a55b1c5a254c0ec93d4109078" +checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca85c467da1bbc8d866eea5deff9cf29ea5f7785054a17da36e65bda9c05845b" +checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327" dependencies = [ "libc", "pyo3-build-config", @@ -3316,26 +3274,26 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ac53762fd065daa3194dd09337a38bd793a188100fd1a9304c4ab312d901771" +checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn", + "syn 2.0.118", ] [[package]] name = "pyo3-macros-backend" -version = "0.29.0" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca3a1557399783172dc5bf39cfca835157732532cba56b71d2292161e53b362" +checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952" dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3373,7 +3331,7 @@ dependencies = [ "rustc-hash 2.1.2", "rustls", "socket2", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tracing", "web-time", @@ -3394,7 +3352,7 @@ dependencies = [ "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.18", + "thiserror 2.0.20", "tinyvec", "tracing", "web-time", @@ -3411,7 +3369,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.52.0", ] [[package]] @@ -3456,6 +3414,17 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -3494,6 +3463,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rand_xorshift" version = "0.4.0" @@ -3533,7 +3508,7 @@ dependencies = [ "rand 0.9.4", "rand_chacha 0.9.0", "simd_helpers", - "thiserror 2.0.18", + "thiserror 2.0.20", "v_frame", "wasm-bindgen", ] @@ -3623,7 +3598,7 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.17", "libredox", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -3672,7 +3647,7 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http 1.4.2", + "http 1.5.0", "http-body 1.0.1", "http-body-util", "hyper", @@ -3724,10 +3699,20 @@ dependencies = [ ] [[package]] -name = "rusqlite" -version = "0.32.1" +name = "rsqlite-vfs" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror 2.0.20", +] + +[[package]] +name = "rusqlite" +version = "0.40.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" dependencies = [ "bitflags", "fallible-iterator", @@ -3735,6 +3720,7 @@ dependencies = [ "hashlink", "libsqlite3-sys", "smallvec", + "sqlite-wasm-rs", ] [[package]] @@ -3938,14 +3924,14 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "indexmap", "itoa", @@ -3998,6 +3984,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + [[package]] name = "sha2" version = "0.10.9" @@ -4105,6 +4102,18 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -4146,6 +4155,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -4163,7 +4183,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -4196,11 +4216,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.18" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl 2.0.18", + "thiserror-impl 2.0.20", ] [[package]] @@ -4211,18 +4231,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] @@ -4342,7 +4362,7 @@ dependencies = [ "derive_builder", "esaxx-rs", "getrandom 0.3.4", - "indicatif 0.18.4", + "indicatif", "itertools 0.14.0", "log", "macro_rules_attribute", @@ -4357,7 +4377,7 @@ dependencies = [ "serde", "serde_json", "spm_precompiled", - "thiserror 2.0.18", + "thiserror 2.0.20", "unicode-normalization-alignments", "unicode-segmentation", "unicode_categories", @@ -4365,9 +4385,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -4387,7 +4407,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -4402,9 +4422,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" dependencies = [ "futures-core", "pin-project-lite", @@ -4416,6 +4436,18 @@ name = "tokio-tungstenite" version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite 0.24.0", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17a073bfed563fa236697a068031408a93cd9522e08abf9933ead3e73411bd71" dependencies = [ "futures-util", "log", @@ -4423,28 +4455,29 @@ dependencies = [ "rustls-pki-types", "tokio", "tokio-rustls", - "tungstenite", + "tungstenite 0.30.0", "webpki-roots 0.26.11", ] [[package]] name = "tokio-util" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", "futures-sink", + "libc", "pin-project-lite", "tokio", ] [[package]] name = "toml" -version = "1.1.3+spec-1.1.0" +version = "1.1.4+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53c96ecdfa941c8fc4fcaed14f99ada8ebed502eef533015095a07e3301d4c3c" +checksum = "3aace63f4bbcdfc2c965b059de67119c89c4017a70d633be6c104910f67056f5" dependencies = [ "indexmap", "serde_core", @@ -4466,9 +4499,9 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.1.2+spec-1.1.0" +version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" dependencies = [ "winnow", ] @@ -4504,7 +4537,7 @@ dependencies = [ "bitflags", "bytes", "futures-util", - "http 1.4.2", + "http 1.5.0", "http-body 1.0.1", "pin-project-lite", "tower", @@ -4521,7 +4554,7 @@ checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233" dependencies = [ "bitflags", "bytes", - "http 1.4.2", + "http 1.5.0", "http-body 1.0.1", "percent-encoding", "pin-project-lite", @@ -4564,7 +4597,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -4744,17 +4777,33 @@ dependencies = [ "byteorder", "bytes", "data-encoding", - "http 1.4.2", + "http 1.5.0", "httparse", "log", "rand 0.8.6", - "rustls", - "rustls-pki-types", - "sha1", + "sha1 0.10.6", "thiserror 1.0.69", "utf-8", ] +[[package]] +name = "tungstenite" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e48ac77174b19c110a50ab2128b24215ac9cb40e0e12e093fb602d175c569d22" +dependencies = [ + "bytes", + "data-encoding", + "http 1.5.0", + "httparse", + "log", + "rand 0.10.2", + "rustls", + "rustls-pki-types", + "sha1 0.11.0", + "thiserror 2.0.20", +] + [[package]] name = "typenum" version = "1.20.1" @@ -4802,9 +4851,9 @@ checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" [[package]] name = "unidiff" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ae26d2e6582eb32eff85cffebf74d20b6510e8b558bbac3a23b48965cf952f" +checksum = "98f3525b540ac79ce6db6f518b91f224c0aac4bffee14da01ccdb2edf5e2be1d" dependencies = [ "encoding_rs", "regex", @@ -4822,25 +4871,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" -[[package]] -name = "ureq" -version = "2.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" -dependencies = [ - "base64 0.22.1", - "flate2", - "log", - "once_cell", - "rustls", - "rustls-pki-types", - "serde", - "serde_json", - "socks", - "url", - "webpki-roots 0.26.11", -] - [[package]] name = "ureq" version = "3.3.0" @@ -4869,7 +4899,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" dependencies = [ "base64 0.22.1", - "http 1.4.2", + "http 1.5.0", "httparse", "log", ] @@ -5047,7 +5077,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.118", "wasm-bindgen-shared", ] @@ -5169,7 +5199,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -5180,7 +5210,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -5213,25 +5243,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", + "windows-targets", ] [[package]] @@ -5249,31 +5261,14 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] @@ -5282,96 +5277,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winnow" version = "1.0.3" @@ -5388,7 +5335,7 @@ dependencies = [ "base64 0.22.1", "deadpool", "futures", - "http 1.4.2", + "http 1.5.0", "http-body-util", "hyper", "hyper-util", @@ -5444,7 +5391,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] @@ -5465,7 +5412,7 @@ checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -5485,7 +5432,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] @@ -5527,7 +5474,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] diff --git a/Dockerfile b/Dockerfile index 8a73b44dd..fc2c3d176 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,7 +47,10 @@ COPY Cargo.toml Cargo.lock rust-toolchain.toml ./ COPY crates/ crates/ COPY headroom/ headroom/ -ARG HEADROOM_EXTRAS=proxy,code +# The standalone Dockerfile must support every backend advertised by +# `headroom proxy --backend`, including Bedrock temporary/SSO credentials. +# Those credentials require botocore (GH #1551), supplied by [bedrock]. +ARG HEADROOM_EXTRAS=proxy,code,bedrock RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/usr/local/cargo/git \ diff --git a/README.md b/README.md index 39a5852a3..0b923e940 100644 --- a/README.md +++ b/README.md @@ -415,7 +415,7 @@ Everything in this repo stays open source (Apache 2.0). The managed offering is uv tool install --python 3.13 "headroom-ai[all]" # CLI, isolated app env pip install "headroom-ai[all]" # Python, everything — includes the `headroom` CLI npm install headroom-ai # TypeScript SDK (library only — no `headroom` CLI) -docker pull ghcr.io/chopratejas/headroom:latest +docker pull ghcr.io/headroomlabs-ai/headroom:latest ``` Granular extras: `[proxy]`, `[mcp]`, `[ml]` (Kompress-v2-base), `[code]`, `[memory]`, `[vector]` (optional HNSW backend — needs a C++ toolchain, not in `[all]`), `[relevance]`, `[image]`, `[agno]`, `[langchain]`, `[evals]`, `[pytorch-mps]` (Apple-GPU memory-embedder offload — set `HEADROOM_EMBEDDER_RUNTIME=pytorch_mps`). Requires **Python 3.10+**. diff --git a/TESTING-copilot-subscription.md b/TESTING-copilot-subscription.md index 69204ca24..2255ccb28 100644 --- a/TESTING-copilot-subscription.md +++ b/TESTING-copilot-subscription.md @@ -107,7 +107,7 @@ There is **no native Windows wheel yet**, so pick one: **A. Mechanism test (easiest — Docker Desktop or WSL2):** ```powershell -$env:HEADROOM_DOCKER_IMAGE = "ghcr.io/chopratejas/headroom:" # ask the maintainer for the tag +$env:HEADROOM_DOCKER_IMAGE = "ghcr.io/headroomlabs-ai/headroom:" # ask the maintainer for the tag # run the Docker-native installer (scripts/install.ps1), then: $env:GITHUB_COPILOT_TOKEN = "" headroom wrap copilot --subscription -- --model gpt-4o -p "Reply with: HEADROOM_OK" diff --git a/crates/headroom-core/Cargo.toml b/crates/headroom-core/Cargo.toml index c47d2a88b..c1f0d5537 100644 --- a/crates/headroom-core/Cargo.toml +++ b/crates/headroom-core/Cargo.toml @@ -22,7 +22,7 @@ tokenizers = "0.22" # with `rustls` (no system OpenSSL dep — keeps the binary static-linkable for # AWS deploys). `from_pretrained` is called once at startup, so blocking is # fine; if a tokio caller needs it later we can wrap in `spawn_blocking`. -hf-hub = { version = "0.4", default-features = false, features = ["ureq", "rustls-tls"] } +hf-hub = { version = "0.5", default-features = false, features = ["ureq", "rustls-tls"] } # `md5` for the CCR cache_key. Python's compression_store hashes the original # diff with MD5 truncated to 24 hex chars; we must match byte-for-byte. md-5 = "0.10" @@ -125,7 +125,7 @@ blake3 = "1" # image may lag behind. Sub-1 MB binary cost. WAL is enabled at # connection-open time (see `ccr/backends/sqlite.rs`); no extra feature # flags required. -rusqlite = { version = "0.32", features = ["bundled"] } +rusqlite = { version = "0.40", features = ["bundled"] } # `redis` for the optional multi-worker CCR backend. Cfg-gated behind # the `redis` feature so deploys that don't need it pay no compile # cost. Default features include the sync `Connection` API used in @@ -196,7 +196,7 @@ redis = ["dep:redis"] [dev-dependencies] proptest = "1" -criterion = { version = "0.5", features = ["html_reports"] } +criterion = { version = "0.8", features = ["html_reports"] } tempfile = "3" [[bench]] diff --git a/crates/headroom-core/src/lib.rs b/crates/headroom-core/src/lib.rs index 8a907045a..a9417bd49 100644 --- a/crates/headroom-core/src/lib.rs +++ b/crates/headroom-core/src/lib.rs @@ -7,6 +7,7 @@ pub mod compression_policy; #[cfg(feature = "ml")] mod onnx_cpu; pub mod relevance; +pub mod rollout; pub mod signals; pub mod tokenizer; pub mod transforms; diff --git a/crates/headroom-core/src/rollout.rs b/crates/headroom-core/src/rollout.rs new file mode 100644 index 000000000..fc9d4fb92 --- /dev/null +++ b/crates/headroom-core/src/rollout.rs @@ -0,0 +1,439 @@ +//! Deterministic runtime-rollout policy and provenance. +//! +//! Rollout channels control behavior in an already-built artifact. They do not +//! select a package, release candidate, or distribution version. Composition +//! roots resolve one immutable snapshot and inject its concrete decisions. + +use serde::Serialize; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeSet; +use std::str::FromStr; + +pub const ROLLOUT_SCHEMA_VERSION: u32 = 1; +pub const ROLLOUT_POLICY_VERSION: &str = "1"; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, PartialOrd, Ord, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum RolloutChannel { + #[default] + Stable, + Beta, + Canary, + Dev, +} + +impl RolloutChannel { + pub fn as_str(self) -> &'static str { + match self { + Self::Stable => "stable", + Self::Beta => "beta", + Self::Canary => "canary", + Self::Dev => "dev", + } + } + + pub fn allows(self, required: Self) -> bool { + self >= required + } +} + +impl FromStr for RolloutChannel { + type Err = (); + + fn from_str(value: &str) -> Result { + match value.trim().to_ascii_lowercase().replace('-', "_").as_str() { + "" | "stable" | "prod" | "production" => Ok(Self::Stable), + "beta" | "preview" => Ok(Self::Beta), + "canary" | "nightly" => Ok(Self::Canary), + "dev" | "development" => Ok(Self::Dev), + _ => Err(()), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Feature { + NativeBedrock, + OpenAiResponsesStreaming, + CanaryProbe, +} + +const ALL_FEATURES: [Feature; 3] = [ + Feature::CanaryProbe, + Feature::NativeBedrock, + Feature::OpenAiResponsesStreaming, +]; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +pub struct FeatureSpec { + pub name: &'static str, + pub available_in: RolloutChannel, + pub default_enabled_in: Option, +} + +impl Feature { + pub fn spec(self) -> FeatureSpec { + match self { + Self::NativeBedrock => FeatureSpec { + name: "native_bedrock", + available_in: RolloutChannel::Stable, + default_enabled_in: Some(RolloutChannel::Stable), + }, + Self::OpenAiResponsesStreaming => FeatureSpec { + name: "openai_responses_streaming", + available_in: RolloutChannel::Stable, + default_enabled_in: Some(RolloutChannel::Stable), + }, + Self::CanaryProbe => FeatureSpec { + name: "canary_probe", + available_in: RolloutChannel::Canary, + default_enabled_in: None, + }, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum FeatureDecisionReason { + Default, + Explicit, + LegacyAlias, + Disabled, + BlockedByChannel, + UnsafeOverride, + NotRequested, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct RolloutConfig { + pub channel: RolloutChannel, + pub requested: BTreeSet, + pub disabled: BTreeSet, + pub unsafe_allow_unstable: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct FeatureDecision { + pub name: &'static str, + pub available_in: RolloutChannel, + pub default_enabled_in: Option, + pub requested: bool, + pub disabled: bool, + pub enabled: bool, + #[serde(rename = "decision")] + pub reason: FeatureDecisionReason, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RolloutSnapshot { + pub schema_version: u32, + pub policy_version: &'static str, + pub registry_digest: String, + pub config: RolloutConfig, + pub decisions: Vec, +} + +impl Default for RolloutSnapshot { + fn default() -> Self { + Self::from_parts("stable", "", "", false) + } +} + +impl RolloutSnapshot { + pub fn from_parts( + channel: &str, + requested: &str, + disabled: &str, + unsafe_allow_unstable: bool, + ) -> Self { + Self::from_parts_with_explicit(channel, requested, disabled, unsafe_allow_unstable, &[]) + } + + pub fn from_parts_with_explicit( + channel: &str, + requested: &str, + disabled: &str, + unsafe_allow_unstable: bool, + explicit: &[Feature], + ) -> Self { + let parsed_channel = RolloutChannel::from_str(channel).unwrap_or_else(|_| { + tracing::warn!(channel, "unknown rollout channel; falling back to stable"); + RolloutChannel::Stable + }); + let valid_names: BTreeSet<_> = ALL_FEATURES + .iter() + .map(|feature| feature.spec().name.to_owned()) + .collect(); + let mut requested_names = validated_names(requested, "requested", &valid_names); + requested_names.extend( + explicit + .iter() + .map(|feature| feature.spec().name.to_owned()), + ); + let disabled_names = validated_names(disabled, "disabled", &valid_names); + let config = RolloutConfig { + channel: parsed_channel, + requested: requested_names, + disabled: disabled_names, + unsafe_allow_unstable, + }; + let decisions = ALL_FEATURES + .iter() + .map(|feature| resolve_feature(*feature, &config)) + .collect(); + Self { + schema_version: ROLLOUT_SCHEMA_VERSION, + policy_version: ROLLOUT_POLICY_VERSION, + registry_digest: registry_digest(), + config, + decisions, + } + } + + pub fn decision(&self, feature: Feature) -> &FeatureDecision { + let name = feature.spec().name; + self.decisions + .iter() + .find(|decision| decision.name == name) + .expect("every registered feature has a decision") + } + + pub fn is_enabled(&self, feature: Feature, _explicit: bool) -> bool { + self.decision(feature).enabled + } + + pub fn enabled(&self) -> BTreeSet { + self.decisions + .iter() + .filter(|decision| decision.enabled) + .map(|decision| decision.name.to_owned()) + .collect() + } + + pub fn qualification_eligible(&self) -> bool { + !self.config.unsafe_allow_unstable + } + + fn canonical_value(&self) -> Value { + json!({ + "schema_version": self.schema_version, + "policy_version": self.policy_version, + "channel": self.config.channel, + "unsafe_override": self.config.unsafe_allow_unstable, + "registry_digest": self.registry_digest, + "features": self.decisions, + }) + } + + pub fn snapshot_digest(&self) -> String { + digest_value(&self.canonical_value()) + } + + pub fn to_value(&self) -> Value { + let mut value = self.canonical_value(); + let object = value + .as_object_mut() + .expect("rollout snapshot is an object"); + object.insert("snapshot_digest".into(), json!(self.snapshot_digest())); + object.insert( + "qualification_eligible".into(), + json!(self.qualification_eligible()), + ); + if !self.qualification_eligible() { + object.insert( + "qualification_ineligible_reason".into(), + json!("unsafe_rollout_override_active"), + ); + } + value + } +} + +fn resolve_feature(feature: Feature, config: &RolloutConfig) -> FeatureDecision { + let spec = feature.spec(); + let requested = config.requested.contains(spec.name); + let disabled = config.disabled.contains(spec.name); + let normally_available = config.channel.allows(spec.available_in); + let (enabled, reason) = if disabled { + (false, FeatureDecisionReason::Disabled) + } else if requested && !normally_available && !config.unsafe_allow_unstable { + (false, FeatureDecisionReason::BlockedByChannel) + } else if requested && !normally_available { + (true, FeatureDecisionReason::UnsafeOverride) + } else if requested { + (true, FeatureDecisionReason::Explicit) + } else if spec + .default_enabled_in + .is_some_and(|minimum| config.channel.allows(minimum)) + { + (true, FeatureDecisionReason::Default) + } else { + (false, FeatureDecisionReason::NotRequested) + }; + FeatureDecision { + name: spec.name, + available_in: spec.available_in, + default_enabled_in: spec.default_enabled_in, + requested, + disabled, + enabled, + reason, + } +} + +fn validated_names(raw: &str, source: &str, valid: &BTreeSet) -> BTreeSet { + let names: BTreeSet<_> = split_feature_names(raw).into_iter().collect(); + for unknown in names.difference(valid) { + tracing::warn!( + feature = unknown, + source, + "unknown rollout feature; ignoring (fail-closed)" + ); + } + names.intersection(valid).cloned().collect() +} + +pub fn split_feature_names(raw: &str) -> Vec { + raw.replace(';', ",") + .split(',') + .filter_map(|part| { + let normalized = normalize_feature_name(part); + (!normalized.is_empty()).then_some(normalized) + }) + .collect() +} + +pub fn normalize_feature_name(raw: impl AsRef) -> String { + raw.as_ref().trim().to_ascii_lowercase().replace('-', "_") +} + +pub fn registry_digest() -> String { + let registry: Vec<_> = ALL_FEATURES.iter().map(|feature| feature.spec()).collect(); + digest_value(&serde_json::to_value(registry).expect("registry is serializable")) +} + +pub fn feature_names() -> BTreeSet<&'static str> { + ALL_FEATURES + .iter() + .map(|feature| feature.spec().name) + .collect() +} + +fn digest_value(value: &Value) -> String { + let canonical = serde_json::to_vec(value).expect("rollout provenance is serializable"); + format!("sha256:{:x}", Sha256::digest(canonical)) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde::Deserialize; + + #[derive(Deserialize)] + struct PolicyVector { + channel: String, + requested: bool, + disabled: bool, + #[serde(rename = "unsafe")] + unsafe_override: bool, + enabled: bool, + decision: String, + } + + #[test] + fn channel_order_matches_python_policy() { + assert!(RolloutChannel::Dev.allows(RolloutChannel::Canary)); + assert!(RolloutChannel::Canary.allows(RolloutChannel::Beta)); + assert!(!RolloutChannel::Stable.allows(RolloutChannel::Canary)); + } + + #[test] + fn stable_blocks_explicit_canary_feature_with_reason() { + let rollout = RolloutSnapshot::from_parts("stable", "canary_probe", "", false); + let decision = rollout.decision(Feature::CanaryProbe); + assert!(!decision.enabled); + assert_eq!(decision.reason, FeatureDecisionReason::BlockedByChannel); + } + + #[test] + fn default_enabled_feature_has_default_reason() { + let rollout = RolloutSnapshot::default(); + let decision = rollout.decision(Feature::NativeBedrock); + assert!(decision.enabled); + assert_eq!(decision.reason, FeatureDecisionReason::Default); + } + + #[test] + fn unsafe_override_crosses_boundary_and_is_ineligible() { + let rollout = RolloutSnapshot::from_parts("stable", "canary_probe", "", true); + assert_eq!( + rollout.decision(Feature::CanaryProbe).reason, + FeatureDecisionReason::UnsafeOverride + ); + assert!(!rollout.qualification_eligible()); + assert_eq!( + rollout.to_value()["qualification_ineligible_reason"], + "unsafe_rollout_override_active" + ); + } + + #[test] + fn disable_beats_default_explicit_and_unsafe() { + for unsafe_override in [false, true] { + let rollout = RolloutSnapshot::from_parts( + "stable", + "native_bedrock", + "native-bedrock", + unsafe_override, + ); + assert_eq!( + rollout.decision(Feature::NativeBedrock).reason, + FeatureDecisionReason::Disabled + ); + } + } + + #[test] + fn provenance_digests_are_deterministic_and_policy_sensitive() { + let first = RolloutSnapshot::from_parts("canary", "canary_probe", "", false); + let second = RolloutSnapshot::from_parts("canary", "canary_probe", "", false); + let changed = RolloutSnapshot::from_parts("stable", "canary_probe", "", false); + assert_eq!(first.registry_digest, second.registry_digest); + assert_eq!(first.snapshot_digest(), second.snapshot_digest()); + assert_ne!(first.snapshot_digest(), changed.snapshot_digest()); + } + + #[test] + fn invalid_inputs_fail_closed() { + let rollout = RolloutSnapshot::from_parts("stabel", "unknown", "unknown", false); + assert_eq!(rollout.config.channel, RolloutChannel::Stable); + assert!(rollout.config.requested.is_empty()); + assert!(rollout.config.disabled.is_empty()); + } + + #[test] + fn shared_python_rust_policy_vectors() { + let vectors: Vec = serde_json::from_str(include_str!( + "../../../tests/fixtures/rollout_policy_vectors.json" + )) + .unwrap(); + for vector in vectors { + let requested = if vector.requested { "canary_probe" } else { "" }; + let disabled = if vector.disabled { "canary_probe" } else { "" }; + let rollout = RolloutSnapshot::from_parts( + &vector.channel, + requested, + disabled, + vector.unsafe_override, + ); + let decision = rollout.decision(Feature::CanaryProbe); + assert_eq!(decision.enabled, vector.enabled); + assert_eq!( + serde_json::to_value(decision.reason).unwrap(), + vector.decision + ); + } + } +} diff --git a/crates/headroom-core/src/tokenizer/estimator.rs b/crates/headroom-core/src/tokenizer/estimator.rs index 5dde222d3..818077be0 100644 --- a/crates/headroom-core/src/tokenizer/estimator.rs +++ b/crates/headroom-core/src/tokenizer/estimator.rs @@ -1,12 +1,35 @@ //! Character-density estimator. Used as a fallback for any tokenizer family //! we haven't wired in yet (Anthropic Claude, Google Gemini, Cohere, …). //! -//! Mirrors `headroom.tokenizers.estimator.EstimatingTokenCounter`. The formula -//! is `ceil(chars / chars_per_token)`. `chars` is *Unicode scalar count*, not -//! byte length, to match Python's `len(text)` semantics on str. +//! Mirrors `headroom.tokenizers.estimator.EstimatingTokenCounter`. Latin chars +//! are priced at `chars_per_token`; dense scripts (CJK / Kana / Hangul / full- +//! width) are priced separately at `CHARS_PER_TOKEN_CJK`, since they tokenize at +//! ~1 token/char and the Latin ratio under-counts them 2-4x. `chars` is a +//! *Unicode scalar count*, not byte length, to match Python's `len(text)`. use super::{Backend, Tokenizer}; +/// Chars-per-token for dense scripts. Byte-identical with Python +/// `EstimatingTokenCounter.CHARS_PER_TOKEN_CJK`. +const CHARS_PER_TOKEN_CJK: f64 = 1.5; + +/// True for a "dense-script" codepoint (CJK ideographs + punctuation, Kana, +/// Hangul, CJK compatibility, half/full-width forms, CJK Ext-A/B). Ranges kept +/// byte-identical with Python `EstimatingTokenCounter.CJK_PATTERN`. +fn is_dense_script(c: char) -> bool { + matches!( + c as u32, + 0x3000..=0x303F // CJK symbols and punctuation + | 0x3040..=0x30FF // Hiragana + Katakana + | 0x3400..=0x4DBF // CJK Unified Ideographs Ext A + | 0x4E00..=0x9FFF // CJK Unified Ideographs + | 0xAC00..=0xD7AF // Hangul syllables + | 0xF900..=0xFAFF // CJK compatibility ideographs + | 0xFF00..=0xFFEF // Half/full-width forms + | 0x20000..=0x2A6DF // CJK Unified Ideographs Ext B + ) +} + #[derive(Debug, Clone, Copy)] pub struct EstimatingCounter { chars_per_token: f64, @@ -42,15 +65,15 @@ impl Tokenizer for EstimatingCounter { if text.is_empty() { return 0; } - // Match Python `EstimatingTokenCounter.count_text`: - // max(1, int(len(text) / chars_per_token + 0.5)) - // Python `int()` truncates toward zero; for non-negative inputs that's - // identical to `as usize` saturating-cast semantics in Rust >= 1.45. - // Adding 0.5 then truncating yields round-half-up. We previously used - // ceil, which over-counted in the middle of the range (e.g. "aaaaa" - // at 4.0 cpt returned 2 here vs 1 in Python). - let chars = text.chars().count() as f64; - let raw = (chars / self.chars_per_token + 0.5) as usize; + // Match Python `EstimatingTokenCounter.count_text` (fixed-ratio path): + // cjk = count_dense_script(text); other = len(text) - cjk + // max(1, int(other / chars_per_token + cjk / CHARS_PER_TOKEN_CJK + 0.5)) + // Dense scripts tokenize at ~1 token/char, so the Latin `chars_per_token` + // under-counts them; price them separately. `int()` truncates toward + // zero (== `as usize` for non-negative); the `+ 0.5` gives round-half-up. + let cjk = text.chars().filter(|&c| is_dense_script(c)).count(); + let other = (text.chars().count() - cjk) as f64; + let raw = (other / self.chars_per_token + cjk as f64 / CHARS_PER_TOKEN_CJK + 0.5) as usize; raw.max(1) } @@ -104,6 +127,27 @@ mod tests { assert_eq!(est.count_text("🦀🦀🦀🦀"), 1); } + #[test] + fn dense_scripts_priced_at_cjk_ratio() { + let est = EstimatingCounter::default(); // 4.0 for Latin + // Pure CJK: cjk=3, other=0 -> 0/4 + 3/1.5 + 0.5 = 2.5 -> int -> 2 + assert_eq!(est.count_text("数据库"), 2); + // 7 CJK -> 7/1.5 + 0.5 = 5.16 -> 5 (the old flat 7/4 -> 2 under-counted ~2.5x) + assert_eq!(est.count_text("数据库连接失败"), 5); + // Kana is dense: 3 hiragana -> 3/1.5 + 0.5 = 2.5 -> 2 + assert_eq!(est.count_text("ひらが"), 2); + // Full-width Latin is dense (U+FF00-FFEF): API -> 2, vs plain "API" -> 1 + assert_eq!(est.count_text("API"), 2); + assert_eq!(est.count_text("API"), 1); + } + + #[test] + fn mixed_ascii_and_cjk_prices_each_separately() { + let est = EstimatingCounter::default(); + // "api数据": other=3, cjk=2 -> 3/4 + 2/1.5 + 0.5 = 0.75+1.33+0.5 = 2.58 -> 2 + assert_eq!(est.count_text("api数据"), 2); + } + #[test] fn min_is_one_for_non_empty_input() { let est = EstimatingCounter::default(); diff --git a/crates/headroom-proxy/Cargo.toml b/crates/headroom-proxy/Cargo.toml index 898645e9d..b1f5088f1 100644 --- a/crates/headroom-proxy/Cargo.toml +++ b/crates/headroom-proxy/Cargo.toml @@ -23,7 +23,7 @@ tower-http = { version = "0.7", features = ["trace", "request-id", "util"] } tracing = { workspace = true } tracing-subscriber = { version = "0.3", features = ["json", "env-filter", "fmt"] } reqwest = { version = "0.12", default-features = false, features = ["stream", "rustls-tls", "http2"] } -tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-webpki-roots"] } +tokio-tungstenite = { version = "0.30", default-features = false, features = ["connect", "rustls-tls-webpki-roots"] } clap = { workspace = true, features = ["derive", "env"] } serde = { workspace = true } serde_json = { workspace = true } @@ -38,7 +38,7 @@ http-body-util = "0.1" hyper = "1" url = "2" humantime = "2" -bytesize = "1" +bytesize = "2" tokio-util = { version = "0.7" } headroom-core = { path = "../headroom-core" } # Phase D PR-D1: native Bedrock InvokeModel route. SigV4 + AWS @@ -98,7 +98,7 @@ md-5 = "0.10" tower = { workspace = true, features = ["util"] } wiremock = "0.6" reqwest = { version = "0.12", default-features = false, features = ["stream", "rustls-tls", "http2", "json"] } -tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-webpki-roots"] } +tokio-tungstenite = { version = "0.30", default-features = false, features = ["connect", "rustls-tls-webpki-roots"] } futures-util = "0.3" tokio = { workspace = true, features = ["macros", "rt-multi-thread", "signal", "net", "io-util", "time", "test-util", "process"] } hyper = { version = "1", features = ["server", "http1", "http2"] } diff --git a/crates/headroom-proxy/src/bedrock/invoke.rs b/crates/headroom-proxy/src/bedrock/invoke.rs index 2571e8bcb..a30e0027a 100644 --- a/crates/headroom-proxy/src/bedrock/invoke.rs +++ b/crates/headroom-proxy/src/bedrock/invoke.rs @@ -615,6 +615,7 @@ mod tests { client: reqwest::Client::new(), bedrock_credentials: None, drift_state: crate::cache_stabilization::drift_detector::DriftState::new(8), + beta_sticky: crate::cache_stabilization::beta_sticky::BetaStickyState::new(8), vertex_token_source: std::sync::Arc::new(crate::vertex::StaticTokenSource::new( "test".to_string(), )), @@ -649,6 +650,7 @@ mod tests { // unit test never observes drift, but `AppState` requires // the field to be populated. drift_state: crate::cache_stabilization::drift_detector::DriftState::new(8), + beta_sticky: crate::cache_stabilization::beta_sticky::BetaStickyState::new(8), // PR-D4: unit tests for the Bedrock URL builder don't // touch the Vertex route, but `AppState` is one struct // — supply a dummy token source so the test compiles. @@ -684,6 +686,7 @@ mod tests { // PR-E6: see above — drift detector is unused by this // test; we just satisfy the struct shape. drift_state: crate::cache_stabilization::drift_detector::DriftState::new(8), + beta_sticky: crate::cache_stabilization::beta_sticky::BetaStickyState::new(8), // PR-D4: unit tests for the Bedrock URL builder don't // touch the Vertex route, but `AppState` is one struct // — supply a dummy token source so the test compiles. diff --git a/crates/headroom-proxy/src/bedrock/invoke_streaming.rs b/crates/headroom-proxy/src/bedrock/invoke_streaming.rs index ce66429f9..62501003c 100644 --- a/crates/headroom-proxy/src/bedrock/invoke_streaming.rs +++ b/crates/headroom-proxy/src/bedrock/invoke_streaming.rs @@ -1014,6 +1014,7 @@ mod tests { // PR-E6: drift detector is unused by this URL-builder // unit test; small capacity to satisfy the struct shape. drift_state: crate::cache_stabilization::drift_detector::DriftState::new(8), + beta_sticky: crate::cache_stabilization::beta_sticky::BetaStickyState::new(8), // PR-D4: unit tests for the Bedrock URL builder don't // touch the Vertex route, but `AppState` is one struct // — supply a dummy token source so the test compiles. @@ -1056,6 +1057,7 @@ mod tests { client: reqwest::Client::new(), bedrock_credentials: None, drift_state: crate::cache_stabilization::drift_detector::DriftState::new(8), + beta_sticky: crate::cache_stabilization::beta_sticky::BetaStickyState::new(8), vertex_token_source: std::sync::Arc::new(crate::vertex::StaticTokenSource::new( "test".to_string(), )), diff --git a/crates/headroom-proxy/src/cache_stabilization/beta_sticky.rs b/crates/headroom-proxy/src/cache_stabilization/beta_sticky.rs new file mode 100644 index 000000000..a8db6a5bc --- /dev/null +++ b/crates/headroom-proxy/src/cache_stabilization/beta_sticky.rs @@ -0,0 +1,620 @@ +//! Session-sticky provider beta headers — Rust port of the Python +//! proxy's `SessionBetaTracker` (PR-A6, `headroom/proxy/helpers.py`). +//! +//! ## Why +//! +//! Provider beta headers (`anthropic-beta`, `openai-beta`) are part of +//! the request bytes that determine the upstream prefix-cache key. +//! Interactive clients (Claude Code, Codex CLI) MAY drop a beta token +//! between turn N and turn N+1 of the same conversation; the cache hot +//! zone is positional, so the next turn's prefix hashes differently and +//! the prefix-cache read misses — the customer silently pays for a full +//! prompt re-write. The Python proxy defeats this with a bounded LRU +//! tracker that unions the client's tokens with every token previously +//! seen for the same `(provider, session)` and forwards the union. +//! +//! The Rust proxy replaces the Python request path in Phase H, which +//! deletes `SessionBetaTracker` with the rest of +//! `headroom/proxy/helpers.py`. Without this port the protection — +//! and its documented operator contract +//! (`docs/content/docs/configuration.mdx`, "Session Beta Header +//! Tracking") — would silently not survive the migration. +//! +//! ## Behaviour contract (parity with Python) +//! +//! - Union client tokens with previously-seen tokens for the session, +//! preserving first-seen order; case-insensitive dedup where the +//! first-seen casing wins. +//! - Keyed by `(provider, session)` so the same session id against +//! Anthropic and OpenAI upstreams keeps independent token sets. +//! - Bounded LRU (`BETA_TRACKER_CAPACITY` sessions): lookups touch +//! recency, overflow evicts the oldest session. +//! - The tracker only ever records tokens the client itself sent. +//! Headroom-added tokens (e.g. memory-tool betas on the Python +//! path) are NOT recorded — the forwarded union is always a subset +//! of values this client already put on the wire, which is what +//! keeps the mechanism consistent with the subscription-stealth +//! invariant (REALIGNMENT invariant #10: "no beta drift"). +//! +//! The operator opt-out lives at the call site: when +//! `Config::beta_header_sticky` is `disabled` the proxy skips the +//! tracker entirely and forwards the client header verbatim (the +//! Python proxy's `HEADROOM_BETA_HEADER_STICKY=disabled` diagnostic +//! mode). That gate is per REALIGNMENT build constraint #4 an explicit +//! loud opt-in, not a silent fallback. +//! +//! Session identity comes from +//! [`super::drift_detector::derive_session_key`] — the same +//! conversation-aware key the drift detector uses (explicit +//! `x-headroom-session-id` when the client declares it, otherwise +//! credential/IP arms folded with a first-message conversation +//! discriminator). +//! +//! ## Divergence from Python: per-conversation, not per-(model, system) +//! +//! The Python tracker keys on the store session id — explicit header, +//! else a hash of `(model, leading system prompt)` — so all parallel +//! conversations sharing a model + system prompt (a Claude Code +//! session and every one of its subagents) share ONE token union and +//! cross-inherit each other's tokens. This port keys on the drift +//! detector's conversation-aware key instead, so each conversation +//! keeps its own union; the integration test +//! `separate_conversations_do_not_leak_tokens` pins that. Deliberate: +//! the `(model, system)` bucket conflating parallel agentic +//! conversations is the exact defect #2085 / #2193 / #2301 chased out +//! of the other session-sticky subsystems. The cost is losing +//! Python's accidental cross-conversation repair (conversation B +//! turn 1 inheriting a token only conversation A ever sent); each +//! conversation's stickiness now starts from its own first sighting, +//! which is also the only variant that can't leak one tenant-visible +//! experiment token into an unrelated conversation's request bytes. + +use std::collections::HashSet; +use std::num::NonZeroUsize; +use std::sync::{Arc, Mutex}; + +use http::header::{HeaderMap, HeaderValue}; +use lru::LruCache; + +use super::drift_detector::session_key_log_prefix; + +/// Maximum number of `(provider, session)` entries tracked. Sessions +/// are keyed per conversation (see module docs), so the working set is +/// the number of concurrently active conversations — same sizing +/// rationale as the drift detector's capacity. Eviction cost is +/// re-learning a live session's dropped tokens from scratch (the next +/// turn forwards the client value verbatim), not a lost request. +pub const BETA_TRACKER_CAPACITY: usize = 1000; + +/// Upstream namespace for a tracked beta-token set. Mirrors the +/// Python tracker's `provider` string key ("anthropic" / "openai"). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum BetaProvider { + /// `/v1/messages` — `anthropic-beta` header. + Anthropic, + /// `/v1/chat/completions` and `/v1/responses` — `openai-beta` + /// header. One namespace for both endpoints, matching the Python + /// proxy's single `provider="openai"` key. + OpenAi, +} + +impl BetaProvider { + /// Stable lower-case label for log fields; matches the Python + /// tracker's provider strings. + pub fn as_str(self) -> &'static str { + match self { + BetaProvider::Anthropic => "anthropic", + BetaProvider::OpenAi => "openai", + } + } + + /// The request header this provider's beta tokens travel in. + pub fn header_name(self) -> &'static str { + match self { + BetaProvider::Anthropic => "anthropic-beta", + BetaProvider::OpenAi => "openai-beta", + } + } +} + +/// Split a comma-separated beta-header value into trimmed, non-empty +/// tokens. Port of the Python `split_beta_tokens` helper. +pub fn split_beta_tokens(value: Option<&str>) -> Vec { + value + .unwrap_or("") + .split(',') + .map(str::trim) + .filter(|t| !t.is_empty()) + .map(str::to_string) + .collect() +} + +/// Per-session ordered token lists, keyed by `(provider, session)`. +type SessionTokenCache = LruCache<(BetaProvider, String), Vec>; + +/// Bounded LRU of beta tokens observed per `(provider, session)`. +/// +/// Cloning shares the underlying map (`Arc`), mirroring +/// [`super::drift_detector::DriftState`] so one instance lives in +/// `AppState` and clones freely into every handler path. +#[derive(Clone)] +pub struct BetaStickyState { + sessions: Arc>, +} + +impl BetaStickyState { + /// Create a tracker bounded to `capacity` sessions. + /// + /// # Panics + /// + /// Panics when `capacity == 0`, mirroring `DriftState::new` (the + /// Python tracker raises `ValueError` on a non-positive bound). + pub fn new(capacity: usize) -> Self { + let cap = NonZeroUsize::new(capacity).expect("BetaStickyState capacity must be > 0"); + Self { + sessions: Arc::new(Mutex::new(LruCache::new(cap))), + } + } + + /// Union `client_value`'s tokens with the session's previously + /// seen tokens, update the session, and return the merged + /// comma-separated value (possibly empty). Port of the Python + /// `SessionBetaTracker.record_and_get_sticky_betas`. + /// + /// On a poisoned lock the tracker fails open: the client value is + /// returned verbatim (trimmed) and state is left untouched — + /// never drop or delay the request for a telemetry-adjacent + /// protection. + pub fn record_and_get_sticky_betas( + &self, + provider: BetaProvider, + session_key: &str, + client_value: Option<&str>, + ) -> String { + let client_tokens = split_beta_tokens(client_value); + + let mut sessions = match self.sessions.lock() { + Ok(guard) => guard, + Err(poisoned) => { + tracing::warn!( + event = "beta_sticky_lock_poisoned", + provider = provider.as_str(), + "beta tracker lock poisoned; forwarding client value verbatim" + ); + drop(poisoned); + return client_tokens.join(","); + } + }; + + let key = (provider, session_key.to_string()); + // `get_mut` touches LRU recency on hit, mirroring the Python + // tracker's move-to-end. + if let Some(merged) = sessions.get_mut(&key) { + // Dedup is case-insensitive with the first-seen casing + // winning. Header values reaching this point are visible + // ASCII (`HeaderValue::to_str` rejects anything else), so + // ASCII lowercasing matches Python's `str.lower()` over + // the reachable domain. + let mut seen: HashSet = merged.iter().map(|t| t.to_ascii_lowercase()).collect(); + for token in client_tokens { + if seen.insert(token.to_ascii_lowercase()) { + merged.push(token); + } + } + return merged.join(","); + } + + let mut merged: Vec = Vec::with_capacity(client_tokens.len()); + let mut seen: HashSet = HashSet::with_capacity(client_tokens.len()); + for token in client_tokens { + if seen.insert(token.to_ascii_lowercase()) { + merged.push(token); + } + } + let joined = merged.join(","); + // `put` on a fresh key evicts the oldest entry once the cache + // is at capacity — the Python tracker's bounded-LRU overflow + // pop. Sessions that never sent a beta token still occupy a + // slot (Python stores their empty list too); the cost is one + // LRU entry, the benefit is identical recency behaviour. + sessions.put(key, merged); + joined + } + + /// Number of tracked sessions (test observability). + #[cfg(test)] + fn active_sessions(&self) -> usize { + self.sessions.lock().map(|c| c.len()).unwrap_or(0) + } +} + +/// Count tokens in a raw header value without allocating a `Vec` +/// (log-field helper; same tokenization as [`split_beta_tokens`]). +fn count_beta_tokens(value: Option<&str>) -> usize { + value + .unwrap_or("") + .split(',') + .filter(|t| !t.trim().is_empty()) + .count() +} + +/// Record the client's beta header for this `(provider, session)` and +/// rewrite the upstream-bound header to the session union when they +/// differ. The full merge site: reads `provider.header_name()` from +/// `outgoing_headers`, unions via the tracker, mutates the map in +/// place. Mirrors the Python handler block (anthropic.py PR-A6): +/// rewrite only when the union is non-empty and differs from the +/// client value; an absent client header gains the union; a session +/// with no tokens anywhere stays header-less. +/// +/// Fail-open contract: a client value that isn't visible ASCII is +/// forwarded verbatim and nothing is recorded (never rewrite what we +/// can't faithfully parse); an unencodable union (unreachable — every +/// token came from a parsed header value) logs and forwards verbatim. +/// +/// Logging: counts only — beta tokens can carry experiment IDs the +/// user hasn't opted to share with Headroom logs (Python +/// `log_beta_header_merge` contract). Python logs every merge at +/// info; here the no-op case drops to debug, matching the drift +/// detector's silent-on-stable precedent, so an info-level +/// `beta_header_merge` always marks an actual cache-affecting +/// rewrite. +pub fn apply_sticky_betas( + tracker: &BetaStickyState, + provider: BetaProvider, + session_key: &str, + outgoing_headers: &mut HeaderMap, + request_id: &str, +) { + let header_name = provider.header_name(); + // Join repeated field lines with "," per RFC 9110 §5.3 list + // semantics BEFORE recording, so a client sending two beta lines + // has both recorded and a later rewrite (which `insert`s a single + // line, dropping the others) can never shrink the upstream token + // set mid-conversation. + let mut parts: Vec<&str> = Vec::new(); + for raw in outgoing_headers.get_all(header_name) { + match raw.to_str() { + Ok(s) => parts.push(s), + Err(_) => { + tracing::debug!( + event = "beta_header_merge_skipped", + request_id = %request_id, + provider = provider.as_str(), + reason = "non_ascii_header_value", + "client beta header is not visible ASCII; forwarding verbatim" + ); + return; + } + } + } + let client_value: Option = if parts.is_empty() { + None + } else { + Some(parts.join(",")) + }; + + let sticky = + tracker.record_and_get_sticky_betas(provider, session_key, client_value.as_deref()); + let rewritten = !sticky.is_empty() && sticky != client_value.as_deref().unwrap_or(""); + if rewritten { + match HeaderValue::from_str(&sticky) { + Ok(value) => { + outgoing_headers.insert(header_name, value); + } + Err(error) => { + tracing::warn!( + event = "beta_header_merge_skipped", + request_id = %request_id, + provider = provider.as_str(), + reason = "unencodable_union", + error = %error, + "sticky beta union not encodable as a header value" + ); + return; + } + } + } + + let client_betas = count_beta_tokens(client_value.as_deref()); + let sticky_betas = count_beta_tokens(Some(&sticky)); + if rewritten { + tracing::info!( + event = "beta_header_merge", + request_id = %request_id, + provider = provider.as_str(), + session_key_hash = %session_key_log_prefix(session_key), + client_betas, + sticky_betas, + "session-sticky beta merge rewrote the upstream header" + ); + } else { + tracing::debug!( + event = "beta_header_merge", + request_id = %request_id, + provider = provider.as_str(), + session_key_hash = %session_key_log_prefix(session_key), + client_betas, + sticky_betas, + "session-sticky beta merge (no-op)" + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // ----------------------------------------------------------------- + // split_beta_tokens — port of the Python tokenizer contract. + // ----------------------------------------------------------------- + + #[test] + fn split_none_and_empty_yield_no_tokens() { + assert!(split_beta_tokens(None).is_empty()); + assert!(split_beta_tokens(Some("")).is_empty()); + assert!(split_beta_tokens(Some(" ")).is_empty()); + assert!(split_beta_tokens(Some(",, ,")).is_empty()); + } + + #[test] + fn split_trims_and_drops_empty_segments() { + assert_eq!( + split_beta_tokens(Some(" a , ,b, c-1 ")), + vec!["a".to_string(), "b".to_string(), "c-1".to_string()] + ); + } + + // ----------------------------------------------------------------- + // record_and_get_sticky_betas — tracker semantics ported from + // tests/test_anthropic_beta_session_sticky.py. + // ----------------------------------------------------------------- + + fn tracker() -> BetaStickyState { + BetaStickyState::new(BETA_TRACKER_CAPACITY) + } + + #[test] + fn first_request_returns_client_tokens() { + let t = tracker(); + let got = t.record_and_get_sticky_betas(BetaProvider::Anthropic, "s1", Some("a,b")); + assert_eq!(got, "a,b"); + } + + #[test] + fn dropped_token_is_reinjected_on_next_turn() { + // The cache-killer this module exists for: turn N sends + // "a,b", turn N+1 drops "b" — the union must restore it. + let t = tracker(); + t.record_and_get_sticky_betas(BetaProvider::Anthropic, "s1", Some("a,b")); + let got = t.record_and_get_sticky_betas(BetaProvider::Anthropic, "s1", Some("a")); + assert_eq!(got, "a,b"); + } + + #[test] + fn union_preserves_first_seen_order() { + let t = tracker(); + t.record_and_get_sticky_betas(BetaProvider::Anthropic, "s1", Some("b,a")); + let got = t.record_and_get_sticky_betas(BetaProvider::Anthropic, "s1", Some("a,c")); + assert_eq!(got, "b,a,c"); + } + + #[test] + fn dedup_is_case_insensitive_first_casing_wins() { + let t = tracker(); + t.record_and_get_sticky_betas( + BetaProvider::Anthropic, + "s1", + Some("Context-Management-2025-06-27"), + ); + let got = t.record_and_get_sticky_betas( + BetaProvider::Anthropic, + "s1", + Some("context-management-2025-06-27"), + ); + assert_eq!(got, "Context-Management-2025-06-27"); + } + + #[test] + fn duplicate_client_tokens_are_deduped() { + let t = tracker(); + let got = t.record_and_get_sticky_betas(BetaProvider::Anthropic, "s1", Some("a,a,b,A")); + assert_eq!(got, "a,b"); + } + + #[test] + fn client_whitespace_is_trimmed_in_union() { + let t = tracker(); + t.record_and_get_sticky_betas(BetaProvider::Anthropic, "s1", Some(" a , b ")); + let got = t.record_and_get_sticky_betas(BetaProvider::Anthropic, "s1", Some("c ")); + assert_eq!(got, "a,b,c"); + } + + #[test] + fn absent_client_value_returns_session_union() { + let t = tracker(); + t.record_and_get_sticky_betas(BetaProvider::Anthropic, "s1", Some("a")); + let got = t.record_and_get_sticky_betas(BetaProvider::Anthropic, "s1", None); + assert_eq!(got, "a"); + } + + #[test] + fn empty_session_and_client_yield_empty_string() { + let t = tracker(); + assert_eq!( + t.record_and_get_sticky_betas(BetaProvider::Anthropic, "s1", None), + "" + ); + } + + #[test] + fn providers_keep_independent_namespaces() { + // Same session id, different providers — token sets must not + // leak across (Python: the (provider, session_id) tuple key). + let t = tracker(); + t.record_and_get_sticky_betas(BetaProvider::Anthropic, "s1", Some("anth-only")); + let got = t.record_and_get_sticky_betas(BetaProvider::OpenAi, "s1", Some("oai-only")); + assert_eq!(got, "oai-only"); + } + + #[test] + fn sessions_keep_independent_token_sets() { + let t = tracker(); + t.record_and_get_sticky_betas(BetaProvider::Anthropic, "s1", Some("a")); + let got = t.record_and_get_sticky_betas(BetaProvider::Anthropic, "s2", Some("b")); + assert_eq!(got, "b"); + } + + #[test] + fn lru_evicts_oldest_session_at_capacity() { + let t = BetaStickyState::new(2); + t.record_and_get_sticky_betas(BetaProvider::Anthropic, "s1", Some("a")); + t.record_and_get_sticky_betas(BetaProvider::Anthropic, "s2", Some("b")); + t.record_and_get_sticky_betas(BetaProvider::Anthropic, "s3", Some("c")); + assert_eq!(t.active_sessions(), 2); + // s1 was evicted: its history is gone, so a bare re-request + // returns only the fresh client value. + let got = t.record_and_get_sticky_betas(BetaProvider::Anthropic, "s1", Some("z")); + assert_eq!(got, "z"); + } + + #[test] + fn lru_hit_touches_recency() { + let t = BetaStickyState::new(2); + t.record_and_get_sticky_betas(BetaProvider::Anthropic, "s1", Some("a")); + t.record_and_get_sticky_betas(BetaProvider::Anthropic, "s2", Some("b")); + // Touch s1 so s2 becomes the eviction candidate. + t.record_and_get_sticky_betas(BetaProvider::Anthropic, "s1", None); + t.record_and_get_sticky_betas(BetaProvider::Anthropic, "s3", Some("c")); + // s1 survived the s3 insert… + assert_eq!( + t.record_and_get_sticky_betas(BetaProvider::Anthropic, "s1", None), + "a" + ); + // …and s2 did not. + assert_eq!( + t.record_and_get_sticky_betas(BetaProvider::Anthropic, "s2", None), + "" + ); + } + + // ----------------------------------------------------------------- + // apply_sticky_betas — header-map plumbing. + // ----------------------------------------------------------------- + + fn header_map(values: &[&str]) -> HeaderMap { + let mut map = HeaderMap::new(); + for v in values { + map.append("anthropic-beta", HeaderValue::from_str(v).unwrap()); + } + map + } + + fn beta_values(map: &HeaderMap) -> Vec { + map.get_all("anthropic-beta") + .iter() + .map(|v| v.to_str().unwrap().to_string()) + .collect() + } + + #[test] + fn apply_rewrites_dropped_token_to_union() { + let t = tracker(); + let mut turn1 = header_map(&["a,b"]); + apply_sticky_betas(&t, BetaProvider::Anthropic, "s1", &mut turn1, "req-1"); + assert_eq!(beta_values(&turn1), vec!["a,b"]); + + let mut turn2 = header_map(&["a"]); + apply_sticky_betas(&t, BetaProvider::Anthropic, "s1", &mut turn2, "req-2"); + assert_eq!(beta_values(&turn2), vec!["a,b"]); + } + + #[test] + fn apply_reinserts_union_when_header_fully_omitted() { + let t = tracker(); + let mut turn1 = header_map(&["a,b"]); + apply_sticky_betas(&t, BetaProvider::Anthropic, "s1", &mut turn1, "req-1"); + + let mut turn2 = HeaderMap::new(); + apply_sticky_betas(&t, BetaProvider::Anthropic, "s1", &mut turn2, "req-2"); + assert_eq!(beta_values(&turn2), vec!["a,b"]); + } + + #[test] + fn apply_never_invents_a_header() { + let t = tracker(); + let mut map = HeaderMap::new(); + apply_sticky_betas(&t, BetaProvider::Anthropic, "s1", &mut map, "req-1"); + assert!(map.get("anthropic-beta").is_none()); + } + + #[test] + fn apply_noop_leaves_header_lines_untouched() { + let t = tracker(); + let mut map = header_map(&["a,b"]); + apply_sticky_betas(&t, BetaProvider::Anthropic, "s1", &mut map, "req-1"); + let mut again = header_map(&["a,b"]); + apply_sticky_betas(&t, BetaProvider::Anthropic, "s1", &mut again, "req-2"); + assert_eq!(beta_values(&again), vec!["a,b"]); + } + + #[test] + fn apply_records_all_repeated_header_lines() { + // RFC 9110 list semantics: two field lines are one list. The + // union must record BOTH lines, so a later rewrite (which + // collapses to a single line) can never shrink the upstream + // token set mid-conversation. + let t = tracker(); + let mut turn1 = header_map(&["a,x", "b"]); + apply_sticky_betas(&t, BetaProvider::Anthropic, "s1", &mut turn1, "req-1"); + // No rewrite on turn 1 (union == joined client list): both + // lines pass through untouched. + assert_eq!(beta_values(&turn1), vec!["a,x", "b"]); + + // Turn 2 drops "x" from the first line: the rewrite must + // carry the full set from both turn-1 lines. + let mut turn2 = header_map(&["a", "b"]); + apply_sticky_betas(&t, BetaProvider::Anthropic, "s1", &mut turn2, "req-2"); + assert_eq!(beta_values(&turn2), vec!["a,x,b"]); + } + + #[test] + fn apply_skips_non_ascii_value_and_records_nothing() { + let t = tracker(); + let mut map = HeaderMap::new(); + map.insert( + "anthropic-beta", + HeaderValue::from_bytes(&[0xfa, 0xfb]).unwrap(), + ); + apply_sticky_betas(&t, BetaProvider::Anthropic, "s1", &mut map, "req-1"); + // Wire bytes untouched… + assert_eq!(map.get("anthropic-beta").unwrap().as_bytes(), &[0xfa, 0xfb]); + // …and nothing recorded: the next ASCII turn sees only its + // own tokens. + let got = t.record_and_get_sticky_betas(BetaProvider::Anthropic, "s1", Some("y")); + assert_eq!(got, "y"); + } + + #[test] + fn concurrent_unions_lose_no_tokens() { + // Port of the Python thread-hammering test: concurrent turns + // on one session must never drop a recorded token. + let t = tracker(); + std::thread::scope(|s| { + for i in 0..8 { + let t = t.clone(); + s.spawn(move || { + let token = format!("tok-{i}"); + for _ in 0..50 { + t.record_and_get_sticky_betas(BetaProvider::Anthropic, "s1", Some(&token)); + } + }); + } + }); + let merged = t.record_and_get_sticky_betas(BetaProvider::Anthropic, "s1", None); + let tokens: HashSet<&str> = merged.split(',').collect(); + for i in 0..8 { + assert!(tokens.contains(format!("tok-{i}").as_str())); + } + } +} diff --git a/crates/headroom-proxy/src/cache_stabilization/drift_detector.rs b/crates/headroom-proxy/src/cache_stabilization/drift_detector.rs index c288d3131..12ca1f709 100644 --- a/crates/headroom-proxy/src/cache_stabilization/drift_detector.rs +++ b/crates/headroom-proxy/src/cache_stabilization/drift_detector.rs @@ -399,8 +399,9 @@ pub fn observe_drift(state: &DriftState, session_key: &str, current: StructuralH /// 16-char hex prefix of SHA-256(session_key). Bounds the log line /// width and never reveals the raw key (which may be a bearer token -/// or API key — see `derive_session_key`). -fn session_key_log_prefix(session_key: &str) -> String { +/// or API key — see `derive_session_key`). `pub(crate)` so the +/// beta-sticky merge site logs the same session identity the same way. +pub(crate) fn session_key_log_prefix(session_key: &str) -> String { let mut hasher = Sha256::new(); hasher.update(session_key.as_bytes()); let digest = hasher.finalize(); diff --git a/crates/headroom-proxy/src/cache_stabilization/mod.rs b/crates/headroom-proxy/src/cache_stabilization/mod.rs index 815c3af21..b5f91a21b 100644 --- a/crates/headroom-proxy/src/cache_stabilization/mod.rs +++ b/crates/headroom-proxy/src/cache_stabilization/mod.rs @@ -12,9 +12,14 @@ //! - **Normalize** request bytes to make cache hits deterministic //! under PAYG mode ([`tool_def_normalize`], PR-E1 / PR-E2; //! [`anthropic_cache_control`], PR-E3; [`openai_cache_key`], PR-E4). -//! These mutate bytes only when the auth-mode gate and per-policy -//! preconditions (e.g. no customer `cache_control` marker) all clear; -//! OAuth and Subscription always passthrough. +//! These mutate *body* bytes only when the auth-mode gate and +//! per-policy preconditions (e.g. no customer `cache_control` +//! marker) all clear; for body mutations, OAuth and Subscription +//! always passthrough. +//! - **Re-echo** client-sent state ([`beta_sticky`]): mutate request +//! *headers* only, on every auth mode, and only ever with values +//! the same client already put on the wire — anti-drift repair of +//! the client's own signal, never injection of Headroom state. //! //! Currently shipped: //! @@ -49,6 +54,16 @@ //! `(model, system, tools)` and inject it so the upstream pins //! cache lookup to a tenant-stable identity. **Mutates the body** //! (only on PAYG) — see its docs for the gating contract. +//! - [`beta_sticky`] — parity port of the Python proxy's PR-A6 +//! `SessionBetaTracker`: per-`(provider, session)` LRU that unions +//! `anthropic-beta` / `openai-beta` tokens across turns so a client +//! dropping a token mid-conversation doesn't rotate the upstream +//! prefix-cache key. **Mutates request headers, never the body**; +//! applies to all auth modes exactly like the Python path (the +//! union only ever contains tokens this client itself sent, so +//! subscription stealth — invariant #10 "no beta drift" — is +//! preserved by construction). Operator opt-out: +//! `--beta-header-sticky disabled`. //! //! Sibling PRs hang additional submodules off this `mod.rs`. Conflict //! resolution between parallel Phase E PRs is intentionally trivial: @@ -56,6 +71,7 @@ //! `mod.rs`'s `pub mod` list. pub mod anthropic_cache_control; +pub mod beta_sticky; pub mod drift_detector; pub mod openai_cache_key; pub mod tool_def_normalize; diff --git a/crates/headroom-proxy/src/config.rs b/crates/headroom-proxy/src/config.rs index 8cc0ae593..181589754 100644 --- a/crates/headroom-proxy/src/config.rs +++ b/crates/headroom-proxy/src/config.rs @@ -1,6 +1,9 @@ //! Configuration for the proxy: CLI flags + env vars. use clap::{Parser, ValueEnum}; +use headroom_core::rollout::{ + feature_names, split_feature_names, Feature, RolloutChannel, RolloutSnapshot, +}; use std::net::SocketAddr; use std::time::Duration; use url::Url; @@ -181,6 +184,48 @@ impl CompressionMode { } } +/// Session-sticky provider beta headers (parity port of the Python +/// proxy's `HEADROOM_BETA_HEADER_STICKY`; see +/// `cache_stabilization::beta_sticky`). +/// +/// When `enabled` (default), the proxy unions each request's +/// `anthropic-beta` / `openai-beta` tokens with the tokens previously +/// seen for the same conversation and forwards the union, so a client +/// dropping a beta token mid-conversation doesn't rotate the upstream +/// prefix-cache key. +/// +/// When `disabled`, the client header is forwarded verbatim and no +/// per-session token state is kept. Diagnostic operator opt-in — NOT +/// a fallback per realignment build constraint #4. +/// +/// Source priority: CLI flag → `HEADROOM_PROXY_BETA_HEADER_STICKY` +/// env var → default (`enabled`). +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] +#[clap(rename_all = "snake_case")] +pub enum BetaHeaderSticky { + /// Union beta tokens per conversation and forward the union. + /// Default. Matches the Python proxy's default behaviour. + Enabled, + /// Forward the client's beta header verbatim; keep no state. + /// Diagnostic-only. + Disabled, +} + +impl BetaHeaderSticky { + /// Stable snake_case name suitable for log fields. + pub fn as_str(self) -> &'static str { + match self { + BetaHeaderSticky::Enabled => "enabled", + BetaHeaderSticky::Disabled => "disabled", + } + } + + /// Convenience: is the sticky union switched on? + pub fn is_enabled(self) -> bool { + matches!(self, BetaHeaderSticky::Enabled) + } +} + #[derive(Debug, Clone, Parser)] #[command( name = "headroom-proxy", @@ -188,6 +233,49 @@ impl CompressionMode { about = "Headroom transparent reverse proxy" )] pub struct CliArgs { + /// Runtime rollout channel that bounds which managed features may run. + /// + /// `stable` admits only features that have completed bake time. `beta` and + /// `canary` admit progressively newer features. `dev` is for local work. + /// Explicit feature requests still cannot cross this boundary unless the + /// unsafe override is set. + #[arg( + long = "rollout-channel", + env = "HEADROOM_ROLLOUT_CHANNEL", + default_value = "stable", + value_parser = parse_rollout_channel, + )] + pub rollout_channel: String, + + /// Comma-separated rollout features to request explicitly. + #[arg( + long = "features", + env = "HEADROOM_FEATURES", + default_value = "", + value_parser = parse_rollout_features, + )] + pub features: String, + + /// Comma-separated rollout features to force off. Disable wins over defaults + /// and explicit enable requests. + #[arg( + long = "disable-features", + env = "HEADROOM_DISABLE_FEATURES", + default_value = "", + value_parser = parse_rollout_features, + )] + pub disable_features: String, + + /// Break-glass override that allows unstable features below their channel. + /// Intended only for emergency mitigation and should be visible in logs. + #[arg( + long = "unsafe-allow-unstable-features", + env = "HEADROOM_UNSAFE_ALLOW_UNSTABLE_FEATURES", + default_value_t = false, + action = clap::ArgAction::Set, + )] + pub unsafe_allow_unstable_features: bool, + /// Address the proxy listens on (e.g. 0.0.0.0:8787). #[arg(long, env = "HEADROOM_PROXY_LISTEN", default_value = "0.0.0.0:8787")] pub listen: SocketAddr, @@ -320,6 +408,28 @@ pub struct CliArgs { )] pub strip_internal_headers: StripInternalHeaders, + /// Session-sticky provider beta headers: union `anthropic-beta` / + /// `openai-beta` tokens per conversation so a client dropping a + /// token mid-conversation doesn't bust the upstream prefix cache. + /// Parity port of the Python proxy's `SessionBetaTracker` (PR-A6). + /// Default `enabled`; `disabled` is a diagnostic operator opt-in. + /// + /// Active only when the compression interceptor is on + /// (`--compression` / `HEADROOM_PROXY_COMPRESSION=1`): with the + /// interceptor off the proxy is a strict byte-pipe and never + /// mutates headers. Startup logs a warning when this is `enabled` + /// while `--compression` is off. + /// + /// Source priority: CLI flag → `HEADROOM_PROXY_BETA_HEADER_STICKY` + /// env var → default (`enabled`). + #[arg( + long = "beta-header-sticky", + env = "HEADROOM_PROXY_BETA_HEADER_STICKY", + value_enum, + default_value_t = BetaHeaderSticky::Enabled, + )] + pub beta_header_sticky: BetaHeaderSticky, + /// Phase C PR-C4: enable the `/v1/responses` SSE streaming /// pipeline. When `true` (default), `Accept: text/event-stream` /// requests on `/v1/responses` flow through the byte-level SSE @@ -475,6 +585,32 @@ fn parse_duration(s: &str) -> Result { humantime::parse_duration(s).map_err(|e| format!("invalid duration `{s}`: {e}")) } +fn parse_rollout_channel(value: &str) -> Result { + value + .parse::() + .map(|channel| channel.as_str().to_owned()) + .map_err(|_| { + format!("unknown rollout channel `{value}` (valid: stable, beta, canary, dev)") + }) +} + +fn parse_rollout_features(value: &str) -> Result { + let valid = feature_names(); + let unknown: Vec<_> = split_feature_names(value) + .into_iter() + .filter(|name| !valid.contains(name.as_str())) + .collect(); + if unknown.is_empty() { + Ok(value.to_owned()) + } else { + Err(format!( + "unknown rollout feature(s): {}; valid: {}", + unknown.join(", "), + valid.into_iter().collect::>().join(", ") + )) + } +} + fn parse_bytes(s: &str) -> Result { s.parse::() .map(|b| b.as_u64()) @@ -484,6 +620,8 @@ fn parse_bytes(s: &str) -> Result { /// Resolved configuration used by the running server. #[derive(Debug, Clone)] pub struct Config { + /// Runtime rollout state resolved from CLI/env. + pub rollout: RolloutSnapshot, pub listen: SocketAddr, pub upstream: Url, pub upstream_timeout: Duration, @@ -517,6 +655,9 @@ pub struct Config { /// upstream-bound requests. PR-A5 default-on guard against /// fingerprinting / leakage of internal flags. pub strip_internal_headers: StripInternalHeaders, + /// Session-sticky provider beta headers (parity port of the + /// Python `SessionBetaTracker`, PR-A6). Default `enabled`. + pub beta_header_sticky: BetaHeaderSticky, /// PR-C4: enable the `/v1/responses` streaming pipeline (SSE /// state-machine + telemetry tee). Default `true`. pub enable_responses_streaming: bool, @@ -555,6 +696,30 @@ pub struct Config { impl Config { pub fn from_cli(args: CliArgs) -> Self { + let mut explicit_features = Vec::new(); + if args.enable_responses_streaming { + explicit_features.push(Feature::OpenAiResponsesStreaming); + } + if args.enable_bedrock_native { + explicit_features.push(Feature::NativeBedrock); + } + // Preserve the pre-rollout rollback controls as legacy disables. Both + // features are stable defaults in the registry, so merely omitting a + // false flag from `explicit_features` would turn it straight back on. + let mut disabled_features = split_feature_names(&args.disable_features); + if !args.enable_responses_streaming { + disabled_features.push(Feature::OpenAiResponsesStreaming.spec().name.to_owned()); + } + if !args.enable_bedrock_native { + disabled_features.push(Feature::NativeBedrock.spec().name.to_owned()); + } + let rollout = RolloutSnapshot::from_parts_with_explicit( + &args.rollout_channel, + &args.features, + &disabled_features.join(","), + args.unsafe_allow_unstable_features, + &explicit_features, + ); let rewrite_host = if args.no_rewrite_host { false } else { @@ -564,6 +729,7 @@ impl Config { .compression_max_body_bytes .unwrap_or(args.max_body_bytes); Self { + rollout: rollout.clone(), listen: args.listen, upstream: args.upstream, upstream_timeout: args.upstream_timeout, @@ -578,9 +744,14 @@ impl Config { cache_control_auto_frozen: args.cache_control_auto_frozen, auth_mode_policy_enforcement: args.auth_mode_policy_enforcement, strip_internal_headers: args.strip_internal_headers, - enable_responses_streaming: args.enable_responses_streaming, + beta_header_sticky: args.beta_header_sticky, + enable_responses_streaming: rollout.is_enabled( + Feature::OpenAiResponsesStreaming, + args.enable_responses_streaming, + ), enable_conversations_passthrough: args.enable_conversations_passthrough, - enable_bedrock_native: args.enable_bedrock_native, + enable_bedrock_native: rollout + .is_enabled(Feature::NativeBedrock, args.enable_bedrock_native), bedrock_region: args.bedrock_region, bedrock_endpoint: args.bedrock_endpoint, aws_profile: args.aws_profile, @@ -594,6 +765,7 @@ impl Config { /// production-default behaviour so existing tests stay unchanged. pub fn for_test(upstream: Url) -> Self { Self { + rollout: RolloutSnapshot::default(), listen: "127.0.0.1:0".parse().unwrap(), upstream, upstream_timeout: Duration::from_secs(60), @@ -621,6 +793,9 @@ impl Config { // from upstream-bound requests. Tests opt out per-case via // `start_proxy_with`. strip_internal_headers: StripInternalHeaders::Enabled, + // Production default: sticky beta-header union per + // conversation (Python-parity). Tests opt out per-case. + beta_header_sticky: BetaHeaderSticky::Enabled, // PR-C4: streaming pipeline + conversations passthrough // both default-on so tests exercise the same paths // production traffic will hit. @@ -644,3 +819,48 @@ impl Config { } } } + +#[cfg(test)] +mod rollout_input_tests { + use super::*; + + #[test] + fn explicit_rollout_inputs_are_strict_and_diagnosable() { + assert_eq!(parse_rollout_channel("CANARY").unwrap(), "canary"); + assert!(parse_rollout_channel("stabel") + .unwrap_err() + .contains("unknown rollout channel")); + assert!(parse_rollout_features("native-bedrock").is_ok()); + let error = parse_rollout_features("native_bedrok").unwrap_err(); + assert!(error.contains("native_bedrok")); + assert!(error.contains("native_bedrock")); + } + + #[test] + fn legacy_false_flags_remain_effective_rollout_disables() { + let args = CliArgs::try_parse_from([ + "headroom-proxy", + "--upstream", + "http://127.0.0.1:9", + "--enable-responses-streaming", + "false", + "--enable-bedrock-native", + "false", + ]) + .unwrap(); + + let config = Config::from_cli(args); + + for feature in [Feature::OpenAiResponsesStreaming, Feature::NativeBedrock] { + let decision = config.rollout.decision(feature); + assert!(!decision.enabled); + assert!(decision.disabled); + assert_eq!( + decision.reason, + headroom_core::rollout::FeatureDecisionReason::Disabled + ); + } + assert!(!config.enable_responses_streaming); + assert!(!config.enable_bedrock_native); + } +} diff --git a/crates/headroom-proxy/src/health.rs b/crates/headroom-proxy/src/health.rs index 683b917a6..5cf17a7f2 100644 --- a/crates/headroom-proxy/src/health.rs +++ b/crates/headroom-proxy/src/health.rs @@ -13,6 +13,11 @@ pub async fn healthz() -> impl IntoResponse { Json(json!({ "ok": true, "service": "headroom-proxy" })) } +/// Effective rollout state of this running Rust proxy process. +pub async fn rollout_status(State(state): State) -> Json { + Json(state.config.rollout.to_value()) +} + /// Upstream health: GETs upstream `/healthz`. Returns 200 when reachable + /// 2xx, 503 otherwise. The endpoint name is reserved by the proxy and is /// not forwarded; operators must not name a real upstream route this. @@ -39,3 +44,18 @@ pub async fn healthz_upstream(State(state): State) -> Response { .into_response(), } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::Config; + + #[tokio::test] + async fn rollout_status_exposes_running_snapshot() { + let state = AppState::new(Config::for_test("http://127.0.0.1:9".parse().unwrap())).unwrap(); + let expected = state.config.rollout.snapshot_digest(); + let Json(payload) = rollout_status(State(state)).await; + assert_eq!(payload["snapshot_digest"], expected); + assert_eq!(payload["qualification_eligible"], true); + } +} diff --git a/crates/headroom-proxy/src/main.rs b/crates/headroom-proxy/src/main.rs index 19e748522..f21d0117f 100644 --- a/crates/headroom-proxy/src/main.rs +++ b/crates/headroom-proxy/src/main.rs @@ -28,9 +28,32 @@ async fn main() -> Result<(), Box> { max_body_bytes = config.max_body_bytes, rewrite_host = config.rewrite_host, graceful_shutdown_timeout_s = config.graceful_shutdown_timeout.as_secs(), + rollout_channel = config.rollout.config.channel.as_str(), + rollout_features_enabled = ?config.rollout.enabled(), + rollout_features_disabled = ?config.rollout.config.disabled, + unsafe_allow_unstable_features = config.rollout.config.unsafe_allow_unstable, + rollout_registry_digest = %config.rollout.registry_digest, + rollout_snapshot_digest = %config.rollout.snapshot_digest(), + qualification_eligible = config.rollout.qualification_eligible(), "headroom-proxy starting" ); + // Session-sticky beta headers only run inside the compression + // interceptor: with `--compression` off the proxy is a strict + // byte-pipe and never mutates headers. Say so loudly at startup — + // an operator reading `beta_header_sticky=enabled` (the default) + // must not believe the protection is active when it isn't. + if config.beta_header_sticky.is_enabled() && !config.compression { + tracing::warn!( + event = "beta_header_sticky_inactive", + beta_header_sticky = config.beta_header_sticky.as_str(), + compression = config.compression, + "beta-header stickiness is enabled but the compression \ + interceptor is off; enable --compression (or \ + HEADROOM_PROXY_COMPRESSION=1) to activate it" + ); + } + let mut state = AppState::new(config.clone())?; // PR-D1: resolve AWS credentials at startup via the `aws-config` diff --git a/crates/headroom-proxy/src/proxy.rs b/crates/headroom-proxy/src/proxy.rs index 7f3d81cd3..1005fb03f 100644 --- a/crates/headroom-proxy/src/proxy.rs +++ b/crates/headroom-proxy/src/proxy.rs @@ -17,6 +17,7 @@ use futures_util::{StreamExt as _, TryStreamExt}; use http_body_util::BodyExt; use crate::cache_stabilization; +use crate::cache_stabilization::beta_sticky::BetaProvider; use crate::cache_stabilization::drift_detector::{ compute_structural_hash, derive_session_key, observe_drift, ApiKind, DriftState, }; @@ -24,7 +25,7 @@ use crate::compression; use crate::config::Config; use crate::error::ProxyError; use crate::headers::{build_forward_request_headers, filter_response_headers}; -use crate::health::{healthz, healthz_upstream}; +use crate::health::{healthz, healthz_upstream, rollout_status}; use crate::websocket::ws_handler; // Phase F PR-F1: imported as `classify_auth_mode` to make the call // site self-documenting. `AuthMode` is re-exported under the same @@ -66,6 +67,13 @@ pub struct AppState { /// request body — so this can be cloned freely into every handler /// path that buffers the body. pub drift_state: DriftState, + /// Session-sticky beta-header tracker (parity port of the Python + /// `SessionBetaTracker`, PR-A6): per-`(provider, session)` LRU of + /// `anthropic-beta` / `openai-beta` tokens, unioned across turns + /// so a client dropping a token mid-conversation doesn't rotate + /// the upstream prefix-cache key. Shares the drift detector's + /// session identity (same `derive_session_key` output). + pub beta_sticky: cache_stabilization::beta_sticky::BetaStickyState, /// PR-D4: GCP ADC bearer-token source for Vertex routes. Default: /// [`crate::vertex::adc::GcpAdcTokenSource`] constructed lazily; /// the actual ADC chain is only resolved when the first Vertex @@ -111,6 +119,9 @@ impl AppState { client, bedrock_credentials: None, drift_state: DriftState::new(DRIFT_DETECTOR_CAPACITY), + beta_sticky: cache_stabilization::beta_sticky::BetaStickyState::new( + cache_stabilization::beta_sticky::BETA_TRACKER_CAPACITY, + ), vertex_token_source, }) } @@ -146,6 +157,7 @@ pub fn build_app(state: AppState) -> Router { let mut router = Router::new() .route("/healthz", get(healthz)) .route("/healthz/upstream", get(healthz_upstream)) + .route("/rollout/status", get(rollout_status)) // PR-D3: Prometheus scrape endpoint. Renders the global // registry in text format. The handler is stateless — no // `AppState` needed — and idempotent across concurrent @@ -707,6 +719,41 @@ pub(crate) async fn forward_http( let session_key = derive_session_key(headers, &client_addr, &parsed, kind); let hash = compute_structural_hash(&parsed, kind); observe_drift(&state.drift_state, &session_key, hash); + + // Session-sticky provider beta headers — port of the + // Python PR-A6 `SessionBetaTracker`. Beta headers are + // part of the bytes that determine the upstream + // prefix-cache key; a client dropping a token between + // turns rotates the key and re-writes the whole + // prefix at the customer's cost. Forward the + // per-conversation union instead. See + // `cache_stabilization::beta_sticky` for the behavior + // contract, the auth-mode rationale (applies to every + // mode, like the Python handler), and the one + // documented divergence from Python (per-conversation + // keying). Reuses the drift detector's `session_key` + // so both cache-stability subsystems agree on + // conversation identity. Mutates upstream-bound + // HEADERS only; body bytes stay untouched (Phase-A + // cache-safety invariant). + if state.config.beta_header_sticky.is_enabled() { + let provider = match endpoint { + compression::CompressibleEndpoint::AnthropicMessages => { + BetaProvider::Anthropic + } + compression::CompressibleEndpoint::OpenAiChatCompletions + | compression::CompressibleEndpoint::OpenAiResponses => { + BetaProvider::OpenAi + } + }; + cache_stabilization::beta_sticky::apply_sticky_betas( + &state.beta_sticky, + provider, + &session_key, + &mut outgoing_headers, + &request_id, + ); + } } } let outcome = match endpoint { diff --git a/crates/headroom-proxy/src/websocket.rs b/crates/headroom-proxy/src/websocket.rs index c3a6358c6..509cd6bfd 100644 --- a/crates/headroom-proxy/src/websocket.rs +++ b/crates/headroom-proxy/src/websocket.rs @@ -218,10 +218,10 @@ async fn run_ws_pump( fn ax_to_tg(m: AxMsg) -> Option { Some(match m { - AxMsg::Text(t) => TgMsg::Text(t.to_string()), - AxMsg::Binary(b) => TgMsg::Binary(b.to_vec()), - AxMsg::Ping(p) => TgMsg::Ping(p.to_vec()), - AxMsg::Pong(p) => TgMsg::Pong(p.to_vec()), + AxMsg::Text(t) => TgMsg::Text(t.to_string().into()), + AxMsg::Binary(b) => TgMsg::Binary(b.to_vec().into()), + AxMsg::Ping(p) => TgMsg::Ping(p.to_vec().into()), + AxMsg::Pong(p) => TgMsg::Pong(p.to_vec().into()), AxMsg::Close(Some(cf)) => TgMsg::Close(Some(TgCloseFrame { code: tokio_tungstenite::tungstenite::protocol::frame::coding::CloseCode::from(cf.code), reason: cf.reason.to_string().into(), diff --git a/crates/headroom-proxy/tests/integration_beta_header_sticky.rs b/crates/headroom-proxy/tests/integration_beta_header_sticky.rs new file mode 100644 index 000000000..d1712be96 --- /dev/null +++ b/crates/headroom-proxy/tests/integration_beta_header_sticky.rs @@ -0,0 +1,529 @@ +//! End-to-end coverage for session-sticky provider beta headers +//! (`cache_stabilization::beta_sticky` — Rust port of the Python +//! proxy's PR-A6 `SessionBetaTracker`). +//! +//! The scenario every test guards: a client (Claude Code, Codex CLI) +//! sends `anthropic-beta: a,b` on turn 1 and drops `b` on turn 2 of +//! the SAME conversation. Beta headers are part of the bytes that +//! determine the upstream prefix-cache key, so the drop rotates the +//! key and the provider re-writes the whole prefix at the customer's +//! cost. The proxy must forward the per-conversation union instead. +//! +//! These tests boot a real Rust proxy in front of a wiremock upstream +//! and assert on the headers/bytes the upstream actually receives: +//! +//! - dropped tokens are re-injected on later turns (Anthropic, +//! OpenAI Chat, OpenAI Responses — all three intercepted routes); +//! - conversation identity works both via the explicit +//! `x-headroom-session-id` opt-in AND via the body-derived +//! conversation discriminator (no explicit header — the realistic +//! Claude Code shape); +//! - the union NEVER invents tokens the client didn't send: no beta +//! header in → no beta header out, and separate conversations don't +//! leak tokens into each other; +//! - `--beta-header-sticky disabled` forwards the client value +//! verbatim (diagnostic opt-out, Python +//! `HEADROOM_BETA_HEADER_STICKY=disabled` parity); +//! - the body is forwarded byte-equal (SHA-256) while the header is +//! rewritten — the mechanism mutates request headers, never body +//! bytes (Phase-A cache-safety contract). + +mod common; + +use common::start_proxy_with; +use serde_json::json; +use sha2::{Digest, Sha256}; +use std::sync::{Arc, Mutex}; +use wiremock::matchers::{method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +/// Everything the upstream saw for one request: selected header +/// values (lower-case names) + raw body bytes. +#[derive(Clone)] +struct Seen { + beta: Option, + session_id_header: Option, + body: Vec, +} + +type Captures = Arc>>; + +/// Mount a capture-everything mock for `route` on the upstream. The +/// `beta_header` name is which provider beta header to record +/// (`anthropic-beta` / `openai-beta`). +async fn mount_capture(upstream: &MockServer, route: &str, beta_header: &'static str) -> Captures { + let captured: Captures = Arc::new(Mutex::new(Vec::new())); + let captured_clone = captured.clone(); + Mock::given(method("POST")) + .and(path(route)) + .respond_with(move |req: &wiremock::Request| { + let get = |name: &str| { + req.headers + .get(name) + .and_then(|v| v.to_str().ok()) + .map(|s| s.to_string()) + }; + captured_clone.lock().unwrap().push(Seen { + beta: get(beta_header), + session_id_header: get("x-headroom-session-id"), + body: req.body.clone(), + }); + ResponseTemplate::new(200).set_body_string(r#"{"ok":true}"#) + }) + .mount(upstream) + .await; + captured +} + +fn anthropic_body(turns: &[(&str, &str)]) -> Vec { + let messages: Vec = turns + .iter() + .map(|(role, content)| json!({"role": role, "content": content})) + .collect(); + serde_json::to_vec(&json!({ + "model": "claude-sonnet-4-5", + "max_tokens": 32, + "messages": messages, + })) + .unwrap() +} + +fn openai_chat_body(turns: &[(&str, &str)]) -> Vec { + let messages: Vec = turns + .iter() + .map(|(role, content)| json!({"role": role, "content": content})) + .collect(); + serde_json::to_vec(&json!({ + "model": "gpt-4o", + "messages": messages, + })) + .unwrap() +} + +fn openai_responses_body(text: &str) -> Vec { + serde_json::to_vec(&json!({ + "model": "gpt-4o", + "input": [{"role": "user", "content": text}], + })) + .unwrap() +} + +async fn post( + client: &reqwest::Client, + url: String, + body: Vec, + headers: &[(&str, &str)], +) -> reqwest::Response { + let mut req = client + .post(url) + .header("content-type", "application/json") + .body(body); + for (name, value) in headers { + req = req.header(*name, *value); + } + req.send().await.expect("proxy reachable") +} + +#[tokio::test] +async fn anthropic_dropped_beta_token_reinjected_with_explicit_session_header() { + let upstream = MockServer::start().await; + let captured = mount_capture(&upstream, "/v1/messages", "anthropic-beta").await; + let proxy = start_proxy_with(&upstream.uri(), |c| { + c.compression = true; + }) + .await; + let client = reqwest::Client::new(); + let url = format!("{}/v1/messages", proxy.url()); + + // Turn 1: two beta tokens. + let resp = post( + &client, + url.clone(), + anthropic_body(&[("user", "hello")]), + &[ + ( + "anthropic-beta", + "context-management-2025-06-27,interleaved-thinking-2025-05-14", + ), + ("x-headroom-session-id", "conv-explicit-1"), + ], + ) + .await; + assert_eq!(resp.status(), 200); + + // Turn 2, same conversation: the client dropped the second token. + let resp = post( + &client, + url, + anthropic_body(&[("user", "hello"), ("assistant", "hi"), ("user", "next")]), + &[ + ("anthropic-beta", "context-management-2025-06-27"), + ("x-headroom-session-id", "conv-explicit-1"), + ], + ) + .await; + assert_eq!(resp.status(), 200); + + let seen = captured.lock().unwrap().clone(); + assert_eq!(seen.len(), 2); + assert_eq!( + seen[0].beta.as_deref(), + Some("context-management-2025-06-27,interleaved-thinking-2025-05-14"), + "turn 1 forwards the client value unchanged" + ); + assert_eq!( + seen[1].beta.as_deref(), + Some("context-management-2025-06-27,interleaved-thinking-2025-05-14"), + "turn 2 must re-inject the dropped token so the upstream \ + prefix-cache key stays byte-stable" + ); + // PR-A5 invariant intact: the internal session header never + // crosses the upstream boundary. + assert!(seen.iter().all(|s| s.session_id_header.is_none())); + + proxy.shutdown().await; +} + +#[tokio::test] +async fn anthropic_conversation_keyed_without_explicit_session_header() { + // The realistic Claude Code shape: no `x-headroom-session-id`; + // conversation identity comes from the credential arm + the + // first-message discriminator inside `derive_session_key`. + let upstream = MockServer::start().await; + let captured = mount_capture(&upstream, "/v1/messages", "anthropic-beta").await; + let proxy = start_proxy_with(&upstream.uri(), |c| { + c.compression = true; + }) + .await; + let client = reqwest::Client::new(); + let url = format!("{}/v1/messages", proxy.url()); + + let auth = ("authorization", "Bearer oauth-workspace-token"); + post( + &client, + url.clone(), + anthropic_body(&[("user", "conversation opener")]), + &[("anthropic-beta", "a,b"), auth], + ) + .await; + // Same conversation (same opener, grown transcript), token "b" + // dropped. + post( + &client, + url, + anthropic_body(&[ + ("user", "conversation opener"), + ("assistant", "reply"), + ("user", "follow-up"), + ]), + &[("anthropic-beta", "a"), auth], + ) + .await; + + let seen = captured.lock().unwrap().clone(); + assert_eq!(seen.len(), 2); + assert_eq!(seen[1].beta.as_deref(), Some("a,b")); + + proxy.shutdown().await; +} + +#[tokio::test] +async fn openai_chat_dropped_beta_token_reinjected() { + let upstream = MockServer::start().await; + let captured = mount_capture(&upstream, "/v1/chat/completions", "openai-beta").await; + let proxy = start_proxy_with(&upstream.uri(), |c| { + c.compression = true; + }) + .await; + let client = reqwest::Client::new(); + let url = format!("{}/v1/chat/completions", proxy.url()); + + post( + &client, + url.clone(), + openai_chat_body(&[("user", "hello")]), + &[ + ("openai-beta", "assistants=v2,realtime=v1"), + ("x-headroom-session-id", "conv-oai-1"), + ], + ) + .await; + post( + &client, + url, + openai_chat_body(&[("user", "hello"), ("assistant", "hi"), ("user", "next")]), + &[ + ("openai-beta", "assistants=v2"), + ("x-headroom-session-id", "conv-oai-1"), + ], + ) + .await; + + let seen = captured.lock().unwrap().clone(); + assert_eq!(seen.len(), 2); + assert_eq!(seen[1].beta.as_deref(), Some("assistants=v2,realtime=v1")); + + proxy.shutdown().await; +} + +#[tokio::test] +async fn openai_responses_dropped_beta_token_reinjected() { + let upstream = MockServer::start().await; + let captured = mount_capture(&upstream, "/v1/responses", "openai-beta").await; + let proxy = start_proxy_with(&upstream.uri(), |c| { + c.compression = true; + }) + .await; + let client = reqwest::Client::new(); + let url = format!("{}/v1/responses", proxy.url()); + + post( + &client, + url.clone(), + openai_responses_body("hello"), + &[ + ("openai-beta", "responses=v1,tools=v2"), + ("x-headroom-session-id", "conv-resp-1"), + ], + ) + .await; + post( + &client, + url, + openai_responses_body("hello again"), + &[ + ("openai-beta", "responses=v1"), + ("x-headroom-session-id", "conv-resp-1"), + ], + ) + .await; + + let seen = captured.lock().unwrap().clone(); + assert_eq!(seen.len(), 2); + assert_eq!(seen[1].beta.as_deref(), Some("responses=v1,tools=v2")); + + proxy.shutdown().await; +} + +#[tokio::test] +async fn anthropic_fully_omitted_beta_header_regains_union() { + // The headline docs claim: "sends a token in turn N and omits it + // in turn N+1" — here the whole header disappears, not just one + // token, and the union must be re-added through real axum/reqwest + // plumbing. + let upstream = MockServer::start().await; + let captured = mount_capture(&upstream, "/v1/messages", "anthropic-beta").await; + let proxy = start_proxy_with(&upstream.uri(), |c| { + c.compression = true; + }) + .await; + let client = reqwest::Client::new(); + let url = format!("{}/v1/messages", proxy.url()); + + post( + &client, + url.clone(), + anthropic_body(&[("user", "hello")]), + &[ + ("anthropic-beta", "context-management-2025-06-27"), + ("x-headroom-session-id", "conv-omit-1"), + ], + ) + .await; + // Turn 2: no anthropic-beta header at all. + post( + &client, + url, + anthropic_body(&[("user", "hello"), ("assistant", "hi"), ("user", "next")]), + &[("x-headroom-session-id", "conv-omit-1")], + ) + .await; + + let seen = captured.lock().unwrap().clone(); + assert_eq!(seen.len(), 2); + assert_eq!( + seen[1].beta.as_deref(), + Some("context-management-2025-06-27"), + "a fully omitted beta header must be restored from session state" + ); + + proxy.shutdown().await; +} + +#[tokio::test] +async fn disabled_flag_forwards_client_value_verbatim() { + use headroom_proxy::config::BetaHeaderSticky; + + let upstream = MockServer::start().await; + let captured = mount_capture(&upstream, "/v1/messages", "anthropic-beta").await; + let proxy = start_proxy_with(&upstream.uri(), |c| { + c.compression = true; + c.beta_header_sticky = BetaHeaderSticky::Disabled; + }) + .await; + let client = reqwest::Client::new(); + let url = format!("{}/v1/messages", proxy.url()); + + post( + &client, + url.clone(), + anthropic_body(&[("user", "hello")]), + &[ + ("anthropic-beta", "a,b"), + ("x-headroom-session-id", "conv-d1"), + ], + ) + .await; + post( + &client, + url, + anthropic_body(&[("user", "hello"), ("assistant", "hi"), ("user", "next")]), + &[ + ("anthropic-beta", "a"), + ("x-headroom-session-id", "conv-d1"), + ], + ) + .await; + + let seen = captured.lock().unwrap().clone(); + assert_eq!(seen.len(), 2); + assert_eq!( + seen[1].beta.as_deref(), + Some("a"), + "disabled mode must forward the dropped-token value verbatim \ + and keep no session state" + ); + + proxy.shutdown().await; +} + +#[tokio::test] +async fn no_client_beta_header_is_never_invented() { + let upstream = MockServer::start().await; + let captured = mount_capture(&upstream, "/v1/messages", "anthropic-beta").await; + let proxy = start_proxy_with(&upstream.uri(), |c| { + c.compression = true; + }) + .await; + let client = reqwest::Client::new(); + let url = format!("{}/v1/messages", proxy.url()); + + for body in [ + anthropic_body(&[("user", "hello")]), + anthropic_body(&[("user", "hello"), ("assistant", "hi"), ("user", "next")]), + ] { + post( + &client, + url.clone(), + body, + &[("x-headroom-session-id", "conv-n1")], + ) + .await; + } + + let seen = captured.lock().unwrap().clone(); + assert_eq!(seen.len(), 2); + assert!( + seen.iter().all(|s| s.beta.is_none()), + "a session that never sent a beta header must never gain one" + ); + + proxy.shutdown().await; +} + +#[tokio::test] +async fn separate_conversations_do_not_leak_tokens() { + let upstream = MockServer::start().await; + let captured = mount_capture(&upstream, "/v1/messages", "anthropic-beta").await; + let proxy = start_proxy_with(&upstream.uri(), |c| { + c.compression = true; + }) + .await; + let client = reqwest::Client::new(); + let url = format!("{}/v1/messages", proxy.url()); + + post( + &client, + url.clone(), + anthropic_body(&[("user", "conversation A")]), + &[ + ("anthropic-beta", "token-a"), + ("x-headroom-session-id", "conv-A"), + ], + ) + .await; + post( + &client, + url, + anthropic_body(&[("user", "conversation B")]), + &[ + ("anthropic-beta", "token-b"), + ("x-headroom-session-id", "conv-B"), + ], + ) + .await; + + let seen = captured.lock().unwrap().clone(); + assert_eq!(seen.len(), 2); + assert_eq!(seen[0].beta.as_deref(), Some("token-a")); + assert_eq!( + seen[1].beta.as_deref(), + Some("token-b"), + "conversation B must not inherit conversation A's tokens" + ); + + proxy.shutdown().await; +} + +#[tokio::test] +async fn body_bytes_stay_byte_equal_while_header_is_rewritten() { + // Cache-safety contract: the sticky union mutates request + // HEADERS only. The forwarded body must remain byte-identical + // (SHA-256) to what the client sent — same assertion idiom as the + // model-sanitizer integration tests. + let upstream = MockServer::start().await; + let captured = mount_capture(&upstream, "/v1/messages", "anthropic-beta").await; + let proxy = start_proxy_with(&upstream.uri(), |c| { + c.compression = true; + }) + .await; + let client = reqwest::Client::new(); + let url = format!("{}/v1/messages", proxy.url()); + + let turn1 = anthropic_body(&[("user", "hello")]); + let turn2 = anthropic_body(&[("user", "hello"), ("assistant", "hi"), ("user", "next")]); + + post( + &client, + url.clone(), + turn1, + &[ + ("anthropic-beta", "a,b"), + ("x-headroom-session-id", "conv-bb"), + ], + ) + .await; + post( + &client, + url, + turn2.clone(), + &[ + ("anthropic-beta", "a"), + ("x-headroom-session-id", "conv-bb"), + ], + ) + .await; + + let seen = captured.lock().unwrap().clone(); + assert_eq!(seen.len(), 2); + // Header was rewritten to the union… + assert_eq!(seen[1].beta.as_deref(), Some("a,b")); + // …but the body bytes are untouched. + assert_eq!( + Sha256::digest(&seen[1].body), + Sha256::digest(&turn2), + "sticky beta union must never mutate body bytes" + ); + + proxy.shutdown().await; +} diff --git a/crates/headroom-proxy/tests/integration_ws.rs b/crates/headroom-proxy/tests/integration_ws.rs index 7f9688d87..1c29aaf3a 100644 --- a/crates/headroom-proxy/tests/integration_ws.rs +++ b/crates/headroom-proxy/tests/integration_ws.rs @@ -53,7 +53,7 @@ async fn ws_text_and_binary_round_trip() { for i in 0..5 { let m = format!("hello-{i}"); - ws.send(Message::Text(m.clone())).await.unwrap(); + ws.send(Message::Text(m.clone().into())).await.unwrap(); let echoed = ws.next().await.unwrap().unwrap(); match echoed { Message::Text(t) => assert_eq!(t.as_str(), m), @@ -62,7 +62,7 @@ async fn ws_text_and_binary_round_trip() { } for i in 0..5u8 { let m: Vec = (0..32u8).map(|b| b ^ i).collect(); - ws.send(Message::Binary(m.clone())).await.unwrap(); + ws.send(Message::Binary(m.clone().into())).await.unwrap(); let echoed = ws.next().await.unwrap().unwrap(); match echoed { Message::Binary(b) => assert_eq!(b.to_vec(), m), diff --git a/dashboard-cache-ttl-main.png b/dashboard-cache-ttl-main.png new file mode 100644 index 000000000..3433b1c56 Binary files /dev/null and b/dashboard-cache-ttl-main.png differ diff --git a/deploy/beacon/package.json b/deploy/beacon/package.json new file mode 100644 index 000000000..b1913f3bd --- /dev/null +++ b/deploy/beacon/package.json @@ -0,0 +1,5 @@ +{ + "name": "headroom-beacon", + "private": true, + "type": "module" +} diff --git a/deploy/beacon/sample-event.json b/deploy/beacon/sample-event.json index 6d5e13bd5..4bacb0ebc 100644 --- a/deploy/beacon/sample-event.json +++ b/deploy/beacon/sample-event.json @@ -284,6 +284,21 @@ "value": { "intValue": "2" } + }, + { + "key": "failure_statuses", + "value": { + "kvlistValue": { + "values": [ + { + "key": "529", + "value": { + "intValue": "2" + } + } + ] + } + } } ] } diff --git a/deploy/beacon/test-rollup.mjs b/deploy/beacon/test-rollup.mjs new file mode 100644 index 000000000..3878ee2d4 --- /dev/null +++ b/deploy/beacon/test-rollup.mjs @@ -0,0 +1,214 @@ +/** + * Self-check for scheduled()'s hourly compaction. node test-rollup.mjs [dir] + * + * The one thing that must never drift: rollupHour() and the QUALIFY in + * headroom-beacon-stats/beacon.sh have to agree on which heartbeat wins. If + * they disagree the reports get quietly wrong rather than loudly broken, so + * this asserts the JS picks exactly the max-seq row per (install, session). + * + * Point it at a directory of real beacon objects to check against the corpus: + * aws s3 sync s3://headroom-telemetry/sessions/dt=.../hh=.../ /tmp/hr/ ... + * node test-rollup.mjs /tmp/hr + * With no argument it runs on a small fixture and needs no network. + */ +import { readdirSync, readFileSync } from 'node:fs'; +import assert from 'node:assert/strict'; +import { oldestRawDay, rollupHour } from './worker.js'; + +// R2 returns at most 1000 keys per list page, so on a real hour (~4,000 +// objects) the cursor loop in rollupHour is load-bearing. The stub paginates at +// a deliberately tiny size so that loop is exercised by every case below: with +// a single-page stub, a regression that dropped the cursor would still print +// "ok" while silently rolling up only the first page of every hour. +const PAGE = 3; + +/** The slice of the R2 binding rollupHour uses, backed by a plain object. */ +function stubBucket(files, { failKeys = new Set() } = {}) { + const written = {}; + const reads = []; + return { + written, + reads, + list: async ({ prefix, cursor, delimiter }) => { + const keys = Object.keys(files) + .filter((k) => k.startsWith(prefix)) + .sort(); + if (delimiter) { + const seen = new Set(); + for (const k of keys) { + const cut = k.indexOf(delimiter, prefix.length); + if (cut >= 0) seen.add(k.slice(0, cut + 1)); + } + return { objects: [], delimitedPrefixes: [...seen], truncated: false }; + } + const start = cursor ? keys.indexOf(cursor) : 0; + const page = keys.slice(start, start + PAGE); + const next = start + PAGE; + return { + objects: page.map((key) => ({ key })), + truncated: next < keys.length, + cursor: next < keys.length ? keys[next] : undefined, + }; + }, + get: async (key) => { + reads.push(key); + if (failKeys.has(key)) throw new Error(`simulated R2 failure: ${key}`); + if (!(key in files)) return null; + return { text: async () => files[key] }; + }, + put: async (key, body) => { + written[key] = body; + }, + }; +} + +const beacon = (install, id, seq) => + JSON.stringify({ resource: { 'headroom.install_id': install }, session: { id, seq } }); + +const PART = 'dt=2026-08-06/hh=14'; + +/** Run rollupHour against a stub bucket and decode whatever it wrote. */ +async function run(files, opts = {}) { + const CORPUS = stubBucket(files, opts); + const spend = { read: 0 }; + let threw = null; + let out = null; + try { + out = await rollupHour({ CORPUS }, PART, spend); + } catch (err) { + threw = err; + } + const body = CORPUS.written[`rollup/${PART}/data.ndjson`]; + return { + threw, + spend, + wrote: out ? out.wrote : 0, + empty: `rollup/${PART}/empty` in CORPUS.written, + keys: Object.keys(CORPUS.written), + rows: body ? body.split('\n').map((l) => JSON.parse(l)) : [], + }; +} + +// 1. Highest seq wins, out-of-order input, one row per (install, session). +// More objects than PAGE, so the list cursor loop runs. +{ + const files = { + [`sessions/${PART}/a.json`]: [beacon('i1', 's1', 3), beacon('i1', 's2', 1)].join('\n'), + [`sessions/${PART}/b.json`]: beacon('i1', 's1', 9), + [`sessions/${PART}/c.json`]: beacon('i1', 's1', 7), + // Same session id under a different install must not collapse together. + [`sessions/${PART}/d.json`]: beacon('i2', 's1', 2), + [`sessions/${PART}/e.json`]: beacon('i1', 's1', 5), + }; + const { rows, spend, threw } = await run(files); + assert.equal(threw, null); + // 5 objects at PAGE=3 is two pages: proves the cursor loop, which is + // load-bearing at the real ~4,000 objects/hour. + assert.ok(Object.keys(files).length > PAGE, 'fixture must span pages'); + assert.equal(spend.read, 5, 'reads every object across every page'); + assert.equal(rows.length, 3, 'one row per (install, session)'); + const seq = Object.fromEntries( + rows.map((r) => [`${r.resource['headroom.install_id']} ${r.session.id}`, r.session.seq]) + ); + assert.deepEqual(seq, { 'i1 s1': 9, 'i1 s2': 1, 'i2 s1': 2 }); +} + +// 2. An unparseable record loses only itself. Content this Worker wrote with +// JSON.stringify never becomes valid later, so blocking the hour on it would +// strand the hour rather than one record. +{ + const files = { + [`sessions/${PART}/a.json`]: '{ this is not json', + [`sessions/${PART}/b.json`]: `\n${beacon('i1', 's1', 4)}\n`, + }; + const { rows, threw } = await run(files); + assert.equal(threw, null, 'corrupt content does not abandon the hour'); + assert.deepEqual(rows.map((r) => r.session.seq), [4], 'survives a corrupt object'); +} + +// 3. A failed get is transient, so the hour must NOT be written — a rollup is +// built once and then trusted forever, so a short read would silently become +// the permanent record. +{ + const files = { + [`sessions/${PART}/a.json`]: beacon('i1', 's1', 1), + [`sessions/${PART}/b.json`]: beacon('i1', 's2', 1), + }; + const { threw, keys } = await run(files, { + failKeys: new Set([`sessions/${PART}/b.json`]), + }); + assert.ok(threw, 'a failed get throws so the hour is retried'); + assert.deepEqual(keys, [], 'nothing written on a partial read'); +} + +// 4. Spend is reported even when the hour throws. Charging a flat guess instead +// lets a run that failed late overshoot the subrequest ceiling. +{ + const files = Object.fromEntries( + Array.from({ length: 7 }, (_, i) => [`sessions/${PART}/o${i}.json`, beacon('i1', `s${i}`, 1)]) + ); + const { threw, spend } = await run(files, { + failKeys: new Set([`sessions/${PART}/o6.json`]), + }); + assert.ok(threw); + assert.equal(spend.read, 7, 'caller sees real spend, not a guess'); +} + +// 5. An empty hour writes a marker, not a zero-byte NDJSON. Without it the hour +// stays "missing" and is re-listed on every run forever. +{ + const { rows, empty, keys } = await run({}); + assert.deepEqual(rows, []); + assert.ok(empty, 'empty hour leaves a marker'); + assert.ok( + keys.every((k) => !k.endsWith('.ndjson')), + 'no zero-byte ndjson for readers to special-case' + ); +} + +// 6. oldestRawDay floors the backfill. A fixed lookback window silently strands +// every hour older than it once analysis stopped reading sessions/. +{ + const CORPUS = stubBucket({ + 'sessions/dt=2026-08-03/hh=01/a.json': beacon('i1', 's1', 1), + 'sessions/dt=2026-08-06/hh=14/b.json': beacon('i1', 's2', 1), + 'sessions/dt=2026-08-07/hh=00/c.json': beacon('i1', 's3', 1), + }); + assert.equal(await oldestRawDay({ CORPUS }), '2026-08-03'); + assert.equal(await oldestRawDay({ CORPUS: stubBucket({}) }), null, 'empty bucket -> null'); +} + +// 7. Against real objects, if a directory was given: same answer as the QUALIFY +// in beacon.sh, which is `count(DISTINCT install||session)` rows, each +// carrying that pair's max seq. +const dir = process.argv[2]; +if (dir) { + const files = {}; + for (const f of readdirSync(dir).filter((f) => f.endsWith('.json'))) { + files[`sessions/${PART}/${f}`] = readFileSync(`${dir}/${f}`, 'utf8'); + } + const { rows, spend, threw } = await run(files); + assert.equal(threw, null); + + const expected = new Map(); + for (const text of Object.values(files)) { + for (const line of text.split('\n')) { + if (!line.trim()) continue; + const r = JSON.parse(line); + const k = `${r.resource?.['headroom.install_id']} ${r.session?.id}`; + expected.set(k, Math.max(expected.get(k) ?? -1, r.session?.seq ?? 0)); + } + } + assert.equal(spend.read, Object.keys(files).length); + assert.equal(rows.length, expected.size, 'row count matches DISTINCT sessions'); + for (const r of rows) { + const k = `${r.resource['headroom.install_id']} ${r.session.id}`; + assert.equal(r.session.seq, expected.get(k), `max seq for ${k}`); + } + console.log( + `real corpus: ${spend.read} objects -> ${rows.length} sessions in 1 object` + + ` (${Math.ceil(spend.read / PAGE)} list pages)` + ); +} + +console.log('ok'); diff --git a/deploy/beacon/worker.js b/deploy/beacon/worker.js index d4e2d0ac2..7fea03001 100644 --- a/deploy/beacon/worker.js +++ b/deploy/beacon/worker.js @@ -28,9 +28,12 @@ * deanonymise install_id, so it is never read. */ -// Mirrors the payload built by _Session.payload(). A key absent here is -// dropped, not stored. Adding a metric means adding it here first — that -// friction is the point. +// Mostly mirrors the payload built by _Session.payload(); an extension may +// also emit its own event carrying one of these top-level keys. A key absent +// here is dropped, not stored. Adding a metric means adding it here first — +// that friction is the point, and it is also the only privacy control that +// works retroactively, so it must land BEFORE any client starts sending the +// key or that traffic is silently discarded and unrecoverable. const ALLOWED_KEYS = [ 'schema_version', 'session', @@ -42,6 +45,14 @@ const ALLOWED_KEYS = [ 'providers', 'models', 'failures', + 'failure_statuses', + // Model-routing summary. Emitted by a routing extension rather than by the + // proxy itself -- see proxy/route_advice.py for the decision seam. Same rule + // as everything above: counters and model ids, no free text. Allowlisted + // here so the corpus can answer what the proxy alone cannot -- a provider's + // real minimum cacheable prefix, how long a cache actually survives, and how + // far predicted cache hits are from the ones that happened. + 'routing', ]; // Resource attributes we keep. Same rule: allowlist, not denylist. @@ -110,7 +121,184 @@ function extract(payload) { return records; } +// ----------------------------------------------------------------- rollup -- +// +// The corpus is one object per heartbeat, ~1KB each — 65k on 2026-08-06 and +// climbing. DuckDB reads them correctly, but a full `pull` is ~100k HTTPS round +// trips for 95MB: minutes of pure per-object latency, no real bytes or compute. +// Listing the bucket alone took 88 seconds. +// +// This job collapses each COMPLETE hour into one object under rollup/, keeping +// only the highest-seq heartbeat per (install, session). One measured hour +// (dt=2026-08-06/hh=14): 3,938 objects and 3,938 rows in, 1 object and 1,061 +// rows out. Analysis reads rollup/**, never sessions/**. Raw is left exactly as +// written, so any rollup can be rebuilt by deleting it. +// +// Hourly rather than daily because every R2 binding call is a subrequest: a day +// is ~65k of them against a 10k-per-invocation ceiling, an hour is ~4k. + +const READ_BUDGET = 60000; // objects per run; see [limits] in wrangler.toml +// A get costs ~45ms of round trip and almost no CPU, so this is what decides +// whether a run finishes: at 20 an hour took ~3 minutes, against a 15-minute +// wall clock for a cron invocation. Raise it if an hour ever stops fitting. +const FANOUT = 100; // concurrent R2 gets + +const partition = (d) => + `dt=${d.toISOString().slice(0, 10)}/hh=${d.toISOString().slice(11, 13)}`; + +/** + * One hour of heartbeats -> one deduped NDJSON object. + * + * Returns `{ read, wrote }`. Spend is reported through the mutable `spend` + * accumulator so the caller still knows it even when this throws: the budget + * has to track real spend, and a flat guess lets a run that failed late + * overshoot the subrequest ceiling and get killed inside an hour that would + * otherwise have succeeded. + * + * Writes nothing unless the whole hour read cleanly. A rollup is built once and + * then treated as done forever, so a partial read would silently become the + * permanent record — better to write nothing and let the next run retry. + */ +export async function rollupHour(env, part, spend = { read: 0 }) { + const best = new Map(); + let failed = 0; // transient: retry the hour + let corrupt = 0; // permanent: record and move on + let cursor; + do { + const page = await env.CORPUS.list({ prefix: `sessions/${part}/`, cursor }); + for (let i = 0; i < page.objects.length; i += FANOUT) { + // allSettled, not all: one transient R2 error among the ~4,000 gets in a + // real hour would otherwise reject the batch and discard the whole hour. + const settled = await Promise.allSettled( + page.objects + .slice(i, i + FANOUT) + .map((o) => env.CORPUS.get(o.key).then((r) => (r ? r.text() : null))) + ); + for (const outcome of settled) { + spend.read++; + // A miss counts as a failure too. The key came from a LIST, so the + // object existed; treating it as empty would quietly shrink the rollup. + if (outcome.status !== 'fulfilled' || outcome.value === null) { + failed++; + continue; + } + for (const line of outcome.value.split('\n')) { + if (!line) continue; + let rec; + try { + rec = JSON.parse(line); + } catch { + // Counted and logged, but NOT a reason to abandon the hour. A + // failed get is transient and worth retrying; content this Worker + // itself wrote with JSON.stringify does not become valid later, so + // blocking on it would strand the hour until its raw objects + // expire and then lose the whole hour instead of one record. + corrupt++; + continue; + } + // A session heartbeats every 5 minutes carrying CUMULATIVE totals, so + // the highest seq IS the whole session and every earlier row is a + // strict subset. Sessions straddle hours, so readers still dedupe + // across rollups on this same key — this only shrinks each hour. + const id = `${rec.resource?.['headroom.install_id']} ${rec.session?.id}`; + const prev = best.get(id); + if (!prev || (rec.session?.seq ?? 0) > (prev.session?.seq ?? 0)) { + best.set(id, rec); + } + } + } + } + cursor = page.truncated ? page.cursor : undefined; + } while (cursor); + + if (failed) { + throw new Error(`${part}: ${failed} of ${spend.read} objects unreadable`); + } + if (corrupt) { + console.error(`rollup ${part}: skipped ${corrupt} unparseable record(s)`); + } + + // A genuinely empty hour gets a marker rather than a zero-byte NDJSON that + // every reader would have to special-case. Without it the hour stays + // "missing" and is re-listed on every run for the life of the bucket. + if (best.size === 0) { + await env.CORPUS.put(`rollup/${part}/empty`, ''); + return { read: spend.read, wrote: 0 }; + } + await env.CORPUS.put( + `rollup/${part}/data.ndjson`, + [...best.values()].map((r) => JSON.stringify(r)).join('\n'), + { httpMetadata: { contentType: 'application/x-ndjson' } } + ); + return { read: spend.read, wrote: best.size }; +} + +/** Oldest `dt=` day still under sessions/, or null. One delimited LIST. */ +export async function oldestRawDay(env) { + const page = await env.CORPUS.list({ prefix: 'sessions/', delimiter: '/' }); + const days = (page.delimitedPrefixes || []) + .map((p) => p.slice('sessions/dt='.length).replace(/\/$/, '')) + .filter((d) => /^\d{4}-\d{2}-\d{2}$/.test(d)) + .sort(); + return days.length ? days[0] : null; +} + export default { + /** Hourly cron. Builds every complete hour back to the oldest raw data. */ + async scheduled(event, env) { + // Backfill reaches all the way to the oldest surviving raw day, NOT a fixed + // window. A fixed window silently strands everything older than it the + // moment analysis stopped reading sessions/ — the raw objects are still + // there, but nothing would ever compact them, so they vanish from every + // report. Bounding by real data instead means the floor rises only when a + // lifecycle rule actually expires the raw objects. + const oldest = await oldestRawDay(env); + if (!oldest) return; + const floorMs = Date.parse(`${oldest}T00:00:00Z`); + if (Number.isNaN(floorMs)) return; + + // Only list from the floor forward. Rollups older than the oldest raw day + // can never be rebuilt, so enumerating them answers nothing — this is what + // keeps the listing bounded by retention rather than by total history. + const done = new Set(); + let cursor; + do { + const page = await env.CORPUS.list({ + prefix: 'rollup/', + startAfter: `rollup/dt=${oldest}`, + cursor, + }); + for (const o of page.objects) { + // Tolerates both `/data.ndjson` and the `/empty` marker. + const rel = o.key.slice('rollup/'.length); + const cut = rel.lastIndexOf('/'); + if (cut > 0) done.add(rel.slice(0, cut)); + } + cursor = page.truncated ? page.cursor : undefined; + } while (cursor); + + // Newest first, so a backlog drains from the present backwards and the + // freshest hour is never the one starved by the budget. Starts one hour + // back: the current hour is still being written to. + let budget = READ_BUDGET; + for (let t = event.scheduledTime - 3600_000; t >= floorMs && budget > 0; t -= 3600_000) { + const part = partition(new Date(t)); + if (done.has(part)) continue; + // Shared with rollupHour so a throw still reports what it spent. + const spend = { read: 0 }; + try { + await rollupHour(env, part, spend); + } catch (err) { + // Newest-first means an hour that always throws — one grown past the + // subrequest ceiling, say — would otherwise block every older hour + // behind it forever. Skip it and keep draining; it has no marker, so + // the next run retries it. + console.error(`rollup ${part} failed after ${spend.read} objects: ${err}`); + } + budget -= spend.read; + } + }, + async fetch(request, env, ctx) { if (request.method !== 'POST') { return new Response('beacon: POST OTLP logs to /v1/logs', { status: 405 }); @@ -134,14 +322,13 @@ export default { } if (records.length === 0) return new Response(null, { status: 204 }); - const now = new Date(); - const day = now.toISOString().slice(0, 10); - const hour = now.toISOString().slice(11, 13); // Hive-style partitioning so DuckDB can prune by date without a catalog. - // ponytail: one object per request. At beacon volume that is a few hundred - // thousand objects a month, which globs fine. Add a daily compaction job - // when the file count starts to slow queries, not before. - const key = `sessions/dt=${day}/hh=${hour}/${crypto.randomUUID()}.json`; + // Shares partition() with the rollup: the cron lists `sessions//`, so + // two independent spellings of this scheme would mean the writer and the + // compactor could drift apart and silently match zero objects. + // ponytail: one object per request. Compacted hourly into rollup/ by + // scheduled() above — analysis reads that, never this. + const key = `sessions/${partition(new Date())}/${crypto.randomUUID()}.json`; const ndjson = records.map((r) => JSON.stringify(r)).join('\n'); // Respond immediately; durability work continues after the response. diff --git a/deploy/beacon/wrangler.toml b/deploy/beacon/wrangler.toml index 412b22045..826dba6bb 100644 --- a/deploy/beacon/wrangler.toml +++ b/deploy/beacon/wrangler.toml @@ -32,6 +32,24 @@ bucket_name = "headroom-telemetry" # npx wrangler secret put METRICS_OTLP_AUTH # Absent = R2 only, which is the right place to start. +# Hourly compaction of sessions/ into rollup/ — see scheduled() in worker.js. +# At :05 so the hour being rolled up is definitely closed. A >=1h interval also +# buys the 15-minute CPU limit instead of 30s, which the backfill run needs. +[triggers] +crons = ["5 * * * *"] + +# Every R2 binding call is a subrequest, and one hour is already ~4k objects. +# The paid default of 10k would cap a run at two hours and stall the backfill +# behind live traffic forever. This only raises a ceiling; a normal run spends +# ~4k. READ_BUDGET in worker.js is what actually bounds the work. +# +# Workers Paid only — on the Free plan this key is rejected outright ("CPU +# limits are not supported for the Free plan"), and the cron could not run +# anyway: Free gives a scheduled handler 10ms of CPU, and parsing an hour of +# heartbeats is tens of ms. +[limits] +subrequests = 100000 + [observability] enabled = true diff --git a/docker-bake.hcl b/docker-bake.hcl index 07397c965..f7ca8aacd 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -10,7 +10,7 @@ target "runtime-default" { inherits = ["_common", "docker-metadata-action"] target = "runtime" args = { - HEADROOM_EXTRAS = "proxy" + HEADROOM_EXTRAS = "proxy,bedrock" RUNTIME_USER = "nonroot" } } @@ -19,7 +19,7 @@ target "runtime" { inherits = ["_common", "docker-metadata-action"] target = "runtime" args = { - HEADROOM_EXTRAS = "proxy" + HEADROOM_EXTRAS = "proxy,bedrock" RUNTIME_USER = "root" } } @@ -28,7 +28,7 @@ target "runtime-nonroot" { inherits = ["_common", "docker-metadata-action"] target = "runtime" args = { - HEADROOM_EXTRAS = "proxy" + HEADROOM_EXTRAS = "proxy,bedrock" RUNTIME_USER = "nonroot" } } @@ -37,7 +37,7 @@ target "runtime-code" { inherits = ["_common", "docker-metadata-action"] target = "runtime" args = { - HEADROOM_EXTRAS = "proxy,code" + HEADROOM_EXTRAS = "proxy,code,bedrock" RUNTIME_USER = "root" } } @@ -46,7 +46,7 @@ target "runtime-code-nonroot" { inherits = ["_common", "docker-metadata-action"] target = "runtime" args = { - HEADROOM_EXTRAS = "proxy,code" + HEADROOM_EXTRAS = "proxy,code,bedrock" RUNTIME_USER = "nonroot" } } @@ -55,7 +55,7 @@ target "runtime-slim" { inherits = ["_common", "docker-metadata-action"] target = "runtime-slim" args = { - HEADROOM_EXTRAS = "proxy" + HEADROOM_EXTRAS = "proxy,bedrock" RUNTIME_USER = "root" } } @@ -64,7 +64,7 @@ target "runtime-slim-nonroot" { inherits = ["_common", "docker-metadata-action"] target = "runtime-slim" args = { - HEADROOM_EXTRAS = "proxy" + HEADROOM_EXTRAS = "proxy,bedrock" RUNTIME_USER = "nonroot" } } @@ -73,7 +73,7 @@ target "runtime-code-slim" { inherits = ["_common", "docker-metadata-action"] target = "runtime-slim" args = { - HEADROOM_EXTRAS = "proxy,code" + HEADROOM_EXTRAS = "proxy,code,bedrock" RUNTIME_USER = "root" } } @@ -82,7 +82,7 @@ target "runtime-code-slim-nonroot" { inherits = ["_common", "docker-metadata-action"] target = "runtime-slim" args = { - HEADROOM_EXTRAS = "proxy,code" + HEADROOM_EXTRAS = "proxy,code,bedrock" RUNTIME_USER = "nonroot" } } diff --git a/docker-compose.yml b/docker-compose.yml index 47b707b2d..204aecb81 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ # 3. point your LLM client at http://localhost:8787 (proxy) # # Just want the proxy without the memory features? You can run the proxy image -# on its own (`docker run -p 8787:8787 ghcr.io/chopratejas/headroom`); the two +# on its own (`docker run -p 8787:8787 ghcr.io/headroomlabs-ai/headroom`); the two # database services below are only required for the memory/relevance features. # # Ports exposed on the host: diff --git a/docs/content/docs/architecture.mdx b/docs/content/docs/architecture.mdx index 089cfca7c..d8f754372 100644 --- a/docs/content/docs/architecture.mdx +++ b/docs/content/docs/architecture.mdx @@ -47,7 +47,7 @@ In proxy mode the server is a FastAPI app with per-provider handlers (Anthropic, The proxy assembles a small, ordered pipeline. Every transform is independent, safe to skip, and **fails open** — on any error it returns the content unchanged and the request still goes through. -1. **Tool-result interceptor** *(opt-in)* — light structural interceptors such as ast-grep Read outlining. Off unless you pass `--intercept-tool-results`. +1. **Tool-result interceptor** *(canary opt-in)* — light structural interceptors such as ast-grep Read outlining. Requires `HEADROOM_ROLLOUT_CHANNEL=canary` plus `--intercept-tool-results`. 2. **CacheAligner** *(off by default)* — a detector that reports dynamic-prefix drift (dates, UUIDs, session tokens). It **never mutates, moves, or rewrites** content. It is disabled by default and hard-disabled inside the proxy; it exists to surface prefix-stability metrics, not to change your messages. 3. **ContentRouter** — the workhorse that does essentially all of the compression. See below. diff --git a/docs/content/docs/ccr.mdx b/docs/content/docs/ccr.mdx index 18c4ff2c4..4d668b07c 100644 --- a/docs/content/docs/ccr.mdx +++ b/docs/content/docs/ccr.mdx @@ -73,13 +73,15 @@ When the LLM calls `headroom_retrieve`: The client never sees CCR tool calls on the Anthropic and OpenAI proxy paths; Headroom resolves them transparently there. - - Native Gemini requests do not yet run the server-side CCR response handler, so - `headroom_retrieve` is not resolved transparently on that path today. Google's - OpenAI-compatible Gemini endpoint can also return - `finish_reason=MALFORMED_FUNCTION_CALL` on large function-response continuations - after CCR retrieval. If you need fully transparent CCR resolution today, use the - Anthropic or OpenAI proxy paths. See [issue #2041](https://github.com/headroomlabs-ai/headroom/issues/2041). + + Buffered native Gemini requests resolve `headroom_retrieve` server-side and + return the model's final response. Streaming native Gemini requests keep the + existing forwarding behavior. When a response contains `headroom_retrieve` + alongside a client-owned function call, Headroom preserves both calls for the + client instead of resolving the mixed response. Google's OpenAI-compatible Gemini endpoint can + also return `finish_reason=MALFORMED_FUNCTION_CALL` on large function-response + continuations after CCR retrieval; that separate limitation remains tracked in + [issue #2041](https://github.com/headroomlabs-ai/headroom/issues/2041). ## Phase 4: Context Tracker diff --git a/docs/content/docs/configuration.mdx b/docs/content/docs/configuration.mdx index 40ee56530..c8a4fca82 100644 --- a/docs/content/docs/configuration.mdx +++ b/docs/content/docs/configuration.mdx @@ -5,6 +5,20 @@ description: All configuration options for the Headroom Python and TypeScript SD Headroom can be configured via the SDK constructor, proxy command line, environment variables, or per-request overrides. +## Runtime Rollout Channels + +Headroom uses rollout channels to control which behaviors an already-installed +artifact may expose. They do not select a package or released version. + +| Variable | Default | Purpose | +|----------|---------|---------| +| `HEADROOM_ROLLOUT_CHANNEL` | `stable` | Selects `stable`, `beta`, `canary`, or `dev`. | +| `HEADROOM_FEATURES` | unset | Comma-separated feature names to request explicitly. | +| `HEADROOM_DISABLE_FEATURES` | unset | Comma-separated feature names to force off. Disable wins over every enable path. | +| `HEADROOM_UNSAFE_ALLOW_UNSTABLE_FEATURES` | unset | Break-glass override for emergency mitigation only. | + +See [Runtime Rollouts](/docs/runtime-rollouts) for policy, provenance, and +contributor rules. If Codex history disappeared after using an older wrapper, see [Recover Codex State](/docs/codex-recovery) before wrapping Codex again. ## SDK Modes (`default_mode` / `headroom_mode`) @@ -294,6 +308,7 @@ headroom proxy --learn --min-evidence 3 | `HEADROOM_REQUEST_TIMEOUT` | Request timeout in seconds | `300` | | `HEADROOM_BETA_HEADER_STICKY` | Controls per-session `anthropic-beta` / `OpenAI-Beta` re-echo. `enabled` (default): the proxy unions beta tokens across turns within a session — if the client sends a token in turn N and omits it in turn N+1, the proxy re-injects it to preserve prefix-cache stability. `disabled`: the client's value is forwarded verbatim with no accumulation. Any other value raises at request time. See [Session Beta Header Tracking](/docs/configuration#session-beta-header-tracking). | `enabled` | | `HEADROOM_BETA_TRACKER_MAX_SESSIONS` | LRU capacity of the in-memory session beta tracker. Once full, the oldest session entry is evicted. | `1000` | +| `HEADROOM_PROXY_BETA_HEADER_STICKY` | Rust proxy: same per-conversation beta-token union as `HEADROOM_BETA_HEADER_STICKY`, applied to `anthropic-beta` / `openai-beta` on the intercepted `/v1/messages`, `/v1/chat/completions`, and `/v1/responses` routes. Requires the compression interceptor (`HEADROOM_PROXY_COMPRESSION=1`) — with it off the Rust proxy is a strict byte-pipe and this flag has no effect (startup warns). Unlike the Python tracker (keyed on model + system prompt), sessions are keyed per conversation, shared with the cache-drift detector — parallel conversations never inherit each other's tokens. `enabled` default; `disabled` forwards the client value verbatim and keeps no state. Tracker capacity is fixed at 1000 sessions. | `enabled` | | `HEADROOM_MODEL_ROUTER_ENABLED` | Enable cost-aware model routing. `1`/`true`/`yes`/`on`/`enabled` turns it on and requires `HEADROOM_MODEL_ROUTES`. See [Cost-aware model routing](/docs/configuration#cost-aware-model-routing). | `off` | | `HEADROOM_MODEL_ROUTES` | JSON array of ordered routing rules for cost-aware model routing (schema below). | -- | | `HEADROOM_THINKING_COMPACT` | Compact plain-text reasoning that models re-send every turn (Kimi/GLM/DeepSeek `reasoning_content` / inline ``): Kompress it on warm turns, drop it on cold turns. No-op for Claude/Codex/OpenAI (encrypted reasoning). See [Cold-prefix hook](#cold-prefix-hook--reasoning-compaction). | `off` | diff --git a/docs/content/docs/meta.json b/docs/content/docs/meta.json index 6a45b0b4c..d716df852 100644 --- a/docs/content/docs/meta.json +++ b/docs/content/docs/meta.json @@ -40,6 +40,7 @@ "vscode-claude-code", "vscode-copilot", "opencode", + "opencode-deepseek", "grok-build", "mcp", "---Configuration---", @@ -57,6 +58,7 @@ "architecture", "ci-cd-flows", "releases", + "runtime-rollouts", "benchmarks", "limitations", "---Help---", diff --git a/docs/content/docs/metrics.mdx b/docs/content/docs/metrics.mdx index ea8c2577f..c07953d6a 100644 --- a/docs/content/docs/metrics.mdx +++ b/docs/content/docs/metrics.mdx @@ -113,7 +113,11 @@ HEADROOM_OTEL_RESOURCE_ATTRIBUTES=deployment.environment=prod | `HEADROOM_OTEL_SERVICE_NAME` | `headroom-proxy` | OTEL `service.name` | | `HEADROOM_OTEL_RESOURCE_ATTRIBUTES` | unset | Comma-separated resource attributes | -Exported counters include `headroom.proxy.requests`, `headroom.proxy.tokens.input`, `headroom.proxy.tokens.output`, `headroom.proxy.tokens.saved`, and `headroom.proxy.cache.read_tokens` / `write_tokens`. +Exported counters include `headroom.proxy.requests`, `headroom.proxy.tokens.input`, and +`headroom.proxy.tokens.output`. `headroom.proxy.tokens.saved` is the all-layer total: +message/compression savings plus tool-schema deferral savings. The component counter +`headroom.proxy.tokens.tool_schema_saved` exposes the deferral portion separately; +`headroom.compression.tokens.saved` remains the compression-pipeline component. Confirm the exporter is live with `curl -s http://localhost:8787/stats | jq .otel`. diff --git a/docs/content/docs/opencode-deepseek.mdx b/docs/content/docs/opencode-deepseek.mdx new file mode 100644 index 000000000..a9a3594af --- /dev/null +++ b/docs/content/docs/opencode-deepseek.mdx @@ -0,0 +1,233 @@ +--- +title: OpenCode + DeepSeek +description: Configure OpenCode to route DeepSeek traffic through the Headroom proxy for compression, output shaping, and savings visibility. +--- + +Save 20-60% on DeepSeek API costs with Headroom's context compression proxy. + +## How it works + +``` +OpenCode → Headroom Proxy (:8787) → DeepSeek API + ↑ compresses input + + shapes output +``` + +The proxy sits between OpenCode and DeepSeek. It compresses tool outputs, logs, +and search results before they reach the model, then shapes responses to be +concise. DeepSeek's API is OpenAI-compatible — one flag and you're running. + +--- + +## 1. Install Headroom + +```bash +pip install headroom-ai +# or via uv: +uv tool install headroom-ai +``` + +You get SmartCrusher (structural compression), the proxy, output shaping, and +the MCP server — everything you need. + +--- + +## 2. Get your DeepSeek API key + +Sign up at [platform.deepseek.com](https://platform.deepseek.com) and generate +an API key. + +Store it somewhere safe: +```bash +export DEEPSEEK_API_KEY="sk-your-deepseek-key-here" +``` + +--- + +## 3. Start the proxy + +```bash +headroom proxy \ + --port 8787 \ + --openai-api-url https://api.deepseek.com/v1 +``` + +The proxy auto-detects `api.deepseek.com` and labels itself "DeepSeek" on the +dashboard. Verify it's running: + +```bash +curl http://127.0.0.1:8787/health +# → "status": "healthy" +``` + +To see which models the proxy exposes: +```bash +curl -s http://127.0.0.1:8787/v1/models \ + -H "Authorization: Bearer sk-your-key" | jq '.data[].id' +``` + +### With output shaping (optional) + +Output shaping makes the model's responses shorter — fewer tokens, lower cost: + +```bash +HEADROOM_ROLLOUT_CHANNEL=beta HEADROOM_OUTPUT_SHAPER=1 HEADROOM_VERBOSITY_LEVEL=2 \ +headroom proxy --port 8787 --openai-api-url https://api.deepseek.com/v1 +``` + +Verbosity levels: + +| Level | Behavior | +|---|---| +| `1` | Skip preambles/postambles | +| `2` | + Don't restate code/file content already in context (**recommended**) | +| `3` | + Omit rationale unless asked | +| `4` | Maximum — fragments, zero fluff | + +--- + +## 4. Configure OpenCode + +**Note:** If you have an existing `~/.config/opencode/opencode.json` (for MCP +servers, etc.), merge the provider section into that file. Having both `.json` +and `.jsonc` in the same directory can cause conflicts. + +Edit `~/.config/opencode/opencode.json`: + +```jsonc +{ + "$schema": "https://opencode.ai/config.json", + "model": "headroom/deepseek-v4-pro", + "provider": { + "headroom": { + "npm": "@ai-sdk/openai-compatible", + "name": "Headroom Proxy", + "options": { + "baseURL": "http://127.0.0.1:8787/v1", + "apiKey": "sk-your-deepseek-key" + }, + "models": { + "deepseek-v4-pro": { + "name": "DeepSeek V4 Pro", + "limit": { "context": 1000000, "output": 384000 } + }, + "deepseek-v4-flash": { + "name": "DeepSeek V4 Flash", + "limit": { "context": 1000000, "output": 384000 } + } + } + } + }, + "mcp": { + "headroom": { + "type": "local", + "command": ["headroom", "mcp", "serve"], + "enabled": true + } + } +} +``` + +**Important:** Only include model IDs that appear in the proxy's `/v1/models` +response. OpenCode validates config models against the proxy's model list. +The current DeepSeek model names are `deepseek-v4-pro` and `deepseek-v4-flash`. +`deepseek-chat` and `deepseek-reasoner` are deprecated compatibility aliases. + +### Model comparison + +| Model | Input / Output (per 1M) | Context | Max Output | +|---|---|---|---| +| `deepseek-v4-pro` | $0.435 / $0.87 | 1M | 384K | +| `deepseek-v4-flash` | $0.14 / $0.28 | 1M | 384K | + +Both models support **thinking mode** for step-by-step reasoning (see below). + +Switch models at any time with `/model` in OpenCode. + +--- + +## 5. Start OpenCode + +```bash +opencode +``` + +Run `/models` to confirm both DeepSeek models appear under "Headroom Proxy". +Select one with `/model deepseek-v4-flash` or `/model deepseek-v4-pro`. + +--- + +## 6. Check savings + +```bash +curl http://127.0.0.1:8787/stats | python3 -m json.tool | grep -A5 compression +``` + +Or open the dashboard at [http://127.0.0.1:8787/dashboard](http://127.0.0.1:8787/dashboard). + +--- + +## Thinking mode (reasoning) + +Both models support thinking mode natively, and DeepSeek enables it by default. +This replaces the deprecated `deepseek-reasoner` (R1) model. +See [DeepSeek's thinking mode docs](https://api-docs.deepseek.com/guides/thinking_mode) +for details on switching between thinking and non-thinking modes. + +--- + +## Common issues + +### "Authentication Fails" / Unauthorized + +The `apiKey` in OpenCode's config is missing or wrong. OpenCode must send the +API key to the proxy, and the proxy forwards it to DeepSeek. Make sure +`"apiKey": "sk-..."` is set under `options`. + +### Models don't appear under "Headroom Proxy" + +1. Verify the proxy is running: `curl http://127.0.0.1:8787/health` +2. Check which models the proxy exposes: `curl -s http://127.0.0.1:8787/v1/models -H "Authorization: Bearer sk-your-key"` +3. Make sure your config model IDs match **exactly** what the proxy returns +4. Don't use both `opencode.json` and `opencode.jsonc` in the same config directory — use one file + +### Models appear but requests fail + +You ran `headroom wrap opencode`. That command replaces your config with Claude +and GPT models. **Do not use `headroom wrap`.** Configure OpenCode manually as +shown above, and launch OpenCode directly with `opencode`. + +### "headroom" command not found + +`uv tool install` puts binaries in `~/.local/bin/`. Add it to your PATH: + +```bash +export PATH="$HOME/.local/bin:$PATH" +``` + +### Output shaping shows no savings + +Output savings are measured against a learned baseline (it compares "what the +model actually emitted" vs "what it would have emitted unshaped"). After a few +sessions, run: + +```bash +headroom learn --verbosity --apply +``` + +This builds the baseline, and `/stats` will show output savings numbers. The +shaper is active immediately — the numbers just need calibration. + +--- + +## What's NOT in this guide + +- **Claude or GPT models** — this setup uses DeepSeek exclusively +- **`headroom wrap`** — do not use it; it overrides the config +- **Deprecated model names** — `deepseek-chat` and `deepseek-reasoner` are + compatibility aliases that will be deprecated on 2026-07-24; use + `deepseek-v4-pro` and `deepseek-v4-flash` instead +- **Kompress (ML compression)** — requires extra dependencies; SmartCrusher + handles the majority of use cases +- **Any code changes** — headroom ships full DeepSeek support natively + (model tables, pricing, tokenizers, domain detection) diff --git a/docs/content/docs/pipeline-extensions.mdx b/docs/content/docs/pipeline-extensions.mdx index c98455e59..a525404ee 100644 --- a/docs/content/docs/pipeline-extensions.mdx +++ b/docs/content/docs/pipeline-extensions.mdx @@ -77,3 +77,26 @@ curl http://localhost:8787/v1/chat/completions \ ``` Internal `x-headroom-*` headers (including this one) are stripped before the request is forwarded upstream by default — see `HEADROOM_STRIP_INTERNAL_HEADERS` in [Configuration](/docs/configuration). + +## Per-request model routing with `request.state.headroom_route` + +`x-headroom-base-url` is client-driven and points at one OpenAI-compatible base. When the choice of model belongs to an extension instead of the caller — a router that picks a cheaper model per turn, say — publish it on the request state and Headroom serves that one request from a backend that speaks the target provider: + +```python +# middleware or an extension holding the request +request.state.headroom_route = SimpleNamespace( + model="moonshot/kimi-k2", # required + provider="moonshot", # optional; inferred from the model id if absent + reason="cheaper at this prefix length", +) +``` + +The contract, in `headroom/proxy/route_advice.py`: + +- **Absent means unchanged.** No advice — or advice that is malformed, names an unknown provider, or fails to build a backend — and the request takes exactly the path it took before. A routing preference can never take traffic down. +- **Duck-typed**, so an extension does not import Headroom to publish one. +- A **native** provider (`anthropic`) needs no backend switch — rewrite `body["model"]` yourself. A foreign one is translated by a `LiteLLMBackend` built for it, and Headroom writes the model id. +- Backends are **built once per provider** and cached; a provider that fails to build is not retried per request. +- Honored on `/v1/messages` and `/v1/chat/completions`, streaming and non-streaming alike. (Not the Responses API, which does not use the backend abstraction.) + +`routemegood` is the reference consumer of this seam: it decides, Headroom routes. diff --git a/docs/content/docs/proxy.mdx b/docs/content/docs/proxy.mdx index 0351e3181..35bb7a77f 100644 --- a/docs/content/docs/proxy.mdx +++ b/docs/content/docs/proxy.mdx @@ -68,7 +68,7 @@ Avoid setting process-wide variables such as `HTTP_PROXY`, `HTTPS_PROXY`, `ALL_P |--------|---------|-------------| | `--mode token` | | Prioritize token compression; prior turns may be rewritten for maximum savings. | | `--mode cache` | default | Freeze prior turns to maximize provider prefix-cache hit rate. This is the effective default (see [Savings profiles](#savings-profiles)). | -| `--intercept-tool-results` | `false` | Opt into tool-result interceptors such as ast-grep Read outlining. | +| `--intercept-tool-results` | `false` | Opt into canary tool-result interceptors such as ast-grep Read outlining. Requires `HEADROOM_ROLLOUT_CHANNEL=canary` (or `dev`). | | `--no-read-lifecycle` | `false` | Disable stale/superseded Read-output compression. | | `--code-aware` / `--no-code-aware` | disabled | Enable or disable AST-based code compression. Requires `headroom-ai[code]`. | | `--code-graph` | `false` | Enable the proxy's live code-graph file watcher for the current project. | @@ -249,7 +249,7 @@ Coding agents re-read the same files repeatedly; these control how stale reads a | Flag / env | Default | Effect | |---|---|---| | `--no-read-lifecycle` | lifecycle on | Stop replacing stale/superseded file reads with CCR markers. | -| `--read-maturation` / `HEADROOM_READ_MATURATION` | `false` | *(Experimental)* Hold freshly-read files out of the prefix cache until the file quiesces. | +| `--read-maturation` / `HEADROOM_READ_MATURATION` | `false` | *(Beta)* Hold freshly-read files out of the prefix cache until the file quiesces. Requires `HEADROOM_ROLLOUT_CHANNEL=beta` (or `dev`). | | `--read-maturation-quiesce-turns` | `5` | Turns of no change before a held read is admitted. | ### Reliability: timeouts, retries, limits diff --git a/docs/content/docs/releases.mdx b/docs/content/docs/releases.mdx index bf58bbffc..c876f4080 100644 --- a/docs/content/docs/releases.mdx +++ b/docs/content/docs/releases.mdx @@ -16,10 +16,11 @@ For the end-to-end visual flow, see [CI/CD Flow Diagrams](/docs/ci-cd-flows). | Package | Type | Registry | Environment Variable | |---------|------|----------|----------------------| | `headroom-ai` | Python | PyPI | `PYPI_PACKAGE` | -| `headroom-ai` | TypeScript SDK | npmjs.org | `NPM_SDK_PACKAGE` | -| `headroom-openclaw` | TypeScript plugin | npmjs.org | `NPM_OPENCLAW_PACKAGE` | -| `@{owner}/headroom-ai` | TypeScript SDK | GitHub Package Registry | — | -| `@{owner}/headroom-openclaw` | TypeScript plugin | GitHub Package Registry | — | +| `headroom-ai` | TypeScript SDK | npmjs.org | `NPM_SDK_PACKAGE` | +| `headroom-openclaw` | TypeScript plugin | npmjs.org | `NPM_OPENCLAW_PACKAGE` | +| `headroom-opencode` | TypeScript plugin | npmjs.org | `NPM_OPENCODE_PACKAGE` | +| `@{owner}/headroom-ai` | TypeScript SDK | GitHub Package Registry | — | +| `@{owner}/headroom-openclaw` | TypeScript plugin | GitHub Package Registry | — | | `headroom-ai-{version}.tar.gz` / `headroom_ai-{version}-py3-none-any.whl` | Python package distributions | GitHub Release (`{owner}/headroom`) | — | | `headroom-ai-{version}.tgz` / `headroom-openclaw-{version}.tgz` | Node release assets | GitHub Release (`{owner}/headroom`) | — | | `ghcr.io/{owner}/headroom` | Docker image | GitHub Container Registry | — | @@ -41,6 +42,7 @@ Release Please calculates the release version from conventional commits and the - `pyproject.toml` - `[project].version` - `headroom/_version.py` - `__version__`, synced at build time - `plugins/openclaw/package.json` - `version`, synced at build time +- `plugins/opencode/package.json` - `version`, synced at build time - `sdk/typescript/package.json` - `version`, synced at build time `release.yml` does not commit back to the repo. Version synchronization happens inside the release build workspace. @@ -98,9 +100,10 @@ Installs the built wheels into representative customer environments and imports Downloads the Python dist artifact and publishes to PyPI via `pypa/gh-action-pypi-publish@release/v1` (trusted publisher). ### publish-npm -Publishes both TypeScript packages to npmjs.org: -- `sdk/typescript/` as `headroom-ai` -- `plugins/openclaw/` as `headroom-openclaw` +Publishes all npm packages to npmjs.org: +- `sdk/typescript/` as `headroom-ai` +- `plugins/openclaw/` as `headroom-openclaw` +- `plugins/opencode/` as `headroom-opencode` ### publish-github-packages Publishes both Node packages to GitHub Package Registry (`npm.pkg.github.com`) using the current repository owner as the npm scope: @@ -124,10 +127,11 @@ All package names, registry URLs, and environment names are defined as top-level env: PYPI_PACKAGE: headroom-ai PYPI_ENVIRONMENT: pypi - NPM_REGISTRY_URL: https://registry.npmjs.org - NPM_SDK_PACKAGE: headroom-ai - NPM_OPENCLAW_PACKAGE: headroom-openclaw - GITHUB_PACKAGES_REGISTRY_URL: https://npm.pkg.github.com + NPM_REGISTRY_URL: https://registry.npmjs.org + NPM_SDK_PACKAGE: headroom-ai + NPM_OPENCLAW_PACKAGE: headroom-openclaw + NPM_OPENCODE_PACKAGE: headroom-opencode + GITHUB_PACKAGES_REGISTRY_URL: https://npm.pkg.github.com ``` To rename a package, update the corresponding constant — all references throughout the workflow update automatically. diff --git a/docs/content/docs/runtime-rollouts.mdx b/docs/content/docs/runtime-rollouts.mdx new file mode 100644 index 000000000..8dead2da9 --- /dev/null +++ b/docs/content/docs/runtime-rollouts.mdx @@ -0,0 +1,174 @@ +--- +title: Runtime Rollouts +description: Deterministic runtime feature control for installed Headroom artifacts. +--- + +Runtime rollout answers one question: **which behaviors may this already-built +Headroom artifact expose in this process?** It is separate from the source and +distribution lifecycle, which decides which commit/artifact is qualified, +released, packaged, and published. + +```bash +HEADROOM_ROLLOUT_CHANNEL=canary headroom proxy +``` + +This runs the installed artifact with canary-eligible runtime features available +according to that artifact's rollout policy. It does **not** install, select, or +run a canary release/version of Headroom. + +## Channels and feature policy + +Channels are ordered `stable < beta < canary < dev`. + +| Channel | Purpose | +|---------|---------| +| `stable` | Default; behavior eligible for normal production use. | +| `beta` | Opt-in behavior backed by automated and limited production evidence. | +| `canary` | Early dogfood behavior still gathering evidence. | +| `dev` | Local development and maintainer experiments. | + +Availability and default enablement are separate registry fields. A feature can +be available in `canary` but remain off until explicitly requested; another can +be available and default-enabled in `stable`. + +Request a named feature: + +```bash +HEADROOM_ROLLOUT_CHANNEL=canary \ +HEADROOM_FEATURES=tool_result_interceptors \ +headroom proxy --intercept-tool-results +``` + +Force it off with the kill switch: + +```bash +HEADROOM_DISABLE_FEATURES=tool_result_interceptors headroom proxy +``` + +## Resolution and precedence + +CLI arguments, environment variables, and typed configuration are resolved once +at configuration construction. The immutable snapshot is injected into the +proxy and transform pipelines; changing the process environment afterward does +not alter a running proxy. + +The existing loopback-only `/admin/runtime-env` endpoint is one narrow +exception: hot-reloading the legacy `HEADROOM_OUTPUT_SHAPER` alias replaces the +proxy's immutable snapshot with a newly resolved snapshot. Channel bounds and +`HEADROOM_DISABLE_FEATURES` still win, and `/stats.rollout` changes with the +effective running decision. Because these overrides are process-local, the +endpoint rejects updates when the built-in server uses multiple workers; restart +the proxy with the desired environment instead. Ambient environment mutation +remains ignored. + +Precedence is deterministic: + +| Condition | Result | +|-----------|--------| +| Explicit disable | Off, even if defaulted, requested, aliased, or unsafe override is active. | +| Requested below its availability channel, unsafe override active | On with `unsafe_override`. | +| Requested below its availability channel | Off with `blocked_by_channel`. | +| Explicit request in an allowed channel | On with `explicit`. | +| Enabled legacy alias in an allowed channel | On with `legacy_alias`. | +| Default-enabled in the active channel | On with `default`. | +| Otherwise | Off with `not_requested`. | + +Legacy feature-specific variables are narrow compatibility aliases only. They +obey channel bounds and explicit disable precedence. + +## Unsafe override and invalid input + +`HEADROOM_UNSAFE_ALLOW_UNSTABLE_FEATURES=1` is a break-glass mechanism. It can +cross a channel boundary for a requested feature, but cannot beat an explicit +disable. The runtime remains usable for debugging and emergency reproduction, +while its snapshot reports: + +```json +{ + "unsafe_override": true, + "qualification_eligible": false, + "qualification_ineligible_reason": "unsafe_rollout_override_active" +} +``` + +The Python resolver logs a warning and falls back to `stable` for an unknown +channel; unknown feature names are warned and ignored (fail-closed). Explicit +Python diagnostics (`headroom rollout status`) and the Rust front proxy's typed +CLI/environment parser reject unknown channels/features and list valid values +before startup. + +## Machine-readable status and provenance + +Inspect a supplied configuration without starting the proxy: + +```bash +headroom rollout status --json +``` + +Inspect the actual running process through the supported black-box endpoint: + +```bash +curl http://127.0.0.1:8787/stats +``` + +The Python proxy publishes the object at `/stats.rollout`. The Rust front proxy, +when deployed, publishes its own effective snapshot at `/rollout/status`; this +keeps each process's distinct feature registry and decisions independently +observable. + +The `/stats.rollout` object and CLI output contain no secrets. They include: + +```json +{ + "schema_version": 1, + "policy_version": "1", + "channel": "stable", + "unsafe_override": false, + "registry_digest": "sha256:...", + "snapshot_digest": "sha256:...", + "qualification_eligible": true, + "features": [ + { + "name": "tool_result_interceptors", + "available_in": "canary", + "default_enabled_in": null, + "requested": false, + "disabled": false, + "enabled": false, + "decision": "not_requested" + } + ] +} +``` + +`schema_version` versions the external JSON contract. `policy_version` versions +the rollout rules. `registry_digest` is SHA-256 over canonical, ordered feature +definitions. `snapshot_digest` identifies the complete effective runtime state. +Equivalent policies/configurations produce equal digests; material policy or +decision changes do not. + +These identities deliberately remain separate from source SHA, artifact SHA-256, +runtime payload SHA-256, and future qualification-policy identities. An external +benchmark can compare `/stats.rollout.registry_digest` and `snapshot_digest` +between A1 passthrough and B Headroom arms without importing Headroom internals. +A mismatch makes the future experiment invalid; benchmark logic itself is out of +scope for runtime rollout. + +## Evidence-backed graduation and rollback + +Features progress from canary through beta toward stable only with linked +deterministic, integration, and benchmark evidence. **Bake time is evidence, not +qualification by itself.** Stable eligibility is followed by release +qualification before behavior becomes a stable default. + +Every rollout-managed behavior must have a fast disable path. Operational +rollback uses `HEADROOM_DISABLE_FEATURES`; source rollback reverts the defining +change. The unsafe override is for diagnostics, not promotion or passing release +evidence. + +Contributors should add named registry entries and tests for default behavior, +explicit request, channel blocking, disable precedence, unsafe behavior, +decision reasons, and provenance rather than reading rollout variables inside +implementation components. Python and Rust registries contain features relevant +to their own runtimes, but share channel ordering, precedence, decision reasons, +fail-closed invalid-input semantics, and deterministic identity semantics. diff --git a/docs/context-mode-integration-analysis.md b/docs/context-mode-integration-analysis.md new file mode 100644 index 000000000..fc21c7bf1 --- /dev/null +++ b/docs/context-mode-integration-analysis.md @@ -0,0 +1,378 @@ +# context-mode → Headroom: enterprise plugin & variant analysis + +Analysis date: 2026-07-29. Sources: `/Users/tcms/demo/context-mode` @ v1.0.169, `/Users/tcms/demo/headroom` @ main. + +--- + +## 1. Bottom line + +context-mode and Headroom attack the same cost problem at **two different layers**, and they do not +overlap where it matters: + +| | context-mode | Headroom | +|---|---|---| +| Interception point | agent **tool-call boundary** (host hooks + MCP) | model **API boundary** (proxy / SDK / MCP) | +| Position relative to context | **pre-context** — data never enters | **in-context** — data already entered, gets squeezed | +| Mechanism | admission control: block, redirect, sandbox, externalize | compression: crush, cache, retrieve | +| Touches the wire request | never | always | +| Loss | lossless (full content in FTS5, queryable) | lossy squeeze + hash rehydrate | + +Headroom's own realignment doc identifies its correct compression target as the **live zone**: +"latest user message content + latest `tool_result` + latest `function_call_output` + latest +`local_shell_call_output`" (`REALIGNMENT/00-overview.md`, Phase B). + +**That is precisely the payload context-mode intercepts one layer earlier.** Headroom Phase B is +building a Rust engine to compress the latest tool result *after* it hits the wire. context-mode +stops that tool result from being produced at all. These are complements, not competitors — and the +upstream position is strictly cheaper: nothing to compress, nothing to cache-invalidate, no +token-validation fallback needed. + +Three strategic unlocks, in order of value: + +1. **Cache safety.** Headroom's #1 identified bug class is prompt-cache busting from request + mutation (5 top-tier cache-killer bugs, `REALIGNMENT/00-overview.md`). context-mode has + *structurally zero* cache-bust risk because it never touches the request body. +2. **Subscription safety.** The realignment flags "fingerprint-class subscription-revocation + risks" from `X-Headroom-*` header leakage, `anthropic-beta` mutation and re-serialization on + OAuth/subscription CLIs. A hook-layer product carries none of this — it is invisible to the + upstream. This is a *deployable-where-the-proxy-can't-go* capability. +3. **Proxy-free deployment.** Headroom's value today requires being in the API path + (`127.0.0.1:8787`). Verified live this session: with the proxy down, `headroom_stats` returns all + zeros and `headroom_compress` no-ops. Enterprises that cannot reroute model traffic (TLS trust, + egress policy, subscription auth) currently get nothing. context-mode's hook+MCP model needs no + interposition. + +Zero references to context-mode exist in the Headroom tree today — clean slate. + +--- + +## 2. context-mode: portable IP inventory + +41,617 lines of TypeScript, 11 MCP tools, 18 host adapters, npm-distributed +(`context-mode@1.0.169`, 8 runtime deps, esbuild-bundled). + +Ranked by *how hard it would be for Headroom to rebuild*: + +### Tier 1 — genuinely hard, no Headroom equivalent + +**1. Cross-host hook adapter layer** — `src/adapters/**` (~10K LOC), `src/adapters/types.ts`, +`src/adapters/detect.ts` (737 lines), `configs/` (18 hosts). +Normalizes three incompatible paradigms — `json-stdio` (Claude Code, Gemini/Qwen, Copilot, Codex, +Kimi, Cursor, Kiro, Antigravity), `ts-plugin` (OpenCode, KiloCode, OpenClaw), `mcp-only` (Zed, Pi, +OMP) — behind one contract: normalized `PreToolUse` / `PostToolUse` / `PreCompact` / +`SessionStart` events, a `PlatformCapabilities` matrix, and a 5-way decision +(`allow | deny | modify | context | ask`). Per-host install, config-format, and self-heal machinery +included (`hooks/heal-partial-install.mjs`, `scripts/plugin-cache-integrity.mjs`). +*Why hard to rebuild:* the value is entirely in the accumulated per-host quirks. There is no spec to +implement against. + +**2. Tool-boundary policy engine** — `src/security.ts` (889 lines). +A real policy decision point, not a regex list: glob→regex compilation, chained-command splitting +(`&&`/`;`/`|` with escape awareness), subshell extraction, deny/ask pattern ingestion from host +settings files, project-boundary containment (`evaluateProjectContainment` — Issue #852: an approved +`ctx_execute_file` cannot escape the repo via a path the user couldn't see), and a +**shell-escape scanner** (`SHELL_ESCAPE_PATTERNS`, `extractShellCommands`) that detects +`execSync`/`subprocess`/etc. embedded inside sandboxed *non-shell* code and re-evaluates the escaped +command against policy. +*Why hard to rebuild:* this is the sandbox-escape prevention layer. Getting it wrong is a CVE. + +**3. Multi-language sandbox executor** — `src/executor.ts` (785), `src/runPool.ts`, +`src/exit-classify.ts`, `src/truncate.ts`. +12 languages, stdout-only egress, timeouts, background detach, output caps, exit classification. +Enforces the "Think in Code" contract: the agent programs the analysis, only the answer enters +context. + +**4. Lossless externalization store** — `src/store.ts` (2,071 lines). +Dual SQLite FTS5 index — a tokenized `chunks` table *plus* a `chunks_trigram` table for +substring/identifier search where BM25 tokenization fails on code — with a `vocabulary` table and +schema migration path. Auto-externalizes any output >100 KB into FTS5 and returns a pointer. +Nothing is discarded; the model queries on demand. + +### Tier 2 — valuable, but partially duplicated in Headroom + +**5. Counterfactual savings accounting** — `src/session/analytics.ts` (3,085 lines), +`src/session/project-attribution.ts`, `src/session/db.ts` (1,726). +`ContextSavings`, `ThinkInCodeComparison`, `RealBytesStats`, `MultiAdapterLifetimeStats`, +`enumerateAdapterDirs()`. Measures *what would have entered context but didn't* — a different and +harder quantity than Headroom's `savings_ledger.py`, which records actual compression deltas. +Session event ledger + `tool_calls` + resume + per-project attribution. + +**6. Multi-vendor pricing catalog** — `src/session/pricing.ts` + `model-prices.json`. +61 curated models × 4 rate buckets (input / output / cache-read / cache-write), refreshed from +litellm, unknown model → `null` rather than a silently wrong Claude rate. +**Overlaps `headroom/pricing/*` heavily. Do not port.** + +### Tier 3 — do not port + +Compression heuristics, memory/graph/relevance, telemetry transport, dashboard, install UX, +update-check. Headroom has all of these, more mature, and Phase B/H is actively consolidating them. + +--- + +## 3. Headroom's actual extension seams + +Verified entry-point groups (all `importlib.metadata`-discovered, all opt-in): + +| Seam | Group | Contract | Source | +|---|---|---|---| +| Proxy extension | `headroom.proxy_extension` | `install(app: FastAPI, config: ProxyConfig) -> None` | `headroom/proxy/extensions.py:52` | +| Pipeline extension | `headroom.pipeline_extension` | `on_pipeline_event(PipelineEvent) -> PipelineEvent \| None` over 11 stages | `headroom/pipeline.py:13,68` | +| Learn plugin | `headroom.learn_plugin` | — | `headroom/learn/registry.py:44` | +| Memory text store | `headroom.memory_text` | — | `headroom/memory/config.py:41`, `factory.py:57` | +| Memory vector store | `headroom.memory_vector` | — | `headroom/memory/config.py:34` | +| Memory store | `headroom.memory_store` | — | `headroom/memory/config.py:25` | +| CCR backend | `headroom.ccr_backend` | — | `headroom/cache/compression_store.py:981` | +| Compression hooks | (subclass, not entry point) | `pre_compress` / `compute_biases` / `post_compress` | `headroom/hooks.py:1-31` | + +Two things worth noting: + +- `headroom/proxy/extensions.py:32` states an explicit **stability contract**: changing + `install(app, config)` or the group name requires a deprecation cycle. This is a supported public + seam, not an accident. +- `headroom/hooks.py:16` says outright: *"Headroom SaaS implements position-aware compression and + cross-turn deduplication via these hooks."* The open-core split is already designed in. + +**The exemplar to copy:** `plugins/headroom-oauth2/` — own `pyproject.toml`, own `LICENSE`, own +`SPEC.md`, registers on `headroom.proxy_extension`, dormant until `--proxy-extension oauth2`, +all config via env, "zero core changes." That is the enterprise plugin template. + +**The precedent to copy:** `headroom/lean_ctx/installer.py` and `headroom/rtk/installer.py` — +Headroom already ships thin installers that adopt sibling products. `plugins/headroom-agent-hooks` +already installs startup hooks into Claude Code and Copilot CLI. The socket exists. + +**The gap:** Headroom has *no tool-boundary interception anywhere*. It sees `tool_use`/`tool_result` +only as message content after the fact (`headroom/parser.py`, `headroom/tokenizers/*`). Its +`PipelineStage` enum has no tool-result stage. Everything context-mode does is upstream of +Headroom's earliest hook. + +--- + +## 4. Proposed plugins & variants + +Ranked by value ÷ effort. + +### P1 — `headroom-recall`: FTS5+trigram lossless store as `headroom.memory_text` + +**What:** port `src/store.ts` behind the existing `headroom.memory_text` seam. + +**Why this first:** it is the smallest diff onto an *already-existing* contract, and it fixes a real +product limitation. Today `headroom_retrieve(hash)` requires you to *know the hash* — the tool +description literally says "hash comes from compression markers like `[N items compressed... hash=abc123]`". +With an FTS5-backed store you get `retrieve-by-query`: "what did that build log say about OOM" +instead of "paste hash abc123". The trigram index matters specifically because BM25 tokenization +loses identifiers and stack frames. + +Composes rather than replaces: `compress` → return squeezed text + hash → store the *original* in +FTS5 → rehydrate by hash **or** by query. Also a natural `headroom.ccr_backend` implementation — +the realignment wants "CCR hardens: persistent backend" (Phase B), and this is one. + +**Enterprise variant:** shared team store, retention/TTL policy, per-project scoping (context-mode +already has `project-attribution.ts`), audit of every retrieval. + +**Effort:** medium. Reimplement in Python/Rust against Headroom's memory interface, or ship the +node store as a sidecar. Do not port the MCP tool surface — only the store. + +### P2 — `headroom-admission`: tool-boundary admission control across 18 hosts + +**What:** context-mode's adapter + hook layer, distributed the way `plugins/openclaw` and +`plugins/opencode` already are (TS package under `plugins/`), reporting savings into Headroom's +`savings_ledger.py` JSONL and emitting Headroom pipeline events. + +**Why:** this is the strategic piece. It gives Headroom: +- a **pre-wire** enforcement point, upstream of Phase B's live-zone engine, with no cache-bust and + no token-validation fallback required; +- coverage of **18 agent hosts** — the realignment's Phase G wants to "extend wrap CLIs (cline, + continue, goose, openhands)"; this is that work already done, and then some; +- a deployment mode that works under **subscription auth**, where the proxy is a revocation risk. + +**Enterprise value — this is the DLP story Headroom cannot currently tell.** A `curl` inside a Bash +tool call never touches the proxy, so Headroom is blind to it. context-mode blocks +`curl`/`wget`/`WebFetch`/inline `fetch()`/`requests.get` at the tool boundary and forces network +egress through `ctx_fetch_and_index`. That converts a token-savings feature into an +**egress-control** feature — a different budget line and a different buyer. + +**Effort:** high, but it's mostly packaging + a reporting bridge, not a rewrite. Keep it TypeScript; +Phase H retires Python *proxy* code but explicitly preserves "CLI wrappers, RTK installer" — the +installer layer is the surviving Python, and it can shell out. + +### P3 — `headroom-policy` (Enterprise, license-gated): the PDP + +**What:** `src/security.ts` as a policy decision point, plus centrally-managed org rulesets. + +Two attach points: the hook layer from P2 (tool-level `allow/deny/ask`), and +`headroom.pipeline_extension` at `PRE_SEND` (prompt-level policy). Feeds `headroom/audit/`. + +**Enterprise features that only make sense paid:** central policy service, org-wide allow/deny +rulesets, project-boundary containment enforcement, shell-escape detection inside sandboxed code, +tamper-evident audit trail, per-team reporting. Gate it with the ELv2 license key (see §6). + +**Effort:** medium. The engine exists and is tested (`tests/security/`, `src/security.ts` 889 lines); +the work is the control plane. + +### P4 — `headroom-sandbox`: Think-in-Code execution + +**What:** `executor.ts` exposed as a Headroom MCP tool (`headroom_execute`), 12 languages, +stdout-only. + +**Why:** this is the mechanism behind context-mode's largest measured savings — +`ctx_execute_file` returns 98% savings across 315 KB of real fixtures (`BENCHMARK.md` Part 1), +versus 82% for index+search (Part 2). Programming the analysis beats compressing the output. + +Must ship *with* P3: the shell-escape scanner is what stops the sandbox being an escape hatch. + +**Effort:** medium-high. Runtime isolation is the hard part; `headroom` already has a `sandbox` extra +in `pyproject.toml` to build on. + +### P5 — `headroom-attribution`: counterfactual savings + per-project cost + +**What:** port the *methodology* from `session/analytics.ts` — `RealBytesStats`, +`ThinkInCodeComparison`, `enumerateAdapterDirs`, `project-attribution.ts` — into Headroom's +`savings_ledger` / `reporting` / `dashboard`. + +**Why:** Headroom measures compression deltas (what it squeezed). context-mode measures the +counterfactual (what never entered). Enterprise buyers want the second number, sliced by team and +repo. Do **not** port `pricing.ts` — `headroom/pricing/*` already does this with litellm resolution. + +**Merge, don't port.** `headroom/audit/reads.py` is already a counterfactual measurement tool over +the same Claude Code transcript corpus (see §8). It has the better mechanism taxonomy — identical +repeat, subset containment, write-readback, stale, line-number scaffolding, context residency, +cache-death windows. `analytics.ts` has the multi-host coverage and per-project attribution it +lacks. Combine the two rather than adding a third implementation. + +**Effort:** low-medium, mostly a metrics-definition merge. + +### Variants (packaging, not code) + +- **Headroom No-Proxy Edition** — P1+P2 only, zero API interposition. Sells to buyers who cannot + reroute model traffic and to every subscription-auth user. Removes the single biggest deployment + blocker Headroom has. +- **Headroom Admission Control (Enterprise)** — P2+P3+P4 with a central policy plane and fleet + enrollment across 18 hosts. Positioned as AI-agent DLP/governance, not token savings. +- **Headroom Fleet** — P5 + `enumerateAdapterDirs` for org-wide rollout state and cost reporting. + +--- + +## 5. Evidence base + +context-mode's `BENCHMARK.md`: 21 scenarios, 376 KB raw → 16.5 KB context, **96% overall**, all +fixtures captured from real tool invocations (Context7, Playwright, `gh`, vitest, tsc, nginx logs, +`git log`, analytics CSV) rather than synthetic. Honest about its weak cases — 13% on a 0.4 KB +Playwright network dump, and Part 2 openly explains why index+search only reaches 50-93% (it returns +exact code blocks rather than summaries, by design). + +Test suite: 125 tests across executor/store/MCP-integration/ecosystem, plus 45 test dirs in `tests/` +covering adapters, security, session, hooks, analytics. + +That's a defensible enough evidence base to reuse in Headroom's own materials, and the fixture corpus +itself is reusable for Headroom's `benchmarks/`. + +--- + +## 6. Blockers — resolve these before writing code + +**1. License incompatibility (hard blocker).** +context-mode is **Elastic License 2.0**, "Copyright 2026 Mert Koseoglu". Headroom is +**Apache-2.0**, "Copyright 2025 Headroom Contributors". + +- ELv2 code **cannot** be merged into the Apache-2.0 core. Not a technicality — it would relicense + Headroom's core. +- ELv2 forbids providing the software "to third parties as a hosted or managed service." That + directly constrains `headroom-managed/`. +- Different copyright holders means this needs an **IP arrangement between entities**, not an + engineering decision. + +The good news: Headroom's plugin architecture is exactly the boundary that makes this tractable. +A separate package with its own `pyproject.toml` and its own `LICENSE`, registered on an entry +point — the `plugins/headroom-oauth2/` shape — can carry ELv2 while core stays Apache-2.0. ELv2 is +also the *right* license for a license-key-gated enterprise tier; it explicitly contemplates one. + +Recommendation: any context-mode-derived code ships as separately-licensed plugin packages under +`plugins/`, never vendored into `headroom/`. Get the IP arrangement in writing first. + +**2. Realignment collision.** +Phases A–I are ~40 PRs / 8–13 weeks and include deleting ~25K LOC. Do not open a new integration +front mid-Phase-B. P1 (`headroom.memory_text` / `ccr_backend`) is the exception — it *serves* Phase +B's "CCR hardens: persistent backend" goal rather than competing with it. + +**3. Phase H direction.** +Python proxy code is being retired. Write nothing new in `headroom/proxy/`. Target the surviving +layers: installers, memory writers, CLI wrappers, and Rust. + +--- + +## 7. Sequencing + +| Order | Item | Gate | +|---|---|---| +| 0 | IP/licensing arrangement | before any code | +| 1 | P1 `headroom-recall` — FTS5 store on `memory_text`/`ccr_backend` | lands inside Phase B, serves it | +| 2 | P2 `headroom-admission` — 18-host hook layer under `plugins/` | after Phase A stabilizes | +| 3 | Variant: **No-Proxy Edition** = P1+P2 | as soon as P2 works on 3+ hosts | +| 4 | P3 `headroom-policy` (Enterprise, ELv2, key-gated) | after P2 | +| 5 | P4 `headroom-sandbox` | with P3, never before | +| 6 | P5 `headroom-attribution` | opportunistic | + +--- + +## 8. Follow-up verification + +All four items flagged as open in the first pass are now resolved. + +**`headroom-managed/` is the SaaS arm, and it is unlicensed.** +`headroom-managed/pyproject.toml`: `name = "headroom-managed"`, `description = "Headroom SaaS +Platform - Managed context window optimization"`, `version = 0.1.0`. It has `app/auth.py`, +`app/middleware/`, `app/routes/`, `app/services/`, `app/models.py`, alembic migrations, and a +`pilot/`. There is **no `license` field and no LICENSE file** — i.e. proprietary by default. + +This *sharpens* the §6 blocker rather than easing it. ELv2 forbids providing the software "to third +parties as a hosted or managed service." The product whose name is literally *Managed* is the one +place context-mode-derived code cannot go without an explicit commercial grant from the copyright +holder. Plan the plugin boundary so that `headroom-managed` consumes only Apache-2.0 core +interfaces, never ELv2 implementations. + +**`headroom/audit/reads.py` does not overlap P3 — and it independently validates the whole thesis.** +It is a *measurement* tool, not an audit trail: it streams Claude Code `*.jsonl` transcripts to size +"the addressable bytes for each Read compression mechanism... so defaults are set from traffic, not +theory." No policy, no tamper-evidence. P3's audit trail remains a gap. + +Two lines in its docstring are the most useful corroboration in either repo: + +- *"context residency — how many assistant turns each Read stays in context (the multiplier on its + prefix-cache read cost; **the case for compress-before-cache-entry**)"* — Headroom is already + arguing, from its own traffic, for moving earlier in the pipeline. context-mode is the terminus of + that argument: compress before **context** entry, not merely before cache entry. +- *"identical repeat — a dedup mechanism for this was prototyped and removed: it measured 0.1% of + Read bytes on real traffic."* — Headroom has already empirically established that + message-history-level dedup is worthless. The addressable bytes are at the tool boundary, not in + history. That is the same conclusion the realignment reached from the cache side, arrived at + independently from the traffic side. + +It *does* overlap **P5** — `audit/reads.py` and context-mode's `session/analytics.ts` are two +independent implementations of counterfactual measurement over the same transcript corpus. Merge +them rather than porting; `audit/reads.py` has the better mechanism taxonomy, `analytics.ts` has +multi-host coverage and per-project attribution. + +**No plugin-authoring docs exist.** `docs/` is a Next.js site (`app/`, `content/`, `components/`); +`wiki/` has nothing on extension authoring (only `macos-deployment.md` matched). `plugins/headroom-oauth2/SPEC.md` +remains the de-facto authoring reference — which means whichever plugin lands first sets the house +style. Worth writing the authoring doc as part of P1. + +**Headroom publishes no benchmark results.** `benchmarks/` is 29 runner scripts with no committed +results artifacts, so no like-for-like number exists to compare against context-mode's 96%. The +comparison has to be run. The harness is there and is unusually strong on exactly the axis that +matters: `prefix_cache_benchmark.py`, `cache_bust_trace_report.py`, `cache_validation_bundle.py`, +`synthetic_token_cache_bust_report.py`, `proxy_mode_benchmark.py`, `agent_cost_benchmark.py`, +`real_world_agent_benchmark.py`. Use it to *prove* the §1 cache-safety claim empirically rather than +asserting it — a measured "zero cache-bust events" result is the strongest possible artifact for the +No-Proxy Edition. + +**Bonus finding — the platform axes are orthogonal.** +`docs/platform-feature-matrix.json` (schema v1, updated 2026-07-06) tracks coverage across +`["linux", "macos", "windows"]` — Headroom's platform axis is **operating system**. context-mode's +platform axis is **agent host** (18 of them). Headroom tracks no host-coverage matrix at all. P2 +therefore fills a dimension that does not currently exist in Headroom's own feature accounting, +which also means it needs a second matrix rather than new rows in this one. + +*Process note:* six subagents were dispatched across this analysis and all six stalled at the +600-second watchdog; one reported "Bash is temporarily unavailable" before dying, so the failures +were tool-layer, not analytical. Every finding in this document was verified directly. diff --git a/docs/observability.md b/docs/observability.md index b6de50dc6..b43f5a970 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -245,6 +245,21 @@ Every label vocabulary is bounded by code, not customer input: `"other"` and a `tracing::warn!` is emitted so wire-format drift surfaces loudly in logs. - `status`: 5-variant enum. +- `tool` (Python-side `wrap_rtk_invocations_total`): bounded by the + set of tools the wrap CLI rewrites, captured by + `headroom.cli.wrap_rtk_metrics`. +- `model` (Python-side `requests_by_model` / + `_cache_requests_by_model`): unlike the Rust path above, the Python + proxy reads `model` from the request body, so it is client-supplied. + It is bounded at record time by `MAX_DISTINCT_MODELS` + (`headroom.telemetry.context`): once the cap is reached, further + distinct models bucket into the `"other"` sentinel and a one-time + warning is logged, mirroring the `tier` discipline above. The + in-memory dicts and the exported `headroom_requests_by_model` series + can never exceed the cap plus `"other"`. + +Every label vocabulary listed above is bounded by code, so no +client-supplied value can drive label cardinality unbounded. There is no code path where a malicious client can drive label cardinality unbounded. diff --git a/docs/package-lock.json b/docs/package-lock.json index 152001ff3..55bff9b41 100644 --- a/docs/package-lock.json +++ b/docs/package-lock.json @@ -35,7 +35,7 @@ "@types/react-dom": "^19.2.3", "ai": "^6.0.149", "openai": "^6.47.0", - "postcss": "^8.5.19", + "postcss": "^8.5.26", "tailwindcss": "^4.2.2", "typescript": "^5.9.3" } @@ -5629,9 +5629,9 @@ "license": "MIT" }, "node_modules/nanoid": { - "version": "3.3.15", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.15.tgz", - "integrity": "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==", + "version": "3.3.18", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", + "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", "funding": [ { "type": "github", @@ -5816,9 +5816,9 @@ } }, "node_modules/postcss": { - "version": "8.5.19", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.19.tgz", - "integrity": "sha512-Mz8SaolMd8nB+G13WkORcxQKHZ/NE4xXevtkJHVuG+guo9/wYKlIMTKAqGdEmYOXR2ijPjTYNHssizdaVSUNdQ==", + "version": "8.5.26", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.26.tgz", + "integrity": "sha512-u82N74LFzG8ca+dD8puPnplTXoGH4fTPpVGuIbt36G3qvNlkvfD0lEAZSxaly3KX8TS/L1A1gsCEmvKmBcVbkQ==", "funding": [ { "type": "opencollective", @@ -5835,7 +5835,7 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.12", + "nanoid": "^3.3.17", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" }, diff --git a/docs/package.json b/docs/package.json index 6a2171274..21abffaf3 100644 --- a/docs/package.json +++ b/docs/package.json @@ -36,7 +36,7 @@ "@types/react-dom": "^19.2.3", "ai": "^6.0.149", "openai": "^6.47.0", - "postcss": "^8.5.19", + "postcss": "^8.5.26", "tailwindcss": "^4.2.2", "typescript": "^5.9.3" }, diff --git a/e2e/wrap/run.py b/e2e/wrap/run.py index 0120adc00..c28a8b1a9 100644 --- a/e2e/wrap/run.py +++ b/e2e/wrap/run.py @@ -790,6 +790,11 @@ def verify_vscode_wrap(base_env: dict[str, str], project_dir: Path) -> None: f'"http://127.0.0.1:{port}{project_prefix}"' in configured, "VS Code wrap should configure the project-scoped proxy URL", ) + assert_true( + f'"github.copilot.advanced.debug.overrideCapiUrl": ' + f'"http://127.0.0.1:{port}{project_prefix}"' in configured, + "VS Code wrap should route Copilot Chat generation through Headroom", + ) assert_true( '"github.copilot.advanced.debug.overrideAuthType": "token"' in configured, "VS Code wrap should configure token auth", @@ -849,8 +854,8 @@ def verify_vscode_claude_wrap(base_env: dict[str, str], project_dir: Path) -> No "VS Code Claude wrap should configure the project-scoped Anthropic URL", ) assert_true( - configured["env"]["ENABLE_TOOL_SEARCH"] == "true", - "VS Code Claude wrap should retain Claude Code tool deferral", + configured["env"]["ENABLE_TOOL_SEARCH"] == "false", + "VS Code Claude wrap should disable tool deferral for webview compatibility", ) assert_true(configured["env"]["KEEP"] == "yes", "Existing Claude env must remain") assert_true(str(settings_path) in output, "Wrap output should identify Claude settings") diff --git a/headroom/_ort.py b/headroom/_ort.py index 4a77d847a..8f10cd7bd 100644 --- a/headroom/_ort.py +++ b/headroom/_ort.py @@ -40,15 +40,40 @@ import importlib.util import logging import os import sys +from importlib.metadata import PackageNotFoundError, version from pathlib import Path logger = logging.getLogger(__name__) _ENV_VAR = "ORT_DYLIB_PATH" +_MIN_RUST_ORT_API_VERSION = (1, 24) # Tri-state module cache: unset sentinel / resolved path / None (no pin). _UNSET = object() _pinned: object = _UNSET +_pinned_from_override = False + + +def _installed_ort_version() -> tuple[int, int] | None: + """Return the installed ONNX Runtime major/minor without importing it.""" + try: + raw = version("onnxruntime") + return tuple(int(part) for part in raw.split(".")[:2]) # type: ignore[return-value] + except (PackageNotFoundError, ValueError): + return None + + +def rust_ort_runtime_compatible() -> bool: + """Whether native Rust detection can safely initialize ORT C API 24. + + A caller-supplied ``ORT_DYLIB_PATH`` remains an explicit override: its + library may be newer than the separately installed Python package. The + auto-pinned package library, however, must advertise at least 1.24. + """ + if _pinned_from_override: + return True + installed = _installed_ort_version() + return installed is not None and installed >= _MIN_RUST_ORT_API_VERSION def ensure_ort_dylib_pinned() -> str | None: @@ -84,12 +109,24 @@ def _resolve_ort_native_library(capi_dir: Path) -> Path | None: def _resolve_and_pin() -> str | None: + global _pinned_from_override try: existing = os.environ.get(_ENV_VAR) if existing: + _pinned_from_override = True logger.debug("%s already set; respecting user override: %s", _ENV_VAR, existing) return existing + installed = _installed_ort_version() + if installed is not None and installed < _MIN_RUST_ORT_API_VERSION: + logger.warning( + "onnxruntime %d.%d exposes an older C API than Rust detection " + "requires (1.24+); leaving %s unset and using Python detection", + *installed, + _ENV_VAR, + ) + return None + spec = importlib.util.find_spec("onnxruntime") if spec is None or not spec.origin: logger.debug( diff --git a/headroom/backends/anyllm.py b/headroom/backends/anyllm.py index 468d20450..c1e02b35f 100644 --- a/headroom/backends/anyllm.py +++ b/headroom/backends/anyllm.py @@ -25,6 +25,44 @@ except ImportError: AnyLLM = None # type: ignore +def _convert_anthropic_tool(tool: dict[str, Any]) -> dict[str, Any]: + """Convert an Anthropic tool definition to the OpenAI function shape. + + any-llm speaks OpenAI, so an Anthropic ``{name, description, input_schema}`` + tool must become ``{type: function, function: {name, description, + parameters}}`` before it is forwarded, or the provider ignores/rejects the + tools array and the model never calls a tool. Mirrors the LiteLLM backend's + converter so both OpenAI-compatible backends send the same shape. + """ + func: dict[str, Any] = {"name": tool.get("name", "")} + if "description" in tool: + func["description"] = tool["description"] + if "input_schema" in tool: + func["parameters"] = tool["input_schema"] + return {"type": "function", "function": func} + + +def _convert_tool_choice(choice: Any) -> Any: + """Convert an Anthropic ``tool_choice`` to the OpenAI shape (mirrors LiteLLM). + + Anthropic: ``{"type": "auto"}``, ``{"type": "any"}``, ``{"type": "tool", + "name": ...}``. OpenAI: ``"auto"``, ``"required"``, ``{"type": "function", + "function": {"name": ...}}``. Passing the raw Anthropic dict through makes + the provider reject or ignore it. + """ + if isinstance(choice, str): + return choice + if isinstance(choice, dict): + choice_type = choice.get("type", "auto") + if choice_type == "auto": + return "auto" + if choice_type == "any": + return "required" + if choice_type == "tool": + return {"type": "function", "function": {"name": choice.get("name", "")}} + return "auto" + + class AnyLLMBackend(Backend): """Backend using any-llm for multi-provider support.""" @@ -251,9 +289,9 @@ class AnyLLMBackend(Backend): if "stop_sequences" in body: kwargs["stop"] = body["stop_sequences"] if "tools" in body: - kwargs["tools"] = body["tools"] + kwargs["tools"] = [_convert_anthropic_tool(t) for t in body["tools"]] if "tool_choice" in body: - kwargs["tool_choice"] = body["tool_choice"] + kwargs["tool_choice"] = _convert_tool_choice(body["tool_choice"]) logger.debug(f"any-llm request: provider={self.provider}, model={original_model}") @@ -301,9 +339,9 @@ class AnyLLMBackend(Backend): if "stop_sequences" in body: kwargs["stop"] = body["stop_sequences"] if "tools" in body: - kwargs["tools"] = body["tools"] + kwargs["tools"] = [_convert_anthropic_tool(t) for t in body["tools"]] if "tool_choice" in body: - kwargs["tool_choice"] = body["tool_choice"] + kwargs["tool_choice"] = _convert_tool_choice(body["tool_choice"]) msg_id = f"msg_{uuid.uuid4().hex[:24]}" @@ -324,42 +362,131 @@ class AnyLLMBackend(Backend): }, ) - yield StreamEvent( - event_type="content_block_start", - data={ - "type": "content_block_start", - "index": 0, - "content_block": {"type": "text", "text": ""}, - }, - ) - stream_response = await self.llm.acompletion(**kwargs) output_tokens = 0 + # Stream text immediately in a single text block, but BUFFER tool + # calls and emit them as complete blocks at the end. OpenAI streams + # parallel tool calls interleaved by index (index 0 and 1 introduced + # together, then a fragment for 0, then for 1), while Anthropic + # requires each content block to be fully emitted — start, deltas, + # stop — before the next opens. Reassembling per index and flushing + # complete blocks keeps every delta inside its own block's start/stop + # for any interleaving. (The previous version pre-opened one text + # block and dropped tool calls entirely; a naive open-on-new-index + # instead mis-sequenced parallel calls, emitting a fragment for an + # already-stopped block.) + current_block_index = -1 + text_block_open = False + # provider tool index -> {"id", "name", "arguments"}, first-seen order + tool_calls: dict[int, dict[str, Any]] = {} + tool_order: list[int] = [] + stop_reason = "end_turn" async for chunk in cast(AsyncIterator[Any], stream_response): - if hasattr(chunk, "choices") and chunk.choices: - delta = chunk.choices[0].delta - if hasattr(delta, "content") and delta.content: + if not (hasattr(chunk, "choices") and chunk.choices): + continue + choice = chunk.choices[0] + delta = choice.delta + + # Map OpenAI finish_reason to the Anthropic stop_reason so a tool + # call or a length truncation is not reported as end_turn. + finish_reason = getattr(choice, "finish_reason", None) + if finish_reason == "tool_calls": + stop_reason = "tool_use" + elif finish_reason == "length": + stop_reason = "max_tokens" + elif finish_reason == "stop": + stop_reason = "end_turn" + + if getattr(delta, "tool_calls", None): + for tc in delta.tool_calls: + idx = tc.index if getattr(tc, "index", None) is not None else 0 + buf = tool_calls.get(idx) + if buf is None: + buf = {"id": None, "name": "", "arguments": ""} + tool_calls[idx] = buf + tool_order.append(idx) + if getattr(tc, "id", None): + buf["id"] = tc.id + func = getattr(tc, "function", None) + if func is not None: + if getattr(func, "name", None): + buf["name"] = func.name + if getattr(func, "arguments", None): + buf["arguments"] += func.arguments + + elif getattr(delta, "content", None): + if not text_block_open: + current_block_index += 1 + text_block_open = True yield StreamEvent( - event_type="content_block_delta", + event_type="content_block_start", data={ - "type": "content_block_delta", - "index": 0, - "delta": {"type": "text_delta", "text": delta.content}, + "type": "content_block_start", + "index": current_block_index, + "content_block": {"type": "text", "text": ""}, }, ) - output_tokens += 1 + yield StreamEvent( + event_type="content_block_delta", + data={ + "type": "content_block_delta", + "index": current_block_index, + "delta": {"type": "text_delta", "text": delta.content}, + }, + ) + output_tokens += 1 - yield StreamEvent( - event_type="content_block_stop", - data={"type": "content_block_stop", "index": 0}, - ) + # Close the text block before any tool blocks (Anthropic orders + # content blocks sequentially, text then tool_use). + if text_block_open: + yield StreamEvent( + event_type="content_block_stop", + data={"type": "content_block_stop", "index": current_block_index}, + ) + + # Flush each buffered tool call as a complete, self-contained block: + # start, one input_json_delta with the reassembled arguments, stop. + for idx in tool_order: + buf = tool_calls[idx] + current_block_index += 1 + tool_id = buf["id"] or f"toolu_{uuid.uuid4().hex[:24]}" + yield StreamEvent( + event_type="content_block_start", + data={ + "type": "content_block_start", + "index": current_block_index, + "content_block": { + "type": "tool_use", + "id": tool_id, + "name": buf["name"], + "input": {}, + }, + }, + ) + if buf["arguments"]: + yield StreamEvent( + event_type="content_block_delta", + data={ + "type": "content_block_delta", + "index": current_block_index, + "delta": { + "type": "input_json_delta", + "partial_json": buf["arguments"], + }, + }, + ) + output_tokens += 1 + yield StreamEvent( + event_type="content_block_stop", + data={"type": "content_block_stop", "index": current_block_index}, + ) yield StreamEvent( event_type="message_delta", data={ "type": "message_delta", - "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "delta": {"stop_reason": stop_reason, "stop_sequence": None}, "usage": {"output_tokens": output_tokens}, }, ) diff --git a/headroom/backends/litellm.py b/headroom/backends/litellm.py index 8430789ce..0297474d4 100644 --- a/headroom/backends/litellm.py +++ b/headroom/backends/litellm.py @@ -413,6 +413,81 @@ PROVIDER_REGISTRY: dict[str, ProviderConfig] = { } +# How long an upstream call may go silent before we give up on it. +# +# WHY THIS EXISTS. There was no timeout here at all, so a request the upstream +# never answered blocked its caller forever. Observed 2026-08-07 under load: +# four agent workers sat on ESTABLISHED connections for 36+ minutes while this +# proxy answered /readyz in 0.11s. No error, no retry, no log line -- the +# client just stops. That is the worst shape a failure can take, because it is +# indistinguishable from slow work and no supervisor can tell the difference. +# +# A float, not an httpx.Timeout, on purpose: litellm expands a float into all +# four httpx phases, so for a STREAMING call this becomes the maximum gap +# BETWEEN CHUNKS rather than a cap on total generation time. A long answer +# streaming steadily is never cut off; a stalled one dies. That is the +# semantic we want, and it falls out of the simpler type. +# +# 600s is deliberately generous -- long enough that no healthy call is at +# risk, short enough that a hang surfaces within a coffee break instead of +# never. +UPSTREAM_TIMEOUT_ENV = "HEADROOM_UPSTREAM_TIMEOUT" +DEFAULT_UPSTREAM_TIMEOUT = 600.0 + + +def _upstream_timeout() -> float: + """Seconds. Never raises; a junk env value must not disable the timeout.""" + import os + + try: + v = float(os.getenv(UPSTREAM_TIMEOUT_ENV, DEFAULT_UPSTREAM_TIMEOUT)) + except (TypeError, ValueError): + return DEFAULT_UPSTREAM_TIMEOUT + # 0 or negative would mean "no timeout" to httpx, which is the bug. + return v if v > 0 else DEFAULT_UPSTREAM_TIMEOUT + + +# Providers that cannot possibly accept an Anthropic `sk-ant-` credential. +# +# Explicit, rather than the inverse "anything not Anthropic": an unrecognised +# provider is usually a compatible or self-hosted gateway, and guessing wrong +# there drops a key that WAS working. Bedrock/Vertex are absent because the +# dispatch sites already skip them entirely (env-based auth). +# +# ponytail: a hand-kept tuple; grow it as targets are confirmed. A registry +# lookup would be the upgrade if this ever outgrows a handful of entries. +_REJECTS_ANTHROPIC_KEY = ("openai", "azure", "gemini") + + +def _caller_key_travels_to(model: str, key: str) -> bool: + """Can this inbound credential authenticate the provider we are about to call? + + The caller authenticates to the PROXY. A routing extension may then rewrite + the model across families mid-request (claude-opus-5 -> gpt-5-mini), and the + caller's key does not travel with that rewrite: we forward `sk-ant-...` to + OpenAI and earn a guaranteed 401, which reads downstream as "the cheap model + failed the task" rather than as the routing bug it is. + + Only an unambiguous mismatch is refused. `sk-ant-` is Anthropic's documented + vendor-specific prefix, so it cannot authenticate one of the providers above. + Every other credential -- a plain Bearer token, an OpenAI-style `sk-` that a + dozen vendors also mint, anything aimed at a compatible or custom gateway -- + is unclassifiable from the string alone and keeps the pass-through. + + Returning False drops the api_key kwarg, so litellm falls back to the target + provider's own env credential: the only key that can work. + """ + if not key.startswith("sk-ant-"): + return True + try: + from litellm import get_llm_provider + + provider = (get_llm_provider(model)[1] or "").lower() + except Exception: # noqa: BLE001 - unclassifiable model, keep pass-through + return True + return provider not in _REJECTS_ANTHROPIC_KEY + + def get_provider_config(provider: str) -> ProviderConfig: """Get provider config, with fallback for unknown providers.""" if provider in PROVIDER_REGISTRY: @@ -622,6 +697,21 @@ class LiteLLMBackend(Backend): if anthropic_model.startswith("arn:aws:"): return f"bedrock/converse/{anthropic_model}" + # Cross-region prefixed IDs are already fully qualified system-defined + # profile IDs — pass through directly. Normalizing and re-looking them + # up in the discovery map can route the request to a wrong or + # unauthorized profile (e.g. an APPLICATION profile in the same account + # that also wraps the same foundation model). This applies whether the + # prefix arrives bare ("us.anthropic...") or already LiteLLM-qualified + # ("bedrock/us.anthropic..."). + _CROSS_REGION_PREFIXES = ("au.", "us.", "eu.", "apac.", "global.") + if anthropic_model.startswith(_CROSS_REGION_PREFIXES): + return f"bedrock/{anthropic_model}" + if anthropic_model.startswith("bedrock/") and anthropic_model[ + len("bedrock/") : + ].startswith(_CROSS_REGION_PREFIXES): + return anthropic_model + normalized = _normalize_bedrock_profile_id(anthropic_model) if normalized and normalized in self._model_map: return self._model_map[normalized] @@ -917,14 +1007,21 @@ class LiteLLMBackend(Backend): _env_auth_providers = ("bedrock", "vertex_ai", "vertex_ai_beta", "sagemaker") if self.provider not in _env_auth_providers: auth_header = headers.get("authorization", headers.get("Authorization", "")) - if auth_header.startswith("Bearer "): - kwargs["api_key"] = auth_header[7:] - elif headers.get("x-api-key"): - kwargs["api_key"] = headers["x-api-key"] + _caller_key = ( + auth_header[7:] + if auth_header.startswith("Bearer ") + else headers.get("x-api-key", "") + ) + # Only forward it if it can actually authenticate the TARGET. + if _caller_key and _caller_key_travels_to(litellm_model, _caller_key): + kwargs["api_key"] = _caller_key logger.debug(f"LiteLLM request: model={litellm_model}") # Make the call + # Bounded, always: an upstream that never answers must not + # block the caller forever. setdefault so an explicit value wins. + kwargs.setdefault("timeout", _upstream_timeout()) response = await acompletion(**kwargs) # Convert to Anthropic format @@ -1022,10 +1119,14 @@ class LiteLLMBackend(Backend): _env_auth_providers = ("bedrock", "vertex_ai", "vertex_ai_beta", "sagemaker") if self.provider not in _env_auth_providers: auth_header = headers.get("authorization", headers.get("Authorization", "")) - if auth_header.startswith("Bearer "): - kwargs["api_key"] = auth_header[7:] - elif headers.get("x-api-key"): - kwargs["api_key"] = headers["x-api-key"] + _caller_key = ( + auth_header[7:] + if auth_header.startswith("Bearer ") + else headers.get("x-api-key", "") + ) + # Only forward it if it can actually authenticate the TARGET. + if _caller_key and _caller_key_travels_to(litellm_model, _caller_key): + kwargs["api_key"] = _caller_key msg_id = f"msg_{uuid.uuid4().hex[:24]}" @@ -1055,6 +1156,9 @@ class LiteLLMBackend(Backend): kwargs["stream_options"] = {"include_usage": True} # Stream content — blocks emitted dynamically based on response + # Bounded, always: an upstream that never answers must not + # block the caller forever. setdefault so an explicit value wins. + kwargs.setdefault("timeout", _upstream_timeout()) response = await acompletion(**kwargs) output_tokens = 0 current_block_index = -1 @@ -1275,14 +1379,21 @@ class LiteLLMBackend(Backend): _env_auth_providers = ("bedrock", "vertex_ai", "vertex_ai_beta", "sagemaker") if self.provider not in _env_auth_providers: auth_header = headers.get("authorization", headers.get("Authorization", "")) - if auth_header.startswith("Bearer "): - kwargs["api_key"] = auth_header[7:] - elif headers.get("x-api-key"): - kwargs["api_key"] = headers["x-api-key"] + _caller_key = ( + auth_header[7:] + if auth_header.startswith("Bearer ") + else headers.get("x-api-key", "") + ) + # Only forward it if it can actually authenticate the TARGET. + if _caller_key and _caller_key_travels_to(litellm_model, _caller_key): + kwargs["api_key"] = _caller_key logger.debug(f"LiteLLM OpenAI request: model={litellm_model}") # Make the call + # Bounded, always: an upstream that never answers must not + # block the caller forever. setdefault so an explicit value wins. + kwargs.setdefault("timeout", _upstream_timeout()) response = await acompletion(**kwargs) # Build the usage block. LiteLLM normalizes prompt-cache stats from @@ -1293,10 +1404,16 @@ class LiteLLMBackend(Backend): # cache_creation_tokens for the OpenAI nested dialect. Surface both # so PrefixCacheTracker.update_from_response on the backend-routed # path observes a stable shape instead of branching on key presence. + # None-guard the core counts (same defensive style as the cache + # fields just below). A provider can leave any of these None on the + # Usage object; emitting None here flows into the OpenAI-shape body, + # and the backend-routed OpenAI handler reads them straight into + # arithmetic and RequestOutcome (output_tokens=..., and + # max(0, prompt_tokens - ...)), which raises TypeError on None. usage_block: dict[str, Any] = { - "prompt_tokens": response.usage.prompt_tokens, - "completion_tokens": response.usage.completion_tokens, - "total_tokens": response.usage.total_tokens, + "prompt_tokens": int(getattr(response.usage, "prompt_tokens", 0) or 0), + "completion_tokens": int(getattr(response.usage, "completion_tokens", 0) or 0), + "total_tokens": int(getattr(response.usage, "total_tokens", 0) or 0), } # Defensive getattr: LiteLLM only attaches these top-level attrs @@ -1447,11 +1564,18 @@ class LiteLLMBackend(Backend): _env_auth_providers = ("bedrock", "vertex_ai", "vertex_ai_beta", "sagemaker") if self.provider not in _env_auth_providers: auth_header = headers.get("authorization", headers.get("Authorization", "")) - if auth_header.startswith("Bearer "): - kwargs["api_key"] = auth_header[7:] - elif headers.get("x-api-key"): - kwargs["api_key"] = headers["x-api-key"] + _caller_key = ( + auth_header[7:] + if auth_header.startswith("Bearer ") + else headers.get("x-api-key", "") + ) + # Only forward it if it can actually authenticate the TARGET. + if _caller_key and _caller_key_travels_to(litellm_model, _caller_key): + kwargs["api_key"] = _caller_key + # Bounded, always: an upstream that never answers must not + # block the caller forever. setdefault so an explicit value wins. + kwargs.setdefault("timeout", _upstream_timeout()) response = await acompletion(**kwargs) async for chunk in response: diff --git a/headroom/binaries.py b/headroom/binaries.py index 3ce4cebbf..229bd817e 100644 --- a/headroom/binaries.py +++ b/headroom/binaries.py @@ -29,6 +29,7 @@ import subprocess import sys import tarfile import tempfile +import time import urllib.error import urllib.parse import urllib.request @@ -245,12 +246,24 @@ def _download(url: str, dest: Path, *, progress: bool = True) -> None: raise OSError(f"binary cache directory is not writable: {dest.parent}") final_url = _mirror_url(url) req = urllib.request.Request(final_url, headers={"User-Agent": "headroom-binaries/1"}) - try: - with urllib.request.urlopen(req, timeout=60) as resp: # noqa: S310 (https) - total = int(resp.headers.get("Content-Length") or 0) - _stream_to(resp, dest, total, label=dest.name, show_progress=progress) - except urllib.error.URLError as e: - raise BinaryFetchError(f"failed to download {final_url}: {e}") from e + attempts = 3 + for attempt in range(1, attempts + 1): + try: + with urllib.request.urlopen(req, timeout=60) as resp: # noqa: S310 (https) + total = int(resp.headers.get("Content-Length") or 0) + _stream_to(resp, dest, total, label=dest.name, show_progress=progress) + return + except urllib.error.URLError as e: + dest.unlink(missing_ok=True) + if attempt == attempts: + raise BinaryFetchError( + f"failed to download {final_url} after {attempts} attempts: {e}" + ) from e + # GitHub release assets occasionally return a transient 5xx or + # reset while redirecting to the object store. A short bounded + # retry keeps proxy startup reliable without hiding persistent + # credential, mirror, or connectivity failures. + time.sleep(0.25 * attempt) def _stream_to(src: Any, dest: Path, total: int, *, label: str, show_progress: bool) -> None: diff --git a/headroom/cache/compression_cache.py b/headroom/cache/compression_cache.py index 71beb4091..00abf07b0 100644 --- a/headroom/cache/compression_cache.py +++ b/headroom/cache/compression_cache.py @@ -135,8 +135,10 @@ class CompressionCache: # `compute_frozen_count` (bounded above by the `min` clamp at # `proxy/handlers/anthropic.py`) and `update_from_result`'s # "unchanged content" tracking. - self._stable_hashes: set[str] = set() - self._first_seen: dict[str, float] = {} + # Ordered mappings preserve set/dict-style membership while allowing + # deterministic oldest-first eviction. + self._stable_hashes: OrderedDict[str, None] = OrderedDict() + self._first_seen: OrderedDict[str, float] = OrderedDict() self._hits: int = 0 self._misses: int = 0 self._total_tokens_saved: int = 0 @@ -172,6 +174,34 @@ class CompressionCache: _, evicted = self._cache.popitem(last=False) self._total_tokens_saved -= evicted.tokens_saved + def _mark_stable_locked(self, content_hash: str) -> None: + """Record a stable hash while bounding retained bookkeeping.""" + self._stable_hashes[content_hash] = None + self._stable_hashes.move_to_end(content_hash) + + while len(self._stable_hashes) > self.max_entries: + self._stable_hashes.popitem(last=False) + + def _record_first_seen_locked(self, content_hash: str, seen_at: float) -> None: + """Record a first-seen timestamp while bounding retained bookkeeping.""" + self._first_seen[content_hash] = seen_at + self._first_seen.move_to_end(content_hash) + + while len(self._first_seen) > self.max_entries: + self._first_seen.popitem(last=False) + + def _prune_expired_first_seen_locked( + self, + now: float, + ttl_seconds: float, + ) -> None: + """Remove first-seen entries whose cache timing window has expired.""" + while self._first_seen: + _, oldest_seen_at = next(iter(self._first_seen.items())) + if now - oldest_seen_at < ttl_seconds: + break + self._first_seen.popitem(last=False) + def mark_stable(self, content_hash: str) -> None: """Mark a content hash as stable (unchanged, not compressed). @@ -180,7 +210,7 @@ class CompressionCache: even though no compressed version exists in the cache. """ with self._lock: - self._stable_hashes.add(content_hash) + self._mark_stable_locked(content_hash) def mark_stable_from_messages(self, messages: list[dict], up_to: int) -> None: """Mark all tool_result hashes in messages[:up_to] as stable.""" @@ -189,7 +219,7 @@ class CompressionCache: if _is_tool_result_message(msg): content = _extract_tool_result_content(msg) if content is not None: - self._stable_hashes.add(self.content_hash(content)) + self._mark_stable_locked(self.content_hash(content)) def should_defer_compression( self, @@ -216,13 +246,18 @@ class CompressionCache: """ with self._lock: now = time.time() + self._prune_expired_first_seen_locked(now, ttl_seconds) + first_seen = self._first_seen.get(content_hash) if first_seen is None: - self._first_seen[content_hash] = now + self._record_first_seen_locked(content_hash, now) return False # First time — compress now (no cache entry to preserve) + age = now - first_seen if age >= ttl_seconds - batch_window: + self._record_first_seen_locked(content_hash, now) return False # Near TTL boundary — compress now (batch window) + return True # Seen recently within TTL — defer to preserve cache def get_stats(self) -> dict: @@ -335,7 +370,7 @@ class CompressionCache: continue if orig_content == comp_content: # Content unchanged — mark as stable for frozen count walk - self._stable_hashes.add(self.content_hash(orig_content)) + self._mark_stable_locked(self.content_hash(orig_content)) continue h = self.content_hash(orig_content) tokens_saved = len(orig_content) // 4 - len(comp_content) // 4 diff --git a/headroom/cache/compression_store.py b/headroom/cache/compression_store.py index 72e6ca34f..bbda697cd 100644 --- a/headroom/cache/compression_store.py +++ b/headroom/cache/compression_store.py @@ -52,6 +52,10 @@ DEFAULT_CCR_TTL_SECONDS = 1800 # session-scale; override via HEADROOM_CCR_TTL_S CCR_TTL_SECONDS_ENV = "HEADROOM_CCR_TTL_SECONDS" _RETRIEVAL_LOG_PREVIEW_CHARS = 4096 +# Previews carry verbatim tool-result content (post-redaction), which makes +# proxy.log too sensitive for users to share in bug reports. Set to +# 0/false/no/off to log byte counts only. +PAYLOAD_PREVIEW_ENV = "HEADROOM_LOG_PAYLOAD_PREVIEW" _SECRET_KEY_VALUE_RE = re.compile( r"(?i)\b([A-Z0-9_-]*(?:API[_-]?KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|AUTH)[A-Z0-9_-]*)" r"(\s*[:=]\s*)([\"']?)([^\"'\s,}]+)" @@ -108,7 +112,21 @@ def _redact_retrieval_log_payload(payload: str) -> str: return _API_KEY_VALUE_RE.sub("sk-[REDACTED]", redacted) +def _payload_preview_enabled() -> bool: + raw = os.environ.get(PAYLOAD_PREVIEW_ENV) + if raw is None: + return True + return raw.strip().lower() not in ("0", "false", "no", "off") + + def _payload_for_retrieval_log(payload: str) -> dict[str, Any]: + if not _payload_preview_enabled(): + return { + "payload_chars": len(payload), + "payload_preview_chars": 0, + "payload_truncated": len(payload) > 0, + "payload_preview": "", + } redacted = _redact_retrieval_log_payload(payload) preview = redacted[:_RETRIEVAL_LOG_PREVIEW_CHARS] truncated = len(redacted) > len(preview) diff --git a/headroom/cache/prefix_tracker.py b/headroom/cache/prefix_tracker.py index d54950191..8157e9e9a 100644 --- a/headroom/cache/prefix_tracker.py +++ b/headroom/cache/prefix_tracker.py @@ -21,6 +21,7 @@ import hashlib import itertools import json import logging +import os import time from collections import OrderedDict from dataclasses import dataclass @@ -229,6 +230,184 @@ def _canonicalize_for_prefix_compare(obj: Any) -> Any: return obj +# Canonical relationships between consecutive histories. These constants are +# strings (rather than an Enum) so they remain cheap to log on the request hot +# path and easy to assert in tests. +RELATION_EXACT = "exact" +RELATION_MESSAGE_APPEND = "message_append" +RELATION_BLOCK_APPEND = "block_append" +RELATION_BLOCK_REWRITE_TAIL = "block_rewrite_tail" +RELATION_DIVERGED = "diverged" + + +@dataclass(frozen=True) +class HistoryRelation: + """How a current message history relates to one recorded last turn. + + ``block_append`` and ``block_rewrite_tail`` are deliberately distinct: + Anthropic should keep the breakpoint on the newest block for a pure append, + but anchor it to ``stable_prefix_blocks - 1`` when the previous tail was + rewritten and therefore can never match a prior cache write (#2671). + """ + + kind: str + message_index: int | None = None + stable_prefix_blocks: int = 0 + stable_suffix_blocks: int = 0 + previous_block_count: int = 0 + current_block_count: int = 0 + + +# A rewritten-tail match is intentionally conservative. The production shape +# behind #2671 has a hundred-plus-block stable prefix and a fixed two-block +# suffix. Requiring both avoids merging sibling sub-calls which merely share a +# short injected preamble or a single generic reminder at the end. +_MIN_REWRITE_PREFIX_BLOCKS = 8 +_MIN_REWRITE_SUFFIX_BLOCKS = 2 + + +def _message_fields_outside_content(message: dict[str, Any]) -> dict[str, Any]: + """Return message identity fields, excluding the block list itself.""" + return {key: value for key, value in message.items() if key != "content"} + + +def _stable_leading_block_run(current: list[Any], previous: list[Any]) -> int: + """Number of canonical-equal blocks at the start of both lists.""" + limit = min(len(current), len(previous)) + run = 0 + while run < limit and current[run] == previous[run]: + run += 1 + return run + + +def _stable_trailing_block_run(current: list[Any], previous: list[Any], *, leading_run: int) -> int: + """Non-overlapping canonical-equal suffix length.""" + limit = min(len(current), len(previous)) - leading_run + run = 0 + while run < limit and current[-(run + 1)] == previous[-(run + 1)]: + run += 1 + return run + + +def _classify_history_canonical( + current_messages: list[Any], previous_messages: list[Any] +) -> HistoryRelation: + """Classify two already-canonical, structurally snapshotted histories.""" + if not previous_messages or len(current_messages) < len(previous_messages): + return HistoryRelation(RELATION_DIVERGED) + + changed: HistoryRelation | None = None + for index, previous_message in enumerate(previous_messages): + current_message = current_messages[index] + if current_message == previous_message: + continue + if changed is not None: + return HistoryRelation(RELATION_DIVERGED) + if not isinstance(previous_message, dict) or not isinstance(current_message, dict): + return HistoryRelation(RELATION_DIVERGED) + if _message_fields_outside_content(previous_message) != _message_fields_outside_content( + current_message + ): + return HistoryRelation(RELATION_DIVERGED) + + previous_blocks = previous_message.get("content") + current_blocks = current_message.get("content") + if not isinstance(previous_blocks, list) or not isinstance(current_blocks, list): + return HistoryRelation(RELATION_DIVERGED) + + previous_count = len(previous_blocks) + current_count = len(current_blocks) + leading = _stable_leading_block_run(current_blocks, previous_blocks) + + # Pure block append. The previous write remains intact and Anthropic's + # lookback can find it, so the breakpoint must advance to the newest + # block and cover the newly appended tail. + if current_count > previous_count and leading == previous_count: + changed = HistoryRelation( + RELATION_BLOCK_APPEND, + message_index=index, + stable_prefix_blocks=leading, + previous_block_count=previous_count, + current_block_count=current_count, + ) + continue + + # Rewritten-tail growth. This is narrower than a fuzzy prefix match: + # message count may not change, content may not shrink, most of the old + # prefix must survive, and a substantial fixed suffix must identify the + # sub-call. Crucially ``leading < previous_count`` proves this is NOT a + # pure append (the bug in the original #2702 discriminator). + trailing = _stable_trailing_block_run(current_blocks, previous_blocks, leading_run=leading) + if ( + len(current_messages) == len(previous_messages) + and current_count >= previous_count + and _MIN_REWRITE_PREFIX_BLOCKS <= leading < previous_count + and leading * 2 >= previous_count + and leading * 2 >= current_count + and trailing >= _MIN_REWRITE_SUFFIX_BLOCKS + ): + changed = HistoryRelation( + RELATION_BLOCK_REWRITE_TAIL, + message_index=index, + stable_prefix_blocks=leading, + stable_suffix_blocks=trailing, + previous_block_count=previous_count, + current_block_count=current_count, + ) + continue + + return HistoryRelation(RELATION_DIVERGED) + + if changed is not None: + return changed + return HistoryRelation( + RELATION_EXACT + if len(current_messages) == len(previous_messages) + else RELATION_MESSAGE_APPEND + ) + + +def classify_history_relation( + current_messages: list[dict[str, Any]], + previous_messages: list[dict[str, Any]], +) -> HistoryRelation: + """Return the canonical cross-turn relationship for two raw histories. + + The canonical projection may drop a whole directive-only message. Refuse + classification when that would shift raw message indices: block replay + always slices the raw lists and must never consume a canonical index as a + raw one. + """ + if not current_messages or not previous_messages: + return HistoryRelation(RELATION_DIVERGED) + current = _lineage_snapshot(_canonicalize_for_prefix_compare(current_messages)) + previous = _lineage_snapshot(_canonicalize_for_prefix_compare(previous_messages)) + prefix_len = len(previous_messages) + if len(previous) != prefix_len: + return HistoryRelation(RELATION_DIVERGED) + if len(_canonicalize_for_prefix_compare(current_messages[:prefix_len])) != prefix_len: + return HistoryRelation(RELATION_DIVERGED) + return _classify_history_canonical(current, previous) + + +def segment_fingerprint(value: Any) -> str: + """Stable hash for non-message provider cache-key segments. + + Cache-control placement and transport annotations are deliberately ignored; + semantic tool/model/thinking changes remain visible. The hash is affinity + metadata only and is never used to reconstruct or forward request content. + """ + canonical = _lineage_snapshot(_canonicalize_for_prefix_compare(value)) + encoded = json.dumps( + canonical, + sort_keys=True, + ensure_ascii=False, + separators=(",", ":"), + default=str, + ) + return hashlib.sha256(encoded.encode()).hexdigest()[:24] + + def extract_cache_stable_delta( current_messages: list[dict[str, Any]], previous_original_messages: list[dict[str, Any]] | None, @@ -251,13 +430,13 @@ def extract_cache_stable_delta( """ if not previous_original_messages or previous_forwarded_messages is None: return None + relation = classify_history_relation(current_messages, previous_original_messages) + if relation.kind not in (RELATION_EXACT, RELATION_MESSAGE_APPEND): + # A same-message block append needs a block-level splice in + # ``overlay_cached_prefix``; slicing only whole messages would silently + # discard its new blocks. Rewritten tails are not append-only deltas. + return None prefix_len = len(previous_original_messages) - if len(current_messages) < prefix_len: - return None - if _canonicalize_for_prefix_compare( - current_messages[:prefix_len] - ) != _canonicalize_for_prefix_compare(previous_original_messages): - return None return ( copy.deepcopy(previous_forwarded_messages), copy.deepcopy(current_messages[prefix_len:]), @@ -281,12 +460,12 @@ def overlay_cached_prefix( the corresponding leading messages so the forwarded prefix stays byte-for-byte what the provider hashed for its cache key. - Safe only when this turn append-only-extends the previous turn (the standard - growing-conversation shape): the previous ORIGINAL messages must be an exact - prefix of the current ORIGINAL messages, and there is exactly one forwarded - message per original. Otherwise the previous forwarded bytes may not - correspond to the same positions, so we return ``optimized_messages`` - unchanged (accept a possible bust rather than forward wrong content). + Safe only when this turn extends the previous turn in a proven positional + shape: either whole-message append or pure block append inside one message. + There must be exactly one previous forwarded message per original. Otherwise + the previous bytes may not correspond to the same positions, so we return + ``optimized_messages`` unchanged (accept a possible bust rather than forward + wrong content). This makes freezing byte-identical in BOTH proxy modes, so the only remaining difference between them is how large a mutable (still-compressible) tail each @@ -311,6 +490,55 @@ def overlay_cached_prefix( n, ) return optimized_messages + + relation = classify_history_relation(current_original_messages, prev_orig) + if relation.kind == RELATION_BLOCK_APPEND and relation.message_index is not None: + message_index = relation.message_index + if message_index < len(optimized_messages): + previous_message = prev_fwd[message_index] + previous_original_message = prev_orig[message_index] + current_message = optimized_messages[message_index] + previous_content = ( + previous_message.get("content") if isinstance(previous_message, dict) else None + ) + previous_original_content = ( + previous_original_message.get("content") + if isinstance(previous_original_message, dict) + else None + ) + current_content = ( + current_message.get("content") if isinstance(current_message, dict) else None + ) + split = ( + len(previous_original_content) + if isinstance(previous_original_content, list) + else -1 + ) + if ( + isinstance(previous_content, list) + and isinstance(previous_original_content, list) + and isinstance(current_content, list) + and len(previous_content) == split + and len(current_content) >= split + and _canonicalize_for_prefix_compare(current_content[:split]) + == _canonicalize_for_prefix_compare(previous_original_content) + ): + merged = copy.deepcopy(previous_message) + merged["content"] = copy.deepcopy(previous_content) + copy.deepcopy( + current_content[split:] + ) + logger.debug( + "overlay: replayed %d forwarded blocks and appended %d new blocks " + "inside message %d", + split, + len(current_content) - split, + message_index, + ) + return ( + list(prev_fwd[:message_index]) + + [merged] + + list(optimized_messages[message_index + 1 :]) + ) # Append-only guard on CONTENT ONLY, message-by-message. Replay the # previously-forwarded (cached, compressed) bytes for the longest LEADING # run of messages that is byte-for-byte (content-canonical) identical to @@ -359,8 +587,77 @@ def overlay_cached_prefix( return list(prev_fwd[:k]) + list(optimized_messages[k:]) +_STABLE_BOUNDARY_ENV = "HEADROOM_STABLE_BOUNDARY_BREAKPOINT" +_MIN_BLOCKS_FOR_RELOCATION = 20 + + +def _stable_boundary_enabled() -> bool: + return os.environ.get(_STABLE_BOUNDARY_ENV, "").strip().lower() not in ( + "0", + "false", + "no", + "off", + ) + + +def _breakpoint_index( + content: list[Any], + message: dict[str, Any], + message_index: int, + previous_forwarded_messages: list[dict[str, Any]] | None, +) -> int: + """Choose newest for appends, stable-prefix end for rewritten tails.""" + newest = len(content) - 1 + if ( + not previous_forwarded_messages + or not _stable_boundary_enabled() + or len(content) < _MIN_BLOCKS_FOR_RELOCATION + or message_index >= len(previous_forwarded_messages) + ): + return newest + previous = previous_forwarded_messages[message_index] + if not isinstance(previous, dict): + return newest + relation = classify_history_relation([message], [previous]) + if relation.kind != RELATION_BLOCK_REWRITE_TAIL: + return newest + logger.debug( + "cache breakpoint anchored to stable run %d/%d blocks in message %d " + "(previous=%d, stable_suffix=%d)", + relation.stable_prefix_blocks, + relation.current_block_count, + message_index, + relation.previous_block_count, + relation.stable_suffix_blocks, + ) + return relation.stable_prefix_blocks - 1 + + +def _client_marker_positions( + client_messages: list[dict[str, Any]], +) -> list[tuple[int, int, dict[str, Any]]]: + """(message index, block index, marker) for every CLIENT cache_control. + + Block-level, not one-per-message: clients mark multiple blocks within a + single long message (Claude Code does this on 1-2-message requests with a + large first message), and the ~20-block lookback applies within a message + just as it does across messages. Only block-style content carries markers. + """ + positions: list[tuple[int, int, dict[str, Any]]] = [] + for i, msg in enumerate(client_messages): + content = msg.get("content") if isinstance(msg, dict) else None + if not isinstance(content, list): + continue + for bi, b in enumerate(content): + if isinstance(b, dict) and isinstance(b.get("cache_control"), dict): + positions.append((i, bi, b["cache_control"])) + return positions + + def normalize_message_cache_control( messages: list[dict[str, Any]], + previous_forwarded_messages: list[dict[str, Any]] | None = None, + client_messages: list[dict[str, Any]] | None = None, ) -> list[dict[str, Any]]: """Own message-level cache_control placement so breakpoints stay bounded. @@ -370,22 +667,35 @@ def normalize_message_cache_control( hard-errors at **>4 cache_control blocks total** (system + tools + messages), so on a long conversation the accumulation eventually 400s. - Fix: strip EVERY message-level cache_control and re-place a **single** - ephemeral breakpoint on the last block of the last block-style message. One - breakpoint caches the whole message prefix up to it, and — because the - provider's cache key is message CONTENT, not marker presence (moving the - breakpoint forward is the documented client pattern and it hits) — stripping - and re-placing markers never busts. system/tools breakpoints live outside - ``messages`` and are left untouched (they still count toward the 4 limit, so - holding messages to one breakpoint leaves room for them). + Fix: strip EVERY message-level cache_control, then re-place markers at the + positions the CLIENT's current request marks (``client_messages``). The + client's positions are load-bearing, not redundant: Anthropic resolves each + breakpoint by walking back **at most ~20 content blocks** for a prior cache + entry, and agentic clients (Claude Code) keep a marker on the previous + turn's newest message precisely so the new turn's write can chain to the + old entry. Collapsing to a single newest-block marker breaks that chain + whenever one turn adds >20 blocks (typical for tool-heavy turns): the + lookback misses, the entire message history silently re-bills as cache + creation, and a marker anchored short of the final block leaves the tail + billing as fully uncached input. Mirroring the client's positions bounds + accumulation identically (the client manages its own 4-marker budget) while + preserving its read/write chaining. - Headroom owns WHERE the breakpoint goes; the client still owns WHAT it says: - the re-placed marker reuses the newest client marker verbatim, so an explicit - ``ttl`` (e.g. ``"1h"``) survives consolidation instead of silently - downgrading to the 5-minute default (#2375). + The provider's cache key is message CONTENT, not marker presence (moving + the breakpoint forward is the documented client pattern and it hits), so + stripping replay leftovers and re-placing markers never busts. system/tools + breakpoints live outside ``messages`` and are left untouched. - Only block-style (list) content can carry cache_control; string content is - left as-is. Returns the input unchanged when there is nothing to normalize. + Headroom owns WHICH BLOCK carries each marker; the client owns the message + positions and the marker values, so an explicit ``ttl`` (e.g. ``"1h"``) + survives per position instead of silently downgrading (#2375). The newest + position uses stable-run anchoring for proven rewritten tails; earlier + positions go on their message's last block. + + Without ``client_messages`` (or when the transformed list no longer aligns + with it), falls back to the legacy single-marker consolidation. Only + block-style (list) content can carry cache_control; string content is left + as-is. Returns the input unchanged when there is nothing to normalize. """ changed = False out: list[dict[str, Any]] = [] @@ -412,13 +722,75 @@ def normalize_message_cache_control( last_block_idx = i else: out.append(msg) - # Re-place exactly one breakpoint on the last block-style message. + + def _place( + target_idx: int, + marker: dict[str, Any], + *, + anchor: bool, + block_idx: int | None = None, + ) -> bool: + msg = out[target_idx] + content = msg.get("content") + if not isinstance(content, list) or not content: + return False + content = list(content) + if block_idx is not None and 0 <= block_idx < len(content): + # Transforms can shift block indices (e.g. a dropped thinking + # block); a slightly-off placement still lands on a stable block + # in the same message, which is harmless — markers are not part + # of the provider's cache key. + breakpoint_index = block_idx + elif anchor: + breakpoint_index = _breakpoint_index( + content, msg, target_idx, previous_forwarded_messages + ) + else: + breakpoint_index = len(content) - 1 + # Anthropic content blocks are dictionaries, but callers can still + # supply mixed list content. Fall back to the newest block rather than + # attempting ``**`` on a scalar stable-boundary element, and skip the + # message entirely when even that is not a dict. + if not isinstance(content[breakpoint_index], dict): + breakpoint_index = len(content) - 1 + if not isinstance(content[breakpoint_index], dict): + return False + content[breakpoint_index] = {**content[breakpoint_index], "cache_control": dict(marker)} + out[target_idx] = {**msg, "content": content} + return True + + # Preferred: mirror the client's marker positions 1:1, block-level. The + # transform pipeline preserves message count, so index alignment is the + # invariant; fall back to legacy consolidation if it ever does not hold, + # or when the client marked nothing (legacy still places one so the + # prefix caches). The newest client marker keeps stable-run anchoring + # when the client placed it on its message's final block (intent: "cache + # through the end"); an explicit mid-message marker is honored verbatim. + if client_messages is not None and len(client_messages) == len(messages): + positions = _client_marker_positions(client_messages) + if positions: + placed_any = False + newest_mi, newest_bi, _ = positions[-1] + newest_client_content = client_messages[newest_mi].get("content") + newest_on_final_block = ( + isinstance(newest_client_content, list) + and newest_bi == len(newest_client_content) - 1 + ) + for mi, bi, marker in positions: + is_newest = (mi, bi) == (newest_mi, newest_bi) + if is_newest and newest_on_final_block: + placed = _place(mi, marker, anchor=True) + else: + placed = _place(mi, marker, anchor=False, block_idx=bi) + placed_any = placed or placed_any + if placed_any or changed: + return out + return messages + + # Legacy: re-place exactly one breakpoint on the last block-style message. if last_block_idx >= 0: - msg = out[last_block_idx] - content = list(msg["content"]) marker = dict(last_marker) if last_marker else {"type": "ephemeral"} - content[-1] = {**content[-1], "cache_control": marker} - out[last_block_idx] = {**msg, "content": content} + _place(last_block_idx, marker, anchor=True) changed = True return out if changed else messages @@ -829,6 +1201,10 @@ class SessionTrackerStore: # value, so a synthetic key can never collide with a client-supplied # x-headroom-session-id. self._lineages: dict[str, OrderedDict[str, list[Any]]] = {} + # Exact non-message cache-key affinity per tracker. Anthropic renders + # tools before system/messages, so two sub-calls with identical history + # but different tool profiles must never share frozen-prefix state. + self._lineage_affinities: dict[str, str | None] = {} self._lineage_counter = itertools.count(1) def get_or_create(self, session_id: str, provider: str) -> PrefixCacheTracker: @@ -854,6 +1230,7 @@ class SessionTrackerStore: session_id: str, provider: str, messages: list[dict[str, Any]] | None = None, + cache_affinity: str | None = None, ) -> PrefixCacheTracker: """Resolve the tracker for THIS conversation within a session id (#2085). @@ -866,10 +1243,11 @@ class SessionTrackerStore: Lineage resolution keys trackers by conversation content instead: reuse the tracker whose previous request messages are a prefix of the - incoming history (client histories are append-only, so a - conversation's next request always extends its previous one); start a - fresh lineage when the history diverges or was rewritten (client-side - compaction — the provider cache line is gone then anyway). + incoming history. It also recognizes a conservative block-level shape + where a large leading run and two-block identity suffix survive while + the middle tail is regenerated; all other rewrites start a fresh + lineage. This keeps #2671's stable cache boundary attached without + merging unrelated parallel sub-calls. Byte-identical histories (templated fan-outs before they diverge) intentionally share a tracker: their provider cache line is identical too, so sharing is harmless. @@ -888,6 +1266,9 @@ class SessionTrackerStore: compares like against like across turns. ``None``/empty (legacy callers, stub stores in tests) falls back to plain :meth:`get_or_create`. + cache_affinity: Stable fingerprint of the provider's non-message + cache-key segments (model/tools/tool choice/thinking). Lineages + with different affinity never share a tracker. Returns: The ``PrefixCacheTracker`` for this conversation's lineage. @@ -918,14 +1299,48 @@ class SessionTrackerStore: family = self._lineages.setdefault(session_id, OrderedDict()) - # Longest recorded chain that prefixes the incoming history wins. + # Strict whole-message continuations win first, then pure block appends. + # Rewritten-tail matches are deliberately last and require a unique best + # structural score; ambiguity starts a fresh lineage instead of making + # sibling sub-calls ping-pong one tracker. + by_length = sorted(family.items(), key=lambda item: len(item[1]), reverse=True) best_key: str | None = None - best_len = -1 - for key, chain in family.items(): - if len(chain) > len(snap) or len(chain) <= best_len: - continue - if snap[: len(chain)] == chain: - best_key, best_len = key, len(chain) + for accepted in ( + (RELATION_EXACT, RELATION_MESSAGE_APPEND), + (RELATION_BLOCK_APPEND,), + ): + for key, chain in by_length: + if self._lineage_affinities.get(key) != cache_affinity: + continue + relation = _classify_history_canonical(snap, chain) + if relation.kind in accepted: + best_key = key + break + if best_key is not None: + break + + if best_key is None: + rewrite_candidates: list[tuple[tuple[int, int, int], str]] = [] + for key, chain in by_length: + if self._lineage_affinities.get(key) != cache_affinity: + continue + relation = _classify_history_canonical(snap, chain) + if relation.kind == RELATION_BLOCK_REWRITE_TAIL: + rewrite_candidates.append( + ( + ( + relation.stable_prefix_blocks, + relation.stable_suffix_blocks, + relation.previous_block_count, + ), + key, + ) + ) + rewrite_candidates.sort(reverse=True) + if rewrite_candidates and ( + len(rewrite_candidates) == 1 or rewrite_candidates[0][0] != rewrite_candidates[1][0] + ): + best_key = rewrite_candidates[0][1] if best_key is None: cap = self._default_config.max_lineages_per_session @@ -963,6 +1378,7 @@ class SessionTrackerStore: # the family before the stamp below. tracker = self.get_or_create(best_key, provider) family[best_key] = snap + self._lineage_affinities[best_key] = cache_affinity return tracker def compute_session_id( @@ -1031,6 +1447,7 @@ class SessionTrackerStore: family = self._lineages[base] for key in [k for k in family if k not in self._trackers]: del family[key] + self._lineage_affinities.pop(key, None) if not family: del self._lineages[base] logger.debug("SessionTrackerStore: cleaned up %d expired sessions", len(expired)) diff --git a/headroom/ccr/marker_resolution.py b/headroom/ccr/marker_resolution.py new file mode 100644 index 000000000..aef4a108f --- /dev/null +++ b/headroom/ccr/marker_resolution.py @@ -0,0 +1,83 @@ +"""Inline resolution of ``<>`` markers on the response path. + +Normal CCR resolution relies on the ``headroom_retrieve`` tool: a marker is +redeemed when the model calls the tool back. That path assumes there's a +subsequent turn in which the model *can* call it. Callers that never see an +injected tool at all — e.g. Headroom running as a LiteLLM guardrail/proxy hop +with no tool-call turn in between (#2509) — have no way to redeem a marker, +so it leaks through as raw text. + +This module provides an explicit, opt-in fallback (``--ccr-inline-resolve``): +scan the outgoing response for markers and substitute the original content +directly, instead of leaving the marker for the model to redeem later. +""" + +from __future__ import annotations + +import json +import logging +import re +from typing import Any + +from ..cache.compression_store import ( + CompressionStore, + format_retrieval_miss_detail, + get_compression_store, +) + +logger = logging.getLogger(__name__) + +# Matches the opaque-blob marker form `<>` (and the +# row-offload form `<>`) emitted by SmartCrusher. +# HASH is 12-24 hex chars; see headroom/ccr/tool_injection.py for the same +# constant used on the injection side. +_MARKER_RE = re.compile(r"<]*>>") + + +def resolve_markers_in_text(text: str, *, store: CompressionStore | None = None) -> str: + """Replace every ``<>`` marker in ``text`` with its original content. + + A miss (expired/evicted/unknown hash) can't be reported back to the + model on this path — there's no tool-call round-trip — so the marker is + left in place with the miss reason appended rather than raising. + """ + if "< str: + hash_key = match.group(1) + entry = resolved_store.retrieve(hash_key) + if entry is not None: + original = entry.original_content + return original if isinstance(original, str) else json.dumps(original) + + get_status = getattr(resolved_store, "get_entry_status", None) + status = get_status(hash_key, clean_expired=True) if callable(get_status) else None + detail = format_retrieval_miss_detail(status) if status else "entry not found" + logger.warning(f"CCR inline-resolve: marker {hash_key} unresolvable ({detail})") + return f"{match.group(0)} [unresolved: {detail}]" + + return _MARKER_RE.sub(_replace, text) + + +def resolve_markers_in_response(response: Any, *, store: CompressionStore | None = None) -> Any: + """Recursively resolve ``<>`` markers in every string field of a payload. + + Walks the full response structure rather than picking out + provider-specific fields (``content`` blocks, ``message.content``, + Responses-API ``output`` items, ...) so it stays correct regardless of + where a marker ends up, and doesn't need per-provider maintenance. + """ + resolved_store = store or get_compression_store() + if isinstance(response, str): + return resolve_markers_in_text(response, store=resolved_store) + if isinstance(response, list): + return [resolve_markers_in_response(item, store=resolved_store) for item in response] + if isinstance(response, dict): + return { + key: resolve_markers_in_response(value, store=resolved_store) + for key, value in response.items() + } + return response diff --git a/headroom/ccr/response_handler.py b/headroom/ccr/response_handler.py index ff93aa981..7cdea7a49 100644 --- a/headroom/ccr/response_handler.py +++ b/headroom/ccr/response_handler.py @@ -58,6 +58,7 @@ class CCRToolResult: content: str success: bool items_retrieved: int = 0 + tool_name: str | None = None @dataclass @@ -211,6 +212,7 @@ class CCRResponseHandler: tool_call_id=ccr_call.tool_call_id, content=content, success=False, + tool_name=ccr_call.tool_name, ) # Retrieval is by hash: always return the full original content. @@ -229,6 +231,7 @@ class CCRResponseHandler: content=content, success=True, items_retrieved=entry.original_item_count, + tool_name=ccr_call.tool_name, ) miss_status = ( @@ -249,6 +252,7 @@ class CCRResponseHandler: tool_call_id=ccr_call.tool_call_id, content=content, success=False, + tool_name=ccr_call.tool_name, ) except Exception as e: @@ -264,6 +268,7 @@ class CCRResponseHandler: tool_call_id=ccr_call.tool_call_id, content=content, success=False, + tool_name=ccr_call.tool_name, ) def _create_tool_result_message( @@ -337,14 +342,13 @@ class CCRResponseHandler: response_data = json.loads(result.content) except json.JSONDecodeError: response_data = {"content": result.content} - parts.append( - { - "functionResponse": { - "name": result.tool_call_id, # tool_call_id contains the function name for Google - "response": response_data, - } - } - ) + function_response = { + "name": result.tool_name or result.tool_call_id, + "response": response_data, + } + if result.tool_name and result.tool_call_id != result.tool_name: + function_response["id"] = result.tool_call_id + parts.append({"functionResponse": function_response}) return { "role": "user", "parts": parts, @@ -399,7 +403,16 @@ class CCRResponseHandler: # echoed back verbatim as `input[]` items — not a single # role/content dict like chat completions. Sentinel key mirrors # `_openai_tool_results`; handle_response() extends on it. - return {"_openai_responses_output_items": response.get("output", [])} + # `.get("output", [])` only falls back when the key is absent, so a + # present-but-null `output` would return None and make the + # `current_messages.extend(...)` in handle_response raise TypeError; + # coerce to a list like the choices branch above. + output_items = response.get("output") + return { + "_openai_responses_output_items": output_items + if isinstance(output_items, list) + else [] + } elif provider == "google": # Google/Gemini format: role is "model", content is in candidates[0].content.parts candidates = response.get("candidates", []) @@ -901,8 +914,17 @@ class StreamingCCRHandler: if "content" in delta and delta["content"]: message["content"] = (message.get("content") or "") + delta["content"] - if "tool_calls" in delta: - for tc_delta in delta["tool_calls"]: + # Guard the value, not just the key: some OpenAI-compatible + # providers include ``"tool_calls": null`` (and ``"function": null``) + # in a delta rather than omitting the key, which would make the + # iteration below raise ``TypeError: 'NoneType' object is not + # iterable`` and abort the whole reconstruction. Mirrors the + # ``and delta["content"]`` value-guard above. + tool_calls = delta.get("tool_calls") + if isinstance(tool_calls, list): + for tc_delta in tool_calls: + if not isinstance(tc_delta, dict): + continue idx = tc_delta.get("index", 0) if idx not in tool_calls_map: tool_calls_map[idx] = { @@ -914,8 +936,8 @@ class StreamingCCRHandler: tc = tool_calls_map[idx] if "id" in tc_delta: tc["id"] = tc_delta["id"] - if "function" in tc_delta: - fn = tc_delta["function"] + fn = tc_delta.get("function") + if isinstance(fn, dict): if "name" in fn: tc["function"]["name"] = fn["name"] if "arguments" in fn: diff --git a/headroom/ccr/tool_calls.py b/headroom/ccr/tool_calls.py index 46e5feacb..44d75ff34 100644 --- a/headroom/ccr/tool_calls.py +++ b/headroom/ccr/tool_calls.py @@ -14,6 +14,7 @@ class CCRToolCall: tool_call_id: str hash_key: str + tool_name: str | None = None def extract_tool_calls(response: dict[str, Any], provider: str) -> list[dict[str, Any]]: @@ -88,6 +89,8 @@ def tool_call_id_for_provider(tool_call: dict[str, Any], provider: str) -> str: if provider == "google": function_call = tool_call.get("functionCall", {}) if isinstance(function_call, dict): + if function_call.get("id"): + return str(function_call["id"]) name = function_call.get("name", CCR_TOOL_NAME) return str(name) return CCR_TOOL_NAME @@ -111,11 +114,14 @@ def parse_ccr_tool_calls( other_calls.append(tool_call) continue + tool_name = None + tool_call_id = tool_call_id_for_provider(tool_call, provider) + if provider == "google": + function_call = tool_call.get("functionCall", {}) + if isinstance(function_call, dict) and function_call.get("id"): + tool_name = str(function_call.get("name", CCR_TOOL_NAME)) ccr_calls.append( - CCRToolCall( - tool_call_id=tool_call_id_for_provider(tool_call, provider), - hash_key=hash_key, - ) + CCRToolCall(tool_call_id=tool_call_id, hash_key=hash_key, tool_name=tool_name) ) return ccr_calls, other_calls diff --git a/headroom/ccr/tool_injection.py b/headroom/ccr/tool_injection.py index 20337285d..86f6cac3f 100644 --- a/headroom/ccr/tool_injection.py +++ b/headroom/ccr/tool_injection.py @@ -16,12 +16,24 @@ from __future__ import annotations import json import re from dataclasses import dataclass, field -from typing import Any +from typing import Any, Protocol, runtime_checkable # Tool name constant - used for matching tool calls CCR_TOOL_NAME = "headroom_retrieve" +@runtime_checkable +class _HashOwnershipStore(Protocol): + """Structural type for verify_ownership()'s store dependency. + + Only needs the existence check — matches CompressionStore.exists() + without coupling this module to the concrete cache implementation + (or requiring test doubles to subclass it). + """ + + def exists(self, hash_key: str, clean_expired: bool = False) -> bool: ... + + def create_ccr_tool_definition( provider: str = "anthropic", ) -> dict[str, Any]: @@ -170,6 +182,11 @@ class CCRToolInjector: inject_tool: bool = True inject_system_instructions: bool = True retrieval_endpoint: str = "/v1/retrieve" + # Store used to verify a scanned marker's hash is actually ours before + # advertising it (issue #2836). None resolves lazily to + # get_compression_store() — request-scoped store if one is set, else the + # global singleton — matching how every other CCR call site resolves it. + compression_store: _HashOwnershipStore | None = None # Detected compression markers _detected_hashes: list[str] = field(default_factory=list) @@ -281,7 +298,16 @@ class CCRToolInjector: return self._detected_hashes def _scan_text(self, text: str) -> None: - """Scan text for compression markers from any compressor.""" + """Scan text for compression markers from any compressor. + + Shape-only: this matches the bracket format any compressor (or, + as it turns out, any *other* context tool) can produce. Callers + that need to know the hash is actually ours — i.e. before + advertising it to the model via the retrieve tool — must call + :meth:`verify_ownership` afterward. Kept separate so this method + stays a pure, store-independent text scan (that's what the + existing marker-format test suite exercises). + """ for pattern in self._marker_patterns: matches = pattern.findall(text) for match in matches: @@ -293,6 +319,59 @@ class CCRToolInjector: if hash_key and hash_key not in self._detected_hashes: self._detected_hashes.append(hash_key) + def verify_ownership(self, store: _HashOwnershipStore | None = None) -> list[str]: + """Drop any detected hash the compression store doesn't recognize. + + The bracket-marker shape (``[... hash=...]``) is not unique to + Headroom — other context tools emit visually identical markers. + Matching shape alone (what :meth:`scan_for_markers` does) adopts + their hashes too: ``has_compressed_content`` goes true and the + retrieve tool + "Available hashes" instruction get injected for a + hash this proxy never stored. The model then calls + ``headroom_retrieve``, gets a guaranteed miss, and re-does the work + it already had (issue #2836). + + Call this after :meth:`scan_for_markers` and before checking + ``has_compressed_content`` / injecting the tool. Uses the same + ``store.exists()`` check the retrieve endpoint itself performs, so + a hash that survives this filter is provably redeemable right now + (or, if it expires between this check and the model's next call, + fails the same way a genuinely-ours stale hash already would — + this only removes hashes that were never ours to begin with). + + Args: + store: Compression store to verify against. Defaults to + ``get_compression_store()`` (request-scoped if set, else + the global singleton) — the same resolution every other + CCR call site uses. + + Returns: + The filtered ``detected_hashes`` list (also updates + ``self.detected_hashes`` in place). + """ + if not self._detected_hashes: + return self._detected_hashes + if store is None: + store = self.compression_store + if store is None: + from headroom.cache.compression_store import get_compression_store + + store = get_compression_store() + + def _safe_exists(hash_key: str) -> bool: + try: + return store.exists(hash_key) + except Exception: + # A store lookup failure must not make CCR verification + # blow up the request; treat as "not ours" (drop the + # marker) — the safe direction, since a dropped real + # marker just means the model can't use the retrieve tool + # for it this turn, the same failure mode as CCR being off. + return False + + self._detected_hashes = [h for h in self._detected_hashes if _safe_exists(h)] + return self._detected_hashes + def inject_tool_definition( self, tools: list[dict[str, Any]] | None, @@ -437,6 +516,10 @@ class CCRToolInjector: tool_was_injected is False if tool was already present (e.g., from MCP). """ self.scan_for_markers(messages) + # Shape-only scanning also matches markers from other context tools; + # drop hashes this proxy never actually stored before they can + # drive tool injection (issue #2836). + self.verify_ownership() if not (self.has_compressed_content or session_has_done_ccr): return messages, tools, False diff --git a/headroom/cli/__init__.py b/headroom/cli/__init__.py index 529a20d65..5901456a0 100644 --- a/headroom/cli/__init__.py +++ b/headroom/cli/__init__.py @@ -25,6 +25,7 @@ from . import ( # noqa: F401 perf, proxy, recover, + rollout, tools, update, wrap, diff --git a/headroom/cli/doctor.py b/headroom/cli/doctor.py index f1f06b380..a45a0cbf1 100644 --- a/headroom/cli/doctor.py +++ b/headroom/cli/doctor.py @@ -30,6 +30,8 @@ from headroom.paths import savings_path from headroom.providers.claude import ( REMOTE_CONTROL_BASE_URL_ENV, REMOTE_CONTROL_SIBLING_GATE_NOTE, + claude_auth_conflict_message, + claude_auth_conflict_sources, detect_claude_code_version, is_custom_anthropic_base_url, remote_control_applies_to_auth, @@ -48,6 +50,13 @@ SKIP = "skip" _LOOPBACK_URL_RE = re.compile(r"https?://(?:127\.0\.0\.1|localhost):(\d+)") _CODEX_BASE_URL_RE = re.compile(r'base_url\s*=\s*"https?://(?:127\.0\.0\.1|localhost):(\d+)') +# Ollama's fixed default port. `ollama launch claude` writes +# ``ANTHROPIC_BASE_URL=http://127.0.0.1:11434`` into the launched Claude Code +# child, which outranks the persistent-install env block and silently bypasses +# the Headroom proxy (issue #2199). Recognized so the routing diagnostic names +# the collision instead of telling the user to re-probe port 11434. +_OLLAMA_DEFAULT_PORT = 11434 + @dataclass class CheckResult: @@ -182,6 +191,39 @@ def check_claude_routing(settings_path: Path, port: int) -> CheckResult: return _classify_routing_url(name, base_url, port, source=str(settings_path)) +def check_claude_auth_conflict( + settings_path: Path, + project_settings_path: Path, + project_local_settings_path: Path, + environ: Mapping[str, str], +) -> CheckResult | None: + """Report contradictory effective Claude credentials without their values.""" + + def settings_env(path: Path) -> dict[str, object]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return {} + env = payload.get("env") if isinstance(payload, dict) else None + return dict(env) if isinstance(env, dict) else {} + + conflict = claude_auth_conflict_sources( + (str(settings_path), settings_env(settings_path)), + (str(project_settings_path), settings_env(project_settings_path)), + (str(project_local_settings_path), settings_env(project_local_settings_path)), + ("shell environment", environ), + ) + if conflict is None: + return None + return CheckResult( + name="claude auth", + status=FAIL, + summary=claude_auth_conflict_message(conflict), + ) + + def check_claude_remote_control_gate( settings_path: Path, environ: Mapping[str, str], @@ -344,6 +386,22 @@ def _classify_routing_url(name: str, url: str, port: int, *, source: str) -> Che ) found_port = int(match.group(1)) if found_port != port: + if found_port == _OLLAMA_DEFAULT_PORT: + # Not a mis-probed Headroom port — this is Ollama's endpoint, so + # `headroom doctor --port 11434` would only chase a red herring. + return CheckResult( + name=name, + status=WARN, + summary=( + f"points at Ollama ({url}), not the Headroom proxy ({source}) — " + "`ollama launch claude` bypasses the persistent Headroom route" + ), + hint=( + "both claim ANTHROPIC_BASE_URL; run Ollama-backed sessions " + "through Headroom by chaining the proxy at its Ollama upstream " + "(see issue #2199)" + ), + ) return CheckResult( name=name, status=WARN, @@ -544,16 +602,26 @@ def doctor(port: int, emit_json: bool) -> None: stats = probe_json(f"{base_url}/stats", timeout=5.0) if livez else None installed = get_version() + project_claude_settings = Path.cwd() / ".claude" / "settings.json" + project_local_claude_settings = Path.cwd() / ".claude" / "settings.local.json" checks = [ check_proxy_liveness(livez, base_url), check_version_drift(livez, installed), check_claude_routing(claude_settings_path(), port), - check_wrap_marker_staleness(Path.cwd() / ".claude" / "settings.local.json"), + check_wrap_marker_staleness(project_local_claude_settings), check_codex_routing(codex_config_path(), port), check_shell_env(os.environ, port), check_savings(stats, savings_path()), check_budget(stats), ] + auth_conflict_check = check_claude_auth_conflict( + claude_settings_path(), + project_claude_settings, + project_local_claude_settings, + os.environ, + ) + if auth_conflict_check is not None: + checks.append(auth_conflict_check) # Lazy resolver: `claude --version` is a Node CLI subprocess (seconds of # cold start, 10s worst-case timeout) — only pay for it when the RC gate # is actually plausible (custom base URL + subscription auth). diff --git a/headroom/cli/init.py b/headroom/cli/init.py index 8a153ee22..2b6f4bf4c 100644 --- a/headroom/cli/init.py +++ b/headroom/cli/init.py @@ -41,6 +41,7 @@ from headroom.install.runtime import ( from headroom.install.state import ManifestError, load_manifest, save_manifest from headroom.install.supervisors import start_supervisor from headroom.providers.claude import TOOL_SEARCH_DEFAULT, TOOL_SEARCH_ENV +from headroom.providers.claude.runtime import TOOL_SEARCH_FOUNDRY_DEFAULT from headroom.providers.codex.install import codex_uses_chatgpt_auth from headroom.providers.codex.threads import retag_to_headroom @@ -181,7 +182,12 @@ def _ensure_claude_hooks(path: Path, profile: str, port: int) -> None: # all into its context window — overflowing it (breaks sub-agent spawns, # forces constant compaction). Keep deferral on; respect a user-set value. # Shares the TOOL_SEARCH_* constants with `wrap` and `install`. - env_map.setdefault(TOOL_SEARCH_ENV, TOOL_SEARCH_DEFAULT) + tool_search_default = ( + TOOL_SEARCH_FOUNDRY_DEFAULT + if os.environ.get("CLAUDE_CODE_USE_FOUNDRY") + else TOOL_SEARCH_DEFAULT + ) + env_map.setdefault(TOOL_SEARCH_ENV, tool_search_default) payload["env"] = env_map hooks = dict(payload.get("hooks") or {}) if isinstance(payload.get("hooks"), dict) else {} diff --git a/headroom/cli/install.py b/headroom/cli/install.py index bf6b79991..50e897ad4 100644 --- a/headroom/cli/install.py +++ b/headroom/cli/install.py @@ -11,6 +11,7 @@ from copy import deepcopy from dataclasses import dataclass import click +from click.core import ParameterSource from headroom._subprocess import run from headroom.install.health import probe_json, probe_ready @@ -36,6 +37,7 @@ from headroom.install.runtime import ( from headroom.install.state import ( ManifestError, delete_manifest, + list_manifests, load_manifest, save_manifest, ) @@ -65,14 +67,82 @@ def install() -> None: """Install and manage persistent Headroom deployments.""" +def _profile_selection_was_explicit() -> bool: + """True when the current command received an explicit ``--profile``. + + An explicit selection must be honored verbatim or rejected, never redirected + to ``HEADROOM_DEPLOYMENT_PROFILE`` or a lone installed deployment: silently + operating ``stop``/``restart``/``remove`` on a different profile than the one + the user typed is dangerous. Only a defaulted (omitted) ``--profile`` is + eligible for the recovery fallback. Outside a Click command context (direct + calls / unit tests) there is no explicit selection to protect. + """ + ctx = click.get_current_context(silent=True) + if ctx is None: + return False + return bool(ctx.get_parameter_source("profile") == ParameterSource.COMMANDLINE) + + +def _missing_profile_error( + name: str, + installed: list[DeploymentManifest], + *, + source: str | None = None, +) -> click.ClickException: + if installed: + names = ", ".join(sorted(m.profile for m in installed)) + hint = f" Installed: {names}. Select one with --profile ." + else: + hint = " No deployments are installed; run `headroom init` or `headroom install apply`." + origin = f" (from {source})" if source else "" + return click.ClickException(f"No deployment profile named '{name}'{origin} is installed.{hint}") + + def _require_manifest(profile: str) -> DeploymentManifest: try: manifest = load_manifest(profile) except ManifestError as e: raise click.ClickException(str(e)) from None - if manifest is None: - raise click.ClickException(f"No deployment profile named '{profile}' is installed.") - return manifest + if manifest is not None: + return manifest + + # The requested profile isn't installed. `headroom init` installs under a + # non-"default" profile name (e.g. init-user), while every lifecycle command + # defaults --profile to "default" -- so on an init'd machine the documented + # bare commands (`headroom install status`, etc.) would all dead-end (#2811). + installed = list_manifests() + + # An EXPLICIT --profile is honored or rejected verbatim, never redirected: a + # typo must not silently act on the env/lone profile (#2832 review). + if _profile_selection_was_explicit(): + raise _missing_profile_error(profile, installed) + + # --profile was defaulted. A non-empty HEADROOM_DEPLOYMENT_PROFILE (which the + # runtime exports) is itself an explicit selection: honor it when installed, + # otherwise fail naming it. It must never fall through to the lone-manifest + # fallback and silently operate on a different deployment (#2832 review). + env_profile = os.environ.get("HEADROOM_DEPLOYMENT_PROFILE", "").strip() + if env_profile: + if env_profile != profile: + try: + resolved = load_manifest(env_profile) + except ManifestError: + resolved = None + if resolved is not None: + return resolved + raise _missing_profile_error(env_profile, installed, source="HEADROOM_DEPLOYMENT_PROFILE") + + # Neither CLI nor environment named a profile. A single installed deployment + # is unambiguous, so use it; otherwise report what is available. + if len(installed) == 1: + return installed[0] + raise _missing_profile_error(profile, installed) + + +def _is_windows() -> bool: + """Return whether this command is running on Windows.""" + + return sys.platform.startswith("win") def _start_deployment(manifest: DeploymentManifest, *, assume_start_lock: bool = False) -> None: @@ -495,7 +565,8 @@ def _echo_installed(manifest: DeploymentManifest, *, prefix: str = "Installed pe is_flag=True, help=( "Opt in to tool_result interceptors (ast-grep Read outliner, etc.) in the " - "persistent runtime. Off by default while this feature ships." + "persistent runtime. This also selects the required canary rollout channel " + "unless --env HEADROOM_ROLLOUT_CHANNEL=... is supplied." ), ) @click.option( @@ -593,6 +664,16 @@ def install_apply( bedrock_profile=bedrock_profile, extra_env=combined_env, ) + if ( + preset == InstallPreset.PERSISTENT_SERVICE.value + and manifest.preset == InstallPreset.PERSISTENT_TASK.value + and _is_windows() + ): + click.echo( + "Warning: persistent-service is not supported on Windows because the " + "Python runner cannot act as a Windows service. Falling back to " + "persistent-task with Task Scheduler." + ) _apply_manifest(manifest) _echo_installed(manifest) diff --git a/headroom/cli/learn.py b/headroom/cli/learn.py index bcbf51466..b17a339ec 100644 --- a/headroom/cli/learn.py +++ b/headroom/cli/learn.py @@ -400,8 +400,9 @@ def _activate_output_shaper(port: int | None = None) -> tuple[str, int]: When a proxy is already running locally we hot-enable it via ``/admin/runtime-env`` (no restart, the same channel ``wrap`` uses), so ``--apply`` actually takes effect. Returns ``(status, port)`` where status is - ``"live"`` (enabled on a running proxy), ``"absent"`` (no reachable proxy), - or ``"error"``. + ``"live"`` (enabled on a running proxy), ``"blocked"`` (the proxy's + rollout channel rejected it), ``"absent"`` (no reachable proxy), or + ``"error"``. """ import json as _json import os as _os @@ -417,7 +418,22 @@ def _activate_output_shaper(port: int | None = None) -> tuple[str, int]: ) try: with urllib.request.urlopen(request, timeout=2) as response: - response.read() + raw_response = response.read() + payload = _json.loads(raw_response) if raw_response else {} + rollout = payload.get("rollout") if isinstance(payload, dict) else None + if isinstance(rollout, dict): + decisions = rollout.get("features") + if isinstance(decisions, list): + output_shaper = next( + ( + item + for item in decisions + if isinstance(item, dict) and item.get("name") == "proxy_output_shaper" + ), + None, + ) + if isinstance(output_shaper, dict) and not output_shaper.get("enabled", False): + return "blocked", resolved_port return "live", resolved_port except (urllib.error.URLError, OSError): # ConnectionRefused (no proxy) or 404 (proxy predates the endpoint). @@ -555,8 +571,17 @@ def _run_verbosity( f"level {best_profile.level} is live now (while HEADROOM_VERBOSITY_LEVEL is unset)." ) click.echo( - " To keep it on across restarts: export HEADROOM_OUTPUT_SHAPER=1 " - "before `headroom wrap ...` (wrap pushes it to the proxy)." + " To keep it on across restarts: export HEADROOM_ROLLOUT_CHANNEL=beta " + "and HEADROOM_OUTPUT_SHAPER=1 before `headroom wrap ...`." + ) + elif status == "blocked": + click.echo( + "\n ⚠ Level written, but the running proxy's rollout channel blocks the " + "beta output shaper." + ) + click.echo( + " Restart it with HEADROOM_ROLLOUT_CHANNEL=beta and " + "HEADROOM_OUTPUT_SHAPER=1; the learned level will be used automatically." ) else: click.echo( @@ -564,9 +589,10 @@ def _run_verbosity( "NOT shaping output yet." ) click.echo( - " Enable it: export HEADROOM_OUTPUT_SHAPER=1 then `headroom wrap ...` " - "(or start `headroom proxy` with it set). The learned level is then used " - "automatically while HEADROOM_VERBOSITY_LEVEL is unset." + " Enable it: export HEADROOM_ROLLOUT_CHANNEL=beta and " + "HEADROOM_OUTPUT_SHAPER=1, then run `headroom wrap ...` (or restart " + "`headroom proxy`). The learned level is then used automatically while " + "HEADROOM_VERBOSITY_LEVEL is unset." ) else: click.echo("\n Dry run — use --apply to persist the level and baseline.") diff --git a/headroom/cli/main.py b/headroom/cli/main.py index 1b74e22df..74fdaad8a 100644 --- a/headroom/cli/main.py +++ b/headroom/cli/main.py @@ -76,6 +76,7 @@ def _register_commands() -> None: perf, # noqa: F401 proxy, # noqa: F401 recover, # noqa: F401 + rollout, # noqa: F401 savings, # noqa: F401 tools, # noqa: F401 update, # noqa: F401 diff --git a/headroom/cli/memory.py b/headroom/cli/memory.py index 51a1fd1f9..9a930233a 100644 --- a/headroom/cli/memory.py +++ b/headroom/cli/memory.py @@ -30,6 +30,8 @@ from ._utils.formatting import ( from ._utils.parsers import parse_duration from .main import main +_REINDEX_PAGE_SIZE = 1_000 + def _default_db_path() -> str: """Resolve the memory DB the proxy/install actually use. @@ -66,6 +68,148 @@ def get_store(db_path: str) -> SQLiteMemoryStore: return SQLiteMemoryStore(db_path) +def _sqlite_table_exists(conn: Any, table_name: str) -> bool: + """Return whether a SQLite table or virtual table has been initialized.""" + row = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", + (table_name,), + ).fetchone() + return row is not None + + +def _remove_from_search_indexes(db_path: str, memory_ids: list[str]) -> bool: + """Remove specific memories from FTS5 and vector search indexes. + + FTS5 cleanup uses a bare sqlite3 connection (FTS5 is built-in). + Vector cleanup requires sqlite-vec to load the vec0 virtual-table + module; when not installed a warning is printed. + + Returns True when both indexes were fully synced, False when any part + of the sync failed. Callers must treat False as a partial failure and + surface it — typically by exiting with a non-zero code so the primary + store mutation is not silently reported as fully successful. + """ + if not memory_ids: + return True + + import sqlite3 + + db = Path(db_path) + ok = True + + # FTS5 table lives in the same memory.db file (built-in, no extension needed). + try: + with sqlite3.connect(str(db)) as conn: + if _sqlite_table_exists(conn, "memory_fts"): + for i in range(0, len(memory_ids), 500): + chunk = memory_ids[i : i + 500] + placeholders = ",".join("?" * len(chunk)) + conn.execute( + f"DELETE FROM memory_fts WHERE memory_id IN ({placeholders})", + chunk, + ) + conn.commit() + except Exception as exc: + print_warning(f"FTS5 index cleanup incomplete: {exc}") + ok = False + + # Vector DB is a sibling file: memory.db -> memory_vectors.db. + # vec_embeddings is a vec0 virtual table — the sqlite-vec extension must be + # loaded on every connection before touching it. + vector_db = db.parent / f"{db.stem}_vectors.db" + if not vector_db.exists(): + return ok + + try: + with sqlite3.connect(str(vector_db)) as conn: + # A sibling database may exist before the optional vector index has + # ever been initialized. That is a valid no-op, not a sync failure. + if not _sqlite_table_exists(conn, "vec_metadata"): + return ok + + try: + import sqlite_vec + except ImportError: + print_warning( + "sqlite-vec is not installed; stale vector index entries may remain. " + "Run 'headroom memory reindex' after installing sqlite-vec to repair." + ) + return False + + conn.enable_load_extension(True) + sqlite_vec.load(conn) + conn.enable_load_extension(False) + for i in range(0, len(memory_ids), 500): + chunk = memory_ids[i : i + 500] + placeholders = ",".join("?" * len(chunk)) + rows = conn.execute( + f"SELECT rowid FROM vec_metadata WHERE memory_id IN ({placeholders})", + chunk, + ).fetchall() + rowids = [r[0] for r in rows] + if rowids: + rph = ",".join("?" * len(rowids)) + conn.execute(f"DELETE FROM vec_embeddings WHERE rowid IN ({rph})", rowids) + conn.execute(f"DELETE FROM vec_metadata WHERE rowid IN ({rph})", rowids) + conn.commit() + except Exception as exc: + print_warning(f"Vector index cleanup incomplete: {exc}") + ok = False + + return ok + + +def _clear_all_search_indexes(db_path: str) -> bool: + """Truncate both search indexes after a full purge. + + Same extension-loading requirement as :func:`_remove_from_search_indexes`. + Returns True on full success, False on any partial failure. + """ + import sqlite3 + + db = Path(db_path) + ok = True + + try: + with sqlite3.connect(str(db)) as conn: + if _sqlite_table_exists(conn, "memory_fts"): + conn.execute("DELETE FROM memory_fts") + conn.commit() + except Exception as exc: + print_warning(f"FTS5 index cleanup incomplete: {exc}") + ok = False + + vector_db = db.parent / f"{db.stem}_vectors.db" + if not vector_db.exists(): + return ok + + try: + with sqlite3.connect(str(vector_db)) as conn: + if not _sqlite_table_exists(conn, "vec_metadata"): + return ok + + try: + import sqlite_vec + except ImportError: + print_warning( + "sqlite-vec is not installed; stale vector index entries may remain. " + "Run 'headroom memory reindex' after installing sqlite-vec to repair." + ) + return False + + conn.enable_load_extension(True) + sqlite_vec.load(conn) + conn.enable_load_extension(False) + conn.execute("DELETE FROM vec_embeddings") + conn.execute("DELETE FROM vec_metadata") + conn.commit() + except Exception as exc: + print_warning(f"Vector index cleanup incomplete: {exc}") + ok = False + + return ok + + def _resolve_memory(store: SQLiteMemoryStore, memory_id: str) -> Memory: """Resolve an exact or unambiguous partial memory ID.""" memory = asyncio.run(store.get(memory_id)) @@ -620,13 +764,37 @@ def edit_memory( mem = matches[0] # Update fields + content_changed = content is not None and content != mem.content if content is not None: mem.content = content if importance is not None: mem.importance = importance + index_ok = True + if content_changed: + # Clear the stale embedding so the memory MCP server re-embeds on + # next startup. Also remove the old FTS5 and vector index entries + # now to avoid serving stale search results until then. + mem.embedding = None + index_ok = _remove_from_search_indexes(db_path, [mem.id]) + # Re-index FTS5 immediately with new content (no embedder needed). + try: + from ..memory.adapters.fts5 import FTS5TextIndex + + fts = FTS5TextIndex(db_path=db_path) + asyncio.run(fts.index_memory(mem)) + except Exception as exc: + print_warning(f"FTS5 re-index incomplete: {exc}") + index_ok = False + # Save asyncio.run(store.save(mem)) + if not index_ok: + print_warning( + f"Updated memory {mem.id[:8]}, but search index sync incomplete. " + "Run 'headroom memory reindex' to repair." + ) + sys.exit(1) print_success(f"Updated memory {mem.id[:8]}") except Exception as e: @@ -775,7 +943,14 @@ def delete_memories( # Delete deleted = asyncio.run(store.delete_batch(resolved_ids)) - print_success(f"Deleted {deleted} memory(ies).") + if _remove_from_search_indexes(db_path, resolved_ids): + print_success(f"Deleted {deleted} memory(ies).") + else: + print_warning( + f"Deleted {deleted} memory(ies) from store, but search index sync " + "incomplete. Run 'headroom memory reindex' to repair." + ) + sys.exit(1) except click.Abort: click.echo("Aborted.") @@ -897,7 +1072,14 @@ def prune_memories( # Delete ids_to_delete = [m.id for m in memories] deleted = asyncio.run(store.delete_batch(ids_to_delete)) - print_success(f"Deleted {deleted} memory(ies).") + if _remove_from_search_indexes(db_path, ids_to_delete): + print_success(f"Deleted {deleted} memory(ies).") + else: + print_warning( + f"Deleted {deleted} memory(ies) from store, but search index sync " + "incomplete. Run 'headroom memory reindex' to repair." + ) + sys.exit(1) except click.BadParameter as e: print_error(str(e)) @@ -952,7 +1134,14 @@ def purge_memories(ctx: click.Context, db_path: str, confirm_flag: bool) -> None # Purge deleted = asyncio.run(store.clear_all()) - print_success(f"Purged {deleted} memory(ies).") + if _clear_all_search_indexes(db_path): + print_success(f"Purged {deleted} memory(ies).") + else: + print_warning( + f"Purged {deleted} memory(ies) from store, but search index sync " + "incomplete. Run 'headroom memory reindex' to repair." + ) + sys.exit(1) except click.Abort: click.echo("Aborted.") @@ -962,6 +1151,143 @@ def purge_memories(ctx: click.Context, db_path: str, confirm_flag: bool) -> None sys.exit(1) +@memory.command("reindex") +@db_path_option +@click.pass_context +def reindex_memories(ctx: click.Context, db_path: str) -> None: + """Rebuild FTS5 search index and remove orphaned vector entries. + + Use this to repair an inconsistent index after a failed delete, prune, + or purge. Run it after installing sqlite-vec to clean up any vector + entries that could not be removed earlier. + + Vector embeddings are not regenerated by this command — they are rebuilt + automatically when the Headroom server next starts. + + \b + Example: + headroom memory reindex + """ + import sqlite3 + + store = get_store(db_path) + + try: + # Page through the complete active store. A fixed cap is destructive: + # clearing FTS and rebuilding only the first N rows drops valid search + # coverage, while using the same truncated ID set for vector cleanup + # misclassifies later primary rows as orphans. + memories: list[Memory] = [] + offset = 0 + while True: + page = asyncio.run( + store.query( + MemoryFilter( + limit=_REINDEX_PAGE_SIZE, + offset=offset, + order_by="created_at", + order_desc=False, + ) + ) + ) + if not page: + break + memories.extend(page) + offset += len(page) + + db = Path(db_path) + ok = True + + # --- FTS5: wipe and rebuild from primary store --- + from ..memory.adapters.fts5 import FTS5TextIndex + + # Construction initializes an absent optional FTS table. Cleanup + # helpers, by contrast, intentionally treat an absent table as a no-op. + fts = FTS5TextIndex(db_path=db_path) + try: + with sqlite3.connect(str(db)) as conn: + conn.execute("DELETE FROM memory_fts") + conn.commit() + except Exception as exc: + print_error(f"Failed to clear FTS5 index: {exc}") + sys.exit(1) + + fts_indexed = 0 + for mem in memories: + try: + asyncio.run(fts.index_memory(mem)) + fts_indexed += 1 + except Exception as exc: + print_warning(f"FTS5: failed to index {mem.id[:8]}: {exc}") + ok = False + + # --- Vector: remove orphaned entries (requires sqlite-vec) --- + vector_db = db.parent / f"{db.stem}_vectors.db" + vector_msg = "" + if vector_db.exists(): + # Orphan detection is based on every primary row, including + # superseded memories that are intentionally omitted from FTS. + with store._get_conn() as conn: + primary_ids = {row[0] for row in conn.execute("SELECT id FROM memories")} + try: + with sqlite3.connect(str(vector_db)) as conn: + if not _sqlite_table_exists(conn, "vec_metadata"): + vector_msg = ", vector index not initialized" + else: + import sqlite_vec + + conn.enable_load_extension(True) + sqlite_vec.load(conn) + conn.enable_load_extension(False) + rows = conn.execute("SELECT memory_id FROM vec_metadata").fetchall() + orphan_ids = [r[0] for r in rows if r[0] not in primary_ids] + if orphan_ids: + for i in range(0, len(orphan_ids), 500): + chunk = orphan_ids[i : i + 500] + ph = ",".join("?" * len(chunk)) + vec_rows = conn.execute( + f"SELECT rowid FROM vec_metadata WHERE memory_id IN ({ph})", + chunk, + ).fetchall() + rowids = [r[0] for r in vec_rows] + if rowids: + rph = ",".join("?" * len(rowids)) + conn.execute( + f"DELETE FROM vec_embeddings WHERE rowid IN ({rph})", + rowids, + ) + conn.execute( + f"DELETE FROM vec_metadata WHERE rowid IN ({rph})", + rowids, + ) + conn.commit() + vector_msg = ( + f", removed {len(orphan_ids)} orphaned vector entry(ies)" + if orphan_ids + else ", vector index clean" + ) + except ImportError: + vector_msg = ( + " (vector index skipped: sqlite-vec not installed — " + "install with: pip install sqlite-vec)" + ) + ok = False + except Exception as exc: + vector_msg = f" (vector index cleanup failed: {exc})" + ok = False + + msg = f"Re-indexed {fts_indexed}/{len(memories)} memories{vector_msg}." + if ok: + print_success(msg) + else: + print_warning(msg) + sys.exit(1) + + except Exception as e: + print_error(f"Failed to reindex: {e}") + sys.exit(1) + + @memory.command("export") @db_path_option @click.option( diff --git a/headroom/cli/output_savings.py b/headroom/cli/output_savings.py index a341b2eea..c3598fc3b 100644 --- a/headroom/cli/output_savings.py +++ b/headroom/cli/output_savings.py @@ -28,7 +28,10 @@ def output_savings() -> None: if not path.exists(): click.echo("No output-savings data yet.") click.echo("Run `headroom learn --verbosity --apply` to seed the baseline,") - click.echo("then enable the shaper (HEADROOM_OUTPUT_SHAPER=1) and send traffic.") + click.echo( + "then enable the beta shaper (HEADROOM_ROLLOUT_CHANNEL=beta " + "HEADROOM_OUTPUT_SHAPER=1) and send traffic." + ) return ledger = SavingsLedger.load(path) diff --git a/headroom/cli/perf.py b/headroom/cli/perf.py index 5c512a073..f3bc65112 100644 --- a/headroom/cli/perf.py +++ b/headroom/cli/perf.py @@ -79,6 +79,8 @@ def perf(hours: float, raw: bool, output_format: str) -> None: "tokens_before", "tokens_after", "tokens_saved", + "message_tokens_saved", + "tool_tokens_saved", "savings_pct", "list_price_per_mtok", ] diff --git a/headroom/cli/proxy.py b/headroom/cli/proxy.py index 30292a4e1..73e2aee8a 100644 --- a/headroom/cli/proxy.py +++ b/headroom/cli/proxy.py @@ -4,6 +4,7 @@ import logging import os import sys import warnings +from importlib import import_module from typing import Any, Literal, cast import click @@ -18,6 +19,38 @@ from headroom.proxy.modes import PROXY_MODE_CACHE, normalize_proxy_mode from .main import main + +def ensure_proxy_dependencies() -> None: + """Verify optional proxy extras are installed before starting or wrapping.""" + required_modules: list[str] = [ + "fastapi", + "uvicorn", + "httpx", + "openai", + "mcp", + "magika", + "zstandard", + "websockets", + "onnxruntime", + "transformers", + "watchdog", + ] + if sys.implementation.name != "pypy": + required_modules.append("orjson") + + try: + for module in required_modules: + import_module(module) + except ImportError as e: + click.secho( + "Error: Proxy dependencies not installed. Run: pip install headroom-ai[proxy]", + fg="red", + err=True, + ) + click.secho(f"Details: {e}", fg="red", err=True) + raise SystemExit(1) from None + + # --------------------------------------------------------------------------- # Startup log suppression. # @@ -258,7 +291,7 @@ def dashboard(port: int, no_open: bool) -> None: is_flag=True, help=( "Opt in to tool_result interceptors (ast-grep Read outliner, etc.). " - "Off by default while this feature ships." + "Requires HEADROOM_ROLLOUT_CHANNEL=canary (or dev)." ), ) @click.option("--no-optimize", is_flag=True, help="Disable optimization (passthrough mode)") @@ -313,6 +346,19 @@ def dashboard(port: int, no_open: bool) -> None: "retrieval marker, so no MCP retrieve tool is needed. Env: HEADROOM_LOSSLESS=1." ), ) +@click.option( + "--ccr-inline-resolve", + is_flag=True, + envvar="HEADROOM_CCR_INLINE_RESOLVE", + help=( + "Resolve <> markers inline on the response path instead of " + "relying on the model to call headroom_retrieve. For callers with no " + "tool-call round-trip to redeem a marker (e.g. Headroom running as a " + "LiteLLM guardrail/proxy hop, see issue #2509). Applies to non-streaming " + "responses only. Off by default. " + "Env: HEADROOM_CCR_INLINE_RESOLVE." + ), +) @click.option( "--no-ccr-proactive-expansion", is_flag=True, @@ -628,7 +674,8 @@ def dashboard(port: int, no_open: bool) -> None: help=( "EXPERIMENTAL: activity-based read maturation — hold fresh Reads " "out of the provider prefix cache and compress them once their " - "file quiesces (env: HEADROOM_READ_MATURATION=1)" + "file quiesces. Requires HEADROOM_ROLLOUT_CHANNEL=beta (or dev); " + "env: HEADROOM_READ_MATURATION=1." ), ) @click.option( @@ -934,6 +981,7 @@ def proxy( tpm: int | None, no_ccr: bool, lossless: bool, + ccr_inline_resolve: bool, no_ccr_proactive_expansion: bool, proxy_extension: tuple[str, ...], compressor: tuple[str, ...], @@ -1017,23 +1065,16 @@ def proxy( Usage with OpenAI-compatible clients: OPENAI_BASE_URL=http://localhost:8787/v1 your-app """ + ensure_proxy_dependencies() + # Import here to avoid slow startup - try: - from headroom.proxy.server import ( - ProxyConfig, - _parse_csv_tools, - _parse_exclude_tools, - _parse_tool_profiles, - run_server, - ) - except ImportError as e: - click.secho( - "Error: Proxy dependencies not installed. Run: pip install headroom-ai[proxy]", - fg="red", - err=True, - ) - click.secho(f"Details: {e}", fg="red", err=True) - raise SystemExit(1) from None + from headroom.proxy.server import ( + ProxyConfig, + _parse_csv_tools, + _parse_exclude_tools, + _parse_tool_profiles, + run_server, + ) # Warn if --learn and --no-learn are both set (--no-learn wins, per docstring) if learn and no_learn: @@ -1066,12 +1107,46 @@ def proxy( err=True, ) + # Resolve rollout inputs once before constructing any rollout-managed + # behavior. The immutable snapshot is injected into ProxyConfig and is also + # what /stats later exposes. + from headroom.rollout import resolve_rollout + + rollout_requests = [] + if intercept_tool_results: + rollout_requests.append("tool_result_interceptors") + if read_maturation: + rollout_requests.append("read_maturation") + rollout_snapshot = resolve_rollout(os.environ, requested=rollout_requests) + + if read_maturation and not rollout_snapshot.is_enabled("read_maturation"): + click.secho( + "error: --read-maturation is not available in the current rollout channel " + f"({rollout_snapshot.channel.value}). Set HEADROOM_ROLLOUT_CHANNEL=beta " + "(or dev), or use HEADROOM_UNSAFE_ALLOW_UNSTABLE_FEATURES=1 for an " + "emergency override.", + fg="red", + err=True, + ) + sys.exit(1) + # Opt-in: turn on tool_result interceptors (ast-grep Read outline, etc.). # Only fetch the bundled CLI tool binaries when the feature is enabled — # otherwise we'd pay a network round-trip and risk a readonly-FS failure # for capabilities the user hasn't asked for. The TransformPipeline reads - # this env var at construction time. + # the resolved snapshot says it is active. if intercept_tool_results: + if not rollout_snapshot.is_enabled("tool_result_interceptors"): + click.secho( + "error: --intercept-tool-results is not available in the current " + f"rollout channel ({rollout_snapshot.channel.value}). Set " + "HEADROOM_ROLLOUT_CHANNEL=canary to dogfood it, or use " + "HEADROOM_UNSAFE_ALLOW_UNSTABLE_FEATURES=1 for emergency override.", + fg="red", + err=True, + ) + sys.exit(1) + from headroom.binaries import ensure_tools resolved_tools = ensure_tools() @@ -1089,7 +1164,6 @@ def proxy( err=True, ) sys.exit(1) - os.environ["HEADROOM_INTERCEPT_ENABLED"] = "1" try: resolved_anthropic_extra_headers = resolve_extra_headers( @@ -1171,6 +1245,7 @@ def proxy( config = ProxyConfig( host=host, port=port, + rollout=rollout_snapshot, anthropic_api_url=provider_api_overrides.anthropic, anthropic_extra_headers=resolved_anthropic_extra_headers, openai_extra_headers=resolved_openai_extra_headers, @@ -1206,6 +1281,7 @@ def proxy( # CCR fully on. ccr_inject_tool=not no_ccr, ccr_inject_marker=not no_ccr, + ccr_resolve_markers_inline=ccr_inline_resolve, lossless=lossless, ccr_proactive_expansion=not no_ccr_proactive_expansion, # Flatten repeat-flag tuple AND any comma-separated values inside it. @@ -1276,7 +1352,7 @@ def proxy( # Read lifecycle: ON by default (use --no-read-lifecycle to disable) read_lifecycle=not no_read_lifecycle, # Read maturation (Mechanism B): experimental, OFF by default - read_maturation=read_maturation, + read_maturation=rollout_snapshot.is_enabled("read_maturation"), read_maturation_quiesce_turns=read_maturation_quiesce_turns, read_maturation_max_hold_turns=read_maturation_max_hold_turns, read_maturation_min_size_bytes=read_maturation_min_size_bytes, diff --git a/headroom/cli/rollout.py b/headroom/cli/rollout.py new file mode 100644 index 000000000..fce624174 --- /dev/null +++ b/headroom/cli/rollout.py @@ -0,0 +1,66 @@ +"""Runtime rollout diagnostics commands.""" + +from __future__ import annotations + +import json +import os + +import click + +from headroom.rollout import RolloutConfigurationError, resolve_rollout + +from .main import main + + +@main.group("rollout") +def rollout_group() -> None: + """Inspect runtime feature-rollout policy (not package releases).""" + + +@rollout_group.command("status") +@click.option("--channel", envvar="HEADROOM_ROLLOUT_CHANNEL") +@click.option("--features", envvar="HEADROOM_FEATURES") +@click.option("--disable-features", envvar="HEADROOM_DISABLE_FEATURES") +@click.option( + "--unsafe-allow-unstable-features", + is_flag=True, + envvar="HEADROOM_UNSAFE_ALLOW_UNSTABLE_FEATURES", +) +@click.option("--json", "json_output", is_flag=True, help="Emit the versioned JSON snapshot.") +def rollout_status( + channel: str | None, + features: str | None, + disable_features: str | None, + unsafe_allow_unstable_features: bool, + json_output: bool, +) -> None: + """Resolve and print the supplied runtime rollout configuration.""" + + env = dict(os.environ) + if channel is not None: + env["HEADROOM_ROLLOUT_CHANNEL"] = channel + if features is not None: + env["HEADROOM_FEATURES"] = features + if disable_features is not None: + env["HEADROOM_DISABLE_FEATURES"] = disable_features + if unsafe_allow_unstable_features: + env["HEADROOM_UNSAFE_ALLOW_UNSTABLE_FEATURES"] = "1" + try: + snapshot = resolve_rollout(env, strict=True) + except RolloutConfigurationError as exc: + raise click.ClickException(str(exc)) from exc + + payload = snapshot.to_dict() + if json_output: + click.echo(json.dumps(payload, sort_keys=True, separators=(",", ":"))) + return + + click.echo(f"Rollout channel: {snapshot.channel.value}") + click.echo(f"Policy: {snapshot.policy_version} ({snapshot.registry_digest})") + click.echo(f"Snapshot: {snapshot.snapshot_digest}") + click.echo(f"Qualification eligible: {str(snapshot.qualification_eligible).lower()}") + for decision in snapshot.decisions: + click.echo( + f" {decision.name}: enabled={str(decision.enabled).lower()} " + f"decision={decision.reason.value}" + ) diff --git a/headroom/cli/update.py b/headroom/cli/update.py index 9b907f126..93796d246 100644 --- a/headroom/cli/update.py +++ b/headroom/cli/update.py @@ -328,13 +328,14 @@ def detect_install_method(extras: str | None = None) -> InstallMethod: 1. git checkout → refuse (`git pull`) 2. editable install → refuse (reinstall from source) - 3. Docker → refuse (pull a new image) + 3. explicit HEADROOM_IN_DOCKER → refuse (official image opt-out) 4. pipx → `pipx upgrade` 5. uv tool → `uv tool upgrade` 6. venv / virtualenv / conda → `sys.executable -m pip install -U` 7. user-site (`pip --user`) → `sys.executable -m pip install -U --user` - 8. externally-managed system Python (PEP 668) → refuse with guidance - 9. writable global Python → `sys.executable -m pip install -U` (last resort) + 8. bare /.dockerenv (system interpreter) → refuse (pull a new image) + 9. externally-managed system Python (PEP 668) → refuse with guidance + 10. writable global Python → `sys.executable -m pip install -U` (last resort) """ if _is_source_checkout(): return InstallMethod( @@ -351,7 +352,13 @@ def detect_install_method(extras: str | None = None) -> InstallMethod: "reinstall with `pip install -U --force-reinstall .`." ), ) - if _in_docker(): + # An EXPLICIT HEADROOM_IN_DOCKER (set by the official image) is a deliberate + # "pull a newer image" opt-out and wins up front, even over a venv. The bare + # /.dockerenv heuristic is handled far lower, after ownership detection, so a + # pip / pipx / uv install inside a devcontainer, Codespace, or docker dev + # image is not shadowed by the mere fact that the environment is a container + # (#2816). + if os.environ.get("HEADROOM_IN_DOCKER", "").strip(): return InstallMethod( kind="docker", can_self_update=False, @@ -404,6 +411,18 @@ def detect_install_method(extras: str | None = None) -> InstallMethod: argv=[sys.executable, "-m", "pip", "install", "-U", "--user", _spec(extras)], ) + # Bare /.dockerenv with no venv / pipx / uv / user-site owner: the install + # belongs to the container's own interpreter, where "pull a newer image" is + # the only real route. An explicit HEADROOM_IN_DOCKER already returned above. + if _in_docker(): + return InstallMethod( + kind="docker", + can_self_update=False, + guidance=( + "Running inside a container — pull a newer Headroom image instead of self-updating." + ), + ) + if _is_externally_managed(): return InstallMethod( kind="system", diff --git a/headroom/cli/wrap.py b/headroom/cli/wrap.py index af22dc45c..09c6642bb 100644 --- a/headroom/cli/wrap.py +++ b/headroom/cli/wrap.py @@ -33,6 +33,8 @@ import sys import time import urllib.parse from collections.abc import Callable +from contextlib import contextmanager +from functools import wraps from pathlib import Path from typing import Any, cast @@ -57,6 +59,7 @@ from headroom._version import normalize_release_version as _normalize_release_ve from headroom.agent_savings import ( apply_agent_savings_env_defaults, ) +from headroom.cli.proxy import ensure_proxy_dependencies from headroom.copilot_auth import ( _API_TOKEN_ENV_VARS, _API_TOKEN_EXPIRES_AT_ENV_VAR, @@ -74,6 +77,8 @@ from headroom.providers.claude import ( REMOTE_CONTROL_BASE_URL_ENV, TOOL_SEARCH_DEFAULT, TOOL_SEARCH_ENV, + claude_auth_conflict_message, + claude_auth_conflict_sources, claude_user_settings_path, configure_vscode_claude_settings, detect_claude_code_version, @@ -87,6 +92,7 @@ from headroom.providers.claude import ( from headroom.providers.claude import ( proxy_base_url as _claude_proxy_base_url, ) +from headroom.providers.claude.runtime import TOOL_SEARCH_FOUNDRY_DEFAULT from headroom.providers.codex import build_launch_env as _build_codex_launch_env from headroom.providers.codex.install import codex_uses_chatgpt_auth from headroom.providers.codex.threads import retag_to_headroom, retag_to_native @@ -256,6 +262,30 @@ def _read_settings_for_write(path: Path) -> dict[str, Any]: return cast("dict[str, Any]", payload) +def _claude_settings_env(path: Path) -> dict[str, object]: + """Read a Claude settings env block for preflight validation.""" + env = _read_settings_for_write(path).get("env") + return dict(env) if isinstance(env, dict) else {} + + +def _raise_on_claude_auth_conflict( + *, + user_settings_path: Path, + project_settings_path: Path, + project_local_settings_path: Path, + environ: dict[str, str], +) -> None: + """Refuse an auth state Claude Code rejects before mutating wrap state.""" + conflict = claude_auth_conflict_sources( + (str(user_settings_path), _claude_settings_env(user_settings_path)), + (str(project_settings_path), _claude_settings_env(project_settings_path)), + (str(project_local_settings_path), _claude_settings_env(project_local_settings_path)), + ("shell environment", environ), + ) + if conflict is not None: + raise click.ClickException(claude_auth_conflict_message(conflict)) + + def _append_text(path: Path, content: str) -> None: """Append to a text file as UTF-8 without translating line endings.""" fsutil.append_text(path, content) @@ -269,13 +299,14 @@ _WRAP_PROXY_TIMEOUT_ML_MODULES = ("torch", "sentence_transformers", "spacy") # Issue #746: Claude Code disables on-demand tool loading (deferral) when # ANTHROPIC_BASE_URL is a custom host and ENABLE_TOOL_SEARCH is unset, which # inflates the local context window by tens of K tokens. Setting the env var -# when we launch Claude Code keeps deferral on. Default to "true" — defer the -# MCP/system tools for maximum context savings, matching native first-party -# behaviour (core built-ins like Read/Edit/Bash are never deferred by Claude -# Code, so the agent loop is unaffected). The key/default are shared with -# `init` and `install` via the Claude provider package to prevent drift. +# when we launch Claude Code keeps deferral on. The generic default stays +# "true" for non-Foundry sessions, while Foundry uses a dedicated compatibility +# default of "false" because its upstream does not support the deferred-tool +# shape. The key/defaults are shared with `init` and `install` via the Claude +# provider package to prevent drift. _TOOL_SEARCH_ENV = TOOL_SEARCH_ENV _TOOL_SEARCH_DEFAULT = TOOL_SEARCH_DEFAULT +_TOOL_SEARCH_FOUNDRY_DEFAULT = TOOL_SEARCH_FOUNDRY_DEFAULT _AGENT_SAVINGS_WRAP_AGENTS = {"claude", "codex", "cursor", "grok", "grok_build"} # 1M context window for `wrap claude` (#1158). Claude Code only sends the @@ -303,6 +334,33 @@ def _resolve_1m_model(current: str | None) -> str: return base if base.endswith(_CONTEXT_1M_SUFFIX) else f"{base}{_CONTEXT_1M_SUFFIX}" +def _apply_1m_to_claude_args(args: tuple[str, ...]) -> tuple[tuple[str, ...], str | None]: + """Add the ``[1m]`` suffix to an explicit ``--model`` in pass-through args. + + Claude Code gives the ``--model`` CLI flag precedence over the + ``ANTHROPIC_MODEL`` env var, so when a user passes both ``--1m`` and + ``--model X`` the env-var suffix is silently shadowed and the session caps at + 200k (#2915). Rewriting the flag's value the same way ``_resolve_1m_model`` + rewrites the env var keeps ``--1m`` effective on the higher-precedence flag. + + Handles ``--model VALUE`` and ``--model=VALUE`` (the first occurrence only, as + Claude Code honours the first). Idempotent via ``_resolve_1m_model``. Returns + ``(new_args, rewritten_value)``; ``rewritten_value`` is ``None`` when no + ``--model`` was present (the env-var path already covers that case). + """ + out = list(args) + for i, arg in enumerate(out): + if arg == "--model" and i + 1 < len(out): + rewritten = _resolve_1m_model(out[i + 1]) + out[i + 1] = rewritten + return tuple(out), rewritten + if arg.startswith("--model="): + rewritten = _resolve_1m_model(arg.split("=", 1)[1]) + out[i] = f"--model={rewritten}" + return tuple(out), rewritten + return tuple(out), None + + def _normalize_tool_search_mode(value: str) -> str: """Validate an ``ENABLE_TOOL_SEARCH`` value and return it normalized. @@ -331,7 +389,8 @@ def _configure_tool_search_env(env: dict[str, str], flag_value: str | None) -> s 1. explicit ``--tool-search`` flag — wins (the user asked for it on the CLI), 2. a pre-existing ``ENABLE_TOOL_SEARCH`` in the environment — respected and left untouched (the user's own Claude Code knob), - 3. the built-in default (``true``). + 3. the built-in mode-specific default (``true`` normally, ``false`` on + Foundry). Returns the value written, or ``None`` when an existing environment value was deliberately left in place. @@ -346,8 +405,11 @@ def _configure_tool_search_env(env: dict[str, str], flag_value: str | None) -> s existing = env.get(_TOOL_SEARCH_ENV) if existing is not None and existing.strip(): return None - env[_TOOL_SEARCH_ENV] = _TOOL_SEARCH_DEFAULT - return _TOOL_SEARCH_DEFAULT + default = ( + _TOOL_SEARCH_FOUNDRY_DEFAULT if env.get("CLAUDE_CODE_USE_FOUNDRY") else _TOOL_SEARCH_DEFAULT + ) + env[_TOOL_SEARCH_ENV] = default + return default # ENABLE_TOOL_SEARCH modes that turn deferral OFF. Everything else Claude Code @@ -415,6 +477,8 @@ def _resolved_tool_search_mode(flag_value: str | None) -> str: existing = os.environ.get(_TOOL_SEARCH_ENV) if existing is not None: probe[_TOOL_SEARCH_ENV] = existing + if os.environ.get("CLAUDE_CODE_USE_FOUNDRY"): + probe["CLAUDE_CODE_USE_FOUNDRY"] = os.environ["CLAUDE_CODE_USE_FOUNDRY"] written = _configure_tool_search_env(probe, flag_value) return written if written is not None else probe.get(_TOOL_SEARCH_ENV, "") @@ -629,6 +693,15 @@ def _start_proxy( proxy_env = os.environ.copy() _scrub_copilot_proxy_seed_env(proxy_env) proxy_env["PYTHONIOENCODING"] = "utf-8" + # `python -m headroom.cli` prepends the launch cwd to sys.path, so running + # `wrap` from a directory that contains a `headroom/` folder (most commonly a + # clone of this repo, whose package lives at /headroom/) shadows the + # installed wheel with the raw source tree, which has no compiled + # `headroom._core`. The proxy then dies with "No module named 'headroom._core'" + # and wrap silently falls back to launching the client unwrapped (#2793). + # PYTHONSAFEPATH disables that cwd prepend (Python 3.11+; a harmless no-op on + # 3.10) so the subprocess always resolves the installed package. + proxy_env["PYTHONSAFEPATH"] = "1" # Vertex AI RST_STREAMs HTTP/2 connections (error_code:2). Force HTTP/1.1 # when wrapping a Vertex-mode client so upstream requests succeed. if os.environ.get("CLAUDE_CODE_USE_VERTEX") or os.environ.get("ANTHROPIC_VERTEX_PROJECT_ID"): @@ -750,7 +823,7 @@ _RETIRED_CONTEXT_TOOL_MESSAGE = ( "rewrote shell commands through a third-party binary Headroom no longer " "manages. Drop --context-tool / --no-context-tool and unset " f"{_RETIRED_CONTEXT_TOOL_ENV}; `headroom wrap` uninstalls what they left " - "behind on first run." + "behind automatically." ) @@ -810,8 +883,10 @@ def _report_context_tool_purge() -> None: default: the Claude ``PreToolUse`` hook, the vendored binaries and the injected hint-file guidance are all durable on disk. Running this once per ``wrap`` / ``unwrap`` invocation is what actually makes the tools go away. - Silent when there is nothing to do, which is the steady state after the first - run, and never fatal — a cleanup failure must not block launching the tool. + Silent when there is nothing to do — the common case once the machine-global + half is stamped done, though the project- and config-directory-scoped half + still runs every launch — and never fatal: a cleanup failure must not block + launching the tool. Reports on **stderr**: some subcommands (``wrap/unwrap openclaw --prepare-only``) emit machine-readable JSON on stdout as their entire @@ -1431,6 +1506,36 @@ def _write_claude_wrap_base_url( return previous +def _write_claude_wrap_tool_search(value: str, *, settings_path: Path | None = None) -> str | None: + """Persist the resolved tool-search mode for daemon-spawned workers. + + Claude Code workers read project settings afresh rather than inheriting + the parent process environment (#2492). Keep this separate from the proxy + URL crash marker: a stale tool-search mode cannot route traffic to a dead + process, and is restored transactionally when the wrap session exits. + """ + path = settings_path or (Path.cwd() / ".claude" / "settings.local.json") + payload = _read_settings_for_write(path) + env_map = dict(payload.get("env") or {}) if isinstance(payload.get("env"), dict) else {} + previous = env_map.get(_TOOL_SEARCH_ENV) + env_map[_TOOL_SEARCH_ENV] = value + payload["env"] = env_map + path.parent.mkdir(parents=True, exist_ok=True) + _write_text(path, json.dumps(payload, indent=2) + "\n") + return previous + + +def _restore_claude_wrap_tool_search( + previous: str | None, *, settings_path: Path | None = None +) -> None: + """Restore the project-local tool-search value written for this session.""" + _restore_claude_wrap_base_url( + previous, + settings_path=settings_path, + _key_override=_TOOL_SEARCH_ENV, + ) + + def _restore_claude_wrap_base_url( previous: str | None, *, @@ -1671,6 +1776,18 @@ def _serena_project_skip_reason(root: Path) -> str | None: Serena's own ``~/.serena`` config directory. A linked git worktree (its top-level ``.git`` is a file, not a directory) is an ephemeral checkout that would pay for its own index at a path that soon disappears. + + A project with no ``.serena/project.yml`` is skipped because the pre-index + cannot succeed there (#2938). ``serena project index`` auto-creates the file + when it is missing, and that auto-creation calls + ``ProjectConfig.autogenerate(interactive=True)``, which asks one ``[y/N]`` + question per additionally-detected language server. The CLI has no + non-interactive switch; the only way to reach the silent branch is to pass + ``--ls/--language`` explicitly, which means Headroom guessing the project's + languages again — exactly the hand-maintained map removed below. Serena's + MCP server generates that file itself (non-interactively) on first start and + indexes lazily on demand, so the pre-index simply resumes from the next + wrap onwards. """ try: resolved = root.resolve() @@ -1681,46 +1798,144 @@ def _serena_project_skip_reason(root: Path) -> str | None: return "$HOME is not a project" if (resolved / ".git").is_file(): return "linked git worktree" + if not (resolved / ".serena" / "project.yml").is_file(): + return "no .serena/project.yml yet — Serena will create it and index on demand" return None +#: Upper bound on the synchronous pre-index. The agent does not launch until +#: this call returns, so the number is a stall budget, not just a safety net. +_SERENA_INDEX_TIMEOUT = 300 + + +def _kill_serena_index_tree(proc: subprocess.Popen) -> None: + """Kill *proc* and everything it spawned (best-effort, never raises). + + ``uvx`` is a launcher: it resolves the environment and then runs the real + ``serena`` executable as a grandchild. Killing only the direct child leaves + that grandchild alive and reparented to PID 1, so every timed-out pre-index + leaked one process that never exits (#2938 — the same failure mode as #615 + and #880). The child is started in its own process group precisely so the + whole tree can be signalled here. + """ + if sys.platform == "win32": + # Windows has no process groups to signal for an already-wedged child; + # ``taskkill /T`` walks the tree by parent PID instead. ``/F`` because a + # process blocked in a read will not act on a graceful close request. + try: + subprocess.run( + ["taskkill", "/F", "/T", "/PID", str(proc.pid)], + capture_output=True, + timeout=10, + check=False, + ) + except Exception: + pass + else: + try: + os.killpg(os.getpgid(proc.pid), signal.SIGKILL) + except Exception: + pass + # Backstop: if the tree kill above did not land, at least the direct child + # goes. Then reap so the parent does not leave a zombie behind, and close + # the capture pipes we opened so the wrap does not carry stray fds into the + # agent it is about to exec. + try: + proc.kill() + except Exception: + pass + try: + proc.wait(timeout=10) + except Exception: + pass + for stream in (proc.stdout, proc.stderr, proc.stdin): + try: + if stream is not None: + stream.close() + except Exception: + pass + + def _index_serena_project(*, verbose: bool = False) -> None: """Warm Serena's symbol cache for the current project (non-fatal). - Runs ``serena project index`` (the same ``uvx --from git+…`` launch used to - start the MCP server) in the project directory so the first symbol query is - not paying for a cold index. Timeout-guarded and best-effort: Serena also - indexes lazily on demand, so a failure or timeout here never blocks the - wrap. + Runs ``serena project index`` (the same ``uvx --from serena-agent`` launch + used to start the MCP server) in the project directory so the first symbol + query is not paying for a cold index. Serena also indexes lazily on demand, + so any failure here is survivable. + + This runs on the launch path, synchronously: the agent starts only once it + returns, so the timeout below is time the user spends staring at nothing. + Two guards keep that bounded (#2938): + + * ``stdin`` is ``DEVNULL``. Serena prompts when it has to auto-create + ``project.yml``, and because stdout is captured the question never + reaches the terminal — an inherited stdin turned that into a silent, + full-timeout hang. EOF makes it fail in about a second instead. + ``_serena_project_skip_reason`` already keeps us out of that state; this + is the belt-and-braces half, and it covers any future Serena prompt too. + * The child gets its own process group so ``_kill_serena_index_tree`` can + take out the ``uvx`` grandchild on timeout rather than orphaning it. """ if shutil.which("uvx") is None: if verbose: click.echo(" Serena: uvx not found — skipping pre-index") return + + popen_kwargs: dict[str, Any] = { + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE, + "stdin": subprocess.DEVNULL, + "text": True, + # ``subprocess.Popen`` directly, so the encoding defaults that + # ``headroom._subprocess.run`` applies have to be repeated here. + "encoding": "utf-8", + "errors": "replace", + "cwd": str(Path.cwd()), + } + if sys.platform == "win32": + popen_kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP + else: + popen_kwargs["start_new_session"] = True + try: - result = run( + proc = subprocess.Popen( [ "uvx", + # PyPI (prebuilt wheels), not the git source that fails to build + # under proot-based filesystems (#2871). "--from", - "git+https://github.com/oraios/serena", + "serena-agent", "serena", "project", "index", ], - capture_output=True, - text=True, - timeout=300, - cwd=str(Path.cwd()), + **popen_kwargs, ) - if result.returncode == 0: - click.echo(" Serena: project pre-indexed (symbol cache warmed)") - elif verbose: - click.echo(f" Serena: pre-index failed ({(result.stderr or '')[:100]})") - except subprocess.TimeoutExpired: - click.echo(" Serena: pre-index timed out (will index on demand)") except Exception as e: if verbose: click.echo(f" Serena: pre-index skipped ({e})") + return + + # Announce the wait. Indexing a large repo legitimately takes minutes and + # the output is captured, so without this line the wrap looks hung. + click.echo(" Serena: pre-indexing project (first run can take a while)…") + try: + _stdout, stderr = proc.communicate(timeout=_SERENA_INDEX_TIMEOUT) + except subprocess.TimeoutExpired: + _kill_serena_index_tree(proc) + click.echo(" Serena: pre-index timed out (will index on demand)") + return + except Exception as e: + _kill_serena_index_tree(proc) + if verbose: + click.echo(f" Serena: pre-index skipped ({e})") + return + + if proc.returncode == 0: + click.echo(" Serena: project pre-indexed (symbol cache warmed)") + elif verbose: + click.echo(f" Serena: pre-index failed ({(stderr or '')[:100]})") def _setup_serena_mcp( @@ -1787,7 +2002,9 @@ def _setup_serena_mcp( # Serena is the active engine here (we passed the detect/uvx guards): steer # the agent toward symbol-level tools, then warm the symbol cache. Both are - # best-effort and non-fatal — neither blocks the wrap. + # best-effort and non-fatal, but the pre-index is *synchronous* — the agent + # does not launch until it returns or hits ``_SERENA_INDEX_TIMEOUT``. See + # ``_index_serena_project`` for how that wait is kept bounded and visible. # # Headroom no longer writes ``.serena/project.yml`` language scoping. Serena # determines the project's languages itself during @@ -3416,7 +3633,7 @@ def _push_runtime_env(port: int, no_proxy: bool) -> None: click.echo(f" Synced output settings to proxy: {', '.join(sorted(payload))}") -def _ensure_proxy( +def _ensure_proxy_unlocked( port: int, no_proxy: bool, *, @@ -3435,7 +3652,13 @@ def _ensure_proxy( copilot_refresh_oauth_token: str | None = None, copilot_api_token_expires_at: float | None = None, ) -> tuple[subprocess.Popen | None, int]: - """Start or verify proxy. Returns (process_handle, actual_port).""" + """Start or verify proxy. Returns (process_handle, actual_port). + + The public ``_ensure_proxy`` wrapper serializes callers per port before + entering this function. Keeping the implementation separate makes the + lock boundary explicit and ensures every health/configuration check runs + under the same startup critical section. + """ helpers = _live_wrap_module() copilot_subscription_seed_requested = ( bool(copilot_api_token) @@ -3802,6 +4025,81 @@ def _ensure_proxy( return None, port +@contextmanager +def _proxy_start_lock(port: int) -> Any: + """Serialize wrap proxy startup across processes sharing a port. + + A proxy can spend tens of seconds loading optional ML components before it + binds its socket. Without this lock, two concurrent ``headroom wrap`` + commands both see an unavailable health endpoint, choose the same port, + and race to spawn a listener. The lock is deliberately held through the + health/configuration checks and startup, then released once the proxy is + ready (or startup fails). Lock files are retained so an interrupted + process cannot create an inode-replacement race for another waiter. + """ + from headroom import paths as _paths + + lock_path = _paths.proxy_start_lock_path(port) + try: + lock_path.parent.mkdir(parents=True, exist_ok=True) + lock_file = open(lock_path, "a+b") # noqa: SIM115 + except OSError: + # Locking is a race-prevention enhancement, not a reason to make wrap + # unusable when a read-only/custom workspace cannot hold state. The + # existing port bind remains the final safety check in that degraded + # environment. + yield + return + with lock_file: + if sys.platform == "win32": + import msvcrt + + # msvcrt.locking operates on bytes from the current file position. + lock_file.seek(0) + if lock_file.read(1) == b"": + lock_file.seek(0) + lock_file.write(b"0") + lock_file.flush() + lock_file.seek(0) + # LK_LOCK has implementation-dependent retry limits. A proxy may + # legitimately take longer than that to load ML components, so + # use the non-blocking primitive in a loop instead. + while True: + try: + msvcrt.locking(lock_file.fileno(), msvcrt.LK_NBLCK, 1) + break + except OSError: + time.sleep(0.05) + try: + yield + finally: + lock_file.seek(0) + msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1) + else: + import fcntl + + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + + +@wraps(_ensure_proxy_unlocked) +def _ensure_proxy( + port: int, + no_proxy: bool, + **kwargs: Any, +) -> tuple[subprocess.Popen | None, int]: + """Start or reuse a proxy without racing another wrap on the same port.""" + if no_proxy: + return _ensure_proxy_unlocked(port, no_proxy, **kwargs) + with _proxy_start_lock(port): + # Re-checking is part of the lock boundary: a concurrent wrapper may + # have finished startup while this caller was waiting for the lock. + return _ensure_proxy_unlocked(port, no_proxy, **kwargs) + + def _client_marker_path(port: int) -> Path: """Path to this process's wrap-client marker for ``port``.""" from headroom import paths as _paths @@ -4463,6 +4761,8 @@ def claude( proxy_holder: list[subprocess.Popen | None] = [None] _saved_base_url: list[str | None] = [None] # previous settings.json value for restore + _tool_search_not_written = object() + _saved_tool_search: list[object | str | None] = [_tool_search_not_written] _settings_foundry: list[bool] = [False] port_holder: list[int] = [port] _settings_vertex: list[bool] = [False] @@ -4471,6 +4771,12 @@ def claude( # early proxy-start failure would make the finally raise UnboundLocalError, # masking the real error and skipping cleanup(). Mirrors the holders above. _wrap_settings_path = Path.cwd() / ".claude" / "settings.local.json" + _raise_on_claude_auth_conflict( + user_settings_path=claude_user_settings_path(), + project_settings_path=Path.cwd() / ".claude" / "settings.json", + project_local_settings_path=_wrap_settings_path, + environ=dict(os.environ), + ) cleanup = _make_cleanup(proxy_holder, port_holder) signal.signal(signal.SIGINT, _ignore_child_sigint) signal.signal(signal.SIGTERM, cleanup) @@ -4694,6 +5000,11 @@ def claude( # Issue #746: keep Claude Code's on-demand tool loading on through the # proxy so tool schemas are not eagerly materialized into local context. _tool_search_value = _configure_tool_search_env(env, tool_search) + _resolved_tool_search_value = env.get(_TOOL_SEARCH_ENV, "") + _saved_tool_search[0] = _write_claude_wrap_tool_search( + _resolved_tool_search_value, + settings_path=_wrap_settings_path, + ) if _tool_search_value is not None: # Describe what the written value actually does: --tool-search # false/0/no/off turns deferral OFF, and the banner must say so @@ -4717,10 +5028,18 @@ def claude( # force it via ANTHROPIC_MODEL on the launched process. if context_1m: env[_ANTHROPIC_MODEL_ENV] = _resolve_1m_model(env.get(_ANTHROPIC_MODEL_ENV)) - click.echo( - f" {_ANTHROPIC_MODEL_ENV}={env[_ANTHROPIC_MODEL_ENV]} " - "(1M context window; issue #1158)" - ) + # An explicit pass-through --model outranks ANTHROPIC_MODEL in Claude + # Code, so add the suffix there too or the env var is silently + # shadowed and the window stays 200k (#2915). Report what will + # actually take effect rather than the shadowed env value. + claude_args, _model_flag_1m = _apply_1m_to_claude_args(claude_args) + if _model_flag_1m is not None: + click.echo(f" --model {_model_flag_1m} (1M context window; issue #1158)") + else: + click.echo( + f" {_ANTHROPIC_MODEL_ENV}={env[_ANTHROPIC_MODEL_ENV]} " + "(1M context window; issue #1158)" + ) result = subprocess.run([claude_bin, *claude_args], env=env) raise SystemExit(result.returncode) @@ -4731,6 +5050,11 @@ def claude( click.echo(f" Error: {e}") raise SystemExit(1) from e finally: + if _saved_tool_search[0] is not _tool_search_not_written: + _restore_claude_wrap_tool_search( + cast(str | None, _saved_tool_search[0]), + settings_path=_wrap_settings_path, + ) _restore_claude_wrap_base_url( _saved_base_url[0], foundry_mode=_settings_foundry[0], @@ -5044,8 +5368,11 @@ def copilot( "automatic model selection." ) + env_wire_api = env.get("COPILOT_PROVIDER_WIRE_API") effective_wire_api = wire_api or ( - _copilot_default_wire_api_for_model(selected_model) if subscription else "completions" + env_wire_api + if env_wire_api in {"completions", "responses"} + else _copilot_default_wire_api_for_model(selected_model) ) env["COPILOT_PROVIDER_TYPE"] = "openai" # Per-project savings: the Copilot CLI cannot send custom headers, so @@ -5184,8 +5511,8 @@ def vscode_copilot( ) -> None: """Run Headroom for GitHub Copilot inside Visual Studio Code. - Transparently overrides Copilot's proxy endpoint, preserving the model - selected in VS Code. It does not edit Codex settings. + Transparently overrides Copilot's proxy and CAPI endpoints, preserving the + model selected in VS Code. It does not edit Codex settings. """ resolution = _require_copilot_subscription_resolution() target_settings = settings_file or vscode_settings_path() @@ -5205,6 +5532,9 @@ def vscode_copilot( click.echo( f' "github.copilot.advanced.debug.overrideProxyUrl": "{vscode_proxy_url(actual_port, _project_name_from_cwd())}",' ) + click.echo( + f' "github.copilot.advanced.debug.overrideCapiUrl": "{vscode_proxy_url(actual_port, _project_name_from_cwd())}",' + ) click.echo(' "github.copilot.advanced.debug.overrideAuthType": "token"') _run_proxy_only_watcher( @@ -5469,6 +5799,9 @@ def _run_codex_wrap( codex_args: tuple, ) -> None: """Execute the Codex wrap flow against the durable Codex home.""" + if not no_proxy: + ensure_proxy_dependencies() + if prepare_only: _prepare_codex_wrap_state( port=port, @@ -6936,6 +7269,20 @@ def opencode( ) subscription_resolution = _require_copilot_subscription_resolution() + # Verify the opencode binary exists BEFORE mutating any config. Otherwise a + # missing binary leaves headroom MCP/Serena/memory entries in the user's + # opencode config and an injected AGENTS.md, then errors with no cleanup -- + # the config-before-verify anti-pattern (#1614). Siblings (claude, codex, + # goose, omp) already check first. `--prepare-only` intentionally writes + # config without launching, so it is exempt. + opencode_bin: str | None = None + if not prepare_only: + opencode_bin = shutil.which("opencode") + if not opencode_bin: + click.echo("Error: 'opencode' not found in PATH.") + click.echo("Install OpenCode: https://opencode.ai") + raise SystemExit(1) + # Snapshot OpenCode config.json BEFORE any wrap-time mutation so # `headroom unwrap opencode` can restore the user's pre-wrap state. _opencode_config_file, _opencode_backup_file = opencode_config_paths() @@ -6976,11 +7323,9 @@ def opencode( inject_opencode_provider_config(port) return - opencode_bin = shutil.which("opencode") - if not opencode_bin: - click.echo("Error: 'opencode' not found in PATH.") - click.echo("Install OpenCode: https://opencode.ai") - raise SystemExit(1) + # Past the prepare-only return the launch path always ran the binary check + # above, so opencode_bin is resolved. + assert opencode_bin is not None # Register our proxy client marker BEFORE _ensure_proxy so that another # wrapper's cleanup sees us as an active client and doesn't terminate a diff --git a/headroom/config.py b/headroom/config.py index 2bc9f4a8a..01f94182c 100644 --- a/headroom/config.py +++ b/headroom/config.py @@ -11,6 +11,7 @@ from enum import Enum from typing import Any, Literal from headroom.models.config import ML_MODEL_DEFAULTS +from headroom.rollout import RolloutSnapshot, resolve_rollout class HeadroomMode(str, Enum): @@ -672,9 +673,14 @@ class HeadroomConfig: content_router_enabled: InitVar[bool | None] = None # Tool-result interceptors (ast-grep Read outline, etc.). Opt-in for now. - # Env var HEADROOM_INTERCEPT_ENABLED=1 also enables (for CLI `--intercept-tool-results`). + # The legacy env alias and this typed request still obey the canary rollout gate. intercept_tool_results: bool = False + # Immutable runtime rollout state. ``None`` is resolved once here so every + # pipeline built from this config observes the same decisions even if the + # process environment later changes. + rollout: RolloutSnapshot | None = None + # Debugging - opt-in diff artifact generation generate_diff_artifact: bool = False # Enable to get detailed transform diffs @@ -682,6 +688,11 @@ class HeadroomConfig: pipeline_extensions: list[Any] = field(default_factory=list) discover_pipeline_extensions: bool = True + def __post_init__(self, content_router_enabled: bool | None = None) -> None: + if self.rollout is None: + requested = ("tool_result_interceptors",) if self.intercept_tool_results else () + self.rollout = resolve_rollout(requested=requested) + def get_context_limit(self, model: str) -> int | None: """ Get context limit for a model from user overrides. diff --git a/headroom/context_tool_cleanup.py b/headroom/context_tool_cleanup.py index d6860d6ff..6dbeb2f14 100644 --- a/headroom/context_tool_cleanup.py +++ b/headroom/context_tool_cleanup.py @@ -10,26 +10,87 @@ Deleting the code is not enough: everything above is *durable state on the user's disk*. Left alone, the Claude hooks keep rewriting every Bash command through binaries Headroom no longer manages, and the injected guidance keeps telling agents to use tools that may not resolve. So ``headroom wrap`` / -``headroom unwrap`` call :func:`purge_context_tool_artifacts` once per run to -remove what earlier versions installed. +``headroom unwrap`` call :func:`purge_context_tool_artifacts` on every run to +remove what earlier versions installed — machine-global artifacts (hooks, +binaries, Claude Code's MCP registration) are removed once per workspace and +then stamped done (see the stamp below), while project- and config-directory- +scoped guidance (``CODEX_HOME`` / ``OPENCODE_HOME`` hint files, Continue's +config) is inspected on every launch, since a later launch can sit in a +different project or point at a different ``CODEX_HOME`` / ``OPENCODE_HOME``. Everything here is idempotent, best-effort and deliberately conservative: * only files Headroom installed (or caused a context tool to install) are - deleted; + deleted. An MCP entry's ``command`` or a hook script's body counts as + Headroom's only when it names a path inside :func:`paths.bin_dir` + (:func:`_references_managed_bin` — which is where the matching rules and + the reasons behind them live); +* a hook entry is Headroom's when it names such a path directly, or — the + common case, since a hook command names a script rather than the binary — + when it names one of the ``~/.claude/hooks`` scripts already classified as + Headroom's, whose verdict it inherits + (:func:`_references_context_tool`, :func:`_names_a_managed_script`). A + Cursor ``hooks.json`` entry naming a script under ``~/.cursor`` cannot + inherit a verdict this way, since the map covers ``~/.claude/hooks`` only; + it is still caught when its ``command`` names the managed directory; +* ``.rtk-hook.sha256`` is never read for its own provenance (it holds a hex + digest, not a path) and instead inherits ``rtk-rewrite.sh``'s + classification; a ``.lean-ctx.bak`` backup inherits ````'s + (:func:`_classify_hook_scripts`); +* a hook script that exists but cannot be read is classified unknown — + deleted by nothing, and named in the report so the user can remove it by + hand; * ``~/.local/bin/{rtk,lean-ctx}`` is unlinked only when it is a symlink into Headroom's own bin directory — a user's own build is never touched; * a JSON config that does not parse is reported and **skipped**, never overwritten (a hand-edited typo must not cost the user their settings); * the tools' own backups of *config* files (``~/.claude.json.lean-ctx.bak`` and friends) are left in place — they hold the user's real settings history. Only - backups of the hook scripts being deleted are cleaned up. + backups of the hook scripts proven to be Headroom's are cleaned up; +* two cases cannot be decided at all, and are accepted as limits rather than + fixed: + + * ``get_lean_ctx_path`` used to check ``PATH`` before Headroom's own bin + directory, so on a machine that already had ``lean-ctx`` on ``PATH``, + the tool that ran was the user's own, and the config it wrote looks + exactly like config the user wrote by hand. That leftover survives the + purge — it still points at a binary that exists, so nothing dangles; + * an rtk hook written *after* #1698 execs a bare ``rtk`` and never mentions + :func:`paths.bin_dir`, so it reads exactly like a hook a user wrote by + hand, and ``rtk-rewrite.sh``, its ``.rtk-hook.sha256`` and its + ``settings.json`` entry all survive while step 3 removes the managed + binary — leaving a hook that silently no-ops (#487, #1698). Earlier + hooks are decidable: Headroom patched the absolute managed path into + them (``_patch_rtk_hook_absolute_path``, removed by #1698), so the + window this misses is rtk setups run between #1698 and the tools' + removal in #2677; +* the marker-fenced guidance block is the one step with no provenance check + to make — ```` is Headroom's own fence, + and no third party writes it. + +Removing the retired integration's machine-global footprint — hook +registrations, hook scripts, PATH symlinks, managed binaries and Claude +Code's own MCP registration — is a one-time migration: the first completed +run of that half stamps ``.context-tools-purged`` beside the managed bin +directory, and every later run skips that half outright. Without the stamp +this would keep rewriting the same machine-wide files on every ``wrap`` +invocation forever, and a user who installs one of these tools *after* the +migration would have Headroom auditing files at each launch for a leftover +that cannot exist there. + +Project- and config-directory-scoped state is not covered by that stamp: a +later invocation can sit in a different project, or point ``CODEX_HOME`` / +``OPENCODE_HOME`` somewhere the stamped run never inspected, and whatever +guidance an earlier Headroom left behind there is still worth removing — so +those steps run on every invocation instead (:func:`_purge_invocation_scoped`). """ from __future__ import annotations import json import os +import posixpath +import re from pathlib import Path from typing import Any @@ -69,52 +130,138 @@ _HOOK_SCRIPTS = ( "lean-ctx-redirect-native", ) +# rtk's integrity digest never names a path (see ``_classify_hook_scripts``) +# and rtk-rewrite.sh is the script it authenticates. +_RTK_DIGEST_NAME = ".rtk-hook.sha256" +_RTK_SCRIPT_NAME = "rtk-rewrite.sh" + # MCP server entries the tools registered, and the config files holding them. # lean-ctx registers itself as an MCP server during ``lean-ctx init``; rtk never # did, but it is matched too so a stale hand-added entry is cleaned up as well. _MCP_SERVER_NAMES = ("lean-ctx", "lean_ctx", "rtk") +# Report-line prefixes meaning "this one is not settled" — a config that would +# not parse, a script that would not read, a file that would not unlink. Such a +# run leaves a leftover behind, so it must not be stamped as the completed +# migration. Emitted by _purge_hook_config, _purge_mcp_entries, +# _purge_fenced_block, _purge_continue_system_messages and _remove_files. +_DEFERRED_PREFIXES = ("skipped ", "could not remove ") + def purge_context_tool_artifacts() -> list[str]: """Remove every rtk / lean-ctx artifact an earlier Headroom version installed. Returns human-readable descriptions of what was removed — plus a line for - any config that had to be skipped because the user must fix it by hand. An - empty list means there was nothing to do, which is the steady state after - the first run. + any config that had to be skipped because the user must fix it by hand. + Machine-global cleanup (hooks, binaries, Claude Code's MCP registration) + runs once and is then skipped via the stamp below; project- and config- + directory-scoped cleanup (hint files, ``CODEX_HOME`` / ``OPENCODE_HOME``, + Continue's config) runs on every call, so a later call in a different + project or a repointed ``CODEX_HOME`` / ``OPENCODE_HOME`` can still report + something even after the global half is long since stamped done. """ + marker = _purge_marker() home = Path.home() project = Path.cwd() report: list[str] = [] - # 1. Hook registrations (Claude Code's settings.json, Cursor's hooks.json). - for config in (home / ".claude" / "settings.json", home / ".cursor" / "hooks.json"): - report += _purge_hook_config(config) + if not marker.exists(): + # Classify every hook script's provenance once, up front: both step 1 + # (is a settings.json entry pointing at *our* script?) and step 2 (is + # the script itself ours?) need the same answer, and each file is + # read once. + hooks_dir = home / ".claude" / "hooks" + hook_classification = _classify_hook_scripts(hooks_dir) - # 2. The generated hook scripts, their integrity digests and stale backups. - hooks_dir = home / ".claude" / "hooks" - report += _remove_files( - *(hooks_dir / name for name in _HOOK_SCRIPTS), - *(hooks_dir / f"{name}.lean-ctx.bak" for name in _HOOK_SCRIPTS), - ) + # 1. Hook registrations (Claude Code's settings.json, Cursor's hooks.json). + for config in (home / ".claude" / "settings.json", home / ".cursor" / "hooks.json"): + report += _purge_hook_config(config, hooks_dir, hook_classification) - # 3. The PATH symlinks, then the managed binaries they pointed at. - for name in ("rtk", "lean-ctx"): - report += _remove_managed_path_link(home / ".local" / "bin" / name) - report += _remove_files(*(paths.bin_dir() / name for name in _BINARY_NAMES)) + # 2. The generated hook scripts, their integrity digests and stale + # backups — only the ones proven to reference Headroom's managed bin + # directory. + managed_names = [name for name in _HOOK_SCRIPTS if hook_classification.get(name)] + report += _remove_files( + *(hooks_dir / name for name in managed_names), + *(hooks_dir / f"{name}.lean-ctx.bak" for name in managed_names), + ) + for name in _HOOK_SCRIPTS: + if hook_classification.get(name, False) is not None: + continue + if name == _RTK_DIGEST_NAME: + report.append( + f"skipped {hooks_dir / name} (inherits {_RTK_SCRIPT_NAME}'s unreadable verdict)" + " — remove any stale hook script by hand" + ) + else: + report.append( + f"skipped {hooks_dir / name} (could not read to verify it was Headroom's)" + " — remove any stale hook script by hand" + ) - # 4. MCP server registrations (lean-ctx registers itself during init). - report += _purge_mcp_entries(home / ".claude.json", "mcpServers") + # 3. The PATH symlinks, then the managed binaries they pointed at. + for name in ("rtk", "lean-ctx"): + report += _remove_managed_path_link(home / ".local" / "bin" / name) + report += _remove_files(*(paths.bin_dir() / name for name in _BINARY_NAMES)) + + # 4. Claude Code's own MCP server registration (lean-ctx registers + # itself during init). OpenCode's is invocation-scoped — see below. + report += _purge_mcp_entries(home / ".claude.json", "mcpServers") + + # Only a global half that settled everything is the completed + # migration. One that could not read a script or parse a config left + # a leftover behind, and the user needs both the reminder on the next + # launch and the cleanup once the permissions or the typo are fixed. + # A deferral in the invocation-scoped half below must not withhold + # this stamp — that half re-runs every time regardless, so nothing is + # lost by stamping the global half done now. + if not any(line.startswith(_DEFERRED_PREFIXES) for line in report): + try: + # Cleanup must not become the first mutation on a pristine + # machine. In particular, ``wrap `` validates + # the binary after the wrap-group migration hook; creating + # ``~/.headroom`` merely to stamp an empty scan violates that + # command's no-side-effects-on-failure contract. Established + # Headroom installs already have the state directory and get + # the one-time fast path; clean machines cheaply rescan until + # some real Headroom state exists. + if marker.parent.is_dir(): + marker.touch() + except OSError: + pass # Unwritable workspace: the purge simply runs again next time. + + report += _purge_invocation_scoped(home, project) + return report + + +def _purge_invocation_scoped(home: Path, project: Path) -> list[str]: + """Steps the one-time stamp must never withhold. + + ``OPENCODE_HOME``'s config, the hint files in ``project`` / + ``CODEX_HOME`` / ``OPENCODE_HOME``, and Continue's config are all a + function of *this* invocation's cwd and environment, not of the machine — + a later run can sit in a different project or point ``CODEX_HOME`` / + ``OPENCODE_HOME`` somewhere the global-half stamp never inspected. Each + step is cheap and side-effect-free when nothing matches, so re-running + them on every invocation costs a handful of reads in the steady state. + """ + report: list[str] = [] report += _purge_mcp_entries(_opencode_home(home) / "opencode.json", "mcp") - - # 5. Marker-fenced guidance in every hint file the wrap harnesses wrote to. for hint_file in _instruction_files(home, project): report += _purge_fenced_block(hint_file) report += _purge_continue_system_messages(project / ".continue" / "config.json") - return report +def _purge_marker() -> Path: + """Path of the "already migrated" stamp. + + Derived from :func:`paths.bin_dir` rather than ``workspace_dir`` so it + cannot escape a temporary tree through ``HEADROOM_WORKSPACE_DIR``. + """ + return paths.bin_dir().parent / ".context-tools-purged" + + def _instruction_files(home: Path, project: Path) -> list[Path]: """Hint files the wrap subcommands injected the context-tool block into. @@ -151,15 +298,194 @@ def _opencode_home(home: Path) -> Path: # --- hook registrations ------------------------------------------------------- -def _references_context_tool(entry: Any) -> bool: - """Whether a hook entry's command is one a retired context tool registered.""" +# Characters that can never be part of a path: a match ending right before +# one of these (or at end-of-string) sits at a real word boundary. Quotes, +# `=`/`:` (`export PATH=":$PATH"`, `BIN=/x`), `;`/`,`/`|` (command +# joiners) and `()` (subshells) all end a path the same way whitespace does. +_PATH_BOUNDARY_CHARS = frozenset("\"'=:;,()|") + +# Splits a command into path-shaped tokens on whitespace plus the same +# boundary punctuation above — used by _names_a_managed_script, which (unlike +# _references_managed_bin's needle-anchored scan) tokenizes the whole command. +_PATH_TOKEN_SPLIT = re.compile(r"[\s" + re.escape("".join(_PATH_BOUNDARY_CHARS)) + r"]+") + + +def _norm_path_text(value: str) -> str: + """Case-fold ``value`` and give it one separator, so paths compare as text.""" + return os.path.normcase(value).replace("\\", "/") + + +def _references_managed_bin(text: str) -> bool: + """Whether ``text`` names a path inside Headroom's managed bin directory. + + A hook command or script body is free text we don't control, so this + scans ``text`` for raw occurrences of the managed directory — the + unresolved and resolved bin directory, and its ``~``-relative form + (home-relative, since a script may reference it unexpanded) — matched + against the text as given and as ``expanduser``'d, case-folded, with both + path separators. Deliberately *not* tokenized on whitespace first: a + quoted or ``$HOME``-derived path can itself contain a space, and slicing + the text into words before searching would sever it. + + A hit is only a real reference at a path boundary on *both* ends. The + character immediately before the match, if any, must be whitespace or a + :data:`_PATH_BOUNDARY_CHARS` character, or the match is just the tail of + some longer, unrelated path segment (e.g. ``/prefix/lean-ctx``) + and is rejected. The match must then be immediately followed by + end-of-string or a :data:`_PATH_BOUNDARY_CHARS` character (an exact + reference, e.g. a bare ``PATH=`` export), or by ``/`` — in which + case the run of characters up to the next boundary is lexically + normalized (``.``/``..`` collapsed) and re-compared, so neither a sibling + directory like ``bin-backup``/``binfoo`` nor a ``bin/../evil`` traversal + can borrow the managed prefix. + # ponytail: boundary-aware substring scan, not a shell parse — upgrade to + # shlex if a command ever embeds a managed path it does not execute. + """ + try: + bin_dir = paths.bin_dir() + resolved_bin_dir = bin_dir.resolve() + except OSError: + return False + + needles = {_norm_path_text(str(bin_dir)), _norm_path_text(str(resolved_bin_dir))} + home = Path.home() + for base in (bin_dir, resolved_bin_dir): + try: + needles.add(_norm_path_text(f"~/{base.relative_to(home).as_posix()}")) + except ValueError: + pass + + for haystack in (text, os.path.expanduser(text)): + normalized_haystack = _norm_path_text(haystack) + if any(_names_managed_dir(normalized_haystack, needle) for needle in needles): + return True + return False + + +def _names_managed_dir(haystack: str, needle: str) -> bool: + """Whether a normalized ``haystack`` names ``needle`` at a path boundary + on both ends: the character right before the match, if any, must be + whitespace or a :data:`_PATH_BOUNDARY_CHARS` character too, or a + user-owned path that merely has the managed directory as a substring + (e.g. ``/prefix/lean-ctx``) would be misread as naming it. + """ + search_from = 0 + while True: + index = haystack.find(needle, search_from) + if index < 0: + return False + end = index + len(needle) + search_from = index + 1 # keep scanning; occurrences may overlap + if index > 0 and not ( + haystack[index - 1].isspace() or haystack[index - 1] in _PATH_BOUNDARY_CHARS + ): + continue + following = haystack[end : end + 1] + if not following or following.isspace() or following in _PATH_BOUNDARY_CHARS: + return True + if following != "/": + continue + tail_end = end + while tail_end < len(haystack) and not ( + haystack[tail_end].isspace() or haystack[tail_end] in _PATH_BOUNDARY_CHARS + ): + tail_end += 1 + candidate = posixpath.normpath(haystack[index:tail_end]) + if candidate == needle or candidate.startswith(needle + "/"): + return True + + +def _classify_hook_scripts(hooks_dir: Path) -> dict[str, bool | None]: + """Classify each existing ``_HOOK_SCRIPTS`` file by whether it is Headroom's. + + ``True`` — the file exists and its body references the managed bin + directory. ``False`` — it exists and does not. ``None`` — it exists but + could not be read, so its provenance is unprovable. A basename with no + file on disk is simply absent from the map. Each file is read at most once. + + ``.rtk-hook.sha256`` holds a hex digest that can never reference a path, + so it is never read; it inherits ``rtk-rewrite.sh``'s classification. + """ + classification: dict[str, bool | None] = {} + for name in _HOOK_SCRIPTS: + if name == _RTK_DIGEST_NAME: + continue + path = hooks_dir / name + if not path.is_file(): + continue + try: + body = fsutil.read_text(path) + except OSError: + classification[name] = None + continue + classification[name] = _references_managed_bin(body) + + if (hooks_dir / _RTK_DIGEST_NAME).is_file(): + classification[_RTK_DIGEST_NAME] = classification.get(_RTK_SCRIPT_NAME, False) + + return classification + + +def _references_context_tool( + entry: Any, hooks_dir: Path, hook_classification: dict[str, bool | None] +) -> bool: + """Whether a hook entry is one a retired context tool registered. + + A command-marker hit alone is not enough — a user could author a script + with a matching name. It must also either name the managed bin directory + directly, or name — as a path resolving to that exact file, not merely + sharing its basename, so a same-named script of the user's own in a + different directory is never caught — a hook script in ``hooks_dir`` the + classification map marks ``True`` (see :func:`_names_a_managed_script`). + """ if not isinstance(entry, dict): return False - command = str(entry.get("command", "")).lower() - return any(marker in command for marker in _HOOK_COMMAND_MARKERS) + command = str(entry.get("command", "")) + if not any(marker in command.lower() for marker in _HOOK_COMMAND_MARKERS): + return False + if _references_managed_bin(command): + return True + return _names_a_managed_script(command, hooks_dir, hook_classification) -def _prune_hooks(hooks: Any) -> tuple[Any, bool]: +def _names_a_managed_script( + command: str, hooks_dir: Path, hook_classification: dict[str, bool | None] +) -> bool: + """Whether ``command`` names, by absolute path, a hook script the map marks ``True``. + + A real command is rarely the bare script path: ``bash