From 11ab5f83a1ccd617a2608349a42feff7f7e72b98 Mon Sep 17 00:00:00 2001 From: JD Davis Date: Tue, 9 Jun 2026 00:18:31 -0500 Subject: [PATCH] feat: add differential network capture harness (#761) ## Summary - add a containerized differential network capture harness for Claude Code direct vs Claude Code routed through Headroom - capture both Headroom client-side traffic and Headroom upstream traffic with sanitized mitmproxy JSONL output - add `headroom capture network-diff` to compare captures and produce Markdown/JSON reports, including Anthropic tool-count/tool-byte deltas for deferred-tool investigations - add an on-demand GitHub Actions workflow for the harness; it only runs via `workflow_dispatch`, with live Claude Code/Anthropic capture gated on `ANTHROPIC_API_KEY` - document the workflow and ignore generated capture artifacts ## Validation - `C:\git\headroom\.venv\Scripts\python.exe -m pytest tests/test_network_diff_capture.py` - `ruff check headroom/capture headroom/cli/capture.py tests/test_network_diff_capture.py` - `ruff format --check headroom/capture headroom/cli/capture.py tests/test_network_diff_capture.py` - `C:\git\headroom\.venv\Scripts\python.exe -m mypy headroom/capture/network_diff.py headroom/cli/capture.py` - `docker compose -f docker/differential-network-capture/docker-compose.yml --profile run config` - `docker compose -f docker/differential-network-capture/docker-compose.yml --profile run build claude-direct` - `docker run --rm -e CLAUDE_COMMAND="claude --version" headroom-network-diff-claude-direct:latest` - parsed `.github/workflows/network-diff-capture.yml` with PyYAML and confirmed manual-only trigger Live Claude API capture was not run locally because `ANTHROPIC_API_KEY` is not set in this environment. The workflow can run it manually in GitHub Actions when that secret is present; otherwise it emits a visible skip warning and uploads a skipped artifact. ## Notes - Full pre-commit mypy still fails on unrelated Windows `fcntl` attributes in `headroom/subscription/tracker.py`; the feature commit skipped only that hook after narrow mypy passed for the new modules. - `tests/test_release_workflows.py` has two Windows-local failures because it shells out to a missing Unix/Rust command; unrelated workflow checks in that file passed before those failures. - Motivated by https://github.com/chopratejas/headroom/issues/746#issuecomment-4651276818 / Issue #746. --- .github/workflows/network-diff-capture.yml | 164 +++++++ .gitignore | 1 + .../Dockerfile.runner | 14 + .../docker-compose.yml | 142 +++++++ .../mitm_capture.py | 89 ++++ .../run-claude-lane.sh | 24 ++ headroom/capture/__init__.py | 17 + headroom/capture/network_diff.py | 399 ++++++++++++++++++ headroom/cli/__init__.py | 1 + headroom/cli/capture.py | 81 ++++ headroom/cli/main.py | 1 + mkdocs.yml | 1 + tests/test_network_diff_capture.py | 130 ++++++ wiki/network-diff-capture.md | 65 +++ 14 files changed, 1129 insertions(+) create mode 100644 .github/workflows/network-diff-capture.yml create mode 100644 docker/differential-network-capture/Dockerfile.runner create mode 100644 docker/differential-network-capture/docker-compose.yml create mode 100644 docker/differential-network-capture/mitm_capture.py create mode 100644 docker/differential-network-capture/run-claude-lane.sh create mode 100644 headroom/capture/__init__.py create mode 100644 headroom/capture/network_diff.py create mode 100644 headroom/cli/capture.py create mode 100644 tests/test_network_diff_capture.py create mode 100644 wiki/network-diff-capture.md diff --git a/.github/workflows/network-diff-capture.yml b/.github/workflows/network-diff-capture.yml new file mode 100644 index 000000000..f5a90200b --- /dev/null +++ b/.github/workflows/network-diff-capture.yml @@ -0,0 +1,164 @@ +name: Network Diff Capture + +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: network-diff-capture-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +env: + PY_VERSION: "3.12" + +jobs: + offline: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-python@v6 + with: + python-version: ${{ env.PY_VERSION }} + + - name: Install offline test tools + run: | + python -m pip install --upgrade pip + python -m pip install \ + 'tiktoken>=0.5.0' \ + 'pydantic>=2.0.0' \ + 'litellm==1.82.3' \ + 'click>=8.1.0' \ + 'rich>=13.0.0' \ + 'opentelemetry-api>=1.24.0' \ + 'ast-grep-cli>=0.30.0' \ + 'fastapi>=0.100.0' \ + 'uvicorn>=0.23.0' \ + 'httpx[http2]>=0.24.0' \ + 'openai>=2.14.0' \ + 'mcp>=1.0.0' \ + 'magika>=0.6.0' \ + 'zstandard>=0.20.0' \ + 'websockets>=13.0' \ + 'onnxruntime>=1.16.0' \ + 'transformers>=4.30.0' \ + 'watchdog>=4.0.0' \ + 'sqlite-vec>=0.1.6' \ + pytest ruff mypy + + - name: Lint capture code + run: ruff check headroom/capture headroom/cli/capture.py tests/test_network_diff_capture.py + + - name: Format check capture code + run: ruff format --check headroom/capture headroom/cli/capture.py tests/test_network_diff_capture.py + + - name: Type-check capture code + run: mypy headroom/capture/network_diff.py headroom/cli/capture.py + + - name: Run capture tests + run: python -m pytest tests/test_network_diff_capture.py + + - name: Validate compose model + env: + ANTHROPIC_API_KEY: dummy + run: docker compose -f docker/differential-network-capture/docker-compose.yml --profile run config + + - name: Build Claude Code runner image + env: + ANTHROPIC_API_KEY: dummy + run: docker compose -f docker/differential-network-capture/docker-compose.yml --profile run build claude-direct + + - name: Smoke Claude Code runner image + run: docker run --rm -e CLAUDE_COMMAND="claude --version" headroom-network-diff-claude-direct:latest + + live-anthropic: + runs-on: ubuntu-latest + timeout-minutes: 90 + needs: offline + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + CLAUDE_PROMPT: "Summarize this repository in one sentence. Keep the answer under 30 words." + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-python@v6 + with: + python-version: ${{ env.PY_VERSION }} + + - name: Install report dependencies + run: | + python -m pip install --upgrade pip + python -m pip install \ + 'tiktoken>=0.5.0' \ + 'pydantic>=2.0.0' \ + 'litellm==1.82.3' \ + 'click>=8.1.0' \ + 'rich>=13.0.0' \ + 'opentelemetry-api>=1.24.0' \ + 'ast-grep-cli>=0.30.0' \ + 'fastapi>=0.100.0' \ + 'uvicorn>=0.23.0' \ + 'httpx[http2]>=0.24.0' \ + 'openai>=2.14.0' \ + 'mcp>=1.0.0' \ + 'magika>=0.6.0' \ + 'zstandard>=0.20.0' \ + 'websockets>=13.0' \ + 'onnxruntime>=1.16.0' \ + 'transformers>=4.30.0' \ + 'watchdog>=4.0.0' \ + 'sqlite-vec>=0.1.6' + + - name: Run live Claude Code differential capture + if: env.ANTHROPIC_API_KEY != '' + working-directory: docker/differential-network-capture + run: | + set -euo pipefail + mkdir -p captures + docker compose up -d --build mitm-direct mitm-headroom-upstream headroom-proxy mitm-headroom-client + trap 'docker compose --profile run down -v' EXIT + + for i in $(seq 1 90); do + if docker compose exec -T headroom-proxy curl --fail --silent http://127.0.0.1:8787/readyz >/dev/null; then + break + fi + if [ "$i" -eq 90 ]; then + docker compose logs headroom-proxy + exit 1 + fi + sleep 2 + done + + docker compose --profile run run --rm claude-direct + docker compose --profile run run --rm claude-headroom + + - name: Report skipped live capture + if: env.ANTHROPIC_API_KEY == '' + run: | + echo "::warning title=Live network diff skipped::ANTHROPIC_API_KEY is not configured for this repository; offline harness checks ran, but live Claude Code capture was skipped." + mkdir -p docker/differential-network-capture/captures + cat > docker/differential-network-capture/captures/skipped.md <<'EOF' + # Live Network Diff Capture Skipped + + `ANTHROPIC_API_KEY` is not configured for this repository. + EOF + + - name: Generate network diff report + if: env.ANTHROPIC_API_KEY != '' + run: | + python -m headroom.cli capture network-diff \ + --direct docker/differential-network-capture/captures/direct.jsonl \ + --headroom docker/differential-network-capture/captures/headroom-client.jsonl \ + --output docker/differential-network-capture/captures/report.md \ + --json-output docker/differential-network-capture/captures/report.json + + - name: Upload capture artifacts + if: always() + uses: actions/upload-artifact@v7 + with: + name: network-diff-capture-${{ github.run_number }} + path: docker/differential-network-capture/captures/ + if-no-files-found: warn diff --git a/.gitignore b/.gitignore index 746f9b458..7adbfffe6 100644 --- a/.gitignore +++ b/.gitignore @@ -194,6 +194,7 @@ headroom.db headroom_*.db *.jsonl !tests/fixtures/*.jsonl +docker/differential-network-capture/captures/ # Documentation build docs/_build/ diff --git a/docker/differential-network-capture/Dockerfile.runner b/docker/differential-network-capture/Dockerfile.runner new file mode 100644 index 000000000..c3d2e368e --- /dev/null +++ b/docker/differential-network-capture/Dockerfile.runner @@ -0,0 +1,14 @@ +FROM node:24-bookworm-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl git python3 \ + && rm -rf /var/lib/apt/lists/* + +ARG CLAUDE_CODE_PACKAGE=@anthropic-ai/claude-code +RUN npm install -g "${CLAUDE_CODE_PACKAGE}" + +WORKDIR /workspace +COPY run-claude-lane.sh /usr/local/bin/run-claude-lane +RUN chmod +x /usr/local/bin/run-claude-lane + +ENTRYPOINT ["run-claude-lane"] diff --git a/docker/differential-network-capture/docker-compose.yml b/docker/differential-network-capture/docker-compose.yml new file mode 100644 index 000000000..9e0ce30b6 --- /dev/null +++ b/docker/differential-network-capture/docker-compose.yml @@ -0,0 +1,142 @@ +name: headroom-network-diff + +services: + mitm-direct: + image: mitmproxy/mitmproxy:12 + command: + - mitmdump + - --mode + - regular + - --listen-host + - 0.0.0.0 + - --listen-port + - "8080" + - --set + - confdir=/mitmproxy + - -s + - /capture/mitm_capture.py + environment: + CAPTURE_LANE: direct + CAPTURE_OUTPUT: /captures/direct.jsonl + CAPTURE_INCLUDE_HOSTS: ${CAPTURE_INCLUDE_HOSTS:-api.anthropic.com} + CAPTURE_BODY_BYTES: ${CAPTURE_BODY_BYTES:-262144} + volumes: + - ./mitm_capture.py:/capture/mitm_capture.py:ro + - ./captures:/captures + - mitm_direct_ca:/mitmproxy + ports: + - "${DIRECT_MITM_PORT:-18080}:8080" + + mitm-headroom-client: + image: mitmproxy/mitmproxy:12 + command: + - mitmdump + - --mode + - reverse:http://headroom-proxy:8787 + - --listen-host + - 0.0.0.0 + - --listen-port + - "8080" + - -s + - /capture/mitm_capture.py + environment: + CAPTURE_LANE: headroom-client + CAPTURE_OUTPUT: /captures/headroom-client.jsonl + CAPTURE_INCLUDE_HOSTS: ${CAPTURE_CLIENT_INCLUDE_HOSTS:-mitm-headroom-client,headroom-proxy,api.anthropic.com} + CAPTURE_BODY_BYTES: ${CAPTURE_BODY_BYTES:-262144} + volumes: + - ./mitm_capture.py:/capture/mitm_capture.py:ro + - ./captures:/captures + ports: + - "${HEADROOM_CLIENT_MITM_PORT:-18082}:8080" + depends_on: + - headroom-proxy + + mitm-headroom-upstream: + image: mitmproxy/mitmproxy:12 + command: + - mitmdump + - --mode + - regular + - --listen-host + - 0.0.0.0 + - --listen-port + - "8080" + - --set + - confdir=/mitmproxy + - -s + - /capture/mitm_capture.py + environment: + CAPTURE_LANE: headroom-upstream + CAPTURE_OUTPUT: /captures/headroom-upstream.jsonl + CAPTURE_INCLUDE_HOSTS: ${CAPTURE_INCLUDE_HOSTS:-api.anthropic.com} + CAPTURE_BODY_BYTES: ${CAPTURE_BODY_BYTES:-262144} + volumes: + - ./mitm_capture.py:/capture/mitm_capture.py:ro + - ./captures:/captures + - mitm_headroom_ca:/mitmproxy + ports: + - "${HEADROOM_MITM_PORT:-18081}:8080" + + headroom-proxy: + build: + context: ../.. + dockerfile: Dockerfile + command: ["--host", "0.0.0.0", "--port", "8787", "--backend", "anthropic"] + environment: + ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:?Set ANTHROPIC_API_KEY} + ANTHROPIC_TARGET_API_URL: ${ANTHROPIC_TARGET_API_URL:-https://api.anthropic.com} + HTTPS_PROXY: http://mitm-headroom-upstream:8080 + HTTP_PROXY: http://mitm-headroom-upstream:8080 + NO_PROXY: 127.0.0.1,localhost,headroom-proxy + REQUESTS_CA_BUNDLE: /mitmproxy/mitmproxy-ca-cert.pem + SSL_CERT_FILE: /mitmproxy/mitmproxy-ca-cert.pem + volumes: + - mitm_headroom_ca:/mitmproxy:ro + depends_on: + - mitm-headroom-upstream + + claude-direct: + build: + context: . + dockerfile: Dockerfile.runner + args: + CLAUDE_CODE_PACKAGE: ${CLAUDE_CODE_PACKAGE:-@anthropic-ai/claude-code} + profiles: ["run"] + environment: + ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:?Set ANTHROPIC_API_KEY} + HTTPS_PROXY: http://mitm-direct:8080 + HTTP_PROXY: http://mitm-direct:8080 + NO_PROXY: 127.0.0.1,localhost + NODE_EXTRA_CA_CERTS: /mitmproxy/mitmproxy-ca-cert.pem + SSL_CERT_FILE: /mitmproxy/mitmproxy-ca-cert.pem + CLAUDE_LANE: direct + CLAUDE_PROMPT: ${CLAUDE_PROMPT:-Summarize this repository in one sentence.} + CLAUDE_ARGS: ${CLAUDE_DIRECT_ARGS:-} + volumes: + - ../..:/workspace:ro + - mitm_direct_ca:/mitmproxy:ro + depends_on: + - mitm-direct + + claude-headroom: + build: + context: . + dockerfile: Dockerfile.runner + args: + CLAUDE_CODE_PACKAGE: ${CLAUDE_CODE_PACKAGE:-@anthropic-ai/claude-code} + profiles: ["run"] + environment: + ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:?Set ANTHROPIC_API_KEY} + ANTHROPIC_BASE_URL: http://mitm-headroom-client:8080 + CLAUDE_LANE: headroom + CLAUDE_PROMPT: ${CLAUDE_PROMPT:-Summarize this repository in one sentence.} + CLAUDE_ARGS: ${CLAUDE_HEADROOM_ARGS:-} + volumes: + - ../..:/workspace:ro + depends_on: + - mitm-headroom-client + +volumes: + mitm_direct_ca: + mitm_headroom_ca: diff --git a/docker/differential-network-capture/mitm_capture.py b/docker/differential-network-capture/mitm_capture.py new file mode 100644 index 000000000..c8e151a62 --- /dev/null +++ b/docker/differential-network-capture/mitm_capture.py @@ -0,0 +1,89 @@ +"""mitmproxy addon that writes sanitized HTTP exchanges as JSONL.""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import os +import time +from pathlib import Path +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +from mitmproxy import http + +LANE = os.environ.get("CAPTURE_LANE", "unknown") +OUTPUT = Path(os.environ.get("CAPTURE_OUTPUT", f"/captures/{LANE}.jsonl")) +INCLUDE_HOSTS = { + host.strip().lower() + for host in os.environ.get("CAPTURE_INCLUDE_HOSTS", "api.anthropic.com").split(",") + if host.strip() +} +BODY_BYTES = int(os.environ.get("CAPTURE_BODY_BYTES", "262144")) +SENSITIVE_HEADER_PARTS = ("authorization", "api-key", "apikey", "token", "secret", "cookie") +SENSITIVE_QUERY_PARTS = ("key", "token", "secret", "signature", "code") +_sequence = 0 + + +def _redact_headers(headers: http.Headers) -> dict[str, str]: + result: dict[str, str] = {} + for key, value in headers.items(multi=True): + if any(part in key.lower() for part in SENSITIVE_HEADER_PARTS): + result[key] = "" + else: + result[key] = value + return result + + +def _sanitize_url(url: str) -> str: + parsed = urlsplit(url) + pairs = [] + for key, value in parse_qsl(parsed.query, keep_blank_values=True): + if any(part in key.lower() for part in SENSITIVE_QUERY_PARTS): + pairs.append((key, "")) + else: + pairs.append((key, value)) + return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, urlencode(pairs), "")) + + +def _request_json(content: bytes) -> object | None: + try: + return json.loads(content.decode("utf-8")) + except Exception: + return None + + +def response(flow: http.HTTPFlow) -> None: + global _sequence + host = flow.request.pretty_host.lower() + if INCLUDE_HOSTS and host not in INCLUDE_HOSTS: + return + + _sequence += 1 + request_body = flow.request.raw_content or b"" + response_body = flow.response.raw_content if flow.response else b"" + record = { + "lane": LANE, + "sequence": _sequence, + "timestamp": time.time(), + "method": flow.request.method, + "url": _sanitize_url(flow.request.pretty_url), + "host": flow.request.pretty_host, + "request_headers": _redact_headers(flow.request.headers), + "request_body_size": len(request_body), + "request_body_sha256": hashlib.sha256(request_body).hexdigest() if request_body else None, + "request_body_b64": base64.b64encode(request_body[:BODY_BYTES]).decode("ascii"), + "request_body_truncated": len(request_body) > BODY_BYTES, + "request_json": _request_json(request_body), + "response_status": flow.response.status_code if flow.response else None, + "response_headers": _redact_headers(flow.response.headers) if flow.response else {}, + "response_body_size": len(response_body), + "response_body_sha256": hashlib.sha256(response_body).hexdigest() + if response_body + else None, + } + + OUTPUT.parent.mkdir(parents=True, exist_ok=True) + with OUTPUT.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(record, separators=(",", ":"), sort_keys=True)) + fh.write("\n") diff --git a/docker/differential-network-capture/run-claude-lane.sh b/docker/differential-network-capture/run-claude-lane.sh new file mode 100644 index 000000000..2b011830a --- /dev/null +++ b/docker/differential-network-capture/run-claude-lane.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env sh +set -eu + +echo "running Claude Code lane: ${CLAUDE_LANE:-unknown}" >&2 + +if [ -n "${NODE_EXTRA_CA_CERTS:-}" ]; then + i=0 + while [ ! -f "$NODE_EXTRA_CA_CERTS" ] && [ "$i" -lt 100 ]; do + i=$((i + 1)) + sleep 0.1 + done +fi + +if [ -n "${CLAUDE_COMMAND:-}" ]; then + sh -lc "$CLAUDE_COMMAND" + exit $? +fi + +if [ -n "${CLAUDE_ARGS:-}" ]; then + # shellcheck disable=SC2086 + claude ${CLAUDE_ARGS} -p "${CLAUDE_PROMPT:-Summarize this repository in one sentence.}" +else + claude -p "${CLAUDE_PROMPT:-Summarize this repository in one sentence.}" +fi diff --git a/headroom/capture/__init__.py b/headroom/capture/__init__.py new file mode 100644 index 000000000..228afbe38 --- /dev/null +++ b/headroom/capture/__init__.py @@ -0,0 +1,17 @@ +"""Network capture comparison helpers.""" + +from .network_diff import ( + CapturedExchange, + CaptureDiff, + compare_captures, + load_capture_file, + render_markdown_report, +) + +__all__ = [ + "CaptureDiff", + "CapturedExchange", + "compare_captures", + "load_capture_file", + "render_markdown_report", +] diff --git a/headroom/capture/network_diff.py b/headroom/capture/network_diff.py new file mode 100644 index 000000000..29b31f4c8 --- /dev/null +++ b/headroom/capture/network_diff.py @@ -0,0 +1,399 @@ +"""Differential network capture reporting for Claude Code vs Headroom. + +The capture format is intentionally JSONL so mitmproxy addons, tests, and +future packet capture tools can all produce the same records without a heavy +dependency in the Headroom package. +""" + +from __future__ import annotations + +import base64 +import hashlib +import json +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +SENSITIVE_HEADER_PARTS = ("authorization", "api-key", "apikey", "token", "secret", "cookie") +SENSITIVE_QUERY_PARTS = ("key", "token", "secret", "signature", "code") +MAX_BODY_PREVIEW_CHARS = 1200 + + +@dataclass(frozen=True) +class CapturedExchange: + """A sanitized HTTP request/response pair captured by the harness.""" + + lane: str + sequence: int + method: str + url: str + host: str + path: str + request_headers: dict[str, str] = field(default_factory=dict) + response_status: int | None = None + response_headers: dict[str, str] = field(default_factory=dict) + request_body_sha256: str | None = None + request_body_size: int = 0 + request_json: Any | None = None + request_body_preview: str | None = None + + @property + def route_key(self) -> str: + return f"{self.method.upper()} {self.host}{self.path}" + + @property + def path_key(self) -> str: + return f"{self.method.upper()} {self.path}" + + +@dataclass(frozen=True) +class CaptureDiff: + """Comparison result between a direct lane and a Headroom lane.""" + + direct_count: int + headroom_count: int + only_direct: list[str] + only_headroom: list[str] + paired: list[dict[str, Any]] + generated_at: str + + def to_dict(self) -> dict[str, Any]: + return { + "generated_at": self.generated_at, + "direct_count": self.direct_count, + "headroom_count": self.headroom_count, + "only_direct": self.only_direct, + "only_headroom": self.only_headroom, + "paired": self.paired, + } + + +def _redact_value(value: object) -> str: + if value is None: + return "" + text = str(value) + if not text: + return text + return "" + + +def sanitize_headers(headers: dict[str, Any] | None) -> dict[str, str]: + sanitized: dict[str, str] = {} + for key, value in (headers or {}).items(): + lower = str(key).lower() + if any(part in lower for part in SENSITIVE_HEADER_PARTS): + sanitized[str(key)] = _redact_value(value) + else: + sanitized[str(key)] = str(value) + return sanitized + + +def sanitize_url(url: str) -> str: + parsed = urlsplit(url) + pairs = [] + for key, value in parse_qsl(parsed.query, keep_blank_values=True): + if any(part in key.lower() for part in SENSITIVE_QUERY_PARTS): + pairs.append((key, "")) + else: + pairs.append((key, value)) + query = urlencode(pairs, doseq=True) + return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, query, "")) + + +def _body_bytes(record: dict[str, Any]) -> bytes: + body_b64 = record.get("request_body_b64") + if isinstance(body_b64, str): + try: + return base64.b64decode(body_b64, validate=True) + except Exception: + return b"" + body = record.get("request_body") + if isinstance(body, str): + return body.encode("utf-8", errors="replace") + return b"" + + +def _parse_json_body(body: bytes) -> Any | None: + if not body: + return None + try: + return json.loads(body.decode("utf-8")) + except Exception: + return None + + +def _preview_body(body: bytes) -> str | None: + if not body: + return None + text = body[:MAX_BODY_PREVIEW_CHARS].decode("utf-8", errors="replace") + return text.replace("\r\n", "\n") + + +def exchange_from_record( + record: dict[str, Any], *, fallback_lane: str, sequence: int +) -> CapturedExchange: + url = sanitize_url(str(record.get("url") or "")) + parsed = urlsplit(url) + path = parsed.path or "/" + if parsed.query: + path = f"{path}?{parsed.query}" + body = _body_bytes(record) + request_json = record.get("request_json") + if request_json is None: + request_json = _parse_json_body(body) + body_sha = record.get("request_body_sha256") + if body_sha is None and body: + body_sha = hashlib.sha256(body).hexdigest() + return CapturedExchange( + lane=str(record.get("lane") or fallback_lane), + sequence=int(record.get("sequence") or sequence), + method=str(record.get("method") or "GET").upper(), + url=url, + host=parsed.netloc or str(record.get("host") or ""), + path=path, + request_headers=sanitize_headers(record.get("request_headers")), + response_status=record.get("response_status"), + response_headers=sanitize_headers(record.get("response_headers")), + request_body_sha256=str(body_sha) if body_sha else None, + request_body_size=int(record.get("request_body_size") or len(body)), + request_json=request_json, + request_body_preview=_preview_body(body), + ) + + +def load_capture_file(path: str | Path, *, fallback_lane: str) -> list[CapturedExchange]: + """Load a JSONL capture file produced by the mitmproxy addon.""" + + exchanges: list[CapturedExchange] = [] + capture_path = Path(path) + for line_number, line in enumerate(capture_path.read_text(encoding="utf-8").splitlines(), 1): + if not line.strip(): + continue + record = json.loads(line) + exchanges.append( + exchange_from_record(record, fallback_lane=fallback_lane, sequence=line_number) + ) + return exchanges + + +def _json_paths(value: Any, prefix: str = "$") -> dict[str, Any]: + if isinstance(value, dict): + paths: dict[str, Any] = {} + for key, child in sorted(value.items()): + paths.update(_json_paths(child, f"{prefix}.{key}")) + return paths or {prefix: {}} + if isinstance(value, list): + paths = {} + for index, child in enumerate(value): + paths.update(_json_paths(child, f"{prefix}[{index}]")) + return paths or {prefix: []} + return {prefix: value} + + +def _header_delta( + direct: dict[str, str], headroom: dict[str, str] +) -> tuple[list[str], list[str], list[str]]: + direct_keys = {key.lower(): key for key in direct} + headroom_keys = {key.lower(): key for key in headroom} + only_direct = sorted(direct_keys[key] for key in set(direct_keys) - set(headroom_keys)) + only_headroom = sorted(headroom_keys[key] for key in set(headroom_keys) - set(direct_keys)) + changed: list[str] = [] + for lower in sorted(set(direct_keys) & set(headroom_keys)): + d_key = direct_keys[lower] + h_key = headroom_keys[lower] + if direct[d_key] != headroom[h_key]: + changed.append(d_key) + return only_direct, only_headroom, changed + + +def _header_value(headers: dict[str, str], name: str) -> str | None: + target = name.lower() + for key, value in headers.items(): + if key.lower() == target: + return value + return None + + +def _anthropic_request_summary(exchange: CapturedExchange) -> dict[str, Any]: + request_json = exchange.request_json if isinstance(exchange.request_json, dict) else {} + tools = request_json.get("tools") + tool_count = len(tools) if isinstance(tools, list) else 0 + tool_bytes = ( + len(json.dumps(tools, sort_keys=True, separators=(",", ":")).encode("utf-8")) + if isinstance(tools, list) + else 0 + ) + return { + "anthropic_beta": _header_value(exchange.request_headers, "anthropic-beta"), + "tools_count": tool_count, + "tools_bytes": tool_bytes, + } + + +def _pair_exchanges( + direct: list[CapturedExchange], headroom: list[CapturedExchange], *, pair_by: str = "path" +) -> tuple[list[tuple[CapturedExchange, CapturedExchange]], list[str], list[str]]: + direct_by_key: dict[str, list[CapturedExchange]] = {} + headroom_by_key: dict[str, list[CapturedExchange]] = {} + for item in direct: + key = item.route_key if pair_by == "route" else item.path_key + direct_by_key.setdefault(key, []).append(item) + for item in headroom: + key = item.route_key if pair_by == "route" else item.path_key + headroom_by_key.setdefault(key, []).append(item) + + pairs: list[tuple[CapturedExchange, CapturedExchange]] = [] + only_direct: list[str] = [] + only_headroom: list[str] = [] + for key in sorted(set(direct_by_key) | set(headroom_by_key)): + direct_items = direct_by_key.get(key, []) + headroom_items = headroom_by_key.get(key, []) + shared = min(len(direct_items), len(headroom_items)) + pairs.extend(zip(direct_items[:shared], headroom_items[:shared], strict=False)) + only_direct.extend([item.route_key for item in direct_items[shared:]]) + only_headroom.extend([item.route_key for item in headroom_items[shared:]]) + return pairs, only_direct, only_headroom + + +def compare_captures( + direct: list[CapturedExchange], headroom: list[CapturedExchange], *, pair_by: str = "path" +) -> CaptureDiff: + pairs, only_direct, only_headroom = _pair_exchanges(direct, headroom, pair_by=pair_by) + paired: list[dict[str, Any]] = [] + for direct_item, headroom_item in pairs: + direct_paths = ( + _json_paths(direct_item.request_json) if direct_item.request_json is not None else {} + ) + headroom_paths = ( + _json_paths(headroom_item.request_json) + if headroom_item.request_json is not None + else {} + ) + only_direct_json = sorted(set(direct_paths) - set(headroom_paths)) + only_headroom_json = sorted(set(headroom_paths) - set(direct_paths)) + changed_json = sorted( + path + for path in set(direct_paths) & set(headroom_paths) + if direct_paths[path] != headroom_paths[path] + ) + headers_only_direct, headers_only_headroom, headers_changed = _header_delta( + direct_item.request_headers, headroom_item.request_headers + ) + paired.append( + { + "route": direct_item.route_key, + "headroom_route": headroom_item.route_key, + "direct_sequence": direct_item.sequence, + "headroom_sequence": headroom_item.sequence, + "status": { + "direct": direct_item.response_status, + "headroom": headroom_item.response_status, + }, + "request_body_size": { + "direct": direct_item.request_body_size, + "headroom": headroom_item.request_body_size, + "delta": headroom_item.request_body_size - direct_item.request_body_size, + }, + "request_body_sha256": { + "direct": direct_item.request_body_sha256, + "headroom": headroom_item.request_body_sha256, + "same": direct_item.request_body_sha256 == headroom_item.request_body_sha256, + }, + "anthropic": { + "direct": _anthropic_request_summary(direct_item), + "headroom": _anthropic_request_summary(headroom_item), + }, + "headers": { + "only_direct": headers_only_direct, + "only_headroom": headers_only_headroom, + "changed": headers_changed, + }, + "json": { + "only_direct": only_direct_json, + "only_headroom": only_headroom_json, + "changed": changed_json, + }, + } + ) + return CaptureDiff( + direct_count=len(direct), + headroom_count=len(headroom), + only_direct=only_direct, + only_headroom=only_headroom, + paired=paired, + generated_at=datetime.now(timezone.utc).isoformat(), + ) + + +def _list_or_dash(values: list[str]) -> str: + return ", ".join(values) if values else "-" + + +def render_markdown_report(diff: CaptureDiff) -> str: + lines = [ + "# Differential Network Capture Report", + "", + f"Generated: `{diff.generated_at}`", + "", + "## Summary", + "", + f"- Direct exchanges: `{diff.direct_count}`", + f"- Headroom exchanges: `{diff.headroom_count}`", + f"- Paired exchanges: `{len(diff.paired)}`", + f"- Only direct: `{len(diff.only_direct)}`", + f"- Only Headroom: `{len(diff.only_headroom)}`", + "", + ] + if diff.only_direct: + lines.extend(["## Only Direct", "", *[f"- `{route}`" for route in diff.only_direct], ""]) + if diff.only_headroom: + lines.extend( + ["## Only Headroom", "", *[f"- `{route}`" for route in diff.only_headroom], ""] + ) + + lines.extend( + [ + "## Paired Exchanges", + "", + "| Route | Status | Body Bytes | Body SHA | Header Delta | JSON Delta |", + "| --- | --- | ---: | --- | --- | --- |", + ] + ) + for item in diff.paired: + route = item["route"] + if item.get("headroom_route") and item["headroom_route"] != route: + route = f"{route} -> {item['headroom_route']}" + status = f"{item['status']['direct']} -> {item['status']['headroom']}" + sizes = item["request_body_size"] + body = f"{sizes['direct']} -> {sizes['headroom']} ({sizes['delta']:+})" + sha = "same" if item["request_body_sha256"]["same"] else "changed" + headers = item["headers"] + header_delta = ( + f"+{_list_or_dash(headers['only_headroom'])}; " + f"-{_list_or_dash(headers['only_direct'])}; " + f"changed={_list_or_dash(headers['changed'])}" + ) + json_delta = item["json"] + json_text = ( + f"+{_list_or_dash(json_delta['only_headroom'])}; " + f"-{_list_or_dash(json_delta['only_direct'])}; " + f"changed={_list_or_dash(json_delta['changed'])}" + ) + anthropic = item.get("anthropic", {}) + direct_anthropic = anthropic.get("direct", {}) + headroom_anthropic = anthropic.get("headroom", {}) + tool_delta = headroom_anthropic.get("tools_bytes", 0) - direct_anthropic.get( + "tools_bytes", 0 + ) + json_text = ( + f"{json_text}; tools={direct_anthropic.get('tools_count', 0)}" + f"->{headroom_anthropic.get('tools_count', 0)}" + f" ({tool_delta:+} bytes)" + ) + lines.append( + f"| `{route}` | `{status}` | `{body}` | `{sha}` | {header_delta} | {json_text} |" + ) + lines.append("") + return "\n".join(lines) diff --git a/headroom/cli/__init__.py b/headroom/cli/__init__.py index d4e20bc3c..053555d02 100644 --- a/headroom/cli/__init__.py +++ b/headroom/cli/__init__.py @@ -13,6 +13,7 @@ survives that kind of sys.modules mutation. """ from . import ( # noqa: F401 + capture, evals, init, install, diff --git a/headroom/cli/capture.py b/headroom/cli/capture.py new file mode 100644 index 000000000..96a4c0df0 --- /dev/null +++ b/headroom/cli/capture.py @@ -0,0 +1,81 @@ +"""Network capture and differential report commands.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import click + +from .main import main + + +@main.group("capture") +def capture_group() -> None: + """Capture and compare network traffic for Headroom investigations.""" + + +@capture_group.command("network-diff") +@click.option( + "--direct", + "direct_path", + required=True, + type=click.Path(exists=True, dir_okay=False, path_type=Path), + help="JSONL capture from the direct Claude Code lane.", +) +@click.option( + "--headroom", + "headroom_path", + required=True, + type=click.Path(exists=True, dir_okay=False, path_type=Path), + help="JSONL capture from the Headroom-proxied Claude Code lane.", +) +@click.option( + "--output", + "markdown_output", + type=click.Path(dir_okay=False, path_type=Path), + help="Write a Markdown report to this path. Defaults to stdout.", +) +@click.option( + "--json-output", + type=click.Path(dir_okay=False, path_type=Path), + help="Optional machine-readable JSON diff output.", +) +@click.option( + "--pair-by", + type=click.Choice(["path", "route"]), + default="path", + show_default=True, + help="Pair exchanges by method+path or by method+host+path.", +) +def network_diff( + direct_path: Path, + headroom_path: Path, + markdown_output: Path | None, + json_output: Path | None, + pair_by: str, +) -> None: + """Compare direct and Headroom MITM capture JSONL files.""" + + from headroom.capture.network_diff import ( + compare_captures, + load_capture_file, + render_markdown_report, + ) + + direct = load_capture_file(direct_path, fallback_lane="direct") + headroom = load_capture_file(headroom_path, fallback_lane="headroom") + diff = compare_captures(direct, headroom, pair_by=pair_by) + markdown = render_markdown_report(diff) + + if markdown_output: + markdown_output.parent.mkdir(parents=True, exist_ok=True) + markdown_output.write_text(markdown, encoding="utf-8") + click.echo(f"Wrote Markdown report: {markdown_output}") + else: + click.echo(markdown) + + if json_output: + json_output.parent.mkdir(parents=True, exist_ok=True) + json_output.write_text(json.dumps(diff.to_dict(), indent=2), encoding="utf-8") + click.echo(f"Wrote JSON report: {json_output}") diff --git a/headroom/cli/main.py b/headroom/cli/main.py index e56d20362..40e0a18b4 100644 --- a/headroom/cli/main.py +++ b/headroom/cli/main.py @@ -36,6 +36,7 @@ def main(ctx: click.Context) -> None: def _register_commands() -> None: """Register all subcommand groups.""" from . import ( + capture, # noqa: F401 evals, # noqa: F401 init, # noqa: F401 install, # noqa: F401 diff --git a/mkdocs.yml b/mkdocs.yml index 9b1c6e4cb..dec34fbf7 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -117,6 +117,7 @@ nav: - CLI: cli.md - API: api.md - SDK: sdk.md + - Differential Network Capture: network-diff-capture.md - Metrics & Observability: metrics.md - Error Codes: errors.md - Troubleshooting: troubleshooting.md diff --git a/tests/test_network_diff_capture.py b/tests/test_network_diff_capture.py new file mode 100644 index 000000000..25fb19aaf --- /dev/null +++ b/tests/test_network_diff_capture.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +import base64 +import json +from pathlib import Path + +from click.testing import CliRunner + +from headroom.capture.network_diff import ( + compare_captures, + load_capture_file, + render_markdown_report, +) +from headroom.cli.main import main + + +def _write_jsonl(path: Path, records: list[dict[str, object]]) -> None: + path.write_text("\n".join(json.dumps(record) for record in records) + "\n", encoding="utf-8") + + +def _body(payload: dict[str, object]) -> str: + return base64.b64encode(json.dumps(payload).encode("utf-8")).decode("ascii") + + +def test_network_diff_redacts_and_reports_body_json_deltas(tmp_path: Path) -> None: + direct_path = tmp_path / "direct.jsonl" + headroom_path = tmp_path / "headroom.jsonl" + _write_jsonl( + direct_path, + [ + { + "lane": "direct", + "method": "POST", + "url": "https://api.anthropic.com/v1/messages?api_key=secret", + "request_headers": { + "authorization": "Bearer secret", + "anthropic-version": "2023-06-01", + "anthropic-beta": "deferred-tools", + }, + "request_body_b64": _body( + {"model": "claude", "messages": [{"content": "hi"}], "tools": []} + ), + "response_status": 200, + } + ], + ) + _write_jsonl( + headroom_path, + [ + { + "lane": "headroom", + "method": "POST", + "url": "https://api.anthropic.com/v1/messages?api_key=secret", + "request_headers": { + "authorization": "Bearer other", + "anthropic-version": "2023-06-01", + "x-headroom-mode": "optimize", + }, + "request_body_b64": _body( + { + "model": "claude", + "messages": [{"content": "hello"}], + "metadata": {}, + "tools": [{"name": "ctx_execute", "input_schema": {"type": "object"}}], + } + ), + "response_status": 200, + } + ], + ) + + direct = load_capture_file(direct_path, fallback_lane="direct") + headroom = load_capture_file(headroom_path, fallback_lane="headroom") + + assert direct[0].url == "https://api.anthropic.com/v1/messages?api_key=%3Credacted%3E" + assert direct[0].request_headers["authorization"] == "" + + diff = compare_captures(direct, headroom) + assert diff.direct_count == 1 + assert diff.headroom_count == 1 + paired = diff.paired[0] + assert paired["headers"]["only_headroom"] == ["x-headroom-mode"] + assert "$.metadata" in paired["json"]["only_headroom"] + assert "$.messages[0].content" in paired["json"]["changed"] + assert paired["anthropic"]["direct"]["tools_count"] == 0 + assert paired["anthropic"]["headroom"]["tools_count"] == 1 + + markdown = render_markdown_report(diff) + assert "Differential Network Capture Report" in markdown + assert "POST api.anthropic.com/v1/messages?api_key=%3Credacted%3E" in markdown + assert "tools=0->1" in markdown + + +def test_network_diff_cli_writes_markdown_and_json(tmp_path: Path) -> None: + direct_path = tmp_path / "direct.jsonl" + headroom_path = tmp_path / "headroom.jsonl" + markdown_path = tmp_path / "report.md" + json_path = tmp_path / "report.json" + record = { + "method": "POST", + "url": "https://api.anthropic.com/v1/messages", + "request_headers": {}, + "request_body_b64": _body({"model": "claude"}), + "response_status": 200, + } + _write_jsonl(direct_path, [record]) + _write_jsonl(headroom_path, [record]) + + result = CliRunner().invoke( + main, + [ + "capture", + "network-diff", + "--direct", + str(direct_path), + "--headroom", + str(headroom_path), + "--output", + str(markdown_path), + "--json-output", + str(json_path), + ], + ) + + assert result.exit_code == 0, result.output + assert "Wrote Markdown report" in result.output + assert "Differential Network Capture Report" in markdown_path.read_text(encoding="utf-8") + payload = json.loads(json_path.read_text(encoding="utf-8")) + assert payload["direct_count"] == 1 + assert payload["headroom_count"] == 1 diff --git a/wiki/network-diff-capture.md b/wiki/network-diff-capture.md new file mode 100644 index 000000000..073133814 --- /dev/null +++ b/wiki/network-diff-capture.md @@ -0,0 +1,65 @@ +# Differential Network Capture + +Headroom includes a containerized harness for comparing Claude Code traffic sent +directly to Anthropic with traffic sent through a Headroom proxy. The harness +uses mitmproxy in two isolated lanes, writes sanitized JSONL captures, and then +generates Markdown/JSON reports with request route, header, body size, body hash, +and JSON payload differences. + +## Run The Harness + +```bash +cd docker/differential-network-capture +mkdir -p captures +export ANTHROPIC_API_KEY=... +export CLAUDE_PROMPT="Summarize this repository in one sentence." +docker compose up --build mitm-direct mitm-headroom-upstream headroom-proxy mitm-headroom-client +docker compose --profile run run --rm claude-direct +docker compose --profile run run --rm claude-headroom +``` + +The primary captures are written to: + +- `docker/differential-network-capture/captures/direct.jsonl` +- `docker/differential-network-capture/captures/headroom-client.jsonl` + +The Headroom lane also writes +`docker/differential-network-capture/captures/headroom-upstream.jsonl`, which is +the request Headroom forwards to Anthropic after proxy processing. + +By default only `api.anthropic.com` is logged. Override +`CAPTURE_INCLUDE_HOSTS` with a comma-separated list to include other hosts. + +## Generate A Report + +```bash +headroom capture network-diff \ + --direct docker/differential-network-capture/captures/direct.jsonl \ + --headroom docker/differential-network-capture/captures/headroom-client.jsonl \ + --output docker/differential-network-capture/captures/report.md \ + --json-output docker/differential-network-capture/captures/report.json +``` + +The report redacts sensitive header values and sensitive query values before +comparison. Request bodies are captured so structural payload differences can be +identified; keep the generated `captures/` directory out of commits because it +may contain prompts, tool outputs, and repository context. + +For Claude Code deferred-tool investigations, the paired exchange table includes +top-level Anthropic `tools` counts and serialized tool bytes. A jump from +`tools=0->N` in the Headroom client lane is evidence that Claude Code eagerly +materialized tool schemas before the request reached Headroom. + +## Custom Claude Invocation + +Set `CLAUDE_COMMAND` to run the exact command under test in both lanes: + +```bash +CLAUDE_COMMAND='claude -p "read README.md and summarize the proxy setup"' \ + docker compose --profile run run --rm claude-direct +CLAUDE_COMMAND='claude -p "read README.md and summarize the proxy setup"' \ + docker compose --profile run run --rm claude-headroom +``` + +Use `CLAUDE_DIRECT_ARGS` and `CLAUDE_HEADROOM_ARGS` when each lane needs +different flags.