diff --git a/headroom/cli/__init__.py b/headroom/cli/__init__.py index 92b28b33a..529a20d65 100644 --- a/headroom/cli/__init__.py +++ b/headroom/cli/__init__.py @@ -18,6 +18,7 @@ from . import ( # noqa: F401 copilot_auth, evals, init, + inspect, install, learn, mcp, diff --git a/headroom/cli/inspect.py b/headroom/cli/inspect.py new file mode 100644 index 000000000..5fe001221 --- /dev/null +++ b/headroom/cli/inspect.py @@ -0,0 +1,183 @@ +"""`headroom inspect` — view original vs compressed message content. + +Headroom already exposes *quantitative* telemetry (token counts, ratios) but no +way to *see* what the compressor changed. This command reads the proxy's +loopback ``/transformations/feed`` endpoint — which carries the pre/post +message snapshots when the proxy runs with ``--log-messages`` — and renders, per +request, the original vs compressed content for each message with the changed +segments highlighted (stdlib unified diff, no new dependencies). See issue #1267. +""" + +from __future__ import annotations + +import difflib +import json +import os +from typing import Any + +import click + +from .main import main + + +def _extract_text(content: Any) -> str: + """Flatten a message's ``content`` (str or list of blocks) to plain text. + + Handles Anthropic/OpenAI text blocks, nested ``tool_result`` content, and + falls back to a JSON dump for unrecognized block shapes so nothing is + silently dropped from the diff. + """ + if content is None: + return "" + if isinstance(content, str): + return content + if isinstance(content, list): + parts: list[str] = [] + for block in content: + if isinstance(block, str): + parts.append(block) + elif isinstance(block, dict): + if isinstance(block.get("text"), str): + parts.append(block["text"]) + elif isinstance(block.get("content"), (str, list)): + parts.append(_extract_text(block.get("content"))) + else: + parts.append(json.dumps(block, ensure_ascii=False, sort_keys=True)) + else: + parts.append(str(block)) + return "\n".join(parts) + return str(content) + + +def _role(msg: Any) -> str: + return str(msg.get("role", "?")) if isinstance(msg, dict) else "?" + + +def _content_of(msg: Any) -> Any: + return msg.get("content") if isinstance(msg, dict) else msg + + +def _render_request(transformation: dict[str, Any], *, full: bool) -> None: + rid = transformation.get("request_id") or "?" + model = transformation.get("model") or "?" + before = transformation.get("input_tokens_original") + after = transformation.get("input_tokens_optimized") + saved = transformation.get("tokens_saved") + pct = transformation.get("savings_percent") + transforms = ", ".join(transformation.get("transforms_applied") or []) or "none" + + click.echo(click.style(f"\n━━ {rid} {model}", bold=True)) + click.echo(f" tokens {before} → {after} (saved {saved}, {pct}%) transforms: {transforms}") + + originals = transformation.get("request_messages") or [] + compressed = transformation.get("compressed_messages") or [] + count = max(len(originals), len(compressed)) + any_change = False + + for i in range(count): + original = originals[i] if i < len(originals) else {} + comp = compressed[i] if i < len(compressed) else {} + original_text = _extract_text(_content_of(original)) + comp_text = _extract_text(_content_of(comp)) + unchanged = original_text == comp_text + if unchanged and not full: + continue + any_change = True + role = _role(original) if original else _role(comp) + click.echo(click.style(f"\n [{i}] {role}", fg="cyan")) + if unchanged: + click.echo(" (unchanged)") + continue + for line in difflib.unified_diff( + original_text.splitlines(), + comp_text.splitlines(), + fromfile="original", + tofile="compressed", + lineterm="", + ): + color: str | None = None + if line.startswith("+") and not line.startswith("+++"): + color = "green" + elif line.startswith("-") and not line.startswith("---"): + color = "red" + elif line.startswith("@@"): + color = "yellow" + click.echo(" " + (click.style(line, fg=color) if color else line)) + + if not any_change: + click.echo( + " (no per-message content changes — savings came from " + "structural / transform-level edits)" + ) + + +@main.command("inspect") +@click.option( + "--port", + "-p", + default=None, + type=click.IntRange(1, 65535), + envvar="HEADROOM_PORT", + help="Proxy port to query (default: 8787, env: HEADROOM_PORT)", +) +@click.option( + "--last", + default=1, + type=click.IntRange(min=1), + help="Show the N most recent requests (default: 1)", +) +@click.option( + "--format", + "output_format", + type=click.Choice(["text", "json"]), + default="text", + help="Output format (default: text). json emits the raw feed for offline diffing.", +) +@click.option( + "--full", + is_flag=True, + help="Show unchanged messages too (default: only messages the compressor changed)", +) +def inspect_cmd(port: int | None, last: int, output_format: str, full: bool) -> None: + """Show original vs compressed content for recent proxy requests. + + \b + Requires a running proxy started with --log-messages (or --log-file), which + captures the pre/post-compression message snapshots the diff reads. + + \b + Examples: + headroom inspect Inspect the most recent request + headroom inspect --last 5 Inspect the 5 most recent requests + headroom inspect --full Include unchanged messages + headroom inspect --format json Raw feed for piping into another tool + """ + from headroom.install.health import probe_json + + resolved_port = port if port is not None else int(os.environ.get("HEADROOM_PORT", "8787")) + base_url = f"http://127.0.0.1:{resolved_port}" + payload = probe_json(f"{base_url}/transformations/feed?limit={last}", timeout=5.0) + + if payload is None: + raise click.ClickException( + f"No reachable proxy on {base_url}. Start one with `headroom proxy` " + "(or pass --port to point at a running instance)." + ) + if not payload.get("log_full_messages"): + raise click.ClickException( + "The proxy isn't capturing message content, so there's nothing to diff. " + "Restart it with `headroom proxy --log-messages`." + ) + + transformations = payload.get("transformations") or [] + if not transformations: + click.echo("No requests recorded yet. Send traffic through the proxy and retry.") + return + + if output_format == "json": + click.echo(json.dumps(transformations, indent=2, ensure_ascii=False)) + return + + # The feed returns oldest→newest; show the most recent first. + for transformation in reversed(transformations): + _render_request(transformation, full=full) diff --git a/headroom/cli/main.py b/headroom/cli/main.py index 384d23fcb..b55c96e1e 100644 --- a/headroom/cli/main.py +++ b/headroom/cli/main.py @@ -68,6 +68,7 @@ def _register_commands() -> None: doctor, # noqa: F401 evals, # noqa: F401 init, # noqa: F401 + inspect, # noqa: F401 install, # noqa: F401 learn, # noqa: F401 mcp, # noqa: F401 diff --git a/tests/test_cli_inspect.py b/tests/test_cli_inspect.py new file mode 100644 index 000000000..b6ab3a790 --- /dev/null +++ b/tests/test_cli_inspect.py @@ -0,0 +1,105 @@ +"""Tests for the `headroom inspect` command (issue #1267). + +The command reads the proxy's loopback ``/transformations/feed`` endpoint and +renders original-vs-compressed content. Tests stub ``probe_json`` so no proxy +is required. + +Tests invoke the real top-level CLI (``main``) so the shipped command path — +including subcommand registration — is exercised, not just the command object. +""" + +from __future__ import annotations + +import json +from unittest.mock import patch + +from click.testing import CliRunner + +from headroom.cli.inspect import _extract_text, _role + + +def _run(args: list[str]): + from headroom.cli.main import main + + return CliRunner().invoke(main, ["inspect", *args]) + + +def test_extract_text_handles_str_and_blocks() -> None: + assert _extract_text("hello") == "hello" + assert _extract_text([{"type": "text", "text": "a"}, {"type": "text", "text": "b"}]) == "a\nb" + # Nested tool_result content. + assert _extract_text([{"type": "tool_result", "content": [{"text": "x"}]}]) == "x" + # Unknown block falls back to JSON, never silently dropped. + assert "foo" in _extract_text([{"weird": "foo"}]) + assert _extract_text(None) == "" + + +def test_role_extraction() -> None: + assert _role({"role": "user"}) == "user" + assert _role("not a dict") == "?" + + +def test_no_proxy_errors_cleanly() -> None: + with patch("headroom.install.health.probe_json", return_value=None): + result = _run([]) + assert result.exit_code != 0 + assert "No reachable proxy" in result.output + + +def test_log_messages_disabled_hint() -> None: + payload = {"transformations": [], "log_full_messages": False} + with patch("headroom.install.health.probe_json", return_value=payload): + result = _run([]) + assert result.exit_code != 0 + assert "--log-messages" in result.output + + +def test_empty_feed_message() -> None: + payload = {"transformations": [], "log_full_messages": True} + with patch("headroom.install.health.probe_json", return_value=payload): + result = _run([]) + assert result.exit_code == 0 + assert "No requests recorded" in result.output + + +def _feed_payload() -> dict: + return { + "log_full_messages": True, + "transformations": [ + { + "request_id": "req-1", + "model": "gpt-4o", + "input_tokens_original": 100, + "input_tokens_optimized": 40, + "tokens_saved": 60, + "savings_percent": 60.0, + "transforms_applied": ["SmartCrusher"], + "request_messages": [ + {"role": "user", "content": "line one\nline two\nline three"}, + ], + "compressed_messages": [ + {"role": "user", "content": "line one\nline three"}, + ], + } + ], + } + + +def test_text_render_shows_diff_and_header() -> None: + with patch("headroom.install.health.probe_json", return_value=_feed_payload()): + result = _run([]) + assert result.exit_code == 0 + out = result.output + assert "req-1" in out + assert "gpt-4o" in out + assert "SmartCrusher" in out + # The removed line shows up on the original side of the diff. + assert "line two" in out + + +def test_json_format_emits_raw_feed() -> None: + with patch("headroom.install.health.probe_json", return_value=_feed_payload()): + result = _run(["--format", "json"]) + assert result.exit_code == 0 + parsed = json.loads(result.output) + assert parsed[0]["request_id"] == "req-1" diff --git a/wiki/cli.md b/wiki/cli.md index 8dad03ee5..7b7c8cbd1 100644 --- a/wiki/cli.md +++ b/wiki/cli.md @@ -26,6 +26,7 @@ This page is the authoritative reference for the **Python Headroom CLI** exposed | `headroom proxy` | Run the Headroom proxy server | **native in container** | | `headroom learn` | Learn from past tool-call failures | **native in container** | | `headroom perf` | Summarize recent proxy performance | **native in container** | +| `headroom inspect` | Show original vs compressed content for recent requests | **native in container** | | `headroom evals ...` | Run memory evaluation workflows | **native in container** | | `headroom memory ...` | Inspect and manage stored memories | **native in container** | | `headroom mcp ...` | Install, inspect, remove, or serve MCP integration | **native in container** | @@ -330,6 +331,30 @@ The command reads `${HEADROOM_WORKSPACE_DIR}/logs/proxy.log` (defaults to `~/.headroom/logs/proxy.log` — see the [Filesystem Contract](filesystem-contract.md)). +## `headroom inspect` + +Show the original vs compressed content for recent requests so you can *see* +what the compressor changed (not just the token counts). Useful for building +trust in compression and debugging quality regressions. + +```bash +headroom inspect # inspect the most recent request +headroom inspect --last 5 # inspect the 5 most recent requests +headroom inspect --full # include unchanged messages +headroom inspect --format json # raw feed for piping into another tool +``` + +| Option | Default | Meaning | +|---|---|---| +| `--port` / `-p` | `8787` | Proxy port to query (env: `HEADROOM_PORT`) | +| `--last` | `1` | Number of most-recent requests to show | +| `--format` | `text` | `text` renders a highlighted diff; `json` emits the raw feed | +| `--full` | off | Include messages the compressor left unchanged | + +`inspect` queries the running proxy's loopback `/transformations/feed` endpoint, +so the proxy must be started with `--log-messages` (or `--log-file`) for the +pre/post-compression snapshots to be captured. + ## `headroom evals` Memory evaluation command group.