mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-08-10 14:27:00 -04:00
942e916368
## Description
Headroom exposes plenty of *quantitative* compression telemetry (token
counts, ratios, `headroom perf`, `/metrics`) but no way to actually
**see what the compressor changed** in the content. That makes it hard
to trust compression or debug a quality regression ("did it drop
something I cared about?").
This adds a `headroom inspect` command (the issue's Option 1). It reads
the proxy's existing loopback `/transformations/feed` endpoint — which
already carries the pre/post-compression message snapshots when the
proxy runs with `--log-messages` — and renders, per request, the
original vs compressed content for each message with the changed
segments highlighted. No new dependencies (stdlib `difflib`).
```
headroom inspect # inspect the most recent request
headroom inspect --last 5 # the 5 most recent
headroom inspect --full # include unchanged messages
headroom inspect --format json # raw feed for offline tooling
```
Per request it shows the model, per-request token counts + savings, the
transforms applied, and a colorized unified diff of each changed message
(red = removed, green = added). Clear errors when no proxy is reachable
or when the proxy wasn't started with `--log-messages`.
Side-by-side / interactive rendering (the fuller form of Option 1) can
follow as a polish pass; this lands the core "see what changed"
capability on data Headroom already captures.
Closes #1267
## Type of Change
- [x] New feature (non-breaking change that adds functionality)
## Changes Made
- `headroom/cli/inspect.py`: new `inspect` command +
content-flattening/diff-render helpers.
- `headroom/cli/__init__.py`, `headroom/cli/main.py`: register the
command.
- `tests/test_cli_inspect.py`: unit tests (content extraction, the
no-proxy / no-`--log-messages` / empty-feed paths, text render, json
output).
- `wiki/cli.md`: document the command + options.
## Testing
- [x] Unit tests pass (`pytest`)
- [x] Linting passes (`ruff check`)
- [x] New tests added
### Test Output
```text
$ pytest tests/test_cli_inspect.py -q
7 passed
$ ruff check headroom/cli/ tests/test_cli_inspect.py
All checks passed!
```
## Real Behavior Proof
- Environment: repo main @ HEAD, local venv
- Exact command / steps: invoked the `inspect` command against a mocked
`/transformations/feed` payload (one request, a user message with a line
removed by SmartCrusher).
- Observed result: header shows `req-1 gpt-4o`, `tokens 100 → 40 (saved
60, 60.0%)`, `transforms: SmartCrusher`, and a unified diff with the
removed line on the original side; no-proxy and missing-`--log-messages`
cases raise actionable errors; `--format json` emits the raw feed.
- Not tested: live end-to-end against a real proxy with `--log-messages`
(the data source — the feed endpoint — is exercised via the mocked
payload that mirrors its shape).
## Review Readiness
- [x] I have performed a self-review
- [x] This PR is ready for human review
## Checklist
- [x] My code follows the project's style guidelines
- [x] I have performed a self-review of my code
- [x] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [x] I have added tests that prove the feature works
105 lines
3.4 KiB
Python
105 lines
3.4 KiB
Python
"""Tests for the `headroom inspect` command (issue #1267).
|
|
|
|
The command reads the proxy's loopback ``/transformations/feed`` endpoint and
|
|
renders original-vs-compressed content. Tests stub ``probe_json`` so no proxy
|
|
is required.
|
|
|
|
Tests invoke the real top-level CLI (``main``) so the shipped command path —
|
|
including subcommand registration — is exercised, not just the command object.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from unittest.mock import patch
|
|
|
|
from click.testing import CliRunner
|
|
|
|
from headroom.cli.inspect import _extract_text, _role
|
|
|
|
|
|
def _run(args: list[str]):
|
|
from headroom.cli.main import main
|
|
|
|
return CliRunner().invoke(main, ["inspect", *args])
|
|
|
|
|
|
def test_extract_text_handles_str_and_blocks() -> None:
|
|
assert _extract_text("hello") == "hello"
|
|
assert _extract_text([{"type": "text", "text": "a"}, {"type": "text", "text": "b"}]) == "a\nb"
|
|
# Nested tool_result content.
|
|
assert _extract_text([{"type": "tool_result", "content": [{"text": "x"}]}]) == "x"
|
|
# Unknown block falls back to JSON, never silently dropped.
|
|
assert "foo" in _extract_text([{"weird": "foo"}])
|
|
assert _extract_text(None) == ""
|
|
|
|
|
|
def test_role_extraction() -> None:
|
|
assert _role({"role": "user"}) == "user"
|
|
assert _role("not a dict") == "?"
|
|
|
|
|
|
def test_no_proxy_errors_cleanly() -> None:
|
|
with patch("headroom.install.health.probe_json", return_value=None):
|
|
result = _run([])
|
|
assert result.exit_code != 0
|
|
assert "No reachable proxy" in result.output
|
|
|
|
|
|
def test_log_messages_disabled_hint() -> None:
|
|
payload = {"transformations": [], "log_full_messages": False}
|
|
with patch("headroom.install.health.probe_json", return_value=payload):
|
|
result = _run([])
|
|
assert result.exit_code != 0
|
|
assert "--log-messages" in result.output
|
|
|
|
|
|
def test_empty_feed_message() -> None:
|
|
payload = {"transformations": [], "log_full_messages": True}
|
|
with patch("headroom.install.health.probe_json", return_value=payload):
|
|
result = _run([])
|
|
assert result.exit_code == 0
|
|
assert "No requests recorded" in result.output
|
|
|
|
|
|
def _feed_payload() -> dict:
|
|
return {
|
|
"log_full_messages": True,
|
|
"transformations": [
|
|
{
|
|
"request_id": "req-1",
|
|
"model": "gpt-4o",
|
|
"input_tokens_original": 100,
|
|
"input_tokens_optimized": 40,
|
|
"tokens_saved": 60,
|
|
"savings_percent": 60.0,
|
|
"transforms_applied": ["SmartCrusher"],
|
|
"request_messages": [
|
|
{"role": "user", "content": "line one\nline two\nline three"},
|
|
],
|
|
"compressed_messages": [
|
|
{"role": "user", "content": "line one\nline three"},
|
|
],
|
|
}
|
|
],
|
|
}
|
|
|
|
|
|
def test_text_render_shows_diff_and_header() -> None:
|
|
with patch("headroom.install.health.probe_json", return_value=_feed_payload()):
|
|
result = _run([])
|
|
assert result.exit_code == 0
|
|
out = result.output
|
|
assert "req-1" in out
|
|
assert "gpt-4o" in out
|
|
assert "SmartCrusher" in out
|
|
# The removed line shows up on the original side of the diff.
|
|
assert "line two" in out
|
|
|
|
|
|
def test_json_format_emits_raw_feed() -> None:
|
|
with patch("headroom.install.health.probe_json", return_value=_feed_payload()):
|
|
result = _run(["--format", "json"])
|
|
assert result.exit_code == 0
|
|
parsed = json.loads(result.output)
|
|
assert parsed[0]["request_id"] == "req-1"
|