feat(perf): add --format {text,json,csv} to headroom perf (#648)

* feat(perf): add structured summary/record builders to analyzer parse_log_files() already returns a fully-structured PerfReport, but the only way to read it was the colored text report. Add reusable machine-readable views so CI guards, dashboards, and agent harnesses can consume perf data without scraping ANSI text: - build_perf_summary(report) -> dict with the aggregated KPIs (savings_pct, cache_hit_pct, by_model, by_transform, ...), mirroring format_report() numbers exactly. - perf_records_as_dicts(report) -> per-record list for --raw output. - PERF_RECORD_FIELDS: shared column order for CSV/raw consumers. Pure additions; no behaviour change to existing callers. Part of #595. * feat(perf): add --format {text,json,csv} to headroom perf Adds a machine-readable output path to the perf command (issue #595): - --format json: aggregated summary (default) or, with --raw, a JSON array of per-record dicts. - --format csv: per-model breakdown (default) or, with --raw, one row per PERF record using the shared PERF_RECORD_FIELDS column order. - --format text (default): unchanged human-readable report. Enables CI guards (jq '.savings_pct < 70'), dashboards, and agent wrappers to consume perf data without scraping ANSI text. Closes #595. * test(perf): cover --format json/csv and structured builders Unit tests for build_perf_summary (totals, savings/cache pct, by_model/by_transform, empty-report zero-division guard) and perf_records_as_dicts, plus CliRunner integration tests for --format json, json --raw, csv, csv --raw, the unchanged text default, and rejection of an unknown format. Part of #595. * fix(perf): rename transform loop var to satisfy mypy The structured-summary builder reused `recs` for both the per-model (list[PerfRecord]) and per-transform (list[TransformRecord]) groupings, so mypy flagged the second assignment as an incompatible-type reuse (analyzer.py:704). Rename the transform loop variable to `t_recs` so each loop keeps a single element type. No behavior change. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --------- Co-authored-by: Kumario1 <ramsakal.ipec@gmail.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
2026-08-27 14:17:10 -04:00 · 2026-06-07 00:43:10 -05:00 · 2026-06-07 00:43:10 -05:00 · 9fe4886cf6
commit 9fe4886cf6
parent fb59f83fab
3 changed files with 374 additions and 6 deletions
--- a/headroom/cli/perf.py
+++ b/headroom/cli/perf.py
@ -1,5 +1,9 @@
 """Performance analysis CLI command."""

+import csv
+import io
+import json
+
 import click

 from .main import main
@ -13,7 +17,14 @@ from .main import main
    help="Analyze logs from the last N hours (default: 168 = 7 days)",
 )
@click.option("--raw", is_flag=True, help="Show raw PERF records instead of report")
-def perf(hours: float, raw: bool) -> None:
+@click.option(
+    "--format",
+    "output_format",
+    type=click.Choice(["text", "json", "csv"]),
+    default="text",
+    help="Output format (default: text). json/csv emit machine-readable data.",
+)
+def perf(hours: float, raw: bool, output_format: str) -> None:
    """Analyze proxy performance from logs.

    \b
@ -26,14 +37,59 @@ def perf(hours: float, raw: bool) -> None:

    \b
    Examples:
-        headroom perf                Analyze last 7 days
-        headroom perf --hours 24     Analyze last 24 hours
-        headroom perf --raw          Show raw parsed records
+        headroom perf                      Analyze last 7 days
+        headroom perf --hours 24           Analyze last 24 hours
+        headroom perf --raw                Show raw parsed records
+        headroom perf --format json        Aggregated report as JSON
+        headroom perf --format csv --hours 24 > last-24h.csv
+        headroom perf --format json --raw  Raw records as a JSON array
    """
-    from headroom.perf.analyzer import format_report, parse_log_files
+    from headroom.perf.analyzer import (
+        PERF_RECORD_FIELDS,
+        build_perf_summary,
+        format_report,
+        parse_log_files,
+        perf_records_as_dicts,
+    )

    report = parse_log_files(last_n_hours=hours)

+    if output_format == "json":
+        payload = perf_records_as_dicts(report) if raw else build_perf_summary(report)
+        click.echo(json.dumps(payload, indent=2))
+        return
+
+    if output_format == "csv":
+        buf = io.StringIO()
+        if raw:
+            writer = csv.DictWriter(buf, fieldnames=PERF_RECORD_FIELDS)
+            writer.writeheader()
+            for rec in perf_records_as_dicts(report):
+                row = dict(rec)
+                # Flatten the transforms list for a single CSV cell.
+                row["transforms"] = ",".join(row.get("transforms", []))
+                writer.writerow(row)
+        else:
+            # Non-raw CSV is the per-model breakdown — the most useful tabular
+            # aggregate for spreadsheets and longitudinal charts.
+            summary = build_perf_summary(report)
+            fieldnames = [
+                "model",
+                "requests",
+                "tokens_before",
+                "tokens_after",
+                "tokens_saved",
+                "savings_pct",
+                "list_price_per_mtok",
+            ]
+            writer = csv.DictWriter(buf, fieldnames=fieldnames)
+            writer.writeheader()
+            for row in summary["by_model"]:
+                writer.writerow(row)
+        click.echo(buf.getvalue(), nl=False)
+        return
+
+    # Default: human-readable text.
    if raw:
        for r in report.perf_records:
            click.echo(
--- a/headroom/perf/analyzer.py
+++ b/headroom/perf/analyzer.py
@ -12,7 +12,7 @@ from __future__ import annotations

 import logging
 import re
-from dataclasses import dataclass, field
+from dataclasses import asdict, dataclass, field
 from datetime import datetime, timedelta

 from headroom import paths as _paths
@ -625,6 +625,128 @@ def format_report(report: PerfReport) -> str:
    return "\n".join(lines)


+# ---------------------------------------------------------------------------
+# Machine-readable views (JSON / CSV) — issue #595
+# ---------------------------------------------------------------------------
+#
+# `parse_log_files()` already returns a fully-structured `PerfReport`; these
+# helpers expose it without forcing CI pipelines, dashboards, or agent
+# harnesses to scrape the colored text report. The aggregate numbers mirror
+# `format_report()` exactly so a JSON consumer and a human reading the report
+# never disagree.
+
+# Column order for the per-record (`--raw`) machine output. Kept as a module
+# constant so the CLI's CSV writer and any external consumer share one source
+# of truth.
+PERF_RECORD_FIELDS = [
+    "timestamp",
+    "request_id",
+    "model",
+    "num_messages",
+    "tokens_before",
+    "tokens_after",
+    "tokens_saved",
+    "cache_read",
+    "cache_write",
+    "cache_hit_pct",
+    "optimization_ms",
+    "transforms",
+]
+
+
+def _pct(saved: int, before: int) -> float:
+    """Reduction percentage, rounded to 1dp, guarding divide-by-zero."""
+    return round(saved / before * 100, 1) if before > 0 else 0.0
+
+
+def build_perf_summary(report: PerfReport) -> dict:
+    """Aggregate a ``PerfReport`` into a JSON-serialisable summary dict.
+
+    The shape mirrors the human-readable ``format_report`` numbers so the same
+    data drives CI regression guards (``jq '.savings_pct < 70'``), dashboards,
+    and end-of-session savings summaries in agent wrappers.
+    """
+    records = report.perf_records
+
+    total_before = sum(r.tokens_before for r in records)
+    total_after = sum(r.tokens_after for r in records)
+    total_saved = sum(r.tokens_saved for r in records)
+
+    total_cr = sum(r.cache_read for r in records)
+    total_cw = sum(r.cache_write for r in records)
+    total_cache = total_cr + total_cw
+    cache_hit_pct = round(total_cr / total_cache * 100, 1) if total_cache > 0 else 0.0
+
+    by_model_groups: dict[str, list[PerfRecord]] = {}
+    for r in records:
+        by_model_groups.setdefault(r.model, []).append(r)
+    by_model = []
+    for model, recs in sorted(by_model_groups.items()):
+        m_before = sum(r.tokens_before for r in recs)
+        m_after = sum(r.tokens_after for r in recs)
+        m_saved = sum(r.tokens_saved for r in recs)
+        by_model.append(
+            {
+                "model": model,
+                "requests": len(recs),
+                "tokens_before": m_before,
+                "tokens_after": m_after,
+                "tokens_saved": m_saved,
+                "savings_pct": _pct(m_saved, m_before),
+                "list_price_per_mtok": _get_list_price(model),
+            }
+        )
+
+    by_transform_groups: dict[str, list[TransformRecord]] = {}
+    for tr in report.transform_records:
+        by_transform_groups.setdefault(tr.name, []).append(tr)
+    by_transform = []
+    for name, t_recs in sorted(
+        by_transform_groups.items(), key=lambda kv: -sum(r.tokens_saved for r in kv[1])
+    ):
+        t_before = sum(r.tokens_before for r in t_recs)
+        t_saved = sum(r.tokens_saved for r in t_recs)
+        by_transform.append(
+            {
+                "transform": name,
+                "uses": len(t_recs),
+                "tokens_before": t_before,
+                "tokens_saved": t_saved,
+                "savings_pct": _pct(t_saved, t_before),
+            }
+        )
+
+    return {
+        "window_hours": report.requested_hours,
+        "actual_window": {
+            "oldest": report.oldest_kept_ts,
+            "newest": report.newest_kept_ts,
+        },
+        "records_filtered_out": report.records_filtered_out,
+        "total_requests": len(records),
+        "total_tokens_before": total_before,
+        "total_tokens_after": total_after,
+        "tokens_saved": total_saved,
+        "savings_pct": _pct(total_saved, total_before),
+        "cache_read_tokens": total_cr,
+        "cache_write_tokens": total_cw,
+        "cache_hit_pct": cache_hit_pct,
+        "by_model": by_model,
+        "by_transform": by_transform,
+        "log_files_read": report.log_files_read,
+        "total_lines_parsed": report.total_lines_parsed,
+    }
+
+
+def perf_records_as_dicts(report: PerfReport) -> list[dict]:
+    """Per-record view of the parsed PERF entries (for ``--raw`` machine output).
+
+    ``transforms`` stays a list so JSON consumers keep structure; the CSV
+    writer flattens it to a comma-joined string at the edge.
+    """
+    return [asdict(r) for r in report.perf_records]
+
+
 def _format_toin_highlights() -> list[str]:
    """Render a human-readable TOIN highlights block from the live store.

--- a/tests/test_cli_perf_format.py
+++ b/tests/test_cli_perf_format.py
@ -0,0 +1,190 @@
+"""Tests for `headroom perf --format {text,json,csv}` (issue #595)."""
+
+from __future__ import annotations
+
+import csv
+import io
+import json
+
+import pytest
+from click.testing import CliRunner
+
+from headroom.cli.main import main
+from headroom.perf import analyzer
+from headroom.perf.analyzer import (
+    PerfRecord,
+    PerfReport,
+    TransformRecord,
+    build_perf_summary,
+    perf_records_as_dicts,
+)
+
+
+@pytest.fixture
+def runner() -> CliRunner:
+    return CliRunner()
+
+
+def _sample_report() -> PerfReport:
+    """A small report with two models, cache numbers, and a transform."""
+    return PerfReport(
+        perf_records=[
+            PerfRecord(
+                timestamp="2026-06-05 10:00:00,000",
+                request_id="hr_1",
+                model="claude-sonnet-4.5",
+                num_messages=10,
+                tokens_before=1000,
+                tokens_after=400,
+                tokens_saved=600,
+                cache_read=800,
+                cache_write=200,
+                cache_hit_pct=80,
+                optimization_ms=12.0,
+                transforms=["content_router"],
+            ),
+            PerfRecord(
+                timestamp="2026-06-05 11:00:00,000",
+                request_id="hr_2",
+                model="claude-opus-4-8",
+                num_messages=4,
+                tokens_before=1000,
+                tokens_after=600,
+                tokens_saved=400,
+                cache_read=200,
+                cache_write=0,
+                cache_hit_pct=100,
+                optimization_ms=8.0,
+                transforms=["content_router"],
+            ),
+        ],
+        transform_records=[
+            TransformRecord(
+                timestamp="2026-06-05 10:00:00,000",
+                name="content_router",
+                tokens_before=2000,
+                tokens_after=1000,
+                tokens_saved=1000,
+            ),
+        ],
+        log_files_read=1,
+        total_lines_parsed=42,
+        requested_hours=24.0,
+        oldest_kept_ts="2026-06-05 10:00:00,000",
+        newest_kept_ts="2026-06-05 11:00:00,000",
+    )
+
+
+# ---------------------------------------------------------------------------
+# Pure builders
+# ---------------------------------------------------------------------------
+
+
+def test_build_perf_summary_totals_and_pct():
+    summary = build_perf_summary(_sample_report())
+
+    assert summary["total_requests"] == 2
+    assert summary["total_tokens_before"] == 2000
+    assert summary["total_tokens_after"] == 1000
+    assert summary["tokens_saved"] == 1000
+    # 1000 / 2000 == 50.0%
+    assert summary["savings_pct"] == 50.0
+    # cache: read 1000, write 200 -> 1000 / 1200 == 83.3%
+    assert summary["cache_read_tokens"] == 1000
+    assert summary["cache_write_tokens"] == 200
+    assert summary["cache_hit_pct"] == 83.3
+    assert summary["window_hours"] == 24.0
+
+
+def test_build_perf_summary_by_model_and_transform():
+    summary = build_perf_summary(_sample_report())
+
+    models = {m["model"]: m for m in summary["by_model"]}
+    assert set(models) == {"claude-sonnet-4.5", "claude-opus-4-8"}
+    assert models["claude-sonnet-4.5"]["tokens_saved"] == 600
+    assert models["claude-sonnet-4.5"]["savings_pct"] == 60.0
+    assert models["claude-opus-4-8"]["savings_pct"] == 40.0
+
+    assert summary["by_transform"][0]["transform"] == "content_router"
+    assert summary["by_transform"][0]["tokens_saved"] == 1000
+    assert summary["by_transform"][0]["uses"] == 1
+
+
+def test_build_perf_summary_empty_report_no_zero_division():
+    summary = build_perf_summary(PerfReport(requested_hours=168.0))
+    assert summary["total_requests"] == 0
+    assert summary["savings_pct"] == 0.0
+    assert summary["cache_hit_pct"] == 0.0
+    assert summary["by_model"] == []
+
+
+def test_perf_records_as_dicts_roundtrips_fields():
+    dicts = perf_records_as_dicts(_sample_report())
+    assert len(dicts) == 2
+    assert dicts[0]["request_id"] == "hr_1"
+    assert dicts[0]["tokens_saved"] == 600
+    # transforms stays a list for JSON consumers
+    assert dicts[0]["transforms"] == ["content_router"]
+
+
+# ---------------------------------------------------------------------------
+# CLI integration
+# ---------------------------------------------------------------------------
+
+
+def _patch_report(monkeypatch, report: PerfReport) -> None:
+    monkeypatch.setattr(analyzer, "parse_log_files", lambda last_n_hours=168.0: report)
+
+
+def test_perf_json_format(runner, monkeypatch):
+    _patch_report(monkeypatch, _sample_report())
+    result = runner.invoke(main, ["perf", "--format", "json"])
+    assert result.exit_code == 0, result.output
+    data = json.loads(result.output)
+    assert data["savings_pct"] == 50.0
+    assert "by_model" in data
+    assert data["total_requests"] == 2
+
+
+def test_perf_json_raw_is_array(runner, monkeypatch):
+    _patch_report(monkeypatch, _sample_report())
+    result = runner.invoke(main, ["perf", "--format", "json", "--raw"])
+    assert result.exit_code == 0, result.output
+    data = json.loads(result.output)
+    assert isinstance(data, list)
+    assert len(data) == 2
+    assert data[0]["request_id"] == "hr_1"
+
+
+def test_perf_csv_by_model(runner, monkeypatch):
+    _patch_report(monkeypatch, _sample_report())
+    result = runner.invoke(main, ["perf", "--format", "csv"])
+    assert result.exit_code == 0, result.output
+    rows = list(csv.DictReader(io.StringIO(result.output)))
+    assert {r["model"] for r in rows} == {"claude-sonnet-4.5", "claude-opus-4-8"}
+    sonnet = next(r for r in rows if r["model"] == "claude-sonnet-4.5")
+    assert sonnet["tokens_saved"] == "600"
+
+
+def test_perf_csv_raw_per_record(runner, monkeypatch):
+    _patch_report(monkeypatch, _sample_report())
+    result = runner.invoke(main, ["perf", "--format", "csv", "--raw"])
+    assert result.exit_code == 0, result.output
+    rows = list(csv.DictReader(io.StringIO(result.output)))
+    assert len(rows) == 2
+    assert rows[0]["request_id"] == "hr_1"
+    # transforms flattened to a string cell
+    assert rows[0]["transforms"] == "content_router"
+
+
+def test_perf_text_default_unchanged(runner, monkeypatch):
+    _patch_report(monkeypatch, _sample_report())
+    result = runner.invoke(main, ["perf"])
+    assert result.exit_code == 0, result.output
+    assert "Headroom Performance Report" in result.output
+
+
+def test_perf_rejects_unknown_format(runner, monkeypatch):
+    _patch_report(monkeypatch, _sample_report())
+    result = runner.invoke(main, ["perf", "--format", "xml"])
+    assert result.exit_code != 0