diff --git a/CHANGELOG.md b/CHANGELOG.md index ae6182f54..4ae691eb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Bug Fixes +* **learn (verbosity):** `--verbosity --apply --all` now aggregates the savings baseline across every project instead of overwriting it per project (last-project-wins), which previously left the output shaper with a tiny, unrepresentative baseline. The applied verbosity level comes from the project with the most samples ([#1288](https://github.com/headroomlabs-ai/headroom/pull/1288)). * **proxy/anthropic:** restore token-mode compression on continued Claude Code turns with a frozen prefix and deferred CCR tool injection. Token mode now runs request-side compression even when the client did not pre-register `headroom_retrieve`, relying on the existing marker-triggered injection override to keep emitted CCR markers redeemable ([#1487](https://github.com/headroomlabs-ai/headroom/issues/1487)). * **subscription:** stop zeroing the 5-hour headroom contribution counters on every poll. The rollover check compared `five_hour.resets_at` with a bare `!=`, but the usage API reports that timestamp with second-level jitter (observed flapping between `01:59:59Z` and `02:00:00Z` on consecutive polls within the same window), so a spurious "5h window rolled over" reset fired every poll interval (~5 min) and the dashboard's per-window savings stuck near 0%. Only a forward jump larger than `_ROLLOVER_MIN_ADVANCE` (1 minute) now counts as a real rollover. * **transforms/content_router:** stop replacing `role="tool"` output with a lossy-unrecoverable summary on the live compression path (refs [#1307](https://github.com/chopratejas/headroom/issues/1307)). `ContentRouter.apply()` routed OpenAI-style `role="tool"` string messages — `Bash`/`grep`/`ls`/`cat` output — through the ML/word-drop summarizers; when the result carried no CCR retrieve marker (CCR off, ratio >= 0.8, or the size-gate fallback) the original was unrecoverable and the agent acted on a fabricated summary. Tool-role string content is now kept verbatim unless the compressed form is CCR-recoverable. Assistant/user text is unaffected, and structurally-lossless passes (SmartCrusher/Log/Search) still apply. The Anthropic `tool_result` block path is tracked separately. diff --git a/headroom/cli/learn.py b/headroom/cli/learn.py index b8a20c405..04cbda325 100644 --- a/headroom/cli/learn.py +++ b/headroom/cli/learn.py @@ -170,12 +170,6 @@ def learn( raise click.UsageError("--all and --project are mutually exclusive.") if llm_judge and not verbosity_mode: raise click.UsageError("--llm-judge only applies with --verbosity.") - if verbosity_mode and analyze_all and apply: - raise click.UsageError( - "--verbosity persists a single global level, so --all --apply would keep " - "only the last project's level. Re-run with one --project (or drop --apply " - "to preview every project)." - ) max_workers = workers if workers is not None else min(os.cpu_count() or 4, 8) @@ -445,7 +439,7 @@ def _run_verbosity( from ..learn.registry import auto_detect_plugins, get_plugin from ..learn.verbosity import analyze from ..paths import ensure_workspace_dir - from ..proxy.output_savings import SavingsLedger + from ..proxy.output_savings import BaselineModel, SavingsLedger # Verbosity mining reads Claude Code transcripts; restrict to that plugin. if agent == "auto": @@ -484,12 +478,26 @@ def _run_verbosity( judge = _make_llm_judge(model or "claude-sonnet-4-6") if llm_judge else None + # Aggregate across all targeted projects. The baseline accumulates so the + # synthetic control reflects every project's transcripts (not just whichever + # one happens to be processed last). The applied verbosity level comes from + # the project with the most samples — the strongest, least noisy signal. + aggregated = BaselineModel() + best_profile = None + best_profile_samples = -1 + analyzed_count = 0 + for proj in targets: session_paths = sorted(proj.data_path.glob("*.jsonl")) if not session_paths: continue profile, baseline = analyze(session_paths, str(proj.project_path), llm_judge=judge) sig = profile.signals + analyzed_count += 1 + aggregated.merge(baseline) + if baseline.total_samples > best_profile_samples: + best_profile_samples = baseline.total_samples + best_profile = profile click.echo(f"\n{'=' * 60}") click.echo(f"Verbosity — {proj.name}") @@ -516,45 +524,49 @@ def _run_verbosity( f"(confidence: {profile.confidence})" ) - if apply: - ws = ensure_workspace_dir() - from datetime import datetime, timezone + if analyzed_count == 0 or best_profile is None: + click.echo("\n No transcripts found in the selected project(s); nothing learned.") + return - profile.learned_at = datetime.now(timezone.utc).isoformat() - profile.save(ws / "verbosity.json") - # Seed the savings baseline: replace baseline, preserve any live - # treatment/control already accumulated. - ledger_path = ws / "output_savings.json" - ledger = SavingsLedger.load(ledger_path) - ledger.baseline = baseline - ledger.save(ledger_path) - click.echo(f"\n [WROTE] {ws / 'verbosity.json'} (level {profile.level})") + if apply: + ws = ensure_workspace_dir() + from datetime import datetime, timezone + + best_profile.learned_at = datetime.now(timezone.utc).isoformat() + best_profile.save(ws / "verbosity.json") + # Seed the savings baseline: replace baseline, preserve any live + # treatment/control already accumulated. + ledger_path = ws / "output_savings.json" + ledger = SavingsLedger.load(ledger_path) + ledger.baseline = aggregated + ledger.save(ledger_path) + click.echo(f"\n [WROTE] {ws / 'verbosity.json'} (level {best_profile.level})") + click.echo( + f" [WROTE] {ledger_path} (baseline: {aggregated.total_samples} samples, " + f"{len(aggregated.strata)} strata across {analyzed_count} project(s))" + ) + # Writing the level is not enough — the shaper is off by default. + # Make --apply actually take effect: hot-enable a running proxy, and + # otherwise tell the user exactly how to turn it on. + status, shaper_port = _activate_output_shaper() + if status == "live": click.echo( - f" [WROTE] {ledger_path} (baseline: {baseline.total_samples} samples, " - f"{len(baseline.strata)} strata)" + f"\n ✓ Output shaper enabled on the running proxy (port {shaper_port}); " + f"level {best_profile.level} is live now (while HEADROOM_VERBOSITY_LEVEL is unset)." + ) + click.echo( + " To keep it on across restarts: export HEADROOM_OUTPUT_SHAPER=1 " + "before `headroom wrap ...` (wrap pushes it to the proxy)." ) - # Writing the level is not enough — the shaper is off by default. - # Make --apply actually take effect: hot-enable a running proxy, and - # otherwise tell the user exactly how to turn it on. - status, shaper_port = _activate_output_shaper() - if status == "live": - click.echo( - f"\n ✓ Output shaper enabled on the running proxy (port {shaper_port}); " - f"level {profile.level} is live now (while HEADROOM_VERBOSITY_LEVEL is unset)." - ) - click.echo( - " To keep it on across restarts: export HEADROOM_OUTPUT_SHAPER=1 " - "before `headroom wrap ...` (wrap pushes it to the proxy)." - ) - else: - click.echo( - "\n ⚠ Level written, but the output shaper is OFF by default — it is " - "NOT shaping output yet." - ) - click.echo( - " Enable it: export HEADROOM_OUTPUT_SHAPER=1 then `headroom wrap ...` " - "(or start `headroom proxy` with it set). The learned level is then used " - "automatically while HEADROOM_VERBOSITY_LEVEL is unset." - ) else: - click.echo("\n Dry run — use --apply to persist the level and baseline.") + click.echo( + "\n ⚠ Level written, but the output shaper is OFF by default — it is " + "NOT shaping output yet." + ) + click.echo( + " Enable it: export HEADROOM_OUTPUT_SHAPER=1 then `headroom wrap ...` " + "(or start `headroom proxy` with it set). The learned level is then used " + "automatically while HEADROOM_VERBOSITY_LEVEL is unset." + ) + else: + click.echo("\n Dry run — use --apply to persist the level and baseline.") diff --git a/headroom/proxy/output_savings.py b/headroom/proxy/output_savings.py index b773774c0..bcff07c7e 100644 --- a/headroom/proxy/output_savings.py +++ b/headroom/proxy/output_savings.py @@ -162,6 +162,16 @@ class _Accum: return 0.0 return max(0.0, (self.sumsq - self.sum * self.sum / self.n) / (self.n - 1)) + def merge(self, other: _Accum) -> None: + """Fold another accumulator's observations into this one. + + n / sum / sumsq are additive, so merging is element-wise addition and + is exactly equivalent to having ``add``-ed both observation streams. + """ + self.n += other.n + self.sum += other.sum + self.sumsq += other.sumsq + def to_dict(self) -> dict[str, float]: return {"n": self.n, "sum": self.sum, "sumsq": self.sumsq} @@ -190,6 +200,19 @@ class BaselineModel: self.strata.setdefault(key, _Accum()).add(output_tokens) self.glob.add(output_tokens) + def merge(self, other: BaselineModel) -> None: + """Fold another baseline's observations into this one. + + Per-stratum and global accumulators are additive, so merging is + element-wise and order-independent — the result is identical to having + observed both corpora against a single model. Used to aggregate a + cross-project baseline from per-project ``analyze`` results without + re-reading transcripts. + """ + for key, acc in other.strata.items(): + self.strata.setdefault(key, _Accum()).merge(acc) + self.glob.merge(other.glob) + def lookup(self, key: str) -> tuple[float, float, int]: """Return ``(mean, var, n)`` for *key* with hierarchical back-off. diff --git a/tests/test_cli_learn.py b/tests/test_cli_learn.py index b5f58bab9..b160ea8d8 100644 --- a/tests/test_cli_learn.py +++ b/tests/test_cli_learn.py @@ -175,6 +175,74 @@ def test_learn_project_lookup_and_apply_flow( assert plugin.writer.calls[0][2] is False +def test_verbosity_all_apply_aggregates_baselines_across_projects( + monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path +) -> None: + import json as _json + + from headroom.proxy.output_savings import BaselineModel, SavingsLedger + + # Two projects, each with a transcript dir holding a dummy session file + # (analyze is faked, so contents are irrelevant — only presence matters). + proj_a_dir = tmp_path / "a" + proj_b_dir = tmp_path / "b" + for d in (proj_a_dir, proj_b_dir): + d.mkdir() + (d / "s.jsonl").write_text("{}") + proj_a = SimpleNamespace(name="a", project_path=tmp_path / "src-a", data_path=proj_a_dir) + proj_b = SimpleNamespace(name="b", project_path=tmp_path / "src-b", data_path=proj_b_dir) + plugin = FakePlugin("claude", "Claude Code", [proj_a, proj_b]) + + # Per-project synthetic baselines. Project A has more samples, so its level + # must be the one applied. + base_a = BaselineModel() + for v in (100, 200, 300): + base_a.observe("opus|new_user_ask|s|tools", v) + base_b = BaselineModel() + base_b.observe("sonnet|unknown|m|notools", 50) + + class _Profile: + def __init__(self, level: int) -> None: + self.level = level + self.confidence = "high" + self.source = "heuristic" + self.rationale = "test" + self.signals: dict[str, object] = {} + self.learned_at: str | None = None + + def save(self, path: object) -> None: + Path(str(path)).write_text(_json.dumps({"level": self.level})) + + results = { + str(proj_a.project_path): (_Profile(1), base_a), + str(proj_b.project_path): (_Profile(3), base_b), + } + + def fake_analyze(session_paths, project_path, llm_judge=None): # noqa: ANN001, ANN201 + return results[project_path] + + monkeypatch.setattr("headroom.learn.registry.get_plugin", lambda name: plugin) + monkeypatch.setattr("headroom.learn.verbosity.analyze", fake_analyze) + monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(tmp_path / "ws")) + + result = runner.invoke( + main, + ["learn", "--agent", "claude", "--verbosity", "--all", "--apply"], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.output + + ledger = SavingsLedger.load(tmp_path / "ws" / "output_savings.json") + # Aggregated, not last-project-wins: both strata present and totals summed. + assert ledger.baseline.total_samples == 4 + assert "opus|new_user_ask|s|tools" in ledger.baseline.strata + assert "sonnet|unknown|m|notools" in ledger.baseline.strata + assert "across 2 project(s)" in result.output + # The applied level comes from the project with the most samples (A → 1). + verbosity = _json.loads((tmp_path / "ws" / "verbosity.json").read_text()) + assert verbosity["level"] == 1 + + def test_learn_reports_missing_requested_project_and_lists_discovered( monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path ) -> None: diff --git a/tests/test_output_savings.py b/tests/test_output_savings.py index 6cde64709..fef965687 100644 --- a/tests/test_output_savings.py +++ b/tests/test_output_savings.py @@ -129,6 +129,29 @@ class TestBaselineModel: assert m2.lookup("k|a|s|tools") == m.lookup("k|a|s|tools") assert m2.total_samples == 3 + def test_merge_is_equivalent_to_observing_both_streams(self): + # Merging two baselines must equal observing every value against one + # model — same totals per stratum and same global fallback. + a = BaselineModel() + for v in (100, 200): + a.observe("opus|new_user_ask|s|tools", v) + b = BaselineModel() + b.observe("opus|new_user_ask|s|tools", 300) + b.observe("sonnet|unknown|m|notools", 50) + + a.merge(b) + + mean, _, n = a.lookup("opus|new_user_ask|s|tools") + assert n == 3 + assert mean == 200.0 # (100 + 200 + 300) / 3 + assert a.total_samples == 4 # 3 + 1 across both strata + + reference = BaselineModel() + for v in (100, 200, 300): + reference.observe("opus|new_user_ask|s|tools", v) + reference.observe("sonnet|unknown|m|notools", 50) + assert a.to_dict() == reference.to_dict() + # --------------------------------------------------------------------------- # synthetic-control estimate