From be876a6c22b75cc38798749f1a29cf828d0b5973 Mon Sep 17 00:00:00 2001 From: Garm Date: Wed, 19 Aug 2026 11:28:22 +0200 Subject: [PATCH] fix(savings): stop scoring unobserved strata against the global mean ``BaselineModel.lookup`` fell back to the all-requests mean whenever prefix back-off found nothing. That fallback is not a control for anything. The baseline is seeded once, by ``learn --verbosity`` reading session history that predates the shaper. It therefore only ever covers the model families the user ran *before* installing -- and it can never be relearned, because every transcript written since is already shaped. So each family they adopt later resolves to a single number derived from a different population. On a real ledger (15,658 baseline samples over 16 strata, all ``opus``; 74,243 treatment requests over 83 strata) that meant: - 48% of shaped requests -- every fable, sonnet, haiku and gpt turn -- scored against one opus-derived mean of 1,083 output tokens. - ``sonnet|new_user_ask|m|notools``: 4,666 requests whose replies average 73 tokens, each credited ~1,010 saved tokens. - 74% of the total reported savings produced by that fallback alone. - Reported reduction 42.6%; 32.1% over the strata the baseline actually observed. Scoring a short no-tool ask against a long tool-calling turn is not a synthetic control, it is a unit conversion. ``lookup`` now returns ``(0.0, 0.0, 0)`` instead, which both estimators already handle as "no evidence" -- the change is which requests qualify, not how they are scored. The global fallback remains available behind ``fall_back_to_global=True`` for callers that want it. Prefix back-off also merges every matching neighbour rather than returning the first one in dict order, so a round-tripped ledger can no longer score the same request differently depending on insertion order. Co-Authored-By: Claude Opus 5 (1M context) --- headroom/proxy/output_savings.py | 32 +++++++++++++++----- tests/test_output_savings.py | 50 ++++++++++++++++++++++++++++++-- 2 files changed, 72 insertions(+), 10 deletions(-) diff --git a/headroom/proxy/output_savings.py b/headroom/proxy/output_savings.py index b0ee6baf3..70641ebc6 100644 --- a/headroom/proxy/output_savings.py +++ b/headroom/proxy/output_savings.py @@ -144,12 +144,23 @@ class BaselineModel: self.strata.setdefault(key, _Accum()).merge(acc) self.glob.merge(other.glob) - def lookup(self, key: str) -> tuple[float, float, int]: + def lookup(self, key: str, *, fall_back_to_global: bool = False) -> tuple[float, float, int]: """Return ``(mean, var, n)`` for *key* with hierarchical back-off. - Falls back by trimming trailing (least-specific) stratum fields, then - to the global mean. Back-off keeps the estimate defined for strata the - baseline never saw, at the cost of specificity. + Falls back by trimming trailing (least-specific) stratum fields, so a + stratum the baseline never saw is still scored against its nearest + observed neighbours. Returns ``(0.0, 0.0, 0)`` when even that finds + nothing -- callers already treat ``n == 0`` as "no evidence". + + ``fall_back_to_global`` restores the old last resort of the + all-requests mean. It is off by default because that mean is not a + control for anything: the baseline is seeded once, from whatever the + user ran *before* installing, so every model family they adopt later + resolves to it. On a real ledger that meant 48% of requests -- sonnet + and fable turns whose replies average 43-770 tokens -- being scored + against one opus-derived mean of 1,083, which alone produced 74% of the + reported savings. Scoring a short no-tool ask against a long + tool-calling turn is not a synthetic control, it is a unit conversion. """ acc = self.strata.get(key) if acc is not None and acc.n > 0: @@ -157,11 +168,16 @@ class BaselineModel: parts = key.split("|") while len(parts) > 1: parts = parts[:-1] - prefix = "|".join(parts) + prefix = "|".join(parts) + "|" + neighbours = _Accum() for k, a in self.strata.items(): - if k.startswith(prefix + "|") and a.n > 0: - return a.mean, a.var, a.n - return self.glob.mean, self.glob.var, self.glob.n + if a.n > 0 and k.startswith(prefix): + neighbours.merge(a) + if neighbours.n > 0: + return neighbours.mean, neighbours.var, neighbours.n + if fall_back_to_global: + return self.glob.mean, self.glob.var, self.glob.n + return 0.0, 0.0, 0 def to_dict(self) -> dict[str, Any]: return { diff --git a/tests/test_output_savings.py b/tests/test_output_savings.py index e9d67a826..2ac898c0f 100644 --- a/tests/test_output_savings.py +++ b/tests/test_output_savings.py @@ -178,14 +178,38 @@ class TestBaselineModel: assert mean == 500.0 assert n == 1 - def test_lookup_falls_back_to_global(self): + def test_an_unobserved_family_is_not_scored_against_the_global_mean(self): + # The baseline is seeded once, from what the user ran before installing. + # Every family adopted later lands here, and the all-requests mean is + # not a control for any of them. m = BaselineModel() m.observe("opus|a|s|tools", 100) m.observe("sonnet|b|m|notools", 300) - mean, _, n = m.lookup("gpt|totally|xl|tools") + assert m.lookup("gpt|totally|xl|tools") == (0.0, 0.0, 0) + + def test_the_global_mean_remains_available_on_request(self): + m = BaselineModel() + m.observe("opus|a|s|tools", 100) + m.observe("sonnet|b|m|notools", 300) + mean, _, n = m.lookup("gpt|totally|xl|tools", fall_back_to_global=True) assert mean == 200.0 # global mean of 100 and 300 assert n == 2 + def test_prefix_backoff_merges_every_neighbour_not_the_first_one_hashed(self): + # Taking the first matching stratum in dict order made the answer + # depend on insertion order, so a round-tripped ledger could score the + # same request differently. + m = BaselineModel() + m.observe("opus|ask|l|tools", 1000) + m.observe("opus|ask|l|notools", 100) + mean, _, n = m.lookup("opus|ask|l|other") + assert (mean, n) == (550.0, 2) + + reversed_order = BaselineModel() + reversed_order.observe("opus|ask|l|notools", 100) + reversed_order.observe("opus|ask|l|tools", 1000) + assert reversed_order.lookup("opus|ask|l|other") == m.lookup("opus|ask|l|other") + def test_roundtrip_serialization(self): m = BaselineModel() for v in (10, 20, 30): @@ -277,6 +301,28 @@ class TestEstimateFromBaseline: # --------------------------------------------------------------------------- +class TestEstimateExcludesUnobservedStrata: + def test_requests_without_baseline_evidence_are_left_out(self): + ledger = SavingsLedger() + for _ in range(10): + ledger.baseline.observe("opus|ask|l|tools", 1000) + ledger.record("treatment", "opus|ask|l|tools", 800) + # A family the baseline never saw, with far shorter replies. Scoring it + # against the global mean would credit ~950 saved tokens per request. + for _ in range(40): + ledger.record("treatment", "sonnet|new_user_ask|m|notools", 50) + + est = ledger.estimate_from_baseline() + assert est.n_requests == 10, "only the observed stratum is scored" + assert abs(est.tokens_saved - 2000) < 1e-6 # 10 * (1000 - 800) + assert abs(est.pct - 20.0) < 1e-6 + + def test_per_request_savings_are_zero_without_evidence(self): + ledger = SavingsLedger() + ledger.baseline.observe("opus|ask|l|tools", 1000) + assert ledger.baseline.lookup("sonnet|new_user_ask|m|notools") == (0.0, 0.0, 0) + + class TestEstimateFromHoldout: def test_none_without_control_data(self): ledger = SavingsLedger()