diff --git a/CHANGELOG.md b/CHANGELOG.md index 8aa6a7a0e..97c1cc944 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +* **proxy:** per-provider attribution in the savings history rollups. Each `/stats-history` bucket (hourly/daily/weekly/monthly) now carries a `by_provider` map breaking down `tokens_saved`, `compression_savings_usd_delta`, `total_input_tokens_delta`, and `total_input_cost_usd_delta` per provider, so consumers can show how savings and spend are distributed across providers within a time period. Providers only appear in a bucket where they moved a counter; legacy history checkpoints with no provider collapse into `"unknown"`. Affected files: `headroom/proxy/savings_tracker.py`, `headroom/proxy/prometheus_metrics.py`. + ### Changed * **deps:** loosen over-pinned constraints and add upper bounds diff --git a/headroom/proxy/prometheus_metrics.py b/headroom/proxy/prometheus_metrics.py index 27ad9fb1c..edfce23c9 100644 --- a/headroom/proxy/prometheus_metrics.py +++ b/headroom/proxy/prometheus_metrics.py @@ -648,6 +648,7 @@ class PrometheusMetrics: model=model, input_tokens=input_tokens, tokens_saved=tokens_saved, + provider=provider, cache_read_tokens=cache_read_tokens, cache_write_tokens=cache_write_tokens, uncached_input_tokens=uncached_input_tokens, diff --git a/headroom/proxy/savings_tracker.py b/headroom/proxy/savings_tracker.py index 600b86bbc..30e6deb8f 100644 --- a/headroom/proxy/savings_tracker.py +++ b/headroom/proxy/savings_tracker.py @@ -115,6 +115,22 @@ def _coerce_float(value: Any, default: float = 0.0) -> float: return default +PROVIDER_UNKNOWN = "unknown" + + +def _normalize_provider(value: Any) -> str: + """Normalize a provider label, falling back to a stable sentinel. + + History checkpoints persisted before per-provider attribution existed have + no provider field, so they collapse into ``PROVIDER_UNKNOWN`` rather than + silently dropping their savings from the per-provider breakdown. + """ + if not isinstance(value, str): + return PROVIDER_UNKNOWN + cleaned = value.strip() + return cleaned or PROVIDER_UNKNOWN + + def _resolve_litellm_model(model: str) -> str: """Resolve model name to one LiteLLM recognizes.""" litellm = _get_litellm_module() @@ -224,6 +240,7 @@ def _normalize_history_entry(entry: Any) -> dict[str, Any] | None: compression_savings_usd = 0.0 total_input_tokens = 0 total_input_cost_usd = 0.0 + provider = PROVIDER_UNKNOWN if isinstance(entry, dict): timestamp = _parse_timestamp(entry.get("timestamp")) @@ -231,6 +248,7 @@ def _normalize_history_entry(entry: Any) -> dict[str, Any] | None: compression_savings_usd = _coerce_float(entry.get("compression_savings_usd")) total_input_tokens = _coerce_int(entry.get("total_input_tokens")) total_input_cost_usd = _coerce_float(entry.get("total_input_cost_usd")) + provider = _normalize_provider(entry.get("provider")) elif isinstance(entry, list | tuple) and len(entry) >= 2: timestamp = _parse_timestamp(entry[0]) total_tokens_saved = _coerce_int(entry[1]) @@ -248,6 +266,7 @@ def _normalize_history_entry(entry: Any) -> dict[str, Any] | None: return { "timestamp": _to_utc_iso(timestamp), + "provider": provider, "total_tokens_saved": total_tokens_saved, "compression_savings_usd": round(compression_savings_usd, 6), "total_input_tokens": total_input_tokens, @@ -344,6 +363,7 @@ class SavingsTracker: *, model: str, tokens_saved: int, + provider: str | None = None, total_input_tokens: int | None = None, total_input_cost_usd: float | None = None, timestamp: datetime | str | None = None, @@ -389,6 +409,7 @@ class SavingsTracker: self._state["history"].append( { "timestamp": _to_utc_iso(timestamp_dt), + "provider": _normalize_provider(provider), "total_tokens_saved": lifetime["tokens_saved"], "compression_savings_usd": lifetime["compression_savings_usd"], "total_input_tokens": lifetime["total_input_tokens"], @@ -405,6 +426,7 @@ class SavingsTracker: model: str, input_tokens: int, tokens_saved: int, + provider: str | None = None, cache_read_tokens: int = 0, cache_write_tokens: int = 0, uncached_input_tokens: int = 0, @@ -508,6 +530,7 @@ class SavingsTracker: self._state["history"].append( { "timestamp": _to_utc_iso(timestamp_dt), + "provider": _normalize_provider(provider), "total_tokens_saved": lifetime["tokens_saved"], "compression_savings_usd": lifetime["compression_savings_usd"], "total_input_tokens": lifetime["total_input_tokens"], @@ -914,6 +937,7 @@ class SavingsTracker: "total_input_tokens": total_input_tokens, "total_input_cost_usd_delta": 0.0, "total_input_cost_usd": total_input_cost_usd, + "by_provider": {}, }, ) entry["tokens_saved"] += delta_tokens @@ -931,4 +955,30 @@ class SavingsTracker: entry["total_input_tokens"] = total_input_tokens entry["total_input_cost_usd"] = round(total_input_cost_usd, 6) + # Attribute this checkpoint's delta to the provider that produced + # it. Each checkpoint comes from a single request, so its delta is + # wholly owned by one provider. Skip no-op checkpoints so providers + # only appear in a bucket where they actually moved a counter. + if delta_tokens or delta_usd or delta_input_tokens or delta_input_cost_usd: + provider = _normalize_provider(point.get("provider")) + prov = entry["by_provider"].setdefault( + provider, + { + "tokens_saved": 0, + "compression_savings_usd_delta": 0.0, + "total_input_tokens_delta": 0, + "total_input_cost_usd_delta": 0.0, + }, + ) + prov["tokens_saved"] += delta_tokens + prov["compression_savings_usd_delta"] = round( + prov["compression_savings_usd_delta"] + delta_usd, + 6, + ) + prov["total_input_tokens_delta"] += delta_input_tokens + prov["total_input_cost_usd_delta"] = round( + prov["total_input_cost_usd_delta"] + delta_input_cost_usd, + 6, + ) + return list(aggregated.values()) diff --git a/tests/test_proxy_savings_history.py b/tests/test_proxy_savings_history.py index 8e95c05f4..ba9391f81 100644 --- a/tests/test_proxy_savings_history.py +++ b/tests/test_proxy_savings_history.py @@ -65,6 +65,7 @@ def test_savings_tracker_helpers_normalize_inputs_and_paths(tmp_path, monkeypatc ["2026-03-27T09:00:00Z", "12", "0.5"] ) == { "timestamp": "2026-03-27T09:00:00Z", + "provider": "unknown", "total_tokens_saved": 12, "compression_savings_usd": 0.5, "total_input_tokens": 0, @@ -122,6 +123,7 @@ def test_savings_tracker_sanitizes_legacy_state_and_applies_retention(tmp_path): assert snapshot["history"] == [ { "timestamp": "2026-03-27T09:00:00Z", + "provider": "unknown", "total_tokens_saved": 30, "compression_savings_usd": 0.03, "total_input_tokens": 0, @@ -190,6 +192,7 @@ def test_record_compression_savings_skips_empty_updates_and_normalizes_timestamp assert snapshot["history"] == [ { "timestamp": "2026-03-27T08:00:00Z", + "provider": "unknown", "total_tokens_saved": 10, "compression_savings_usd": 0.01, "total_input_tokens": 120, @@ -197,6 +200,7 @@ def test_record_compression_savings_skips_empty_updates_and_normalizes_timestamp }, { "timestamp": "2026-03-27T12:34:00Z", + "provider": "unknown", "total_tokens_saved": 15, "compression_savings_usd": 0.015, "total_input_tokens": 180, @@ -522,6 +526,83 @@ def test_savings_tracker_rollups_preserve_spend_and_input_history(tmp_path, monk ] +def test_savings_tracker_rollup_attributes_savings_per_provider(tmp_path, monkeypatch): + path = tmp_path / "proxy_savings.json" + tracker = SavingsTracker( + path=str(path), + max_history_points=100, + max_history_age_days=30, + ) + monkeypatch.setattr( + "headroom.proxy.savings_tracker._estimate_compression_savings_usd", + lambda model, tokens_saved: tokens_saved / 1000.0, + ) + + # Two providers active in the same hour bucket. + tracker.record_compression_savings( + model="claude-3-5-sonnet", + tokens_saved=100, + provider="anthropic", + total_input_tokens=120, + total_input_cost_usd=0.24, + timestamp="2026-03-27T09:10:00Z", + ) + tracker.record_compression_savings( + model="gpt-4o", + tokens_saved=40, + provider="openai", + total_input_tokens=200, + total_input_cost_usd=0.40, + timestamp="2026-03-27T09:40:00Z", + ) + # Only anthropic active in the next hour bucket. + tracker.record_compression_savings( + model="claude-3-5-sonnet", + tokens_saved=25, + provider="anthropic", + total_input_tokens=260, + total_input_cost_usd=0.52, + timestamp="2026-03-27T10:05:00Z", + ) + # A legacy-style record with no provider collapses into "unknown". + tracker.record_compression_savings( + model="gpt-4o", + tokens_saved=15, + total_input_tokens=320, + total_input_cost_usd=0.64, + timestamp="2026-03-27T11:00:00Z", + ) + + hourly = tracker.history_response()["series"]["hourly"] + + first = hourly[0] + assert first["tokens_saved"] == 140 + assert set(first["by_provider"]) == {"anthropic", "openai"} + assert first["by_provider"]["anthropic"]["tokens_saved"] == 100 + assert first["by_provider"]["anthropic"]["total_input_tokens_delta"] == 120 + assert first["by_provider"]["anthropic"]["compression_savings_usd_delta"] == pytest.approx(0.1) + assert first["by_provider"]["anthropic"]["total_input_cost_usd_delta"] == pytest.approx(0.24) + assert first["by_provider"]["openai"]["tokens_saved"] == 40 + assert first["by_provider"]["openai"]["total_input_tokens_delta"] == 80 + assert first["by_provider"]["openai"]["compression_savings_usd_delta"] == pytest.approx(0.04) + assert first["by_provider"]["openai"]["total_input_cost_usd_delta"] == pytest.approx(0.16) + # Per-provider deltas sum back to the bucket total. + assert ( + first["by_provider"]["anthropic"]["tokens_saved"] + + first["by_provider"]["openai"]["tokens_saved"] + == first["tokens_saved"] + ) + + second = hourly[1] + assert set(second["by_provider"]) == {"anthropic"} + assert second["by_provider"]["anthropic"]["tokens_saved"] == 25 + assert second["by_provider"]["anthropic"]["total_input_tokens_delta"] == 60 + + third = hourly[2] + assert set(third["by_provider"]) == {"unknown"} + assert third["by_provider"]["unknown"]["tokens_saved"] == 15 + + def test_stats_history_defaults_to_compact_history_but_can_return_full_history( tmp_path, monkeypatch ):