Merge pull request #224 from gglucass/codex/compact-stats-history-default

Compact /stats-history responses by default

CHANGELOG.md

@@ -30,6 +30,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   savings, logs, and telemetry resolve to the bind-mounted `.headroom` path.
   See [`wiki/filesystem-contract.md`](wiki/filesystem-contract.md).
 
+### Changed
+- **`/stats-history` now returns compact checkpoint history by default** — the
+  JSON response keeps recent checkpoints dense while evenly sampling older
+  checkpoints so long-running installs do not return ever-growing payloads.
+  Add `history_mode=full` to fetch the full retained checkpoint list, or
+  `history_mode=none` to skip it entirely while still receiving the derived
+  hourly/daily/weekly/monthly rollups. Responses now include a
+  `history_summary` block describing stored versus returned points.
+
 ## [0.5.22] - 2026-04-11
 
 ### Added

headroom/dashboard/templates/dashboard.html

@@ -1616,6 +1616,9 @@
                 async downloadHistory(format = 'json', series = null) {
                     const selectedSeries = series || this.historySelectedSeriesKey;
                     const params = new URLSearchParams({ format, series: selectedSeries });
+                    if (format === 'json' && selectedSeries === 'history') {
+                        params.set('history_mode', 'full');
+                    }
                     const response = await fetch('/stats-history?' + params.toString());
                     if (!response.ok) throw new Error('Failed to export history');
 

headroom/proxy/savings_tracker.py

@@ -29,6 +29,7 @@ DEFAULT_SAVINGS_FILE = "proxy_savings.json"
 SCHEMA_VERSION = 2
 DEFAULT_MAX_HISTORY_POINTS = 5000
 DEFAULT_MAX_HISTORY_AGE_DAYS = 365
+DEFAULT_MAX_RESPONSE_HISTORY_POINTS = 500
 DEFAULT_DISPLAY_SESSION_INACTIVITY_MINUTES = 60
 
 LITELLM_AVAILABLE = importlib.util.find_spec("litellm") is not None
@@ -311,11 +312,19 @@ class SavingsTracker:
         path: str | None = None,
         max_history_points: int = DEFAULT_MAX_HISTORY_POINTS,
         max_history_age_days: int = DEFAULT_MAX_HISTORY_AGE_DAYS,
+        max_response_history_points: int = DEFAULT_MAX_RESPONSE_HISTORY_POINTS,
         display_session_inactivity_minutes: int = (DEFAULT_DISPLAY_SESSION_INACTIVITY_MINUTES),
     ) -> None:
         self._path = Path(path or get_default_savings_storage_path())
         self._max_history_points = max_history_points
         self._max_history_age_days = max_history_age_days
+        self._max_response_history_points = max(
+            _coerce_int(
+                max_response_history_points,
+                DEFAULT_MAX_RESPONSE_HISTORY_POINTS,
+            ),
+            1,
+        )
         self._display_session_inactivity_minutes = max(
             _coerce_int(
                 display_session_inactivity_minutes,
@@ -524,16 +533,17 @@ class SavingsTracker:
             "retention": snapshot["retention"],
         }
 
-    def history_response(self) -> dict[str, Any]:
+    def history_response(self, history_mode: str = "compact") -> dict[str, Any]:
         """Return frontend-friendly historical data for `/stats-history`."""
         snapshot = self.snapshot()
-        history = snapshot["history"]
+        raw_history = snapshot["history"]
         series = {
-            "hourly": self._build_rollup(history, bucket="hour"),
-            "daily": self._build_rollup(history, bucket="day"),
-            "weekly": self._build_rollup(history, bucket="week"),
-            "monthly": self._build_rollup(history, bucket="month"),
+            "hourly": self._build_rollup(raw_history, bucket="hour"),
+            "daily": self._build_rollup(raw_history, bucket="day"),
+            "weekly": self._build_rollup(raw_history, bucket="week"),
+            "monthly": self._build_rollup(raw_history, bucket="month"),
         }
+        history = self._history_for_response(raw_history, mode=history_mode)
         return {
             "schema_version": snapshot["schema_version"],
             "generated_at": _to_utc_iso(_utc_now()),
@@ -549,6 +559,12 @@ class SavingsTracker:
                 "available_series": ["history", *series.keys()],
             },
             "retention": snapshot["retention"],
+            "history_summary": {
+                "mode": history_mode,
+                "stored_points": len(raw_history),
+                "returned_points": len(history),
+                "compacted": len(history) < len(raw_history),
+            },
         }
 
     def export_rows(self, series: str = "history") -> list[dict[str, Any]]:
@@ -604,6 +620,7 @@ class SavingsTracker:
                 "retention": {
                     "max_history_points": self._max_history_points,
                     "max_history_age_days": self._max_history_age_days,
+                    "max_response_history_points": self._max_response_history_points,
                 },
             }
 
@@ -727,6 +744,53 @@ class SavingsTracker:
 
         self._state["history"] = history
 
+    def _history_for_response(
+        self,
+        history: list[dict[str, Any]],
+        *,
+        mode: str,
+    ) -> list[dict[str, Any]]:
+        if mode == "none":
+            return []
+        if mode == "full":
+            return [dict(item) for item in history]
+        return self._compact_history(history)
+
+    def _compact_history(self, history: list[dict[str, Any]]) -> list[dict[str, Any]]:
+        if len(history) <= self._max_response_history_points:
+            return [dict(item) for item in history]
+
+        # Keep the recent tail dense for charts while evenly sampling older
+        # checkpoints so long-running installs don't return unbounded payloads.
+        recent_points = min(
+            max(self._max_response_history_points // 3, 50),
+            self._max_response_history_points - 1,
+        )
+        recent = history[-recent_points:]
+        older = history[:-recent_points]
+        older_slots = self._max_response_history_points - len(recent)
+        if older_slots <= 0 or not older:
+            return [dict(item) for item in recent[-self._max_response_history_points :]]
+
+        if older_slots == 1:
+            sampled_older = [older[0]]
+        else:
+            sampled_older = [
+                older[((len(older) - 1) * index) // (older_slots - 1)]
+                for index in range(older_slots)
+            ]
+
+        compacted: list[dict[str, Any]] = []
+        seen_timestamps: set[str] = set()
+        for point in [*sampled_older, *recent]:
+            timestamp = point.get("timestamp")
+            if not isinstance(timestamp, str) or timestamp in seen_timestamps:
+                continue
+            seen_timestamps.add(timestamp)
+            compacted.append(dict(point))
+
+        return compacted
+
     def _save_locked(self) -> None:
         try:
             self._path.parent.mkdir(parents=True, exist_ok=True)

headroom/proxy/server.py

@@ -1668,6 +1668,7 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
     async def stats_history(
         format: Literal["json", "csv"] = "json",
         series: Literal["history", "hourly", "daily", "weekly", "monthly"] = "history",
+        history_mode: Literal["compact", "full", "none"] = "compact",
     ):
         """Get durable proxy compression history plus display-session state."""
         if format == "csv":
@@ -1678,7 +1679,7 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
                 headers={"Content-Disposition": f'attachment; filename="{filename}"'},
             )
 
-        return proxy.metrics.savings_tracker.history_response()
+        return proxy.metrics.savings_tracker.history_response(history_mode=history_mode)
 
     @app.get("/transformations/feed")
     async def transformations_feed(limit: int = 20):

tests/test_proxy_savings_history.py

@@ -130,6 +130,7 @@ def test_savings_tracker_sanitizes_legacy_state_and_applies_retention(tmp_path):
     assert snapshot["retention"] == {
         "max_history_points": 1,
         "max_history_age_days": 2,
+        "max_response_history_points": 500,
     }
 
 
@@ -484,6 +485,60 @@ def test_savings_tracker_rollups_preserve_spend_and_input_history(tmp_path, monk
     ]
 
 
+def test_stats_history_defaults_to_compact_history_but_can_return_full_history(
+    tmp_path, monkeypatch
+):
+    path = tmp_path / "proxy_savings.json"
+    tracker = SavingsTracker(
+        path=str(path),
+        max_history_points=100,
+        max_history_age_days=30,
+        max_response_history_points=5,
+    )
+    monkeypatch.setattr(
+        "headroom.proxy.savings_tracker._estimate_compression_savings_usd",
+        lambda model, tokens_saved: tokens_saved / 1000.0,
+    )
+
+    for i in range(8):
+        tracker.record_compression_savings(
+            model="gpt-4o",
+            tokens_saved=10,
+            total_input_tokens=(i + 1) * 100,
+            total_input_cost_usd=(i + 1) * 0.1,
+            timestamp=f"2026-03-27T09:{i:02d}:00Z",
+        )
+
+    compact = tracker.history_response()
+    assert compact["history_summary"] == {
+        "mode": "compact",
+        "stored_points": 8,
+        "returned_points": 5,
+        "compacted": True,
+    }
+    assert len(compact["history"]) == 5
+    assert compact["history"][0]["timestamp"] == "2026-03-27T09:00:00Z"
+    assert compact["history"][-1]["timestamp"] == "2026-03-27T09:07:00Z"
+
+    full = tracker.history_response(history_mode="full")
+    assert full["history_summary"] == {
+        "mode": "full",
+        "stored_points": 8,
+        "returned_points": 8,
+        "compacted": False,
+    }
+    assert len(full["history"]) == 8
+
+    none = tracker.history_response(history_mode="none")
+    assert none["history"] == []
+    assert none["history_summary"] == {
+        "mode": "none",
+        "stored_points": 8,
+        "returned_points": 0,
+        "compacted": True,
+    }
+
+
 def test_stats_history_persists_across_restarts_and_stats_stays_compatible(tmp_path, monkeypatch):
     savings_path = tmp_path / "proxy_savings.json"
     monkeypatch.setenv("HEADROOM_SAVINGS_PATH", str(savings_path))
@@ -533,6 +588,12 @@ def test_stats_history_persists_across_restarts_and_stats_stays_compatible(tmp_p
         assert history_data["series"]["hourly"][0]["total_input_cost_usd_delta"] == pytest.approx(
             0.24
         )
+        assert history_data["history_summary"] == {
+            "mode": "compact",
+            "stored_points": 1,
+            "returned_points": 1,
+            "compacted": False,
+        }
 
         assert stats_data["display_session"] == history_data["display_session"]
         assert (
@@ -560,6 +621,11 @@ def test_stats_history_persists_across_restarts_and_stats_stays_compatible(tmp_p
         assert updated["series"]["daily"][0]["total_input_tokens_delta"] == 240
         assert updated["series"]["daily"][0]["total_input_cost_usd_delta"] == pytest.approx(0.48)
 
+        full = client.get("/stats-history?history_mode=full").json()
+        assert full["history_summary"]["mode"] == "full"
+        assert full["history_summary"]["stored_points"] == 2
+        assert full["history_summary"]["returned_points"] == 2
+
         persisted = json.loads(savings_path.read_text())
         assert persisted["lifetime"]["tokens_saved"] == 55
         assert persisted["lifetime"]["total_input_tokens"] == 240

wiki/metrics.md

@@ -100,7 +100,7 @@ curl http://localhost:8787/stats-history
 
 ```json
 {
-  "schema_version": 1,
+  "schema_version": 2,
   "generated_at": "2026-03-27T09:10:00Z",
   "lifetime": {
     "tokens_saved": 12500,
@@ -123,6 +123,12 @@ curl http://localhost:8787/stats-history
     "default_format": "json",
     "available_formats": ["json", "csv"],
     "available_series": ["history", "hourly", "daily", "weekly", "monthly"]
+  },
+  "history_summary": {
+    "mode": "compact",
+    "stored_points": 2048,
+    "returned_points": 500,
+    "compacted": true
   }
 }
 ```
@@ -132,11 +138,17 @@ compression history. It survives proxy restarts, tolerates missing or malformed
 state files, and powers the historical view in `/dashboard`. It now includes
 hourly, daily, weekly, and monthly chart-ready rollups.
 
+By default, the `history` array is compacted for transport efficiency. Use
+`history_mode=full` when you explicitly need the full retained checkpoint list,
+or `history_mode=none` when you only need the aggregate rollups and lifetime
+totals.
+
 For export-friendly downloads:
 
 ```bash
 curl "http://localhost:8787/stats-history?format=csv&series=daily"
 curl "http://localhost:8787/stats-history?format=csv&series=monthly"
+curl "http://localhost:8787/stats-history?history_mode=full"
 ```
 
 CSV exports are available for `history`, `hourly`, `daily`, `weekly`, and

wiki/proxy.md

@@ -241,8 +241,10 @@ curl http://localhost:8787/stats-history
 other Headroom frontends. It returns:
 
 - lifetime proxy compression totals
-- bounded persisted checkpoint history
+- compact checkpoint history by default, with `history_mode=full` available for
+  export/debug flows
 - derived hourly, daily, weekly, and monthly rollups for charts
+- a `history_summary` block describing stored versus returned checkpoint counts
 - UTC timestamps throughout
 
 By default the proxy stores this history at
@@ -258,6 +260,7 @@ daily/weekly/monthly rollups and built-in JSON / CSV export buttons.
 ```bash
 curl "http://localhost:8787/stats-history?format=csv&series=weekly"
 curl "http://localhost:8787/stats-history?format=csv&series=monthly"
+curl "http://localhost:8787/stats-history?history_mode=full"
 ```
 
 ### Prometheus Metrics