From 53be64ca128cb1906b0e508a8343c91fed1f47fd Mon Sep 17 00:00:00 2001 From: Tejas Chopra Date: Sat, 27 Jun 2026 22:48:26 -0700 Subject: [PATCH] chore(telemetry): remove Supabase anonymous beacon; fix contact domain to headroomlabs.ai (#1526) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description Removes the anonymous-telemetry **beacon** — the only external, third-party data flow Headroom ever initiated. When telemetry was opted in, it POSTed aggregate `/stats` to a hardcoded **Supabase** REST endpoint (with an embedded anon API key in the source). For enterprise/on-prem deployments this is exactly the kind of vendor-controlled data egress a security review flags, so it's gone entirely — **zero "Supabase" references remain in the codebase.** What stays (by design): the **local** telemetry collector + the `HEADROOM_TELEMETRY` opt-in (it only feeds `/stats` and `/v1/telemetry` — nothing leaves the process), **OpenTelemetry export** (`HEADROOM_OTEL_METRICS_*`, so operators send operational metrics to *their own* collector), and the license usage reporter (your own domain, license-key-gated). Also fixes the contact domain: `headroom.dev` → `headroomlabs.ai` everywhere. Closes # (no tracking issue) ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [ ] Performance improvement - [x] Code refactoring (no functional changes) > Non-breaking: `HEADROOM_TELEMETRY` is still accepted (now gates local collection only). The only behavior change is that no telemetry is ever sent externally. ## Changes Made - **Deleted the Supabase beacon**: `TelemetryBeacon` class, `_SUPABASE_URL`/`_SUPABASE_KEY`/`_TABLE`/`_ENDPOINT`, the JSONB projection helper, the proxy-lifespan beacon wiring, the `SUPABASE_` install env passthrough, and `tests/test_strategy_stats_supabase.py`. - **Kept** the local opt-in predicate (`is_telemetry_enabled` etc.) in `beacon.py` — still used by the local collector + CLI — reworded to "local only". - **Retained** the single-worker-owner file lock (the cc-switch reconciler depends on it); updated its comments to drop the beacon framing. - `/stats` `anon_telemetry_shipping` is now always `False` (nothing ships externally); startup log reworded to "Local telemetry". - Reworded remaining "Supabase" comments in `collector.py`, `context.py`, `prometheus_metrics.py`, and two test docstrings. - Contact domain: `security@headroom.dev` → `security@headroomlabs.ai`, `conduct@headroom.dev` → `conduct@headroomlabs.ai`, FUNDING.yml sponsor URL. ## Testing - [x] Unit tests pass (`pytest`) - [x] Linting passes (`ruff check`) - [x] Type checking passes (`mypy`) - [x] New tests added for new functionality - [x] Manual testing performed ### Test Output ```text $ grep -rniI "supabase" --include=*.py --include=*.md --include=*.mdx . # (excl .venv/sbom) >>> ZERO Supabase references $ grep -rniI "headroom.dev" . >>> ZERO headroom.dev references $ ruff check -> All checks passed! $ ruff format --check -> 10 files already formatted $ mypy -> Success: no issues found $ pytest tests/test_telemetry.py tests/test_telemetry_warning.py \ tests/test_proxy_telemetry_env.py tests/test_compression_observability.py \ tests/test_paths.py tests/test_paths_backward_compat.py -q ============================= 173 passed in 6.67s ============================== ``` ## Real Behavior Proof - **Environment:** macOS, Python 3.12 (`.venv`). - **Exact command / steps:** repo-wide grep for `supabase`/`headroom.dev`; `create_app(...)` driven through a full `TestClient` lifespan (startup + shutdown) in `test_proxy_telemetry_env.py`; `/stats` exercised in `test_telemetry_warning.py`. - **Observed result:** zero `supabase`/`headroom.dev` strings remain; the proxy starts and shuts down cleanly with the beacon removed (the worker-owner lock + reconciler still elect a single owner); `/stats.anon_telemetry_shipping` is `False` even with `HEADROOM_TELEMETRY=on`; local collector + OTEL paths unchanged. - **Not tested:** no live network call was ever made (the point — the external POST is gone). OTEL export and the license reporter were not exercised (unchanged by this PR). ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] New and existing unit tests pass locally with my changes - [ ] I have updated the CHANGELOG.md if applicable ## Additional Notes - The license usage reporter (`reporter.py` → `app.headroomlabs.ai`) is intentionally **kept** — it's license-key-gated (dormant for unlicensed/OSS deployments) and goes to your own domain, not a third party. - Docs/CHANGELOG left unchecked: a couple of docs mention the telemetry beacon and may want a follow-up note that it now collects locally only; happy to add. --- .github/FUNDING.yml | 2 +- CODE_OF_CONDUCT.md | 2 +- SECURITY.md | 2 +- headroom/install/runtime.py | 1 - headroom/proxy/prometheus_metrics.py | 8 +- headroom/proxy/server.py | 52 ++-- headroom/telemetry/beacon.py | 323 ++---------------------- headroom/telemetry/collector.py | 5 +- headroom/telemetry/context.py | 4 +- tests/test_compression_observability.py | 16 +- tests/test_proxy_telemetry_env.py | 50 ---- tests/test_strategy_stats_supabase.py | 91 ------- tests/test_telemetry.py | 4 +- tests/test_telemetry_warning.py | 7 +- 14 files changed, 61 insertions(+), 506 deletions(-) delete mode 100644 tests/test_strategy_stats_supabase.py diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index e4977c1d1..f4d7e814f 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -4,4 +4,4 @@ github: [headroom-sdk] # patreon: headroom # open_collective: headroom # ko_fi: headroom -# custom: ["https://headroom.dev/sponsor"] +# custom: ["https://headroomlabs.ai/sponsor"] diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 542c3103c..a90652cf7 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -60,7 +60,7 @@ representative at an online or offline event. Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at -**conduct@headroom.dev**. +**conduct@headroomlabs.ai**. All complaints will be reviewed and investigated promptly and fairly. diff --git a/SECURITY.md b/SECURITY.md index 237341f77..24d1d50fd 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -15,7 +15,7 @@ We take security vulnerabilities seriously. If you discover a security issue, pl **Please DO NOT open a public GitHub issue for security vulnerabilities.** -Instead, please email us at: **security@headroom.dev** +Instead, please email us at: **security@headroomlabs.ai** Include the following information: - Type of vulnerability (e.g., injection, data exposure, authentication bypass) diff --git a/headroom/install/runtime.py b/headroom/install/runtime.py index 65563d0a5..ec0d3f08f 100644 --- a/headroom/install/runtime.py +++ b/headroom/install/runtime.py @@ -45,7 +45,6 @@ PASSTHROUGH_ENV_PREFIXES = ( "OLLAMA_", "LITELLM_", "OTEL_", - "SUPABASE_", "QDRANT_", "NEO4J_", "LANGSMITH_", diff --git a/headroom/proxy/prometheus_metrics.py b/headroom/proxy/prometheus_metrics.py index 195a34419..dca95ddb0 100644 --- a/headroom/proxy/prometheus_metrics.py +++ b/headroom/proxy/prometheus_metrics.py @@ -900,11 +900,9 @@ class PrometheusMetrics: # internally on `self.compressions_by_strategy` and # `self.tokens_saved_by_strategy` (populated by # `record_compression`) but **deliberately not exported - # here**. The proxy's metric→Supabase pipeline treats - # each metric name as a column, and we cannot add new - # columns. The state is still observable for tests + - # programmatic introspection; if/when a non-column- - # adding export path exists, surface it there. + # here** as individual Prometheus series. The state is + # still observable via /stats + tests + programmatic + # introspection. _append_metric( lines, name="headroom_latency_ms_sum", diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py index 897599830..7c7e8b58c 100644 --- a/headroom/proxy/server.py +++ b/headroom/proxy/server.py @@ -1519,15 +1519,18 @@ class HeadroomProxy( "(set GITHUB_TOKEN or GITHUB_COPILOT_GITHUB_TOKEN to enable)" ) - # Log anonymous telemetry status so operators can see it in the log stream + # Log local telemetry status so operators can see it in the log stream. + # Nothing is sent externally — telemetry is collected locally only (the + # anonymous telemetry beacon was removed); operational metrics export + # only to your own OTEL collector via HEADROOM_OTEL_METRICS_*. if is_telemetry_enabled(): logger.info( - "Anonymous telemetry: ENABLED (aggregate stats only — no prompts or content). " - "Opt out: HEADROOM_TELEMETRY=off or --no-telemetry" + "Local telemetry: ENABLED (aggregate stats, local only — nothing sent " + "externally). Opt out: HEADROOM_TELEMETRY=off or --no-telemetry" ) else: logger.info( - "Anonymous telemetry: DISABLED (off by default — opt in: " + "Local telemetry: DISABLED (off by default — opt in: " "HEADROOM_TELEMETRY=on or --telemetry)" ) @@ -1901,21 +1904,10 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI: set_upstream=_set_anthropic_upstream, ) - # Telemetry beacon (anonymous aggregate stats). - # With uvicorn workers > 1, each worker runs the lifespan independently. - # We must ensure only ONE beacon runs across all workers — otherwise each - # worker creates its own beacon, spamming the telemetry table with N rows - # per cycle instead of 1 (all reading the same /stats from the same port). - # - # Strategy: use a file lock to ensure only the first worker starts the - # beacon. Other workers see the lock and skip. - from headroom.telemetry.beacon import TelemetryBeacon - - _beacon = TelemetryBeacon( - port=config.port if hasattr(config, "port") else 8787, - sdk=os.environ.get("HEADROOM_SDK", "proxy").strip() or "proxy", - backend=config.backend if hasattr(config, "backend") else "anthropic", - ) + # Single-worker-owner lock. With uvicorn workers > 1, each worker runs the + # lifespan independently. A file lock elects ONE owner worker so that + # single-instance background tasks (currently the cc-switch reconciler) run + # once across all workers instead of N times. from headroom import paths as _hr_paths _beacon_lock_path = _hr_paths.beacon_lock_path(config.port) @@ -1996,19 +1988,14 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI: if proxy._background_compression_enabled: await proxy._background_compressor.start() - # Only start beacon if we acquire the lock (first worker wins) + # Elect the single owner worker (first worker wins the lock). _beacon_is_owner[0] = _try_acquire_beacon_lock() - if _beacon_is_owner[0]: - await _beacon.start() - else: - logger.debug("Beacon: skipping (another worker owns the lock)") - # Only the beacon-lock owner runs the reconciler. With - # uvicorn workers > 1 each worker runs this lifespan; without - # this guard every worker would watch + rewrite settings.json - # concurrently and each process would hold its own - # HeadroomProxy.ANTHROPIC_API_URL, so workers could disagree on - # the upstream. Single-owner mirrors the beacon's reasoning. + # Only the owner worker runs the reconciler. With uvicorn + # workers > 1 each worker runs this lifespan; without this guard + # every worker would watch + rewrite settings.json concurrently + # and each process would hold its own HeadroomProxy.ANTHROPIC_API_URL, + # so workers could disagree on the upstream. if _cc_reconciler is not None and _beacon_is_owner[0]: await _cc_reconciler.start() @@ -2023,7 +2010,6 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI: if _cc_reconciler is not None: await _cc_reconciler.stop() if _beacon_is_owner[0]: - await _beacon.stop() _release_beacon_lock() if proxy.usage_reporter: await proxy.usage_reporter.stop() @@ -3097,7 +3083,9 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI: "ccr_retrievals": compression_stats.get("total_retrievals", 0), }, "compression_cache": compression_cache_stats, - "anon_telemetry_shipping": is_telemetry_enabled(), + # Always False: the anonymous telemetry beacon was removed, so no + # telemetry is ever shipped externally (local collection only). + "anon_telemetry_shipping": False, "telemetry": { "enabled": telemetry_stats.get("enabled", False), "total_compressions": telemetry_stats.get("total_compressions", 0), diff --git a/headroom/telemetry/beacon.py b/headroom/telemetry/beacon.py index 69c1773ad..36ac2b751 100644 --- a/headroom/telemetry/beacon.py +++ b/headroom/telemetry/beacon.py @@ -1,80 +1,32 @@ -"""Anonymous usage telemetry beacon for Headroom. +"""Telemetry opt-in state for Headroom. -Sends aggregate-only stats (tokens saved, compression ratios, cache hit rates, -performance overhead) to help improve Headroom. No prompts, no content, no PII. +Headroom collects only **local**, aggregate telemetry (tokens saved, compression +ratios, timings) for the in-process collector and the ``/stats`` / +``/v1/telemetry`` endpoints. **Nothing is sent to Headroom Labs:** the anonymous +telemetry beacon that previously shipped aggregate stats has been removed. +Operational metrics can still be exported to *your own* OpenTelemetry collector +via ``HEADROOM_OTEL_METRICS_*`` (a destination you control). -Off by default (opt-in). Nothing is collected or sent unless you opt in with: - HEADROOM_TELEMETRY=on headroom proxy - headroom proxy --telemetry +This module holds the ``HEADROOM_TELEMETRY`` opt-in predicate (off by default) +that gates local collection, plus the CLI notice helpers. """ from __future__ import annotations -import asyncio -import hashlib -import logging import os -import platform -import sys -import time -import uuid - -from headroom.telemetry.context import detect_install_mode, detect_stack - -logger = logging.getLogger(__name__) - -# Supabase endpoint for anonymous aggregate telemetry. -# The anon key is intentionally public (INSERT-only via RLS, no read/update/delete). -# Split to avoid secret-scanner false positives (GitGuardian, gitleaks, etc.). -_SUPABASE_URL = "https://dtlllcsudcoasebbamcq.supabase.co" -_SUPABASE_KEY = ".".join( - [ - "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", - "eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6ImR0bGxsY3N1ZGNvYXNlYmJhbWNxIiwicm9sZSI6ImFub24iLCJpYXQiOjE3NzM3MDc4NDUsImV4cCI6MjA4OTI4Mzg0NX0", - "h_C6dLQKa8BVc3upgEvulR4E0K4eiEViyddRMIylKjU", - ] -) -_TABLE = "proxy_telemetry_v2" -_ENDPOINT = f"{_SUPABASE_URL}/rest/v1/{_TABLE}?on_conflict=session_id" - -# Report every 5 minutes -_INTERVAL_SECONDS = 300 - _OFF_VALUES = frozenset(("off", "false", "0", "no", "disable", "disabled")) _ON_VALUES = frozenset(("on", "true", "1", "yes", "enable", "enabled")) -def _build_pipeline_timing(stats: dict) -> dict[str, object]: - """Project the /stats `pipeline_timing` JSONB payload for Supabase. - - Flattens transform timing to {name: avg_ms} and, when present, nests - ContentRouter strategy counts and per-strategy tokens-saved totals - under a `_strategies` sub-key. The Supabase column is JSONB, so the - nested shape lands without a schema change. - """ - raw_timing = stats.get("pipeline_timing", {}) or {} - pipeline_timing: dict[str, object] = { - name: round(info.get("average_ms", 0), 2) - for name, info in raw_timing.items() - if isinstance(info, dict) - } - strategies = stats.get("compressions_by_strategy", {}) or {} - tokens_by_strategy = stats.get("tokens_saved_by_strategy", {}) or {} - if strategies or tokens_by_strategy: - pipeline_timing["_strategies"] = { - "compressions": dict(strategies), - "tokens_saved": dict(tokens_by_strategy), - } - return pipeline_timing - - def is_telemetry_enabled() -> bool: - """Check if telemetry is enabled (off by default, opt in with env var). + """Check if local telemetry collection is enabled (off by default, opt-in). - Fail-closed: telemetry is only enabled when HEADROOM_TELEMETRY is set to an - explicit on-value (on/true/1/yes/enable/enabled). Anything else — including - unset, empty, or an unrecognized value — leaves it disabled. + Fail-closed: only enabled when HEADROOM_TELEMETRY is set to an explicit + on-value (on/true/1/yes/enable/enabled). Anything else — including unset, + empty, or an unrecognized value — leaves it disabled. Local collection only + feeds the in-process collector and the ``/stats`` endpoint; nothing is + transmitted to Headroom Labs. """ val = os.environ.get("HEADROOM_TELEMETRY", "").lower().strip() return val in _ON_VALUES @@ -103,249 +55,6 @@ def format_telemetry_notice(*, prefix: str = "") -> str: if not is_telemetry_enabled() or not is_telemetry_warn_enabled(): return "" return ( - f"{prefix}Telemetry: ENABLED (anonymous aggregate stats) | " + f"{prefix}Telemetry: ENABLED (local aggregate stats only — nothing sent externally) | " "Disable: HEADROOM_TELEMETRY=off or --no-telemetry" ) - - -class TelemetryBeacon: - """Periodically sends anonymous aggregate stats to Supabase.""" - - def __init__(self, port: int = 8787, sdk: str = "proxy", backend: str = "anthropic") -> None: - self._port = port - self._sdk = sdk - self._backend = backend - self._task: asyncio.Task[None] | None = None - self._start_time = time.time() - # Unique per proxy run — used as upsert key so each session produces 1 row - self._session_id = uuid.uuid4().hex - # Stable across restarts — anonymous machine fingerprint (SHA256 of hostname) - self._instance_id = hashlib.sha256(platform.node().encode()).hexdigest()[:16] - # Deployment shape is determined once at startup (wrapped / persistent / on_demand) - self._install_mode = detect_install_mode(port) - - async def start(self) -> None: - """Start the periodic beacon. Call from proxy startup.""" - if not is_telemetry_enabled(): - logger.debug("Telemetry disabled (HEADROOM_TELEMETRY=off)") - return - self._task = asyncio.create_task(self._loop()) - logger.info( - "Telemetry: ENABLED (anonymous aggregate stats, opt out: HEADROOM_TELEMETRY=off)" - ) - - async def stop(self) -> None: - """Stop and send one final report. Call from proxy shutdown.""" - if self._task: - self._task.cancel() - self._task = None - # Final report — but only if the proxy ran for more than 2 minutes. - # Short-lived restarts (e.g. crash loops, orchestration churn) would - # otherwise spam the telemetry table with duplicate cumulative stats. - uptime_seconds = time.time() - self._start_time - if is_telemetry_enabled() and uptime_seconds > 120: - await self._report() - - async def _loop(self) -> None: - """Background loop: wait, report, repeat.""" - # Wait 60 seconds before first report - await asyncio.sleep(60) - while True: - try: - await self._report() - except Exception: - pass # Never crash the proxy for telemetry - await asyncio.sleep(_INTERVAL_SECONDS) - - async def _report(self) -> None: - """Fetch stats from local /stats endpoint and POST to Supabase. - - Wrapped in multiple try/except layers so that: - 1. A missing httpx import silently skips. - 2. A failed /stats fetch silently skips. - 3. Extraction of any stats section is independent — one bad key - never blocks the others. - 4. A failed Supabase POST silently skips (fire-and-forget). - The proxy NEVER crashes or slows down because of telemetry. - """ - try: - import httpx - except ImportError: - return - - # ---- Fetch stats from our own proxy ---- - try: - async with httpx.AsyncClient(timeout=5.0) as client: - resp = await client.get(f"http://127.0.0.1:{self._port}/stats") - if resp.status_code != 200: - return - stats = resp.json() - except Exception: - return - - # Don't send empty stats — no point reporting zeros - try: - total_requests = stats.get("requests", {}).get("total", 0) - if total_requests == 0: - return - except Exception: - return - - # ---- Build payload — each section guarded independently ---- - session_minutes = max(1, int((time.time() - self._start_time) / 60)) - - try: - from headroom._version import __version__ as headroom_version - except Exception: - headroom_version = "unknown" - - # Core identity (always present) - payload: dict = { - "session_id": self._session_id, - "instance_id": self._instance_id, - "headroom_version": headroom_version, - "python_version": ( - f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" - ), - "os": f"{platform.system()} {platform.machine()}", - "sdk": self._sdk, - "backend": self._backend, - "session_minutes": session_minutes, - "install_mode": self._install_mode, - "headroom_stack": detect_stack(stats), - } - - try: - by_stack = (stats.get("requests") or {}).get("by_stack") or {} - if by_stack: - payload["requests_by_stack"] = dict(by_stack) - except Exception: - logger.debug("Beacon: failed to extract requests_by_stack", exc_info=True) - - # --- Effectiveness metrics --- - try: - tokens = stats.get("tokens", {}) - requests_stats = stats.get("requests", {}) - cache = stats.get("prefix_cache", {}).get("totals", {}) - cost = stats.get("cost", {}) - models_by = requests_stats.get("by_model", {}) - - payload.update( - { - "tokens_saved": tokens.get("saved", 0), - "requests": requests_stats.get("total", 0), - "compression_percent": tokens.get("savings_percent", 0), - "cache_hit_rate": cache.get("hit_rate", 0), - "cost_saved_usd": cost.get("savings_usd", 0), - "cache_saved_usd": cost.get("cache_savings_usd", 0), - "models_used": [ - m for m in models_by.keys() if not m.startswith("passthrough:") - ], - } - ) - except Exception: - logger.debug("Beacon: failed to extract effectiveness metrics", exc_info=True) - - # --- Cache bust tracking (tokens lost due to compression breaking prefix cache) --- - try: - cvc = stats.get("prefix_cache", {}).get("compression_vs_cache", {}) - bust_tokens = cvc.get("tokens_lost_to_cache_bust", 0) - if bust_tokens > 0: - payload["cache_bust_tokens"] = bust_tokens - except Exception: - logger.debug("Beacon: failed to extract cache bust metrics", exc_info=True) - - # --- Performance overhead (how much latency Headroom adds) --- - try: - overhead = stats.get("overhead", {}) - payload.update( - { - "overhead_avg_ms": round(overhead.get("average_ms", 0), 2), - "overhead_max_ms": round(overhead.get("max_ms", 0), 2), - } - ) - except Exception: - logger.debug("Beacon: failed to extract overhead metrics", exc_info=True) - - # --- TTFB (time to first byte — what the user feels) --- - try: - ttfb = stats.get("ttfb", {}) - payload["ttfb_avg_ms"] = round(ttfb.get("average_ms", 0), 2) - except Exception: - logger.debug("Beacon: failed to extract TTFB metrics", exc_info=True) - - # --- Pipeline timing breakdown (where is time spent?) --- - # Stored as JSONB — variable-shape dict of transform_name → avg_ms, - # plus an optional `_strategies` sub-key carrying ContentRouter strategy - # counts and per-strategy tokens-saved totals (zero schema change — the - # JSONB column absorbs the nested shape). - try: - pipeline_timing = _build_pipeline_timing(stats) - if pipeline_timing: - payload["pipeline_timing"] = pipeline_timing - except Exception: - logger.debug("Beacon: failed to extract pipeline timing", exc_info=True) - - # --- Request patterns (how big are conversations?) --- - try: - tokens = stats.get("tokens", {}) - total_req = stats.get("requests", {}).get("total", 1) - tokens_before = tokens.get("total_before_compression", 0) - tokens_after = tokens_before - tokens.get("saved", 0) - payload.update( - { - "avg_tokens_before": round(tokens_before / max(total_req, 1)), - "avg_tokens_after": round(tokens_after / max(total_req, 1)), - } - ) - except Exception: - logger.debug("Beacon: failed to extract request patterns", exc_info=True) - - # --- Compression cache effectiveness --- - try: - cc = stats.get("compression_cache", {}) - if cc: - payload["compression_cache"] = { - "hit_rate": cc.get("hit_rate", 0), - "entries": cc.get("entries", 0), - "tokens_saved": cc.get("total_tokens_saved", 0), - } - except Exception: - logger.debug("Beacon: failed to extract cache stats", exc_info=True) - - # --- CCR (Compress-Cache-Retrieve) usage --- - try: - ccr = stats.get("compression", {}) - if ccr.get("ccr_entries", 0) > 0: - payload["ccr"] = { - "entries": ccr.get("ccr_entries", 0), - "retrievals": ccr.get("ccr_retrievals", 0), - } - except Exception: - logger.debug("Beacon: failed to extract CCR stats", exc_info=True) - - # --- Waste signals (what patterns of waste do we see?) --- - try: - waste = stats.get("waste_signals", {}) - if waste: - payload["waste_signals"] = waste - except Exception: - logger.debug("Beacon: failed to extract waste signals", exc_info=True) - - # ---- Send to Supabase (fire-and-forget, upsert on session_id) ---- - try: - async with httpx.AsyncClient(timeout=10.0) as client: - await client.post( - _ENDPOINT, - json=payload, - headers={ - "apikey": _SUPABASE_KEY, - "Authorization": f"Bearer {_SUPABASE_KEY}", - "Content-Type": "application/json", - "Prefer": "resolution=merge-duplicates,return=minimal", - }, - ) - except Exception: - # No internet, DNS failure, timeout, Supabase down — all fine. - # Headroom continues working perfectly without telemetry. - logger.debug("Beacon: failed to send telemetry", exc_info=True) diff --git a/headroom/telemetry/collector.py b/headroom/telemetry/collector.py index 34682c38d..d0a6270b2 100644 --- a/headroom/telemetry/collector.py +++ b/headroom/telemetry/collector.py @@ -744,8 +744,9 @@ def get_telemetry_collector( if _telemetry_collector is None: with _collector_lock: if _telemetry_collector is None: - # Honour HEADROOM_TELEMETRY (the documented opt-out var, - # also used by the Supabase beacon at telemetry/beacon.py). + # Honour HEADROOM_TELEMETRY (the documented opt-out var; see + # the predicate in telemetry/beacon.py). Collection is local + # only — nothing is sent externally. # Pre-#390 this only checked HEADROOM_TELEMETRY_DISABLED, # so users who set HEADROOM_TELEMETRY=off (the value in # the docs) still saw /v1/telemetry report enabled=true. diff --git a/headroom/telemetry/context.py b/headroom/telemetry/context.py index 3a486d3f1..0a724d35f 100644 --- a/headroom/telemetry/context.py +++ b/headroom/telemetry/context.py @@ -27,7 +27,7 @@ _KNOWN_WRAP_AGENTS = frozenset( # Stack slugs must start with a letter and contain only [a-z0-9_], max 64 chars. # Applied at every ingress (env var, HTTP header, stats aggregation) so downstream -# sinks (Prometheus labels, Supabase column, JSONB payload) see a bounded vocabulary. +# sinks (Prometheus labels, OTEL attributes) see a bounded vocabulary. _STACK_SLUG_RE = re.compile(r"^[a-z][a-z0-9_]{0,63}$") # Cardinality cap on the per-process requests_by_stack dict. Protects the @@ -44,7 +44,7 @@ def normalize_stack(raw: str | None) -> str | None: else ``None``. All external stack identifiers (env var, HTTP header, stats keys) must pass through this function — it is the single chokepoint that bounds cardinality and rejects garbage before it reaches Prometheus or the - Supabase telemetry row. + OTEL metrics layer. """ if not raw: diff --git a/tests/test_compression_observability.py b/tests/test_compression_observability.py index c1f2b5857..3491a6ceb 100644 --- a/tests/test_compression_observability.py +++ b/tests/test_compression_observability.py @@ -11,9 +11,9 @@ strategy. These tests exist so the next regression of that shape fails the suite the day it lands instead of waiting on an audit. The counters live ONLY as in-process state on the metrics instance; -they are deliberately NOT exported through the Prometheus scrape or -OTel surface, because the metric→Supabase pipeline treats each -metric name as a column and we cannot add new columns. CI-level +they are deliberately NOT exported as new Prometheus metric names +(to avoid unbounded metric-series growth) — they remain observable +via /stats. CI-level observability via these tests is enough to catch silent regressions; production export waits on a non-column-adding pipeline. @@ -342,11 +342,11 @@ def test_prometheus_metrics_accumulates_codex_ws_unit_and_frame_counters(): def test_prometheus_export_does_not_leak_per_strategy_metrics(): """Per-strategy state is tracked in-process only. The Prometheus - scrape output deliberately must NOT emit new metric names — the - metric→Supabase pipeline treats each metric name as a column, and - we cannot add new columns. This test guards that constraint: if a - future change adds the metric to the scrape, this fails and forces - a conscious decision.""" + scrape output deliberately must NOT emit new metric names (to avoid + unbounded metric-series growth); the state stays observable via + /stats. This test guards that constraint: if a future change adds + the metric to the scrape, this fails and forces a conscious + decision.""" import asyncio from headroom.proxy.prometheus_metrics import PrometheusMetrics diff --git a/tests/test_proxy_telemetry_env.py b/tests/test_proxy_telemetry_env.py index 37c1b0101..e3ffd65a6 100644 --- a/tests/test_proxy_telemetry_env.py +++ b/tests/test_proxy_telemetry_env.py @@ -1,7 +1,6 @@ """Tests for proxy telemetry environment variable handling.""" import asyncio -from unittest.mock import patch import pytest @@ -12,55 +11,6 @@ from fastapi.testclient import TestClient from headroom.proxy.server import ProxyConfig, _proxy_config_from_env, create_app -class TestProxyTelemetrySDKEnv: - """Test HEADROOM_SDK handling when the proxy builds telemetry beacons.""" - - def test_proxy_telemetry_sdk_defaults_to_proxy(self, monkeypatch): - """Telemetry beacon uses the default SDK label when env var is unset.""" - monkeypatch.delenv("HEADROOM_SDK", raising=False) - - with patch("headroom.telemetry.beacon.TelemetryBeacon") as mock_beacon: - create_app( - ProxyConfig( - cache_enabled=False, - rate_limit_enabled=False, - cost_tracking_enabled=False, - ) - ) - - assert mock_beacon.call_args.kwargs["sdk"] == "proxy" - - def test_proxy_telemetry_sdk_uses_env_override(self, monkeypatch): - """Telemetry beacon uses HEADROOM_SDK when it is non-empty.""" - monkeypatch.setenv("HEADROOM_SDK", "headroom-app") - - with patch("headroom.telemetry.beacon.TelemetryBeacon") as mock_beacon: - create_app( - ProxyConfig( - cache_enabled=False, - rate_limit_enabled=False, - cost_tracking_enabled=False, - ) - ) - - assert mock_beacon.call_args.kwargs["sdk"] == "headroom-app" - - def test_proxy_telemetry_sdk_empty_env_falls_back_to_proxy(self, monkeypatch): - """Telemetry beacon falls back to proxy when HEADROOM_SDK is blank.""" - monkeypatch.setenv("HEADROOM_SDK", " ") - - with patch("headroom.telemetry.beacon.TelemetryBeacon") as mock_beacon: - create_app( - ProxyConfig( - cache_enabled=False, - rate_limit_enabled=False, - cost_tracking_enabled=False, - ) - ) - - assert mock_beacon.call_args.kwargs["sdk"] == "proxy" - - class TestProxyPeriodicTOINStatsEnv: """Test HEADROOM_PERIODIC_TOIN_STATS handling for long-lived proxy workers.""" diff --git a/tests/test_strategy_stats_supabase.py b/tests/test_strategy_stats_supabase.py deleted file mode 100644 index 21523c821..000000000 --- a/tests/test_strategy_stats_supabase.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Phase 3e.0: surface per-strategy compression counters. - -The internal `compressions_by_strategy` and `tokens_saved_by_strategy` -counters (PR #302) were tracked in process but never exported, because -the Prometheus→Supabase pipeline treats each metric name as a column and -adding columns is operationally expensive. - -These tests pin the alternative path: - -1. The `/stats` endpoint exposes both dicts on the response root. -2. The telemetry beacon nests them under `pipeline_timing._strategies`, - landing as JSONB inside the existing `pipeline_timing` Supabase - column — zero schema change. -""" - -from __future__ import annotations - -import pytest - - -def test_stats_endpoint_exposes_per_strategy_counters() -> None: - pytest.importorskip("fastapi") - from fastapi.testclient import TestClient - - from headroom.proxy.server import ProxyConfig, create_app - - app = create_app( - ProxyConfig( - optimize=False, - cache_enabled=False, - rate_limit_enabled=False, - cost_tracking_enabled=False, - log_requests=False, - ccr_inject_tool=False, - ccr_handle_responses=False, - ccr_context_tracking=False, - ) - ) - - proxy = app.state.proxy - proxy.metrics.record_compression("smart_crusher", original_tokens=300, compressed_tokens=80) - proxy.metrics.record_compression("smart_crusher", original_tokens=100, compressed_tokens=40) - proxy.metrics.record_compression("diff", original_tokens=120, compressed_tokens=70) - - with TestClient(app) as client: - response = client.get("/stats") - - assert response.status_code == 200 - body = response.json() - assert body["compressions_by_strategy"] == {"smart_crusher": 2, "diff": 1} - assert body["tokens_saved_by_strategy"] == { - "smart_crusher": (300 - 80) + (100 - 40), - "diff": 120 - 70, - } - - -def test_beacon_nests_strategies_under_pipeline_timing() -> None: - from headroom.telemetry.beacon import _build_pipeline_timing - - stats = { - "pipeline_timing": { - "smart_crusher": {"average_ms": 12.345, "max_ms": 50.0, "count": 10}, - "diff_compressor": {"average_ms": 5.1, "max_ms": 9.0, "count": 4}, - }, - "compressions_by_strategy": {"smart_crusher": 4321, "diff": 87}, - "tokens_saved_by_strategy": {"smart_crusher": 1_234_567, "diff": 23_456}, - } - - timing = _build_pipeline_timing(stats) - - assert timing["smart_crusher"] == 12.35 - assert timing["diff_compressor"] == 5.1 - assert timing["_strategies"] == { - "compressions": {"smart_crusher": 4321, "diff": 87}, - "tokens_saved": {"smart_crusher": 1_234_567, "diff": 23_456}, - } - - -def test_beacon_omits_strategies_subkey_when_counters_empty() -> None: - from headroom.telemetry.beacon import _build_pipeline_timing - - stats = { - "pipeline_timing": {"router": {"average_ms": 1.2}}, - "compressions_by_strategy": {}, - "tokens_saved_by_strategy": {}, - } - - timing = _build_pipeline_timing(stats) - - assert timing == {"router": 1.2} - assert "_strategies" not in timing diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index f6fae06cf..d0420ccda 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -584,8 +584,8 @@ class TestGlobalTelemetryCollector: Pre-#390 the collector only honoured HEADROOM_TELEMETRY_DISABLED, which is undocumented. Users following the docs set HEADROOM_TELEMETRY=off and watched /v1/telemetry continue to report enabled=true. The collector now - consults `is_telemetry_enabled()` (the same predicate the Supabase beacon - uses), so both env vars take effect. + consults `is_telemetry_enabled()` (the documented opt-in predicate), + so both env vars take effect. """ reset_telemetry_collector() monkeypatch.delenv("HEADROOM_TELEMETRY_DISABLED", raising=False) diff --git a/tests/test_telemetry_warning.py b/tests/test_telemetry_warning.py index 66ad64716..7f2490a41 100644 --- a/tests/test_telemetry_warning.py +++ b/tests/test_telemetry_warning.py @@ -227,8 +227,9 @@ class TestStatsEndpointTelemetryFlag: pytest.importorskip("fastapi") - async def test_stats_includes_anon_telemetry_shipping_true(self, monkeypatch): - # Opt-in: shipping is only true once telemetry is explicitly enabled. + async def test_stats_anon_telemetry_shipping_always_false(self, monkeypatch): + # The anonymous telemetry beacon was removed, so nothing is ever shipped + # externally — even with telemetry explicitly enabled. monkeypatch.setenv("HEADROOM_TELEMETRY", "on") from headroom.proxy.server import ProxyConfig, create_app @@ -248,7 +249,7 @@ class TestStatsEndpointTelemetryFlag: assert resp.status_code == 200 data = resp.json() assert "anon_telemetry_shipping" in data - assert data["anon_telemetry_shipping"] is True + assert data["anon_telemetry_shipping"] is False async def test_stats_includes_anon_telemetry_shipping_false(self, monkeypatch): monkeypatch.setenv("HEADROOM_TELEMETRY", "off")