mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-08-27 14:17:10 -04:00
Token-level cache hit rate, compression-vs-cache tracking, dashboard SQL, security plan
Cache stats: - hit_rate is now token-level (cache_read / total_input) not request-level - Track uncached_input_tokens per provider in metrics - Preserve request_hit_rate as secondary metric Compression-vs-cache: - Detect when compression busts the prefix cache (expected_cached - actual_read) - Two simple session-level numbers: tokens_saved vs cache_bust_tokens - Log CACHE-BUST per request, aggregate in /stats and telemetry beacon - Single new column in proxy_telemetry_v2: cache_bust_tokens Dashboard infra: - SQL for dashboard_summary table + pg_cron hourly refresh - Hourly + daily aggregation from proxy_telemetry_v2 - Upgrade scripts for adding hourly_stats and cache bust columns
This commit is contained in:
chopratejas
parent
f4160f9e47
commit
14d00c6c74
1656 changed files with 259772 additions and 1 deletions
200
sql/create_dashboard_summary.sql
Normal file
200
sql/create_dashboard_summary.sql
Normal file
|
|
@ -0,0 +1,200 @@
|
|||
-- Dashboard summary table + pg_cron hourly refresh
|
||||
-- Run this in the Supabase SQL Editor
|
||||
|
||||
-- 1. Create the summary table (single row, updated hourly)
|
||||
CREATE TABLE IF NOT EXISTS dashboard_summary (
|
||||
id text PRIMARY KEY DEFAULT 'current',
|
||||
updated_at timestamptz DEFAULT now(),
|
||||
total_tokens_saved bigint DEFAULT 0,
|
||||
total_cost_saved numeric DEFAULT 0,
|
||||
total_requests int DEFAULT 0,
|
||||
unique_instances int DEFAULT 0,
|
||||
active_days int DEFAULT 0,
|
||||
daily_stats jsonb DEFAULT '[]'::jsonb,
|
||||
hourly_stats jsonb DEFAULT '[]'::jsonb,
|
||||
top_instances jsonb DEFAULT '[]'::jsonb,
|
||||
os_breakdown jsonb DEFAULT '{}'::jsonb,
|
||||
version_breakdown jsonb DEFAULT '{}'::jsonb
|
||||
);
|
||||
|
||||
-- 2. RLS: anon can SELECT (public dashboard), only postgres can write
|
||||
ALTER TABLE dashboard_summary ENABLE ROW LEVEL SECURITY;
|
||||
|
||||
CREATE POLICY "Public read access" ON dashboard_summary
|
||||
FOR SELECT USING (true);
|
||||
|
||||
-- 3. The aggregation function (called by pg_cron)
|
||||
CREATE OR REPLACE FUNCTION refresh_dashboard_summary()
|
||||
RETURNS void AS $$
|
||||
DECLARE
|
||||
_daily jsonb;
|
||||
_hourly jsonb;
|
||||
_top jsonb;
|
||||
_os jsonb;
|
||||
_versions jsonb;
|
||||
_total_tokens bigint;
|
||||
_total_cost numeric;
|
||||
_total_requests int;
|
||||
_unique_instances int;
|
||||
_active_days int;
|
||||
BEGIN
|
||||
-- Daily totals: MAX per instance per day (beacon is cumulative), then SUM across instances
|
||||
WITH instance_daily AS (
|
||||
SELECT
|
||||
instance_id,
|
||||
created_at::date AS day,
|
||||
MAX(COALESCE(tokens_saved, 0)) AS tokens_saved,
|
||||
MAX(COALESCE(cost_saved_usd, 0)) AS cost_saved,
|
||||
MAX(COALESCE(requests, 0)) AS requests
|
||||
FROM proxy_telemetry_v2
|
||||
GROUP BY instance_id, created_at::date
|
||||
),
|
||||
daily_agg AS (
|
||||
SELECT
|
||||
day,
|
||||
SUM(tokens_saved) AS tokens_saved,
|
||||
SUM(cost_saved)::numeric(12,2) AS cost_saved,
|
||||
SUM(requests) AS requests,
|
||||
COUNT(DISTINCT instance_id) AS instances
|
||||
FROM instance_daily
|
||||
GROUP BY day
|
||||
ORDER BY day
|
||||
)
|
||||
SELECT
|
||||
COALESCE(jsonb_agg(jsonb_build_object(
|
||||
'date', day,
|
||||
'tokens_saved', tokens_saved,
|
||||
'cost_saved', cost_saved,
|
||||
'requests', requests,
|
||||
'instances', instances
|
||||
) ORDER BY day), '[]'::jsonb),
|
||||
COALESCE(SUM(tokens_saved), 0),
|
||||
COALESCE(SUM(cost_saved), 0),
|
||||
COALESCE(SUM(requests), 0),
|
||||
COUNT(DISTINCT day)
|
||||
INTO _daily, _total_tokens, _total_cost, _total_requests, _active_days
|
||||
FROM daily_agg;
|
||||
|
||||
-- Hourly totals: last 48 hours, MAX per instance per hour, then SUM across instances
|
||||
WITH instance_hourly AS (
|
||||
SELECT
|
||||
instance_id,
|
||||
date_trunc('hour', created_at) AS hour,
|
||||
MAX(COALESCE(tokens_saved, 0)) AS tokens_saved,
|
||||
MAX(COALESCE(cost_saved_usd, 0)) AS cost_saved,
|
||||
MAX(COALESCE(requests, 0)) AS requests
|
||||
FROM proxy_telemetry_v2
|
||||
WHERE created_at >= now() - interval '48 hours'
|
||||
GROUP BY instance_id, date_trunc('hour', created_at)
|
||||
),
|
||||
hourly_agg AS (
|
||||
SELECT
|
||||
hour,
|
||||
SUM(tokens_saved) AS tokens_saved,
|
||||
SUM(cost_saved)::numeric(12,2) AS cost_saved,
|
||||
SUM(requests) AS requests,
|
||||
COUNT(DISTINCT instance_id) AS instances
|
||||
FROM instance_hourly
|
||||
GROUP BY hour
|
||||
ORDER BY hour
|
||||
)
|
||||
SELECT COALESCE(jsonb_agg(jsonb_build_object(
|
||||
'hour', to_char(hour, 'YYYY-MM-DD HH24:MI'),
|
||||
'tokens_saved', tokens_saved,
|
||||
'cost_saved', cost_saved,
|
||||
'requests', requests,
|
||||
'instances', instances
|
||||
) ORDER BY hour), '[]'::jsonb)
|
||||
INTO _hourly
|
||||
FROM hourly_agg;
|
||||
|
||||
-- Unique instances
|
||||
SELECT COUNT(DISTINCT instance_id) INTO _unique_instances FROM proxy_telemetry_v2;
|
||||
|
||||
-- Top 20 instances by total tokens saved
|
||||
WITH instance_totals AS (
|
||||
SELECT
|
||||
instance_id,
|
||||
SUM(max_tokens) AS tokens_saved,
|
||||
SUM(max_cost)::numeric(12,2) AS cost_saved,
|
||||
MAX(os) AS os,
|
||||
MAX(version) AS version
|
||||
FROM (
|
||||
SELECT
|
||||
instance_id,
|
||||
created_at::date,
|
||||
MAX(COALESCE(tokens_saved, 0)) AS max_tokens,
|
||||
MAX(COALESCE(cost_saved_usd, 0)) AS max_cost,
|
||||
MAX(os) AS os,
|
||||
MAX(headroom_version) AS version
|
||||
FROM proxy_telemetry_v2
|
||||
GROUP BY instance_id, created_at::date
|
||||
) sub
|
||||
GROUP BY instance_id
|
||||
ORDER BY tokens_saved DESC
|
||||
LIMIT 20
|
||||
)
|
||||
SELECT COALESCE(jsonb_agg(jsonb_build_object(
|
||||
'instance_id', LEFT(instance_id, 8),
|
||||
'tokens_saved', tokens_saved,
|
||||
'cost_saved', cost_saved,
|
||||
'os', SPLIT_PART(COALESCE(os, '?'), ' ', 1),
|
||||
'version', version
|
||||
) ORDER BY tokens_saved DESC), '[]'::jsonb)
|
||||
INTO _top
|
||||
FROM instance_totals;
|
||||
|
||||
-- OS breakdown
|
||||
SELECT COALESCE(jsonb_object_agg(os_name, cnt), '{}'::jsonb)
|
||||
INTO _os
|
||||
FROM (
|
||||
SELECT SPLIT_PART(COALESCE(os, '?'), ' ', 1) AS os_name, COUNT(*) AS cnt
|
||||
FROM proxy_telemetry_v2
|
||||
GROUP BY os_name
|
||||
) sub;
|
||||
|
||||
-- Version breakdown
|
||||
SELECT COALESCE(jsonb_object_agg(COALESCE(headroom_version, '?'), cnt), '{}'::jsonb)
|
||||
INTO _versions
|
||||
FROM (
|
||||
SELECT headroom_version, COUNT(*) AS cnt
|
||||
FROM proxy_telemetry_v2
|
||||
GROUP BY headroom_version
|
||||
) sub;
|
||||
|
||||
-- Upsert the single summary row
|
||||
INSERT INTO dashboard_summary (id, updated_at, total_tokens_saved, total_cost_saved,
|
||||
total_requests, unique_instances, active_days, daily_stats, hourly_stats,
|
||||
top_instances, os_breakdown, version_breakdown)
|
||||
VALUES ('current', now(), _total_tokens, _total_cost, _total_requests,
|
||||
_unique_instances, _active_days, _daily, _hourly, _top, _os, _versions)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
updated_at = EXCLUDED.updated_at,
|
||||
total_tokens_saved = EXCLUDED.total_tokens_saved,
|
||||
total_cost_saved = EXCLUDED.total_cost_saved,
|
||||
total_requests = EXCLUDED.total_requests,
|
||||
unique_instances = EXCLUDED.unique_instances,
|
||||
active_days = EXCLUDED.active_days,
|
||||
daily_stats = EXCLUDED.daily_stats,
|
||||
hourly_stats = EXCLUDED.hourly_stats,
|
||||
top_instances = EXCLUDED.top_instances,
|
||||
os_breakdown = EXCLUDED.os_breakdown,
|
||||
version_breakdown = EXCLUDED.version_breakdown;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- 4. Run it once to populate
|
||||
SELECT refresh_dashboard_summary();
|
||||
|
||||
-- 5. Enable pg_cron extension (if not already)
|
||||
CREATE EXTENSION IF NOT EXISTS pg_cron;
|
||||
|
||||
-- 6. Schedule hourly refresh (runs at minute 7 to avoid :00 congestion)
|
||||
SELECT cron.schedule(
|
||||
'refresh-dashboard',
|
||||
'7 * * * *',
|
||||
'SELECT refresh_dashboard_summary()'
|
||||
);
|
||||
|
||||
-- Verify the schedule
|
||||
SELECT * FROM cron.job;
|
||||
169
sql/upgrade_dashboard_v2.sql
Normal file
169
sql/upgrade_dashboard_v2.sql
Normal file
|
|
@ -0,0 +1,169 @@
|
|||
-- Upgrade dashboard_summary: add hourly_stats column + update refresh function
|
||||
-- Run this in Supabase SQL Editor (safe to run on existing table)
|
||||
|
||||
-- 1. Add hourly_stats column if missing
|
||||
ALTER TABLE dashboard_summary
|
||||
ADD COLUMN IF NOT EXISTS hourly_stats jsonb DEFAULT '[]'::jsonb;
|
||||
|
||||
-- 2. Replace the refresh function with hourly support
|
||||
CREATE OR REPLACE FUNCTION refresh_dashboard_summary()
|
||||
RETURNS void AS $$
|
||||
DECLARE
|
||||
_daily jsonb;
|
||||
_hourly jsonb;
|
||||
_top jsonb;
|
||||
_os jsonb;
|
||||
_versions jsonb;
|
||||
_total_tokens bigint;
|
||||
_total_cost numeric;
|
||||
_total_requests int;
|
||||
_unique_instances int;
|
||||
_active_days int;
|
||||
BEGIN
|
||||
-- Daily totals: MAX per instance per day (beacon is cumulative), then SUM across instances
|
||||
WITH instance_daily AS (
|
||||
SELECT
|
||||
instance_id,
|
||||
created_at::date AS day,
|
||||
MAX(COALESCE(tokens_saved, 0)) AS tokens_saved,
|
||||
MAX(COALESCE(cost_saved_usd, 0)) AS cost_saved,
|
||||
MAX(COALESCE(requests, 0)) AS requests
|
||||
FROM proxy_telemetry_v2
|
||||
GROUP BY instance_id, created_at::date
|
||||
),
|
||||
daily_agg AS (
|
||||
SELECT
|
||||
day,
|
||||
SUM(tokens_saved) AS tokens_saved,
|
||||
SUM(cost_saved)::numeric(12,2) AS cost_saved,
|
||||
SUM(requests) AS requests,
|
||||
COUNT(DISTINCT instance_id) AS instances
|
||||
FROM instance_daily
|
||||
GROUP BY day
|
||||
ORDER BY day
|
||||
)
|
||||
SELECT
|
||||
COALESCE(jsonb_agg(jsonb_build_object(
|
||||
'date', day,
|
||||
'tokens_saved', tokens_saved,
|
||||
'cost_saved', cost_saved,
|
||||
'requests', requests,
|
||||
'instances', instances
|
||||
) ORDER BY day), '[]'::jsonb),
|
||||
COALESCE(SUM(tokens_saved), 0),
|
||||
COALESCE(SUM(cost_saved), 0),
|
||||
COALESCE(SUM(requests), 0),
|
||||
COUNT(DISTINCT day)
|
||||
INTO _daily, _total_tokens, _total_cost, _total_requests, _active_days
|
||||
FROM daily_agg;
|
||||
|
||||
-- Hourly totals: last 48 hours, MAX per instance per hour, then SUM across instances
|
||||
WITH instance_hourly AS (
|
||||
SELECT
|
||||
instance_id,
|
||||
date_trunc('hour', created_at) AS hour,
|
||||
MAX(COALESCE(tokens_saved, 0)) AS tokens_saved,
|
||||
MAX(COALESCE(cost_saved_usd, 0)) AS cost_saved,
|
||||
MAX(COALESCE(requests, 0)) AS requests
|
||||
FROM proxy_telemetry_v2
|
||||
WHERE created_at >= now() - interval '48 hours'
|
||||
GROUP BY instance_id, date_trunc('hour', created_at)
|
||||
),
|
||||
hourly_agg AS (
|
||||
SELECT
|
||||
hour,
|
||||
SUM(tokens_saved) AS tokens_saved,
|
||||
SUM(cost_saved)::numeric(12,2) AS cost_saved,
|
||||
SUM(requests) AS requests,
|
||||
COUNT(DISTINCT instance_id) AS instances
|
||||
FROM instance_hourly
|
||||
GROUP BY hour
|
||||
ORDER BY hour
|
||||
)
|
||||
SELECT COALESCE(jsonb_agg(jsonb_build_object(
|
||||
'hour', to_char(hour, 'YYYY-MM-DD HH24:MI'),
|
||||
'tokens_saved', tokens_saved,
|
||||
'cost_saved', cost_saved,
|
||||
'requests', requests,
|
||||
'instances', instances
|
||||
) ORDER BY hour), '[]'::jsonb)
|
||||
INTO _hourly
|
||||
FROM hourly_agg;
|
||||
|
||||
-- Unique instances
|
||||
SELECT COUNT(DISTINCT instance_id) INTO _unique_instances FROM proxy_telemetry_v2;
|
||||
|
||||
-- Top 20 instances by total tokens saved
|
||||
WITH instance_totals AS (
|
||||
SELECT
|
||||
instance_id,
|
||||
SUM(max_tokens) AS tokens_saved,
|
||||
SUM(max_cost)::numeric(12,2) AS cost_saved,
|
||||
MAX(os) AS os,
|
||||
MAX(version) AS version
|
||||
FROM (
|
||||
SELECT
|
||||
instance_id,
|
||||
created_at::date,
|
||||
MAX(COALESCE(tokens_saved, 0)) AS max_tokens,
|
||||
MAX(COALESCE(cost_saved_usd, 0)) AS max_cost,
|
||||
MAX(os) AS os,
|
||||
MAX(headroom_version) AS version
|
||||
FROM proxy_telemetry_v2
|
||||
GROUP BY instance_id, created_at::date
|
||||
) sub
|
||||
GROUP BY instance_id
|
||||
ORDER BY tokens_saved DESC
|
||||
LIMIT 20
|
||||
)
|
||||
SELECT COALESCE(jsonb_agg(jsonb_build_object(
|
||||
'instance_id', LEFT(instance_id, 8),
|
||||
'tokens_saved', tokens_saved,
|
||||
'cost_saved', cost_saved,
|
||||
'os', SPLIT_PART(COALESCE(os, '?'), ' ', 1),
|
||||
'version', version
|
||||
) ORDER BY tokens_saved DESC), '[]'::jsonb)
|
||||
INTO _top
|
||||
FROM instance_totals;
|
||||
|
||||
-- OS breakdown
|
||||
SELECT COALESCE(jsonb_object_agg(os_name, cnt), '{}'::jsonb)
|
||||
INTO _os
|
||||
FROM (
|
||||
SELECT SPLIT_PART(COALESCE(os, '?'), ' ', 1) AS os_name, COUNT(*) AS cnt
|
||||
FROM proxy_telemetry_v2
|
||||
GROUP BY os_name
|
||||
) sub;
|
||||
|
||||
-- Version breakdown
|
||||
SELECT COALESCE(jsonb_object_agg(COALESCE(headroom_version, '?'), cnt), '{}'::jsonb)
|
||||
INTO _versions
|
||||
FROM (
|
||||
SELECT headroom_version, COUNT(*) AS cnt
|
||||
FROM proxy_telemetry_v2
|
||||
GROUP BY headroom_version
|
||||
) sub;
|
||||
|
||||
-- Upsert the single summary row
|
||||
INSERT INTO dashboard_summary (id, updated_at, total_tokens_saved, total_cost_saved,
|
||||
total_requests, unique_instances, active_days, daily_stats, hourly_stats,
|
||||
top_instances, os_breakdown, version_breakdown)
|
||||
VALUES ('current', now(), _total_tokens, _total_cost, _total_requests,
|
||||
_unique_instances, _active_days, _daily, _hourly, _top, _os, _versions)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
updated_at = EXCLUDED.updated_at,
|
||||
total_tokens_saved = EXCLUDED.total_tokens_saved,
|
||||
total_cost_saved = EXCLUDED.total_cost_saved,
|
||||
total_requests = EXCLUDED.total_requests,
|
||||
unique_instances = EXCLUDED.unique_instances,
|
||||
active_days = EXCLUDED.active_days,
|
||||
daily_stats = EXCLUDED.daily_stats,
|
||||
hourly_stats = EXCLUDED.hourly_stats,
|
||||
top_instances = EXCLUDED.top_instances,
|
||||
os_breakdown = EXCLUDED.os_breakdown,
|
||||
version_breakdown = EXCLUDED.version_breakdown;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- 3. Refresh now to populate hourly data
|
||||
SELECT refresh_dashboard_summary();
|
||||
19
sql/upgrade_telemetry_cache_bust.sql
Normal file
19
sql/upgrade_telemetry_cache_bust.sql
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
-- Simplify cache bust tracking in proxy_telemetry_v2
|
||||
-- Run in Supabase SQL Editor
|
||||
|
||||
-- 1. Drop the over-engineered columns from the previous version
|
||||
ALTER TABLE proxy_telemetry_v2
|
||||
DROP COLUMN IF EXISTS cache_bust_count,
|
||||
DROP COLUMN IF EXISTS cache_bust_net_negative,
|
||||
DROP COLUMN IF EXISTS cache_bust_verdict;
|
||||
|
||||
-- 2. Keep just one column: tokens that lost their cache discount due to compression.
|
||||
-- Compare with tokens_saved to see if compression is net-positive:
|
||||
-- tokens_saved > cache_bust_tokens → compression wins
|
||||
-- tokens_saved < cache_bust_tokens → should freeze more of the prefix
|
||||
ALTER TABLE proxy_telemetry_v2
|
||||
ADD COLUMN IF NOT EXISTS cache_bust_tokens bigint DEFAULT 0;
|
||||
|
||||
-- 3. Drop the over-engineered dashboard column too
|
||||
ALTER TABLE dashboard_summary
|
||||
DROP COLUMN IF EXISTS cache_bust_stats;
|
||||
Loading…
Add table
Add a link
Reference in a new issue