diff --git a/docs/components/community-charts.tsx b/docs/components/community-charts.tsx deleted file mode 100644 index 8887126c0..000000000 --- a/docs/components/community-charts.tsx +++ /dev/null @@ -1,346 +0,0 @@ -'use client' - -import { useState } from 'react' -import { - AreaChart, Area, BarChart, Bar, - XAxis, YAxis, Tooltip, ResponsiveContainer, CartesianGrid, -} from 'recharts' - -// --- Embedded telemetry data --- -const DATA = { - total_tokens_saved: 41750654085, - total_cost_saved: 176635.62, - total_requests: 1194154, - unique_instances: 889, - active_days: 14, - daily_stats: [ - { date: '2026-03-30', requests: 1050, instances: 17, cost_saved: 42.29, tokens_saved: 7560502 }, - { date: '2026-03-31', requests: 26526, instances: 149, cost_saved: 1904.15, tokens_saved: 1004245860 }, - { date: '2026-04-01', requests: 53480, instances: 117, cost_saved: 3353.50, tokens_saved: 860455498 }, - { date: '2026-04-02', requests: 64906, instances: 123, cost_saved: 18007.10, tokens_saved: 3431477414 }, - { date: '2026-04-03', requests: 99872, instances: 162, cost_saved: 29238.20, tokens_saved: 6159073542 }, - { date: '2026-04-04', requests: 90992, instances: 147, cost_saved: 12863.30, tokens_saved: 1864667449 }, - { date: '2026-04-05', requests: 162541, instances: 142, cost_saved: 49802.60, tokens_saved: 11990261722 }, - { date: '2026-04-06', requests: 127494, instances: 174, cost_saved: 11533.70, tokens_saved: 3850552409 }, - { date: '2026-04-07', requests: 158840, instances: 201, cost_saved: 14260.90, tokens_saved: 4113326538 }, - { date: '2026-04-08', requests: 84459, instances: 186, cost_saved: 8383.12, tokens_saved: 2317153362 }, - { date: '2026-04-09', requests: 137856, instances: 208, cost_saved: 10663.80, tokens_saved: 2058570508 }, - { date: '2026-04-10', requests: 111043, instances: 192, cost_saved: 9893.92, tokens_saved: 1979992305 }, - { date: '2026-04-11', requests: 65851, instances: 147, cost_saved: 5853.92, tokens_saved: 1521911387 }, - { date: '2026-04-12', requests: 9244, instances: 27, cost_saved: 835.12, tokens_saved: 591405589 }, - ], - hourly_stats: [ - { hour: '2026-04-10 06:00', requests: 8548, instances: 9, cost_saved: 577.12, tokens_saved: 198669219 }, - { hour: '2026-04-10 07:00', requests: 8289, instances: 18, cost_saved: 456.34, tokens_saved: 172834374 }, - { hour: '2026-04-10 08:00', requests: 8066, instances: 23, cost_saved: 559.56, tokens_saved: 196181344 }, - { hour: '2026-04-10 09:00', requests: 4890, instances: 16, cost_saved: 206.95, tokens_saved: 124914685 }, - { hour: '2026-04-10 10:00', requests: 9531, instances: 21, cost_saved: 421.51, tokens_saved: 236470420 }, - { hour: '2026-04-10 11:00', requests: 5170, instances: 13, cost_saved: 174.45, tokens_saved: 114113494 }, - { hour: '2026-04-10 12:00', requests: 10982, instances: 20, cost_saved: 388.05, tokens_saved: 166308102 }, - { hour: '2026-04-10 13:00', requests: 8001, instances: 16, cost_saved: 228.53, tokens_saved: 134921875 }, - { hour: '2026-04-10 14:00', requests: 4950, instances: 15, cost_saved: 208.56, tokens_saved: 119763470 }, - { hour: '2026-04-10 15:00', requests: 9000, instances: 17, cost_saved: 1739.51, tokens_saved: 166911747 }, - { hour: '2026-04-10 16:00', requests: 12569, instances: 17, cost_saved: 1130.82, tokens_saved: 312958735 }, - { hour: '2026-04-10 17:00', requests: 12057, instances: 17, cost_saved: 443.07, tokens_saved: 203763560 }, - { hour: '2026-04-10 18:00', requests: 14011, instances: 18, cost_saved: 1018.06, tokens_saved: 294928335 }, - { hour: '2026-04-10 19:00', requests: 12599, instances: 17, cost_saved: 915.21, tokens_saved: 327535343 }, - { hour: '2026-04-10 20:00', requests: 10522, instances: 16, cost_saved: 2484.87, tokens_saved: 206735552 }, - { hour: '2026-04-10 21:00', requests: 7125, instances: 16, cost_saved: 928.27, tokens_saved: 289265979 }, - { hour: '2026-04-10 22:00', requests: 4481, instances: 9, cost_saved: 190.60, tokens_saved: 137301814 }, - { hour: '2026-04-10 23:00', requests: 4033, instances: 5, cost_saved: 154.82, tokens_saved: 126989500 }, - { hour: '2026-04-11 00:00', requests: 8998, instances: 12, cost_saved: 770.90, tokens_saved: 254824094 }, - { hour: '2026-04-11 01:00', requests: 13729, instances: 12, cost_saved: 668.12, tokens_saved: 330717273 }, - { hour: '2026-04-11 02:00', requests: 3738, instances: 2, cost_saved: 152.18, tokens_saved: 126547280 }, - { hour: '2026-04-11 03:00', requests: 4449, instances: 4, cost_saved: 185.03, tokens_saved: 134597314 }, - { hour: '2026-04-11 04:00', requests: 5961, instances: 9, cost_saved: 303.65, tokens_saved: 157734893 }, - { hour: '2026-04-11 05:00', requests: 6550, instances: 8, cost_saved: 248.84, tokens_saved: 158834743 }, - { hour: '2026-04-11 06:00', requests: 5429, instances: 8, cost_saved: 268.11, tokens_saved: 155129772 }, - { hour: '2026-04-11 07:00', requests: 5671, instances: 11, cost_saved: 225.25, tokens_saved: 143244185 }, - { hour: '2026-04-11 08:00', requests: 7258, instances: 12, cost_saved: 264.80, tokens_saved: 158618368 }, - { hour: '2026-04-11 09:00', requests: 8732, instances: 7, cost_saved: 301.46, tokens_saved: 157720064 }, - { hour: '2026-04-11 10:00', requests: 6058, instances: 6, cost_saved: 230.22, tokens_saved: 155936384 }, - { hour: '2026-04-11 11:00', requests: 7615, instances: 13, cost_saved: 444.21, tokens_saved: 288328690 }, - { hour: '2026-04-11 12:00', requests: 6749, instances: 10, cost_saved: 716.35, tokens_saved: 542054609 }, - { hour: '2026-04-11 13:00', requests: 6276, instances: 10, cost_saved: 2259.74, tokens_saved: 572384133 }, - { hour: '2026-04-11 14:00', requests: 8858, instances: 16, cost_saved: 846.04, tokens_saved: 602153567 }, - { hour: '2026-04-11 15:00', requests: 9284, instances: 12, cost_saved: 811.16, tokens_saved: 581585147 }, - { hour: '2026-04-11 16:00', requests: 7390, instances: 16, cost_saved: 749.69, tokens_saved: 566925112 }, - { hour: '2026-04-11 17:00', requests: 11558, instances: 13, cost_saved: 1023.99, tokens_saved: 623709492 }, - { hour: '2026-04-11 18:00', requests: 6154, instances: 10, cost_saved: 709.22, tokens_saved: 546162461 }, - { hour: '2026-04-11 19:00', requests: 5711, instances: 11, cost_saved: 660.33, tokens_saved: 548094492 }, - { hour: '2026-04-11 20:00', requests: 9287, instances: 13, cost_saved: 800.20, tokens_saved: 575078608 }, - { hour: '2026-04-11 21:00', requests: 5858, instances: 9, cost_saved: 675.77, tokens_saved: 547657183 }, - { hour: '2026-04-11 22:00', requests: 5523, instances: 13, cost_saved: 1133.82, tokens_saved: 638333840 }, - { hour: '2026-04-11 23:00', requests: 7668, instances: 8, cost_saved: 903.46, tokens_saved: 592286174 }, - { hour: '2026-04-12 00:00', requests: 5723, instances: 9, cost_saved: 655.44, tokens_saved: 543699655 }, - { hour: '2026-04-12 01:00', requests: 5291, instances: 8, cost_saved: 673.29, tokens_saved: 552129148 }, - { hour: '2026-04-12 02:00', requests: 6689, instances: 9, cost_saved: 698.83, tokens_saved: 551696076 }, - { hour: '2026-04-12 03:00', requests: 6532, instances: 8, cost_saved: 749.06, tokens_saved: 569436595 }, - { hour: '2026-04-12 04:00', requests: 4805, instances: 3, cost_saved: 644.29, tokens_saved: 540846848 }, - { hour: '2026-04-12 05:00', requests: 5181, instances: 6, cost_saved: 642.90, tokens_saved: 539173642 }, - { hour: '2026-04-12 06:00', requests: 4739, instances: 1, cost_saved: 637.17, tokens_saved: 537999942 }, - ], - top_instances: [ - { os: 'Windows', version: '0.5.18', cost_saved: 36325.40, instance_id: '1d5d8ed0', tokens_saved: 8852060567 }, - { os: 'Linux', version: '0.5.19', cost_saved: 2188.80, instance_id: '96d9632f', tokens_saved: 2395311403 }, - { os: 'Windows', version: '0.5.17', cost_saved: 11878.30, instance_id: '1e850cb3', tokens_saved: 2375786993 }, - { os: 'Windows', version: '0.5.17', cost_saved: 9080.36, instance_id: '0cdfb8e9', tokens_saved: 1816351278 }, - { os: 'Linux', version: '0.5.18', cost_saved: 7580.30, instance_id: '7456b0f9', tokens_saved: 1635744469 }, - { os: 'Darwin', version: '0.5.16', cost_saved: 1772.99, instance_id: '1661b732', tokens_saved: 1488844495 }, - { os: 'Darwin', version: '0.5.17', cost_saved: 4667.23, instance_id: '08bf5ae1', tokens_saved: 933450700 }, - { os: 'Darwin', version: '0.5.18', cost_saved: 0, instance_id: 'e20f01b6', tokens_saved: 565038755 }, - { os: 'Linux', version: '0.5.18', cost_saved: 538.37, instance_id: '5b0795b2', tokens_saved: 557489086 }, - { os: 'Windows', version: '0.5.18', cost_saved: 1773.69, instance_id: 'eff9e644', tokens_saved: 503362483 }, - { os: 'Windows', version: '0.5.18', cost_saved: 1812.66, instance_id: '388102c7', tokens_saved: 485305324 }, - { os: 'Linux', version: '0.5.17', cost_saved: 2297.45, instance_id: '3ee70444', tokens_saved: 468622655 }, - { os: 'Darwin', version: '0.5.16', cost_saved: 319.65, instance_id: '6e758d1e', tokens_saved: 437502391 }, - { os: 'Linux', version: '0.5.18', cost_saved: 1461.02, instance_id: 'b2e71a28', tokens_saved: 415224504 }, - { os: 'Darwin', version: '0.5.21', cost_saved: 509.44, instance_id: '8b3795aa', tokens_saved: 353583838 }, - { os: 'Linux', version: '0.5.17', cost_saved: 1663.48, instance_id: 'b6a1a735', tokens_saved: 334438796 }, - { os: 'Linux', version: '0.5.21', cost_saved: 1603.77, instance_id: 'd0c16fa0', tokens_saved: 322890921 }, - { os: 'Linux', version: '0.5.18', cost_saved: 1580.12, instance_id: '4140eb00', tokens_saved: 316804284 }, - { os: 'Darwin', version: '0.5.21', cost_saved: 1052.77, instance_id: '2b11b55c', tokens_saved: 308601543 }, - { os: 'Darwin', version: '0.5.17', cost_saved: 1473.78, instance_id: '1ede777b', tokens_saved: 296950447 }, - ], -} - -// --- Formatters --- - -function fmt(n: number): string { - if (n >= 1e9) return `${(n / 1e9).toFixed(1)}B` - if (n >= 1e6) return `${(n / 1e6).toFixed(1)}M` - if (n >= 1e3) return `${(n / 1e3).toFixed(1)}K` - return n.toFixed(0) -} - -function fmtUsd(n: number): string { - if (n >= 1e3) return `$${(n / 1e3).toFixed(1)}K` - return `$${n.toFixed(0)}` -} - -const MONTHS = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec'] - -function fmtDateDaily(d: string): string { - const [, m, day] = d.split('-') - return `${MONTHS[parseInt(m, 10) - 1]} ${parseInt(day, 10)}` -} - -function fmtDateHourly(d: string): string { - // "2026-04-10 06:00" → "Apr 10 6am" - const [date, time] = d.split(' ') - const [, m, day] = date.split('-') - const hour = parseInt(time.split(':')[0], 10) - const ampm = hour >= 12 ? 'pm' : 'am' - const h12 = hour === 0 ? 12 : hour > 12 ? hour - 12 : hour - return `${MONTHS[parseInt(m, 10) - 1]} ${parseInt(day, 10)} ${h12}${ampm}` -} - -// --- Components --- - -const PURPLE = 'hsl(262, 52%, 56%)' -const PURPLE_LIGHT = 'hsl(262, 60%, 65%)' - -type Metric = 'cost_saved' | 'requests' | 'tokens_saved' -type TimeRange = 'daily' | 'hourly' - -function ToggleButton({ active, onClick, children }: { active: boolean; onClick: () => void; children: React.ReactNode }) { - return ( - - ) -} - -function CustomTooltip({ active, payload, label }: any) { - if (!active || !payload?.length) return null - return ( -
-

{label}

- {payload.map((p: any) => ( -

- {p.dataKey === 'cost_saved' ? fmtUsd(p.value) : fmt(p.value)} -

- ))} -
- ) -} - -function StatsCards() { - return ( -
- {[ - { label: 'Cost Saved', value: fmtUsd(DATA.total_cost_saved) }, - { label: 'Requests Optimized', value: fmt(DATA.total_requests) }, - { label: 'Active Instances', value: fmt(DATA.unique_instances) }, - { label: 'Active Days', value: String(DATA.active_days) }, - ].map(s => ( -
- {s.value} - {s.label} -
- ))} -
- ) -} - -function AreaChartSection() { - const [metric, setMetric] = useState('tokens_saved') - const [timeRange, setTimeRange] = useState('hourly') - - const isHourly = timeRange === 'hourly' - const chartData: { label: string; requests: number; instances: number; cost_saved: number; tokens_saved: number }[] = !isHourly - ? DATA.daily_stats.map(d => ({ label: fmtDateDaily(d.date), requests: d.requests, instances: d.instances, cost_saved: d.cost_saved, tokens_saved: d.tokens_saved })) - : DATA.hourly_stats.map(d => ({ label: fmtDateHourly(d.hour), requests: d.requests, instances: d.instances, cost_saved: d.cost_saved, tokens_saved: d.tokens_saved })) - - const metricLabels: Record = { - cost_saved: 'Cost Saved', - requests: 'Requests', - tokens_saved: 'Tokens Saved', - } - - const tickFormatter = (v: number) => - metric === 'cost_saved' ? fmtUsd(v) : fmt(v) - - return ( -
-
-
- {(['cost_saved', 'requests', 'tokens_saved'] as Metric[]).map(m => ( - setMetric(m)}> - {metricLabels[m]} - - ))} -
-
- {(['hourly', 'daily'] as TimeRange[]).map(t => ( - setTimeRange(t)}> - {t === 'hourly' ? 'Last 48h' : 'Daily'} - - ))} -
-
-
- - - - - - - - - - - - } /> - - - -
-
- ) -} - -function BarChartSection() { - const barData = DATA.top_instances.slice(0, 10).map(i => ({ - name: `${i.instance_id} (${i.os})`, - tokens_saved: i.tokens_saved, - cost_saved: i.cost_saved, - })) - - return ( -
-
- - - - - - { - if (!active || !payload?.length) return null - const d = payload[0].payload - return ( -
-

{d.name}

-

{fmt(d.tokens_saved)} tokens

-

{fmtUsd(d.cost_saved)} saved

-
- ) - }} /> - - - -
-
- ) -} - -function DataTable() { - return ( -
-
- - - - - - - - - - - - {DATA.top_instances.map((inst, i) => ( - - - - - - - - ))} - -
InstanceOSVersionTokens SavedCost Saved
{inst.instance_id}{inst.os}{inst.version}{fmt(inst.tokens_saved)}{fmtUsd(inst.cost_saved)}
-
-
- ) -} - -export function CommunityCharts({ section }: { section?: 'stats' | 'area' | 'bar' | 'table' }) { - if (section === 'stats') return - if (section === 'area') return - if (section === 'bar') return - if (section === 'table') return - - // Render all if no section specified - return ( -
- - - - -
- ) -} diff --git a/docs/components/community-stats-header.tsx b/docs/components/community-stats-header.tsx deleted file mode 100644 index b2d9d8d97..000000000 --- a/docs/components/community-stats-header.tsx +++ /dev/null @@ -1,34 +0,0 @@ -import { fetchCommunityStats, fmtNum, fmtUsd } from '@/lib/telemetry'; - -/** - * Server component that renders the top-level stats for the community savings page. - * Fetches live data from Supabase at render time. - */ -export async function CommunityStatsHeader() { - const data = await fetchCommunityStats(); - - const stats = [ - { value: fmtNum(data.total_tokens_saved), label: 'Tokens Saved' }, - { value: fmtUsd(data.total_cost_saved), label: 'Cost Saved' }, - { value: fmtNum(data.total_requests), label: 'Requests Optimized' }, - { value: fmtNum(data.unique_instances), label: 'Active Instances' }, - ]; - - return ( -
- {stats.map((s) => ( -
- - {s.value} - - - {s.label} - -
- ))} -
- ); -} diff --git a/docs/components/live-stats.tsx b/docs/components/live-stats.tsx deleted file mode 100644 index 2d8f2790b..000000000 --- a/docs/components/live-stats.tsx +++ /dev/null @@ -1,43 +0,0 @@ -import Link from 'next/link'; -import { fetchCommunityStats, fmtNum, fmtUsd } from '@/lib/telemetry'; - -/** - * Server component that fetches live stats from Supabase at render time. - * Falls back to hardcoded data if the API is unreachable. - */ -export async function LiveStats() { - const data = await fetchCommunityStats(); - - const stats = [ - { value: fmtNum(data.total_tokens_saved), label: 'Tokens Saved' }, - { value: fmtUsd(data.total_cost_saved), label: 'Cost Saved' }, - { value: fmtNum(data.total_requests), label: 'Requests Optimized' }, - { value: fmtNum(data.unique_instances), label: 'Active Instances' }, - ]; - - return ( -
-
- {stats.map((s) => ( -
- - {s.value} - - - {s.label} - -
- ))} -
- - View detailed charts and breakdowns → - -
- ); -} diff --git a/docs/components/marketing.tsx b/docs/components/marketing.tsx index 8b3c8f669..99af69f9e 100644 --- a/docs/components/marketing.tsx +++ b/docs/components/marketing.tsx @@ -2,43 +2,6 @@ import Link from 'next/link'; import { Button } from './button'; import { CodeBlock } from './code-block'; -// --- Live Stats Grid --- - -const liveStats = [ - { value: '$176.6K', label: 'Cost Saved' }, - { value: '1.19M', label: 'Requests Optimized' }, - { value: '889', label: 'Active Instances' }, - { value: '14', label: 'Active Days' }, -]; - -export function LiveStats() { - return ( -
-
- {liveStats.map((s) => ( -
- - {s.value} - - - {s.label} - -
- ))} -
- - View detailed charts and breakdowns → - -
- ); -} - // --- Key Features Grid --- const features: { diff --git a/docs/components/mdx.tsx b/docs/components/mdx.tsx index 37e680523..176819c66 100644 --- a/docs/components/mdx.tsx +++ b/docs/components/mdx.tsx @@ -9,10 +9,6 @@ import { KeyFeatures, FrameworkIntegrations, } from './marketing'; -import { LiveStats } from './live-stats'; -import { CommunityStatsHeader } from './community-stats-header'; -import { StatsSection } from './stats'; -import { CommunityCharts } from './community-charts'; const generator = createGenerator(); @@ -26,10 +22,6 @@ export function getMDXComponents(components?: MDXComponents) { TypeTable, Tab, Tabs, - StatsSection, - CommunityCharts, - CommunityStatsHeader, - LiveStats, KeyFeatures, FrameworkIntegrations, ...components, diff --git a/docs/components/stats.tsx b/docs/components/stats.tsx deleted file mode 100644 index 2f53fbef0..000000000 --- a/docs/components/stats.tsx +++ /dev/null @@ -1,71 +0,0 @@ -import { Map } from '@/components/map' - -export function StatsSection() { - return ( -
-
-
- -
-
-
-
-
-

- The Context Optimization Layer for{' '} - - LLM Applications - -

-

- Compress everything your AI agent reads.{' '} - - Same answers, fraction of the tokens. - -

-
-
-
- - 87 % - -

- - Token Reduction - -

-
-
- - 100 % - -

- - Accuracy - -

-
-
- 6 -

- - Algorithms - -

-
-
- - 100 + - -

- - Providers - -

-
-
-
-
-
- ) -} diff --git a/docs/content/docs/architecture.mdx b/docs/content/docs/architecture.mdx index 5a903b09d..089cfca7c 100644 --- a/docs/content/docs/architecture.mdx +++ b/docs/content/docs/architecture.mdx @@ -1,11 +1,11 @@ --- title: Architecture -description: How Headroom's three-stage compression pipeline works, from message parsing through transform execution to provider cache optimization. +description: How Headroom compresses LLM traffic — from request interception through the ContentRouter compression pipeline to provider cache optimization. --- -Headroom sits between your application and the LLM provider. It intercepts messages, compresses them intelligently, and forwards the optimized request. The response comes back unchanged. +Headroom sits between your application and the LLM provider. It intercepts the request, compresses the parts that carry the most redundant tokens — tool outputs, file reads, logs, search results — and forwards the optimized request upstream. The provider's response is returned unchanged. -## High-Level Flow +## High-level flow ``` +---------------------------------------------------------------+ @@ -14,119 +14,107 @@ Headroom sits between your application and the LLM provider. It intercepts messa | v +---------------------------------------------------------------+ -| HEADROOM CLIENT | -| +-----------+ +------------+ +---------+ | -| | ANALYZE | > | TRANSFORM | > | CALL | | -| | (Parser) | | (Pipeline)| | (API) | | -| +-----------+ +------------+ +---------+ | -| | | | | -| v v v | -| Count tokens Apply compressions Send to LLM provider | -| Detect waste Preserve meaning Log metrics | +| HEADROOM | +| Proxy (FastAPI) · Python compress() · TS compress() | +| | | +| v | +| Transform pipeline ──▶ ContentRouter | +| (detect content type, route to one compressor) | +| | | +| v | +| Backend (direct · LiteLLM · any-llm) | +---------------------------------------------------------------+ | v +---------------------------------------------------------------+ -| OPENAI / ANTHROPIC / GOOGLE | +| OPENAI · ANTHROPIC · GOOGLE · BEDROCK · 100+ | +---------------------------------------------------------------+ ``` -## Entry Points +## Entry points -Headroom can be used in three ways, all feeding into the same pipeline: +Headroom can be used three ways, all feeding the same compression pipeline: -| Entry Point | How It Works | Code Changes | +| Entry point | How it works | Code changes | |-------------|-------------|--------------| -| **SDK Mode** | Wrap your LLM client with `HeadroomClient` | Minimal -- swap client constructor | -| **Proxy Mode** | Run `headroom proxy` and point your client at it | Zero -- just change the base URL | -| **Integrations** | LangChain, Vercel AI SDK, Agno adapters | Framework-specific setup | +| **Proxy mode** | Run `headroom proxy` and point your client's base URL at it | Zero — just change the base URL | +| **SDK mode** | Call `compress()` (Python or TypeScript) on your messages before you send them | Minimal — one function call | +| **Integrations** | LangChain, Vercel AI SDK, Agno, Strands, LiteLLM, MCP adapters | Framework-specific setup | -## The Transform Pipeline +In proxy mode the server is a FastAPI app with per-provider handlers (Anthropic, OpenAI, Gemini, Bedrock) that each run the same compression pipeline before forwarding through the selected [backend](/docs/proxy#cloud-providers). -Messages flow through a sequence of transforms. Each transform is independent, safe to skip, and fails gracefully (returns original content unchanged). +## The compression pipeline -### Stage 1: Cache Aligner +The proxy assembles a small, ordered pipeline. Every transform is independent, safe to skip, and **fails open** — on any error it returns the content unchanged and the request still goes through. -Detects dynamic content (dates, UUIDs, session tokens) in your system prompt and reports prefix metrics. Keep the stable prefix and live context separated in the caller so provider caches (Anthropic `cache_control`, OpenAI prefix caching) can hit on repeated calls. +1. **Tool-result interceptor** *(opt-in)* — light structural interceptors such as ast-grep Read outlining. Off unless you pass `--intercept-tool-results`. +2. **CacheAligner** *(off by default)* — a detector that reports dynamic-prefix drift (dates, UUIDs, session tokens). It **never mutates, moves, or rewrites** content. It is disabled by default and hard-disabled inside the proxy; it exists to surface prefix-stability metrics, not to change your messages. +3. **ContentRouter** — the workhorse that does essentially all of the compression. See below. -``` -Observed: "You are helpful. Current Date: 2024-12-15" - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Changes daily = cache miss every day +The pipeline **never drops or reorders messages**. Earlier versions shipped a "context manager" stage (rolling-window / intelligent-context scoring) that deleted old turns to make the request fit the context window. That stage was **removed** — the `RollingWindowConfig`, `IntelligentContextConfig`, and `ScoringWeights` classes no longer exist. Headroom now does *live-zone-only* compression: it compresses content in place and leaves the message list intact. See [Context Management](/docs/context-management). -Caller layout: - "You are helpful." [stable prefix] - "[Context: Current Date: 2024-12-15]" [live context] -``` +### ContentRouter -Overhead: sub-millisecond. +ContentRouter detects the type of each content block and dispatches it to exactly one compressor: -### Stage 2: Smart Crusher +| Detected content | Compressor | Typical savings | +|---|---|---| +| JSON arrays (tool outputs) | SmartCrusher | 70–90% | +| Source code | CodeAwareCompressor (opt-in; off by default) | 40–70% | +| Search / grep results | SearchCompressor | 80–95% | +| Build / test logs | LogCompressor | 85–95% | +| Diffs | DiffCompressor | 40–80% | +| HTML | HTMLExtractor (trafilatura) | ~95% | +| Tabular (CSV/TSV/markdown tables) | TabularCompressor | 60–90% | +| Structured config (YAML/TOML/INI) | ConfigCompressor | 40–70% | +| Plain text | TextCrusher | 30–60% | +| Anything else | Kompress (ML fallback) | varies | -Analyzes tool output content and compresses it using statistical methods. This is where the bulk of token savings come from. +To avoid recompressing the same content, ContentRouter keeps a **two-tier, TTL-bounded cache**: a *skip set* of content already known not to compress, and a *result cache* of previously compressed output. Default TTL is 30 minutes. -**What it does:** +### Rust core -1. Parses JSON arrays in tool outputs -2. Runs field-level statistical analysis (variance, uniqueness, change points) -3. Selects a representative subset using the Kneedle algorithm on bigram coverage -4. Preserves errors, anomalies, and distribution boundaries unconditionally -5. Factors out constant fields shared by all items +The heaviest compressors run in a native Rust extension — `headroom._core`, built with PyO3 — that the proxy loads at startup. SmartCrusher and the search/log/diff compressors and content detection are Rust-backed; the Python classes you import are thin, API-compatible shims over them. The ML fallback (**Kompress**, a ModernBERT token compressor) runs separately through ONNX Runtime, either locally or offloaded to a remote endpoint (see [Text & Logs](/docs/text-and-logs)). -**Strategies by content type:** +## Compression modes and provider caches -| Content | Strategy | Typical Savings | -|---------|----------|-----------------| -| JSON arrays of dicts | Statistical sampling + anomaly preservation | 83--95% | -| JSON arrays of strings | Dedup + adaptive sampling | 60--90% | -| JSON arrays of numbers | Statistical summary + outlier preservation | 70--85% | -| Build/test logs | Pattern clustering | 85--94% | -| HTML | Article extraction (trafilatura-based) | ~95% | +Headroom runs in one of two modes (`--mode`, default `cache`): -**Item retention split:** 30% from array start (schema), 15% from end (recency), 55% by importance score. Error items are always kept regardless of budget. +- **`cache` mode (default)** — compresses only the newest delta in each turn and forwards prior turns byte-faithfully, so the provider's prefix cache is never invalidated mid-conversation. Best for coding agents and any long, multi-turn session. +- **`token` mode** — prioritizes raw token removal and may recompress earlier turns, trading some cache stability for maximum savings. -Overhead: 1--50ms for typical payloads. Scales linearly with input size. +Because prefix caching is where most of the cost savings live on multi-turn workloads, keeping the stable prefix intact matters. What each provider's cache buys you: -### Stage 3: Context Manager +| Provider | Mechanism | Savings on cached tokens | +|----------|-----------|--------------------------| +| Anthropic | `cache_control` on the stable prefix | up to ~90% | +| OpenAI | automatic prefix caching | up to ~50% | +| Google | `CachedContent` API | up to ~75% | -Ensures the final message array fits within the model's context window. - -**Rolling Window** (default): Drops oldest messages first, preserving system prompt and recent turns. Tool calls and their responses are dropped as atomic units. - -**Intelligent Context** (advanced): Scores every message on six dimensions (recency, semantic similarity, TOIN importance, error indicators, forward references, token density) and drops the lowest-scored messages first. Dropped messages are stored in CCR for potential retrieval. - -Overhead: sub-millisecond for Rolling Window; depends on scoring config for Intelligent Context. - -## Provider Cache Optimization - -After the pipeline, Headroom applies provider-specific cache hints: - -| Provider | Mechanism | Savings | -|----------|-----------|---------| -| Anthropic | `cache_control` blocks on stable prefix | Up to 90% on cached tokens | -| OpenAI | Prefix alignment for automatic caching | Up to 50% on cached tokens | -| Google | `CachedContent` API | Up to 75% on cached tokens | +See [Cache Optimization](/docs/cache-optimization) and [Savings Profiles](/docs/proxy#savings-profiles) for how the two modes interact with the built-in profiles. ## CCR: Compress-Cache-Retrieve -When SmartCrusher compresses a tool output or Intelligent Context drops messages, the original content is stored in a local compression cache. If the LLM needs the full data, it can request retrieval via a `headroom_retrieve` tool call. This makes compression reversible. +Compression is **reversible**. When ContentRouter compresses a tool output, the original is stored in a local Compress-Cache-Retrieve (CCR) store. If the model needs the full data, it calls a `headroom_retrieve` tool and gets the original back. ``` -Compress: 1000 items -> 15 items (stored original in CCR) -Cache: Hash-indexed local store (SQLite) -Retrieve: LLM calls headroom_retrieve("abc123") -> original 1000 items +Compress: 1000 items -> 15 items (original stored in CCR) +Cache: hash-indexed local SQLite store +Retrieve: model calls headroom_retrieve("abc123") -> original 1000 items ``` +CCR is **on by default**. Disable the markers and the injected retrieval tool with `--no-ccr`, or run a marker-free, format-native lossless mode with `--lossless`. See [Reversible Compression (CCR)](/docs/ccr). + ## TOIN: Tool Output Intelligence Network -TOIN learns compression patterns across sessions and users. When a tool is used repeatedly, TOIN builds up statistics about which fields matter, which items get retrieved, and what compression strategies work best. These learned patterns feed back into SmartCrusher and Intelligent Context scoring. +TOIN learns which fields matter for a given tool over repeated calls — which items get retrieved, which fields carry signal — and feeds that back into SmartCrusher's importance scoring so compression gets sharper for the tools you actually use. -Cold start: For new tool types, TOIN falls back to statistical heuristics. Patterns build up over time as tools are used. +TOIN is **local and observation-only**: it aggregates statistics on your own machine (or your own proxy instance). Nothing about your tools or traffic is shared across users or sent off the box. For a brand-new tool type it falls back to statistical heuristics and improves as it observes more calls. -## What Headroom Does NOT Touch +## What Headroom does not rewrite -- **User messages**: Never compressed (the user's intent must be preserved exactly) -- **System prompts**: Content preserved; dynamic parts are reported so callers can keep them outside the stable prefix -- **Code**: Passes through unchanged unless tree-sitter AST compression is explicitly enabled -- **Model responses**: Returned unchanged from the provider -- **Short content**: Tool outputs under 200 tokens pass through (overhead exceeds savings) +- **Your prompt text** — the natural-language instructions you write are preserved. Compression targets bulk content blocks (tool outputs, file reads, logs), not your intent. +- **System prompts** — preserved by default so the hottest part of the prefix cache stays stable. A savings profile can opt into compacting them. +- **Code** — passes through unchanged unless AST-based code compression is explicitly enabled (off by default). +- **Model responses** — returned from the provider unchanged. +- **Short content** — blocks below the minimum-token threshold pass through (overhead would exceed savings). diff --git a/docs/content/docs/benchmarks.mdx b/docs/content/docs/benchmarks.mdx index f17da6c86..db7b0cfd6 100644 --- a/docs/content/docs/benchmarks.mdx +++ b/docs/content/docs/benchmarks.mdx @@ -1,9 +1,9 @@ --- title: Benchmarks -description: Compression performance, accuracy preservation, latency overhead, and real-world production telemetry from 250+ Headroom proxy instances. +description: Compression performance, accuracy preservation, and latency overhead, measured on reproducible local benchmarks. --- -Headroom's core promise: compress context without losing accuracy. This page covers compression benchmarks, accuracy evaluations, latency overhead, and production telemetry. +Headroom's core promise: compress context without losing accuracy. This page covers compression benchmarks, accuracy evaluations, and latency overhead. Every number below is measured locally and reproducible (see [Reproducing Results](#reproducing-results)). For local inference, the main benefit is often faster prompt processing rather than lower API spend. See [Local LLM prefill benchmarking](/docs/local-llm-prefill) for a reproducible passthrough-vs-optimized proxy workflow. @@ -107,45 +107,10 @@ Compression pays for itself in latency for 11 of 12 tested scenarios against Cla ContentRouter accounts for 91--98% of pipeline cost on average. CacheAligner is sub-millisecond. -## Production Telemetry - -Real-world data from **50,000+ proxy sessions** across 250+ unique instances (March--April 2026). Collected via anonymous telemetry (opt-in: `HEADROOM_TELEMETRY=on`; telemetry is off by default). - -### Proxy Overhead - -| Percentile | Latency | -|---|---| -| **Median (P50)** | **52ms** | -| P90 | 309ms | -| P99 | 4,172ms | -| Mean | 161ms | - -The median 52ms overhead is negligible compared to LLM inference time (typically 2--10 seconds). - -### Compression Rate - -| Percentile | Compression | -|---|---| -| P25 | 4.8% | -| **Median** | **4.8%** | -| P75 | 6.9% | -| Mean | 11.3% | - -Median compression is modest because many requests are short conversational turns. Heavy tool-use sessions (file reads, shell output) see 40--80% compression. - -### Fleet Summary - -| Metric | Value | -|---|---| -| Clean instances | 249 | -| Total tokens saved | 1.4 billion | -| Total savings | ~$4,000 | -| OS distribution | Linux 57%, macOS 38%, Windows 5% | - ## Reproducing Results ```bash -git clone https://github.com/chopratejas/headroom.git +git clone https://github.com/headroomlabs-ai/headroom.git cd headroom pip install -e ".[evals,html]" pytest tests/test_evals/ -v -s diff --git a/docs/content/docs/cache-optimization.mdx b/docs/content/docs/cache-optimization.mdx index 3f9f199d0..baf3831d3 100644 --- a/docs/content/docs/cache-optimization.mdx +++ b/docs/content/docs/cache-optimization.mdx @@ -5,6 +5,10 @@ description: Stabilize message prefixes for provider KV cache hits and configure LLM providers cache prompt prefixes to avoid reprocessing identical input on repeated calls. Headroom's **CacheAligner** is detector-only, so it surfaces prefix drift, reports observability data, and leaves message assembly to the caller. + +CacheAligner is **disabled by default** and hard-disabled inside the proxy — it never runs on your traffic unless you explicitly enable it, and even then it only *reports* metrics; it never repairs a prefix. In proxy mode, prefix-cache stability comes from **cache mode** (`--mode cache`, the default), which compresses only the newest delta and forwards prior turns byte-faithfully. See [Savings Profiles](/docs/proxy#savings-profiles). + + ## What CacheAligner reports System prompts often contain dynamic content, such as dates, session IDs, and timestamps, that changes between requests. Even a single character difference at the start of a prompt invalidates the entire provider cache. diff --git a/docs/content/docs/code-compression.mdx b/docs/content/docs/code-compression.mdx index 32b59c89b..f407e96b1 100644 --- a/docs/content/docs/code-compression.mdx +++ b/docs/content/docs/code-compression.mdx @@ -11,7 +11,7 @@ Naive truncation breaks code. Cutting a function in half leaves invalid syntax t - **Syntax validity** -- output always parses correctly - **Structural preservation** -- imports, signatures, types, decorators are kept intact -- **Lightweight** -- ~50MB (tree-sitter) vs ~1GB for LLMLingua +- **Lightweight** -- ~50MB of tree-sitter parsers, loaded lazily and cached ## Supported Languages @@ -150,7 +150,8 @@ pip install "headroom-ai[code]" Tree-sitter parsers are lazy-loaded and cached. You can free memory when done: ```python -from headroom.transforms import is_tree_sitter_available, unload_tree_sitter +from headroom.transforms import is_tree_sitter_available +from headroom.transforms.code_compressor import unload_tree_sitter # Check if tree-sitter is installed print(is_tree_sitter_available()) # True diff --git a/docs/content/docs/community-savings.mdx b/docs/content/docs/community-savings.mdx deleted file mode 100644 index a0ec8b702..000000000 --- a/docs/content/docs/community-savings.mdx +++ /dev/null @@ -1,22 +0,0 @@ ---- -title: Community Savings -description: Aggregate savings from Headroom instances across the community. Anonymous telemetry data — no prompts, no content, no PII. ---- - -Real-time aggregate metrics from Headroom proxy instances worldwide. All data is anonymous — only token counts, compression ratios, and cost estimates are collected. Telemetry is off by default; [opt in](https://github.com/chopratejas/headroom/blob/main/headroom/telemetry/beacon.py) with `HEADROOM_TELEMETRY=on`. - -## Overview - - - -## Savings Over Time - - - -## Top Savings by Instance - - - -## Instance Details - - diff --git a/docs/content/docs/configuration.mdx b/docs/content/docs/configuration.mdx index dbf68c022..0be97f2ab 100644 --- a/docs/content/docs/configuration.mdx +++ b/docs/content/docs/configuration.mdx @@ -284,7 +284,7 @@ headroom proxy --learn --min-evidence 3 |----------|-------------|---------| | `HEADROOM_HOST` | Proxy bind host | `127.0.0.1` | | `HEADROOM_PORT` | Proxy bind port | `8787` | -| `HEADROOM_MODE` | Proxy optimization mode: `token` or `cache` | `token` | +| `HEADROOM_MODE` | Proxy optimization mode: `token` or `cache` | `cache` | | `HEADROOM_WORKERS` | Uvicorn worker count | `1` | | `HEADROOM_LIMIT_CONCURRENCY` | Maximum concurrent connections before 503 | `1000` | | `HEADROOM_MAX_CONNECTIONS` | Maximum upstream HTTP connections | `500` | @@ -292,7 +292,7 @@ headroom proxy --learn --min-evidence 3 | `HEADROOM_KEEPALIVE_EXPIRY` | Seconds an idle upstream keep-alive connection is kept open | `90` | | `HEADROOM_HTTP_PROXY` | HTTP proxy URL for upstream provider requests only; HTTPS provider APIs use CONNECT | -- | | `HEADROOM_BUDGET` | Daily budget limit in USD | -- | -| `HEADROOM_TELEMETRY` | Set to `on` to opt in to anonymous telemetry | `off` (opt-in) | +| `HEADROOM_TELEMETRY` | Set to `on` for **local-only** usage stats (powers your own `/stats` and dashboard; nothing is sent externally) | `off` | | `HEADROOM_STATELESS` | Set to `true` to disable filesystem writes | `false` | | `HEADROOM_MODEL_LIMITS` | Custom model config (JSON string or file path) | -- | | `HEADROOM_BASE_URL` | Base URL of the Headroom proxy (TypeScript SDK) | `http://localhost:8787` | @@ -302,7 +302,6 @@ headroom proxy --learn --min-evidence 3 | `HEADROOM_SAVINGS_PATH` | Override persistent savings file location. Always wins when set. | derived from `${HEADROOM_WORKSPACE_DIR}` | | `HEADROOM_TOIN_PATH` | Override TOIN telemetry file location. Always wins when set. | derived from `${HEADROOM_WORKSPACE_DIR}` | | `HEADROOM_SUBSCRIPTION_STATE_PATH` | Override subscription tracker state file. Always wins when set. | derived from `${HEADROOM_WORKSPACE_DIR}` | -| `HEADROOM_TELEMETRY` | Set to `on` to opt in to anonymous telemetry | `off` | | `HEADROOM_PERIODIC_TOIN_STATS` | Controls periodic TOIN stats logging in long-lived proxy workers. Set to `0`, `false`, `off`, or `no` to disable the 5-minute stats loop without disabling TOIN learning or request-time feedback. | `true` | | `HEADROOM_MEMORY_INJECTION_MODE` | Memory-context routing mode: `live_zone_tail` (default) or `disabled`. The legacy `system_prompt` mode was retired by PR-A2; supplying it raises. | `live_zone_tail` | | `HEADROOM_PROXY_PYTHON_FORWARDER_MODE` | Python forwarder serialization mode. `byte_faithful` (default) forwards original request bytes verbatim when no transform mutated the body and re-serializes canonically only when needed — keeps Anthropic prompt-cache hit-rate intact. `legacy_json_kwarg` is an explicit operator opt-in for emergency rollback to the historical `httpx ... json=body` behavior. NOT a fallback — only flip on explicit operator decision. | `byte_faithful` | @@ -373,7 +372,7 @@ precedence rules: explicit argument > per-resource env var > derived from canonical root > default. Every legacy env var continues to work unchanged. -See the **[Filesystem Contract](https://github.com/chopratejas/headroom/blob/main/wiki/filesystem-contract.md)** +See the **[Filesystem Contract](https://github.com/headroomlabs-ai/headroom/blob/main/wiki/filesystem-contract.md)** page for the full bucket table, plugin-author guidance, and the Docker naming overlap note (`HEADROOM_WORKSPACE` is *not* the same as `HEADROOM_WORKSPACE_DIR`). diff --git a/docs/content/docs/docker-install.mdx b/docs/content/docs/docker-install.mdx index d1332800f..747330cd9 100644 --- a/docs/content/docs/docker-install.mdx +++ b/docs/content/docs/docker-install.mdx @@ -10,13 +10,13 @@ Run Headroom without installing Python or Node.js on the host. The install scrip ### Linux ```bash -curl -fsSL https://raw.githubusercontent.com/chopratejas/headroom/main/scripts/install.sh | bash +curl -fsSL https://raw.githubusercontent.com/headroomlabs-ai/headroom/main/scripts/install.sh | bash ``` ### macOS (bash 4.3+) ```bash -curl -fsSL https://raw.githubusercontent.com/chopratejas/headroom/main/scripts/install.sh | "$(brew --prefix bash)/bin/bash" +curl -fsSL https://raw.githubusercontent.com/headroomlabs-ai/headroom/main/scripts/install.sh | "$(brew --prefix bash)/bin/bash" ``` Stock `/bin/bash` on macOS is 3.2, so install a newer bash first (for example via Homebrew) and run the installer with that shell. The installed wrapper pins that same bash interpreter so later invocations stay on the supported runtime. @@ -24,13 +24,13 @@ Stock `/bin/bash` on macOS is 3.2, so install a newer bash first (for example vi ### Windows PowerShell ```powershell -irm https://raw.githubusercontent.com/chopratejas/headroom/main/scripts/install.ps1 | iex +irm https://raw.githubusercontent.com/headroomlabs-ai/headroom/main/scripts/install.ps1 | iex ``` ## What the installer does 1. Verifies Docker is installed and available. -2. Pulls `ghcr.io/chopratejas/headroom:latest` by default, or reuses / pulls `HEADROOM_DOCKER_IMAGE` when you set a custom image override. +2. Pulls `ghcr.io/headroomlabs-ai/headroom:latest` by default, or reuses / pulls `HEADROOM_DOCKER_IMAGE` when you set a custom image override. 3. Installs a `headroom` wrapper into `~/.local/bin` or `~/bin`. 4. Updates shell startup files so the wrapper directory is on `PATH`. @@ -44,7 +44,7 @@ The wrapper keeps Headroom inside Docker and mounts host state back into the con Port `8787` stays the default, so `http://localhost:8787` works the same way as a native install. -Published releases also push versioned GHCR tags such as `ghcr.io/chopratejas/headroom:0.5.26`, and those images are built with the same synced package version used for the matching PyPI and npm release. +Published releases also push versioned GHCR tags such as `ghcr.io/headroomlabs-ai/headroom:0.5.26`, and those images are built with the same synced package version used for the matching PyPI and npm release. ## How the wrapper behaves @@ -66,7 +66,7 @@ docker run --rm -it \ -p 8787:8787 \ -v "$PWD:/workspace" \ -w /workspace \ - ghcr.io/chopratejas/headroom:latest \ + ghcr.io/headroomlabs-ai/headroom:latest \ headroom proxy --host 0.0.0.0 --port 8787 ``` diff --git a/docs/content/docs/how-compression-works.mdx b/docs/content/docs/how-compression-works.mdx index aa1d23dea..5ea69b257 100644 --- a/docs/content/docs/how-compression-works.mdx +++ b/docs/content/docs/how-compression-works.mdx @@ -1,26 +1,26 @@ --- title: How Compression Works -description: Understand Headroom's three-stage compression pipeline, automatic content routing, and how different content types are compressed. +description: Understand Headroom's compression pipeline, automatic content routing, and how different content types are compressed. --- Headroom automatically detects what kind of content you're sending and routes it to the right compressor. You don't need to configure anything -- just call `compress()` and the pipeline handles the rest. -## The Three-Stage Pipeline +## The Pipeline -Every request flows through three stages: +Every request flows through a short pipeline: ``` ┌──────────────┐ ┌────────────────┐ │ CacheAligner │────>│ ContentRouter │ -│ │ │ │ -│ Report │ │ Detect type & │ -│ prefix drift │ │ route to best │ +│ (off by │ │ │ +│ default) │ │ Detect type & │ +│ report drift │ │ route to best │ │ for cache │ │ compressor │ └──────────────┘ └────────────────┘ ``` -1. **CacheAligner** detects dynamic content (dates, user context) in your system prompt and reports prefix drift so the caller can keep the static prefix cacheable across requests. -2. **ContentRouter** inspects each tool output and routes it to the optimal compressor -- SmartCrusher for JSON arrays, CodeAwareCompressor for source code, LogCompressor for build output, and so on. +1. **CacheAligner** *(detector-only, off by default)* reports dynamic-prefix drift (dates, session context) so callers can keep the static prefix cacheable. It never rewrites your messages. +2. **ContentRouter** inspects each content block and routes it to one compressor -- SmartCrusher for JSON arrays, CodeAwareCompressor for source code, LogCompressor for build output, and so on. This is where essentially all compression happens. ## Content Type Detection @@ -33,8 +33,11 @@ The router auto-detects content type by analyzing structure and patterns. No man | Search results | `file:line:content` format | SearchCompressor | 80-95% | | Build/test logs | Timestamps, log levels, pytest/npm markers | LogCompressor | 85-95% | | Diffs | Unified diff format | DiffCompressor | 60-80% | -| HTML | Tag structure | HTMLCompressor | 50-70% | -| Plain text | Fallback | TextCompressor | 60-80% | +| HTML | Tag structure | HTMLExtractor | 50-70% | +| Tabular (CSV/TSV/markdown tables) | Delimited rows / table syntax | TabularCompressor | 60-90% | +| Structured config (YAML/TOML/INI) | Config syntax | ConfigCompressor | 40-70% | +| Plain text | Text with no stronger signal | TextCrusher | 30-60% | +| Anything else | ML fallback | Kompress | varies | ## Quick Start @@ -153,7 +156,7 @@ When you call `compress()`, here is the full sequence: 1. **Content detection** -- Magika (ML-based) or pattern matching identifies the content type 2. **Structure extraction** -- A handler extracts a structure mask marking what to preserve -3. **Compression** -- Non-structural content is compressed (SmartCrusher, LLMLingua, or text utilities) +3. **Compression** -- Non-structural content is compressed (SmartCrusher, Kompress ML, or text utilities like TextCrusher) 4. **CCR storage** -- If enabled, the original is stored for retrieval when the LLM needs full context diff --git a/docs/content/docs/index.mdx b/docs/content/docs/index.mdx index 6afd7e368..75d3d2a84 100644 --- a/docs/content/docs/index.mdx +++ b/docs/content/docs/index.mdx @@ -3,8 +3,6 @@ title: Introduction description: Headroom is the context optimization layer for LLM applications. Compress tool outputs, DB results, file reads, and RAG results before they reach the model. Same answers, fraction of the tokens. --- - - Headroom compresses everything your AI agent reads -- tool outputs, database results, file reads, RAG retrievals, API responses -- before it reaches the LLM. The model sees less noise, responds faster, and costs less. ## Quick preview @@ -36,10 +34,6 @@ print(f"Saved {result.tokens_saved} tokens ({result.compression_ratio:.0%})") -## Community stats - - - ## What gets compressed | Content type | What happens | Typical savings | diff --git a/docs/content/docs/installation.mdx b/docs/content/docs/installation.mdx index d1b0b6c18..08c666a33 100644 --- a/docs/content/docs/installation.mdx +++ b/docs/content/docs/installation.mdx @@ -61,7 +61,7 @@ args = ["mcp", "serve"] ``` Native Intel macOS installs are currently tracked in -[chopratejas/headroom#525](https://github.com/chopratejas/headroom/issues/525). +[headroomlabs-ai/headroom#525](https://github.com/headroomlabs-ai/headroom/issues/525). Use [Docker-Native Install](/docs/docker-install) on Intel Macs until native wheel support lands. @@ -140,7 +140,7 @@ To install the prerequisites: If you'd rather avoid the native toolchain entirely, run Headroom through Docker — see the [Docker](#docker) section below. Native -Windows wheels are tracked in [#636](https://github.com/chopratejas/headroom/issues/636). +Windows wheels are tracked in [#636](https://github.com/headroomlabs-ai/headroom/issues/636). ### pipx @@ -218,8 +218,8 @@ node -e "const h = require('headroom-ai'); console.log('headroom-ai loaded')" Pre-built images are published to GitHub Container Registry on every release. ```bash -docker pull ghcr.io/chopratejas/headroom:latest -docker run -p 8787:8787 ghcr.io/chopratejas/headroom:latest +docker pull ghcr.io/headroomlabs-ai/headroom:latest +docker run -p 8787:8787 ghcr.io/headroomlabs-ai/headroom:latest ``` @@ -281,8 +281,8 @@ included in any extra or Docker image — see |---|---|---| | `HEADROOM_PORT` | `8787` | Port the proxy listens on | | `HEADROOM_HOST` | `127.0.0.1` | Host the proxy binds to | -| `HEADROOM_MODE` | `token` | Default optimization mode: `token` or `cache` | -| `HEADROOM_TELEMETRY` | `off` (opt-in) | Set to `on` to opt in to anonymous telemetry | +| `HEADROOM_MODE` | `cache` | Default optimization mode: `token` or `cache` | +| `HEADROOM_TELEMETRY` | `off` | Set to `on` for local-only usage stats (nothing is sent externally) | | `HEADROOM_REQUEST_TIMEOUT` | `300` | Request timeout in seconds | ### TypeScript SDK diff --git a/docs/content/docs/litellm.mdx b/docs/content/docs/litellm.mdx index 86ca64758..aa10b5509 100644 --- a/docs/content/docs/litellm.mdx +++ b/docs/content/docs/litellm.mdx @@ -6,7 +6,7 @@ description: Add Headroom compression to LiteLLM with a single callback. Works w Headroom integrates with [LiteLLM](https://github.com/BerriAI/litellm) as a callback that compresses messages before they reach any provider. One line to enable, works with all 100+ LiteLLM-supported providers. -Looking for the proxy's `--backend litellm-vertex` / `vertex_ai` / `bedrock` options +Looking for the proxy's `--backend vertex_ai` / `bedrock` options instead? Those make the Headroom **proxy** call cloud providers through LiteLLM — a different mechanism from the callback documented here. See [Cloud providers](/docs/proxy#cloud-providers). diff --git a/docs/content/docs/memory.mdx b/docs/content/docs/memory.mdx index c0b919b67..cc56829c7 100644 --- a/docs/content/docs/memory.mdx +++ b/docs/content/docs/memory.mdx @@ -86,16 +86,16 @@ response = client2.chat.completions.create( ## Memory Categories -Memories are categorized for better organization and retrieval: +Each memory carries a free-form `category` string -- pass any label you like. These are the conventional categories Headroom uses for organization and retrieval: | Category | Description | Examples | |----------|-------------|----------| -| `PREFERENCE` | Likes, dislikes, preferred approaches | "Prefers Python", "Likes dark mode" | -| `FACT` | Identity, role, constraints | "Works at fintech startup", "Senior engineer" | -| `CONTEXT` | Current goals, ongoing tasks | "Migrating to microservices", "Working on auth" | -| `ENTITY` | Information about entities | "Project Apollo uses React", "Team lead is Sarah" | -| `DECISION` | Decisions made | "Chose PostgreSQL over MySQL" | -| `INSIGHT` | Derived insights | "User tends to prefer typed languages" | +| `"preference"` | Likes, dislikes, preferred approaches | "Prefers Python", "Likes dark mode" | +| `"fact"` | Identity, role, constraints | "Works at fintech startup", "Senior engineer" | +| `"context"` | Current goals, ongoing tasks | "Migrating to microservices", "Working on auth" | +| `"entity"` | Information about entities | "Project Apollo uses React", "Team lead is Sarah" | +| `"decision"` | Decisions made | "Chose PostgreSQL over MySQL" | +| `"insight"` | Derived insights | "User tends to prefer typed languages" | ## Memory API @@ -132,7 +132,7 @@ print(f"By category: {stats['categories']}") When facts change, Headroom creates a **supersession chain** that preserves history: ```python -from headroom.memory import HierarchicalMemory, MemoryCategory +from headroom.memory import HierarchicalMemory, MemoryFilter memory = await HierarchicalMemory.create() @@ -140,7 +140,7 @@ memory = await HierarchicalMemory.create() orig = await memory.add( content="User works at Google", user_id="alice", - category=MemoryCategory.FACT, + category="fact", ) # User changes jobs -- supersede the old memory diff --git a/docs/content/docs/meta.json b/docs/content/docs/meta.json index 430384417..f05e8c7fc 100644 --- a/docs/content/docs/meta.json +++ b/docs/content/docs/meta.json @@ -6,7 +6,6 @@ "installation", "docker-install", "persistent-installs", - "community-savings", "---Compression---", "how-compression-works", "smart-crusher", @@ -33,6 +32,8 @@ "langchain", "agno", "strands", + "crewai", + "autogen", "litellm", "claude-code-vertex", "claude-code-azure-foundry", diff --git a/docs/content/docs/proxy.mdx b/docs/content/docs/proxy.mdx index ed2fb01d0..f40cba1c4 100644 --- a/docs/content/docs/proxy.mdx +++ b/docs/content/docs/proxy.mdx @@ -22,7 +22,7 @@ headroom proxy \ --budget 100.0 ``` -Telemetry is **off by default** (opt-in). Opt in with `HEADROOM_TELEMETRY=on` or `--telemetry`. +Telemetry is **local-only and off by default**. `HEADROOM_TELEMETRY=on` (or `--telemetry`) turns on in-process usage stats that power your own `/stats`, `/metrics`, and dashboard — **nothing is sent to Headroom Labs**. (The anonymous aggregate beacon that older versions shipped has been removed from the code.) ## CLI options @@ -50,8 +50,8 @@ Telemetry is **off by default** (opt-in). Opt in with `HEADROOM_TELEMETRY=on` or | `--gemini-api-url` | Gemini default | Custom Gemini API URL | | `--backend` | `anthropic` | Backend: `anthropic`, `bedrock`, `openrouter`, `anyllm`, or `litellm-` | | `--bedrock-api-url` | None | Bedrock InvokeModel upstream for the `/model/{id}/invoke` passthrough routes (see [Bedrock via a local gateway](#bedrock-via-a-local-gateway)) | -| `--telemetry` | `false` | Opt in to anonymous telemetry (off by default) | -| `--no-telemetry` | `false` | Force anonymous telemetry off (already the default) | +| `--telemetry` | `false` | Enable local, in-process usage stats (for your own `/stats` and dashboard; nothing leaves the machine) | +| `--no-telemetry` | `false` | Force local telemetry off (already the default) | | `--stateless` | `false` | Disable filesystem writes and keep runtime state in memory | Use `--http-proxy` or `HEADROOM_HTTP_PROXY` when only provider API traffic should go through a proxy: @@ -195,6 +195,123 @@ HEADROOM_SAVINGS_PROFILE=balanced HEADROOM_TARGET_RATIO=0.15 headroom proxy For permanent custom profiles, see the profile definitions in `headroom/agent_savings.py`. Each profile is an `AgentSavingsProfile` dataclass with fields for compression mode, target ratio, turn protection, and pipeline toggles. +## Configuration in depth + +Proxy behavior is set by three layers, **each overriding the one before**: + +1. **Savings profile** (`HEADROOM_SAVINGS_PROFILE`) — seeds a whole posture (mode, keep-ratio, which roles get compressed, Kompress on/off). Default `coding`. See [Savings profiles](#savings-profiles). +2. **Environment variables** — nearly every CLI flag has an `HEADROOM_*` twin, which is what you'll use in Docker, systemd, or CI. +3. **CLI flags** — the most explicit; they win over env and profile. + +The tables below group the knobs by what they control. They aren't exhaustive (`headroom proxy --help` prints the full list), but they cover what real deployments actually touch. Unless noted, every option is off/unset by default and safe to ignore. + +### Compression tuning + +Fine-grained control over what gets compressed and how hard. Most users pick a [profile](#savings-profiles) instead and never touch these. + +| Flag / env | Default | Effect | +|---|---|---| +| `--mode` / `HEADROOM_MODE` | `cache` | `cache` compresses only the newest delta (prefix-cache safe); `token` maximizes removal. | +| `--target-ratio` / `HEADROOM_TARGET_RATIO` | unset | Keep-ratio for ML text compression; lower = more aggressive (e.g. `0.10`). | +| `HEADROOM_MIN_TOKENS` | `500` | Minimum block size before a tool output is compressed. | +| `--compress-user-messages` / `HEADROOM_COMPRESS_USER_MESSAGES` | `false` | Compress content inside user-role messages (tool results live there). The `coding` profile turns this on. | +| `HEADROOM_COMPRESS_SYSTEM_MESSAGES` | unset | Compress system prompts. Off by default to keep the hottest cache prefix stable. | +| `HEADROOM_PROTECT_RECENT` | profile | Never compress the N most recent turns. | +| `--protect-tool-results` / `HEADROOM_PROTECT_TOOL_RESULTS` | empty | Comma-separated tool names whose output is never lossy-compressed. | +| `--compressor` (repeatable) / `HEADROOM_COMPRESSORS` | all | Restrict to specific compressors: `smart_crusher,kompress,code_aware,search,log,tabular,config,html,image`. | +| `--code-aware` / `--no-code-aware` | off | AST-based [code compression](/docs/code-compression). Requires `headroom-ai[code]`. | + +### Kompress (ML compression) + +Kompress is the ModernBERT/ONNX compressor that ContentRouter falls back to for prose and unstructured text. It can run in-process or be offloaded to a hosted endpoint. + +| Flag / env | Default | Effect | +|---|---|---| +| `--disable-kompress` / `HEADROOM_DISABLE_KOMPRESS` | `false` | Turn off ML compression; keep the structural compressors. | +| `--disable-kompress-anthropic` / `--disable-kompress-openai` | inherit | Per-provider override. | +| `--force-kompress-all` / `HEADROOM_FORCE_KOMPRESS_ALL` | `false` | Route *all* content through Kompress, bypassing per-type selection. | +| `HEADROOM_KOMPRESS_ENDPOINT` | none | Offload ML compression to a remote `/compress` endpoint (e.g. a Modal deployment) instead of running the model locally. | +| `HEADROOM_KOMPRESS_ENDPOINT_TOKEN` | none | Bearer token for the remote endpoint. | +| `HEADROOM_KOMPRESS_BACKEND` | `auto` | Compute backend: `auto`, `onnx_cpu`, `onnx_coreml`, `pytorch`, `pytorch_mps`. | + +### Reversible compression (CCR) and lossless mode + +By default Headroom stores originals so the model can recover them via `headroom_retrieve`. See [Reversible Compression](/docs/ccr). + +| Flag / env | Default | Effect | +|---|---|---| +| `--no-ccr` / `HEADROOM_NO_CCR` | CCR on | Disable retrieval markers **and** the injected `headroom_retrieve` tool. | +| `--lossless` / `HEADROOM_LOSSLESS` | `false` | Format-native lossless compaction only — no CCR marker, no retrieval tool. | +| `--no-ccr-proactive-expansion` | expansion on | Stop proactively re-expanding compressed content when the model appears to need it. | + +### File-read handling + +Coding agents re-read the same files repeatedly; these control how stale reads are handled without busting the prefix cache. + +| Flag / env | Default | Effect | +|---|---|---| +| `--no-read-lifecycle` | lifecycle on | Stop replacing stale/superseded file reads with CCR markers. | +| `--read-maturation` / `HEADROOM_READ_MATURATION` | `false` | *(Experimental)* Hold freshly-read files out of the prefix cache until the file quiesces. | +| `--read-maturation-quiesce-turns` | `5` | Turns of no change before a held read is admitted. | + +### Reliability: timeouts, retries, limits + +| Flag / env | Default | Effect | +|---|---|---| +| `--request-timeout-seconds` / `HEADROOM_REQUEST_TIMEOUT` | `300` | Upstream request timeout (seconds). | +| `--connect-timeout-seconds` | `10` | Upstream connect timeout (seconds). | +| `--retry-max-attempts` | `3` | Upstream retries on transient failure. | +| `--limit-concurrency` | `1000` | Concurrent connections before returning 503. | +| `--rpm` / `--tpm` | `60` / `100000` | Requests- and tokens-per-minute rate limits (disable with `--no-rate-limit`). | +| `--budget` / `--budget-period` | none / `daily` | Spend cap in USD per period; over-budget requests get 429. | +| `--workers` / `HEADROOM_WORKERS` | `1` | Uvicorn worker processes. | + +### Tool search and MCP + +Defers large tool schemas so they don't sit in every request. See [MCP](/docs/mcp). + +| Env | Scope | Effect | +|---|---|---| +| `HEADROOM_TOOL_SEARCH` | proxy (server-side) | Defer MCP/system tool schemas behind a `search_tools` tool. The `coding` profile enables it. | +| `ENABLE_TOOL_SEARCH` | client (Claude Code) | Keep Claude Code's own deferred tool-loading active behind a custom base URL ([issue #746](https://github.com/headroomlabs-ai/headroom/issues/746)). Set automatically by `headroom wrap`. | + +### Cost-aware model routing + +Rewrite the upstream model per request — for example, send small, tool-free calls to a cheaper model. Opt-in and off by default. Configure with `HEADROOM_MODEL_ROUTER_ENABLED` plus `HEADROOM_MODEL_ROUTES`; see [Cost-aware model routing](/docs/configuration#cost-aware-model-routing). + +### Observability + +| Flag / env | Default | Effect | +|---|---|---| +| `--telemetry` / `HEADROOM_TELEMETRY` | off | **Local-only** usage stats for your own `/stats`, `/metrics`, and dashboard. Nothing leaves the machine. | +| `--log-file` / `HEADROOM_LOG_FILE` | none | JSONL request/response log. | +| `--log-messages` | `false` | Include full message bodies in the log (may contain sensitive data). | +| `HEADROOM_OTEL_METRICS_ENABLED` | `false` | Export OpenTelemetry metrics (`HEADROOM_OTEL_METRICS_ENDPOINT`, …). | +| `HEADROOM_LANGFUSE_ENABLED` | `false` | Emit Langfuse traces (`LANGFUSE_PUBLIC_KEY` / `LANGFUSE_SECRET_KEY`). | + +See [Metrics](/docs/metrics) for the Prometheus and Grafana setup. + +### Security and networking + +| Flag / env | Default | Effect | +|---|---|---| +| `HEADROOM_PROXY_TOKEN` | none | Require a bearer token (`X-Headroom-Proxy-Token`) from non-loopback callers. | +| `--offline` / `HEADROOM_OFFLINE` | `false` | Air-gap mode: hard-disable **all** egress (telemetry, update checks, license reporting, model downloads). | +| `--stateless` / `HEADROOM_STATELESS` | `false` | Keep all state in memory; no filesystem writes (disables logs, memory, TOIN). | +| `HEADROOM_STRIP_INTERNAL_HEADERS` | `enabled` | Strip internal `x-headroom-*` headers before forwarding upstream. | +| `HEADROOM_TLS_STRICT` | strict | Set `0` to relax CA-constraint checks behind a corporate TLS-inspection proxy. | + +### Performance + +| Flag / env | Default | Effect | +|---|---|---| +| `--embedding-server` / `HEADROOM_EMBEDDING_SERVER` | off | Share one ONNX embedder across workers (~600 MB RSS saved). | +| `--compression-max-workers` / `HEADROOM_COMPRESSION_MAX_WORKERS` | CPU count | Bound the CPU-bound compression threadpool. | + + +For programmatic deployment you can pass an entire proxy config as JSON via `HEADROOM_PROXY_CONFIG_JSON`, or point `HEADROOM_CONFIG_DIR` / `HEADROOM_WORKSPACE_DIR` at custom roots (see [Filesystem Contract](/docs/filesystem-contract)). + + ## API endpoints ### `GET /health` diff --git a/docs/content/docs/text-and-logs.mdx b/docs/content/docs/text-and-logs.mdx index fc0c11f44..6514d43a8 100644 --- a/docs/content/docs/text-and-logs.mdx +++ b/docs/content/docs/text-and-logs.mdx @@ -10,7 +10,7 @@ Headroom provides specialized compressors for text-based content that isn't JSON | `SearchCompressor` | grep/ripgrep output | Relevant matches, file diversity | 80-95% | | `LogCompressor` | Build/test logs | Errors, stack traces, summaries | 85-95% | | `DiffCompressor` | Unified diffs | Changed lines, context | 60-80% | -| `TextCompressor` | General text | Relevant paragraphs, anchors | 60-80% | +| `TextCrusher` | General text | Relevant sentences, anchors | 30-60% | | `KompressCompressor` | General text fallback | Learned token scoring via ONNX | 30-50% | ## SearchCompressor @@ -47,9 +47,11 @@ print(result.compressed) from headroom.transforms import SearchCompressor, SearchCompressorConfig config = SearchCompressorConfig( - max_results=50, # Keep up to 50 matches - preserve_file_diversity=True, # Ensure different files represented - relevance_threshold=0.3, # Minimum relevance score to keep + max_total_matches=30, # Cap total matches kept across all files + max_matches_per_file=5, # Cap matches kept per file (diversity) + max_files=15, # Cap number of distinct files kept + boost_errors=True, # Prioritize lines that look like errors + context_keywords=["auth"], # Extra terms to bias selection toward ) compressor = SearchCompressor(config) @@ -112,21 +114,22 @@ compressor = DiffCompressor() result = compressor.compress(diff_output) ``` -## TextCompressor +## TextCrusher -General-purpose text compression with anchor preservation. Best for documentation, README files, and prose content. +Extractive prose compression -- it keeps the most relevant input sentences verbatim (selection, not rewriting). Best for documentation, README files, and prose content. `TextCrusher` lives in its own module rather than the `headroom.transforms` package root: ```python -from headroom.transforms import TextCompressor +from headroom.transforms.text_crusher import TextCrusher long_text = """ ... thousands of lines of documentation ... """ -compressor = TextCompressor() +compressor = TextCrusher() result = compressor.compress(long_text, context="authentication") print(result.compressed) +print(f"{result.original_tokens} -> {result.compressed_tokens} tokens") ``` **What gets preserved:** @@ -166,7 +169,8 @@ if detection.content_type == ContentType.SEARCH_RESULTS: elif detection.content_type == ContentType.BUILD_OUTPUT: result = LogCompressor().compress(content) elif detection.content_type == ContentType.PLAIN_TEXT: - result = TextCompressor().compress(content, context="process") + from headroom.transforms.text_crusher import TextCrusher + result = TextCrusher().compress(content, context="process") ``` ## When Each Compressor Is Used @@ -178,7 +182,7 @@ The ContentRouter selects the right compressor automatically. Here's when each f | `file:line:content` lines | SearchCompressor | grep/ripgrep output format | | pytest, npm, cargo markers | LogCompressor | Build tool output patterns | | `---/+++` and `@@` markers | DiffCompressor | Unified diff format | -| Prose, documentation | TextCompressor | Fallback for non-structured text | +| Prose, documentation | TextCrusher | Fallback for non-structured text | | Long plain text | KompressCompressor | ContentRouter fallback | ## Performance @@ -188,5 +192,5 @@ The ContentRouter selects the right compressor automatically. Here's when each f | SearchCompressor | 1,000 matches | 30-50 matches | ~2ms | | LogCompressor | 5,000 lines | 100-200 lines | ~3ms | | DiffCompressor | Large diff | Changed hunks only | ~2ms | -| TextCompressor | 10,000 chars | 2,000 chars | ~2ms | +| TextCrusher | 10,000 chars | 2,000 chars | ~2ms | | KompressCompressor | Plain text | 50-70% of original | model-dependent | diff --git a/docs/content/docs/troubleshooting.mdx b/docs/content/docs/troubleshooting.mdx index 14552856a..a7c68a2ac 100644 --- a/docs/content/docs/troubleshooting.mdx +++ b/docs/content/docs/troubleshooting.mdx @@ -202,7 +202,7 @@ ENABLE_TOOL_SEARCH=true ANTHROPIC_BASE_URL=http://localhost:8787 claude When deferral is off, the proxy log also prints a one-time hint naming the fix. -See [issue #746](https://github.com/chopratejas/headroom/issues/746) for the full analysis. +See [issue #746](https://github.com/headroomlabs-ai/headroom/issues/746) for the full analysis. Anthropic's VSCode extension webview does not currently render the deferred-tool @@ -529,4 +529,4 @@ print(f"Tokens: {result.tokens_before} -> {result.tokens_after}") 1. Enable debug logging and check the output 2. Use `simulate()` to see what transforms would apply 3. Run `validate_setup()` for configuration issues -4. File an issue at [github.com/chopratejas/headroom](https://github.com/chopratejas/headroom/issues) with your Headroom version, Python version, provider, debug log output, and minimal reproduction code +4. File an issue at [github.com/headroomlabs-ai/headroom](https://github.com/headroomlabs-ai/headroom/issues) with your Headroom version, Python version, provider, debug log output, and minimal reproduction code diff --git a/docs/lib/layout.shared.ts b/docs/lib/layout.shared.ts index de5a8389d..aad9c0be4 100644 --- a/docs/lib/layout.shared.ts +++ b/docs/lib/layout.shared.ts @@ -5,6 +5,6 @@ export function baseOptions(): BaseLayoutProps { nav: { title: 'Headroom', }, - githubUrl: 'https://github.com/chopratejas/headroom', + githubUrl: 'https://github.com/headroomlabs-ai/headroom', }; } diff --git a/docs/lib/shared.ts b/docs/lib/shared.ts index 554f6987c..5a5291942 100644 --- a/docs/lib/shared.ts +++ b/docs/lib/shared.ts @@ -2,7 +2,7 @@ export const docsRoute = '/docs'; export const docsContentRoute = '/llms.mdx/docs'; export const gitConfig = { - user: 'chopratejas', + user: 'headroomlabs-ai', repo: 'headroom', branch: 'main', }; diff --git a/docs/lib/telemetry.ts b/docs/lib/telemetry.ts deleted file mode 100644 index 3f949d13a..000000000 --- a/docs/lib/telemetry.ts +++ /dev/null @@ -1,46 +0,0 @@ -export interface CommunityStats { - total_tokens_saved: number; - total_cost_saved: number; - total_requests: number; - unique_instances: number; -} - -const fallbackStats: CommunityStats = { - total_tokens_saved: 0, - total_cost_saved: 0, - total_requests: 0, - unique_instances: 0, -}; - -export function fmtNum(value: number) { - return new Intl.NumberFormat('en-US', { - notation: value >= 10000 ? 'compact' : 'standard', - maximumFractionDigits: 1, - }).format(value); -} - -export function fmtUsd(value: number) { - return new Intl.NumberFormat('en-US', { - style: 'currency', - currency: 'USD', - notation: value >= 10000 ? 'compact' : 'standard', - maximumFractionDigits: value >= 1000 ? 1 : 2, - }).format(value); -} - -export async function fetchCommunityStats(): Promise { - const endpoint = process.env.NEXT_PUBLIC_COMMUNITY_STATS_URL; - if (!endpoint) return fallbackStats; - - try { - const response = await fetch(endpoint, { next: { revalidate: 300 } }); - if (!response.ok) return fallbackStats; - - return { - ...fallbackStats, - ...(await response.json()), - }; - } catch { - return fallbackStats; - } -}