From c1d2eec588e7fd49f85eabf8199770da6374fa67 Mon Sep 17 00:00:00 2001 From: chopratejas Date: Wed, 13 May 2026 17:36:06 -0700 Subject: [PATCH] docs: improve discoverability for AI agents and search crawlers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Several signals AI agents and search engines use to discover and install a project were misaligned or missing: * ``docs/app/layout.tsx`` set ``metadataBase`` to ``https://chopratejas.github.io/headroom/`` while the live docs run on Vercel — every page's ``og:url`` and ``twitter:url`` resolved to a URL that returns 404 for ``/llms.txt``. Now points at the live Vercel host (overridable via ``NEXT_PUBLIC_SITE_URL`` for a future custom domain). Adds explicit ``openGraph`` and ``twitter`` metadata so social shares render a card with the project's pitch. * No ``llms.txt`` at the GitHub repo root. AI agents crawling ``github.com/chopratejas/headroom/`` saw only the README. The new ``llms.txt`` follows the llmstxt.org convention: 1-line pitch, canonical docs links, copy-paste install commands (pip / npm / Docker / proxy / ``headroom wrap``), and entry points for the library, proxy, MCP server, and SDK integrations. Points at the Fumadocs-generated ``/llms.txt`` and ``/llms-full.txt`` for the full picture. * ``pyproject.toml`` ``Documentation`` URL pointed at the GitHub README anchor. Updated to point at the docs site so PyPI visitors land on searchable docs, and adds an ``AI / LLM Index`` URL pointing at the Fumadocs ``/llms.txt``. * No explicit AI-bot allow list. Added ``docs/app/robots.ts`` (Next 13+ App Router convention) with explicit allows for GPTBot, ClaudeBot, PerplexityBot, Google-Extended, OAI-SearchBot, ChatGPT-User, Cohere-AI, CCBot, and Applebot-Extended. Wildcard allow as the catch-all. Advertises the sitemap. * No ``sitemap.xml`` route. Added ``docs/app/sitemap.ts`` that pulls every Fumadocs page out of ``source`` (same source backing ``/llms.txt``, search, and OG images) so search and AI crawlers can enumerate doc pages without scraping HTML. * README didn't tell AI agents where to look. Added a 2-line pointer near the top nav row: read ``/llms.txt`` here, or fetch the live index / full docs blob. Also tightened the GitHub repo description and added five topics (``claude-code``, ``cursor``, ``tokens``, ``prompt-engineering``, ``typescript``) via ``gh repo edit`` — that's already live on the repo, not part of this commit. No Python or Rust code changes; ``make ci-precheck`` was run to confirm the test slice still passes. --- README.md | 7 ++++- docs/app/layout.tsx | 31 +++++++++++++++++++-- docs/app/robots.ts | 42 ++++++++++++++++++++++++++++ docs/app/sitemap.ts | 39 ++++++++++++++++++++++++++ llms.txt | 67 +++++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 8 ++++-- 6 files changed, 188 insertions(+), 6 deletions(-) create mode 100644 docs/app/robots.ts create mode 100644 docs/app/sitemap.ts create mode 100644 llms.txt diff --git a/README.md b/README.md index 7757d599a..27474f27e 100644 --- a/README.md +++ b/README.md @@ -26,9 +26,14 @@ Install · Proof · Agents · - Discord + Discord · + llms.txt

+

+ AI agents / LLMs: read /llms.txt here, or fetch the live index / full docs blob. +

+ --- > Headroom compresses everything your AI agent reads — tool outputs, logs, RAG chunks, files, and conversation history — before it reaches the LLM. Same answers, fraction of the tokens. diff --git a/docs/app/layout.tsx b/docs/app/layout.tsx index 1e5e74d69..6850d0d6d 100644 --- a/docs/app/layout.tsx +++ b/docs/app/layout.tsx @@ -7,14 +7,39 @@ const inter = Inter({ subsets: ['latin'], }); +// Canonical URL for the live docs. ``metadataBase`` resolves the og:url +// and twitter:url for every page; pointing it at the actual live site +// is what lets crawlers (search + LLM) follow the right canonical and +// pick up ``/llms.txt`` / ``/sitemap.xml`` / og images. Override at +// build time via ``NEXT_PUBLIC_SITE_URL`` (e.g. when promoting to a +// custom domain). +const SITE_URL = process.env.NEXT_PUBLIC_SITE_URL ?? 'https://headroom-docs.vercel.app'; + export const metadata: Metadata = { title: { - default: 'Headroom', + default: 'Headroom — Context Optimization Layer for AI Agents', template: '%s | Headroom', }, description: - 'Compress everything your AI agent reads. Same answers, fraction of the tokens.', - metadataBase: new URL('https://chopratejas.github.io/headroom/'), + 'Compress everything your AI agent reads — tool outputs, logs, files, RAG chunks. Same answers, fraction of the tokens. Library, proxy, MCP server. Local-first. Apache 2.0.', + metadataBase: new URL(SITE_URL), + alternates: { + canonical: '/', + }, + openGraph: { + type: 'website', + siteName: 'Headroom', + title: 'Headroom — Context Optimization Layer for AI Agents', + description: + 'Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 60–95% fewer tokens, same answers.', + url: '/', + }, + twitter: { + card: 'summary_large_image', + title: 'Headroom — Context Optimization Layer for AI Agents', + description: + 'Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 60–95% fewer tokens, same answers.', + }, }; export default function Layout({ children }: LayoutProps<'/'>) { diff --git a/docs/app/robots.ts b/docs/app/robots.ts new file mode 100644 index 000000000..a316c6dc6 --- /dev/null +++ b/docs/app/robots.ts @@ -0,0 +1,42 @@ +// Next.js App Router robots convention (Next 13+). The default Next +// behaviour allows everything; this file makes the intent explicit so +// AI-bot operators that read an opt-in list (GPTBot, ClaudeBot, +// PerplexityBot, Google-Extended, etc.) see a clear green light, and +// so the sitemap is discoverable. +// +// Headroom docs are open-source documentation we WANT indexed. If a +// future page should be excluded, add it to the ``disallow`` list of +// the relevant rule. + +import type { MetadataRoute } from 'next'; + +const SITE_URL = process.env.NEXT_PUBLIC_SITE_URL ?? 'https://headroom-docs.vercel.app'; + +export default function robots(): MetadataRoute.Robots { + return { + rules: [ + // Bot-specific allows. These names are the literal user-agent + // strings each operator publishes. Listing them explicitly is + // the documented way to opt INTO AI-training / AI-search + // indexing — silence (no rule) is treated as opt-out by some + // operators (notably Google-Extended). + { userAgent: 'GPTBot', allow: '/' }, + { userAgent: 'OAI-SearchBot', allow: '/' }, + { userAgent: 'ChatGPT-User', allow: '/' }, + { userAgent: 'ClaudeBot', allow: '/' }, + { userAgent: 'Claude-Web', allow: '/' }, + { userAgent: 'anthropic-ai', allow: '/' }, + { userAgent: 'PerplexityBot', allow: '/' }, + { userAgent: 'Perplexity-User', allow: '/' }, + { userAgent: 'Google-Extended', allow: '/' }, + { userAgent: 'cohere-ai', allow: '/' }, + { userAgent: 'CCBot', allow: '/' }, + { userAgent: 'Applebot-Extended', allow: '/' }, + // Catch-all so traditional search crawlers also see an + // explicit allow. + { userAgent: '*', allow: '/' }, + ], + sitemap: `${SITE_URL}/sitemap.xml`, + host: SITE_URL, + }; +} diff --git a/docs/app/sitemap.ts b/docs/app/sitemap.ts new file mode 100644 index 000000000..b96d26d40 --- /dev/null +++ b/docs/app/sitemap.ts @@ -0,0 +1,39 @@ +// Next.js App Router sitemap convention (Next 13+). Pulls every page +// out of the Fumadocs ``source`` (same source that backs ``/llms.txt``, +// search, and the OG image generator) and emits a valid sitemap.xml. +// +// Search engines and AI crawlers use this to enumerate every doc page +// without scraping HTML. The ``robots.ts`` route advertises the +// sitemap URL so well-behaved crawlers find it on the first GET. + +import type { MetadataRoute } from 'next'; +import { source } from '@/lib/source'; + +const SITE_URL = process.env.NEXT_PUBLIC_SITE_URL ?? 'https://headroom-docs.vercel.app'; + +export default function sitemap(): MetadataRoute.Sitemap { + const now = new Date(); + + // Static top-level routes (home page; docs index is covered by the + // page enumeration below). + const staticRoutes: MetadataRoute.Sitemap = [ + { + url: `${SITE_URL}/`, + lastModified: now, + changeFrequency: 'weekly', + priority: 1.0, + }, + ]; + + // Every Fumadocs page (introduction, quickstart, installation, + // integrations, …). ``page.url`` is the relative URL like + // ``/docs/quickstart``; ``page.data`` carries the front-matter. + const docPages: MetadataRoute.Sitemap = source.getPages().map((page) => ({ + url: `${SITE_URL}${page.url}`, + lastModified: now, + changeFrequency: 'weekly' as const, + priority: 0.8, + })); + + return [...staticRoutes, ...docPages]; +} diff --git a/llms.txt b/llms.txt new file mode 100644 index 000000000..cd006b021 --- /dev/null +++ b/llms.txt @@ -0,0 +1,67 @@ +# Headroom + +> Context optimization layer for LLM applications. Compress tool outputs, logs, files, and RAG chunks before they reach the model. Same answers, 60–95% fewer tokens. Library, proxy, and MCP server. Apache 2.0, local-first. + +Headroom is shipped as a Python package (`headroom-ai`), a TypeScript package (`headroom-ai`), an OpenAI + Anthropic-compatible HTTP proxy (`headroom proxy`), and an MCP server (`headroom_compress`, `headroom_retrieve`, `headroom_stats` tools). All four modes use the same compression pipeline: per-content-type compressors (JSON, code, logs, diffs, text) feed into a Compress-Cache-Retrieve (CCR) store so compression stays reversible — the LLM can ask for the original whenever it wants. + +The canonical, always-current documentation index lives at the docs site below. If you can fetch one URL, fetch that one; the entries here are a hand-curated subset. + +## Canonical docs (start here) + +- [Live llms.txt (full doc index)](https://headroom-docs.vercel.app/llms.txt): Auto-generated index of every doc page with descriptions. +- [Live llms-full.txt (every doc page concatenated)](https://headroom-docs.vercel.app/llms-full.txt): One Markdown blob containing every doc page. Use when you can spend the tokens for full context. +- [Docs site](https://headroom-docs.vercel.app/docs): Human-browsable docs with search. +- [GitHub repo](https://github.com/chopratejas/headroom): Source, issues, releases. +- [PyPI package](https://pypi.org/project/headroom-ai/): Python install. +- [npm package](https://www.npmjs.com/package/headroom-ai): TypeScript install. + +## Install (copy-paste-runnable) + +- Python: `pip install headroom-ai` (add `[all]` for every optional extra) +- TypeScript / Node: `npm install headroom-ai` (or `pnpm add headroom-ai`, `bun add headroom-ai`) +- Docker: `docker run -p 8787:8787 ghcr.io/chopratejas/headroom:latest` +- Run the proxy: `headroom proxy --port 8787` then point any client at `http://127.0.0.1:8787` +- Wrap an agent in one command: `headroom wrap claude` (also: `codex`, `cursor`, `aider`, `copilot`, `gemini`) + +## Entry points + +- [Quickstart](https://headroom-docs.vercel.app/docs/quickstart): 5-minute end-to-end (install → compress → call the model). +- [Installation](https://headroom-docs.vercel.app/docs/installation): All install paths, extras, Docker tags, env vars. +- [Proxy server](https://headroom-docs.vercel.app/docs/proxy): Run as a local HTTP proxy in front of OpenAI / Anthropic / Gemini. +- [MCP server](https://headroom-docs.vercel.app/docs/mcp): `headroom_compress`, `headroom_retrieve`, `headroom_stats` for Claude Code / Cursor / any MCP host. +- [API reference](https://headroom-docs.vercel.app/docs/api-reference): Python + TypeScript `compress()` API. + +## How it works + +- [How compression works](https://headroom-docs.vercel.app/docs/how-compression-works): Three-stage pipeline + automatic content routing. +- [SmartCrusher](https://headroom-docs.vercel.app/docs/smart-crusher): Statistical JSON / array compression (70–90% on tool outputs). +- [Code compression](https://headroom-docs.vercel.app/docs/code-compression): AST-aware via tree-sitter (preserves imports, signatures, types). +- [Text & log compression](https://headroom-docs.vercel.app/docs/text-and-logs): Search results, build logs, diffs. +- [CCR (reversible)](https://headroom-docs.vercel.app/docs/ccr): Compress-Cache-Retrieve — originals never deleted; LLM retrieves on demand. + +## SDK / framework integrations + +- [Anthropic SDK](https://headroom-docs.vercel.app/docs/anthropic-sdk): `withHeadroom(anthropic)` wrapper. +- [OpenAI SDK](https://headroom-docs.vercel.app/docs/openai-sdk): `withHeadroom(openai)` wrapper. +- [Vercel AI SDK](https://headroom-docs.vercel.app/docs/vercel-ai-sdk): Middleware + `withHeadroom()`. +- [LangChain](https://headroom-docs.vercel.app/docs/langchain): Chat models, memory, retrievers, agents. +- [Agno](https://headroom-docs.vercel.app/docs/agno): Model wrapping + observability hooks. +- [Strands](https://headroom-docs.vercel.app/docs/strands): Model wrapping + hook-based tool output compression. +- [LiteLLM](https://headroom-docs.vercel.app/docs/litellm): Single callback; works with all 100+ LiteLLM providers. + +## Memory & cross-agent state + +- [Persistent memory](https://headroom-docs.vercel.app/docs/memory): Per-project SQLite + HNSW vector store. No cross-project bleed (GH #462). +- [SharedContext](https://headroom-docs.vercel.app/docs/shared-context): Compressed inter-agent context handoffs. +- [Failure learning](https://headroom-docs.vercel.app/docs/failure-learning): Offline analysis writes corrections to `CLAUDE.md` / `AGENTS.md`. + +## Operations + +- [Configuration](https://headroom-docs.vercel.app/docs/configuration): Env vars, config file, per-call overrides. +- [Benchmarks](https://headroom-docs.vercel.app/docs/benchmarks): Token-savings numbers across content types. +- [Troubleshooting](https://headroom-docs.vercel.app/docs/troubleshooting): Common failure modes and fixes. +- [Limitations](https://headroom-docs.vercel.app/docs/limitations): What Headroom won't do well today. + +## Licensing + +Apache 2.0. Use commercially, modify, redistribute. Data stays on the user's machine when running the library, proxy, or MCP server locally. No telemetry by default. diff --git a/pyproject.toml b/pyproject.toml index 0ac0beafb..5da91e4d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -230,11 +230,15 @@ all = [ headroom = "headroom.cli:main" [project.urls] -Homepage = "https://github.com/chopratejas/headroom" -Documentation = "https://github.com/chopratejas/headroom#readme" +Homepage = "https://headroom-docs.vercel.app" +Documentation = "https://headroom-docs.vercel.app/docs" Repository = "https://github.com/chopratejas/headroom" Issues = "https://github.com/chopratejas/headroom/issues" Changelog = "https://github.com/chopratejas/headroom/blob/main/CHANGELOG.md" +# llms.txt convention (llmstxt.org) — point AI agents / LLM crawlers +# at the auto-generated docs index so they can resolve install paths +# and entry points without a follow-up fetch. +"AI / LLM Index" = "https://headroom-docs.vercel.app/llms.txt" # Maturin builds a single wheel containing both the Python source under # `headroom/` AND the compiled Rust extension `headroom/_core.so` (cdylib