mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-08-27 14:17:10 -04:00
Several signals AI agents and search engines use to discover and install a project were misaligned or missing: * ``docs/app/layout.tsx`` set ``metadataBase`` to ``https://chopratejas.github.io/headroom/`` while the live docs run on Vercel — every page's ``og:url`` and ``twitter:url`` resolved to a URL that returns 404 for ``/llms.txt``. Now points at the live Vercel host (overridable via ``NEXT_PUBLIC_SITE_URL`` for a future custom domain). Adds explicit ``openGraph`` and ``twitter`` metadata so social shares render a card with the project's pitch. * No ``llms.txt`` at the GitHub repo root. AI agents crawling ``github.com/chopratejas/headroom/`` saw only the README. The new ``llms.txt`` follows the llmstxt.org convention: 1-line pitch, canonical docs links, copy-paste install commands (pip / npm / Docker / proxy / ``headroom wrap``), and entry points for the library, proxy, MCP server, and SDK integrations. Points at the Fumadocs-generated ``/llms.txt`` and ``/llms-full.txt`` for the full picture. * ``pyproject.toml`` ``Documentation`` URL pointed at the GitHub README anchor. Updated to point at the docs site so PyPI visitors land on searchable docs, and adds an ``AI / LLM Index`` URL pointing at the Fumadocs ``/llms.txt``. * No explicit AI-bot allow list. Added ``docs/app/robots.ts`` (Next 13+ App Router convention) with explicit allows for GPTBot, ClaudeBot, PerplexityBot, Google-Extended, OAI-SearchBot, ChatGPT-User, Cohere-AI, CCBot, and Applebot-Extended. Wildcard allow as the catch-all. Advertises the sitemap. * No ``sitemap.xml`` route. Added ``docs/app/sitemap.ts`` that pulls every Fumadocs page out of ``source`` (same source backing ``/llms.txt``, search, and OG images) so search and AI crawlers can enumerate doc pages without scraping HTML. * README didn't tell AI agents where to look. Added a 2-line pointer near the top nav row: read ``/llms.txt`` here, or fetch the live index / full docs blob. Also tightened the GitHub repo description and added five topics (``claude-code``, ``cursor``, ``tokens``, ``prompt-engineering``, ``typescript``) via ``gh repo edit`` — that's already live on the repo, not part of this commit. No Python or Rust code changes; ``make ci-precheck`` was run to confirm the test slice still passes.
39 lines
1.4 KiB
TypeScript
39 lines
1.4 KiB
TypeScript
// Next.js App Router sitemap convention (Next 13+). Pulls every page
|
|
// out of the Fumadocs ``source`` (same source that backs ``/llms.txt``,
|
|
// search, and the OG image generator) and emits a valid sitemap.xml.
|
|
//
|
|
// Search engines and AI crawlers use this to enumerate every doc page
|
|
// without scraping HTML. The ``robots.ts`` route advertises the
|
|
// sitemap URL so well-behaved crawlers find it on the first GET.
|
|
|
|
import type { MetadataRoute } from 'next';
|
|
import { source } from '@/lib/source';
|
|
|
|
const SITE_URL = process.env.NEXT_PUBLIC_SITE_URL ?? 'https://headroom-docs.vercel.app';
|
|
|
|
export default function sitemap(): MetadataRoute.Sitemap {
|
|
const now = new Date();
|
|
|
|
// Static top-level routes (home page; docs index is covered by the
|
|
// page enumeration below).
|
|
const staticRoutes: MetadataRoute.Sitemap = [
|
|
{
|
|
url: `${SITE_URL}/`,
|
|
lastModified: now,
|
|
changeFrequency: 'weekly',
|
|
priority: 1.0,
|
|
},
|
|
];
|
|
|
|
// Every Fumadocs page (introduction, quickstart, installation,
|
|
// integrations, …). ``page.url`` is the relative URL like
|
|
// ``/docs/quickstart``; ``page.data`` carries the front-matter.
|
|
const docPages: MetadataRoute.Sitemap = source.getPages().map((page) => ({
|
|
url: `${SITE_URL}${page.url}`,
|
|
lastModified: now,
|
|
changeFrequency: 'weekly' as const,
|
|
priority: 0.8,
|
|
}));
|
|
|
|
return [...staticRoutes, ...docPages];
|
|
}
|