From 285176be54e1d179676dcf205de44d5893f8efa5 Mon Sep 17 00:00:00 2001 From: Tejas Chopra Date: Fri, 24 Jul 2026 15:06:43 -0700 Subject: [PATCH] fix(tokenizers): price Claude against a real BPE (tiktoken o200k) not a char estimate (#2543) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Description Claude's tokenizer is not public, so `get_tokenizer("claude-*")` returned `EstimatingTokenCounter(chars_per_token=3.5)` — a **character-ratio estimate**. The estimator's chars-per-token flips with the *detected content type* (JSON 3.2 / code 3.5 / English 4.0 / CJK 1.5), so: - the **same bytes** count differently before vs after compression → a fold could appear to *increase* tokens; - it **disagrees with the pipeline's** estimator on the same content. Calibration vs a real BPE (tiktoken `o200k_base`) on representative content: | content | char-est(3.5) / o200k | |---|---| | English | **1.28×** (overcount 28%) | | code | **0.83×** (undercount 17%) | | JSON | 0.97× | | diff | 1.02× | i.e. the estimate is off **−17% … +28%**, and the error is content-dependent (non-uniform) — which is exactly what produces impossible `tok_after > tok_before` deltas. This prices Claude against **tiktoken `o200k_base`** — a real, deterministic, **monotone** BPE. It's not Claude's exact vocab, but it's within ~10–20% and, crucially, **consistent before/after**, which is what compression ratios and context-pressure gating actually need. Closes # ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) - [x] Performance improvement - [ ] Breaking change - [ ] Documentation update - [ ] Code refactoring (no functional changes) ## Changes Made - `TiktokenCounter.__init__` accepts an explicit `encoding` override (so a model can be priced against a chosen encoding regardless of name-based resolution). - `_create_anthropic` now returns `TiktokenCounter(model, encoding="o200k_base")`, **failing open to the character estimator** if the tiktoken vocab can't be loaded (reuses the existing `load_encoding` timeout/guard). - Updated the `MODEL_PATTERNS` comment and `test_get_anthropic_model` to the new contract; added `test_claude_priced_with_real_bpe_not_char_estimate`. **Blast radius is contained:** `content_router` keeps its own module-level estimator for routing decisions, so only the **handler's reported counts** change. `tiktoken` is already a dependency. **Composes with #2542** (tokenizer-consistent before/after): that PR makes both endpoints use *one* tokenizer; this PR makes that one tokenizer a *real BPE*. Together they fully eliminate the inflated/phantom lines. This PR is based on `main` and is independently valid. ## Calibration note (please review) The one behavioral consumer of the handler's count is the background-compression gate (`original_tokens >= _background_compression_min_tokens`). Because o200k differs from the old estimate by ~±20% depending on content, that threshold now trips on slightly different requests. It's *more* accurate, but the threshold was tuned against the estimator — worth a re-check. No other gate consumes the handler count (routing uses `content_router`'s estimator). ## Testing - [x] Unit tests pass (`pytest`) - [x] Linting passes (`ruff check` + `ruff format --check`) - [x] Type checking passes (`mypy`) - [x] New tests added - [x] Manual testing performed ### Test Output ```text $ ruff check && ruff format --check All checks passed! 4 files already formatted $ mypy headroom/tokenizers/registry.py headroom/tokenizers/tiktoken_counter.py Success: no issues found in 2 source files $ pytest tests/test_tokenizer.py tests/test_tokenizers.py -q 55 passed, 14 skipped ``` ## Real Behavior Proof - **Unit:** `get_tokenizer("claude-opus-4-8")` → `TiktokenCounter(encoding_name="o200k_base")`; `count_text(sample)` == `tiktoken.get_encoding("o200k_base").encode(sample)` exactly; a `tool_result` shrunk 300→3 words drops 508→13 tokens (folds register). - **Live proxy** (Anthropic path, `--proxy-extension lossless_guard`): foldable request logs `tok_before=694 tok_after=231 tok_saved=463 transforms=turn_hook` — real o200k-scale numbers (vs the estimator's 607 for the same payload), clean fold, no inflation. - **Not tested:** exact agreement with Anthropic's *own* count_tokens API (o200k is a proxy; a follow-up could calibrate against it offline). GPT paths unchanged (already tiktoken). ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [ ] Documentation — N/A (internal; docstrings updated) - [x] My changes generate no new warnings - [x] New and existing unit tests pass locally with my changes - [x] I did **not** edit `CHANGELOG.md` ## Additional Notes Follow-ups (not here): (1) extend the same real-BPE treatment to other private-tokenizer providers (Gemini/Cohere/Moonshot), each with its own calibration; (2) optional opt-in calibration against Anthropic's `count_tokens` API to true-up the ~10–20% absolute offset. --- headroom/tokenizers/registry.py | 34 ++++++++++++++++---- headroom/tokenizers/tiktoken_counter.py | 8 +++-- tests/test_claude_session_mode_benchmark.py | 15 ++++++++- tests/test_tokenizer.py | 35 +++++++++++++++++++++ tests/test_tokenizers.py | 10 ++++-- 5 files changed, 91 insertions(+), 11 deletions(-) diff --git a/headroom/tokenizers/registry.py b/headroom/tokenizers/registry.py index ad59aeec0..eddce6fa6 100644 --- a/headroom/tokenizers/registry.py +++ b/headroom/tokenizers/registry.py @@ -36,7 +36,9 @@ MODEL_PATTERNS: list[tuple[str, str]] = [ (r"^curie", "tiktoken"), (r"^babbage", "tiktoken"), (r"^ada", "tiktoken"), - # Anthropic models -> estimation (Claude uses custom tokenizer) + # Anthropic models -> real BPE proxy (Claude's tokenizer is private; priced + # against tiktoken o200k_base for consistent, monotone counts — see + # _create_anthropic) (r"^claude-", "anthropic"), # Llama family -> huggingface (when available) (r"^llama", "huggingface"), @@ -335,13 +337,33 @@ class TokenizerRegistry: return EstimatingTokenCounter() def _create_anthropic(self, model: str) -> TokenCounter: - """Create Anthropic tokenizer. + """Create Anthropic (Claude) tokenizer. - Anthropic uses a custom tokenizer that's not publicly available. - We use estimation calibrated for Claude models. + Anthropic's tokenizer isn't public. Rather than a character-ratio + estimate — whose chars-per-token flips with the detected content type + (JSON 3.2 / code 3.5 / English 4.0), so compressing text can appear to + *increase* tokens and two components disagree on the same bytes — price + Claude against a real BPE (tiktoken ``o200k_base``) as a stable, monotone + proxy. It is not Claude's exact vocab, but it is deterministic, + consistent before/after, and within ~10-20% of Claude's real counts — + which is what compression ratios and context-pressure gating need. Falls + back to the character estimator if the tiktoken vocab can't be loaded. """ - # Claude models use ~3.5 chars per token on average - return EstimatingTokenCounter(chars_per_token=3.5) + try: + from .tiktoken_counter import ( + TiktokenCounter, + TiktokenLoadError, + load_encoding, + ) + + try: + load_encoding("o200k_base") + except TiktokenLoadError: + logger.info("tiktoken o200k_base unavailable for %s; using char estimator", model) + return EstimatingTokenCounter(chars_per_token=3.5) + return TiktokenCounter(model, encoding="o200k_base") + except Exception: # pragma: no cover - defensive; keep counting alive + return EstimatingTokenCounter(chars_per_token=3.5) def _create_google(self, model: str) -> TokenCounter: """Create Google tokenizer. diff --git a/headroom/tokenizers/tiktoken_counter.py b/headroom/tokenizers/tiktoken_counter.py index 91065e40f..5c05f308d 100644 --- a/headroom/tokenizers/tiktoken_counter.py +++ b/headroom/tokenizers/tiktoken_counter.py @@ -212,15 +212,19 @@ class TiktokenCounter(BaseTokenizer): MESSAGE_OVERHEAD = 3 REPLY_OVERHEAD = 3 - def __init__(self, model: str = "gpt-4o"): + def __init__(self, model: str = "gpt-4o", encoding: str | None = None): """Initialize tiktoken counter. Args: model: Model name to determine encoding. Defaults to 'gpt-4o' (o200k_base encoding). + encoding: Explicit tiktoken encoding name (e.g. 'o200k_base') that + overrides model-based resolution. Used to price + private-tokenizer models (Claude) against a real BPE proxy + instead of a character estimate. """ self.model = model - self.encoding_name = get_encoding_for_model(model) + self.encoding_name = encoding or get_encoding_for_model(model) self._encoding = None # Lazy load @property diff --git a/tests/test_claude_session_mode_benchmark.py b/tests/test_claude_session_mode_benchmark.py index 20c824796..d02c9162d 100644 --- a/tests/test_claude_session_mode_benchmark.py +++ b/tests/test_claude_session_mode_benchmark.py @@ -97,7 +97,20 @@ def test_load_session_replay_groups_assistant_request_events(tmp_path: Path) -> def test_simulation_and_winner_logic() -> None: - tool_blob = '{"rows":[1,2,3,4]}' * 80 + # Realistic varied tool output. A pathologically repetitive blob (the same + # JSON object * N) is a degenerate case for a real BPE tokenizer — it merges + # the repetition to near-nothing — so a token-mode rewrite can cost MORE real + # tokens than the original, which the old character estimate masked by + # over-counting the repetition. Varied records keep the fixture representative + # of real agent tool output, where the rewrite is a genuine win. + tool_blob = json.dumps( + { + "rows": [ + {"id": i, "label": f"row-{i}", "value": (i * 37) % 100, "ok": i % 3 == 0} + for i in range(150) + ] + } + ) turn1 = ReplayTurn( session_id="s1", project_key="C--git-demo", diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py index f95295ae1..c5d23a1dd 100644 --- a/tests/test_tokenizer.py +++ b/tests/test_tokenizer.py @@ -22,6 +22,41 @@ class FakeTokenCounter: return sum(len(str(msg.get("content", "")).split()) for msg in messages) +def test_claude_priced_with_real_bpe_not_char_estimate() -> None: + """Claude has no public tokenizer, so we price it against a real BPE + (tiktoken o200k_base) instead of a content-adaptive character estimate — + otherwise before/after counts drift between components and compressing text + can appear to *increase* tokens. A tool_result fold must always register as + a reduction; and when the vocab is available the count is the exact o200k + count (proving it is a real BPE, not a chars/token ratio).""" + from headroom.tokenizers import get_tokenizer + + tok = get_tokenizer("claude-opus-4-8") + + long_msg = [ + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "t", "content": "alpha " * 300}], + } + ] + short_msg = [ + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "t", "content": "alpha " * 3}], + } + ] + assert tok.count_messages(long_msg) > tok.count_messages(short_msg) # fold visible + + try: + import tiktoken + + enc = tiktoken.get_encoding("o200k_base") + except Exception: # vocab unavailable → estimator fallback; monotonicity above still holds + return + sample = "The quick brown fox jumps over the lazy dog. " * 10 + assert tok.count_text(sample) == len(enc.encode(sample)) + + def test_tokenizer_delegates_to_counter() -> None: counter = FakeTokenCounter() tokenizer = Tokenizer(counter, model="gpt-4o") diff --git a/tests/test_tokenizers.py b/tests/test_tokenizers.py index 7b76bb699..524fb38a0 100644 --- a/tests/test_tokenizers.py +++ b/tests/test_tokenizers.py @@ -363,9 +363,15 @@ class TestTokenizerRegistry: assert isinstance(tokenizer, TiktokenCounter) def test_get_anthropic_model(self): - """Test getting tokenizer for Anthropic model.""" + """Anthropic (Claude) has no public tokenizer, so it is priced against a + real BPE proxy (tiktoken o200k_base) rather than a character estimate — + for consistent, monotone before/after counts. Falls back to the estimator + only when the tiktoken vocab can't be loaded.""" tokenizer = get_tokenizer("claude-3-sonnet") - assert isinstance(tokenizer, EstimatingTokenCounter) + if isinstance(tokenizer, TiktokenCounter): + assert tokenizer.encoding_name == "o200k_base" + else: # vocab unavailable in this environment → documented fallback + assert isinstance(tokenizer, EstimatingTokenCounter) def test_get_unknown_model_fallback(self): """Test fallback for unknown model."""