diff --git a/headroom/tokenizers/registry.py b/headroom/tokenizers/registry.py index ad59aeec0..eddce6fa6 100644 --- a/headroom/tokenizers/registry.py +++ b/headroom/tokenizers/registry.py @@ -36,7 +36,9 @@ MODEL_PATTERNS: list[tuple[str, str]] = [ (r"^curie", "tiktoken"), (r"^babbage", "tiktoken"), (r"^ada", "tiktoken"), - # Anthropic models -> estimation (Claude uses custom tokenizer) + # Anthropic models -> real BPE proxy (Claude's tokenizer is private; priced + # against tiktoken o200k_base for consistent, monotone counts — see + # _create_anthropic) (r"^claude-", "anthropic"), # Llama family -> huggingface (when available) (r"^llama", "huggingface"), @@ -335,13 +337,33 @@ class TokenizerRegistry: return EstimatingTokenCounter() def _create_anthropic(self, model: str) -> TokenCounter: - """Create Anthropic tokenizer. + """Create Anthropic (Claude) tokenizer. - Anthropic uses a custom tokenizer that's not publicly available. - We use estimation calibrated for Claude models. + Anthropic's tokenizer isn't public. Rather than a character-ratio + estimate — whose chars-per-token flips with the detected content type + (JSON 3.2 / code 3.5 / English 4.0), so compressing text can appear to + *increase* tokens and two components disagree on the same bytes — price + Claude against a real BPE (tiktoken ``o200k_base``) as a stable, monotone + proxy. It is not Claude's exact vocab, but it is deterministic, + consistent before/after, and within ~10-20% of Claude's real counts — + which is what compression ratios and context-pressure gating need. Falls + back to the character estimator if the tiktoken vocab can't be loaded. """ - # Claude models use ~3.5 chars per token on average - return EstimatingTokenCounter(chars_per_token=3.5) + try: + from .tiktoken_counter import ( + TiktokenCounter, + TiktokenLoadError, + load_encoding, + ) + + try: + load_encoding("o200k_base") + except TiktokenLoadError: + logger.info("tiktoken o200k_base unavailable for %s; using char estimator", model) + return EstimatingTokenCounter(chars_per_token=3.5) + return TiktokenCounter(model, encoding="o200k_base") + except Exception: # pragma: no cover - defensive; keep counting alive + return EstimatingTokenCounter(chars_per_token=3.5) def _create_google(self, model: str) -> TokenCounter: """Create Google tokenizer. diff --git a/headroom/tokenizers/tiktoken_counter.py b/headroom/tokenizers/tiktoken_counter.py index 91065e40f..5c05f308d 100644 --- a/headroom/tokenizers/tiktoken_counter.py +++ b/headroom/tokenizers/tiktoken_counter.py @@ -212,15 +212,19 @@ class TiktokenCounter(BaseTokenizer): MESSAGE_OVERHEAD = 3 REPLY_OVERHEAD = 3 - def __init__(self, model: str = "gpt-4o"): + def __init__(self, model: str = "gpt-4o", encoding: str | None = None): """Initialize tiktoken counter. Args: model: Model name to determine encoding. Defaults to 'gpt-4o' (o200k_base encoding). + encoding: Explicit tiktoken encoding name (e.g. 'o200k_base') that + overrides model-based resolution. Used to price + private-tokenizer models (Claude) against a real BPE proxy + instead of a character estimate. """ self.model = model - self.encoding_name = get_encoding_for_model(model) + self.encoding_name = encoding or get_encoding_for_model(model) self._encoding = None # Lazy load @property diff --git a/tests/test_claude_session_mode_benchmark.py b/tests/test_claude_session_mode_benchmark.py index 20c824796..d02c9162d 100644 --- a/tests/test_claude_session_mode_benchmark.py +++ b/tests/test_claude_session_mode_benchmark.py @@ -97,7 +97,20 @@ def test_load_session_replay_groups_assistant_request_events(tmp_path: Path) -> def test_simulation_and_winner_logic() -> None: - tool_blob = '{"rows":[1,2,3,4]}' * 80 + # Realistic varied tool output. A pathologically repetitive blob (the same + # JSON object * N) is a degenerate case for a real BPE tokenizer — it merges + # the repetition to near-nothing — so a token-mode rewrite can cost MORE real + # tokens than the original, which the old character estimate masked by + # over-counting the repetition. Varied records keep the fixture representative + # of real agent tool output, where the rewrite is a genuine win. + tool_blob = json.dumps( + { + "rows": [ + {"id": i, "label": f"row-{i}", "value": (i * 37) % 100, "ok": i % 3 == 0} + for i in range(150) + ] + } + ) turn1 = ReplayTurn( session_id="s1", project_key="C--git-demo", diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py index f95295ae1..c5d23a1dd 100644 --- a/tests/test_tokenizer.py +++ b/tests/test_tokenizer.py @@ -22,6 +22,41 @@ class FakeTokenCounter: return sum(len(str(msg.get("content", "")).split()) for msg in messages) +def test_claude_priced_with_real_bpe_not_char_estimate() -> None: + """Claude has no public tokenizer, so we price it against a real BPE + (tiktoken o200k_base) instead of a content-adaptive character estimate — + otherwise before/after counts drift between components and compressing text + can appear to *increase* tokens. A tool_result fold must always register as + a reduction; and when the vocab is available the count is the exact o200k + count (proving it is a real BPE, not a chars/token ratio).""" + from headroom.tokenizers import get_tokenizer + + tok = get_tokenizer("claude-opus-4-8") + + long_msg = [ + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "t", "content": "alpha " * 300}], + } + ] + short_msg = [ + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "t", "content": "alpha " * 3}], + } + ] + assert tok.count_messages(long_msg) > tok.count_messages(short_msg) # fold visible + + try: + import tiktoken + + enc = tiktoken.get_encoding("o200k_base") + except Exception: # vocab unavailable → estimator fallback; monotonicity above still holds + return + sample = "The quick brown fox jumps over the lazy dog. " * 10 + assert tok.count_text(sample) == len(enc.encode(sample)) + + def test_tokenizer_delegates_to_counter() -> None: counter = FakeTokenCounter() tokenizer = Tokenizer(counter, model="gpt-4o") diff --git a/tests/test_tokenizers.py b/tests/test_tokenizers.py index 7b76bb699..524fb38a0 100644 --- a/tests/test_tokenizers.py +++ b/tests/test_tokenizers.py @@ -363,9 +363,15 @@ class TestTokenizerRegistry: assert isinstance(tokenizer, TiktokenCounter) def test_get_anthropic_model(self): - """Test getting tokenizer for Anthropic model.""" + """Anthropic (Claude) has no public tokenizer, so it is priced against a + real BPE proxy (tiktoken o200k_base) rather than a character estimate — + for consistent, monotone before/after counts. Falls back to the estimator + only when the tiktoken vocab can't be loaded.""" tokenizer = get_tokenizer("claude-3-sonnet") - assert isinstance(tokenizer, EstimatingTokenCounter) + if isinstance(tokenizer, TiktokenCounter): + assert tokenizer.encoding_name == "o200k_base" + else: # vocab unavailable in this environment → documented fallback + assert isinstance(tokenizer, EstimatingTokenCounter) def test_get_unknown_model_fallback(self): """Test fallback for unknown model."""