diff --git a/headroom/tokenizers/estimator.py b/headroom/tokenizers/estimator.py index 89fb0d6d1..a28fc12d6 100644 --- a/headroom/tokenizers/estimator.py +++ b/headroom/tokenizers/estimator.py @@ -30,6 +30,8 @@ class EstimatingTokenCounter(BaseTokenizer): - Base: ~4 characters per token (calibrated against GPT-4) - Adjustments for code, URLs, numbers, whitespace - Special handling for JSON structure + - CJK / Kana / Hangul characters priced at ~1 token each (these scripts + tokenize far denser than Latin text) Example: counter = EstimatingTokenCounter() @@ -41,6 +43,13 @@ class EstimatingTokenCounter(BaseTokenizer): CHARS_PER_TOKEN = 4.0 # Average for English text CHARS_PER_TOKEN_CODE = 3.5 # Code is denser CHARS_PER_TOKEN_JSON = 3.2 # JSON has more structure + # CJK / Kana / Hangul scripts tokenize at roughly 0.6-1.7 tokens *per + # character* (cl100k_base ~1.0-1.7, DeepSeek/Qwen native ~0.6-0.8), versus + # ~0.25 tokens/char for English. A flat 4.0 ratio under-counts them ~4-6x, + # so dense-script codepoints are priced separately. 1.5 chars/token keeps + # the estimate on the conservative (slight-overestimate) side for native + # CJK tokenizers while staying close for cl100k_base. + CHARS_PER_TOKEN_CJK = 1.5 # Patterns for content type detection CODE_PATTERN = re.compile( @@ -50,6 +59,21 @@ class EstimatingTokenCounter(BaseTokenizer): re.MULTILINE, ) JSON_PATTERN = re.compile(r"^\s*[\[\{]") + # Dense scripts where one character is worth roughly one token: CJK + # symbols/punctuation, Hiragana/Katakana, CJK Unified (+ Ext A), Hangul, + # CJK compatibility ideographs, fullwidth forms, and astral CJK extensions. + CJK_PATTERN = re.compile( + "[" + "\u3000-\u303f" # CJK symbols and punctuation + "\u3040-\u30ff" # Hiragana + Katakana + "\u3400-\u4dbf" # CJK Unified Ideographs Extension A + "\u4e00-\u9fff" # CJK Unified Ideographs + "\uac00-\ud7af" # Hangul syllables + "\uf900-\ufaff" # CJK compatibility ideographs + "\uff00-\uffef" # Halfwidth and fullwidth forms + "\U00020000-\U0002a6df" # CJK Unified Ideographs Extension B + "]" + ) URL_PATTERN = re.compile(r"https?://\S+") UUID_PATTERN = re.compile( r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", re.IGNORECASE @@ -83,14 +107,32 @@ class EstimatingTokenCounter(BaseTokenizer): # Auto-detect content type and adjust ratio ratio = self._detect_ratio(text) - # Apply ratio with minimum of 1 token - base_count = int(len(text) / ratio + 0.5) + # Price dense scripts (CJK/Kana/Hangul) separately: they tokenize at + # roughly one token per character, so applying the Latin ratio to them + # under-counts by 4-6x. The remaining characters keep the detected ratio. + cjk_chars = self._count_cjk_chars(text) + other_chars = len(text) - cjk_chars + base_count = int(other_chars / ratio + cjk_chars / self.CHARS_PER_TOKEN_CJK + 0.5) # Add overhead for special patterns overhead = self._count_special_overhead(text) return max(1, base_count + overhead) + def _count_cjk_chars(self, text: str) -> int: + """Count dense-script (CJK/Kana/Hangul/fullwidth) codepoints. + + These scripts encode at ~1 token per character, unlike Latin text + (~4 chars per token), so they are priced with CHARS_PER_TOKEN_CJK. + + Args: + text: Text to analyze. + + Returns: + Number of dense-script characters in the text. + """ + return len(self.CJK_PATTERN.findall(text)) + def _detect_ratio(self, text: str) -> float: """Detect optimal chars-per-token ratio based on content. diff --git a/tests/test_tokenizers.py b/tests/test_tokenizers.py index bc1b3b19b..6c89d8f32 100644 --- a/tests/test_tokenizers.py +++ b/tests/test_tokenizers.py @@ -208,6 +208,47 @@ def hello(): count = counter.count_text(code_text) assert count > 0 + def test_count_text_cjk_not_underestimated(self): + """CJK text must not be priced at the Latin ~4-chars/token ratio. + + Regression: count_text divided the whole string length by the Latin + ratio (4.0), so 100 Chinese characters estimated ~25 tokens while real + tokenizers (cl100k_base / DeepSeek / Qwen) produce ~60-150. Dense + scripts tokenize at roughly one token per character, so the estimate + must be far above len/4 and on the order of the character count. + """ + counter = EstimatingTokenCounter() + text = "你好世界" * 25 # 100 CJK characters + count = counter.count_text(text) + # Old behavior returned len/4 == 25; require clearly above that floor. + assert count > len(text) / 3 + # And in the right ballpark for one-token-per-char scripts. + assert count >= int(len(text) * 0.6) + + def test_count_text_cjk_japanese_and_korean(self): + """Japanese (Kana) and Korean (Hangul) are also dense scripts.""" + counter = EstimatingTokenCounter() + for text in ("こんにちは世界" * 10, "안녕하세요" * 10): + count = counter.count_text(text) + assert count >= int(len(text) * 0.6) + + def test_count_text_mixed_latin_cjk(self): + """Mixed text prices the Latin part and the CJK part independently.""" + counter = EstimatingTokenCounter() + latin = "The quick brown fox jumps over the lazy dog. " # 45 chars + cjk = "今天天气很好" # 6 CJK chars + mixed = counter.count_text(latin + cjk) + # Must exceed the all-Latin estimate of the same length, since the CJK + # tail is priced denser than 4 chars/token. + latin_only = counter.count_text(latin + "x" * len(cjk)) + assert mixed > latin_only + + def test_count_text_latin_unchanged(self): + """Pure-Latin estimates are unchanged by the CJK adjustment.""" + counter = EstimatingTokenCounter() + text = "Hello, world!" + assert 2 <= counter.count_text(text) <= 6 + def test_repr(self): """Test string representation.""" counter = EstimatingTokenCounter()