diff --git a/crates/headroom-core/src/transforms/search_compressor.rs b/crates/headroom-core/src/transforms/search_compressor.rs index e6f79d93d..5ffed0e3a 100644 --- a/crates/headroom-core/src/transforms/search_compressor.rs +++ b/crates/headroom-core/src/transforms/search_compressor.rs @@ -70,6 +70,36 @@ use md5::{Digest, Md5}; use crate::ccr::CcrStore; use crate::signals::{ImportanceContext, LineImportanceDetector}; + +/// True for CJK ideographs, kana, and Hangul. Code-point ranges kept +/// byte-identical with the Python `_is_cjk_char` for search-compressor parity. +fn is_cjk_char(c: char) -> bool { + matches!( + c as u32, + 0x3040..=0x30FF | 0x3400..=0x4DBF | 0x4E00..=0x9FFF | 0xAC00..=0xD7AF | 0xF900..=0xFAFF + ) +} + +/// CJK character bigrams from the CJK runs of a (lowercased) query, so a +/// spaceless CJK query can match content. Mirrors the Python `_cjk_bigrams`. +fn cjk_bigrams(text: &str) -> BTreeSet { + let mut out = BTreeSet::new(); + let mut run: Vec = Vec::new(); + for c in text.chars() { + if is_cjk_char(c) { + run.push(c); + } else { + for w in run.windows(2) { + out.insert(w.iter().collect::()); + } + run.clear(); + } + } + for w in run.windows(2) { + out.insert(w.iter().collect::()); + } + out +} use crate::transforms::adaptive_sizer::compute_optimal_k; // ─── Types ────────────────────────────────────────────────────────────── @@ -363,10 +393,15 @@ impl SearchCompressor { pub fn score_matches(&self, files: &mut BTreeMap, context: &str) { let context_lower = context.to_ascii_lowercase(); - let context_words: Vec<&str> = context_lower + // Dedup like Python's `set`; count length in CHARS (not bytes) to match + // Python codepoints; and add CJK char bigrams so a spaceless CJK query + // (no whitespace words to split on) can still match content. + let mut context_words: BTreeSet = context_lower .split_whitespace() - .filter(|w| w.len() > 2) + .filter(|w| w.chars().count() > 2) + .map(|w| w.to_string()) .collect(); + context_words.extend(cjk_bigrams(&context_lower)); for fm in files.values_mut() { for m in &mut fm.matches { @@ -374,7 +409,7 @@ impl SearchCompressor { let content_lower = m.content.to_ascii_lowercase(); for w in &context_words { - if content_lower.contains(w) { + if content_lower.contains(w.as_str()) { score += 0.3; } } @@ -682,6 +717,14 @@ mod tests { ); } + #[test] + fn cjk_bigrams_from_runs() { + let b = cjk_bigrams("认证令牌"); + assert!(b.contains("认证") && b.contains("证令") && b.contains("令牌") && b.len() == 3); + assert!(cjk_bigrams("hello").is_empty()); + assert!(cjk_bigrams("a认b证").is_empty()); // isolated CJK chars -> no pair + } + #[test] fn parses_ripgrep_context_line() { assert_eq!( diff --git a/headroom/transforms/search_compressor.py b/headroom/transforms/search_compressor.py index d9316c23b..8a1c64631 100644 --- a/headroom/transforms/search_compressor.py +++ b/headroom/transforms/search_compressor.py @@ -50,6 +50,36 @@ from typing import Any, cast logger = logging.getLogger(__name__) +def _is_cjk_char(c: str) -> bool: + """True for CJK ideographs, kana, and Hangul. Code-point ranges kept + byte-identical with the Rust `is_cjk_char` for search-compressor parity.""" + o = ord(c) + return ( + 0x3040 <= o <= 0x30FF + or 0x3400 <= o <= 0x4DBF + or 0x4E00 <= o <= 0x9FFF + or 0xAC00 <= o <= 0xD7AF + or 0xF900 <= o <= 0xFAFF + ) + + +def _cjk_bigrams(text: str) -> set[str]: + """CJK character bigrams from the CJK runs of a (lowercased) query, so a + spaceless CJK query can match content. Mirrors the Rust `cjk_bigrams`.""" + out: set[str] = set() + run: list[str] = [] + for c in text: + if _is_cjk_char(c): + run.append(c) + else: + for i in range(len(run) - 1): + out.add(run[i] + run[i + 1]) + run = [] + for i in range(len(run) - 1): + out.add(run[i] + run[i + 1]) + return out + + # ─── Public dataclasses (preserve existing import surface) ────────────────── @@ -223,14 +253,21 @@ class SearchCompressor: Stays Python so the legacy direct-call test surface keeps working without rebuilding through Rust on every test. The - scoring constants must mirror Rust `SearchCompressor::score_matches` - — Rust unit tests pin Rust's behavior; the parity assertion at - the bottom of this module pins both sides agree. + scoring constants mirror Rust `SearchCompressor::score_matches`, + pinned by Rust unit tests and Python tests over the same inputs: + word-overlap and CJK-bigram scoring are byte-equal. (The error- + boost keyword set still diverges for a few terms fixed only on + the Rust side -- see keyword_detector; there is no cross-impl + assertion, so this equality is test-pinned, not mechanically + enforced.) """ from headroom.transforms.error_detection import PRIORITY_PATTERNS_SEARCH context_lower = context.lower() + # Dedup whitespace words (len>2 by codepoints), and add CJK char bigrams + # so a spaceless CJK query can match content. context_words = {w for w in context_lower.split() if len(w) > 2} + context_words |= _cjk_bigrams(context_lower) for fm in file_matches.values(): for match in fm.matches: diff --git a/tests/test_search_compressor_cjk.py b/tests/test_search_compressor_cjk.py new file mode 100644 index 000000000..5b2facd2e --- /dev/null +++ b/tests/test_search_compressor_cjk.py @@ -0,0 +1,36 @@ +"""CJK-aware relevance scoring in the search compressor. + +The relevance scorer tokenized the query on whitespace, so a spaceless CJK query +matched content only when the WHOLE query was a literal substring of a line. CJK +char bigrams now let a longer CJK query boost lines that share a substring. The +Rust<->Python parity was also hardened (dedup like Python's set; char-length +filter instead of bytes). These exercise the Python legacy scorer that mirrors +Rust. +""" + +from headroom.transforms.search_compressor import ( + SearchCompressor, + SearchCompressorConfig, + _cjk_bigrams, +) + + +def test_cjk_bigrams_from_runs(): + assert _cjk_bigrams("认证令牌") == {"认证", "证令", "令牌"} + assert _cjk_bigrams("hello world") == set() # ASCII -> no CJK bigrams + assert _cjk_bigrams("a认b证") == set() # isolated CJK chars -> no adjacent pair + + +def test_score_matches_cjk_query_bigrams_boost(): + compressor = SearchCompressor(SearchCompressorConfig(boost_errors=False, context_keywords=[])) + content = "\n".join( + [ + "src/a.py:10:认证令牌已过期需要重新登录", + "src/b.py:2:plain ascii content here", + ] + ) + parsed = compressor._parse_search_results(content) + # the whole query is NOT a substring of the content line, but its bigrams are + compressor._score_matches(parsed, "认证令牌缓存淘汰策略") + assert parsed["src/a.py"].matches[0].score > 0 # 认证/证令/令牌 bigrams match + assert parsed["src/b.py"].matches[0].score == 0