diff --git a/crates/headroom-core/src/transforms/text_crusher/crusher.rs b/crates/headroom-core/src/transforms/text_crusher/crusher.rs index e75a3c722..73a80486b 100644 --- a/crates/headroom-core/src/transforms/text_crusher/crusher.rs +++ b/crates/headroom-core/src/transforms/text_crusher/crusher.rs @@ -362,7 +362,19 @@ fn tokens_ascii(text: &str) -> Vec { out } -/// CJK path: ICU WordSegmenter (dictionary) word units; alnum-bearing, lowercased. +/// Fold full-width ASCII variants (A-Z, 0-9, full-width punctuation) to their +/// half-width form, and the ideographic space to a normal space. Real CJK text +/// mixes these with normal ASCII; folding makes a token match regardless of +/// width. Only the internal token KEY is folded -- the kept output stays verbatim. +fn width_fold(c: char) -> char { + match c as u32 { + 0xFF01..=0xFF5E => char::from_u32(c as u32 - 0xFEE0).unwrap_or(c), + 0x3000 => ' ', + _ => c, + } +} + +/// CJK path: ICU WordSegmenter (dictionary) word units; width-folded, lowercased. fn tokens_icu(text: &str) -> Vec { let seg = *WORD_SEGMENTER; let mut out = Vec::new(); @@ -371,7 +383,7 @@ fn tokens_icu(text: &str) -> Vec { if b > prev { let w = text[prev..b].trim(); if !w.is_empty() && w.chars().any(|c| c.is_alphanumeric()) { - out.push(w.to_lowercase()); + out.push(w.chars().map(width_fold).collect::().to_lowercase()); } prev = b; } @@ -562,6 +574,22 @@ mod tests { ); } + #[test] + fn fullwidth_ascii_folds_to_halfwidth() { + // full-width "API" inside CJK must fold to the same token as "api", + // so dedup/relevance match across width variants. + let toks = tokens("认证API密钥"); + assert!( + toks.iter().any(|t| t == "api"), + "full-width ASCII should fold to 'api': {toks:?}" + ); + // full-width digits too + assert!( + tokens("端口8080").iter().any(|t| t == "8080"), + "full-width digits should fold" + ); + } + #[test] fn cjk_relevance_keeps_query_match() { let needle = "认证令牌的缓存策略采用最近最少使用淘汰算法来管理过期。";