From f244d3a7765ea1908c2ed0c02882bd488df3dd3f Mon Sep 17 00:00:00 2001 From: Stephen Dennis Date: Mon, 20 Jul 2026 11:38:47 -0600 Subject: [PATCH] fix(functions): wordpos() indexes by grapheme cluster, not byte offset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit wordpos(str, charpos, sep) documents charpos as a 1-based character position and its siblings wordstart()/wordend() are grapheme-correct, but the implementation used charpos as a raw byte offset (cp[charpos-1]) while bounding it against strip_color()'s code-point count. For any multi-byte UTF-8 string it returned the wrong word (e.g. wordpos("héllo world", 6) gave 1 instead of 2 — byte 6 is the 'o', but grapheme 6 is the space). Bound charpos against the grapheme-cluster count (utf8_cluster_count over the byte length) and resolve it to a byte pointer by walking clusters (utf8_next_grapheme), mirroring wordstart()/wordend(). Also passes the byte length (not the code-point count) to trim_space_sep_LEN(), fixing a latent length mismatch. Behaviour is identical for pure-ASCII input (one cluster == one byte == one code point). Adds testcases/wordpos_fn.mux TC003: accented 'é' (2-byte, proves byte->character) and a skin-tone wave grapheme cluster (2 code points, proves cluster- not code-point-indexing), plus a register-fed runtime case. Expected values derived from this tree's muxscript. Updates the survey-function-matrix.md entry. Full smoke 1307/1307. Co-Authored-By: Claude Fable 5 --- docs/survey-function-matrix.md | 18 ++++++++++------- mux/modules/engine/functions.cpp | 29 +++++++++++++++++++++++----- testcases/smoke.flat | 4 +++- testcases/wordpos_fn.mux | 33 ++++++++++++++++++++++++++++++-- 4 files changed, 69 insertions(+), 15 deletions(-) diff --git a/docs/survey-function-matrix.md b/docs/survey-function-matrix.md index 254d72964..e4f7f287e 100644 --- a/docs/survey-function-matrix.md +++ b/docs/survey-function-matrix.md @@ -261,17 +261,21 @@ moving between them. | :--- | :--- | :--- | | Word index -> word text | `extract()`, `first()`, `rest()`, `last()` | covered | | Grapheme offset -> grapheme slice | `mid()`, `strdelete()`, `strinsert()`, `strreplace()` | covered | -| Byte offset -> containing word | `wordpos(str, charpos, sep)` | present (byte-offset, not grapheme) | +| Grapheme position -> containing word | `wordpos(str, charpos, sep)` | **covered** | | Word index -> grapheme start/end | `wordstart()`, `wordend()` | **covered** | | String -> grapheme list | `graphemes()` | **covered** | | Grapheme list -> string | -- | **missing** (no list-consuming joiner; `strcat()` is variadic-args only) | -Note: `wordpos()` indexes into the color-stripped UTF-8 buffer by byte -position (`cp[charpos - 1]`), not by grapheme cluster. Despite the -documentation saying "character position", the implementation is -byte-oriented. For ASCII strings the distinction is invisible, but for -multi-byte UTF-8 the results will be wrong if the caller passes a -grapheme count instead of a byte offset. +Note: `wordpos()` now indexes by grapheme cluster (fixed): it bounds +`charpos` against the grapheme count and walks grapheme clusters +(`utf8_next_grapheme`) to resolve the position, matching its "character +position" documentation and the grapheme-correct `wordstart()`/`wordend()`. +The previous implementation used `charpos` as a raw byte offset +(`cp[charpos - 1]`) while bounding against the code-point count, so it +returned the wrong word for multi-byte UTF-8. Behaviour is unchanged for +pure-ASCII input (one cluster == one byte == one code point). Regression +coverage: `testcases/wordpos_fn.mux` TC003 (accented `é` and a skin-tone +grapheme cluster). ### Best Additions diff --git a/mux/modules/engine/functions.cpp b/mux/modules/engine/functions.cpp index ea37c8873..6a09ba64c 100644 --- a/mux/modules/engine/functions.cpp +++ b/mux/modules/engine/functions.cpp @@ -5223,16 +5223,35 @@ static FUNCTION(fun_wordpos) return; } - size_t ncp; - UTF8 *cp = strip_color(fargs[0], 0, &ncp); + size_t nBytes; + UTF8 *cp = strip_color(fargs[0], &nBytes, nullptr); unsigned int charpos = mux_atol(fargs[1]); + // charpos is a 1-based grapheme (character) position, matching this + // function's documentation and the grapheme-correct wordstart()/wordend(). + // The old code used charpos as a raw byte offset (&cp[charpos-1]) while + // bounding it against the code-point count from strip_color()'s third + // out-parameter, so it returned the wrong word for any multi-byte UTF-8 + // text. Bound by the grapheme count and resolve charpos to a byte pointer by + // walking grapheme clusters over the byte length. This is identical to the + // old behaviour for pure-ASCII input (one cluster == one byte == one point). + size_t nGraphemes = utf8_cluster_count(cp, nBytes); + if ( charpos > 0 - && charpos <= ncp) + && charpos <= nGraphemes) { + // Resolve the (charpos-1) leading grapheme clusters to a byte pointer. + UTF8 *tp = cp; + size_t remaining = nBytes; + for (unsigned int g = 1; g < charpos; g++) + { + mux_cursor cluster = utf8_next_grapheme(tp, remaining); + tp += cluster.m_byte; + remaining -= cluster.m_byte; + } + size_t ncp_trimmed; - UTF8 *tp = &(cp[charpos - 1]); - cp = trim_space_sep_LEN(cp, ncp, sep, &ncp_trimmed); + cp = trim_space_sep_LEN(cp, nBytes, sep, &ncp_trimmed); UTF8 *xp = split_token(&cp, sep); int i; diff --git a/testcases/smoke.flat b/testcases/smoke.flat index e60af6355..1b53d88b8 100644 --- a/testcases/smoke.flat +++ b/testcases/smoke.flat @@ -12180,7 +12180,9 @@ >257 "@if cand(eq(wordpos(This is a test,1), 1),eq(wordpos(This is a test,6), 2),eq(wordpos(This is a test,9), 3),eq(wordpos(This is a test,14), 4),strmatch(wordpos(This is a test,0), #-1*),strmatch(wordpos(This is a test,99), #-1*))={@log smoke=TC001: wordpos char-position mapping. Succeeded.},{@log smoke=TC001: wordpos char-position mapping. Failed (p1=[wordpos(This is a test,1)] p6=[wordpos(This is a test,6)] p9=[wordpos(This is a test,9)] p14=[wordpos(This is a test,14)] p0=[wordpos(This is a test,0)] p99=[wordpos(This is a test,99)]).}" >258 -"@if cand(eq(wordpos(This is a test,5), 2),eq(wordpos(a|bb|ccc,6,|), 3),eq(wordpos(a|bb|ccc,3,|), 2))={@log smoke=TC002: wordpos separators and custom delimiter. Succeeded.;@trig me/tr.done},{@log smoke=TC002: wordpos separators and custom delimiter. Failed (p5=[wordpos(This is a test,5)] p6|=[wordpos(a|bb|ccc,6,|)] p3|=[wordpos(a|bb|ccc,3,|)]).;@trig me/tr.done}" +"@if cand(eq(wordpos(This is a test,5), 2),eq(wordpos(a|bb|ccc,6,|), 3),eq(wordpos(a|bb|ccc,3,|), 2))={@log smoke=TC002: wordpos separators and custom delimiter. Succeeded.},{@log smoke=TC002: wordpos separators and custom delimiter. Failed (p5=[wordpos(This is a test,5)] p6|=[wordpos(a|bb|ccc,6,|)] p3|=[wordpos(a|bb|ccc,3,|)]).}" +>259 +"@if cand(eq(wordpos(h[chr(233)]llo world,1), 1),eq(wordpos(h[chr(233)]llo world,6), 2),eq(wordpos(h[chr(233)]llo world,11), 2),strmatch(wordpos(h[chr(233)]llo world,12), #-1*),eq(wordpos(a[chr(128075)][chr(127995)] b,2), 1),eq(wordpos(a[chr(128075)][chr(127995)] b,3), 2),eq(wordpos(a[chr(128075)][chr(127995)] b,4), 2),strmatch(wordpos(a[chr(128075)][chr(127995)] b,5), #-1*),eq(wordpos(setr(0,h[chr(233)]llo world),6), 2))={@log smoke=TC003: wordpos grapheme-position indexing (UTF-8). Succeeded.;@trig me/tr.done},{@log smoke=TC003: wordpos grapheme-position indexing (UTF-8). Failed (acc6=[wordpos(h[chr(233)]llo world,6)] acc11=[wordpos(h[chr(233)]llo world,11)] acc12=[wordpos(h[chr(233)]llo world,12)] emoji3=[wordpos(a[chr(128075)][chr(127995)] b,3)] emoji4=[wordpos(a[chr(128075)][chr(127995)] b,4)] rt6=[setr(0,h[chr(233)]llo world)][wordpos(r(0),6)]).;@trig me/tr.done}" >260 "@log smoke=End wordpos() test cases.;@notify smoke" < diff --git a/testcases/wordpos_fn.mux b/testcases/wordpos_fn.mux index b728494b0..5be51eb6e 100644 --- a/testcases/wordpos_fn.mux +++ b/testcases/wordpos_fn.mux @@ -44,11 +44,40 @@ eq(wordpos(a|bb|ccc,3,|), 2) )= { - @log smoke=TC002: wordpos separators and custom delimiter. Succeeded.; + @log smoke=TC002: wordpos separators and custom delimiter. Succeeded. + }, + { + @log smoke=TC002: wordpos separators and custom delimiter. Failed (p5=[wordpos(This is a test,5)] p6|=[wordpos(a|bb|ccc,6,|)] p3|=[wordpos(a|bb|ccc,3,|)]). + } +- +# +# Test Case #3 - charpos is a 1-based GRAPHEME position, not a byte offset. +# "hllo world" has 11 graphemes (the accented e is 2 UTF-8 bytes): +# graphemes h(1) e(2) l(3) l(4) o(5) space(6) w(7) o(8) r(9) l(10) d(11). +# Grapheme 6 is the space -> word 2; a byte-indexed implementation would map +# position 6 to the 'o' (byte 6) and wrongly return word 1. The skin-tone +# wave "a b" is one grapheme cluster of two code points, so +# its 4 graphemes are a(1) wave(2) space(3) b(4) -- proving cluster (not byte +# or code-point) indexing. Expected values derived from this tree's muxscript. +# +&tr.tc003 test_wordpos_fn= + @if cand( + eq(wordpos(h[chr(233)]llo world,1), 1), + eq(wordpos(h[chr(233)]llo world,6), 2), + eq(wordpos(h[chr(233)]llo world,11), 2), + strmatch(wordpos(h[chr(233)]llo world,12), #-1*), + eq(wordpos(a[chr(128075)][chr(127995)] b,2), 1), + eq(wordpos(a[chr(128075)][chr(127995)] b,3), 2), + eq(wordpos(a[chr(128075)][chr(127995)] b,4), 2), + strmatch(wordpos(a[chr(128075)][chr(127995)] b,5), #-1*), + eq(wordpos(setr(0,h[chr(233)]llo world),6), 2) + )= + { + @log smoke=TC003: wordpos grapheme-position indexing (UTF-8). Succeeded.; @trig me/tr.done }, { - @log smoke=TC002: wordpos separators and custom delimiter. Failed (p5=[wordpos(This is a test,5)] p6|=[wordpos(a|bb|ccc,6,|)] p3|=[wordpos(a|bb|ccc,3,|)]).; + @log smoke=TC003: wordpos grapheme-position indexing (UTF-8). Failed (acc6=[wordpos(h[chr(233)]llo world,6)] acc11=[wordpos(h[chr(233)]llo world,11)] acc12=[wordpos(h[chr(233)]llo world,12)] emoji3=[wordpos(a[chr(128075)][chr(127995)] b,3)] emoji4=[wordpos(a[chr(128075)][chr(127995)] b,4)] rt6=[setr(0,h[chr(233)]llo world)][wordpos(r(0),6)]).; @trig me/tr.done } -