mmo-ref/tinymux
1
0
Fork 0
mirror of https://github.com/brazilofmux/tinymux synced 2026-08-13 00:23:11 -04:00
Code Issues Projects Releases Packages Wiki Activity
tinymux/mux/lib/utf8_normalize.cpp
Stephen Dennis b0dcfc9834 perf(lib): take utf8_normalize_nfc_slow's 512 KB off the stack 
NFC_MAX_CODEPOINTS is LBUF_SIZE * 2, so NFCCodePoint cps[] is
65,536 entries of 8 bytes (UTF32 cp; int ccc) -- 524,288 bytes, and
the whole of the function's 524,496-byte frame.  It is the slow path
of a routine every piece of string handling reaches, so entering it
cost half a megabyte of stack.

Frame drops 524,496 -> 288 bytes.

This is the largest fixable frame outside the engine.  The larger ones
in libmux are all in color_ops.c -- co_splice at 589,856, co_reverse at
524,304, four more at 262,208 -- and those cannot be fixed this way:
mux/rv64/Makefile compiles exactly three objects into the freestanding
blob (softlib.o, color_ops.o, unicode_tables.o), which has no malloc
and no .bss.  a7a816ea8 left co_reverse's 512 KB alone for that reason.
utf8_normalize.cpp is not one of the three, so it may use .bss.

Safe as a plain function-local static, with no claim guard, because
there is exactly one claim site: utf8_normalize_nfc_slow neither
recurses (zero self-calls) nor evaluates softcode, and none of
DecomposeOne, CanonicalOrder or CanonicalCompose call back into it, so
no two activations are ever live on one thread at once.

make test: 32 passed, 1 skipped (stubslave, not configured), 0 failed.

Coverage here is indirect -- there is no unit test for normalization in
tests/ -- but the combining-character cases reach this path: graphemes
26 cases, comp 33, chr 13, all succeeding.  The change is a storage
class only; no logic is touched.
2026-08-03 10:50:34 -06:00

820 lines
24 KiB
C++
Raw Permalink Blame History

/*! \file utf8_normalize.cpp
* \brief NFC normalization for UTF-8 strings.
*
* Implements Unicode NFC normalization (Canonical Decomposition followed by
* Canonical Composition) using DFA state machines for all Unicode property
* lookups. Hangul composition and decomposition are handled algorithmically.
*
* All operations are locale-independent per Unicode default algorithms.
*/
#include "copyright.h"
#include "autoconf.h"
#include "config.h"
#include "core.h"
// Hangul constants (Unicode 3.0+ algorithmic composition/decomposition).
//
#define HANGUL_SBASE 0xAC00
#define HANGUL_LBASE 0x1100
#define HANGUL_VBASE 0x1161
#define HANGUL_TBASE 0x11A7
#define HANGUL_LCOUNT 19
#define HANGUL_VCOUNT 21
#define HANGUL_TCOUNT 28
#define HANGUL_NCOUNT (HANGUL_VCOUNT * HANGUL_TCOUNT) // 588
#define HANGUL_SCOUNT (HANGUL_LCOUNT * HANGUL_NCOUNT) // 11172
// ---------------------------------------------------------------------------
// UTF-8 <-> UTF-32 helpers.
// ---------------------------------------------------------------------------
// Decode a single UTF-32 code point from a UTF-8 byte sequence.
// Advances *pp past the code point. Returns UNI_EOF on error.
//
static UTF32 utf8_Decode(const UTF8 **pp, const UTF8 *pEnd)
{
const UTF8 *p = *pp;
if (p >= pEnd)
{
return UNI_EOF;
}
int n = utf8_FirstByte[*p];
if (n <= 0 || n >= UTF8_CONTINUE)
{
// Continuation byte or invalid — skip one byte.
(*pp)++;
return UNI_EOF;
}
if (p + n > pEnd)
{
// Truncated sequence at end of buffer.
*pp = pEnd;
return UNI_EOF;
}
for (int i = 1; i < n; i++)
{
if (UTF8_CONTINUE != utf8_FirstByte[p[i]])
{
// Invalid continuation byte — skip one byte.
(*pp)++;
return UNI_EOF;
}
}
UTF32 cp = utf8_decode_raw(p, n);
if (!utf8_is_valid_scalar(cp, n))
{
(*pp)++;
return UNI_EOF;
}
*pp = p + n;
return cp;
}
// Encode a UTF-32 code point to UTF-8.
// Returns the number of bytes written (1-4), or 0 on error.
//
static int utf8_Encode(UTF32 cp, UTF8 *buf)
{
if (cp < 0x80)
{
buf[0] = static_cast<UTF8>(cp);
return 1;
}
else if (cp < 0x800)
{
buf[0] = static_cast<UTF8>(0xC0 | (cp >> 6));
buf[1] = static_cast<UTF8>(0x80 | (cp & 0x3F));
return 2;
}
else if (cp < 0x10000)
{
buf[0] = static_cast<UTF8>(0xE0 | (cp >> 12));
buf[1] = static_cast<UTF8>(0x80 | ((cp >> 6) & 0x3F));
buf[2] = static_cast<UTF8>(0x80 | (cp & 0x3F));
return 3;
}
else if (cp <= 0x10FFFF)
{
buf[0] = static_cast<UTF8>(0xF0 | (cp >> 18));
buf[1] = static_cast<UTF8>(0x80 | ((cp >> 12) & 0x3F));
buf[2] = static_cast<UTF8>(0x80 | ((cp >> 6) & 0x3F));
buf[3] = static_cast<UTF8>(0x80 | (cp & 0x3F));
return 4;
}
return 0;
}
// ---------------------------------------------------------------------------
// DFA traversal helpers.
// ---------------------------------------------------------------------------
// Run the integer-valued DFA (CCC, NFC_QC) on a single code point's bytes.
// Returns the accepting state offset (the integer value), or the default.
// Templated on SBT type because the generator uses unsigned char for small
// tables and unsigned short for larger ones.
//
template<typename T>
static int RunIntegerDFA(
const unsigned char *itt,
const unsigned short *sot,
const T *sbt,
int nStartState,
int nAcceptStart,
int nDefault,
const UTF8 *pStart,
const UTF8 *pEnd)
{
int iState = nStartState;
const UTF8 *p = pStart;
while (p < pEnd && iState < nAcceptStart)
{
unsigned char ch = *p++;
unsigned char iColumn = itt[ch];
unsigned short iOffset = sot[iState];
for (;;)
{
int y = sbt[iOffset];
if (y < 128)
{
if (iColumn < y)
{
iState = sbt[iOffset + 1];
break;
}
else
{
iColumn = static_cast<unsigned char>(iColumn - y);
iOffset += 2;
}
}
else
{
y = 256 - y;
if (iColumn < y)
{
iState = sbt[iOffset + iColumn + 1];
break;
}
else
{
iColumn = static_cast<unsigned char>(iColumn - y);
iOffset = static_cast<unsigned short>(iOffset + y + 1);
}
}
}
}
if (iState < nAcceptStart)
{
return nDefault;
}
return iState - nAcceptStart;
}
// Get Canonical Combining Class for a code point.
//
static int GetCCC(const UTF8 *pStart, const UTF8 *pEnd)
{
return RunIntegerDFA(
tr_ccc_nfcqc_itt, tr_ccc_nfcqc_sot, tr_ccc_nfcqc_sbt,
TR_CCC_NFCQC_START_STATE, TR_CCC_NFCQC_ACCEPTING_STATES_START,
0,
pStart, pEnd) / 3;
}
// Combined CCC + NFC_QC lookup — single DFA traversal instead of two.
//
static void GetCCCandNFCQC(const UTF8 *pStart, const UTF8 *pEnd,
int &ccc, int &nfcqc)
{
int combined = RunIntegerDFA(
tr_ccc_nfcqc_itt, tr_ccc_nfcqc_sot, tr_ccc_nfcqc_sbt,
TR_CCC_NFCQC_START_STATE, TR_CCC_NFCQC_ACCEPTING_STATES_START,
0,
pStart, pEnd);
ccc = combined / 3;
nfcqc = combined % 3;
}
// Get NFD decomposition for a code point.
// Returns the string_desc* or nullptr if no decomposition.
//
static const string_desc *GetNFD(const UTF8 *p, bool &bXor)
{
unsigned short iState = TR_NFD_START_STATE;
do
{
unsigned char ch = *p++;
unsigned char iColumn = tr_nfd_itt[ch];
unsigned short iOffset = tr_nfd_sot[iState];
for (;;)
{
int y = tr_nfd_sbt[iOffset];
if (y < 128)
{
if (iColumn < y)
{
iState = tr_nfd_sbt[iOffset + 1];
break;
}
else
{
iColumn = static_cast<unsigned char>(iColumn - y);
iOffset += 2;
}
}
else
{
y = 256 - y;
if (iColumn < y)
{
iState = tr_nfd_sbt[iOffset + iColumn + 1];
break;
}
else
{
iColumn = static_cast<unsigned char>(iColumn - y);
iOffset = static_cast<unsigned short>(iOffset + y + 1);
}
}
}
} while (iState < TR_NFD_ACCEPTING_STATES_START);
int idx = iState - TR_NFD_ACCEPTING_STATES_START;
if (TR_NFD_DEFAULT == idx)
{
bXor = false;
return nullptr;
}
bXor = (TR_NFD_XOR_START <= idx);
return tr_nfd_ott + idx - 1;
}
// Look up NFC composition pair via the two-code-point DFA.
// Returns the composed code point, or 0 if no composition.
//
static UTF32 ComposeViaTable(const UTF8 *pStarter, int nStarterBytes,
const UTF8 *pCombining, int nCombiningBytes)
{
int iState = TR_NFC_COMPOSE_START_STATE;
// Feed starter bytes.
//
for (int i = 0; i < nStarterBytes && iState < TR_NFC_COMPOSE_ACCEPTING_STATES_START; i++)
{
unsigned char ch = pStarter[i];
unsigned char iColumn = tr_nfc_compose_itt[ch];
unsigned short iOffset = tr_nfc_compose_sot[iState];
for (;;)
{
int y = tr_nfc_compose_sbt[iOffset];
if (y < 128)
{
if (iColumn < y)
{
iState = tr_nfc_compose_sbt[iOffset + 1];
break;
}
else
{
iColumn = static_cast<unsigned char>(iColumn - y);
iOffset += 2;
}
}
else
{
y = 256 - y;
if (iColumn < y)
{
iState = tr_nfc_compose_sbt[iOffset + iColumn + 1];
break;
}
else
{
iColumn = static_cast<unsigned char>(iColumn - y);
iOffset = static_cast<unsigned short>(iOffset + y + 1);
}
}
}
}
// Feed combining bytes.
//
for (int i = 0; i < nCombiningBytes && iState < TR_NFC_COMPOSE_ACCEPTING_STATES_START; i++)
{
unsigned char ch = pCombining[i];
unsigned char iColumn = tr_nfc_compose_itt[ch];
unsigned short iOffset = tr_nfc_compose_sot[iState];
for (;;)
{
int y = tr_nfc_compose_sbt[iOffset];
if (y < 128)
{
if (iColumn < y)
{
iState = tr_nfc_compose_sbt[iOffset + 1];
break;
}
else
{
iColumn = static_cast<unsigned char>(iColumn - y);
iOffset += 2;
}
}
else
{
y = 256 - y;
if (iColumn < y)
{
iState = tr_nfc_compose_sbt[iOffset + iColumn + 1];
break;
}
else
{
iColumn = static_cast<unsigned char>(iColumn - y);
iOffset = static_cast<unsigned short>(iOffset + y + 1);
}
}
}
}
if (iState < TR_NFC_COMPOSE_ACCEPTING_STATES_START)
{
return 0; // No composition.
}
int idx = iState - TR_NFC_COMPOSE_ACCEPTING_STATES_START;
if (0 == idx)
{
return 0; // Default state = no composition.
}
return tr_nfc_compose_nfc_compose_result[idx];
}
// Compose two code points. Handles Hangul algorithmically, then
// falls back to the DFA table.
//
static UTF32 Compose(UTF32 cp1, UTF32 cp2)
{
// Hangul L + V -> LV
//
if ( HANGUL_LBASE <= cp1
&& cp1 < HANGUL_LBASE + HANGUL_LCOUNT
&& HANGUL_VBASE <= cp2
&& cp2 < HANGUL_VBASE + HANGUL_VCOUNT)
{
return HANGUL_SBASE
+ (cp1 - HANGUL_LBASE) * HANGUL_NCOUNT
+ (cp2 - HANGUL_VBASE) * HANGUL_TCOUNT;
}
// Hangul LV + T -> LVT
//
if ( HANGUL_SBASE <= cp1
&& cp1 < HANGUL_SBASE + HANGUL_SCOUNT
&& 0 == ((cp1 - HANGUL_SBASE) % HANGUL_TCOUNT)
&& HANGUL_TBASE < cp2
&& cp2 < HANGUL_TBASE + HANGUL_TCOUNT)
{
return cp1 + (cp2 - HANGUL_TBASE);
}
// Table lookup via DFA.
//
UTF8 buf1[4], buf2[4];
int n1 = utf8_Encode(cp1, buf1);
int n2 = utf8_Encode(cp2, buf2);
if (n1 <= 0 || n2 <= 0)
{
return 0;
}
return ComposeViaTable(buf1, n1, buf2, n2);
}
// ---------------------------------------------------------------------------
// NFC Quick Check: utf8_is_nfc()
// ---------------------------------------------------------------------------
bool utf8_is_nfc(const UTF8 *src, size_t nSrc)
{
const UTF8 *p = src;
const UTF8 *pEnd = src + nSrc;
int lastCCC = 0;
while (p < pEnd)
{
// ASCII fast path. Every code point U+0000..U+007F has CCC = 0 and
// NFC_QC = Yes, so neither the quick-check verdict nor the canonical
// ordering test below can be changed by one. Verified exhaustively
// against this file's own tables: all 128 return ccc = 0, qc = 0,
// while 114 of U+0080..U+03FF do not, so the property is specific to
// ASCII rather than vacuous.
//
// This is worth a branch because net.cpp normalizes every command
// line from every player and those lines are overwhelmingly ASCII.
// Without it each byte pays a full DFA traversal in GetCCCandNFCQC:
// measured 5.2 ns/char before, 0.5 ns/char after (#1907).
//
// lastCCC = 0 is what the general path below would store for a
// CCC = 0 code point, so the ordering test is unaffected.
//
if (*p < 0x80)
{
lastCCC = 0;
p++;
continue;
}
const UTF8 *pStart = p;
int n = utf8_FirstByte[*p];
if (n <= 0 || n >= UTF8_CONTINUE)
{
return false;
}
if (p + n > pEnd)
{
return false;
}
for (int i = 1; i < n; i++)
{
if (UTF8_CONTINUE != utf8_FirstByte[p[i]])
{
return false;
}
}
UTF32 cp = utf8_decode_raw(p, n);
if (!utf8_is_valid_scalar(cp, n))
{
return false;
}
// Combined CCC + NFC_QC lookup — single DFA traversal.
//
int ccc, qc;
GetCCCandNFCQC(pStart, pStart + n, ccc, qc);
if (0 != qc)
{
return false; // No or Maybe
}
if (ccc != 0 && lastCCC > ccc)
{
// Combining marks out of canonical order.
return false;
}
lastCCC = ccc;
p = pStart + n;
}
return true;
}
// ---------------------------------------------------------------------------
// NFC Normalization: utf8_normalize_nfc()
//
// Algorithm (UAX #15):
// 1. Decompose: Expand each code point to its NFD form (canonical
// decomposition, recursive). Hangul syllables decomposed algorithmically.
// 2. Reorder: Sort combining marks by Canonical Combining Class (stable).
// 3. Compose: Combine starter + combining mark pairs back into precomposed
// forms where possible. Hangul composed algorithmically.
// ---------------------------------------------------------------------------
// Maximum code points we can handle in a single normalization buffer.
// A LBUF_SIZE string is at most LBUF_SIZE code points (ASCII case).
// After decomposition, each code point can expand to at most ~4 code points.
// In practice, decomposition rarely exceeds 2:1 expansion.
//
#define NFC_MAX_CODEPOINTS (LBUF_SIZE * 2)
struct NFCCodePoint
{
UTF32 cp;
int ccc;
};
// Decompose a single code point into the buffer. Handles Hangul
// algorithmically and uses the NFD DFA table for everything else.
//
static void DecomposeOne(UTF32 cp, NFCCodePoint *buf, int &n, int maxN)
{
// Hangul syllable decomposition.
//
if (HANGUL_SBASE <= cp && cp < HANGUL_SBASE + HANGUL_SCOUNT)
{
int sIndex = cp - HANGUL_SBASE;
UTF32 l = HANGUL_LBASE + sIndex / HANGUL_NCOUNT;
UTF32 v = HANGUL_VBASE + (sIndex % HANGUL_NCOUNT) / HANGUL_TCOUNT;
UTF32 t = HANGUL_TBASE + sIndex % HANGUL_TCOUNT;
if (n < maxN) { buf[n].cp = l; buf[n].ccc = 0; n++; }
if (n < maxN) { buf[n].cp = v; buf[n].ccc = 0; n++; }
if (t != HANGUL_TBASE && n < maxN)
{
buf[n].cp = t; buf[n].ccc = 0; n++;
}
return;
}
// Table lookup.
//
UTF8 encoded[4];
int nBytes = utf8_Encode(cp, encoded);
if (nBytes <= 0)
{
return;
}
bool bXor;
const string_desc *sd = GetNFD(encoded, bXor);
if (nullptr == sd)
{
// No decomposition — emit the code point as-is.
//
if (n < maxN)
{
UTF8 enc2[4];
int nb2 = utf8_Encode(cp, enc2);
buf[n].cp = cp;
buf[n].ccc = GetCCC(enc2, enc2 + nb2);
n++;
}
return;
}
// The string_desc contains the decomposed UTF-8 bytes.
// If bXor, we need to XOR the original bytes with the pattern.
//
UTF8 decomposed[32];
size_t nDecomp;
if (bXor)
{
// XOR decomposition: XOR each byte of the original encoding
// with the pattern bytes.
//
nDecomp = sd->n_bytes;
if (nDecomp > sizeof(decomposed)) nDecomp = sizeof(decomposed);
for (size_t i = 0; i < nDecomp; i++)
{
decomposed[i] = encoded[i] ^ sd->p[i];
}
}
else
{
// Literal decomposition.
//
nDecomp = sd->n_bytes;
if (nDecomp > sizeof(decomposed)) nDecomp = sizeof(decomposed);
memcpy(decomposed, sd->p, nDecomp);
}
// Parse the decomposed UTF-8 into code points and recursively decompose.
// (The table already contains fully recursive decompositions, but we
// look up CCC for each resulting code point.)
//
const UTF8 *dp = decomposed;
const UTF8 *dpEnd = decomposed + nDecomp;
while (dp < dpEnd && n < maxN)
{
const UTF8 *dpStart = dp;
UTF32 dcp = utf8_Decode(&dp, dpEnd);
if (UNI_EOF == dcp)
{
break;
}
// The table output is already fully decomposed, so no further
// recursion needed. Just look up CCC.
//
UTF8 enc3[4];
int nb3 = utf8_Encode(dcp, enc3);
buf[n].cp = dcp;
buf[n].ccc = (nb3 > 0) ? GetCCC(enc3, enc3 + nb3) : 0;
n++;
}
}
// Canonical ordering: stable sort combining marks by CCC.
// Starters (CCC=0) are never reordered.
//
static void CanonicalOrder(NFCCodePoint *buf, int n)
{
// Simple insertion sort — combining mark sequences are short.
//
for (int i = 1; i < n; i++)
{
if (buf[i].ccc != 0)
{
NFCCodePoint tmp = buf[i];
int j = i;
while (j > 0 && buf[j-1].ccc > tmp.ccc && buf[j-1].ccc != 0)
{
buf[j] = buf[j-1];
j--;
}
buf[j] = tmp;
}
}
}
// Canonical composition step.
//
static void CanonicalCompose(NFCCodePoint *buf, int &n)
{
if (n < 2)
{
return;
}
// Find the starter (leftmost CCC=0 code point).
//
int starterIdx = -1;
for (int i = 0; i < n; i++)
{
if (0 == buf[i].ccc)
{
starterIdx = i;
break;
}
}
if (starterIdx < 0)
{
return;
}
int lastCCC = -1;
for (int i = starterIdx + 1; i < n; i++)
{
int ccc = buf[i].ccc;
// UAX #15 D115: B is blocked from starter A if and only if there is
// some character C between A and B where CCC(C) = 0 or
// CCC(C) >= CCC(B).
//
// Note there is no exemption for CCC(B) = 0. The opposite holds: when
// B is itself a starter, CCC(B) = 0, so every intervening mark
// satisfies CCC(C) >= 0 and blocks it. An earlier `&& ccc != 0` here
// removed protection from exactly those sequences, so NFC composed
// Hangul jamo across an intervening mark -- U+B3C4 U+032B U+11C1
// became U+B3DE U+032B, consuming the jongseong and changing the
// syllable (#1905).
//
// lastCCC is only ever assigned from a non-zero ccc, so the
// lastCCC != -1 test already means "an intervening mark was seen";
// adjacent starters still compose normally.
//
bool blocked = (lastCCC != -1 && lastCCC >= ccc);
if (!blocked)
{
UTF32 composed = Compose(buf[starterIdx].cp, buf[i].cp);
if (0 != composed)
{
// Replace the starter with the composed character.
//
buf[starterIdx].cp = composed;
// Remove buf[i] by shifting.
//
for (int j = i; j < n - 1; j++)
{
buf[j] = buf[j+1];
}
n--;
i--;
// Reset lastCCC since we modified the sequence.
//
lastCCC = -1;
continue;
}
}
if (0 == ccc)
{
// New starter.
//
starterIdx = i;
lastCCC = -1;
}
else
{
lastCCC = ccc;
}
}
}
// The NFD working buffer below is NFC_MAX_CODEPOINTS * sizeof(NFCCodePoint),
// which is 512 KB. A frame that large makes GCC's -fstack-clash-protection
// (enabled by default on Debian/Ubuntu, so it appears in no CFLAGS listing)
// emit a prologue loop that touches all 128 of its 4 KB pages on entry:
//
// lea -0x80000(%rsp),%r11
// loop:
// sub $0x1000,%rsp
// orq $0x0,(%rsp) ; probe the page
// cmp %r11,%rsp
// jne loop ; 128 iterations
//
// A prologue runs before any early-out in the body, so folding this buffer
// into utf8_normalize_nfc() would charge those 128 probes to every caller --
// including the already-NFC fast path, which is the common case and touches
// the buffer not at all. Measured at ~51% of the total CPU of a single
// softcode command that calls chr() in a loop, with 93% of the function's
// samples landing on the `cmp` above.
//
// Keep this split. Do not move the array back into utf8_normalize_nfc(),
// and do not shrink NFC_MAX_CODEPOINTS -- LBUF_SIZE * 2 is the worst-case
// decomposition bound and lowering it is a correctness regression, not an
// optimization.
//
// Correctness note: this helper is not reentrant-sensitive. It does not
// recurse, and DecomposeOne/CanonicalOrder/CanonicalCompose/utf8_Encode are
// all pure, so the buffer cannot be observed across calls.
//
#if defined(_MSC_VER)
#define NFC_NOINLINE __declspec(noinline)
#elif defined(__GNUC__) || defined(__clang__)
#define NFC_NOINLINE __attribute__((noinline))
#else
#define NFC_NOINLINE
#endif
static NFC_NOINLINE void utf8_normalize_nfc_slow(const UTF8 *src, size_t nSrc,
UTF8 *dst, size_t nDstMax, size_t *pnDst)
{
// Step 1: Decompose all code points to NFD.
//
// NFC_MAX_CODEPOINTS is LBUF_SIZE * 2, so at LBUF_SIZE 32768 this table is
// 512 KB -- the whole of this function's stack frame. It is the slow path
// of a routine every piece of string handling reaches, so it should not be
// costing half a megabyte of stack to enter.
//
// Safe as static: utf8_normalize_nfc_slow neither recurses nor evaluates
// softcode. It calls only utf8_Decode, utf8_Encode, DecomposeOne,
// CanonicalOrder, CanonicalCompose and memcpy, none of which call back into
// it, so no two activations are ever live on one thread at the same time.
// (utf8_normalize.cpp is not one of the three sources compiled into the
// freestanding rv64 blob, so unlike color_ops.c it may use .bss.)
//
static thread_local NFCCodePoint cps[NFC_MAX_CODEPOINTS];
int nCps = 0;
const UTF8 *p = src;
const UTF8 *pEnd = src + nSrc;
while (p < pEnd && nCps < NFC_MAX_CODEPOINTS)
{
UTF32 cp = utf8_Decode(&p, pEnd);
if (UNI_EOF == cp)
{
continue;
}
DecomposeOne(cp, cps, nCps, NFC_MAX_CODEPOINTS);
}
// Step 2: Canonical ordering (sort combining marks by CCC).
//
CanonicalOrder(cps, nCps);
// Step 3: Canonical composition.
//
CanonicalCompose(cps, nCps);
// Step 4: Encode back to UTF-8.
//
size_t nOut = 0;
for (int i = 0; i < nCps; i++)
{
UTF8 enc[4];
int nb = utf8_Encode(cps[i].cp, enc);
if (nb > 0 && nOut + nb <= nDstMax)
{
memcpy(dst + nOut, enc, nb);
nOut += nb;
}
}
*pnDst = nOut;
}
void utf8_normalize_nfc(const UTF8 *src, size_t nSrc, UTF8 *dst, size_t nDstMax, size_t *pnDst)
{
*pnDst = 0;
// Quick check: if already NFC, just copy. This path must stay free of
// any large local, or it inherits the probe loop described above.
//
if (utf8_is_nfc(src, nSrc))
{
size_t nCopy = (nSrc < nDstMax) ? nSrc : nDstMax;
memcpy(dst, src, nCopy);
*pnDst = nCopy;
return;
}
utf8_normalize_nfc_slow(src, nSrc, dst, nDstMax, pnDst);
}
Reference in a new issue View git blame Copy permalink