/*! \file utf8_normalize.cpp * \brief NFC normalization for UTF-8 strings. * * Implements Unicode NFC normalization (Canonical Decomposition followed by * Canonical Composition) using DFA state machines for all Unicode property * lookups. Hangul composition and decomposition are handled algorithmically. * * All operations are locale-independent per Unicode default algorithms. */ #include "copyright.h" #include "autoconf.h" #include "config.h" #include "core.h" // Hangul constants (Unicode 3.0+ algorithmic composition/decomposition). // #define HANGUL_SBASE 0xAC00 #define HANGUL_LBASE 0x1100 #define HANGUL_VBASE 0x1161 #define HANGUL_TBASE 0x11A7 #define HANGUL_LCOUNT 19 #define HANGUL_VCOUNT 21 #define HANGUL_TCOUNT 28 #define HANGUL_NCOUNT (HANGUL_VCOUNT * HANGUL_TCOUNT) // 588 #define HANGUL_SCOUNT (HANGUL_LCOUNT * HANGUL_NCOUNT) // 11172 // --------------------------------------------------------------------------- // UTF-8 <-> UTF-32 helpers. // --------------------------------------------------------------------------- // Decode a single UTF-32 code point from a UTF-8 byte sequence. // Advances *pp past the code point. Returns UNI_EOF on error. // static UTF32 utf8_Decode(const UTF8 **pp, const UTF8 *pEnd) { const UTF8 *p = *pp; if (p >= pEnd) { return UNI_EOF; } int n = utf8_FirstByte[*p]; if (n <= 0 || n >= UTF8_CONTINUE) { // Continuation byte or invalid — skip one byte. (*pp)++; return UNI_EOF; } if (p + n > pEnd) { // Truncated sequence at end of buffer. *pp = pEnd; return UNI_EOF; } for (int i = 1; i < n; i++) { if (UTF8_CONTINUE != utf8_FirstByte[p[i]]) { // Invalid continuation byte — skip one byte. (*pp)++; return UNI_EOF; } } UTF32 cp = utf8_decode_raw(p, n); if (!utf8_is_valid_scalar(cp, n)) { (*pp)++; return UNI_EOF; } *pp = p + n; return cp; } // Encode a UTF-32 code point to UTF-8. // Returns the number of bytes written (1-4), or 0 on error. // static int utf8_Encode(UTF32 cp, UTF8 *buf) { if (cp < 0x80) { buf[0] = static_cast(cp); return 1; } else if (cp < 0x800) { buf[0] = static_cast(0xC0 | (cp >> 6)); buf[1] = static_cast(0x80 | (cp & 0x3F)); return 2; } else if (cp < 0x10000) { buf[0] = static_cast(0xE0 | (cp >> 12)); buf[1] = static_cast(0x80 | ((cp >> 6) & 0x3F)); buf[2] = static_cast(0x80 | (cp & 0x3F)); return 3; } else if (cp <= 0x10FFFF) { buf[0] = static_cast(0xF0 | (cp >> 18)); buf[1] = static_cast(0x80 | ((cp >> 12) & 0x3F)); buf[2] = static_cast(0x80 | ((cp >> 6) & 0x3F)); buf[3] = static_cast(0x80 | (cp & 0x3F)); return 4; } return 0; } // --------------------------------------------------------------------------- // DFA traversal helpers. // --------------------------------------------------------------------------- // Run the integer-valued DFA (CCC, NFC_QC) on a single code point's bytes. // Returns the accepting state offset (the integer value), or the default. // Templated on SBT type because the generator uses unsigned char for small // tables and unsigned short for larger ones. // template static int RunIntegerDFA( const unsigned char *itt, const unsigned short *sot, const T *sbt, int nStartState, int nAcceptStart, int nDefault, const UTF8 *pStart, const UTF8 *pEnd) { int iState = nStartState; const UTF8 *p = pStart; while (p < pEnd && iState < nAcceptStart) { unsigned char ch = *p++; unsigned char iColumn = itt[ch]; unsigned short iOffset = sot[iState]; for (;;) { int y = sbt[iOffset]; if (y < 128) { if (iColumn < y) { iState = sbt[iOffset + 1]; break; } else { iColumn = static_cast(iColumn - y); iOffset += 2; } } else { y = 256 - y; if (iColumn < y) { iState = sbt[iOffset + iColumn + 1]; break; } else { iColumn = static_cast(iColumn - y); iOffset = static_cast(iOffset + y + 1); } } } } if (iState < nAcceptStart) { return nDefault; } return iState - nAcceptStart; } // Get Canonical Combining Class for a code point. // static int GetCCC(const UTF8 *pStart, const UTF8 *pEnd) { return RunIntegerDFA( tr_ccc_nfcqc_itt, tr_ccc_nfcqc_sot, tr_ccc_nfcqc_sbt, TR_CCC_NFCQC_START_STATE, TR_CCC_NFCQC_ACCEPTING_STATES_START, 0, pStart, pEnd) / 3; } // Combined CCC + NFC_QC lookup — single DFA traversal instead of two. // static void GetCCCandNFCQC(const UTF8 *pStart, const UTF8 *pEnd, int &ccc, int &nfcqc) { int combined = RunIntegerDFA( tr_ccc_nfcqc_itt, tr_ccc_nfcqc_sot, tr_ccc_nfcqc_sbt, TR_CCC_NFCQC_START_STATE, TR_CCC_NFCQC_ACCEPTING_STATES_START, 0, pStart, pEnd); ccc = combined / 3; nfcqc = combined % 3; } // Get NFD decomposition for a code point. // Returns the string_desc* or nullptr if no decomposition. // static const string_desc *GetNFD(const UTF8 *p, bool &bXor) { unsigned short iState = TR_NFD_START_STATE; do { unsigned char ch = *p++; unsigned char iColumn = tr_nfd_itt[ch]; unsigned short iOffset = tr_nfd_sot[iState]; for (;;) { int y = tr_nfd_sbt[iOffset]; if (y < 128) { if (iColumn < y) { iState = tr_nfd_sbt[iOffset + 1]; break; } else { iColumn = static_cast(iColumn - y); iOffset += 2; } } else { y = 256 - y; if (iColumn < y) { iState = tr_nfd_sbt[iOffset + iColumn + 1]; break; } else { iColumn = static_cast(iColumn - y); iOffset = static_cast(iOffset + y + 1); } } } } while (iState < TR_NFD_ACCEPTING_STATES_START); int idx = iState - TR_NFD_ACCEPTING_STATES_START; if (TR_NFD_DEFAULT == idx) { bXor = false; return nullptr; } bXor = (TR_NFD_XOR_START <= idx); return tr_nfd_ott + idx - 1; } // Look up NFC composition pair via the two-code-point DFA. // Returns the composed code point, or 0 if no composition. // static UTF32 ComposeViaTable(const UTF8 *pStarter, int nStarterBytes, const UTF8 *pCombining, int nCombiningBytes) { int iState = TR_NFC_COMPOSE_START_STATE; // Feed starter bytes. // for (int i = 0; i < nStarterBytes && iState < TR_NFC_COMPOSE_ACCEPTING_STATES_START; i++) { unsigned char ch = pStarter[i]; unsigned char iColumn = tr_nfc_compose_itt[ch]; unsigned short iOffset = tr_nfc_compose_sot[iState]; for (;;) { int y = tr_nfc_compose_sbt[iOffset]; if (y < 128) { if (iColumn < y) { iState = tr_nfc_compose_sbt[iOffset + 1]; break; } else { iColumn = static_cast(iColumn - y); iOffset += 2; } } else { y = 256 - y; if (iColumn < y) { iState = tr_nfc_compose_sbt[iOffset + iColumn + 1]; break; } else { iColumn = static_cast(iColumn - y); iOffset = static_cast(iOffset + y + 1); } } } } // Feed combining bytes. // for (int i = 0; i < nCombiningBytes && iState < TR_NFC_COMPOSE_ACCEPTING_STATES_START; i++) { unsigned char ch = pCombining[i]; unsigned char iColumn = tr_nfc_compose_itt[ch]; unsigned short iOffset = tr_nfc_compose_sot[iState]; for (;;) { int y = tr_nfc_compose_sbt[iOffset]; if (y < 128) { if (iColumn < y) { iState = tr_nfc_compose_sbt[iOffset + 1]; break; } else { iColumn = static_cast(iColumn - y); iOffset += 2; } } else { y = 256 - y; if (iColumn < y) { iState = tr_nfc_compose_sbt[iOffset + iColumn + 1]; break; } else { iColumn = static_cast(iColumn - y); iOffset = static_cast(iOffset + y + 1); } } } } if (iState < TR_NFC_COMPOSE_ACCEPTING_STATES_START) { return 0; // No composition. } int idx = iState - TR_NFC_COMPOSE_ACCEPTING_STATES_START; if (0 == idx) { return 0; // Default state = no composition. } return tr_nfc_compose_nfc_compose_result[idx]; } // Compose two code points. Handles Hangul algorithmically, then // falls back to the DFA table. // static UTF32 Compose(UTF32 cp1, UTF32 cp2) { // Hangul L + V -> LV // if ( HANGUL_LBASE <= cp1 && cp1 < HANGUL_LBASE + HANGUL_LCOUNT && HANGUL_VBASE <= cp2 && cp2 < HANGUL_VBASE + HANGUL_VCOUNT) { return HANGUL_SBASE + (cp1 - HANGUL_LBASE) * HANGUL_NCOUNT + (cp2 - HANGUL_VBASE) * HANGUL_TCOUNT; } // Hangul LV + T -> LVT // if ( HANGUL_SBASE <= cp1 && cp1 < HANGUL_SBASE + HANGUL_SCOUNT && 0 == ((cp1 - HANGUL_SBASE) % HANGUL_TCOUNT) && HANGUL_TBASE < cp2 && cp2 < HANGUL_TBASE + HANGUL_TCOUNT) { return cp1 + (cp2 - HANGUL_TBASE); } // Table lookup via DFA. // UTF8 buf1[4], buf2[4]; int n1 = utf8_Encode(cp1, buf1); int n2 = utf8_Encode(cp2, buf2); if (n1 <= 0 || n2 <= 0) { return 0; } return ComposeViaTable(buf1, n1, buf2, n2); } // --------------------------------------------------------------------------- // NFC Quick Check: utf8_is_nfc() // --------------------------------------------------------------------------- bool utf8_is_nfc(const UTF8 *src, size_t nSrc) { const UTF8 *p = src; const UTF8 *pEnd = src + nSrc; int lastCCC = 0; while (p < pEnd) { // ASCII fast path. Every code point U+0000..U+007F has CCC = 0 and // NFC_QC = Yes, so neither the quick-check verdict nor the canonical // ordering test below can be changed by one. Verified exhaustively // against this file's own tables: all 128 return ccc = 0, qc = 0, // while 114 of U+0080..U+03FF do not, so the property is specific to // ASCII rather than vacuous. // // This is worth a branch because net.cpp normalizes every command // line from every player and those lines are overwhelmingly ASCII. // Without it each byte pays a full DFA traversal in GetCCCandNFCQC: // measured 5.2 ns/char before, 0.5 ns/char after (#1907). // // lastCCC = 0 is what the general path below would store for a // CCC = 0 code point, so the ordering test is unaffected. // if (*p < 0x80) { lastCCC = 0; p++; continue; } const UTF8 *pStart = p; int n = utf8_FirstByte[*p]; if (n <= 0 || n >= UTF8_CONTINUE) { return false; } if (p + n > pEnd) { return false; } for (int i = 1; i < n; i++) { if (UTF8_CONTINUE != utf8_FirstByte[p[i]]) { return false; } } UTF32 cp = utf8_decode_raw(p, n); if (!utf8_is_valid_scalar(cp, n)) { return false; } // Combined CCC + NFC_QC lookup — single DFA traversal. // int ccc, qc; GetCCCandNFCQC(pStart, pStart + n, ccc, qc); if (0 != qc) { return false; // No or Maybe } if (ccc != 0 && lastCCC > ccc) { // Combining marks out of canonical order. return false; } lastCCC = ccc; p = pStart + n; } return true; } // --------------------------------------------------------------------------- // NFC Normalization: utf8_normalize_nfc() // // Algorithm (UAX #15): // 1. Decompose: Expand each code point to its NFD form (canonical // decomposition, recursive). Hangul syllables decomposed algorithmically. // 2. Reorder: Sort combining marks by Canonical Combining Class (stable). // 3. Compose: Combine starter + combining mark pairs back into precomposed // forms where possible. Hangul composed algorithmically. // --------------------------------------------------------------------------- // Maximum code points we can handle in a single normalization buffer. // A LBUF_SIZE string is at most LBUF_SIZE code points (ASCII case). // After decomposition, each code point can expand to at most ~4 code points. // In practice, decomposition rarely exceeds 2:1 expansion. // #define NFC_MAX_CODEPOINTS (LBUF_SIZE * 2) struct NFCCodePoint { UTF32 cp; int ccc; }; // Decompose a single code point into the buffer. Handles Hangul // algorithmically and uses the NFD DFA table for everything else. // static void DecomposeOne(UTF32 cp, NFCCodePoint *buf, int &n, int maxN) { // Hangul syllable decomposition. // if (HANGUL_SBASE <= cp && cp < HANGUL_SBASE + HANGUL_SCOUNT) { int sIndex = cp - HANGUL_SBASE; UTF32 l = HANGUL_LBASE + sIndex / HANGUL_NCOUNT; UTF32 v = HANGUL_VBASE + (sIndex % HANGUL_NCOUNT) / HANGUL_TCOUNT; UTF32 t = HANGUL_TBASE + sIndex % HANGUL_TCOUNT; if (n < maxN) { buf[n].cp = l; buf[n].ccc = 0; n++; } if (n < maxN) { buf[n].cp = v; buf[n].ccc = 0; n++; } if (t != HANGUL_TBASE && n < maxN) { buf[n].cp = t; buf[n].ccc = 0; n++; } return; } // Table lookup. // UTF8 encoded[4]; int nBytes = utf8_Encode(cp, encoded); if (nBytes <= 0) { return; } bool bXor; const string_desc *sd = GetNFD(encoded, bXor); if (nullptr == sd) { // No decomposition — emit the code point as-is. // if (n < maxN) { UTF8 enc2[4]; int nb2 = utf8_Encode(cp, enc2); buf[n].cp = cp; buf[n].ccc = GetCCC(enc2, enc2 + nb2); n++; } return; } // The string_desc contains the decomposed UTF-8 bytes. // If bXor, we need to XOR the original bytes with the pattern. // UTF8 decomposed[32]; size_t nDecomp; if (bXor) { // XOR decomposition: XOR each byte of the original encoding // with the pattern bytes. // nDecomp = sd->n_bytes; if (nDecomp > sizeof(decomposed)) nDecomp = sizeof(decomposed); for (size_t i = 0; i < nDecomp; i++) { decomposed[i] = encoded[i] ^ sd->p[i]; } } else { // Literal decomposition. // nDecomp = sd->n_bytes; if (nDecomp > sizeof(decomposed)) nDecomp = sizeof(decomposed); memcpy(decomposed, sd->p, nDecomp); } // Parse the decomposed UTF-8 into code points and recursively decompose. // (The table already contains fully recursive decompositions, but we // look up CCC for each resulting code point.) // const UTF8 *dp = decomposed; const UTF8 *dpEnd = decomposed + nDecomp; while (dp < dpEnd && n < maxN) { const UTF8 *dpStart = dp; UTF32 dcp = utf8_Decode(&dp, dpEnd); if (UNI_EOF == dcp) { break; } // The table output is already fully decomposed, so no further // recursion needed. Just look up CCC. // UTF8 enc3[4]; int nb3 = utf8_Encode(dcp, enc3); buf[n].cp = dcp; buf[n].ccc = (nb3 > 0) ? GetCCC(enc3, enc3 + nb3) : 0; n++; } } // Canonical ordering: stable sort combining marks by CCC. // Starters (CCC=0) are never reordered. // static void CanonicalOrder(NFCCodePoint *buf, int n) { // Simple insertion sort — combining mark sequences are short. // for (int i = 1; i < n; i++) { if (buf[i].ccc != 0) { NFCCodePoint tmp = buf[i]; int j = i; while (j > 0 && buf[j-1].ccc > tmp.ccc && buf[j-1].ccc != 0) { buf[j] = buf[j-1]; j--; } buf[j] = tmp; } } } // Canonical composition step. // static void CanonicalCompose(NFCCodePoint *buf, int &n) { if (n < 2) { return; } // Find the starter (leftmost CCC=0 code point). // int starterIdx = -1; for (int i = 0; i < n; i++) { if (0 == buf[i].ccc) { starterIdx = i; break; } } if (starterIdx < 0) { return; } int lastCCC = -1; for (int i = starterIdx + 1; i < n; i++) { int ccc = buf[i].ccc; // UAX #15 D115: B is blocked from starter A if and only if there is // some character C between A and B where CCC(C) = 0 or // CCC(C) >= CCC(B). // // Note there is no exemption for CCC(B) = 0. The opposite holds: when // B is itself a starter, CCC(B) = 0, so every intervening mark // satisfies CCC(C) >= 0 and blocks it. An earlier `&& ccc != 0` here // removed protection from exactly those sequences, so NFC composed // Hangul jamo across an intervening mark -- U+B3C4 U+032B U+11C1 // became U+B3DE U+032B, consuming the jongseong and changing the // syllable (#1905). // // lastCCC is only ever assigned from a non-zero ccc, so the // lastCCC != -1 test already means "an intervening mark was seen"; // adjacent starters still compose normally. // bool blocked = (lastCCC != -1 && lastCCC >= ccc); if (!blocked) { UTF32 composed = Compose(buf[starterIdx].cp, buf[i].cp); if (0 != composed) { // Replace the starter with the composed character. // buf[starterIdx].cp = composed; // Remove buf[i] by shifting. // for (int j = i; j < n - 1; j++) { buf[j] = buf[j+1]; } n--; i--; // Reset lastCCC since we modified the sequence. // lastCCC = -1; continue; } } if (0 == ccc) { // New starter. // starterIdx = i; lastCCC = -1; } else { lastCCC = ccc; } } } // The NFD working buffer below is NFC_MAX_CODEPOINTS * sizeof(NFCCodePoint), // which is 512 KB. A frame that large makes GCC's -fstack-clash-protection // (enabled by default on Debian/Ubuntu, so it appears in no CFLAGS listing) // emit a prologue loop that touches all 128 of its 4 KB pages on entry: // // lea -0x80000(%rsp),%r11 // loop: // sub $0x1000,%rsp // orq $0x0,(%rsp) ; probe the page // cmp %r11,%rsp // jne loop ; 128 iterations // // A prologue runs before any early-out in the body, so folding this buffer // into utf8_normalize_nfc() would charge those 128 probes to every caller -- // including the already-NFC fast path, which is the common case and touches // the buffer not at all. Measured at ~51% of the total CPU of a single // softcode command that calls chr() in a loop, with 93% of the function's // samples landing on the `cmp` above. // // Keep this split. Do not move the array back into utf8_normalize_nfc(), // and do not shrink NFC_MAX_CODEPOINTS -- LBUF_SIZE * 2 is the worst-case // decomposition bound and lowering it is a correctness regression, not an // optimization. // // Correctness note: this helper is not reentrant-sensitive. It does not // recurse, and DecomposeOne/CanonicalOrder/CanonicalCompose/utf8_Encode are // all pure, so the buffer cannot be observed across calls. // #if defined(_MSC_VER) #define NFC_NOINLINE __declspec(noinline) #elif defined(__GNUC__) || defined(__clang__) #define NFC_NOINLINE __attribute__((noinline)) #else #define NFC_NOINLINE #endif static NFC_NOINLINE void utf8_normalize_nfc_slow(const UTF8 *src, size_t nSrc, UTF8 *dst, size_t nDstMax, size_t *pnDst) { // Step 1: Decompose all code points to NFD. // // NFC_MAX_CODEPOINTS is LBUF_SIZE * 2, so at LBUF_SIZE 32768 this table is // 512 KB -- the whole of this function's stack frame. It is the slow path // of a routine every piece of string handling reaches, so it should not be // costing half a megabyte of stack to enter. // // Safe as static: utf8_normalize_nfc_slow neither recurses nor evaluates // softcode. It calls only utf8_Decode, utf8_Encode, DecomposeOne, // CanonicalOrder, CanonicalCompose and memcpy, none of which call back into // it, so no two activations are ever live on one thread at the same time. // (utf8_normalize.cpp is not one of the three sources compiled into the // freestanding rv64 blob, so unlike color_ops.c it may use .bss.) // static thread_local NFCCodePoint cps[NFC_MAX_CODEPOINTS]; int nCps = 0; const UTF8 *p = src; const UTF8 *pEnd = src + nSrc; while (p < pEnd && nCps < NFC_MAX_CODEPOINTS) { UTF32 cp = utf8_Decode(&p, pEnd); if (UNI_EOF == cp) { continue; } DecomposeOne(cp, cps, nCps, NFC_MAX_CODEPOINTS); } // Step 2: Canonical ordering (sort combining marks by CCC). // CanonicalOrder(cps, nCps); // Step 3: Canonical composition. // CanonicalCompose(cps, nCps); // Step 4: Encode back to UTF-8. // size_t nOut = 0; for (int i = 0; i < nCps; i++) { UTF8 enc[4]; int nb = utf8_Encode(cps[i].cp, enc); if (nb > 0 && nOut + nb <= nDstMax) { memcpy(dst + nOut, enc, nb); nOut += nb; } } *pnDst = nOut; } void utf8_normalize_nfc(const UTF8 *src, size_t nSrc, UTF8 *dst, size_t nDstMax, size_t *pnDst) { *pnDst = 0; // Quick check: if already NFC, just copy. This path must stay free of // any large local, or it inherits the probe loop described above. // if (utf8_is_nfc(src, nSrc)) { size_t nCopy = (nSrc < nDstMax) ? nSrc : nDstMax; memcpy(dst, src, nCopy); *pnDst = nCopy; return; } utf8_normalize_nfc_slow(src, nSrc, dst, nDstMax, pnDst); }