mmo-ref/otclient-redemption
1
0
Fork 0
mirror of https://github.com/mehah/otclient synced 2026-08-15 16:29:06 -04:00
Code Issues Projects Releases Packages Wiki Activity

refactor: string encoding to use utf8cpp and add tests (#1443)

Browse source 
Update string encoding implementation to use utf8cpp

This change updates the string encoding implementation to use the utf8cpp library for improved robustness and consistency across platforms. It refactors the encoding functions in src/framework/stdext/string.cpp, adds comprehensive unit tests, updates build configurations to include utf8cpp, and documents the new encoding policy and behavior.

String encoding improvements

Refactored all string encoding functions in src/framework/stdext/string.cpp to use the utf8cpp library, providing strict UTF-8 validation, robust conversions between UTF-8, Latin-1, and UTF-16, and consistent error handling.

Added a new documentation file docs/string-encoding-policy.md detailing the updated encoding policy, error handling, dependencies, and testing strategy.

Testing enhancements

Added a new unit test suite in tests/stdext/string_encoding_test.cpp covering UTF-8 validation, conversions, roundtrip consistency, control character handling, and platform-specific UTF-16 conversions.

Updated the test build configuration to include the new test directory and sources.

Build system updates

Updated src/CMakeLists.txt to require and link the utf8cpp library for all platforms, ensuring proper integration.

Added utf8cpp as a dependency in vcpkg.json for package management.
This commit is contained in:
Eduardo Dantas 2025-11-20 02:08:02 -03:00 • committed by GitHub
parent bf6e3c523a
commit 67460c09bb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 249 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

112
tests/stdext/string_encoding_test.cpp Normal file
View file

@ -0,0 +1,112 @@
#include <gtest/gtest.h>
#include <string>
#include <framework/stdext/string.h>
namespace {
TEST(StringEncoding, Utf8Validation)
{
EXPECT_TRUE(stdext::is_valid_utf8("Hello World"));
EXPECT_TRUE(stdext::is_valid_utf8(""));
EXPECT_TRUE(stdext::is_valid_utf8("ASCII 123"));
EXPECT_TRUE(stdext::is_valid_utf8("Café"));
EXPECT_TRUE(stdext::is_valid_utf8("日本語"));
EXPECT_TRUE(stdext::is_valid_utf8("🎉🎊"));
EXPECT_FALSE(stdext::is_valid_utf8("\x80"));
EXPECT_FALSE(stdext::is_valid_utf8("\xFF"));
EXPECT_FALSE(stdext::is_valid_utf8("\xC0\x80"));
EXPECT_FALSE(stdext::is_valid_utf8("\xF5\x80\x80\x80"));
EXPECT_FALSE(stdext::is_valid_utf8("\xC2"));
EXPECT_FALSE(stdext::is_valid_utf8("\xED\xA0\x80"));
}
TEST(StringEncoding, Utf8ToLatin1)
{
EXPECT_EQ(stdext::utf8_to_latin1("Hello"), "Hello");
EXPECT_EQ(stdext::utf8_to_latin1("123"), "123");
EXPECT_EQ(stdext::utf8_to_latin1("\t\r\n"), "\t\r\n");
EXPECT_EQ(stdext::utf8_to_latin1("Café"), "Caf\xE9");
EXPECT_EQ(stdext::utf8_to_latin1("Über"), "\xDC""ber");
EXPECT_EQ(stdext::utf8_to_latin1("naïve"), "na\xEF""ve");
EXPECT_EQ(stdext::utf8_to_latin1("Hello 世界"), "Hello ");
EXPECT_EQ(stdext::utf8_to_latin1("🎉"), "");
EXPECT_EQ(stdext::utf8_to_latin1("\xFF\xFE"), "");
EXPECT_EQ(stdext::utf8_to_latin1("\xC0\x80"), "");
EXPECT_EQ(stdext::utf8_to_latin1("\x01\x02\x03"), "");
EXPECT_EQ(stdext::utf8_to_latin1("\x1F"), "");
EXPECT_EQ(stdext::utf8_to_latin1("\x80\x90\x9F"), "");
EXPECT_EQ(stdext::utf8_to_latin1(""), "");
EXPECT_EQ(stdext::utf8_to_latin1(std::string("\x00", 1)), "");
EXPECT_EQ(stdext::utf8_to_latin1(std::string("\xC2\xA0")), "\xA0");
EXPECT_EQ(stdext::utf8_to_latin1("ÿ"), "\xFF"); // U+00FF
}
TEST(StringEncoding, Latin1ToUtf8)
{
EXPECT_EQ(stdext::latin1_to_utf8("Hello"), "Hello");
EXPECT_EQ(stdext::latin1_to_utf8("123"), "123");
EXPECT_EQ(stdext::latin1_to_utf8("\t\r\n"), "\t\r\n");
EXPECT_EQ(stdext::latin1_to_utf8("Caf\xE9"), "Café");
EXPECT_EQ(stdext::latin1_to_utf8("\xDC""ber"), "Über");
EXPECT_EQ(stdext::latin1_to_utf8("na\xEF""ve"), "naïve");
std::string latin1All;
latin1All.reserve(256);
for (int i = 0; i < 256; ++i) {
latin1All += static_cast<char>(i);
}
const auto utf8Result = stdext::latin1_to_utf8(latin1All);
EXPECT_FALSE(utf8Result.empty());
EXPECT_TRUE(stdext::is_valid_utf8(utf8Result));
EXPECT_EQ(stdext::latin1_to_utf8(""), "");
EXPECT_TRUE(stdext::is_valid_utf8(stdext::latin1_to_utf8(std::string("\x00", 1))));
}
TEST(StringEncoding, Roundtrip)
{
const std::string ascii = "Hello World 123!";
EXPECT_EQ(stdext::latin1_to_utf8(stdext::utf8_to_latin1(ascii)), ascii);
const std::string latin1 = "Caf\xE9 naïve";
EXPECT_EQ(stdext::utf8_to_latin1(stdext::latin1_to_utf8(latin1)), latin1);
EXPECT_EQ(stdext::utf8_to_latin1(stdext::latin1_to_utf8("")), "");
}
#ifdef WIN32
TEST(StringEncoding, Utf16Conversions)
{
EXPECT_EQ(stdext::utf8_to_utf16("Hello"), L"Hello");
EXPECT_EQ(stdext::utf16_to_utf8(L"Hello"), "Hello");
EXPECT_EQ(stdext::utf8_to_utf16("Café"), L"Café");
EXPECT_EQ(stdext::utf16_to_utf8(L"Café"), "Café");
EXPECT_EQ(stdext::utf8_to_utf16("🎉"), L"🎉");
EXPECT_EQ(stdext::utf16_to_utf8(L"🎉"), "🎉");
EXPECT_TRUE(stdext::utf8_to_utf16("\xFF\xFE").empty());
const std::wstring invalidSurrogate = L"\xD800";
EXPECT_TRUE(stdext::utf16_to_utf8(invalidSurrogate).empty());
EXPECT_EQ(stdext::latin1_to_utf16("Caf\xE9"), L"Café");
EXPECT_EQ(stdext::utf16_to_latin1(L"Café"), "Caf\xE9");
EXPECT_EQ(stdext::utf8_to_utf16(""), L"");
EXPECT_EQ(stdext::utf16_to_utf8(L""), "");
}
#endif
}