mirror of
https://github.com/EmberEmu/Ember
synced 2026-08-14 22:32:05 -04:00
410 lines
No EOL
9.1 KiB
C++
410 lines
No EOL
9.1 KiB
C++
/*
|
||
* Copyright (c) 2022 Ember
|
||
*
|
||
* This Source Code Form is subject to the terms of the Mozilla Public
|
||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||
*/
|
||
|
||
#include <shared/utility/UTF8.h>
|
||
#include <shared/utility/Utility.h>
|
||
#include <gtest/gtest.h>
|
||
#include <string>
|
||
|
||
using namespace ember;
|
||
|
||
/*
|
||
* Most UTF8 test cases are adapted from:
|
||
* https://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt
|
||
*
|
||
* Almost all of these are basic tests for whichever UTF8
|
||
* library is being used by the utilities rather than Ember's
|
||
* own code
|
||
*/
|
||
|
||
TEST(TextUtilities, StringLength_ASCII) {
|
||
std::string str = "Pretty boring test text";
|
||
auto len = utility::utf8::length(str);
|
||
EXPECT_EQ(len, 23);
|
||
|
||
str = "";
|
||
len = utility::utf8::length(str);
|
||
EXPECT_EQ(len, 0);
|
||
}
|
||
|
||
TEST(TextUtilities, StringLength_UTF8) {
|
||
std::string str = "日本語テキスト";
|
||
auto len = utility::utf8::length(str);
|
||
EXPECT_EQ(len, 7);
|
||
|
||
str = "ŏϭեۛåȆɷςƐҕђǹǹǍ";
|
||
len = utility::utf8::length(str);
|
||
EXPECT_EQ(len, 14);
|
||
|
||
str = "翻译不好的文本";
|
||
len = utility::utf8::length(str);
|
||
EXPECT_EQ(len, 7);
|
||
|
||
str = "";
|
||
len = utility::utf8::length(str);
|
||
EXPECT_EQ(len, 0);
|
||
}
|
||
|
||
TEST(TextUtilities, DISABLED_NameFormat_ASCII) {
|
||
// Lower
|
||
std::string str = "bankalt";
|
||
auto formatted = utility::utf8::name_format(str, std::locale());
|
||
EXPECT_EQ("Bankalt", formatted);
|
||
|
||
// Upper
|
||
str = "CHAOSVEX";
|
||
formatted = utility::utf8::name_format(str, std::locale());
|
||
EXPECT_EQ("Chaosvex", formatted);
|
||
|
||
// Mixed
|
||
str = "cHaoSvEx";
|
||
formatted = utility::utf8::name_format(str, std::locale());
|
||
EXPECT_EQ("Chaosvex", formatted);
|
||
}
|
||
|
||
TEST(TextUtilities, IsValid_UTF8_MixedSequences) {
|
||
std::string text = "CHAOSVEX";
|
||
bool res = utility::utf8::is_valid(text);
|
||
EXPECT_EQ(res, true);
|
||
|
||
text = "日本語テキスト";
|
||
res = utility::utf8::is_valid(text);
|
||
EXPECT_EQ(res, true);
|
||
|
||
const char* bad_text = "\xf0\x28\x8c\x28";
|
||
res = utility::utf8::is_valid(bad_text, 4);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
TEST(TextUtilities, IsValid_UTF8_OverlongSequences) {
|
||
{
|
||
const char* bad_text = "\xc0\xaf";
|
||
auto res = utility::utf8::is_valid(bad_text, 2);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "\xe0\x80\xaf";
|
||
auto res = utility::utf8::is_valid(bad_text, 3);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "\xf0\x80\x80\xaf";
|
||
auto res = utility::utf8::is_valid(bad_text, 4);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "\xf8\x80\x80\x80\xaf";
|
||
auto res = utility::utf8::is_valid(bad_text, 5);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "\xfc\x80\x80\x80\x80\xaf";
|
||
auto res = utility::utf8::is_valid(bad_text, 6);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
}
|
||
|
||
TEST(TextUtilities, IsValid_UTF8_UnexpectedContinuation) {
|
||
{
|
||
const char* bad_text = "\x80";
|
||
auto res = utility::utf8::is_valid(bad_text, 1);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "\xbf";
|
||
auto res = utility::utf8::is_valid(bad_text, 1);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
}
|
||
|
||
TEST(TextUtilities, IsValid_UTF8_OverlongNul) {
|
||
{
|
||
const char* bad_text = "\xc0\x80";
|
||
auto res = utility::utf8::is_valid(bad_text, 2);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "\xe0\x80\x80";
|
||
auto res = utility::utf8::is_valid(bad_text, 3);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "\xf0\x80\x80\x80";
|
||
auto res = utility::utf8::is_valid(bad_text, 4);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "\xf8\x80\x80\x80\x80";
|
||
auto res = utility::utf8::is_valid(bad_text, 5);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "\xfc\x80\x80\x80\x80\x80";
|
||
auto res = utility::utf8::is_valid(bad_text, 6);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
}
|
||
|
||
TEST(TextUtilities, IsValid_UTF8_SingleUTF16Surrogates) {
|
||
{
|
||
const char* bad_text = "\xed\xa0\x80";
|
||
auto res = utility::utf8::is_valid(bad_text, 3);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "\xed\xad\xbf";
|
||
auto res = utility::utf8::is_valid(bad_text, 3);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "\xed\xae\x80";
|
||
auto res = utility::utf8::is_valid(bad_text, 3);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "\xed\xaf\xbf";
|
||
auto res = utility::utf8::is_valid(bad_text, 3);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "\xed\xb0\x80";
|
||
auto res = utility::utf8::is_valid(bad_text, 3);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "\xed\xbe\x80";
|
||
auto res = utility::utf8::is_valid(bad_text, 3);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "\xed\xbf\xbf";
|
||
auto res = utility::utf8::is_valid(bad_text, 3);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
}
|
||
|
||
TEST(TextUtilities, IsValid_UTF8_PairedUTF16Surrogates) {
|
||
{
|
||
const char* bad_text = "\xed\xa0\x80\xed\xb0\x80";
|
||
auto res = utility::utf8::is_valid(bad_text, 6);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "\xed\xa0\x80\xed\xbf\xbf";
|
||
auto res = utility::utf8::is_valid(bad_text, 6);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "\xed\xad\xbf\xed\xb0\x80";
|
||
auto res = utility::utf8::is_valid(bad_text, 6);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "\xed\xad\xbf\xed\xbf\xbf";
|
||
auto res = utility::utf8::is_valid(bad_text, 6);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "\xed\xae\x80\xed\xb0\x80";
|
||
auto res = utility::utf8::is_valid(bad_text, 6);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "\xed\xae\x80\xed\xbf\xbf";
|
||
auto res = utility::utf8::is_valid(bad_text, 6);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "\xed\xaf\xbf\xed\xb0\x80";
|
||
auto res = utility::utf8::is_valid(bad_text, 6);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "\xed\xaf\xbf\xed\xbf\xbf";
|
||
auto res = utility::utf8::is_valid(bad_text, 6);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
}
|
||
|
||
TEST(TextUtilities, IsValid_UTF8_MaxOverlongSequences) {
|
||
{
|
||
const char* bad_text = "\xc1\xbf";
|
||
auto res = utility::utf8::is_valid(bad_text, 2);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "\xe0\x9f\xbf";
|
||
auto res = utility::utf8::is_valid(bad_text, 3);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "\xf0\x8f\xbf\xbf";
|
||
auto res = utility::utf8::is_valid(bad_text, 4);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "\xf8\x87\xbf\xbf\xbf";
|
||
auto res = utility::utf8::is_valid(bad_text, 5);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "\xfc\x83\xbf\xbf\xbf\xbf";
|
||
auto res = utility::utf8::is_valid(bad_text, 6);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
}
|
||
|
||
TEST(TextUtilities, IsValid_UTF8_ImpossibleBytes) {
|
||
{
|
||
const char* bad_text = "\xfe\xfe\xff\xff";
|
||
auto res = utility::utf8::is_valid(bad_text, 4);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "\xfe";
|
||
auto res = utility::utf8::is_valid(bad_text, 1);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "\xff";
|
||
auto res = utility::utf8::is_valid(bad_text, 1);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "\xff\xfe";
|
||
auto res = utility::utf8::is_valid(bad_text, 2);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
|
||
{
|
||
const char* bad_text = "<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>";
|
||
auto res = utility::utf8::is_valid(bad_text, 5);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
}
|
||
|
||
TEST(TextUtilities, IsValid_UTF8_Noncharacters) {
|
||
{
|
||
const char* bad_text = "";
|
||
auto res = utility::utf8::is_valid(bad_text, 32);
|
||
EXPECT_EQ(res, false);
|
||
}
|
||
}
|
||
|
||
TEST(TextUtilities, ConsecutiveCheck_ASCII_Case_Insensitive) {
|
||
std::string text = "";
|
||
auto res = utility::max_consecutive(text);
|
||
EXPECT_EQ(res, 0);
|
||
|
||
text = "f";
|
||
res = utility::max_consecutive(text);
|
||
EXPECT_EQ(res, 1);
|
||
|
||
text = "ff";
|
||
res = utility::max_consecutive(text);
|
||
EXPECT_EQ(res, 2);
|
||
|
||
text = "fff";
|
||
res = utility::max_consecutive(text);
|
||
EXPECT_EQ(res, 3);
|
||
|
||
text = "foo";
|
||
res = utility::max_consecutive(text);
|
||
EXPECT_EQ(res, 2);
|
||
|
||
text = "foofooof";
|
||
res = utility::max_consecutive(text);
|
||
EXPECT_EQ(res, 3);
|
||
|
||
text = "FOOBAAAR";
|
||
res = utility::max_consecutive(text);
|
||
EXPECT_EQ(res, 3);
|
||
|
||
text = "FOOBAaAR";
|
||
res = utility::max_consecutive(text);
|
||
EXPECT_EQ(res, 2);
|
||
}
|
||
|
||
TEST(TextUtilities, ConsecutiveCheck_ASCII_CaseSensitive) {
|
||
std::string text = "";
|
||
auto res = utility::max_consecutive(text, true);
|
||
EXPECT_EQ(res, 0);
|
||
|
||
text = "Fff";
|
||
res = utility::max_consecutive(text, true);
|
||
EXPECT_EQ(res, 3);
|
||
|
||
text = "FffOooo";
|
||
res = utility::max_consecutive(text, true);
|
||
EXPECT_EQ(res, 4);
|
||
}
|
||
|
||
TEST(TextUtilities, DISABLED_ConsecutiveCheck_UTF8_CaseInsensitive) {
|
||
std::string text = "";
|
||
auto res = utility::utf8::max_consecutive(text, true);
|
||
EXPECT_EQ(res, 0);
|
||
|
||
text = "Fff";
|
||
res = utility::utf8::max_consecutive(text, true);
|
||
EXPECT_EQ(res, 3);
|
||
|
||
text = "FffOooo";
|
||
res = utility::utf8::max_consecutive(text, true);
|
||
EXPECT_EQ(res, 4);
|
||
}
|
||
|
||
TEST(TextUtilities, ConsecutiveCheck_UTF8_CaseSensitive) {
|
||
std::string text = "";
|
||
auto res = utility::utf8::max_consecutive(text);
|
||
EXPECT_EQ(res, 0);
|
||
|
||
text = "テキスト";
|
||
res = utility::utf8::max_consecutive(text);
|
||
EXPECT_EQ(res, 1);
|
||
|
||
text = "テテキスト";
|
||
res = utility::utf8::max_consecutive(text);
|
||
EXPECT_EQ(res, 2);
|
||
|
||
text = "テテキススス";
|
||
res = utility::utf8::max_consecutive(text);
|
||
EXPECT_EQ(res, 3);
|
||
|
||
text = "テテキスススト";
|
||
res = utility::utf8::max_consecutive(text);
|
||
EXPECT_EQ(res, 3);
|
||
} |