mmo-ref/mudlet
1
0
Fork 0
mirror of https://github.com/Mudlet/Mudlet synced 2026-08-13 18:26:27 -04:00
Code Issues Projects Releases Packages Wiki Activity
mudlet/test/TMxpEntityTagHandlerTest.cpp
Vadim Peretokin 4d36d19274
fix: MXP text now shows correctly on non-English games (#9491) 
#### Brief overview of PR changes/additions

MXP element attribute values - such as `SEND` link targets/hints and
`<!ENTITY>` values - were always decoded as UTF-8, ignoring the
session's negotiated encoding. On non-UTF-8 games (Latin1, GBK, Big5,
WINDOWS-1251, ...) any non-ASCII bytes in an attribute were mis-decoded,
so MXP links and entities carried garbled text.

This threads the active session encoding into `TMxpNodeBuilder` (set by
`TMxpProcessor` before parsing each character) and decodes attribute
names/values through a new shared `TStringUtils::decodeBytes` helper.
That helper also replaces the duplicated decode logic previously inline
in `TMxpProcessor::decodeRawBytes`, so the attribute path and the raw
tag/content path now interpret bytes identically. An empty
(not-yet-negotiated) encoding continues to be treated as UTF-8,
preserving prior behaviour for default sessions.

#### Motivation for adding to Mudlet

Players on non-English MUDs saw corrupted text in MXP links and custom
entities whenever those carried accented or non-Latin characters - the
attribute parser was the only MXP path still hardcoding UTF-8 while
element content already honoured the session encoding.

#### Other info (issues closed, discussion etc)

Added regression tests:
- `TMxpEntityTagHandlerTest`: quoted and unquoted `<!ENTITY>` values in
a WINDOWS-1251 session decode to "Гроза"; a UTF-8 session still resolves
the same value unchanged.
- `TMxpSendTagHandlerTest`: a `SEND href` with Latin1 bytes decodes to
"señor"; an empty (default) session encoding still decodes attributes as
UTF-8.

The WINDOWS-1251 entity test was verified to fail on the baseline
(producing U+FFFD replacement characters) before the fix. All 14
MXP/encoding/entity unit tests pass.

Assisted-by: Claude:claude-opus-4-8
2026-07-27 19:55:26 +02:00

256 lines
9.7 KiB
C++
Raw Blame History

/***************************************************************************
* Copyright (C) 2022 by Gustavo Sousa - gustavocms@gmail.com *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
* along with this program; if not, write to the *
* Free Software Foundation, Inc., *
* 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. *
***************************************************************************/
#include "TMxpEntityTagHandler.h"
#include "TMxpStubClient.h"
#include <QTest>
#include <TMxpProcessor.h>
#include <TMxpTagParser.h>
#include <TMxpTagProcessor.h>
class TMxpEntityTagHandlerTest : public QObject {
Q_OBJECT
private:
static QSharedPointer<MxpNode> parseNode(const QString &tagText) {
auto nodes = TMxpTagParser::parseToMxpNodeList(tagText);
return !nodes.empty() ? nodes.first() : nullptr;
}
void processInput(TMxpProcessor &processor, const std::string& input) {
for (char ch : input) {
processor.processMxpInput(ch, true);
}
}
private slots:
void testPublish() {
TMxpStubClient stub;
TMxpTagProcessor processor;
auto tag = parseNode("<!EN ob \"street lamp\" publish>");
processor.handleNode(processor, stub, tag.get());
QCOMPARE(processor.getEntityResolver().getResolution("&ob;"),
"street lamp");
QCOMPARE(stub.mPublishedEntityName, "&ob;");
QCOMPARE(stub.mPublishedEntityValue, "street lamp");
}
void testPrivate() {
TMxpStubClient stub;
TMxpTagProcessor processor;
auto tag = parseNode("<!EN ob \"street lamp\" private>");
processor.handleNode(processor, stub, tag.get());
QCOMPARE(processor.getEntityResolver().getResolution("&ob;"),
"street lamp");
QCOMPARE(stub.mPublishedEntityName, "");
QCOMPARE(stub.mPublishedEntityValue, "");
}
void testAddRemoveStart() {
TMxpStubClient stub;
TMxpTagProcessor processor;
processor.getEntityResolver().registerEntity("&entity;", "v1");
QCOMPARE(processor.getEntityResolver().getResolution("&entity;"), "v1");
processor.handleNode(processor, stub,
parseNode("<!EN entity \"v2\" add>").get());
QCOMPARE(processor.getEntityResolver().getResolution("&entity;"), "v1|v2");
processor.handleNode(processor, stub,
parseNode("<!EN entity \"v1\" remove>").get());
QCOMPARE(processor.getEntityResolver().getResolution("&entity;"), "v2");
processor.handleNode(processor, stub,
parseNode("<!EN entity \"v2\" remove>").get());
QCOMPARE(processor.getEntityResolver().getResolution("&entity;"), "");
}
void testAddRemoveEnd() {
TMxpStubClient stub;
TMxpTagProcessor processor;
processor.getEntityResolver().registerEntity("&entity;", "v1");
QCOMPARE(processor.getEntityResolver().getResolution("&entity;"), "v1");
processor.handleNode(processor, stub,
parseNode("<!EN entity \"v2\" add>").get());
QCOMPARE(processor.getEntityResolver().getResolution("&entity;"), "v1|v2");
processor.handleNode(processor, stub,
parseNode("<!EN entity \"v2\" remove>").get());
QCOMPARE(processor.getEntityResolver().getResolution("&entity;"), "v1");
}
void testAddRemovePartOfItemValue() {
TMxpStubClient stub;
TMxpTagProcessor processor;
processor.getEntityResolver().registerEntity("&entity;", "my text example");
QCOMPARE(processor.getEntityResolver().getResolution("&entity;"),
"my text example");
processor.handleNode(processor, stub,
parseNode("<!EN entity \"my value\" add>").get());
QCOMPARE(processor.getEntityResolver().getResolution("&entity;"),
"my text example|my value");
processor.handleNode(processor, stub,
parseNode("<!EN entity \"my text\" remove>").get());
QCOMPARE(processor.getEntityResolver().getResolution("&entity;"),
"my text example|my value");
processor.handleNode(processor, stub,
parseNode("<!EN entity \"example\" remove>").get());
QCOMPARE(processor.getEntityResolver().getResolution("&entity;"),
"my text example|my value");
processor.handleNode(processor, stub,
parseNode("<!EN entity \"value\" remove>").get());
QCOMPARE(processor.getEntityResolver().getResolution("&entity;"),
"my text example|my value");
processor.handleNode(processor, stub,
parseNode("<!EN entity \"my\" remove>").get());
QCOMPARE(processor.getEntityResolver().getResolution("&entity;"),
"my text example|my value");
processor.handleNode(processor, stub,
parseNode("<!EN entity \"my v\" remove>").get());
QCOMPARE(processor.getEntityResolver().getResolution("&entity;"),
"my text example|my value");
}
void testInterpolation() {
TMxpStubClient stub;
TMxpProcessor processor(&stub);
processor.setMode(6); // !EN and SEND are SECURE tags
std::string input =
"<!EN ob \"street lamp\" private><send href=\"kill &ob;\">kill</send>";
processInput(processor, input);
QCOMPARE(stub.mHrefs.size(), 1);
QCOMPARE(stub.mHrefs[0], "send([[kill street lamp]])");
QCOMPARE(stub.mHints.size(), 1);
QCOMPARE(stub.mHints[0], "kill street lamp");
}
void testDelete() {
TMxpStubClient stub;
TMxpTagProcessor processor;
processor.getEntityResolver().registerEntity("&entity;", "v1");
QCOMPARE(processor.getEntityResolver().getResolution("&entity;"), "v1");
processor.handleNode(processor, stub,
parseNode("<!EN entity delete>").get());
QCOMPARE(processor.getEntityResolver().getResolution("&entity;"),
"&entity;");
processor.getEntityResolver().registerEntity("&myEntity;", "v2");
QCOMPARE(processor.getEntityResolver().getResolution("&myEntity;"), "v2");
processor.handleNode(processor, stub,
parseNode("<!EN myEntity delete>").get());
QCOMPARE(processor.getEntityResolver().getResolution("&myEntity;"),
"&myEntity;");
}
void testEntityModification() {
TMxpStubClient stub;
TMxpTagProcessor processor;
processor.getEntityResolver().registerEntity("&entity;", "v1");
QCOMPARE(processor.getEntityResolver().getResolution("&entity;"), "v1");
processor.handleNode(processor, stub, parseNode("<!EN entity ''>").get());
QCOMPARE(processor.getEntityResolver().getResolution("&entity;"), "");
processor.handleNode(processor, stub, parseNode("<!EN entity V2>").get());
QCOMPARE(processor.getEntityResolver().getResolution("&entity;"), "V2");
processor.handleNode(processor, stub, parseNode("<!EN entity>").get());
QCOMPARE(processor.getEntityResolver().getResolution("&entity;"), "");
processor.handleNode(processor, stub, parseNode("<!EN entity V3>").get());
QCOMPARE(processor.getEntityResolver().getResolution("&entity;"), "V3");
processor.handleNode(processor, stub, parseNode("<!EN entity \"\">").get());
QCOMPARE(processor.getEntityResolver().getResolution("&entity;"), "");
}
// <!ENTITY> values must be decoded with the session encoding rather than
// hardcoded UTF-8 - covers quoted and unquoted values in a WINDOWS-1251
// session ("Гроза" = bytes C3 F0 EE E7 E0)
void testNonUtf8SessionEntityValue() {
TMxpStubClient stub;
stub.mEncoding = QByteArrayLiteral("WINDOWS-1251");
TMxpProcessor processor(&stub);
processor.setMode(6);
processInput(processor, "<!ENTITY storm \"\xC3\xF0\xEE\xE7\xE0\">");
processInput(processor, "<!ENTITY storm2 \xC3\xF0\xEE\xE7\xE0>");
TEntityResolver &resolver =
processor.getMxpTagProcessor().getEntityResolver();
QCOMPARE(resolver.getResolution("&storm;"), QString("Гроза"));
QCOMPARE(resolver.getResolution("&storm2;"), QString("Гроза"));
}
// UTF-8 sessions must keep resolving non-Latin1 entity values unchanged
void testUtf8SessionEntityValue() {
TMxpStubClient stub;
TMxpProcessor processor(&stub);
processor.setMode(6);
processInput(processor, "<!ENTITY storm \"Гроза\">");
QCOMPARE(
processor.getMxpTagProcessor().getEntityResolver().getResolution(
"&storm;"),
QString("Гроза"));
}
void testEmptyEntityInterpolation() {
TMxpStubClient stub;
TMxpProcessor mxpProcessor(&stub);
mxpProcessor.setMode(6);
std::string input = "<!en entity ''><send href=\"examine ob&entity;\" "
"hint=\"examine&entity;\">examine</send>";
processInput(mxpProcessor, input);
QCOMPARE(stub.mHrefs.size(), 1);
QCOMPARE(stub.mHrefs[0], "send([[examine ob]])");
QCOMPARE(stub.mHints.size(), 1);
QCOMPARE(stub.mHints[0], "examine");
}
};
#include "TMxpEntityTagHandlerTest.moc"
QTEST_MAIN(TMxpEntityTagHandlerTest)
Reference in a new issue View git blame Copy permalink