Superminaren/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-27 21:46:57 +02:00
Code Issues Packages Projects Releases Wiki Activity

common/peg : handle invalid utf-8 sequences in the AST (#29161)

Browse Source 
* common/peg : handle invalid utf-8 sequences in the AST

* cont : return maximal subpart per Unicode recommendations

* cont : remove strict argument
This commit is contained in:
Aldehir Rojas
2026-09-20 06:51:40 -05:00
committed by GitHub
parent 3cf03257f2
commit 3d82ef62d4
7 changed files with 173 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/peg-parser/test-unicode.cpp
+41 -7
View File
@@ -273,19 +273,35 @@ void test_unicode(testing &t) {
});
t.test("malformed UTF-8", [](testing &t) {
std::vector<test_case> test_cases {
struct passthrough_case {
std::string input;
std::string expected_text;
std::string expected_sanitized;
};
std::vector<passthrough_case> test_cases {
// Invalid UTF-8 bytes
{std::string("Hello\xFF\xFE"), "", COMMON_PEG_PARSE_RESULT_FAIL},
{std::string("Hello\xFF\xFE</tag>"), std::string("Hello\xFF\xFE"), "Hello\xEF\xBF\xBD\xEF\xBF\xBD"},
// Continuation byte without lead byte
{std::string("Hello\x80World"), "", COMMON_PEG_PARSE_RESULT_FAIL},
{std::string("Hello\x80World</tag>"), std::string("Hello\x80World"), "Hello\xEF\xBF\xBDWorld"},
// Invalid continuation byte
{std::string("\xC3\x28"), "", COMMON_PEG_PARSE_RESULT_FAIL},
// Invalid continuation byte, the lead byte is dropped and '(' survives
{std::string("\xC3\x28</tag>"), std::string("\xC3\x28"), "\xEF\xBF\xBD("},
// Two good bytes of a 3-byte sequence then a bad third byte, the prefix is replaced once and the third byte is kept
{std::string("\xE4\xB8" "A</tag>"), std::string("\xE4\xB8" "A"), "\xEF\xBF\xBD" "A"},
{std::string("\xE4\xB8</tag>"), std::string("\xE4\xB8"), "\xEF\xBF\xBD"},
// Truncated sequence in a complete input, the leftover prefix is replaced once
{std::string("Hello\xE4\xB8"), std::string("Hello\xE4\xB8"), "Hello\xEF\xBF\xBD"},
// Valid multi-byte content around the bad byte is left alone
{std::string("\xE4\xBD\xA0\xFF\xE5\xA5\xBD</tag>"), std::string("\xE4\xBD\xA0\xFF\xE5\xA5\xBD"), "\xE4\xBD\xA0\xEF\xBF\xBD\xE5\xA5\xBD"},
};
auto parser = build_peg_parser([](common_peg_parser_builder& p) {
return p.until("</tag>");
return p.tag("body", p.until("</tag>")) + p.optional(p.literal("</tag>"));
});
for (size_t i = 0; i < test_cases.size(); i++) {
@@ -296,10 +312,28 @@ void test_unicode(testing &t) {
common_peg_parse_context ctx(tc.input);
auto result = parser.parse(ctx);
assert_result_equal(t, tc.expected_result, result.type);
assert_result_equal(t, COMMON_PEG_PARSE_RESULT_SUCCESS, result.type);
const auto & node = ctx.ast.get(result.nodes[0]);
t.assert_equal("raw text", tc.expected_text, std::string(node.text));
t.assert_equal("sanitized text", tc.expected_sanitized, node.sanitized_text());
});
}
});
t.test("malformed UTF-8 rescanned by backtracking", [](testing &t) {
// The failed alternative and the lookahead scan the same bad byte, it must only be recorded once
auto parser = build_peg_parser([](common_peg_parser_builder& p) {
return (p.until("<a>") + p.literal("<a>")) | (p.peek(p.until("<b>")) + p.until("<b>") + p.literal("<b>"));
});
std::string input("x\xFFy<b>");
common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
assert_result_equal(t, COMMON_PEG_PARSE_RESULT_SUCCESS, result.type);
t.assert_equal("invalid count", 1u, result.invalid_utf8.size());
t.assert_equal("invalid offset", 1u, result.invalid_utf8[0].pos);
});
});
t.test("json_string parser", [](testing &t) {