Superminaren/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-27 21:46:57 +02:00
Code Issues Packages Projects Releases Wiki Activity

common/peg : handle invalid utf-8 sequences in the AST (#29161)

Browse Source 
* common/peg : handle invalid utf-8 sequences in the AST

* cont : return maximal subpart per Unicode recommendations

* cont : remove strict argument
This commit is contained in:
Aldehir Rojas
2026-09-20 06:51:40 -05:00
committed by GitHub
parent 3cf03257f2
commit 3d82ef62d4
7 changed files with 173 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/peg-parser.cpp
+50 -24
View File
@@ -166,6 +166,25 @@ common_peg_ast_id common_peg_ast_arena::find_by_rule(const common_peg_ast_node &
return COMMON_PEG_INVALID_AST_ID;
}
std::string common_peg_ast_node::sanitized_text() const {
if (invalid_utf8.empty()) {
return std::string(text);
}
std::string out;
out.reserve(text.size() + 2 * invalid_utf8.size());
size_t seg_start = start;
for (const auto & invalid : invalid_utf8) {
out.append(text.data() + (seg_start - start), invalid.pos - seg_start);
out.append("\xEF\xBF\xBD");
seg_start = invalid.pos + invalid.len;
}
out.append(text.data() + (seg_start - start), end - seg_start);
return out;
}
void common_peg_ast_arena::visit(common_peg_ast_id id, const common_peg_ast_visitor & visitor) const {
if (id == COMMON_PEG_INVALID_AST_ID) {
return;
@@ -282,6 +301,7 @@ struct parser_executor {
auto pos = start_pos;
std::vector<common_peg_ast_id> nodes;
std::vector<common_peg_invalid_utf8> invalid_utf8;
for (size_t i = 0; i < p.children.size(); i++) {
const auto & child_id = p.children[i];
@@ -306,13 +326,14 @@ struct parser_executor {
if (!result.nodes.empty()) {
nodes.insert(nodes.end(), result.nodes.begin(), result.nodes.end());
}
invalid_utf8.insert(invalid_utf8.end(), result.invalid_utf8.begin(), result.invalid_utf8.end());
if (result.need_more_input()) {
ctx.parse_depth--;
if (ctx.is_debug()) {
fprintf(stderr, "%sSEQ -> NEED_MORE\n", debug_indent().c_str());
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, result.end, std::move(nodes));
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, result.end, std::move(nodes), std::move(invalid_utf8));
}
pos = result.end;
@@ -322,7 +343,7 @@ struct parser_executor {
if (ctx.is_debug()) {
fprintf(stderr, "%sSEQ -> SUCCESS at %zu->%zu\n", debug_indent().c_str(), start_pos, pos);
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, std::move(nodes));
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, std::move(nodes), std::move(invalid_utf8));
}
common_peg_parse_result operator()(const common_peg_choice_parser & p) {
@@ -370,6 +391,7 @@ struct parser_executor {
auto pos = start_pos;
int match_count = 0;
std::vector<common_peg_ast_id> nodes;
std::vector<common_peg_invalid_utf8> invalid_utf8;
// Try to match up to max_count times (or unlimited if max_count is -1)
while (p.max_count == -1 || match_count < p.max_count) {
@@ -400,6 +422,7 @@ struct parser_executor {
if (!result.nodes.empty()) {
nodes.insert(nodes.end(), result.nodes.begin(), result.nodes.end());
}
invalid_utf8.insert(invalid_utf8.end(), result.invalid_utf8.begin(), result.invalid_utf8.end());
pos = result.end;
match_count++;
@@ -410,13 +433,14 @@ struct parser_executor {
if (!result.nodes.empty()) {
nodes.insert(nodes.end(), result.nodes.begin(), result.nodes.end());
}
invalid_utf8.insert(invalid_utf8.end(), result.invalid_utf8.begin(), result.invalid_utf8.end());
ctx.parse_depth--;
if (ctx.is_debug()) {
fprintf(stderr, "%sREPEAT -> NEED_MORE (count=%d, nodes=%zu)\n", debug_indent().c_str(),
match_count, nodes.size());
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, result.end, std::move(nodes));
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, result.end, std::move(nodes), std::move(invalid_utf8));
}
// Child failed - stop trying
@@ -434,7 +458,7 @@ struct parser_executor {
fprintf(stderr, "%sREPEAT -> NEED_MORE (not enough matches: %d < %d)\n", debug_indent().c_str(),
match_count, p.min_count);
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos, std::move(nodes));
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos, std::move(nodes), std::move(invalid_utf8));
}
if (ctx.is_debug()) {
fprintf(stderr, "%sREPEAT -> FAIL (not enough matches: %d < %d)\n", debug_indent().c_str(), match_count,
@@ -448,7 +472,7 @@ struct parser_executor {
fprintf(stderr, "%sREPEAT -> SUCCESS (count=%d, nodes=%zu)\n", debug_indent().c_str(), match_count,
nodes.size());
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, std::move(nodes));
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, std::move(nodes), std::move(invalid_utf8));
}
common_peg_parse_result operator()(const common_peg_and_parser & p) {
@@ -664,23 +688,23 @@ struct parser_executor {
// Scan input and check for delimiters
size_t pos = start_pos;
size_t last_valid_pos = start_pos;
std::vector<common_peg_invalid_utf8> invalid_utf8;
while (pos < ctx.input.size()) {
auto utf8_result = common_parse_utf8_codepoint(ctx.input, pos);
if (utf8_result.status == utf8_parse_result::INCOMPLETE) {
// Incomplete UTF-8 sequence
if (!ctx.is_lenient()) {
// Input is complete but UTF-8 is incomplete = malformed
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos);
}
// Return what we have so far (before incomplete sequence)
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, last_valid_pos);
if (utf8_result.status == utf8_parse_result::INCOMPLETE && ctx.is_lenient()) {
// The rest of the sequence may still arrive, return what we have so far
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, last_valid_pos, {}, std::move(invalid_utf8));
}
if (utf8_result.status == utf8_parse_result::INVALID) {
// Malformed UTF-8
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos);
if (utf8_result.status != utf8_parse_result::SUCCESS) {
// Malformed UTF-8, or a sequence truncated by the end of a complete input.
// A delimiter cannot start inside bytes that fail to decode, so consume them and move on
invalid_utf8.push_back({pos, utf8_result.bytes_consumed});
pos += utf8_result.bytes_consumed;
last_valid_pos = pos;
continue;
}
// Check if a delimiter starts at this position
@@ -688,12 +712,12 @@ struct parser_executor {
if (match == common_trie::COMPLETE_MATCH) {
// Found a complete delimiter, return everything before it
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos);
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, {}, std::move(invalid_utf8));
}
if (match == common_trie::PARTIAL_MATCH) {
// Found a partial match extending to end of input, return everything before it
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos);
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, {}, std::move(invalid_utf8));
}
pos += utf8_result.bytes_consumed;
@@ -702,9 +726,9 @@ struct parser_executor {
if (last_valid_pos == ctx.input.size() && ctx.is_lenient()) {
// Reached the end of a partial stream, there might still be more input that we need to consume.
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, last_valid_pos);
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, last_valid_pos, {}, std::move(invalid_utf8));
}
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, last_valid_pos);
return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, last_valid_pos, {}, std::move(invalid_utf8));
}
common_peg_parse_result operator()(const common_peg_schema_parser & p) {
@@ -728,10 +752,11 @@ struct parser_executor {
result.end,
text,
std::move(result.nodes),
result.need_more_input()
result.need_more_input(),
result.invalid_utf8
);
return common_peg_parse_result(result.type, result.start, result.end, { node_id });
return common_peg_parse_result(result.type, result.start, result.end, { node_id }, std::move(result.invalid_utf8));
}
return result;
@@ -757,10 +782,11 @@ struct parser_executor {
result.end,
text,
std::move(result.nodes),
result.need_more_input()
result.need_more_input(),
result.invalid_utf8
);
return common_peg_parse_result(result.type, result.start, result.end, { node_id });
return common_peg_parse_result(result.type, result.start, result.end, { node_id }, std::move(result.invalid_utf8));
}
return result;