common/peg : handle invalid utf-8 sequences in the AST (#29161)

* common/peg : handle invalid utf-8 sequences in the AST

* cont : return maximal subpart per Unicode recommendations

* cont : remove strict argument

tests/test-chat-peg-parser.cpp

@@ -23,6 +23,7 @@ static void test_command7_parser_compare(testing & t);
 static void test_prefix_tool_names(testing & t);
 static void test_tagged_peg_parser(testing & t);
 static void test_permute(testing & t);
+static void test_invalid_utf8(testing & t);
 
 int main(int argc, char * argv[]) {
     testing t(std::cout);
@@ -42,6 +43,7 @@ int main(int argc, char * argv[]) {
     t.test("prefix tool names", test_prefix_tool_names);
     t.test("tagged peg parser", test_tagged_peg_parser);
     t.test("permute", test_permute);
+    t.test("invalid utf8", test_invalid_utf8);
 
     return t.summary();
 }
@@ -1069,3 +1071,36 @@ static void test_permute(testing & t) {
         )""", gbnf_of(parser));
     });
 }
+
+static void test_invalid_utf8(testing & t) {
+    auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) {
+        return "<think>" + p.reasoning(p.until("</think>")) + "</think>" + p.content(p.rest()) + p.end();
+    });
+
+    t.test("replaced in reasoning and content", [&](testing & t) {
+        std::string input("<think>plan\xFF\xFE</think>caf\xC3\xA9 \x80 done");
+        common_peg_parse_context ctx(input);
+        auto result = parser.parse(ctx);
+        t.assert_true("success", result.success());
+
+        common_chat_msg msg;
+        auto mapper = common_chat_peg_mapper(msg);
+        mapper.from_ast(ctx.ast, result);
+
+        t.assert_equal("reasoning", "plan\xEF\xBF\xBD\xEF\xBF\xBD", msg.reasoning_content);
+        t.assert_equal("content", "caf\xC3\xA9 \xEF\xBF\xBD done", msg.content);
+    });
+
+    t.test("partial input keeps trailing incomplete sequence out", [&](testing & t) {
+        std::string input("<think>x</think>a\x80" "b\xE4\xB8");
+        common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
+        auto result = parser.parse(ctx);
+        t.assert_true("not fail", !result.fail());
+
+        common_chat_msg msg;
+        auto mapper = common_chat_peg_mapper(msg);
+        mapper.from_ast(ctx.ast, result);
+
+        t.assert_equal("content", "a\xEF\xBF\xBD" "b", msg.content);
+    });
+}