rename: common_chat_syntax (now contains format)

ggml-org · ochafik · May 25, 2025 · Mar 12, 2025 · Mar 12, 2025 · Mar 12, 2025
commit c46d4da4c2b7f3bfb9e3d555930d0ab2febf8a2e
diff --git a/common/chat-parser.cpp b/common/chat-parser.cpp
@@ -10,8 +10,8 @@
 
 using json = nlohmann::ordered_json;
 
-common_chat_msg_parser::common_chat_msg_parser(const std::string & input, bool is_partial, const common_chat_reasoning_syntax & reasoning_syntax)
-    : input_(input), is_partial_(is_partial), reasoning_syntax_(reasoning_syntax)
+common_chat_msg_parser::common_chat_msg_parser(const std::string & input, bool is_partial, const common_chat_syntax & syntax)
+    : input_(input), is_partial_(is_partial), syntax_(syntax)
 {
     result_.role = "assistant";
 
@@ -127,14 +127,14 @@ void common_chat_msg_parser::consume_literal(const std::string & literal) {
 }
 
 void common_chat_msg_parser::try_consume_think_tags(const common_regex & start_think_regex, const common_regex & end_think_regex) {
-    if (reasoning_syntax_.format != COMMON_REASONING_FORMAT_NONE) {
-        if (reasoning_syntax_.thinking_forced_open || try_consume_regex(start_think_regex)) {
+    if (syntax_.reasoning_format != COMMON_REASONING_FORMAT_NONE) {
+        if (syntax_.thinking_forced_open || try_consume_regex(start_think_regex)) {
             if (auto res = try_find_regex(end_think_regex)) {
                 result_.reasoning_content = res->prelude;
                 consume_spaces();
             } else {
                 result_.reasoning_content = consume_rest();
-                if (!reasoning_syntax_.thinking_forced_open) {
+                if (!syntax_.thinking_forced_open) {
                     incomplete("Failed to find end of reasoning tag " + end_think_regex.str());
                 }
                 return;
@@ -218,7 +218,7 @@ std::optional<common_json> common_chat_msg_parser::try_consume_json(
         // No healing marker, just return the parsed json
         return result;
     }
-    if (!is_partial_) {
+    if (!is_partial()) {
         incomplete("JSON is incomplete");
         return std::nullopt; // Actually unreachable
     }

diff --git a/common/chat-parser.h b/common/chat-parser.h
@@ -16,14 +16,14 @@ class common_chat_msg_partial_exception : public std::runtime_error {
 class common_chat_msg_parser {
     std::string input_;
     bool is_partial_;
-    common_chat_reasoning_syntax reasoning_syntax_;
+    common_chat_syntax syntax_;
 
     size_t pos_ = 0;
     common_chat_msg result_;
     std::string healing_marker_;
 
   public:
-    common_chat_msg_parser(const std::string & input, bool is_partial, const common_chat_reasoning_syntax & reasoning_syntax);
+    common_chat_msg_parser(const std::string & input, bool is_partial, const common_chat_syntax & syntax);
     const std::string & input() const { return input_; }
     size_t pos() const { return pos_; }
     const std::string & healing_marker() const { return healing_marker_; }

diff --git a/common/chat.cpp b/common/chat.cpp
@@ -578,17 +578,22 @@ static void parse_json_tool_calls(
                     // get_function_name signalled us that we should skip this match and treat it as content.
                     from = res->groups[0].begin + 1;
                     continue;
-                } else {
-                    from = std::string::npos;
                 }
+                from = std::string::npos;
+
                 builder.add_content(res->prelude);
-                if (auto partial = builder.try_consume_json({{}})) {
-                    std::string arguments = partial->json.dump();
-                    if (!builder.add_tool_call(name, "", arguments, partial->healing_marker)) {
-                        builder.incomplete("incomplete tool call");
+                auto maybe_raw_python = name == "python" && allow_raw_python;
+                if (builder.input()[builder.pos()] == '{' || !maybe_raw_python) {
+                    if (auto partial = builder.try_consume_json({{}})) {
+                        std::string arguments = partial->json.dump();
+                        if (!builder.add_tool_call(name, "", arguments, partial->healing_marker)) {
+                            builder.incomplete("incomplete tool call");
+                        }
+                        builder.consume_regex(close_regex);
                     }
-                    builder.consume_regex(close_regex);
-                } else if (name == "python" && allow_raw_python) {
+                    continue;
+                } 
+                if (maybe_raw_python) {
                     auto code = builder.consume_rest();
                     std::string arguments;
                     common_healing_marker healing_marker;
@@ -602,13 +607,11 @@ static void parse_json_tool_calls(
                         builder.incomplete("incomplete tool call");
                     }
                     return;
-                } else {
-                    builder.incomplete("incomplete tool call");
-                    return;
                 }
-            } else {
-                break;
+                builder.incomplete("incomplete tool call");
+                return;
             }
+            break;
         }
         if (block_close) {
             builder.consume_regex(*block_close);
@@ -1238,14 +1241,18 @@ static common_chat_params common_chat_params_init_functionary_v3_2(const common_
                 std::string args_pattern = "[\\s\\S]*";
                 auto args_rule = builder.add_schema(name + "-args", parameters);
                 if (name == "python") {
-                    args_pattern = "\\{" + args_pattern;
                     args_rule = builder.add_rule(name + "-maybe-raw-args", args_rule + " | [^{] .*");
+                } else {
+                    args_pattern = "\\{" + args_pattern;
+                }
+                auto call_rule = builder.add_rule(name + "-call", "\"" + name + "\\n\" " + args_rule);
+                first_tool_rules.push_back(call_rule);
+                if (inputs.parallel_tool_calls) {
+                    subsequent_tool_rules.push_back(builder.add_rule(name + "-call2", "\">>>\" " + call_rule));
                 }
-                first_tool_rules.push_back(builder.add_rule(name + "-call", "( \"assistant<|end_header_id|>\\n\" )? \"" + name + "\\n\" " + args_rule));
-                subsequent_tool_rules.push_back(builder.add_rule(name + "-call2", "\">>>" + name + "\\n\" " + args_rule));
                 data.grammar_triggers.push_back({
                     COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL,
-                    "((?:[\\s\\S]*?>>>)?" + regex_escape(name) + "\n)" + args_pattern,
+                    "((?:[\\s\\S]+?>>>)?" + regex_escape(name) + "\n)" + args_pattern,
                 });
             });
             data.preserved_tokens = {
@@ -1771,20 +1778,20 @@ static void common_chat_parse(common_chat_msg_parser & builder, common_chat_form
     builder.finish();
 }
 
-common_chat_msg common_chat_parse(const std::string & input, common_chat_format format, bool is_partial, const common_chat_reasoning_syntax & reasoning_syntax) {
-    common_chat_msg_parser builder(input, is_partial, reasoning_syntax);
+common_chat_msg common_chat_parse(const std::string & input, bool is_partial, const common_chat_syntax & syntax) {
+    common_chat_msg_parser builder(input, is_partial, syntax);
     try {
-        common_chat_parse(builder, format);
+        common_chat_parse(builder, syntax.format);
     } catch (const common_chat_msg_partial_exception & ex) {
         LOG_DBG("Partial parse: %s\n", ex.what());
         if (!is_partial) {
             throw std::runtime_error(ex.what());
         }
     }
     auto msg = builder.result();
-    switch (reasoning_syntax.format) {
+    switch (syntax.reasoning_format) {
         case COMMON_REASONING_FORMAT_DEEPSEEK:
-            if (!msg.reasoning_content.empty() && reasoning_syntax.inlined_in_content) {
+            if (!msg.reasoning_content.empty() && syntax.reasoning_in_content) {
                 std::string content = "<think>" + msg.reasoning_content;
                 if (!is_partial || !msg.content.empty()) {
                     content += "</think>";

diff --git a/common/chat.h b/common/chat.h
@@ -123,10 +123,12 @@ struct common_chat_params {
     std::vector<std::string>            additional_stops;
 };
 
-struct common_chat_reasoning_syntax {
-    common_reasoning_format format = COMMON_REASONING_FORMAT_NONE;
-    bool inlined_in_content        = false;
-    bool thinking_forced_open      = false;
+struct common_chat_syntax {
+    common_chat_format       format                = COMMON_CHAT_FORMAT_CONTENT_ONLY;
+    common_reasoning_format  reasoning_format      = COMMON_REASONING_FORMAT_NONE;
+    // Whether reasoning_content should be inlined in the content (e.g. for reasoning_format=deepseek in stream mode)
+    bool                     reasoning_in_content  = false;
+    bool                     thinking_forced_open  = false;
 };
 
 // Check if the template supplied via "--chat-template" is supported or not. Returns true if it's valid
@@ -166,7 +168,7 @@ std::string common_chat_format_example(
     bool use_jinja);
 
 std::string               common_chat_format_name(common_chat_format format);
-common_chat_msg           common_chat_parse(const std::string & input, common_chat_format format, bool is_partial = false, const common_chat_reasoning_syntax & reasoning_syntax = {});
+common_chat_msg           common_chat_parse(const std::string & input, bool is_partial, const common_chat_syntax & syntax);
 
 common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::string & tool_choice);
 

diff --git a/examples/server/server.cpp b/examples/server/server.cpp
@@ -1,3 +1,4 @@
+#include "chat.h"
 #include "utils.hpp"
 
 #include "arg.h"
@@ -117,8 +118,7 @@ struct slot_params {
     oaicompat_type               oaicompat                 = OAICOMPAT_TYPE_NONE;
     std::string                  oaicompat_model;
     std::string                  oaicompat_cmpl_id;
-    common_chat_format           oaicompat_chat_format      = COMMON_CHAT_FORMAT_CONTENT_ONLY;
-    common_chat_reasoning_syntax oaicompat_reasoning_syntax;
+    common_chat_syntax           oaicompat_chat_syntax;
 
     json to_json() const {
         std::vector<std::string> samplers;
@@ -174,7 +174,10 @@ struct slot_params {
             {"grammar_lazy",              sampling.grammar_lazy},
             {"grammar_triggers",          grammar_triggers},
             {"preserved_tokens",          sampling.preserved_tokens},
-            {"chat_format",               common_chat_format_name(oaicompat_chat_format)},
+            {"chat_format",               common_chat_format_name(oaicompat_chat_syntax.format)},
+            {"reasoning_format",          (oaicompat_chat_syntax.reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK ? "deepseek" : "none")},
+            {"reasoning_in_content",      oaicompat_chat_syntax.reasoning_in_content},
+            {"thinking_forced_open",      oaicompat_chat_syntax.thinking_forced_open},
             {"samplers",                  samplers},
             {"speculative.n_max",         speculative.n_max},
             {"speculative.n_min",         speculative.n_min},
@@ -349,14 +352,14 @@ struct server_task {
         {
             auto it = data.find("chat_format");
             if (it != data.end()) {
-                params.oaicompat_chat_format = static_cast<common_chat_format>(it->get<int>());
-                SRV_INF("Chat format: %s\n", common_chat_format_name(params.oaicompat_chat_format).c_str());
+                params.oaicompat_chat_syntax.format = static_cast<common_chat_format>(it->get<int>());
+                SRV_INF("Chat format: %s\n", common_chat_format_name(params.oaicompat_chat_syntax.format).c_str());
             } else {
-                params.oaicompat_chat_format = defaults.oaicompat_chat_format;
+                params.oaicompat_chat_syntax.format = defaults.oaicompat_chat_syntax.format;
             }
-            params.oaicompat_reasoning_syntax.format = params_base.reasoning_format;
-            params.oaicompat_reasoning_syntax.inlined_in_content = params.stream;
-            params.oaicompat_reasoning_syntax.thinking_forced_open = json_value(data, "thinking_forced_open", false);
+            params.oaicompat_chat_syntax.reasoning_format = params_base.reasoning_format;
+            params.oaicompat_chat_syntax.reasoning_in_content = params.stream;
+            params.oaicompat_chat_syntax.thinking_forced_open = json_value(data, "thinking_forced_open", false);
         }
 
         {
@@ -632,7 +635,7 @@ struct server_task_result_cmpl_final : server_task_result {
    
CCAC
 oaicompat_type     oaicompat                = OAICOMPAT_TYPE_NONE;
     std::string        oaicompat_model;
     std::string        oaicompat_cmpl_id;
-    common_chat_format oaicompat_chat_format    = COMMON_CHAT_FORMAT_CONTENT_ONLY;
+    common_chat_syntax oaicompat_chat_syntax;
     common_chat_msg    oaicompat_msg;
 
     virtual int get_index() override {
@@ -2335,9 +2338,8 @@ struct server_context {
         SRV_DBG("Parsing chat message: %s\n", slot.generated_text.c_str());
         auto new_msg = common_chat_parse(
             slot.generated_text,
-            slot.params.oaicompat_chat_format,
             /* is_partial= */ true,
-            slot.params.oaicompat_reasoning_syntax);
+            slot.params.oaicompat_chat_syntax);
         if (!new_msg.empty()) {
             slot.generated_msg = new_msg;
         }
@@ -2347,7 +2349,6 @@ struct server_context {
         // res->previous_content = slot.generated_text.substr(0, slot.generated_text.size() - tkn.text_to_send.size());
         // res->oaicompat_chat_format = slot.params.oaicompat_chat_format;
 
-
         // populate res.probs_output
         if (slot.params.sampling.n_probs > 0) {
             res->prob_output = tkn; // copy the token probs
@@ -2391,10 +2392,9 @@ struct server_context {
         SRV_DBG("Parsing chat message: %s\n", res->content.c_str());
         res->oaicompat_msg         = slot.generated_msg = common_chat_parse(
             res->content,
-            slot.params.oaicompat_chat_format,
             /* is_partial= */ slot.stop == STOP_TYPE_LIMIT,
-            slot.params.oaicompat_reasoning_syntax);
-        res->oaicompat_chat_format = slot.params.oaicompat_chat_format;
+            slot.params.oaicompat_chat_syntax);
+        res->oaicompat_chat_syntax = slot.params.oaicompat_chat_syntax;
 
         // populate res.probs_output
         if (slot.params.sampling.n_probs > 0) {