8000 `server`: fix format of streamed tool call deltas (diff name, fix id … · bandoti/llama.cpp@d74e94c · GitHub
[go: up one dir, main page]

Skip to content

Commit d74e94c

Browse files
authored
server: fix format of streamed tool call deltas (diff name, fix id location) (ggml-org#13800)
* fix deltas of tool_call.function.name * fix tool_call.id (was in tool_call.function.id!) + add function type * add tool_call.type * populate empty tool_call.function.arguments on first delta
1 parent f13847c commit d74e94c

File tree

3 files changed

+17
-17
lines changed

3 files changed

+17
-17
lines changed

common/chat.cpp

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -106,9 +106,9 @@ std::vector<common_chat_msg_diff> common_chat_msg_diff::compute_diffs(const comm
106106
if (!args_diff.empty() || pref.id != newf.id) {
107107
auto & diff = diffs.emplace_back();
108108
diff.tool_call_index = idx;
109-
diff.tool_call_delta.name = newf.name;
110109
if (pref.id != newf.id) {
111110
diff.tool_call_delta.id = newf.id;
111+
diff.tool_call_delta.name = newf.name;
112112
}
113113
diff.tool_call_delta.arguments = args_diff;
114114
}
@@ -392,22 +392,19 @@ template <> json common_chat_msg_diff_to_json_oaicompat(const common_chat_msg_di
392392
delta["content"] = diff.content_delta;
393393
}
394394
if (diff.tool_call_index != std::string::npos) {
395+
json tool_call;
396+
tool_call["index"] = diff.tool_call_index;
397+
if (!diff.tool_call_delta.id.empty()) {
398+
tool_call["id"] = diff.tool_call_delta.id;
399+
tool_call["type"] = "function";
400+
}
395401
json function = json::object();
396402
if (!diff.tool_call_delta.name.empty()) {
397403
function["name"] = diff.tool_call_delta.name;
398404
}
399-
if (!diff.tool_call_delta.id.empty()) {
400-
function["id"] = diff.tool_call_delta.id;
401-
}
402-
if (!diff.tool_call_delta.arguments.empty()) {
403-
function["arguments"] = diff.tool_call_delta.arguments;
404-
}
405-
delta["tool_calls"] = json::array({
406-
json {
407-
{"index", diff.tool_call_index},
408-
{"function", function}
409-
}
410-
});
405+
function["arguments"] = diff.tool_call_delta.arguments;
406+
tool_call["function"] = function;
407+
delta["tool_calls"] = json::array({tool_call});
411408
}
412409
return delta;
413410
}

tests/test-chat.cpp

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1356,8 +1356,7 @@ static void test_msg_diffs_compute() {
13561356

13571357
common_chat_msg_diff diff12;
13581358
diff12.tool_call_index = 0;
1359-
diff12.tool_call_delta.name = "special_function";
1360-
// Note: id doesnt change here.
1359+
// Note: neither id nor name change here.
13611360
diff12.tool_call_delta.arguments = "g1\": 1}";
13621361

13631362
assert_equals(

tools/server/tests/utils.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -328,6 +328,10 @@ def make_any_request(
328328
if 'function' not in tc:
329329
raise ValueError(f"Expected function type, got {tc['type']}")
330330
if tc['index'] >= len(tool_calls):
331+
assert 'id' in tc
332+
assert tc.get('type') == 'function'
333+
assert 'function' in tc and 'name' in tc['function'] and len(tc['function']['name']) > 0, \
334+
f"Expected function call with name, got {tc.get('function')}"
331335
tool_calls.append(dict(
332336
id="",
333337
type="function",
@@ -340,10 +344,10 @@ def make_any_request(
340344
if tc.get('id') is not None:
341345
tool_call['id'] = tc['id']
342346
fct = tc['function']
347+
assert 'id' not in fct, f"Function call should not have id: {fct}"
343348
if fct.get('name') is not None:
344-
tool_call['function']['name'] = fct['name']
349+
tool_call['function']['name'] = tool_call['function'].get('name', '') + fct['name']
345350
if fct.get('arguments') is not None:
346-
assert len(fct['arguments']) > 0, f'Expected non empty arguments delta!'
347351
tool_call['function']['arguments'] += fct['arguments']
348352

349353
print(f'Streamed response had {content_parts} content parts, {tool_call_parts} tool call parts incl. {arguments_parts} arguments parts')

0 commit comments

Comments
 (0)
0