8000 Adding logprobs to /v1/completions (#11344) · mglambda/llama.cpp@38348c6 · GitHub
[go: up one dir, main page]

Skip to content

Commit 38348c6

Browse files
jpodivinmglambda
authored andcommitted
Adding logprobs to /v1/completions (ggml-org#11344)
Signed-off-by: Jiri Podivin <jpodivin@redhat.com>
1 parent b1237b9 commit 38348c6

File tree

1 file changed

+5
-0
lines changed

1 file changed

+5
-0
lines changed

examples/server/server.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,11 @@ struct server_task {
267267
params.speculative.n_min = std::max(params.speculative.n_min, 2);
268268
params.speculative.n_max = std::max(params.speculative.n_max, 0);
269269

270+
// Use OpenAI API logprobs only if n_probs wasn't provided
271+
if (data.contains("logprobs") && params.sampling.n_probs == defaults.sampling.n_probs){
272+
params.sampling.n_probs = json_value(data, "logprobs", defaults.sampling.n_probs);
273+
}
274+
270275
if (data.contains("lora")) {
271276
if (data.at("lora").is_array()) {
272277
params.lora = parse_lora_request(params_base.lora_adapters, data.at("lora"));

0 commit comments

Comments
 (0)
0