server : fix prompt caching for repeated prompts (ggml-org#5420)

ristew · web-flow · commit 7c777fcd5dd4 · 2024-02-09T12:49:49.000+02:00
diff --git a/examples/server/server.cpp b/examples/server/server.cpp
@@ -1592,10 +1592,6 @@ struct llama_server_context
                         LOG_TEE("slot %d : in cache: %i tokens | to process: %i tokens\n", slot.id, slot.n_past, slot.num_prompt_tokens_processed);
                     }
 
-                    LOG_TEE("slot %d : kv cache rm - [%d, end)\n", slot.id, (int) system_tokens.size() + slot.n_past);
-
-                    llama_kv_cache_seq_rm(ctx, slot.id, system_tokens.size() + slot.n_past, -1);
-
                     slot.cache_tokens = prompt_tokens;
 
                     if (slot.n_past == slot.num_prompt_tokens && slot.n_past > 0)
@@ -1609,6 +1605,10 @@ struct llama_server_context
                         }
                     }
 
+                    LOG_TEE("slot %d : kv cache rm - [%d, end)\n", slot.id, (int) system_tokens.size() + slot.n_past);
+
+                    llama_kv_cache_seq_rm(ctx, slot.id, system_tokens.size() + slot.n_past, -1);
+
                     LOG_VERBOSE("prompt ingested", {
                                                     {"n_past",  slot.n_past},
                                                     {"cached",  tokens_to_str(ctx, slot.cache_tokens.cbegin(), slot.cache_tokens.cbegin() + slot.n_past)},

Original file line number	Diff line number	Diff line change
`@@ -1592,10 +1592,6 @@ struct llama_server_context`
`1592`	`1592`	`LOG_TEE("slot %d : in cache: %i tokens \| to process: %i tokens\n", slot.id, slot.n_past, slot.num_prompt_tokens_processed);`
`1593`	`1593`	`}`
`1594`	`1594`
`1595`		`- LOG_TEE("slot %d : kv cache rm - [%d, end)\n", slot.id, (int) system_tokens.size() + slot.n_past);`
`1596`		`-`
`1597`		`- llama_kv_cache_seq_rm(ctx, slot.id, system_tokens.size() + slot.n_past, -1);`
`1598`		`-`
`1599`	`1595`	`slot.cache_tokens = prompt_tokens;`
`1600`	`1596`
`1601`	`1597`	`if (slot.n_past == slot.num_prompt_tokens && slot.n_past > 0)`
`@@ -1609,6 +1605,10 @@ struct llama_server_context`
`1609`	`1605`	`}`
`1610`	`1606`	`}`
`1611`	`1607`
	`1608`	`+ LOG_TEE("slot %d : kv cache rm - [%d, end)\n", slot.id, (int) system_tokens.size() + slot.n_past);`
	`1609`	`+`
	`1610`	`+ llama_kv_cache_seq_rm(ctx, slot.id, system_tokens.size() + slot.n_past, -1);`
	`1611`	`+`
`1612`	`1612`	`LOG_VERBOSE("prompt ingested", {`
`1613`	`1613`	`{"n_past", slot.n_past},`
`1614`	`1614`	`{"cached", tokens_to_str(ctx, slot.cache_tokens.cbegin(), slot.cache_tokens.cbegin() + slot.n_past)},`