8000 kv-cache : refactor + add llama_memory_state_i by ggerganov · Pull Request #13746 · ggml-org/llama.cpp · GitHub
[go: up one dir, main page]
Skip to content

kv-cache : refactor + add llama_memory_state_i #13746

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 14 commits into from
May 31, 2025
Merged

kv-cache : refactor + add llama_memory_state_i #13746

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
773b6e3
kv-cache : simplify the "struct llama_kv_cache" interface
ggerganov May 25, 2025
9fc50dc
kv-cache : revert the (n_swa + n_ubatch) change (for next PR)
ggerganov May 25, 2025
c2c3591
kv-cache : some comments
ggerganov May 25, 2025
8856782
context : fix graph reserve for multiple sequences
ggerganov May 25, 2025
bffb9d4
kv-cache : fix typo [no ci]
ggerganov May 25, 2025
32cc9ea
kv-cache : fix find_slot() logic for free slots
ggerganov May 25, 2025
f97de9b
llama : add TODO for deprecating the defrag API in the future
ggerganov May 26, 2025
7764d91
kv-cache : improve find_slot() using min/max seq pos info
ggerganov May 27, 2025
780bba9
llama : handle aborts and compute errors
ggerganov May 28, 2025
dbcfa5f
memory : extract state into llama_memory_state
ggerganov May 28, 2025
f2ded9d
kv-cache : add comments
ggerganov May 30, 2025
e230e51
server : update batching logic to reset n_batch on successful decode
ggerganov May 31, 2025
3cf5186
server : upon full re-processing, remove the sequence from the cache
ggerganov May 31, 2025
71619f2
kv-cache : add TODO for doing split_equal when split_simple fails
ggerganov May 31, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
kv-cache : add comments 
ggml-ci
  • Loading branch information
@ggerganov
ggerganov committed May 30, 2025
commit f2ded9d44b18afac15673d94ed741dfca0b55472
13 changes: 10 additions & 3 deletions src/llama-kv-cache.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -493,9 +493,13 @@ int32_t llama_kv_cache_unified::find_slot(const llama_ubatch & ubatch) const {
LLAMA_LOG_WARN("\n%s\n", ss.c_str());
}

LLAMA_LOG_WARN("kv_cells: n_swa = %4d, min[0] = %5d, max[0] = %5d\n", n_swa, cells.seq_pos_min(0), cells.seq_pos_max(0));
LLAMA_LOG_WARN("kv_cells: n_swa = %4d, min[1] = %5d, max[1] = %5d\n", n_swa, cells.seq_pos_min(1), cells.seq_pos_max(1));
LLAMA_LOG_WARN("kv_cells: n_swa = %4d, min[2] = %5d, max[2] = %5d\n", n_swa, cells.seq_pos_min(2), cells.seq_pos_max(2));
for (int s = 0; s < LLAMA_MAX_PARALLEL_SEQUENCES; ++s) {
if (cells.seq_pos_min(s) < 0) {
continue;
}

LLAMA_LOG_WARN("kv_cells: n_swa = %4d, min[%d] = %5d, max[%d] = %5d\n", n_swa, s, cells.seq_pos_min(s), s, cells.seq_pos_max(s));
}
#endif

uint32_t n_tested = 0;
Expand Down Expand Up @@ -538,6 +542,9 @@ int32_t llama_kv_cache_unified::find_slot(const llama_ubatch & ubatch) const {
const llama_seq_id seq_id_cell = cells.seq_get(head_cur + i);

// SWA mask
// note: we insert only in the cell with minimum pos in order to preserve the invariant that
// all positions between [pos_min, pos_max] for each sequence will be present in the cache
// ref: https://github.com/ggml-org/llama.cpp/pull/13746#issuecomment-2916057092
if (pos_cell == seq_pos_min[seq_id_cell] &&
is_masked_swa(pos_cell, cells.seq_pos_max(seq_id_cell) + 1)) {
seq_pos_min[seq_id_cell]++;
Expand Down
5 changes: 5 additions & 0 deletions src/llama-kv-cells.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@ class llama_kv_cells_unified {
}
}

// clear a non-empty cell
void rm(uint32_t i) {
assert(i < pos.size());
assert(pos[i] != -1);
Expand Down Expand Up @@ -202,13 +203,15 @@ class llama_kv_cells_unified {
return false;
}

// number of different sequences in the cell
int seq_count(uint32_t i) const {
assert(i < pos.size());
assert(pos[i] != -1);

return seq[i].count();
}

// check if the cell contains seq_id
bool seq_has(uint32_t i, llama_seq_id seq_id) const {
assert(i < pos.size());
assert(seq_id >= 0);
Expand All @@ -226,6 +229,8 @@ class llama_kv_cells_unified {
seq_pos[seq_id].insert(pos[i]);
}

// return the sequence id of this cell
// note: call only for cells with exactly one sequence
llama_seq_id seq_get(uint32_t i) const {
assert(seq[i].count() == 1);

Expand Down
0