8000 `quantize`: add imatrix and dataset metadata in GGUF by phymbert · Pull Request #6658 · ggml-org/llama.cpp · GitHub
[go: up one dir, main page]
Skip to content

quantize: add imatrix and dataset metadata in GGUF #6658

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 16 commits into from
Apr 26, 2024
Merged

quantize: add imatrix and dataset metadata in GGUF #6658

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
d766c5a
imatrix: save the dataset file used in the output file
phymbert Apr 13, 2024
01e7795
llama: support kv overrides type string string
phymbert Apr 13, 2024
a9202fb
common: factorize KV Overrides parsing between common and server
phymbert Apr 13, 2024
262c95a
quantize: add imatrix n entries and dataset KV metadata
phymbert Apr 13, 2024
cbc43aa
llama: remove kv override str_value initialization as it does not com…
phymbert Apr 13, 2024
851de16
quantize: add imatrix m_last_call as `quantize.imatrix.chunks_count`
phymbert Apr 13, 2024
0d82da6
quantize: add imatrix filename in KV
phymbert Apr 13, 2024
ea0ad80
Merge remote-tracking branch 'refs/remotes/origin/master' into hp/qua…
phymbert Apr 19, 2024
82e4187
llama: add llama_model_kv_override_free
phymbert Apr 19, 2024
aa0e28f
common: add llama_model_kv_override_free
phymbert Apr 20, 2024
2606bc9
Merge remote-tracking branch 'refs/remotes/origin/master' into hp/qua…
phymbert Apr 20, 2024
4bd2664
llama: finally move the string KV override value to the stack
phymbert Apr 20, 2024
5cf8ccb
llama : minor
ggerganov Apr 21, 2024
8360e0c
no need to add a NUL to the std::vector, std::string can be initializ…
phymbert Apr 21, 2024
b54eede
Merge remote-tracking branch 'refs/remotes/origin/master' into hp/qua…
phymbert Apr 26, 2024
bcbdd28
kv override: ensure string termination
phymbert Apr 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
common: add llama_model_kv_override_free 
common: free kv override if used after model loading
  • Loading branch information
@phymbert
phymbert committed Apr 20, 2024
commit aa0e28f8fcc540e4c01b71ff65761551518edc27
12 changes: 12 additions & 0 deletions common/common.cpp 8000
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2232,6 +2232,10 @@ std::tuple<struct llama_model *, struct llama_context *> llama_init_from_gpt_par
return std::make_tuple(nullptr, nullptr);
}

if (!params.kv_overrides.empty()) {
llama_model_kv_override_free(params.kv_overrides.data());
}

auto cparams = llama_context_params_from_gpt_params(params);

llama_context * lctx = llama_new_context_with_model(model, cparams);
Expand Down Expand Up @@ -2952,3 +2956,11 @@ llama_control_vector_data llama_control_vector_load(const std::vector<llama_cont

return result;
}

void llama_model_kv_override_free(struct llama_model_kv_override * kv_overrides) {
for (const struct llama_model_kv_override *p = kv_overrides; p->key[0] != 0; p++) {
if (p->tag == LLAMA_KV_OVERRIDE_TYPE_STR) {
delete p->str_value;
}
}
}
3 changes: 3 additions & 0 deletions common/common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,9 @@ struct gpt_params {

bool parse_kv_override(const char * data, std::vector<llama_model_kv_override> & overrides);

// Frees all allocated memory
void llama_model_kv_override_free(struct llama_model_kv_override * ctx);

bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params);

bool gpt_params_parse(int argc, char ** argv, gpt_params & params);
Expand Down
8 changes: 0 additions & 8 deletions llama.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14987,14 +14987,6 @@ void llama_free_model(struct llama_model * model) {
delete model;
}

void llama_model_kv_override_free(struct llama_model_kv_override * kv_overrides) {
for (const struct llama_model_kv_override *p = kv_overrides; p->key[0] != 0; p++) {
if (p->tag == LLAMA_KV_OVERRIDE_TYPE_STR) {
delete p->str_value;
}
}
}

struct llama_context * llama_new_context_with_model(
struct llama_model * model,
struct llama_context_params params) {
Expand Down
3 changes: 0 additions & 3 deletions llama.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,9 +209,6 @@ extern "C" {
};
};

// Frees all allocated memory
LLAMA_API void llama_model_kv_override_free(struct llama_model_kv_override * ctx);

struct llama_model_params {
int32_t n_gpu_layers; // number of layers to store in VRAM
enum llama_split_mode split_mode; // how to split the model across multiple GPUs
Expand Down
0