8000 Adjust prompt processing thread count seperately from n_threads by netrunnereve · Pull Request #2534 · ggml-org/llama.cpp · GitHub
[go: up one dir, main page]
Skip to content

Adjust prompt processing thread count seperately from n_threads #2534

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 19 commits into from
Closed

Adjust prompt processing thread count seperately from n_threads #2534

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
5f02218
test pp_threads
netrunnereve Aug 6, 2023
1de711d
builds fine
netrunnereve Aug 6, 2023
590feea
add printout of pp_threads
netrunnereve Aug 6, 2023
215e2f2
only activate pp_threads for main for now
netrunnereve Aug 6, 2023
ce6d86e
fix
netrunnereve Aug 6, 2023
0480362
remove from llama_context_params
8000 netrunnereve Aug 6, 2023
5624a29
Merge branch 'ggerganov:master' into master
netrunnereve Aug 9, 2023
d854348
perplexity only uses pp_threads
netrunnereve Aug 9, 2023
be26777
add pp_threads support to other files
netrunnereve Aug 9, 2023
193f295
Update llama.cpp
netrunnereve Aug 9, 2023
3919e67
Update README.md
netrunnereve Aug 9, 2023
49f0bfd
Update README.md
netrunnereve Aug 9, 2023
01f45e1
manual merge with llama.cpp master
netrunnereve Aug 9, 2023
400dcce
Merge branch 'ggerganov:master' into master
netrunnereve Aug 10, 2023
a129a31
Merge branch 'ggerganov:master' into master
netrunnereve Aug 18, 2023
1c154e9
lazy fix for llama-bench (runs without pp_threads support)
netrunnereve Aug 18, 2023
d50ccb0
manual merge
netrunnereve Aug 24, 2023
471e469
pre gguf merge
netrunnereve Aug 24, 2023
8209b5d
revert llama_eval, create main example
netrunnereve Aug 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
revert llama_eval, create main example
  • Loading branch information
@netrunnereve
netrunnereve committed Aug 25, 2023
commit 8209b5d6a2349baf7c6f6682ff50ca6a26b52c54
10 changes: 6 additions & 4 deletions examples/main/main.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ int main(int argc, char ** argv) {
fprintf(stderr, "%s: testing memory usage for n_batch = %d, n_ctx = %d\n", __func__, params.n_batch, params.n_ctx);

const std::vector<llama_token> tmp(params.n_batch, llama_token_bos());
llama_eval(ctx, tmp.data(), tmp.size(), params.n_ctx, params.n_threads, params.pp_threads);
llama_eval(ctx, tmp.data(), tmp.size(), params.n_ctx, params.n_threads);
}

llama_print_timings(ctx);
Expand Down Expand Up @@ -406,7 +406,7 @@ int main(int argc, char ** argv) {
// do one empty run to warm up the model
{
const std::vector<llama_token> tmp = { llama_token_bos(), };
llama_eval(ctx, tmp.data(), tmp.size(), 0, params.n_threads, params.pp_threads);
llama_eval(ctx, tmp.data(), tmp.size(), 0, params.n_threads);
llama_reset_timings(ctx);
}

Expand Down Expand Up @@ -513,7 +513,8 @@ int main(int argc, char ** argv) {

for (int i = 0; i < input_size; i += params.n_batch) {
int n_eval = std::min(input_size - i, params.n_batch);
if (llama_eval(ctx_guidance, input_buf + i, n_eval, n_past_guidance, params.n_threads, params.pp_threads)) {
int eval_thr = n_eval > 1 ? params.pp_threads : params.n_threads;
if (llama_eval(ctx_guidance, input_buf + i, n_eval, n_past_guidance, eval_thr)) {
fprintf(stderr, "%s : failed to eval\n", __func__);
return 1;
}
Expand All @@ -527,7 +528,8 @@ int main(int argc, char ** argv) {
if (n_eval > params.n_batch) {
n_eval = params.n_batch;
}
if (llama_eval(ctx, &embd[i], n_eval, n_past, params.n_threads, params.pp_threads)) {
int eval_thr = n_eval > 1 ? params.pp_threads : params.n_threads;
if (llama_eval(ctx, &embd[i], n_eval, n_past, eval_thr)) {
fprintf(stderr, "%s : failed to eval\n", __func__);
return 1;
}
Expand Down
19 changes: 7 additions & 12 deletions llama.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1787,7 +1787,6 @@ static struct ggml_cgraph * llama_build_graph(
// - n_tokens number of tokens
// - n_past: the context size so far
// - n_threads: number of threads to use for inference
// - pp_threads: number of threads to use for prompt processing
//
static bool llama_eval_internal(
llama_context & lctx,
Expand All @@ -1796,7 +1795,6 @@ static bool llama_eval_internal(
int n_tokens,
int n_past,
int n_threads,
int pp_threads,
const char * cgraph_fname) {

LLAMA_ASSERT((!tokens && embd) || (tokens && !embd));
Expand Down Expand Up @@ -1840,8 +1838,7 @@ static bool llama_eval_internal(

// for big prompts, if BLAS is enabled, it is better to use only one thread
// otherwise, the threads are spin-lock waiting for the BLAS calls and are degrading the performance
pp_threads = N >= 32 && ggml_cpu_has_blas() && !ggml_cpu_has_gpublas() ? 1 : pp_threads;
n_threads = N > 1 ? pp_threads : n_threads;
n_threads = N >= 32 && ggml_cpu_has_blas() && !ggml_cpu_has_gpublas() ? 1 : n_threads;

struct ggml_tensor * res = gf->nodes[gf->n_nodes - 1];
struct ggml_tensor * embeddings = gf->nodes[gf->n_nodes - 2];
Expand Down Expand Up @@ -3487,7 +3484,7 @@ struct llama_context * llama_new_context_with_model(
if (ggml_mpi_rank(ctx->ctx_mpi) > 0) {
// Enter a blocking eval loop with dummy input, letting rank=0 drive the process
const std::vector<llama_token> tmp(ctx->model.hparams.n_ctx, llama_token_bos());
while (!llama_eval(ctx, tmp.data(), tmp.size(), 0, 0, 0)) {};
while (!llama_eval(ctx, tmp.data(), tmp.size(), 0, 0)) {};
llama_backend_free();
exit(1);
}
Expand Down Expand Up @@ -4179,9 +4176,8 @@ int llama_eval(
const llama_token * tokens,
int n_tokens,
int n_past,
int n_threads,
int pp_threads) {
if (!llama_eval_internal(*ctx, tokens, nullptr, n_tokens, n_past, n_threads, pp_threads, nullptr)) {
int n_threads) {
if (!llama_eval_internal(*ctx, tokens, nullptr, n_tokens, n_past, n_threads, nullptr)) {
LLAMA_LOG_ERROR("%s: failed to eval\n", __func__);
return 1;
}
Expand All @@ -4202,9 +4198,8 @@ int llama_eval_embd(
const float * embd,
int n_tokens,
int n_past,
int n_threads,
int pp_threads) {
if (!llama_eval_internal(*ctx, nullptr, embd, n_tokens, n_past, n_threads, pp_threads, nullptr)) {
int n_threads) {
if (!llama_eval_internal(*ctx, nullptr, embd, n_tokens, n_past, n_threads, nullptr)) {
LLAMA_LOG_ERROR("%s: failed to eval\n", __func__);
return 1;
}
Expand All @@ -4225,7 +4220,7 @@ int llama_eval_export(struct llama_context * ctx, const char * fname) {

const std::vector<llama_token> tmp(n_batch, llama_token_bos());

if (!llama_eval_internal(*ctx, tmp.data(), nullptr, tmp.size(), n_ctx, 1, 1, fname)) {
if (!llama_eval_internal(*ctx, tmp.data(), nullptr, tmp.size(), n_ctx, 1, fname)) {
LLAMA_LOG_ERROR("%s: failed to eval\n", __func__);
return 1;
}
Expand Down
6 changes: 2 additions & 4 deletions llama.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -308,17 +308,15 @@ extern "C" {
const llama_token * tokens,
int n_tokens,
int n_past,
int n_threads,
int pp_threads);
int n_threads);

// Same as llama_eval, but use float matrix input directly.
LLAMA_API int llama_eval_embd(
struct llama_context * ctx,
const float * embd,
int n_tokens,
int n_past,
int n_threads,
int pp_threads);
int n_threads);

// Export a static computation graph for context of 511 and batch size of 1
// NOTE: since this functionality is mostly for debugging and demonstration purposes, we hardcode these
Expand Down
0