Bugfix: n_batch should always be <= n_ctx

abetlen · abetlen · commit 5075c16fccae · 2023-04-04T13:08:21.000-04:00
diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py
@@ -68,7 +68,7 @@ def __init__(
             maxlen=self.last_n_tokens_size,
         )
         self.tokens_consumed = 0
-        self.n_batch = n_batch
+        self.n_batch = min(n_ctx, n_batch)
 
         self.n_threads = n_threads or multiprocessing.cpu_count()
 

Original file line number	Diff line number	Diff line change
`@@ -68,7 +68,7 @@ def __init__(`
`68`	`68`	`maxlen=self.last_n_tokens_size,`
`69`	`69`	`)`
`70`	`70`	`self.tokens_consumed = 0`
`71`		`- self.n_batch = n_batch`
	`71`	`+ self.n_batch = min(n_ctx, n_batch)`
`72`	`72`
`73`	`73`	`self.n_threads = n_threads or multiprocessing.cpu_count()`
`74`	`74`