Merge pull request abetlen#108 from eiery/main

abetlen · web-flow · commit f37456133a40 · 2023-04-24T13:48:09.000-04:00
Update n_batch default to 512 to match upstream llama.cpp
diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py
@@ -37,7 +37,7 @@ def __init__(
         use_mlock: bool = False,
         embedding: bool = False,
         n_threads: Optional[int] = None,
-        n_batch: int = 8,
+        n_batch: int = 512,
         last_n_tokens_size: int = 64,
         lora_base: Optional[str] = None,
         lora_path: Optional[str] = None,
