fix(server): Use split_mode from model settings (abetlen#1594)

grider-withourai · abetlen · web-flow · commit 66d5cddbc319 · 2024-07-17T18:09:28.000-04:00
Co-authored-by: Andrei &lt;abetlen@gmail.com&gt;
diff --git a/llama_cpp/server/model.py b/llama_cpp/server/model.py
@@ -223,6 +223,7 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama:
             **kwargs,
             # Model Params
             n_gpu_layers=settings.n_gpu_layers,
+            split_mode=settings.split_mode,
             main_gpu=settings.main_gpu,
             tensor_split=settings.tensor_split,
             vocab_only=settings.vocab_only,
