add hf_tokenizer_path in server ModelSettings

jeffrey-fong · jeffrey-fong · commit 9594d5c888e9 · 2024-01-23T15:49:48.000Z
diff --git a/llama_cpp/server/model.py b/llama_cpp/server/model.py
@@ -119,6 +119,8 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama:
             use_mmap=settings.use_mmap,
             use_mlock=settings.use_mlock,
             kv_overrides=kv_overrides,
+            # Tokenizer Params (optionally for Functionary function calling)
+            hf_tokenizer_path=settings.hf_tokenizer_path,
             # Context Params
             seed=settings.seed,
             n_ctx=settings.n_ctx,
diff --git a/llama_cpp/server/settings.py b/llama_cpp/server/settings.py
@@ -56,6 +56,11 @@ class ModelSettings(BaseSettings):
         default=None,
         description="List of model kv overrides in the format key=type:value where type is one of (bool, int, float). Valid true values are (true, TRUE, 1), otherwise false.",
     )
+    # Tokenizer Params
+    hf_tokenizer_path: Optional[str] = Field(
+        default=None,
+        description="Override llama.cpp tokenizer with HF AutoTokenizer from this path if provided.",
+    )
     # Context Params
     seed: int = Field(
         default=llama_cpp.LLAMA_DEFAULT_SEED, description="Random seed. -1 for random."
