fix: pass chat handler not chat formatter for huggingface autotokenizer and tokenizer_config formats.

abetlen · abetlen · commit 24f39454e91c · 2024-01-21T18:38:04.000-05:00
diff --git a/llama_cpp/server/model.py b/llama_cpp/server/model.py
@@ -78,7 +78,7 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama:
                 settings.hf_pretrained_model_name_or_path is not None
             ), "hf_pretrained_model_name_or_path must be set for hf-autotokenizer"
             chat_handler = (
-                llama_cpp.llama_chat_format.hf_autotokenizer_to_chat_formatter(
+                llama_cpp.llama_chat_format.hf_autotokenizer_to_chat_completion_handler(
                     settings.hf_pretrained_model_name_or_path
                 )
             )
@@ -87,7 +87,7 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama:
                 settings.hf_tokenizer_config_path is not None
             ), "hf_tokenizer_config_path must be set for hf-tokenizer-config"
             chat_handler = (
-                llama_cpp.llama_chat_format.hf_tokenizer_config_to_chat_formatter(
+                llama_cpp.llama_chat_format.hf_tokenizer_config_to_chat_completion_handler(
                     json.load(open(settings.hf_tokenizer_config_path))
                 )
             )

Original file line number	Diff line number	Diff line change
`@@ -78,7 +78,7 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama:`
`78`	`78`	`settings.hf_pretrained_model_name_or_path is not None`
`79`	`79`	`), "hf_pretrained_model_name_or_path must be set for hf-autotokenizer"`
`80`	`80`	`chat_handler = (`
`81`		`- llama_cpp.llama_chat_format.hf_autotokenizer_to_chat_formatter(`
	`81`	`+ llama_cpp.llama_chat_format.hf_autotokenizer_to_chat_completion_handler(`
`82`	`82`	`settings.hf_pretrained_model_name_or_path`
`83`	`83`	`)`
`84`	`84`	`)`
`@@ -87,7 +87,7 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama:`
`87`	`87`	`settings.hf_tokenizer_config_path is not None`
`88`	`88`	`), "hf_tokenizer_config_path must be set for hf-tokenizer-config"`
`89`	`89`	`chat_handler = (`
`90`		`- llama_cpp.llama_chat_format.hf_tokenizer_config_to_chat_formatter(`
	`90`	`+ llama_cpp.llama_chat_format.hf_tokenizer_config_to_chat_completion_handler(`
`91`	`91`	`json.load(open(settings.hf_tokenizer_config_path))`
`92`	`92`	`)`
`93`	`93`	`)`