feat: Update llama.cpp

abetlen · abetlen · commit c3fc80a2cf5e · 2024-09-01T21:06:18.000-04:00
diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py
@@ -276,6 +276,7 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa
 #     LLAMA_VOCAB_TYPE_BPE  = 2, // GPT-2 tokenizer based on byte-level BPE
 #     LLAMA_VOCAB_TYPE_WPM  = 3, // BERT tokenizer based on WordPiece
 #     LLAMA_VOCAB_TYPE_UGM  = 4, // T5 tokenizer based on Unigram
+#     LLAMA_VOCAB_TYPE_RWKV = 5, // RWKV tokenizer based on greedy tokenization
 # };
 LLAMA_VOCAB_TYPE_NONE = 0
 """For models without vocab"""
@@ -287,6 +288,8 @@ def byref(obj: CtypesCData, offset: Optional[int] = None) -> CtypesRef[CtypesCDa
 """BERT tokenizer based on WordPiece"""
 LLAMA_VOCAB_TYPE_UGM = 4
 """T5 tokenizer based on Unigram"""
+LLAMA_VOCAB_TYPE_RWKV = 5
+"""RWKV tokenizer based on greedy tokenization"""
 
 
 # // pre-tokenization types
diff --git a/vendor/llama.cpp b/vendor/llama.cpp
@@ -1 +1 @@
-Subproject commit a47667cff41f5a198eb791974e0afcc1cddd3229
+Subproject commit 8f1d81a0b6f50b9bad72db0b6fcd299ad9ecd48c
