Update llama.cpp

abetlen · abetlen · commit 3a29d65f456c · 2023-08-26T23:36:24.000-04:00
diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py
@@ -455,7 +455,7 @@ def detokenize(self, tokens: List[int]) -> bytes:
             output += bytes(buffer[:n])
         # NOTE: Llama1 models automatically added a space at the start of the prompt
         # this line removes a leading space if the first token is a beginning of sentence token
-        return output[1:] if len(tokens) > 0 and tokens[0] == self.token_bos() else output
+        return output
 
     def set_cache(self, cache: Optional[BaseLlamaCache]):
         """Set the cache.
diff --git a/tests/test_llama.py b/tests/test_llama.py
@@ -14,16 +14,16 @@ def test_llama_cpp_tokenization():
 
     tokens = llama.tokenize(text)
     assert tokens[0] == llama.token_bos()
-    assert tokens == [1, 15043, 2787]
+    assert tokens == [1, 10994, 2787]
     detokenized = llama.detokenize(tokens)
     assert detokenized == text
 
     tokens = llama.tokenize(text, add_bos=False)
     assert tokens[0] != llama.token_bos()
-    assert tokens == [15043, 2787]
+    assert tokens == [10994, 2787]
 
     detokenized = llama.detokenize(tokens)
-    assert detokenized != text
+    assert detokenized == text
 
 
 @pytest.mark.skip(reason="bug in tokenization where leading space is always inserted even if not after eos")
diff --git a/vendor/llama.cpp b/vendor/llama.cpp
@@ -1 +1 @@
-Subproject commit 232caf3c1581a6cb023571780ff41dc2d66d1ca0
+Subproject commit c1ac54b77aaba10d029084d152be786102010eb2
