potential out of bounds read

eiz · eiz · commit c67e5ef641ae · 2023-03-17T21:02:42.000-07:00
diff --git a/utils.cpp b/utils.cpp
@@ -324,7 +324,7 @@ struct llama_tokenizer {
         while (!text.empty()) {
             llama_sp_symbol sym;
             size_t char_len = utf8_len(text.data()[0]);
-            sym.text = std::string_view(text.data(), char_len);
+            sym.text = std::string_view(text.data(), std::min(text.size(), char_len));
             sym.prev = index - 1;
             text.remove_prefix(char_len);
             sym.next = text.empty() ? -1 : index + 1;
