common : add comments

ggml-org · ggerganov · Aug 27, 2023 · Aug 26, 2023 · Aug 26, 2023 · Aug 26, 2023
commit 1e7a033f10891e502e19b19ebc7da918409fe21e
diff --git a/common/common.h b/common/common.h
@@ -116,15 +116,21 @@ struct llama_context_params llama_context_params_from_gpt_params(const gpt_param
 // Vocab utils
 //
 
+// tokenizes a string into a vector of tokens
+// should work similar to Python's `tokenizer.encode`
 std::vector<llama_token> llama_tokenize(
         struct llama_context * ctx,
            const std::string & text,
                         bool   add_bos);
 
+// tokenizes a token into a piece
+// should work similar to Python's `tokenizer.id_to_piece`
 std::string llama_token_to_piece(
         const struct llama_context * ctx,
                        llama_token   token);
 
+// detokenizes a vector of tokens into a string
+// should work similar to Python's `tokenizer.decode`
 // removes the leading space from the first non-BOS token
 std::string llama_detokenize(
                          llama_context * ctx,

diff --git a/llama.h b/llama.h
@@ -384,7 +384,7 @@ extern "C" {
     // Token Id -> Piece.
     // Uses the vocabulary in the provided context.
     // Does not write null terminator to the buffer.
-    // Use code is responsible to remove the leading whitespace of the first non-BOS token.
+    // User code is responsible to remove the leading whitespace of the first non-BOS token when decoding multiple tokens.
     LLAMA_API int llama_token_to_piece(
             const struct llama_context * ctx,
                            llama_token   token,