llama : fix llama_token_is_prefix

ggerganov · Oct 11, 2024 · 6384002 · 6384002
1 parent 8131fce
commit 6384002
Showing 1 changed file with 3 additions and 3 deletions.
diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp
@@ -1865,14 +1865,14 @@ bool llama_token_is_prefix_impl(
     char text_buf_0[128];
     char text_buf_1[128];
 
-    const int32_t len0 = llama_token_to_piece_impl(vocab, token0, text_buf_0, 128, 0, false);
-    const int32_t len1 = llama_token_to_piece_impl(vocab, token1, text_buf_1, 128, 0, false);
+    const int32_t len0 = llama_token_to_piece_impl(vocab, token0, text_buf_0, sizeof(text_buf_0) - 1, 0, false);
+    const int32_t len1 = llama_token_to_piece_impl(vocab, token1, text_buf_1, sizeof(text_buf_1) - 1, 0, false);
 
     if (len0 <= 0 || len1 <= 0) {
         return false;
     }
 
-    return len0 < len1 && memcmp(text_buf_0, text_buf_1, len0) == 0;
+    return len0 <= len1 && memcmp(text_buf_0, text_buf_1, len0) == 0;
 }
 
 int32_t llama_detokenize_impl(