From 5651be511735cb89f522ee3449eba433385a14cb Mon Sep 17 00:00:00 2001 From: Branden Butler Date: Fri, 10 Nov 2023 09:59:22 -0600 Subject: [PATCH] Support special tokens and not adding BOS to prompt in speculative --- examples/speculative/speculative.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/speculative/speculative.cpp b/examples/speculative/speculative.cpp index 3a8e278110c20e..0aae69d16f7b3d 100644 --- a/examples/speculative/speculative.cpp +++ b/examples/speculative/speculative.cpp @@ -94,9 +94,13 @@ int main(int argc, char ** argv) { } } - // tokenize the prompt + + // Tokenize the prompt + const bool add_bos = llama_vocab_type(llama_get_model(ctx_tgt)) == LLAMA_VOCAB_TYPE_SPM; + LOG("add_bos: %d\n", add_bos); + std::vector inp; - inp = ::llama_tokenize(ctx_tgt, params.prompt, true); + inp = ::llama_tokenize(ctx_tgt, params.prompt, add_bos, true); const int max_context_size = llama_n_ctx(ctx_tgt); const int max_tokens_list_size = max_context_size - 4;