diff --git a/convert.py b/convert.py
index 27d187daca9fa..448b6f0f35ba7 100755
--- a/convert.py
+++ b/convert.py
@@ -846,7 +846,12 @@ def add_meta_vocab(self, vocab: Vocab) -> None:
             scores.append(score)
             toktypes.append(toktype)
 
-        self.gguf.add_tokenizer_model("llama")
+        if isinstance(vocab, SentencePieceVocab):
+            self.gguf.add_tokenizer_model("llama")
+        elif isinstance(vocab, BpeVocab):
+            self.gguf.add_tokenizer_model("gpt2")
+        else:
+            raise ValueError(f'Unknown vocab type: Not BpeVocab or SentencePieceVocab')
         self.gguf.add_token_list(tokens)
         self.gguf.add_token_scores(scores)
         self.gguf.add_token_types(toktypes)