Fix : Nemotron tokenizer for GGUF format (huggingface#35836)

fix nemotron gguf
elvircrn · Feb 13, 2025 · 87a49ed · 87a49ed
1 parent 96f7d49
commit 87a49ed
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 2 deletions.
diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py
@@ -339,6 +339,7 @@
             ("musicgen_melody", ("T5Tokenizer", "T5TokenizerFast" if is_tokenizers_available() else None)),
             ("mvp", ("MvpTokenizer", "MvpTokenizerFast" if is_tokenizers_available() else None)),
             ("myt5", ("MyT5Tokenizer", None)),
+            ("nemotron", (None, "PreTrainedTokenizerFast" if is_tokenizers_available() else None)),
             ("nezha", ("BertTokenizer", "BertTokenizerFast" if is_tokenizers_available() else None)),
             (
                 "nllb",

diff --git a/tests/quantization/ggml/test_ggml.py b/tests/quantization/ggml/test_ggml.py
@@ -835,9 +835,9 @@ def test_nemotron_q6_k(self):
 
         tokenizer = AutoTokenizer.from_pretrained(self.nemotron_model_id, gguf_file=self.q6_k_nemotron_model_id)
         text = tokenizer(self.example_text, return_tensors="pt")["input_ids"]
-        out = model.generate(text, max_new_tokens=10)
+        out = model.generate(text, max_new_tokens=16)
 
-        EXPECTED_TEXT = "'Hello. hotmail.com.'"
+        EXPECTED_TEXT = "Hello.▁hotmail.com</s>"
         self.assertEqual(tokenizer.decode(out[0], skip_special_tokens=True), EXPECTED_TEXT)
 
     def test_gemma2_q3_k(self):