fix: Updated test_embeded_special_tokens for luke and mluke models (#…

…32413) Fixed tokenizertests for luke, mluke models.
huggingface · Aug 7, 2024 · 2f92db4 · 2f92db4
1 parent 607bdd1
commit 2f92db4
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 6 deletions.
diff --git a/tests/models/luke/test_tokenization_luke.py b/tests/models/luke/test_tokenization_luke.py
@@ -146,11 +146,9 @@ def test_embeded_special_tokens(self):
                 # token_type_ids should put 0 everywhere
                 self.assertEqual(sum(tokens_r["token_type_ids"]), sum(tokens_p["token_type_ids"]))
 
-                # token_type_ids should put 0 everywhere
-                self.assertEqual(sum(tokens_r["token_type_ids"]), sum(tokens_p["token_type_ids"]))
-
                 # attention_mask should put 1 everywhere, so sum over length should be 1
                 self.assertEqual(
+                    sum(tokens_r["attention_mask"]) / len(tokens_r["attention_mask"]),
                     sum(tokens_p["attention_mask"]) / len(tokens_p["attention_mask"]),
                 )
 

diff --git a/tests/models/mluke/test_tokenization_mluke.py b/tests/models/mluke/test_tokenization_mluke.py
@@ -109,11 +109,9 @@ def test_embeded_special_tokens(self):
                 # token_type_ids should put 0 everywhere
                 self.assertEqual(sum(tokens_r["token_type_ids"]), sum(tokens_p["token_type_ids"]))
 
-                # token_type_ids should put 0 everywhere
-                self.assertEqual(sum(tokens_r["token_type_ids"]), sum(tokens_p["token_type_ids"]))
-
                 # attention_mask should put 1 everywhere, so sum over length should be 1
                 self.assertEqual(
+                    sum(tokens_r["attention_mask"]) / len(tokens_r["attention_mask"]),
                     sum(tokens_p["attention_mask"]) / len(tokens_p["attention_mask"]),
                 )