deprecate warning

huggingface · Jul 25, 2024 · ec6f78a · ec6f78a
1 parent 5728ea9
commit ec6f78a
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py
@@ -1595,6 +1595,10 @@ def __init__(self, **kwargs):
 
         # By default, cleaning tokenization spaces for both fast and slow tokenizers
         self.clean_up_tokenization_spaces = kwargs.pop("clean_up_tokenization_spaces", False)
+        warnings.warn(
+            "The `clean_up_tokenization_spaces` argument will soon be deprecated. It currently defaults to False if not passed.",
+            FutureWarning,
+        )
 
         # By default, do not split special tokens for both fast and slow tokenizers
         self.split_special_tokens = kwargs.pop("split_special_tokens", False)