[SW_196667] fix(nltk): Revert 1247. Updated the version. added the pu…

…nkt_tab download (#1258) Co-authored-by: regisss <15324346+regisss@users.noreply.github.com>
huggingface · Aug 15, 2024 · ff8f95f · ff8f95f
1 parent 998ff47
commit ff8f95f
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 1 deletion.
diff --git a/examples/summarization/requirements.txt b/examples/summarization/requirements.txt
@@ -2,7 +2,7 @@ datasets >= 2.4.0
 sentencepiece != 0.1.92
 protobuf
 rouge-score
-nltk == 3.8.1
+nltk
 py7zr
 torch >= 1.3
 evaluate
diff --git a/examples/summarization/run_summarization.py b/examples/summarization/run_summarization.py
@@ -80,6 +80,8 @@ def check_optimum_habana_min_version(*a, **b):
     with FileLock(".lock") as lock:
         nltk.download("punkt", quiet=True)
 
+nltk.download("punkt_tab")  # Needed for version 3.8.2
+
 # A list of all multilingual tokenizer which require lang attribute.
 MULTILINGUAL_TOKENIZERS = [MBartTokenizer, MBartTokenizerFast, MBart50Tokenizer, MBart50TokenizerFast]