From 2023a3e3335fa4ed9d784de20e4a6a43c520793c Mon Sep 17 00:00:00 2001 From: Klaus Hipp Date: Fri, 26 Jan 2024 23:42:13 +0100 Subject: [PATCH] Fix input data file extension in examples --- examples/flax/language-modeling/run_bart_dlm_flax.py | 3 ++- examples/flax/language-modeling/run_clm_flax.py | 3 ++- examples/flax/language-modeling/run_mlm_flax.py | 3 ++- examples/flax/language-modeling/run_t5_mlm_flax.py | 3 ++- examples/pytorch/language-modeling/run_clm_no_trainer.py | 3 ++- examples/pytorch/language-modeling/run_mlm_no_trainer.py | 3 ++- examples/pytorch/language-modeling/run_plm.py | 3 ++- examples/pytorch/multiple-choice/run_swag.py | 3 ++- examples/pytorch/multiple-choice/run_swag_no_trainer.py | 3 ++- .../question-answering/run_qa_beam_search_no_trainer.py | 4 +++- examples/pytorch/question-answering/run_qa_no_trainer.py | 4 +++- .../pytorch/summarization/run_summarization_no_trainer.py | 3 ++- examples/pytorch/token-classification/run_ner.py | 4 +++- examples/pytorch/token-classification/run_ner_no_trainer.py | 3 ++- examples/pytorch/translation/run_translation_no_trainer.py | 3 ++- .../jax-projects/model_parallel/run_clm_mp.py | 3 ++- examples/research_projects/luke/run_luke_ner_no_trainer.py | 3 ++- examples/research_projects/mlm_wwm/run_mlm_wwm.py | 3 ++- examples/research_projects/performer/run_mlm_performer.py | 3 ++- examples/tensorflow/language-modeling/run_mlm.py | 3 ++- examples/tensorflow/multiple-choice/run_swag.py | 3 ++- examples/tensorflow/token-classification/run_ner.py | 3 ++- .../run_{{cookiecutter.example_shortcut}}.py | 3 ++- 23 files changed, 49 insertions(+), 23 deletions(-) diff --git a/examples/flax/language-modeling/run_bart_dlm_flax.py b/examples/flax/language-modeling/run_bart_dlm_flax.py index 8603482218b4..f5369299a6d4 100644 --- a/examples/flax/language-modeling/run_bart_dlm_flax.py +++ b/examples/flax/language-modeling/run_bart_dlm_flax.py @@ -558,9 +558,10 @@ def main(): data_files = {} if data_args.train_file is not None: data_files["train"] = data_args.train_file + extension = data_args.train_file.split(".")[-1] if data_args.validation_file is not None: data_files["validation"] = data_args.validation_file - extension = data_args.train_file.split(".")[-1] + extension = data_args.validation_file.split(".")[-1] if extension == "txt": extension = "text" datasets = load_dataset( diff --git a/examples/flax/language-modeling/run_clm_flax.py b/examples/flax/language-modeling/run_clm_flax.py index 48d924f9bb39..2145b242bb1c 100755 --- a/examples/flax/language-modeling/run_clm_flax.py +++ b/examples/flax/language-modeling/run_clm_flax.py @@ -449,9 +449,10 @@ def main(): dataset_args = {} if data_args.train_file is not None: data_files["train"] = data_args.train_file + extension = data_args.train_file.split(".")[-1] if data_args.validation_file is not None: data_files["validation"] = data_args.validation_file - extension = data_args.train_file.split(".")[-1] + extension = data_args.validation_file.split(".")[-1] if extension == "txt": extension = "text" dataset_args["keep_linebreaks"] = data_args.keep_linebreaks diff --git a/examples/flax/language-modeling/run_mlm_flax.py b/examples/flax/language-modeling/run_mlm_flax.py index 39fc5e783637..d0aa82c6b8fe 100755 --- a/examples/flax/language-modeling/run_mlm_flax.py +++ b/examples/flax/language-modeling/run_mlm_flax.py @@ -485,9 +485,10 @@ def main(): data_files = {} if data_args.train_file is not None: data_files["train"] = data_args.train_file + extension = data_args.train_file.split(".")[-1] if data_args.validation_file is not None: data_files["validation"] = data_args.validation_file - extension = data_args.train_file.split(".")[-1] + extension = data_args.validation_file.split(".")[-1] if extension == "txt": extension = "text" datasets = load_dataset( diff --git a/examples/flax/language-modeling/run_t5_mlm_flax.py b/examples/flax/language-modeling/run_t5_mlm_flax.py index 45d3fe32bcf9..fa6a5742236c 100755 --- a/examples/flax/language-modeling/run_t5_mlm_flax.py +++ b/examples/flax/language-modeling/run_t5_mlm_flax.py @@ -599,9 +599,10 @@ def main(): data_files = {} if data_args.train_file is not None: data_files["train"] = data_args.train_file + extension = data_args.train_file.split(".")[-1] if data_args.validation_file is not None: data_files["validation"] = data_args.validation_file - extension = data_args.train_file.split(".")[-1] + extension = data_args.validation_file.split(".")[-1] if extension == "txt": extension = "text" datasets = load_dataset( diff --git a/examples/pytorch/language-modeling/run_clm_no_trainer.py b/examples/pytorch/language-modeling/run_clm_no_trainer.py index 15d513b0c928..d2855a514a82 100755 --- a/examples/pytorch/language-modeling/run_clm_no_trainer.py +++ b/examples/pytorch/language-modeling/run_clm_no_trainer.py @@ -345,9 +345,10 @@ def main(): dataset_args = {} if args.train_file is not None: data_files["train"] = args.train_file + extension = args.train_file.split(".")[-1] if args.validation_file is not None: data_files["validation"] = args.validation_file - extension = args.train_file.split(".")[-1] + extension = args.validation_file.split(".")[-1] if extension == "txt": extension = "text" dataset_args["keep_linebreaks"] = not args.no_keep_linebreaks diff --git a/examples/pytorch/language-modeling/run_mlm_no_trainer.py b/examples/pytorch/language-modeling/run_mlm_no_trainer.py index d9b8120a98e8..68253c33a35f 100755 --- a/examples/pytorch/language-modeling/run_mlm_no_trainer.py +++ b/examples/pytorch/language-modeling/run_mlm_no_trainer.py @@ -351,9 +351,10 @@ def main(): data_files = {} if args.train_file is not None: data_files["train"] = args.train_file + extension = args.train_file.split(".")[-1] if args.validation_file is not None: data_files["validation"] = args.validation_file - extension = args.train_file.split(".")[-1] + extension = args.validation_file.split(".")[-1] if extension == "txt": extension = "text" raw_datasets = load_dataset(extension, data_files=data_files) diff --git a/examples/pytorch/language-modeling/run_plm.py b/examples/pytorch/language-modeling/run_plm.py index 66451247e06a..1a744083b18a 100755 --- a/examples/pytorch/language-modeling/run_plm.py +++ b/examples/pytorch/language-modeling/run_plm.py @@ -328,9 +328,10 @@ def main(): data_files = {} if data_args.train_file is not None: data_files["train"] = data_args.train_file + extension = data_args.train_file.split(".")[-1] if data_args.validation_file is not None: data_files["validation"] = data_args.validation_file - extension = data_args.train_file.split(".")[-1] + extension = data_args.validation_file.split(".")[-1] if extension == "txt": extension = "text" raw_datasets = load_dataset(extension, data_files=data_files, cache_dir=model_args.cache_dir) diff --git a/examples/pytorch/multiple-choice/run_swag.py b/examples/pytorch/multiple-choice/run_swag.py index a1cfcfdddafa..2eaf97b70335 100755 --- a/examples/pytorch/multiple-choice/run_swag.py +++ b/examples/pytorch/multiple-choice/run_swag.py @@ -311,9 +311,10 @@ def main(): data_files = {} if data_args.train_file is not None: data_files["train"] = data_args.train_file + extension = data_args.train_file.split(".")[-1] if data_args.validation_file is not None: data_files["validation"] = data_args.validation_file - extension = data_args.train_file.split(".")[-1] + extension = data_args.validation_file.split(".")[-1] raw_datasets = load_dataset( extension, data_files=data_files, diff --git a/examples/pytorch/multiple-choice/run_swag_no_trainer.py b/examples/pytorch/multiple-choice/run_swag_no_trainer.py index 9ad725483291..529ce2eae3cb 100755 --- a/examples/pytorch/multiple-choice/run_swag_no_trainer.py +++ b/examples/pytorch/multiple-choice/run_swag_no_trainer.py @@ -357,9 +357,10 @@ def main(): data_files = {} if args.train_file is not None: data_files["train"] = args.train_file + extension = args.train_file.split(".")[-1] if args.validation_file is not None: data_files["validation"] = args.validation_file - extension = args.train_file.split(".")[-1] + extension = args.validation_file.split(".")[-1] raw_datasets = load_dataset(extension, data_files=data_files) # Trim a number of training examples if args.debug: diff --git a/examples/pytorch/question-answering/run_qa_beam_search_no_trainer.py b/examples/pytorch/question-answering/run_qa_beam_search_no_trainer.py index 905189d0d41a..4103dd0014ec 100644 --- a/examples/pytorch/question-answering/run_qa_beam_search_no_trainer.py +++ b/examples/pytorch/question-answering/run_qa_beam_search_no_trainer.py @@ -362,11 +362,13 @@ def main(): data_files = {} if args.train_file is not None: data_files["train"] = args.train_file + extension = args.train_file.split(".")[-1] if args.validation_file is not None: data_files["validation"] = args.validation_file + extension = args.validation_file.split(".")[-1] if args.test_file is not None: data_files["test"] = args.test_file - extension = args.train_file.split(".")[-1] + extension = args.test_file.split(".")[-1] raw_datasets = load_dataset(extension, data_files=data_files, field="data") # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at # https://huggingface.co/docs/datasets/loading_datasets. diff --git a/examples/pytorch/question-answering/run_qa_no_trainer.py b/examples/pytorch/question-answering/run_qa_no_trainer.py index 1a58f6ce442f..0c6794b16f21 100755 --- a/examples/pytorch/question-answering/run_qa_no_trainer.py +++ b/examples/pytorch/question-answering/run_qa_no_trainer.py @@ -410,11 +410,13 @@ def main(): data_files = {} if args.train_file is not None: data_files["train"] = args.train_file + extension = args.train_file.split(".")[-1] if args.validation_file is not None: data_files["validation"] = args.validation_file + extension = args.validation_file.split(".")[-1] if args.test_file is not None: data_files["test"] = args.test_file - extension = args.train_file.split(".")[-1] + extension = args.test_file.split(".")[-1] raw_datasets = load_dataset(extension, data_files=data_files, field="data") # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at # https://huggingface.co/docs/datasets/loading_datasets. diff --git a/examples/pytorch/summarization/run_summarization_no_trainer.py b/examples/pytorch/summarization/run_summarization_no_trainer.py index 96ccb552ed16..826c8b64a4ec 100644 --- a/examples/pytorch/summarization/run_summarization_no_trainer.py +++ b/examples/pytorch/summarization/run_summarization_no_trainer.py @@ -404,9 +404,10 @@ def main(): data_files = {} if args.train_file is not None: data_files["train"] = args.train_file + extension = args.train_file.split(".")[-1] if args.validation_file is not None: data_files["validation"] = args.validation_file - extension = args.train_file.split(".")[-1] + extension = args.validation_file.split(".")[-1] raw_datasets = load_dataset(extension, data_files=data_files) # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at # https://huggingface.co/docs/datasets/loading_datasets. diff --git a/examples/pytorch/token-classification/run_ner.py b/examples/pytorch/token-classification/run_ner.py index 8d7a67cd1571..13ce7d98771d 100755 --- a/examples/pytorch/token-classification/run_ner.py +++ b/examples/pytorch/token-classification/run_ner.py @@ -311,11 +311,13 @@ def main(): data_files = {} if data_args.train_file is not None: data_files["train"] = data_args.train_file + extension = data_args.train_file.split(".")[-1] if data_args.validation_file is not None: data_files["validation"] = data_args.validation_file + extension = data_args.validation_file.split(".")[-1] if data_args.test_file is not None: data_files["test"] = data_args.test_file - extension = data_args.train_file.split(".")[-1] + extension = data_args.test_file.split(".")[-1] raw_datasets = load_dataset(extension, data_files=data_files, cache_dir=model_args.cache_dir) # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at # https://huggingface.co/docs/datasets/loading_datasets. diff --git a/examples/pytorch/token-classification/run_ner_no_trainer.py b/examples/pytorch/token-classification/run_ner_no_trainer.py index 02bbd12d22ba..439730e77d26 100755 --- a/examples/pytorch/token-classification/run_ner_no_trainer.py +++ b/examples/pytorch/token-classification/run_ner_no_trainer.py @@ -339,9 +339,10 @@ def main(): data_files = {} if args.train_file is not None: data_files["train"] = args.train_file + extension = args.train_file.split(".")[-1] if args.validation_file is not None: data_files["validation"] = args.validation_file - extension = args.train_file.split(".")[-1] + extension = args.validation_file.split(".")[-1] raw_datasets = load_dataset(extension, data_files=data_files) # Trim a number of training examples if args.debug: diff --git a/examples/pytorch/translation/run_translation_no_trainer.py b/examples/pytorch/translation/run_translation_no_trainer.py index c4764b5ee4a7..87f7ba6bcfad 100644 --- a/examples/pytorch/translation/run_translation_no_trainer.py +++ b/examples/pytorch/translation/run_translation_no_trainer.py @@ -384,9 +384,10 @@ def main(): data_files = {} if args.train_file is not None: data_files["train"] = args.train_file + extension = args.train_file.split(".")[-1] if args.validation_file is not None: data_files["validation"] = args.validation_file - extension = args.train_file.split(".")[-1] + extension = args.validation_file.split(".")[-1] raw_datasets = load_dataset(extension, data_files=data_files) # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at # https://huggingface.co/docs/datasets/loading_datasets. diff --git a/examples/research_projects/jax-projects/model_parallel/run_clm_mp.py b/examples/research_projects/jax-projects/model_parallel/run_clm_mp.py index 4ff4bd559d8c..a72e5cff861c 100644 --- a/examples/research_projects/jax-projects/model_parallel/run_clm_mp.py +++ b/examples/research_projects/jax-projects/model_parallel/run_clm_mp.py @@ -297,9 +297,10 @@ def main(): data_files = {} if data_args.train_file is not None: data_files["train"] = data_args.train_file + extension = data_args.train_file.split(".")[-1] if data_args.validation_file is not None: data_files["validation"] = data_args.validation_file - extension = data_args.train_file.split(".")[-1] + extension = data_args.validation_file.split(".")[-1] if extension == "txt": extension = "text" dataset = load_dataset(extension, data_files=data_files, cache_dir=model_args.cache_dir) diff --git a/examples/research_projects/luke/run_luke_ner_no_trainer.py b/examples/research_projects/luke/run_luke_ner_no_trainer.py index e03c665e4ec2..cac487b059d7 100644 --- a/examples/research_projects/luke/run_luke_ner_no_trainer.py +++ b/examples/research_projects/luke/run_luke_ner_no_trainer.py @@ -285,9 +285,10 @@ def main(): data_files = {} if args.train_file is not None: data_files["train"] = args.train_file + extension = args.train_file.split(".")[-1] if args.validation_file is not None: data_files["validation"] = args.validation_file - extension = args.train_file.split(".")[-1] + extension = args.validation_file.split(".")[-1] raw_datasets = load_dataset(extension, data_files=data_files) # Trim a number of training examples if args.debug: diff --git a/examples/research_projects/mlm_wwm/run_mlm_wwm.py b/examples/research_projects/mlm_wwm/run_mlm_wwm.py index 3a7326d38219..629026bdb20a 100644 --- a/examples/research_projects/mlm_wwm/run_mlm_wwm.py +++ b/examples/research_projects/mlm_wwm/run_mlm_wwm.py @@ -271,9 +271,10 @@ def main(): data_files = {} if data_args.train_file is not None: data_files["train"] = data_args.train_file + extension = data_args.train_file.split(".")[-1] if data_args.validation_file is not None: data_files["validation"] = data_args.validation_file - extension = data_args.train_file.split(".")[-1] + extension = data_args.validation_file.split(".")[-1] if extension == "txt": extension = "text" datasets = load_dataset(extension, data_files=data_files) diff --git a/examples/research_projects/performer/run_mlm_performer.py b/examples/research_projects/performer/run_mlm_performer.py index 7c1f418815be..4261d9c184b7 100644 --- a/examples/research_projects/performer/run_mlm_performer.py +++ b/examples/research_projects/performer/run_mlm_performer.py @@ -517,9 +517,10 @@ def generate_batch_splits(samples_idx: np.ndarray, batch_size: int) -> np.ndarra data_files = {} if data_args.train_file is not None: data_files["train"] = data_args.train_file + extension = data_args.train_file.split(".")[-1] if data_args.validation_file is not None: data_files["validation"] = data_args.validation_file - extension = data_args.train_file.split(".")[-1] + extension = data_args.validation_file.split(".")[-1] if extension == "txt": extension = "text" datasets = load_dataset(extension, data_files=data_files) diff --git a/examples/tensorflow/language-modeling/run_mlm.py b/examples/tensorflow/language-modeling/run_mlm.py index 5be9e0219b71..38bdc71d984d 100755 --- a/examples/tensorflow/language-modeling/run_mlm.py +++ b/examples/tensorflow/language-modeling/run_mlm.py @@ -341,9 +341,10 @@ def main(): data_files = {} if data_args.train_file is not None: data_files["train"] = data_args.train_file + extension = data_args.train_file.split(".")[-1] if data_args.validation_file is not None: data_files["validation"] = data_args.validation_file - extension = data_args.train_file.split(".")[-1] + extension = data_args.validation_file.split(".")[-1] if extension == "txt": extension = "text" raw_datasets = load_dataset( diff --git a/examples/tensorflow/multiple-choice/run_swag.py b/examples/tensorflow/multiple-choice/run_swag.py index 8572ec98e1ae..4706fbc7e6dd 100644 --- a/examples/tensorflow/multiple-choice/run_swag.py +++ b/examples/tensorflow/multiple-choice/run_swag.py @@ -320,9 +320,10 @@ def main(): data_files = {} if data_args.train_file is not None: data_files["train"] = data_args.train_file + extension = data_args.train_file.split(".")[-1] if data_args.validation_file is not None: data_files["validation"] = data_args.validation_file - extension = data_args.train_file.split(".")[-1] + extension = data_args.validation_file.split(".")[-1] raw_datasets = load_dataset( extension, data_files=data_files, diff --git a/examples/tensorflow/token-classification/run_ner.py b/examples/tensorflow/token-classification/run_ner.py index 84b2ab702a17..c6921f9f9aa3 100644 --- a/examples/tensorflow/token-classification/run_ner.py +++ b/examples/tensorflow/token-classification/run_ner.py @@ -260,9 +260,10 @@ def main(): data_files = {} if data_args.train_file is not None: data_files["train"] = data_args.train_file + extension = data_args.train_file.split(".")[-1] if data_args.validation_file is not None: data_files["validation"] = data_args.validation_file - extension = data_args.train_file.split(".")[-1] + extension = data_args.validation_file.split(".")[-1] raw_datasets = load_dataset( extension, data_files=data_files, diff --git a/templates/adding_a_new_example_script/{{cookiecutter.directory_name}}/run_{{cookiecutter.example_shortcut}}.py b/templates/adding_a_new_example_script/{{cookiecutter.directory_name}}/run_{{cookiecutter.example_shortcut}}.py index f01283ae08fc..2ed4a4d8af90 100755 --- a/templates/adding_a_new_example_script/{{cookiecutter.directory_name}}/run_{{cookiecutter.example_shortcut}}.py +++ b/templates/adding_a_new_example_script/{{cookiecutter.directory_name}}/run_{{cookiecutter.example_shortcut}}.py @@ -730,9 +730,10 @@ def main(): data_files = {} if args.train_file is not None: data_files["train"] = args.train_file + extension = args.train_file.split(".")[-1] if args.validation_file is not None: data_files["validation"] = args.validation_file - extension = args.train_file.split(".")[-1] + extension = args.validation_file.split(".")[-1] raw_datasets = load_dataset(extension, data_files=data_files) # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at # https://huggingface.co/docs/datasets/loading_datasets.