Reorganize file utils (huggingface#16264)

* Split file_utils in several submodules

* Fixes

* Add back more objects

* More fixes

* Who exactly decided to import that from there?

* Second suggestion to code with code review

* Revert wront move

* Fix imports

* Adapt all imports

* Adapt all imports everywhere

* Revert this import, will fix in a separate commit

ISSUES.md

@@ -71,7 +71,7 @@ You are not required to read the following guidelines before opening an issue. H
      File "/transformers/src/transformers/__init__.py", line 34, in <module>
        from . import dependency_versions_check
      File "/transformers/src/transformers/dependency_versions_check.py", line 34, in <module>
-       from .file_utils import is_tokenizers_available
+       from .utils import is_tokenizers_available
      File "/transformers/src/transformers/file_utils.py", line 40, in <module>
        from tqdm.auto import tqdm
     ModuleNotFoundError: No module named 'tqdm.auto'
@@ -124,7 +124,7 @@ You are not required to read the following guidelines before opening an issue. H
      File "/transformers/src/transformers/__init__.py", line 34, in <module>
        from . import dependency_versions_check
      File "/transformers/src/transformers/dependency_versions_check.py", line 34, in <module>
-       from .file_utils import is_tokenizers_available
+       from .utils import is_tokenizers_available
      File "/transformers/src/transformers/file_utils.py", line 40, in <module>
        from tqdm.auto import tqdm
    ModuleNotFoundError: No module named 'tqdm.auto'

examples/flax/image-captioning/run_image_captioning_flax.py

@@ -52,7 +52,7 @@
     HfArgumentParser,
     is_tensorboard_available,
 )
-from transformers.file_utils import get_full_repo_name, is_offline_mode
+from transformers.utils import get_full_repo_name, is_offline_mode
 
 
 logger = logging.getLogger(__name__)

examples/flax/language-modeling/run_clm_flax.py

@@ -57,8 +57,8 @@
     is_tensorboard_available,
     set_seed,
 )
-from transformers.file_utils import get_full_repo_name
 from transformers.testing_utils import CaptureLogger
+from transformers.utils import get_full_repo_name
 
 
 logger = logging.getLogger(__name__)

examples/flax/language-modeling/run_mlm_flax.py

@@ -58,7 +58,7 @@
     is_tensorboard_available,
     set_seed,
 )
-from transformers.file_utils import get_full_repo_name
+from transformers.utils import get_full_repo_name
 
 
 MODEL_CONFIG_CLASSES = list(FLAX_MODEL_FOR_MASKED_LM_MAPPING.keys())

examples/flax/language-modeling/run_t5_mlm_flax.py

@@ -56,8 +56,8 @@
     is_tensorboard_available,
     set_seed,
 )
-from transformers.file_utils import get_full_repo_name
 from transformers.models.t5.modeling_flax_t5 import shift_tokens_right
+from transformers.utils import get_full_repo_name
 
 
 MODEL_CONFIG_CLASSES = list(FLAX_MODEL_FOR_MASKED_LM_MAPPING.keys())

examples/flax/question-answering/run_qa.py

@@ -53,8 +53,7 @@
     PreTrainedTokenizerFast,
     is_tensorboard_available,
 )
-from transformers.file_utils import get_full_repo_name
-from transformers.utils import check_min_version
+from transformers.utils import check_min_version, get_full_repo_name
 from utils_qa import postprocess_qa_predictions
 
 

examples/flax/summarization/run_summarization_flax.py

@@ -54,7 +54,7 @@
     HfArgumentParser,
     is_tensorboard_available,
 )
-from transformers.file_utils import get_full_repo_name, is_offline_mode
+from transformers.utils import get_full_repo_name, is_offline_mode
 
 
 logger = logging.getLogger(__name__)

examples/flax/text-classification/run_flax_glue.py

@@ -48,8 +48,7 @@
     TrainingArguments,
     is_tensorboard_available,
 )
-from transformers.file_utils import get_full_repo_name
-from transformers.utils import check_min_version
+from transformers.utils import check_min_version, get_full_repo_name
 
 
 logger = logging.getLogger(__name__)

examples/flax/token-classification/run_flax_ner.py

@@ -47,8 +47,7 @@
     HfArgumentParser,
     is_tensorboard_available,
 )
-from transformers.file_utils import get_full_repo_name
-from transformers.utils import check_min_version
+from transformers.utils import check_min_version, get_full_repo_name
 from transformers.utils.versions import require_version
 
 

examples/flax/vision/run_image_classification.py

@@ -53,7 +53,7 @@
     is_tensorboard_available,
     set_seed,
 )
-from transformers.file_utils import get_full_repo_name
+from transformers.utils import get_full_repo_name
 
 
 logger = logging.getLogger(__name__)

examples/legacy/seq2seq/old_test_tatoeba_conversion.py

@@ -16,9 +16,9 @@
 import tempfile
 import unittest
 
-from transformers.file_utils import cached_property
 from transformers.models.marian.convert_marian_tatoeba_to_pytorch import DEFAULT_REPO, TatoebaConverter
 from transformers.testing_utils import slow
+from transformers.utils import cached_property
 
 
 @unittest.skipUnless(os.path.exists(DEFAULT_REPO), "Tatoeba directory does not exist.")

examples/legacy/seq2seq/seq2seq_trainer.py

@@ -19,7 +19,6 @@
 from torch.utils.data import DistributedSampler, RandomSampler
 
 from transformers import PreTrainedModel, Trainer, logging
-from transformers.file_utils import is_torch_tpu_available
 from transformers.integrations import is_fairscale_available
 from transformers.models.fsmt.configuration_fsmt import FSMTConfig
 from transformers.optimization import (
@@ -34,6 +33,7 @@
 )
 from transformers.trainer_pt_utils import get_tpu_sampler
 from transformers.training_args import ParallelMode
+from transformers.utils import is_torch_tpu_available
 
 
 if is_fairscale_available():

examples/legacy/seq2seq/utils.py

@@ -34,8 +34,8 @@
 
 from sentence_splitter import add_newline_to_end_of_each_sentence
 from transformers import BartTokenizer, EvalPrediction, PreTrainedTokenizer, T5Tokenizer
-from transformers.file_utils import cached_property
 from transformers.models.bart.modeling_bart import shift_tokens_right
+from transformers.utils import cached_property
 
 
 try:

examples/pytorch/language-modeling/run_clm_no_trainer.py

@@ -51,7 +51,7 @@
     get_scheduler,
     set_seed,
 )
-from transformers.file_utils import get_full_repo_name
+from transformers.utils import get_full_repo_name
 from transformers.utils.versions import require_version
 
 

examples/pytorch/language-modeling/run_mlm_no_trainer.py

@@ -51,7 +51,7 @@
     get_scheduler,
     set_seed,
 )
-from transformers.file_utils import get_full_repo_name
+from transformers.utils import get_full_repo_name
 from transformers.utils.versions import require_version
 
 

examples/pytorch/multiple-choice/run_swag.py

@@ -41,10 +41,9 @@
     default_data_collator,
     set_seed,
 )
-from transformers.file_utils import PaddingStrategy
 from transformers.tokenization_utils_base import PreTrainedTokenizerBase
 from transformers.trainer_utils import get_last_checkpoint
-from transformers.utils import check_min_version
+from transformers.utils import PaddingStrategy, check_min_version
 
 
 # Will error if the minimal version of Transformers is not installed. Remove at your own risks.

examples/pytorch/multiple-choice/run_swag_no_trainer.py

@@ -50,7 +50,7 @@
     get_scheduler,
     set_seed,
 )
-from transformers.file_utils import PaddingStrategy, get_full_repo_name
+from transformers.utils import PaddingStrategy, get_full_repo_name
 
 
 logger = logging.getLogger(__name__)

examples/pytorch/question-answering/run_qa_beam_search_no_trainer.py

@@ -47,8 +47,7 @@
     get_scheduler,
     set_seed,
 )
-from transformers.file_utils import get_full_repo_name
-from transformers.utils import check_min_version
+from transformers.utils import check_min_version, get_full_repo_name
 from transformers.utils.versions import require_version
 from utils_qa import postprocess_qa_predictions_with_beam_search
 

examples/pytorch/question-answering/run_qa_no_trainer.py

@@ -49,8 +49,7 @@
     get_scheduler,
     set_seed,
 )
-from transformers.file_utils import get_full_repo_name
-from transformers.utils import check_min_version
+from transformers.utils import check_min_version, get_full_repo_name
 from transformers.utils.versions import require_version
 from utils_qa import postprocess_qa_predictions
 

examples/pytorch/speech-pretraining/run_wav2vec2_pretraining_no_trainer.py

@@ -42,8 +42,8 @@
     is_wandb_available,
     set_seed,
 )
-from transformers.file_utils import get_full_repo_name
 from transformers.models.wav2vec2.modeling_wav2vec2 import _compute_mask_indices, _sample_negative_indices
+from transformers.utils import get_full_repo_name
 
 
 logger = logging.getLogger(__name__)

examples/pytorch/summarization/run_summarization.py

@@ -45,9 +45,8 @@
     Seq2SeqTrainingArguments,
     set_seed,
 )
-from transformers.file_utils import is_offline_mode
 from transformers.trainer_utils import get_last_checkpoint
-from transformers.utils import check_min_version
+from transformers.utils import check_min_version, is_offline_mode
 from transformers.utils.versions import require_version
 
 

examples/pytorch/summarization/run_summarization_no_trainer.py

@@ -49,7 +49,7 @@
     get_scheduler,
     set_seed,
 )
-from transformers.file_utils import get_full_repo_name, is_offline_mode
+from transformers.utils import get_full_repo_name, is_offline_mode
 from transformers.utils.versions import require_version
 
 

examples/pytorch/test_pytorch_examples.py

@@ -25,8 +25,8 @@
 import torch
 
 from transformers import ViTMAEForPreTraining, Wav2Vec2ForPreTraining
-from transformers.file_utils import is_apex_available
 from transformers.testing_utils import CaptureLogger, TestCasePlus, get_gpu_count, slow, torch_device
+from transformers.utils import is_apex_available
 
 
 SRC_DIRS = [

examples/pytorch/text-classification/run_glue_no_trainer.py

@@ -40,7 +40,7 @@
     get_scheduler,
     set_seed,
 )
-from transformers.file_utils import get_full_repo_name
+from transformers.utils import get_full_repo_name
 from transformers.utils.versions import require_version
 
 

examples/pytorch/token-classification/run_ner_no_trainer.py

@@ -48,7 +48,7 @@
     get_scheduler,
     set_seed,
 )
-from transformers.file_utils import get_full_repo_name
+from transformers.utils import get_full_repo_name
 from transformers.utils.versions import require_version
 
 

examples/pytorch/translation/run_translation_no_trainer.py

@@ -50,7 +50,7 @@
     get_scheduler,
     set_seed,
 )
-from transformers.file_utils import get_full_repo_name
+from transformers.utils import get_full_repo_name
 from transformers.utils.versions import require_version
 
 

examples/tensorflow/multiple-choice/run_swag.py

@@ -43,9 +43,8 @@
     create_optimizer,
     set_seed,
 )
-from transformers.file_utils import PaddingStrategy
 from transformers.tokenization_utils_base import PreTrainedTokenizerBase
-from transformers.utils import check_min_version
+from transformers.utils import PaddingStrategy, check_min_version
 
 
 # Will error if the minimal version of Transformers is not installed. Remove at your own risks.

examples/tensorflow/question-answering/run_qa.py

@@ -41,8 +41,7 @@
     TFTrainingArguments,
     set_seed,
 )
-from transformers.file_utils import CONFIG_NAME, TF2_WEIGHTS_NAME
-from transformers.utils import check_min_version
+from transformers.utils import CONFIG_NAME, TF2_WEIGHTS_NAME, check_min_version
 from utils_qa import postprocess_qa_predictions
 
 

examples/tensorflow/summarization/run_summarization.py

@@ -43,9 +43,8 @@
     create_optimizer,
     set_seed,
 )
-from transformers.file_utils import is_offline_mode
 from transformers.trainer_utils import get_last_checkpoint
-from transformers.utils import check_min_version
+from transformers.utils import check_min_version, is_offline_mode
 from transformers.utils.versions import require_version
 
 

examples/tensorflow/text-classification/run_text_classification.py

@@ -37,7 +37,7 @@
     TFTrainingArguments,
     set_seed,
 )
-from transformers.file_utils import CONFIG_NAME, TF2_WEIGHTS_NAME
+from transformers.utils import CONFIG_NAME, TF2_WEIGHTS_NAME
 
 
 os.environ["TF_CPP_MIN_LOG_LEVEL"] = "1"  # Reduce the amount of console output from TF