From 7051c5fcc8c13e67063b20cda4ee7b8462f168bd Mon Sep 17 00:00:00 2001 From: Lucain Date: Tue, 21 Jan 2025 16:08:31 +0100 Subject: [PATCH] Remove deprecated `get_cached_models` (#35809) * Remove deprecated get_cached_models * imports --- src/transformers/file_utils.py | 1 - src/transformers/utils/__init__.py | 1 - src/transformers/utils/hub.py | 42 +----------------------------- 3 files changed, 1 insertion(+), 43 deletions(-) diff --git a/src/transformers/file_utils.py b/src/transformers/file_utils.py index c13d506a6137..4fae91f43fdb 100644 --- a/src/transformers/file_utils.py +++ b/src/transformers/file_utils.py @@ -71,7 +71,6 @@ copy_func, default_cache_path, define_sagemaker_information, - get_cached_models, get_file_from_repo, get_torch_version, has_file, diff --git a/src/transformers/utils/__init__.py b/src/transformers/utils/__init__.py index 919f9f8bde03..e5aedf5916fa 100755 --- a/src/transformers/utils/__init__.py +++ b/src/transformers/utils/__init__.py @@ -91,7 +91,6 @@ define_sagemaker_information, download_url, extract_commit_hash, - get_cached_models, get_file_from_repo, has_file, http_user_agent, diff --git a/src/transformers/utils/hub.py b/src/transformers/utils/hub.py index b7194ec579da..843b9ce667c2 100644 --- a/src/transformers/utils/hub.py +++ b/src/transformers/utils/hub.py @@ -25,7 +25,7 @@ import warnings from concurrent import futures from pathlib import Path -from typing import Dict, List, Optional, Tuple, Union +from typing import Dict, List, Optional, Union from urllib.parse import urlparse from uuid import uuid4 @@ -60,7 +60,6 @@ hf_raise_for_status, send_telemetry, ) -from huggingface_hub.utils._deprecation import _deprecate_method from requests.exceptions import HTTPError from . import __version__, logging @@ -165,45 +164,6 @@ def is_remote_url(url_or_filename): return parsed.scheme in ("http", "https") -# TODO: remove this once fully deprecated -# TODO? remove from './examples/research_projects/lxmert/utils.py' as well -# TODO? remove from './examples/research_projects/visual_bert/utils.py' as well -@_deprecate_method(version="4.39.0", message="This method is outdated and does not support the new cache system.") -def get_cached_models(cache_dir: Union[str, Path] = None) -> List[Tuple]: - """ - Returns a list of tuples representing model binaries that are cached locally. Each tuple has shape `(model_url, - etag, size_MB)`. Filenames in `cache_dir` are use to get the metadata for each model, only urls ending with *.bin* - are added. - - Args: - cache_dir (`Union[str, Path]`, *optional*): - The cache directory to search for models within. Will default to the transformers cache if unset. - - Returns: - List[Tuple]: List of tuples each with shape `(model_url, etag, size_MB)` - """ - if cache_dir is None: - cache_dir = TRANSFORMERS_CACHE - elif isinstance(cache_dir, Path): - cache_dir = str(cache_dir) - if not os.path.isdir(cache_dir): - return [] - - cached_models = [] - for file in os.listdir(cache_dir): - if file.endswith(".json"): - meta_path = os.path.join(cache_dir, file) - with open(meta_path, encoding="utf-8") as meta_file: - metadata = json.load(meta_file) - url = metadata["url"] - etag = metadata["etag"] - if url.endswith(".bin"): - size_MB = os.path.getsize(meta_path.strip(".json")) / 1e6 - cached_models.append((url, etag, size_MB)) - - return cached_models - - def define_sagemaker_information(): try: instance_data = requests.get(os.environ["ECS_CONTAINER_METADATA_URI"]).json()