Skip to content

Commit

Permalink
[pre-commit.ci] auto fixes from pre-commit.com hooks
Browse files Browse the repository at this point in the history
for more information, see https://pre-commit.ci
  • Loading branch information
pre-commit-ci[bot] committed May 22, 2024
1 parent eca4faa commit c682cae
Show file tree
Hide file tree
Showing 3 changed files with 10 additions and 8 deletions.
4 changes: 2 additions & 2 deletions pangeo_forge_recipes/openers.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ def open_url(
cache: Optional[CacheFSSpecTarget] = None,
secrets: Optional[Dict] = None,
open_kwargs: Optional[Dict] = None,
fsspec_sync_patch: bool = False
fsspec_sync_patch: bool = False,
) -> OpenFileType:
"""Open a string-based URL with fsspec.
Expand All @@ -30,7 +30,7 @@ def open_url(
kw = open_kwargs or {}
if cache is not None:
# this has side effects
cache.cache_file(url, secrets, fsspec_sync_patch **kw)
cache.cache_file(url, secrets, fsspec_sync_patch**kw)
open_file = cache.open_file(url, mode="rb")
else:
open_file = _get_opener(url, secrets, fsspec_sync_patch, **kw)
Expand Down
9 changes: 7 additions & 2 deletions pangeo_forge_recipes/storage.py
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,9 @@ def _full_path(self, path: str) -> str:
class CacheFSSpecTarget(FlatFSSpecTarget):
"""Alias for FlatFSSpecTarget"""

def cache_file(self, fname: str, secrets: Optional[dict], fsspec_sync_patch=False, **open_kwargs) -> None:
def cache_file(
self, fname: str, secrets: Optional[dict], fsspec_sync_patch=False, **open_kwargs
) -> None:
# check and see if the file already exists in the cache
logger.info(f"Caching file '{fname}'")
input_opener = _get_opener(fname, secrets, fsspec_sync_patch, **open_kwargs)
Expand Down Expand Up @@ -228,10 +230,13 @@ def _get_opener(fname, secrets, fsspec_sync_patch, **open_kwargs):
logger.debug("Attempting to enable synchronous filesystem implementations in FSSpec")
try:
from httpfs_sync.core import SyncHTTPFileSystem

SyncHTTPFileSystem.overwrite_async_registration()
logger.debug("Synchronous HTTP implementation enabled.")
except ImportError:
logger.warning("httpfs_sync could not be imported. Falling back to default (async) http implementation.")
logger.warning(
"httpfs_sync could not be imported. Falling back to default (async) http implementation."
)

fname = fname if not secrets else _add_query_string_secrets(fname, secrets)
return fsspec.open(fname, mode="rb", **open_kwargs)
Expand Down
5 changes: 1 addition & 4 deletions pangeo_forge_recipes/transforms.py
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,6 @@ class OpenURLWithFSSpec(beam.PTransform):
max_concurrency: Optional[int] = None
fsspec_sync_patch: bool = False


def expand(self, pcoll):
if isinstance(self.cache, str):
cache = CacheFSSpecTarget.from_url(self.cache)
Expand All @@ -170,9 +169,7 @@ def expand(self, pcoll):
open_kwargs=self.open_kwargs,
)
return pcoll | MapWithConcurrencyLimit(
open_url,
kwargs=kws,
max_concurrency=self.max_concurrency
open_url, kwargs=kws, max_concurrency=self.max_concurrency
)


Expand Down

0 comments on commit c682cae

Please sign in to comment.