Merge pull request #15 from observingClouds/fix_isdir

improve ls error handling

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## unreleased
+- Fix `isdir()` function call [#15](https://github.com/observingClouds/ecmwfspec/pull/15)
+- Add UPath support for ec-protocol [#15](https://github.com/observingClouds/ecmwfspec/pull/15)
+- Raise specific errors when `ls` fails due to PermissionError or FileNotFoundError [#15](https://github.com/observingClouds/ecmwfspec/pull/15)
+
 ## 0.0.2
 - Add support for `ectmp` file paths [#2](https://github.com/observingClouds/ecmwfspec/issues/2)
 - Fix file listing when using additional flags [#9](https://github.com/observingClouds/ecmwfspec/issues/9)

ecmwfspec/core.py

@@ -22,6 +22,7 @@
 )
 
 from fsspec.spec import AbstractFileSystem
+from upath import UPath
 
 from . import ecfs_wrapper as ecfs
 
@@ -435,3 +436,14 @@ def _open(
             encoding=kwargs.get("encoding"),
             file_permissions=self.file_permissions,
         )
+
+
+class ECFSPath(UPath):
+    @property
+    def path(self) -> str:
+        path = "/".join(self.parts)
+
+        if not path.startswith("/"):
+            path = "/" + path
+
+        return path

ecmwfspec/ecfs_wrapper.py

@@ -48,19 +48,25 @@ def ls(
         command.insert(-1, "-R")
 
     result = subprocess.run(
-        command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True
+        command, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
     )
     logger.debug(result.stdout)
-
-    if result.stderr is not None:
-        logger.error(result.stderr)
-        raise Exception("Error running command: {}".format(command))
-
-    files = result.stdout.split("\n")  # type: ignore
-    files = [f for f in files if f != ""]
+    if result.returncode != 0:
+        logger.debug(result.stderr)
+        if "Permission denied" in result.stderr:
+            raise PermissionError(result.stderr)
+        elif "File does not exist" in result.stderr:
+            raise FileNotFoundError(result.stderr)
+        else:
+            raise Exception(result.stderr)
+
+    result_lines = result.stdout.split("\n")
+    result_lines = [f for f in result_lines if f != ""]
 
     if detail:
-        files = [f.split() for f in files]
+        files = [f.split() for f in result_lines]
+    else:
+        files = result_lines  # type: ignore
 
     df = pd.DataFrame(files, columns=columns)
 

pyproject.toml

@@ -6,6 +6,8 @@ readme = "README.md"
 dependencies = [
     "fsspec",
     "dask",
+    "universal-pathlib",
+    "typing_extensions; python_version<'3.11'",
     "xarray"
 ]
 
@@ -69,5 +71,8 @@ exclude_lines = ["pragma: no cover"]
 ec = "ecmwfspec.ECFileSystem"
 ectmp = "ecmwfspec.ECTmpFileSystem"
 
+[project.entry-points."universal_pathlib.implementations"]
+ec = "ecmwfspec.core:ECFSPath"
+
 [tool.pdm.scripts]
 post_install = "pre-commit install"