Refactor file handling in ParquetDatasetMetadata; update version to 0…

….9.3.13
legout · Dec 10, 2024 · 0ceed0c · 0ceed0c
1 parent 9432630
commit 0ceed0c
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 12 deletions.
diff --git a/pydala/metadata.py b/pydala/metadata.py
@@ -286,18 +286,17 @@ def update_file_metadata(self, files: list[str] | None = None, **kwargs) -> None
         Returns:
             None
         """
-        new_files = []
+        new_files = files or []
         rm_files = []
-        # Add new files to file_metadata
+
         if not files:
             files = self._ls_files()
 
             if self.has_file_metadata:
                 new_files += sorted(set(files) - set(self.files_in_file_metadata))
                 rm_files += sorted(set(self.files_in_file_metadata) - set(files))
-
-        else:
-            new_files += files
+            else:
+                new_files += files
 
         if new_files:
             self._collect_file_metadata(files=new_files, **kwargs)
@@ -343,8 +342,7 @@ def _get_unified_schema(
             pyarrow.Schema: The unified schema for the dataset.
 
         """
-        # if not self.has_file_metadata:
-        #    self.update_file_metadata()
+
         if self.has_file_metadata:
             new_files = sorted(
                 (set(self.files_in_file_metadata) - set(self.files_in_metadata))
@@ -398,8 +396,6 @@ def _repair_file_schemas(
         # get unified schema
         if schema is None:
             schema, _ = self._get_unified_schema()
-            #     ts_unit=ts_unit, tz=tz, use_large_string=use_large_string, sort=sort
-            # )
 
         files_to_repair = [
             f
@@ -410,8 +406,6 @@ def _repair_file_schemas(
         if format_version is None and self.has_metadata:
             format_version = self.metadata.format_version
 
-        # find files to repair
-        # files with different schema or format version
         if format_version is not None:
             files_to_repair += [
                 f

diff --git a/pyproject.toml b/pyproject.toml
@@ -23,7 +23,7 @@ name = "pydala2"
 readme = "README.md"
 #repository = "https://github.com/legout/pydala2"
 requires-python = ">= 3.10"
-version = "0.9.3.12"
+version = "0.9.3.13"
 
 [project.optional-dependencies]
 legacy = ["polars-lts-cpu>=0.20.4"]