diff --git a/pydala/metadata.py b/pydala/metadata.py index ae343bd..3d033f7 100644 --- a/pydala/metadata.py +++ b/pydala/metadata.py @@ -286,18 +286,17 @@ def update_file_metadata(self, files: list[str] | None = None, **kwargs) -> None Returns: None """ - new_files = [] + new_files = files or [] rm_files = [] - # Add new files to file_metadata + if not files: files = self._ls_files() if self.has_file_metadata: new_files += sorted(set(files) - set(self.files_in_file_metadata)) rm_files += sorted(set(self.files_in_file_metadata) - set(files)) - - else: - new_files += files + else: + new_files += files if new_files: self._collect_file_metadata(files=new_files, **kwargs) @@ -343,8 +342,7 @@ def _get_unified_schema( pyarrow.Schema: The unified schema for the dataset. """ - # if not self.has_file_metadata: - # self.update_file_metadata() + if self.has_file_metadata: new_files = sorted( (set(self.files_in_file_metadata) - set(self.files_in_metadata)) @@ -398,8 +396,6 @@ def _repair_file_schemas( # get unified schema if schema is None: schema, _ = self._get_unified_schema() - # ts_unit=ts_unit, tz=tz, use_large_string=use_large_string, sort=sort - # ) files_to_repair = [ f @@ -410,8 +406,6 @@ def _repair_file_schemas( if format_version is None and self.has_metadata: format_version = self.metadata.format_version - # find files to repair - # files with different schema or format version if format_version is not None: files_to_repair += [ f diff --git a/pyproject.toml b/pyproject.toml index 9480b80..abaf1e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ name = "pydala2" readme = "README.md" #repository = "https://github.com/legout/pydala2" requires-python = ">= 3.10" -version = "0.9.3.12" +version = "0.9.3.13" [project.optional-dependencies] legacy = ["polars-lts-cpu>=0.20.4"]