-
Notifications
You must be signed in to change notification settings - Fork 15
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
CLN: Remove last usages of the Any type #489
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -147,15 +147,18 @@ class DerivedObjectDescriptor: | |
"table", | ||
"dictionary", | ||
] | ||
efolder: Literal[ | ||
"maps", | ||
"polygons", | ||
"points", | ||
"cubes", | ||
"grids", | ||
"tables", | ||
"dictionaries", | ||
] | ||
efolder: ( | ||
Literal[ | ||
"maps", | ||
"polygons", | ||
"points", | ||
"cubes", | ||
"grids", | ||
"tables", | ||
"dictionaries", | ||
] | ||
| str | ||
) | ||
fmt: str | ||
extension: str | ||
spec: Dict[str, Any] | ||
|
@@ -230,8 +233,8 @@ class ObjectDataProvider: | |
""" | ||
|
||
# input fields | ||
obj: Any | ||
dataio: Any | ||
obj: types.Inferrable | ||
dataio: dataio.ExportData | ||
meta_existing: dict = field(default_factory=dict) | ||
|
||
# result properties; the most important is metadata which IS the 'data' part in | ||
|
@@ -262,7 +265,7 @@ def _derive_name_stratigraphy(self) -> DerivedNamedStratigraphy: | |
name = derive_name(self.dataio, self.obj) | ||
|
||
# next check if usename has a "truename" and/or aliases from the config | ||
strat = self.dataio.config.get("stratigraphy") # shortform | ||
strat = self.dataio.config.get("stratigraphy", {}) | ||
no_start_or_missing_name = strat is None or name not in strat | ||
|
||
rv = DerivedNamedStratigraphy( | ||
|
@@ -658,11 +661,11 @@ def _derive_spec_bbox_dataframe( | |
) -> SpecificationAndBoundingBox: | ||
"""Process/collect the data items for DataFrame.""" | ||
logger.info("Process data metadata for DataFrame (tables)") | ||
df: pd.DataFrame = self.obj | ||
assert isinstance(self.obj, pd.DataFrame) | ||
return SpecificationAndBoundingBox( | ||
spec=specification.TableSpecification( | ||
columns=list(df.columns), | ||
size=int(df.size), | ||
columns=list(self.obj.columns), | ||
size=int(self.obj.size), | ||
).model_dump( | ||
mode="json", | ||
exclude_none=True, | ||
|
@@ -675,11 +678,13 @@ def _derive_spec_bbox_arrowtable( | |
) -> SpecificationAndBoundingBox: | ||
"""Process/collect the data items for Arrow table.""" | ||
logger.info("Process data metadata for arrow (tables)") | ||
table = self.obj | ||
from pyarrow import Table | ||
|
||
assert isinstance(self.obj, Table) | ||
return SpecificationAndBoundingBox( | ||
spec=specification.TableSpecification( | ||
columns=list(table.column_names), | ||
size=table.num_columns * table.num_rows, | ||
columns=list(self.obj.column_names), | ||
size=self.obj.num_columns * self.obj.num_rows, | ||
).model_dump( | ||
mode="json", | ||
exclude_none=True, | ||
|
@@ -699,6 +704,9 @@ def _get_columns(self) -> list[str]: | |
columns = list(self.obj.columns) | ||
else: | ||
logger.debug("arrow") | ||
from pyarrow import Table | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I wonder if and when these lazy imports start to become numerous enough to bite us later on... the import doesn't seem excessively slow to my mind, about half a second (at least as far as Python is concerned). Probably few enough to be OK now, but will there be more? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Ive wondered so as well. But i think for now, its fine. But if i go back and look at older versions it looks like there was a period of time where we could not really upon pyarrow being installed. |
||
|
||
assert isinstance(self.obj, Table) | ||
columns = self.obj.column_names | ||
logger.debug("Available columns in table %s ", columns) | ||
return columns | ||
|
@@ -813,7 +821,7 @@ def _derive_from_existing(self) -> None: | |
# TODO: Clean up types below. | ||
self.time0, self.time1 = parse_timedata(self.meta_existing["data"]) # type: ignore | ||
|
||
def _process_content(self) -> tuple[str, dict | None]: | ||
def _process_content(self) -> tuple[str | dict, dict | None]: | ||
"""Work with the `content` metadata""" | ||
|
||
# content == "unset" is not wanted, but in case metadata has been produced while | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Discussed with @jcrivenaes ✅