Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reduce the number of warnings in writer tests #2691

Merged
merged 7 commits into from
Dec 15, 2023
Merged

Reduce the number of warnings in writer tests #2691

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
4df285d
Fix proj authority usage
pnuu Dec 15, 2023
013b49f
Fix/suppress PROJ warnings about losing projection information
pnuu Dec 15, 2023
755ec6b
Use datetime64[ns] in CF writer
pnuu Dec 15, 2023
cd4cd73
Catch warning about pretty time formatting
pnuu Dec 15, 2023
36b09d1
Catch warning of invalid NetCDF dataset name
pnuu Dec 15, 2023
8ea9e30
Add area definitions to remove unnecessary warnings
pnuu Dec 15, 2023
612e927
Handle warnings from encoding time in CF
pnuu Dec 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions satpy/cf/coords.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -291,8 +291,8 @@ def add_time_bounds_dimension(ds: xr.Dataset, time: str = "time") -> xr.Dataset:
if start_time is not None)
end_time = min(end_time for end_time in end_times
if end_time is not None)
ds["time_bnds"] = xr.DataArray([[np.datetime64(start_time),
np.datetime64(end_time)]],
ds["time_bnds"] = xr.DataArray([[np.datetime64(start_time, "ns"),
np.datetime64(end_time, "ns")]],
dims=["time", "bnds_1d"])
ds[time].attrs["bounds"] = "time_bnds"
ds[time].attrs["standard_name"] = "time"
Expand Down
16 changes: 15 additions & 1 deletion satpy/tests/test_writers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -548,13 +548,20 @@ def setUp(self):
import tempfile
from datetime import datetime

from pyresample.geometry import AreaDefinition

from satpy.scene import Scene

adef = AreaDefinition(
"test", "test", "test", "EPSG:4326",
100, 200, (-180., -90., 180., 90.),
)
ds1 = xr.DataArray(
da.zeros((100, 200), chunks=50),
dims=("y", "x"),
attrs={"name": "test",
"start_time": datetime(2018, 1, 1, 0, 0, 0)}
"start_time": datetime(2018, 1, 1, 0, 0, 0),
"area": adef}
)
self.scn = Scene()
self.scn["test"] = ds1
Expand Down Expand Up @@ -650,15 +657,22 @@ def setup_method(self):
import tempfile
from datetime import datetime

from pyresample.geometry import AreaDefinition

from satpy.scene import Scene

adef = AreaDefinition(
"test", "test", "test", "EPSG:4326",
100, 200, (-180., -90., 180., 90.),
)
ds1 = xr.DataArray(
da.zeros((100, 200), chunks=50),
dims=("y", "x"),
attrs={
"name": "test",
"start_time": datetime(2018, 1, 1, 0, 0, 0),
"sensor": "fake_sensor",
"area": adef,
}
)
ds2 = ds1.copy()
Expand Down
45 changes: 28 additions & 17 deletions satpy/tests/writer_tests/test_cf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,8 @@ def test_save_dataset_a_digit_no_prefix_include_attr(self):
scn = Scene()
scn["1"] = xr.DataArray([1, 2, 3])
with TempFile() as filename:
scn.save_datasets(filename=filename, writer="cf", include_orig_name=True, numeric_name_prefix="")
with pytest.warns(UserWarning, match=r"Invalid NetCDF dataset name"):
scn.save_datasets(filename=filename, writer="cf", include_orig_name=True, numeric_name_prefix="")
with xr.open_dataset(filename) as f:
np.testing.assert_array_equal(f["1"][:], [1, 2, 3])
assert "original_name" not in f["1"].attrs
Expand Down Expand Up @@ -208,8 +209,10 @@ def test_groups(self):
attrs={"name": "HRV", "start_time": tstart, "end_time": tend})

with TempFile() as filename:
scn.save_datasets(filename=filename, writer="cf", groups={"visir": ["IR_108", "VIS006"], "hrv": ["HRV"]},
pretty=True)
with pytest.warns(UserWarning, match=r"Cannot pretty-format"):
scn.save_datasets(filename=filename, writer="cf",
groups={"visir": ["IR_108", "VIS006"], "hrv": ["HRV"]},
pretty=True)

nc_root = xr.open_dataset(filename)
assert "history" in nc_root.attrs
Expand Down Expand Up @@ -240,11 +243,11 @@ def test_single_time_value(self):
test_array = np.array([[1, 2], [3, 4]])
scn["test-array"] = xr.DataArray(test_array,
dims=["x", "y"],
coords={"time": np.datetime64("2018-05-30T10:05:00")},
coords={"time": np.datetime64("2018-05-30T10:05:00", "ns")},
attrs=dict(start_time=start_time,
end_time=end_time))
with TempFile() as filename:
scn.save_datasets(filename=filename, writer="cf")
scn.save_datasets(filename=filename, writer="cf", encoding={"time": {"units": "seconds since 2018-01-01"}})
with xr.open_dataset(filename, decode_cf=True) as f:
np.testing.assert_array_equal(f["time"], scn["test-array"]["time"])
bounds_exp = np.array([[start_time, end_time]], dtype="datetime64[m]")
Expand All @@ -255,13 +258,14 @@ def test_time_coordinate_on_a_swath(self):
scn = Scene()
test_array = np.array([[1, 2], [3, 4], [5, 6], [7, 8]])
times = np.array(["2018-05-30T10:05:00", "2018-05-30T10:05:01",
"2018-05-30T10:05:02", "2018-05-30T10:05:03"], dtype=np.datetime64)
"2018-05-30T10:05:02", "2018-05-30T10:05:03"], dtype="datetime64[ns]")
scn["test-array"] = xr.DataArray(test_array,
dims=["y", "x"],
coords={"time": ("y", times)},
attrs=dict(start_time=times[0], end_time=times[-1]))
with TempFile() as filename:
scn.save_datasets(filename=filename, writer="cf", pretty=True)
scn.save_datasets(filename=filename, writer="cf", pretty=True,
encoding={"time": {"units": "seconds since 2018-01-01"}})
with xr.open_dataset(filename, decode_cf=True) as f:
np.testing.assert_array_equal(f["time"], scn["test-array"]["time"])

Expand All @@ -273,11 +277,15 @@ def test_bounds(self):
test_array = np.array([[1, 2], [3, 4]]).reshape(2, 2, 1)
scn["test-array"] = xr.DataArray(test_array,
dims=["x", "y", "time"],
coords={"time": [np.datetime64("2018-05-30T10:05:00")]},
coords={"time": [np.datetime64("2018-05-30T10:05:00", "ns")]},
attrs=dict(start_time=start_time,
end_time=end_time))
with TempFile() as filename:
scn.save_datasets(filename=filename, writer="cf")
with warnings.catch_warnings():
# The purpose is to use the default time encoding, silence the warning
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

so the default time encoding triggers a warning?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, the default of xarray.coding.times.CFDatetimeCoder seems to be days since <epoch> and the datatype we end up is integers, so fractional days are not possible without the library converting to float. I don't know how the time dtype would be set to float by us, and this was the only case I saw of this warning so didn't use much time investigating.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, so then maybe something for the next iteration of warning smashing.

warnings.filterwarnings("ignore", category=UserWarning,
message=r"Times can't be serialized faithfully to int64 with requested units")
scn.save_datasets(filename=filename, writer="cf")
# Check decoded time coordinates & bounds
with xr.open_dataset(filename, decode_cf=True) as f:
bounds_exp = np.array([[start_time, end_time]], dtype="datetime64[m]")
Expand Down Expand Up @@ -307,16 +315,17 @@ def test_bounds_minimum(self):
test_arrayB = np.array([[1, 2], [3, 5]]).reshape(2, 2, 1)
scn["test-arrayA"] = xr.DataArray(test_arrayA,
dims=["x", "y", "time"],
coords={"time": [np.datetime64("2018-05-30T10:05:00")]},
coords={"time": [np.datetime64("2018-05-30T10:05:00", "ns")]},
attrs=dict(start_time=start_timeA,
end_time=end_timeA))
scn["test-arrayB"] = xr.DataArray(test_arrayB,
dims=["x", "y", "time"],
coords={"time": [np.datetime64("2018-05-30T10:05:00")]},
coords={"time": [np.datetime64("2018-05-30T10:05:00", "ns")]},
attrs=dict(start_time=start_timeB,
end_time=end_timeB))
with TempFile() as filename:
scn.save_datasets(filename=filename, writer="cf")
scn.save_datasets(filename=filename, writer="cf",
encoding={"time": {"units": "seconds since 2018-01-01"}})
with xr.open_dataset(filename, decode_cf=True) as f:
bounds_exp = np.array([[start_timeA, end_timeB]], dtype="datetime64[m]")
np.testing.assert_array_equal(f["time_bnds"], bounds_exp)
Expand All @@ -330,14 +339,15 @@ def test_bounds_missing_time_info(self):
test_arrayB = np.array([[1, 2], [3, 5]]).reshape(2, 2, 1)
scn["test-arrayA"] = xr.DataArray(test_arrayA,
dims=["x", "y", "time"],
coords={"time": [np.datetime64("2018-05-30T10:05:00")]},
coords={"time": [np.datetime64("2018-05-30T10:05:00", "ns")]},
attrs=dict(start_time=start_timeA,
end_time=end_timeA))
scn["test-arrayB"] = xr.DataArray(test_arrayB,
dims=["x", "y", "time"],
coords={"time": [np.datetime64("2018-05-30T10:05:00")]})
coords={"time": [np.datetime64("2018-05-30T10:05:00", "ns")]})
with TempFile() as filename:
scn.save_datasets(filename=filename, writer="cf")
scn.save_datasets(filename=filename, writer="cf",
encoding={"time": {"units": "seconds since 2018-01-01"}})
with xr.open_dataset(filename, decode_cf=True) as f:
bounds_exp = np.array([[start_timeA, end_timeA]], dtype="datetime64[m]")
np.testing.assert_array_equal(f["time_bnds"], bounds_exp)
Expand All @@ -350,11 +360,12 @@ def test_unlimited_dims_kwarg(self):
test_array = np.array([[1, 2], [3, 4]])
scn["test-array"] = xr.DataArray(test_array,
dims=["x", "y"],
coords={"time": np.datetime64("2018-05-30T10:05:00")},
coords={"time": np.datetime64("2018-05-30T10:05:00", "ns")},
attrs=dict(start_time=start_time,
end_time=end_time))
with TempFile() as filename:
scn.save_datasets(filename=filename, writer="cf", unlimited_dims=["time"])
scn.save_datasets(filename=filename, writer="cf", unlimited_dims=["time"],
encoding={"time": {"units": "seconds since 2018-01-01"}})
with xr.open_dataset(filename) as f:
assert set(f.encoding["unlimited_dims"]) == {"time"}

Expand Down
18 changes: 16 additions & 2 deletions satpy/tests/writer_tests/test_geotiff.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,19 @@

def _get_test_datasets_2d():
"""Create a single 2D test dataset."""
from pyresample.geometry import AreaDefinition

adef = AreaDefinition(
"test", "test", "test", "EPSG:4326",
100, 200, (-180., -90., 180., 90.),
)
ds1 = xr.DataArray(
da.zeros((100, 200), chunks=50),
dims=("y", "x"),
attrs={"name": "test",
"start_time": datetime.utcnow(),
"units": "K"}
"units": "K",
"area": adef}
)
return [ds1]

Expand All @@ -54,12 +61,19 @@ def _get_test_datasets_2d_nonlinear_enhancement():

def _get_test_datasets_3d():
"""Create a single 3D test dataset."""
from pyresample.geometry import AreaDefinition

adef = AreaDefinition(
"test", "test", "test", "EPSG:4326",
100, 200, (-180., -90., 180., 90.),
)
ds1 = xr.DataArray(
da.zeros((3, 100, 200), chunks=50),
dims=("bands", "y", "x"),
coords={"bands": ["R", "G", "B"]},
attrs={"name": "test",
"start_time": datetime.utcnow()}
"start_time": datetime.utcnow(),
"area": adef}
)
return [ds1]

Expand Down
Loading