Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor area_mean_time_series #750

Merged
merged 23 commits into from
May 30, 2024
Merged

Refactor area_mean_time_series #750

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
9c7418e
Refactor area_mean_time_series
tomvothecoder Oct 10, 2023
c947c1a
Add inital refactor work
tomvothecoder May 9, 2024
d12f027
Rename `region_to_data` to `metrics_dict`
tomvothecoder May 9, 2024
96332a2
Add `Dataset._get_year_str()` to pad years < 1000 for Xarray subsetting
tomvothecoder May 10, 2024
f682107
Refactor functions in `run_diag()`
tomvothecoder May 10, 2024
c5b06a7
Fix incorrect yearly mean API call and make plotter work
tomvothecoder May 13, 2024
21e3fc6
Clean up code in driver
tomvothecoder May 13, 2024
3c16a50
Update run script
tomvothecoder May 13, 2024
7c388ef
Fix exception with ref var causing QFLX to not generate
tomvothecoder May 13, 2024
47f44ad
Add regression testing json notebook
tomvothecoder May 14, 2024
fb40ef3
Add replicate functions for ccb slice flag
tomvothecoder May 24, 2024
0fc8605
Add `_get_non_submonthly_time_slice()`
tomvothecoder May 24, 2024
cfb30b9
Remove unused methods
tomvothecoder May 24, 2024
226656f
Add missing bounds for time series datasets
tomvothecoder May 24, 2024
ab88dfc
Fix tests
tomvothecoder May 24, 2024
2f0c0e1
Update docstrings
tomvothecoder May 29, 2024
3e1687c
Update docstrings
tomvothecoder May 29, 2024
f1d61d7
Refactor getting end time into single function
tomvothecoder May 29, 2024
6908a31
Update e3sm_diags/driver/utils/dataset_xr.py
tomvothecoder May 29, 2024
24b6571
Add fix for centering time for non-submonthly data
tomvothecoder May 30, 2024
8236dda
Fix viewer description in driver
tomvothecoder May 30, 2024
7de853c
Fix `qflxconvert_units()` not keeping attrs
tomvothecoder May 30, 2024
b48b73e
Fix unit tests
tomvothecoder May 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions auxiliary_tools/cdat_regression_testing/662-area-mean-time-series/annual_avgs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
# %%
import cdms2
import cdutil
import xcdat as xc

# %%
# CDAT
# Time coordinates at the first day of the month (1850-01-01)
fn1 = "/global/cfs/cdirs/e3sm/e3sm_diags/test_model_data_for_acme_diags/time-series/E3SM_v1/FSNS_185001_201312.nc"
var_key = "FSNS"

ds_cdat2 = cdms2.open(fn1)
var = ds_cdat2(var_key)

var_avg = cdutil.averager(var, axis="xy")
var_avg_year = cdutil.YEAR(var_avg)

print(var_avg_year.getTime().asComponentTime()[-3:-1])
# [2011-7-2 12:0:0.0, 2012-7-2 12:0:0.0]

# %%
# xCDAT
ds_xcdat = xc.open_dataset(fn1, decode_times=True)
ds_xcdat_avg = ds_xcdat.spatial.average(var_key, axis=["X", "Y"])
ds_xcdat_avg_year = ds_xcdat_avg.temporal.group_average(var_key, freq="year")

var_avg_year_xc = ds_xcdat_avg_year[var_key]

print(var_avg_year_xc.time.values[-3:-1])
# [cftime.DatetimeNoLeap(2012, 1, 1, 0, 0, 0, 0, has_year_zero=True)
# cftime.DatetimeNoLeap(2013, 1, 1, 0, 0, 0, 0, has_year_zero=True)]
154 changes: 154 additions & 0 deletions ...liary_tools/cdat_regression_testing/662-area-mean-time-series/debug/759_slice_flag_ccb.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
"""
https://github.com/E3SM-Project/e3sm_diags/blob/633b52c314325e605fe7f62687cc4d00e5a0a3d5/e3sm_diags/driver/utils/dataset.py#L665-L672
if sub_monthly:
start_time = "{}-01-01".format(start_year)
end_time = "{}-01-01".format(str(int(end_year) + 1))
slice_flag = "co"
else:
start_time = "{}-01-15".format(start_year)
end_time = "{}-12-15".format(end_year)
slice_flag = "ccb"
var_time = fin(var, time=(start_time, end_time, slice_flag))(squeeze=1)
"""
# %%
import cdms2
import xcdat as xc

# Parameters
# Time coordinates at the first day of the month (1850-01-01)
fn1 = "/global/cfs/cdirs/e3sm/e3sm_diags/test_model_data_for_acme_diags/time-series/E3SM_v1/FSNS_185001_201312.nc"
var = "FSNS"
start_time = "2011-01-15"
end_time = "2013-12-15"

slice_flag = "ccb"


ds_cdat2 = cdms2.open(fn1)


# "ccb" slice
# RESULT: Adds a coordinate to the end because the end time of the dataset is
# 2013-12-01, and "ccb" only allows the right side to be closed. It will use
# bounds to determine the right bound value to add the coordinate point.

# Example:
# 1. Actual end time: 2013-12-01
# 2. Slice end time: 2013-12-15
# 3. Bound values: [2013-12-01, 2014-1-1]
# 4. Use 2014-1-1 as last coordinate point.
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
var_s = ds_cdat2(var, time=(start_time, end_time, slice_flag))(squeeze=1)

time_s = var_s.getTime()
time_s_dt = time_s.asComponentTime()

"""
[2011-2-1 0:0:0.0,
2011-3-1 0:0:0.0,
2011-4-1 0:0:0.0,
2011-5-1 0:0:0.0,
2011-6-1 0:0:0.0,
2011-7-1 0:0:0.0,
2011-8-1 0:0:0.0,
2011-9-1 0:0:0.0,
2011-10-1 0:0:0.0,
2011-11-1 0:0:0.0,
2011-12-1 0:0:0.0,
2012-1-1 0:0:0.0,
2012-2-1 0:0:0.0,
2012-3-1 0:0:0.0,
2012-4-1 0:0:0.0,
2012-5-1 0:0:0.0,
2012-6-1 0:0:0.0,
2012-7-1 0:0:0.0,
2012-8-1 0:0:0.0,
2012-9-1 0:0:0.0,
2012-10-1 0:0:0.0,
2012-11-1 0:0:0.0,
2012-12-1 0:0:0.0,
2013-1-1 0:0:0.0,
2013-2-1 0:0:0.0,
2013-3-1 0:0:0.0,
2013-4-1 0:0:0.0,
2013-5-1 0:0:0.0,
2013-6-1 0:0:0.0,
2013-7-1 0:0:0.0,
2013-8-1 0:0:0.0,
2013-9-1 0:0:0.0,
2013-10-1 0:0:0.0,
2013-11-1 0:0:0.0,
2013-12-1 0:0:0.0,
2014-1-1 0:0:0.0]
"""

# 36 2014-1-1 0:0:0.0
print(len(time_s), time_s_dt[-1])
# [59829. 59860.]
print(time_s.getBounds()[-1])

# ~~~~~~~

# xCDAT
ds_xcdat = xc.open_dataset(fn1, decode_times=True)

ds_xcdat_s = ds_xcdat.sel(time=slice(start_time, end_time))

# 35 2013-12-01 00:00:00
print(len(ds_xcdat_s.time), ds_xcdat_s.time.values[-1])

# [cftime.DatetimeNoLeap(2013, 11, 1, 0, 0, 0, 0, has_year_zero=True)
# cftime.DatetimeNoLeap(2013, 12, 1, 0, 0, 0, 0, has_year_zero=True)]
print(ds_xcdat_s.time_bnds.values[-1])


# No slice
# ~~~~~~~
var_ns = ds_cdat2(var, time=(start_time, end_time))(squeeze=1)
time_ns = var_ns.getTime()
time_ns_dt = time_ns.asComponentTime()

"""
[2011-2-1 0:0:0.0,
2011-3-1 0:0:0.0,
2011-4-1 0:0:0.0,
2011-5-1 0:0:0.0,
2011-6-1 0:0:0.0,
2011-7-1 0:0:0.0,
2011-8-1 0:0:0.0,
2011-9-1 0:0:0.0,
2011-10-1 0:0:0.0,
2011-11-1 0:0:0.0,
2011-12-1 0:0:0.0,
2012-1-1 0:0:0.0,
2012-2-1 0:0:0.0,
2012-3-1 0:0:0.0,
2012-4-1 0:0:0.0,
2012-5-1 0:0:0.0,
2012-6-1 0:0:0.0,
2012-7-1 0:0:0.0,
2012-8-1 0:0:0.0,
2012-9-1 0:0:0.0,
2012-10-1 0:0:0.0,
2012-11-1 0:0:0.0,
2012-12-1 0:0:0.0,
2013-1-1 0:0:0.0,
2013-2-1 0:0:0.0,
2013-3-1 0:0:0.0,
2013-4-1 0:0:0.0,
2013-5-1 0:0:0.0,
2013-6-1 0:0:0.0,
2013-7-1 0:0:0.0,
2013-8-1 0:0:0.0,
2013-9-1 0:0:0.0,
2013-10-1 0:0:0.0,
2013-11-1 0:0:0.0,
2013-12-1 0:0:0.0]
"""

# 35 2013-12-1 0:0:0.0
print(len(time_ns), time_ns_dt[-1])
# [59799. 59829.]
print(time_ns.getBounds()[-1])
137 changes: 137 additions & 0 deletions auxiliary_tools/cdat_regression_testing/662-area-mean-time-series/debug/759_slice_flag_co.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
# %%
import cdms2
import xcdat as xc

# Parameters
# Time coordinates at the first day of the month (1850-01-01)
fn1 = "/global/cfs/cdirs/e3sm/e3sm_diags/test_model_data_for_acme_diags/time-series/E3SM_v1/FSNS_185001_201312.nc"
var = "FSNS"
start_time = "2011-01-01"
end_time = "2014-01-01"

# "co" - YYYY-01-01 +1 for the end year
slice_flag = "co"


ds_cdat = cdms2.open(fn1)

# With slice -- nothing changes because "co" allows the right side to be open,
# The actual end time (2013-12-01) is within the slice end time (2014-01-01).
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
var_s = ds_cdat(var, time=(start_time, end_time, slice_flag))(squeeze=1)

time_s = var_s.getTime()
time_s_dt = time_s.asComponentTime()

"""
[2011-1-1 0:0:0.0,
2011-2-1 0:0:0.0,
2011-3-1 0:0:0.0,
2011-4-1 0:0:0.0,
2011-5-1 0:0:0.0,
2011-6-1 0:0:0.0,
2011-7-1 0:0:0.0,
2011-8-1 0:0:0.0,
2011-9-1 0:0:0.0,
2011-10-1 0:0:0.0,
2011-11-1 0:0:0.0,
2011-12-1 0:0:0.0,
2012-1-1 0:0:0.0,
2012-2-1 0:0:0.0,
2012-3-1 0:0:0.0,
2012-4-1 0:0:0.0,
2012-5-1 0:0:0.0,
2012-6-1 0:0:0.0,
2012-7-1 0:0:0.0,
2012-8-1 0:0:0.0,
2012-9-1 0:0:0.0,
2012-10-1 0:0:0.0,
2012-11-1 0:0:0.0,
2012-12-1 0:0:0.0,
2013-1-1 0:0:0.0,
2013-2-1 0:0:0.0,
2013-3-1 0:0:0.0,
2013-4-1 0:0:0.0,
2013-5-1 0:0:0.0,
2013-6-1 0:0:0.0,
2013-7-1 0:0:0.0,
2013-8-1 0:0:0.0,
2013-9-1 0:0:0.0,
2013-10-1 0:0:0.0,
2013-11-1 0:0:0.0,
2013-12-1 0:0:0.0]
"""


# 36 2013-12-1 0:0:0.0
print(len(time_s), time_s_dt[-1])
# [59799. 59829.]
print(time_s.getBounds()[-1])


# ~~~~~~~

# xCDAT
ds_xcdat = xc.open_dataset(fn1, decode_times=True)

ds_xcdat_s = ds_xcdat.sel(time=slice(start_time, end_time))

# 35 2013-12-01 00:00:00
print(len(ds_xcdat_s.time), ds_xcdat_s.time.values[-1])

# [cftime.DatetimeNoLeap(2013, 11, 1, 0, 0, 0, 0, has_year_zero=True)
# cftime.DatetimeNoLeap(2013, 12, 1, 0, 0, 0, 0, has_year_zero=True)]
print(ds_xcdat_s.time_bnds.values[-1])


# No slice
# RESULT: Adds a coordinate to the end ("ccn" is default)
# ~~~~~~~
var_ns = ds_cdat(var, time=(start_time, end_time))(squeeze=1)
time_ns = var_ns.getTime()
time_ns_dt = time_ns.asComponentTime()

"""
[2011-1-1 0:0:0.0,
2011-2-1 0:0:0.0,
2011-3-1 0:0:0.0,
2011-4-1 0:0:0.0,
2011-5-1 0:0:0.0,
2011-6-1 0:0:0.0,
2011-7-1 0:0:0.0,
2011-8-1 0:0:0.0,
2011-9-1 0:0:0.0,
2011-10-1 0:0:0.0,
2011-11-1 0:0:0.0,
2011-12-1 0:0:0.0,
2012-1-1 0:0:0.0,
2012-2-1 0:0:0.0,
2012-3-1 0:0:0.0,
2012-4-1 0:0:0.0,
2012-5-1 0:0:0.0,
2012-6-1 0:0:0.0,
2012-7-1 0:0:0.0,
2012-8-1 0:0:0.0,
2012-9-1 0:0:0.0,
2012-10-1 0:0:0.0,
2012-11-1 0:0:0.0,
2012-12-1 0:0:0.0,
2013-1-1 0:0:0.0,
2013-2-1 0:0:0.0,
2013-3-1 0:0:0.0,
2013-4-1 0:0:0.0,
2013-5-1 0:0:0.0,
2013-6-1 0:0:0.0,
2013-7-1 0:0:0.0,
2013-8-1 0:0:0.0,
2013-9-1 0:0:0.0,
2013-10-1 0:0:0.0,
2013-11-1 0:0:0.0,
2013-12-1 0:0:0.0,
2014-1-1 0:0:0.0]
"""

# 37 2014-1-1 0:0:0.0
print(len(time_ns), time_ns_dt[-1])
# [59829. 59860.]
print(time_ns.getBounds()[-1])
Loading
Loading