From dda2c615f456cf90fd398fa81e5085d9aa80eb12 Mon Sep 17 00:00:00 2001 From: SarahAlidoost Date: Thu, 11 Feb 2021 16:56:20 +0100 Subject: [PATCH 01/12] remove lazy_regrid.py because now esmvalcore takes care of it --- .../diag_scripts/hydrology/lazy_regrid.py | 103 ------------------ 1 file changed, 103 deletions(-) delete mode 100644 esmvaltool/diag_scripts/hydrology/lazy_regrid.py diff --git a/esmvaltool/diag_scripts/hydrology/lazy_regrid.py b/esmvaltool/diag_scripts/hydrology/lazy_regrid.py deleted file mode 100644 index e62f5bf6c1..0000000000 --- a/esmvaltool/diag_scripts/hydrology/lazy_regrid.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Lazy regridding, because this is not supported by iris (yet). - -Iris issue requesting the feature: -https://github.com/SciTools/iris/issues/3700 -""" -import copy - -import iris -import numpy as np - -HORIZONTAL_SCHEMES = { - 'linear': iris.analysis.Linear(extrapolation_mode='mask'), - 'linear_extrapolate': - iris.analysis.Linear(extrapolation_mode='extrapolate'), - 'nearest': iris.analysis.Nearest(extrapolation_mode='mask'), - 'area_weighted': iris.analysis.AreaWeighted(), -} -"""Supported horizontal regridding schemes.""" - - -def _compute_chunks(src, tgt): - """Compute the chunk sizes needed to regrid src to tgt.""" - block_bytes = 50 * (1 << 20) # 50 MB block size - - if src.dtype == np.float32: - dtype_bytes = 4 # size of float32 in bytes - else: - dtype_bytes = 8 # size of float64 in bytes - - ntime = src.coord('time').shape[0] - tgt_nlat = tgt.coord('latitude').shape[0] - tgt_nlon = tgt.coord('longitude').shape[0] - - # Define blocks along the time dimension - min_nblocks = int(ntime * tgt_nlat * tgt_nlon * dtype_bytes / block_bytes) - min_nblocks = max(min_nblocks, 1) - timefull = ntime // min_nblocks - timepart = ntime % timefull - - nfullblocks = ntime // timefull - npartblocks = int(timepart > 0) - - time_chunks = (timefull, ) * nfullblocks + (timepart, ) * npartblocks - src_chunks = ( - time_chunks, - (src.coord('latitude').shape[0], ), - (src.coord('longitude').shape[0], ), - ) - tgt_chunks = ( - time_chunks, - (tgt_nlat, ), - (tgt_nlon, ), - ) - - return src_chunks, tgt_chunks - - -def _regrid_data(src, tgt, scheme): - """Regrid data from cube src onto grid of cube tgt.""" - src_chunks, tgt_chunks = _compute_chunks(src, tgt) - - # Define the block regrid function - if scheme not in HORIZONTAL_SCHEMES: - raise ValueError(f"Regridding scheme {scheme} not supported, " - f"choose from {HORIZONTAL_SCHEMES.keys()}.") - regridder = HORIZONTAL_SCHEMES[scheme].regridder(src, tgt) - - def regrid(block): - tlen = block.shape[0] - cube = src[:tlen].copy(block) - return regridder(cube).core_data() - - # Regrid - data = src.core_data().rechunk(src_chunks).map_blocks( - regrid, - dtype=src.dtype, - chunks=tgt_chunks, - ) - - return data - - -def lazy_regrid(src, tgt, scheme): - """Regrid cube src onto the grid of cube tgt.""" - data = _regrid_data(src, tgt, scheme) - - result = iris.cube.Cube(data) - result.metadata = copy.deepcopy(src.metadata) - - def copy_coords(src_coords, add_method): - for coord in src_coords: - dims = src.coord_dims(coord) - if coord == src.coord('longitude'): - coord = tgt.coord('longitude') - elif coord == src.coord('latitude'): - coord = tgt.coord('latitude') - result_coord = coord.copy() - add_method(result_coord, dims) - - copy_coords(src.dim_coords, result.add_dim_coord) - copy_coords(src.aux_coords, result.add_aux_coord) - - return result From 6269955bea4cbc5e0ade70897894720d6cb114ea Mon Sep 17 00:00:00 2001 From: SarahAlidoost Date: Thu, 11 Feb 2021 16:57:18 +0100 Subject: [PATCH 02/12] replace lazy_regrid with regrid from esmvalcore --- esmvaltool/diag_scripts/hydrology/wflow.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/esmvaltool/diag_scripts/hydrology/wflow.py b/esmvaltool/diag_scripts/hydrology/wflow.py index 58a7ddfd94..102aadf17c 100644 --- a/esmvaltool/diag_scripts/hydrology/wflow.py +++ b/esmvaltool/diag_scripts/hydrology/wflow.py @@ -2,12 +2,12 @@ import logging from pathlib import Path -import iris import numpy as np from osgeo import gdal +import iris +from esmvalcore.preprocessor import regrid from esmvaltool.diag_scripts.hydrology.derive_evspsblpot import debruin_pet -from esmvaltool.diag_scripts.hydrology.lazy_regrid import lazy_regrid from esmvaltool.diag_scripts.shared import (ProvenanceLogger, get_diagnostic_filename, group_metadata, run_diagnostic) @@ -97,7 +97,7 @@ def regrid_temperature(src_temp, src_height, target_height, scheme): src_slt = src_temp.copy(data=src_temp.core_data() + src_dtemp.core_data()) # Interpolate sea-level temperature to target grid - target_slt = lazy_regrid(src_slt, target_height, scheme) + target_slt = regrid(src_slt, target_height, scheme) # Convert sea-level temperature to new target elevation target_dtemp = lapse_rate_correction(target_height) @@ -216,7 +216,7 @@ def main(cfg): logger.info("Processing variable precipitation_flux") scheme = cfg['regrid'] - pr_dem = lazy_regrid(all_vars['pr'], dem, scheme) + pr_dem = regrid(all_vars['pr'], dem, scheme) logger.info("Processing variable temperature") tas_dem = regrid_temperature( @@ -229,12 +229,12 @@ def main(cfg): logger.info("Processing variable potential evapotranspiration") if 'evspsblpot' in all_vars: pet = all_vars['evspsblpot'] - pet_dem = lazy_regrid(pet, dem, scheme) + pet_dem = regrid(pet, dem, scheme) else: logger.info("Potential evapotransporation not available, deriving") - psl_dem = lazy_regrid(all_vars['psl'], dem, scheme) - rsds_dem = lazy_regrid(all_vars['rsds'], dem, scheme) - rsdt_dem = lazy_regrid(all_vars['rsdt'], dem, scheme) + psl_dem = regrid(all_vars['psl'], dem, scheme) + rsds_dem = regrid(all_vars['rsds'], dem, scheme) + rsdt_dem = regrid(all_vars['rsdt'], dem, scheme) pet_dem = debruin_pet( tas=tas_dem, psl=psl_dem, From d75d9cfff6ecfb901820d30bd4c7b7c6ef3fd8b8 Mon Sep 17 00:00:00 2001 From: SarahAlidoost Date: Thu, 11 Feb 2021 16:57:57 +0100 Subject: [PATCH 03/12] use correct file format for wflow dem file --- esmvaltool/recipes/hydrology/recipe_wflow.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/esmvaltool/recipes/hydrology/recipe_wflow.yml b/esmvaltool/recipes/hydrology/recipe_wflow.yml index f568901a67..7b33a6b321 100644 --- a/esmvaltool/recipes/hydrology/recipe_wflow.yml +++ b/esmvaltool/recipes/hydrology/recipe_wflow.yml @@ -4,17 +4,17 @@ documentation: description: | Pre-processes climate data for the WFlow hydrological model. - + authors: - kalverla_peter - camphuijsen_jaro - alidoost_sarah - aerts_jerom - andela_bouwe - + projects: - ewatercycle - + references: - acknow_project @@ -53,5 +53,5 @@ diagnostics: script: script: hydrology/wflow.py basin: Meuse - dem_file: 'wflow/wflow_dem_Meuse.nc' + dem_file: 'wflow_parameterset/meuse/staticmaps/wflow_dem.map' regrid: area_weighted From 0c4e78ac887bc3ece16232fa60803ac58e80623f Mon Sep 17 00:00:00 2001 From: SarahAlidoost Date: Fri, 12 Feb 2021 15:58:40 +0100 Subject: [PATCH 04/12] sort import statements --- esmvaltool/diag_scripts/hydrology/wflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvaltool/diag_scripts/hydrology/wflow.py b/esmvaltool/diag_scripts/hydrology/wflow.py index 102aadf17c..24f604b5d5 100644 --- a/esmvaltool/diag_scripts/hydrology/wflow.py +++ b/esmvaltool/diag_scripts/hydrology/wflow.py @@ -4,9 +4,9 @@ import numpy as np from osgeo import gdal +from esmvalcore.preprocessor import regrid import iris -from esmvalcore.preprocessor import regrid from esmvaltool.diag_scripts.hydrology.derive_evspsblpot import debruin_pet from esmvaltool.diag_scripts.shared import (ProvenanceLogger, get_diagnostic_filename, From 4a243920c73c538c527967289e56891ba89df4b5 Mon Sep 17 00:00:00 2001 From: SarahAlidoost Date: Fri, 12 Feb 2021 15:58:58 +0100 Subject: [PATCH 05/12] change time periods --- esmvaltool/recipes/hydrology/recipe_wflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esmvaltool/recipes/hydrology/recipe_wflow.yml b/esmvaltool/recipes/hydrology/recipe_wflow.yml index 7b33a6b321..dc9bb148a6 100644 --- a/esmvaltool/recipes/hydrology/recipe_wflow.yml +++ b/esmvaltool/recipes/hydrology/recipe_wflow.yml @@ -40,7 +40,7 @@ diagnostics: mip: day preprocessor: rough_cutout start_year: 1990 - end_year: 1990 + end_year: 2001 pr: *daily_var # evspsblpot: # doesn't exist for ERA-Interim. # Reconstruct evspsblpot using: From 64e5f574c9f9e45a15a4662417e59fa27468a99b Mon Sep 17 00:00:00 2001 From: SarahAlidoost Date: Mon, 15 Feb 2021 09:19:58 +0100 Subject: [PATCH 06/12] revise years in hype recipe --- esmvaltool/recipes/hydrology/recipe_hype.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esmvaltool/recipes/hydrology/recipe_hype.yml b/esmvaltool/recipes/hydrology/recipe_hype.yml index 9a500938ed..02f1ccba8a 100644 --- a/esmvaltool/recipes/hydrology/recipe_hype.yml +++ b/esmvaltool/recipes/hydrology/recipe_hype.yml @@ -43,8 +43,8 @@ diagnostics: variables: tas: &var mip: day - start_year: 1979 - end_year: 1979 + start_year: 1990 + end_year: 2018 preprocessor: temperature tasmin: *var tasmax: *var From 33bf3db8e68417b66c709a3826ca8eca3d7e482d Mon Sep 17 00:00:00 2001 From: SarahAlidoost Date: Tue, 16 Feb 2021 17:38:35 +0100 Subject: [PATCH 07/12] bring back lazy_regrid and refactor it --- .../diag_scripts/hydrology/lazy_regrid.py | 60 +++++++++++++++++++ esmvaltool/diag_scripts/hydrology/wflow.py | 14 ++--- 2 files changed, 67 insertions(+), 7 deletions(-) create mode 100644 esmvaltool/diag_scripts/hydrology/lazy_regrid.py diff --git a/esmvaltool/diag_scripts/hydrology/lazy_regrid.py b/esmvaltool/diag_scripts/hydrology/lazy_regrid.py new file mode 100644 index 0000000000..698b6ad6d0 --- /dev/null +++ b/esmvaltool/diag_scripts/hydrology/lazy_regrid.py @@ -0,0 +1,60 @@ +"""Lazy regridding, because this is not supported by iris (yet). + +Iris issue requesting the feature: +https://github.com/SciTools/iris/issues/3808 +""" +import numpy as np +import iris + +HORIZONTAL_SCHEMES = { + 'linear': iris.analysis.Linear(extrapolation_mode='mask'), + 'linear_extrapolate': + iris.analysis.Linear(extrapolation_mode='extrapolate'), + 'nearest': iris.analysis.Nearest(extrapolation_mode='mask'), + 'area_weighted': iris.analysis.AreaWeighted(), +} +"""Supported horizontal regridding schemes.""" + + +def _compute_chunks(src, tgt): + """Compute the chunk sizes needed to regrid src to tgt.""" + block_bytes = 50 * (1 << 20) # 50 MB block size + + if src.dtype == np.float32: + dtype_bytes = 4 # size of float32 in bytes + else: + dtype_bytes = 8 # size of float64 in bytes + + ntime = src.coord('time').shape[0] + tgt_nlat = tgt.coord('latitude').shape[0] + tgt_nlon = tgt.coord('longitude').shape[0] + + # Define blocks along the time dimension + min_nblocks = int(ntime * tgt_nlat * tgt_nlon * dtype_bytes / block_bytes) + min_nblocks = max(min_nblocks, 1) + timefull = ntime // min_nblocks + timepart = ntime % timefull + + nfullblocks = ntime // timefull + npartblocks = int(timepart > 0) + + time_chunks = (timefull, ) * nfullblocks + (timepart, ) * npartblocks + src_chunks = ( + time_chunks, + (src.coord('latitude').shape[0], ), + (src.coord('longitude').shape[0], ), + ) + + return src_chunks + + +def lazy_regrid(src, tgt, scheme): + """Regrid cube src onto the grid of cube tgt.""" + src_chunks = _compute_chunks(src, tgt) + + if scheme not in HORIZONTAL_SCHEMES: + raise ValueError(f"Regridding scheme {scheme} not supported, " + f"choose from {HORIZONTAL_SCHEMES.keys()}.") + regridder = HORIZONTAL_SCHEMES[scheme].regridder(src, tgt) + src.data = src.lazy_data().rechunk(src_chunks) + return regridder(src) diff --git a/esmvaltool/diag_scripts/hydrology/wflow.py b/esmvaltool/diag_scripts/hydrology/wflow.py index 24f604b5d5..c0d5c73ead 100644 --- a/esmvaltool/diag_scripts/hydrology/wflow.py +++ b/esmvaltool/diag_scripts/hydrology/wflow.py @@ -4,10 +4,10 @@ import numpy as np from osgeo import gdal -from esmvalcore.preprocessor import regrid import iris from esmvaltool.diag_scripts.hydrology.derive_evspsblpot import debruin_pet +from esmvaltool.diag_scripts.hydrology.lazy_regrid import lazy_regrid from esmvaltool.diag_scripts.shared import (ProvenanceLogger, get_diagnostic_filename, group_metadata, run_diagnostic) @@ -97,7 +97,7 @@ def regrid_temperature(src_temp, src_height, target_height, scheme): src_slt = src_temp.copy(data=src_temp.core_data() + src_dtemp.core_data()) # Interpolate sea-level temperature to target grid - target_slt = regrid(src_slt, target_height, scheme) + target_slt = lazy_regrid(src_slt, target_height, scheme) # Convert sea-level temperature to new target elevation target_dtemp = lapse_rate_correction(target_height) @@ -216,7 +216,7 @@ def main(cfg): logger.info("Processing variable precipitation_flux") scheme = cfg['regrid'] - pr_dem = regrid(all_vars['pr'], dem, scheme) + pr_dem = lazy_regrid(all_vars['pr'], dem, scheme) logger.info("Processing variable temperature") tas_dem = regrid_temperature( @@ -229,12 +229,12 @@ def main(cfg): logger.info("Processing variable potential evapotranspiration") if 'evspsblpot' in all_vars: pet = all_vars['evspsblpot'] - pet_dem = regrid(pet, dem, scheme) + pet_dem = lazy_regrid(pet, dem, scheme) else: logger.info("Potential evapotransporation not available, deriving") - psl_dem = regrid(all_vars['psl'], dem, scheme) - rsds_dem = regrid(all_vars['rsds'], dem, scheme) - rsdt_dem = regrid(all_vars['rsdt'], dem, scheme) + psl_dem = lazy_regrid(all_vars['psl'], dem, scheme) + rsds_dem = lazy_regrid(all_vars['rsds'], dem, scheme) + rsdt_dem = lazy_regrid(all_vars['rsdt'], dem, scheme) pet_dem = debruin_pet( tas=tas_dem, psl=psl_dem, From b231f037e807d91612b25f1e30958b5576e11d09 Mon Sep 17 00:00:00 2001 From: SarahAlidoost Date: Tue, 16 Feb 2021 17:41:15 +0100 Subject: [PATCH 08/12] remove recipe_hype from this PR --- esmvaltool/recipes/hydrology/recipe_hype.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esmvaltool/recipes/hydrology/recipe_hype.yml b/esmvaltool/recipes/hydrology/recipe_hype.yml index 02f1ccba8a..9a500938ed 100644 --- a/esmvaltool/recipes/hydrology/recipe_hype.yml +++ b/esmvaltool/recipes/hydrology/recipe_hype.yml @@ -43,8 +43,8 @@ diagnostics: variables: tas: &var mip: day - start_year: 1990 - end_year: 2018 + start_year: 1979 + end_year: 1979 preprocessor: temperature tasmin: *var tasmax: *var From 19cb0fee9b928d2bfb205626204a21cf6d4fe95e Mon Sep 17 00:00:00 2001 From: SarahAlidoost Date: Wed, 17 Feb 2021 14:11:22 +0100 Subject: [PATCH 09/12] rename function lazy_regrid to rechunk_and_regrid --- esmvaltool/diag_scripts/hydrology/wflow.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/esmvaltool/diag_scripts/hydrology/wflow.py b/esmvaltool/diag_scripts/hydrology/wflow.py index c0d5c73ead..4cb5af1d58 100644 --- a/esmvaltool/diag_scripts/hydrology/wflow.py +++ b/esmvaltool/diag_scripts/hydrology/wflow.py @@ -7,7 +7,7 @@ import iris from esmvaltool.diag_scripts.hydrology.derive_evspsblpot import debruin_pet -from esmvaltool.diag_scripts.hydrology.lazy_regrid import lazy_regrid +from esmvaltool.diag_scripts.hydrology.lazy_regrid import rechunk_and_regrid from esmvaltool.diag_scripts.shared import (ProvenanceLogger, get_diagnostic_filename, group_metadata, run_diagnostic) @@ -97,7 +97,7 @@ def regrid_temperature(src_temp, src_height, target_height, scheme): src_slt = src_temp.copy(data=src_temp.core_data() + src_dtemp.core_data()) # Interpolate sea-level temperature to target grid - target_slt = lazy_regrid(src_slt, target_height, scheme) + target_slt = rechunk_and_regrid(src_slt, target_height, scheme) # Convert sea-level temperature to new target elevation target_dtemp = lapse_rate_correction(target_height) @@ -216,7 +216,7 @@ def main(cfg): logger.info("Processing variable precipitation_flux") scheme = cfg['regrid'] - pr_dem = lazy_regrid(all_vars['pr'], dem, scheme) + pr_dem = rechunk_and_regrid(all_vars['pr'], dem, scheme) logger.info("Processing variable temperature") tas_dem = regrid_temperature( @@ -229,12 +229,12 @@ def main(cfg): logger.info("Processing variable potential evapotranspiration") if 'evspsblpot' in all_vars: pet = all_vars['evspsblpot'] - pet_dem = lazy_regrid(pet, dem, scheme) + pet_dem = rechunk_and_regrid(pet, dem, scheme) else: logger.info("Potential evapotransporation not available, deriving") - psl_dem = lazy_regrid(all_vars['psl'], dem, scheme) - rsds_dem = lazy_regrid(all_vars['rsds'], dem, scheme) - rsdt_dem = lazy_regrid(all_vars['rsdt'], dem, scheme) + psl_dem = rechunk_and_regrid(all_vars['psl'], dem, scheme) + rsds_dem = rechunk_and_regrid(all_vars['rsds'], dem, scheme) + rsdt_dem = rechunk_and_regrid(all_vars['rsdt'], dem, scheme) pet_dem = debruin_pet( tas=tas_dem, psl=psl_dem, From 427c473106e71afcebaccf5f1088d71cf61ad54f Mon Sep 17 00:00:00 2001 From: SarahAlidoost Date: Wed, 17 Feb 2021 14:11:56 +0100 Subject: [PATCH 10/12] replace iris with esmvalcore.processor.regrid --- .../diag_scripts/hydrology/lazy_regrid.py | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/esmvaltool/diag_scripts/hydrology/lazy_regrid.py b/esmvaltool/diag_scripts/hydrology/lazy_regrid.py index 698b6ad6d0..0f737cc77f 100644 --- a/esmvaltool/diag_scripts/hydrology/lazy_regrid.py +++ b/esmvaltool/diag_scripts/hydrology/lazy_regrid.py @@ -4,15 +4,14 @@ https://github.com/SciTools/iris/issues/3808 """ import numpy as np -import iris - -HORIZONTAL_SCHEMES = { - 'linear': iris.analysis.Linear(extrapolation_mode='mask'), - 'linear_extrapolate': - iris.analysis.Linear(extrapolation_mode='extrapolate'), - 'nearest': iris.analysis.Nearest(extrapolation_mode='mask'), - 'area_weighted': iris.analysis.AreaWeighted(), -} +from esmvalcore.preprocessor import regrid + +HORIZONTAL_SCHEMES = [ + 'linear', + 'linear_extrapolate', + 'nearest', + 'area_weighted', +] """Supported horizontal regridding schemes.""" @@ -48,13 +47,12 @@ def _compute_chunks(src, tgt): return src_chunks -def lazy_regrid(src, tgt, scheme): +def rechunk_and_regrid(src, tgt, scheme): """Regrid cube src onto the grid of cube tgt.""" src_chunks = _compute_chunks(src, tgt) if scheme not in HORIZONTAL_SCHEMES: raise ValueError(f"Regridding scheme {scheme} not supported, " - f"choose from {HORIZONTAL_SCHEMES.keys()}.") - regridder = HORIZONTAL_SCHEMES[scheme].regridder(src, tgt) + f"choose from {HORIZONTAL_SCHEMES}.") src.data = src.lazy_data().rechunk(src_chunks) - return regridder(src) + return regrid(src, tgt, scheme) From 0f90cb0756050731a8e191eacfa66c00f25585e2 Mon Sep 17 00:00:00 2001 From: SarahAlidoost Date: Thu, 18 Feb 2021 09:34:32 +0100 Subject: [PATCH 11/12] rename lazy_regrid to compute_chunks, refactor the script and wflow diagnostics --- .../{lazy_regrid.py => compute_chunks.py} | 30 ++++--------------- esmvaltool/diag_scripts/hydrology/wflow.py | 10 ++++++- 2 files changed, 15 insertions(+), 25 deletions(-) rename esmvaltool/diag_scripts/hydrology/{lazy_regrid.py => compute_chunks.py} (58%) diff --git a/esmvaltool/diag_scripts/hydrology/lazy_regrid.py b/esmvaltool/diag_scripts/hydrology/compute_chunks.py similarity index 58% rename from esmvaltool/diag_scripts/hydrology/lazy_regrid.py rename to esmvaltool/diag_scripts/hydrology/compute_chunks.py index 0f737cc77f..3b35646a3c 100644 --- a/esmvaltool/diag_scripts/hydrology/lazy_regrid.py +++ b/esmvaltool/diag_scripts/hydrology/compute_chunks.py @@ -1,21 +1,15 @@ -"""Lazy regridding, because this is not supported by iris (yet). +"""Re-chunk the time dimension, to be used by the regrid processor. -Iris issue requesting the feature: +For large cubes, regridding to a high resolution grid increases +the size of the data. To solve this, we re-chunk the time dimension. + +Related iris issue: https://github.com/SciTools/iris/issues/3808 """ import numpy as np -from esmvalcore.preprocessor import regrid - -HORIZONTAL_SCHEMES = [ - 'linear', - 'linear_extrapolate', - 'nearest', - 'area_weighted', -] -"""Supported horizontal regridding schemes.""" -def _compute_chunks(src, tgt): +def compute_chunks(src, tgt): """Compute the chunk sizes needed to regrid src to tgt.""" block_bytes = 50 * (1 << 20) # 50 MB block size @@ -43,16 +37,4 @@ def _compute_chunks(src, tgt): (src.coord('latitude').shape[0], ), (src.coord('longitude').shape[0], ), ) - return src_chunks - - -def rechunk_and_regrid(src, tgt, scheme): - """Regrid cube src onto the grid of cube tgt.""" - src_chunks = _compute_chunks(src, tgt) - - if scheme not in HORIZONTAL_SCHEMES: - raise ValueError(f"Regridding scheme {scheme} not supported, " - f"choose from {HORIZONTAL_SCHEMES}.") - src.data = src.lazy_data().rechunk(src_chunks) - return regrid(src, tgt, scheme) diff --git a/esmvaltool/diag_scripts/hydrology/wflow.py b/esmvaltool/diag_scripts/hydrology/wflow.py index 4cb5af1d58..d3de40151d 100644 --- a/esmvaltool/diag_scripts/hydrology/wflow.py +++ b/esmvaltool/diag_scripts/hydrology/wflow.py @@ -6,8 +6,9 @@ from osgeo import gdal import iris +from esmvalcore.preprocessor import regrid from esmvaltool.diag_scripts.hydrology.derive_evspsblpot import debruin_pet -from esmvaltool.diag_scripts.hydrology.lazy_regrid import rechunk_and_regrid +from esmvaltool.diag_scripts.hydrology.compute_chunks import compute_chunks from esmvaltool.diag_scripts.shared import (ProvenanceLogger, get_diagnostic_filename, group_metadata, run_diagnostic) @@ -39,6 +40,13 @@ def create_provenance_record(): return record +def rechunk_and_regrid(src, tgt, scheme): + """Rechunk cube src and regrid it onto the grid of cube tgt.""" + src_chunks = compute_chunks(src, tgt) + src.data = src.lazy_data().rechunk(src_chunks) + return regrid(src, tgt, scheme) + + def get_input_cubes(metadata): """Create a dict with all (preprocessed) input files.""" provenance = create_provenance_record() From d32e806b8cf5ff0e9ef593bd0e696453db0b82d7 Mon Sep 17 00:00:00 2001 From: SarahAlidoost Date: Thu, 18 Feb 2021 09:41:50 +0100 Subject: [PATCH 12/12] revise the docstring in compute_chunks.py --- esmvaltool/diag_scripts/hydrology/compute_chunks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esmvaltool/diag_scripts/hydrology/compute_chunks.py b/esmvaltool/diag_scripts/hydrology/compute_chunks.py index 3b35646a3c..56b3f7b43d 100644 --- a/esmvaltool/diag_scripts/hydrology/compute_chunks.py +++ b/esmvaltool/diag_scripts/hydrology/compute_chunks.py @@ -1,7 +1,7 @@ """Re-chunk the time dimension, to be used by the regrid processor. -For large cubes, regridding to a high resolution grid increases -the size of the data. To solve this, we re-chunk the time dimension. +For large cubes, regridding to a high resolution grid increases the size +of the data. To reduce memory use, we re-chunk the time dimension. Related iris issue: https://github.com/SciTools/iris/issues/3808