From 69496dfdc96bcde318174cd7b404dbd0bb143895 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Fri, 16 Dec 2022 22:23:49 +0000 Subject: [PATCH 01/11] Initial recipe sketch (not working atm) --- PyQG_subgrid_forcing_Ross2022/meta.yaml | 0 PyQG_subgrid_forcing_Ross2022/recipe.py | 27 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 PyQG_subgrid_forcing_Ross2022/meta.yaml create mode 100644 PyQG_subgrid_forcing_Ross2022/recipe.py diff --git a/PyQG_subgrid_forcing_Ross2022/meta.yaml b/PyQG_subgrid_forcing_Ross2022/meta.yaml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/PyQG_subgrid_forcing_Ross2022/recipe.py b/PyQG_subgrid_forcing_Ross2022/recipe.py new file mode 100644 index 0000000000..b8a2b2eca7 --- /dev/null +++ b/PyQG_subgrid_forcing_Ross2022/recipe.py @@ -0,0 +1,27 @@ +import pangeo_forge_recipes +from pangeo_forge_recipes.patterns import pattern_from_file_sequence +from pangeo_forge_recipes.recipes import XarrayZarrRecipe + +# from pangeo_forge_recipes.recipes import setup_logging +# setup_logging() + +# key: file name, value: globus address, time chunk size +file_dict = { + "eddy/high_res.zarr": ("https://g-402b74.00888.8540.data.globus.org/eddy/high_res.zarr", 11), + "eddy/low_res.zarr": ("https://g-402b74.00888.8540.data.globus.org/eddy/low_res.zarr", 87), + "eddy/forcing1.zarr": ("https://g-402b74.00888.8540.data.globus.org/eddy/forcing1.zarr", 1), + "eddy/forcing2.zarr": ("https://g-402b74.00888.8540.data.globus.org/eddy/forcing2.zarr", 1), + "eddy/forcing3.zarr": ("https://g-402b74.00888.8540.data.globus.org/eddy/forcing3.zarr", 1), + "jet/high_res.zarr": ("https://g-402b74.00888.8540.data.globus.org/jet/high_res.zarr", 11), + "jet/low_res.zarr": ("https://g-402b74.00888.8540.data.globus.org/jet/low_res.zarr", 87), + "jet/forcing1.zarr": ("https://g-402b74.00888.8540.data.globus.org/jet/forcing1.zarr", 1), + "jet/forcing2.zarr": ("https://g-402b74.00888.8540.data.globus.org/jet/forcing2.zarr", 1), + "jet/forcing3.zarr": ("https://g-402b74.00888.8540.data.globus.org/jet/forcing3.zarr", 1) +} + + +pattern = lambda v: pattern_from_file_sequence([v[0]], concat_dim="time", file_type="unknown") +xr2zarr = lambda v: XarrayZarrRecipe(pattern(v), target_chunks={"time": v[1]}, cache_inputs=False, xarray_open_kwargs={"engine": "zarr"}) +recipes = {k: xr2zarr(v) for k, v in mydict.items()} + +recipes["eddy/lores.zarr"].to_function()() From 02c8784114b94822d30bfbfc77bfff6568c357ac Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Fri, 16 Dec 2022 22:35:43 +0000 Subject: [PATCH 02/11] Fix dumb mistakes in recipe.py (not working, but for the right reason) --- PyQG_subgrid_forcing_Ross2022/recipe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PyQG_subgrid_forcing_Ross2022/recipe.py b/PyQG_subgrid_forcing_Ross2022/recipe.py index b8a2b2eca7..d5711171a7 100644 --- a/PyQG_subgrid_forcing_Ross2022/recipe.py +++ b/PyQG_subgrid_forcing_Ross2022/recipe.py @@ -22,6 +22,6 @@ pattern = lambda v: pattern_from_file_sequence([v[0]], concat_dim="time", file_type="unknown") xr2zarr = lambda v: XarrayZarrRecipe(pattern(v), target_chunks={"time": v[1]}, cache_inputs=False, xarray_open_kwargs={"engine": "zarr"}) -recipes = {k: xr2zarr(v) for k, v in mydict.items()} +recipes = {k: xr2zarr(v) for k, v in file_dict.items()} -recipes["eddy/lores.zarr"].to_function()() +[recipe.to_function()() for recipe in recipes.values()] # probably not idiomatic, plz fix From 5bfd20a3e573513f77192b03d4025809ea9f3f43 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Tue, 20 Dec 2022 15:44:47 +0000 Subject: [PATCH 03/11] partial metadata --- PyQG_subgrid_forcing_Ross2022/meta.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/PyQG_subgrid_forcing_Ross2022/meta.yaml b/PyQG_subgrid_forcing_Ross2022/meta.yaml index e69de29bb2..aaf0cc4238 100644 --- a/PyQG_subgrid_forcing_Ross2022/meta.yaml +++ b/PyQG_subgrid_forcing_Ross2022/meta.yaml @@ -0,0 +1,10 @@ +title: 'PyQG Subgrid Forcing (Ross et al., 2022)' +description: 'Data from "Benchmarking of machine learning ocean subgrid parameterizations in an idealized model" by Ross et al., 2022' +recipes: + - id: PyQG_Ross2022 + object: 'recipe:recipe' +maintainers: + - name: 'Christopher Dupuis' + github: cmdupuis3 +bakery: + id: 'pangeo-ldeo-nsf-earthcube' \ No newline at end of file From feb97a5c329bf47432ad8194edcbfa25b4ce6178 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 4 Jan 2023 23:37:56 +0000 Subject: [PATCH 04/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- PyQG_subgrid_forcing_Ross2022/meta.yaml | 2 +- PyQG_subgrid_forcing_Ross2022/recipe.py | 31 ++++++++++++++----------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/PyQG_subgrid_forcing_Ross2022/meta.yaml b/PyQG_subgrid_forcing_Ross2022/meta.yaml index aaf0cc4238..4deb84ace1 100644 --- a/PyQG_subgrid_forcing_Ross2022/meta.yaml +++ b/PyQG_subgrid_forcing_Ross2022/meta.yaml @@ -7,4 +7,4 @@ maintainers: - name: 'Christopher Dupuis' github: cmdupuis3 bakery: - id: 'pangeo-ldeo-nsf-earthcube' \ No newline at end of file + id: 'pangeo-ldeo-nsf-earthcube' diff --git a/PyQG_subgrid_forcing_Ross2022/recipe.py b/PyQG_subgrid_forcing_Ross2022/recipe.py index d5711171a7..8426bc5030 100644 --- a/PyQG_subgrid_forcing_Ross2022/recipe.py +++ b/PyQG_subgrid_forcing_Ross2022/recipe.py @@ -7,21 +7,26 @@ # key: file name, value: globus address, time chunk size file_dict = { - "eddy/high_res.zarr": ("https://g-402b74.00888.8540.data.globus.org/eddy/high_res.zarr", 11), - "eddy/low_res.zarr": ("https://g-402b74.00888.8540.data.globus.org/eddy/low_res.zarr", 87), - "eddy/forcing1.zarr": ("https://g-402b74.00888.8540.data.globus.org/eddy/forcing1.zarr", 1), - "eddy/forcing2.zarr": ("https://g-402b74.00888.8540.data.globus.org/eddy/forcing2.zarr", 1), - "eddy/forcing3.zarr": ("https://g-402b74.00888.8540.data.globus.org/eddy/forcing3.zarr", 1), - "jet/high_res.zarr": ("https://g-402b74.00888.8540.data.globus.org/jet/high_res.zarr", 11), - "jet/low_res.zarr": ("https://g-402b74.00888.8540.data.globus.org/jet/low_res.zarr", 87), - "jet/forcing1.zarr": ("https://g-402b74.00888.8540.data.globus.org/jet/forcing1.zarr", 1), - "jet/forcing2.zarr": ("https://g-402b74.00888.8540.data.globus.org/jet/forcing2.zarr", 1), - "jet/forcing3.zarr": ("https://g-402b74.00888.8540.data.globus.org/jet/forcing3.zarr", 1) + 'eddy/high_res.zarr': ('https://g-402b74.00888.8540.data.globus.org/eddy/high_res.zarr', 11), + 'eddy/low_res.zarr': ('https://g-402b74.00888.8540.data.globus.org/eddy/low_res.zarr', 87), + 'eddy/forcing1.zarr': ('https://g-402b74.00888.8540.data.globus.org/eddy/forcing1.zarr', 1), + 'eddy/forcing2.zarr': ('https://g-402b74.00888.8540.data.globus.org/eddy/forcing2.zarr', 1), + 'eddy/forcing3.zarr': ('https://g-402b74.00888.8540.data.globus.org/eddy/forcing3.zarr', 1), + 'jet/high_res.zarr': ('https://g-402b74.00888.8540.data.globus.org/jet/high_res.zarr', 11), + 'jet/low_res.zarr': ('https://g-402b74.00888.8540.data.globus.org/jet/low_res.zarr', 87), + 'jet/forcing1.zarr': ('https://g-402b74.00888.8540.data.globus.org/jet/forcing1.zarr', 1), + 'jet/forcing2.zarr': ('https://g-402b74.00888.8540.data.globus.org/jet/forcing2.zarr', 1), + 'jet/forcing3.zarr': ('https://g-402b74.00888.8540.data.globus.org/jet/forcing3.zarr', 1), } -pattern = lambda v: pattern_from_file_sequence([v[0]], concat_dim="time", file_type="unknown") -xr2zarr = lambda v: XarrayZarrRecipe(pattern(v), target_chunks={"time": v[1]}, cache_inputs=False, xarray_open_kwargs={"engine": "zarr"}) +pattern = lambda v: pattern_from_file_sequence([v[0]], concat_dim='time', file_type='unknown') +xr2zarr = lambda v: XarrayZarrRecipe( + pattern(v), + target_chunks={'time': v[1]}, + cache_inputs=False, + xarray_open_kwargs={'engine': 'zarr'}, +) recipes = {k: xr2zarr(v) for k, v in file_dict.items()} -[recipe.to_function()() for recipe in recipes.values()] # probably not idiomatic, plz fix +[recipe.to_function()() for recipe in recipes.values()] # probably not idiomatic, plz fix From de89306b21bf2e114de0e6225329e53dbe9d8bba Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Thu, 5 Jan 2023 17:33:51 +0000 Subject: [PATCH 05/11] PyQG: trying to make pre-commit happy --- PyQG_subgrid_forcing_Ross2022/recipe.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/PyQG_subgrid_forcing_Ross2022/recipe.py b/PyQG_subgrid_forcing_Ross2022/recipe.py index 8426bc5030..318231dc3f 100644 --- a/PyQG_subgrid_forcing_Ross2022/recipe.py +++ b/PyQG_subgrid_forcing_Ross2022/recipe.py @@ -1,4 +1,4 @@ -import pangeo_forge_recipes + from pangeo_forge_recipes.patterns import pattern_from_file_sequence from pangeo_forge_recipes.recipes import XarrayZarrRecipe @@ -20,13 +20,17 @@ } -pattern = lambda v: pattern_from_file_sequence([v[0]], concat_dim='time', file_type='unknown') -xr2zarr = lambda v: XarrayZarrRecipe( - pattern(v), - target_chunks={'time': v[1]}, - cache_inputs=False, - xarray_open_kwargs={'engine': 'zarr'}, -) +def pattern(v): + return pattern_from_file_sequence([v[0]], concat_dim='time', file_type='unknown') + +def xr2zarr(v): + return XarrayZarrRecipe( + pattern(v), + target_chunks={'time': v[1]}, + cache_inputs=False, + xarray_open_kwargs={'engine': 'zarr'}, + ) + recipes = {k: xr2zarr(v) for k, v in file_dict.items()} [recipe.to_function()() for recipe in recipes.values()] # probably not idiomatic, plz fix From cd5b16d56f012731389902f6f20a23db1014d725 Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Thu, 5 Jan 2023 17:37:37 +0000 Subject: [PATCH 06/11] PyQG: pre-commit whitespace --- PyQG_subgrid_forcing_Ross2022/recipe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyQG_subgrid_forcing_Ross2022/recipe.py b/PyQG_subgrid_forcing_Ross2022/recipe.py index 318231dc3f..6a5f692c10 100644 --- a/PyQG_subgrid_forcing_Ross2022/recipe.py +++ b/PyQG_subgrid_forcing_Ross2022/recipe.py @@ -20,7 +20,7 @@ } -def pattern(v): +def pattern(v): return pattern_from_file_sequence([v[0]], concat_dim='time', file_type='unknown') def xr2zarr(v): From 21af5806dacfa9cd6c653852f303d0a5f16a3a45 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 10 Jan 2023 00:09:32 +0000 Subject: [PATCH 07/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- PyQG_subgrid_forcing_Ross2022/recipe.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/PyQG_subgrid_forcing_Ross2022/recipe.py b/PyQG_subgrid_forcing_Ross2022/recipe.py index 6a5f692c10..423065d60d 100644 --- a/PyQG_subgrid_forcing_Ross2022/recipe.py +++ b/PyQG_subgrid_forcing_Ross2022/recipe.py @@ -1,4 +1,3 @@ - from pangeo_forge_recipes.patterns import pattern_from_file_sequence from pangeo_forge_recipes.recipes import XarrayZarrRecipe @@ -23,6 +22,7 @@ def pattern(v): return pattern_from_file_sequence([v[0]], concat_dim='time', file_type='unknown') + def xr2zarr(v): return XarrayZarrRecipe( pattern(v), @@ -31,6 +31,7 @@ def xr2zarr(v): xarray_open_kwargs={'engine': 'zarr'}, ) + recipes = {k: xr2zarr(v) for k, v in file_dict.items()} [recipe.to_function()() for recipe in recipes.values()] # probably not idiomatic, plz fix From 9e636cbc7f74a7355f13a6e7644bbf31c379eb7b Mon Sep 17 00:00:00 2001 From: cmdupuis3 Date: Tue, 10 Jan 2023 00:30:05 +0000 Subject: [PATCH 08/11] move PyQG recipe to correct place --- .../PyQG_subgrid_forcing_Ross2022}/meta.yaml | 0 .../PyQG_subgrid_forcing_Ross2022}/recipe.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {PyQG_subgrid_forcing_Ross2022 => recipes/PyQG_subgrid_forcing_Ross2022}/meta.yaml (100%) rename {PyQG_subgrid_forcing_Ross2022 => recipes/PyQG_subgrid_forcing_Ross2022}/recipe.py (100%) diff --git a/PyQG_subgrid_forcing_Ross2022/meta.yaml b/recipes/PyQG_subgrid_forcing_Ross2022/meta.yaml similarity index 100% rename from PyQG_subgrid_forcing_Ross2022/meta.yaml rename to recipes/PyQG_subgrid_forcing_Ross2022/meta.yaml diff --git a/PyQG_subgrid_forcing_Ross2022/recipe.py b/recipes/PyQG_subgrid_forcing_Ross2022/recipe.py similarity index 100% rename from PyQG_subgrid_forcing_Ross2022/recipe.py rename to recipes/PyQG_subgrid_forcing_Ross2022/recipe.py From 009be5b1ea4274c3351042bad4c6e4bf57f88db1 Mon Sep 17 00:00:00 2001 From: Christopher Dupuis <45972964+cmdupuis3@users.noreply.github.com> Date: Mon, 9 Jan 2023 19:42:29 -0500 Subject: [PATCH 09/11] Update recipes/PyQG_subgrid_forcing_Ross2022/meta.yaml Co-authored-by: Charles Stern <62192187+cisaacstern@users.noreply.github.com> --- recipes/PyQG_subgrid_forcing_Ross2022/meta.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/recipes/PyQG_subgrid_forcing_Ross2022/meta.yaml b/recipes/PyQG_subgrid_forcing_Ross2022/meta.yaml index 4deb84ace1..30a1164eed 100644 --- a/recipes/PyQG_subgrid_forcing_Ross2022/meta.yaml +++ b/recipes/PyQG_subgrid_forcing_Ross2022/meta.yaml @@ -1,8 +1,7 @@ title: 'PyQG Subgrid Forcing (Ross et al., 2022)' description: 'Data from "Benchmarking of machine learning ocean subgrid parameterizations in an idealized model" by Ross et al., 2022' recipes: - - id: PyQG_Ross2022 - object: 'recipe:recipe' + dict_object: 'recipe:recipes' maintainers: - name: 'Christopher Dupuis' github: cmdupuis3 From 3157bb11e93d117db15b37b5b385f8ea7423093d Mon Sep 17 00:00:00 2001 From: Christopher Dupuis <45972964+cmdupuis3@users.noreply.github.com> Date: Mon, 9 Jan 2023 19:43:00 -0500 Subject: [PATCH 10/11] PyQG: fix last line Co-authored-by: Charles Stern <62192187+cisaacstern@users.noreply.github.com> --- recipes/PyQG_subgrid_forcing_Ross2022/recipe.py | 1 - 1 file changed, 1 deletion(-) diff --git a/recipes/PyQG_subgrid_forcing_Ross2022/recipe.py b/recipes/PyQG_subgrid_forcing_Ross2022/recipe.py index 423065d60d..60e6234889 100644 --- a/recipes/PyQG_subgrid_forcing_Ross2022/recipe.py +++ b/recipes/PyQG_subgrid_forcing_Ross2022/recipe.py @@ -34,4 +34,3 @@ def xr2zarr(v): recipes = {k: xr2zarr(v) for k, v in file_dict.items()} -[recipe.to_function()() for recipe in recipes.values()] # probably not idiomatic, plz fix From ebae2053f1eb1e7d7909935574efa77b77c72ed0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 10 Jan 2023 00:44:56 +0000 Subject: [PATCH 11/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- recipes/PyQG_subgrid_forcing_Ross2022/recipe.py | 1 - 1 file changed, 1 deletion(-) diff --git a/recipes/PyQG_subgrid_forcing_Ross2022/recipe.py b/recipes/PyQG_subgrid_forcing_Ross2022/recipe.py index 60e6234889..4ae81838dc 100644 --- a/recipes/PyQG_subgrid_forcing_Ross2022/recipe.py +++ b/recipes/PyQG_subgrid_forcing_Ross2022/recipe.py @@ -33,4 +33,3 @@ def xr2zarr(v): recipes = {k: xr2zarr(v) for k, v in file_dict.items()} -