Skip to content

Commit

Permalink
Added downloader for era5 data.
Browse files Browse the repository at this point in the history
  • Loading branch information
@fstrnad
fstrnad committed Sep 4, 2023
1 parent f2bd3b3 commit 07a2387
Show file tree
Hide file tree
Showing 5 changed files with 86 additions and 68 deletions.
2 changes: 1 addition & 1 deletion geoutils/geodata/base_dataset.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ class BaseDataset():

def __init__(
self,
var_name=None,
data_nc=None,
var_name=None,
time_range=None,
lon_range=[-180, 180],
lat_range=[-90, 90],
Expand Down
106 changes: 46 additions & 60 deletions geoutils/geodata/downloader/download_era5.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
import numpy as np
from cdo import Cdo
import argparse
from importlib import reload
reload(tu)


def str2bool(v):
Expand All @@ -26,6 +28,7 @@ def str2bool(v):
'u10': '10m_u_component_of_wind',
'v10': '10m_v_component_of_wind',
'sst': 'sea_surface_temperature',
'ttr': 'top_net_thermal_radiation',
}


Expand All @@ -41,7 +44,11 @@ def download_era5(variable, plevels=None,
start_day=1, end_day=31,
start_month='Jan', end_month='Dec',
start_hour=0, end_hour=23,
run=True, **kwargs):
filename=None,
folder='./',
daymean=False,
run=True,
**kwargs):
"""Download ERA5 data for a given variable.
Args:
Expand Down Expand Up @@ -72,33 +79,46 @@ def download_era5(variable, plevels=None,
run = True

years = np.arange(starty, endy+1, 1)
smonths = tu.get_index_of_month(month=start_month)
emonths = tu.get_index_of_month(month=end_month)
months = np.arange(smonths, emonths+1, 1)
sdays = np.arange(start_day, end_day+1, 1)

smonths = tu.get_month_number(start_month)
emonths = tu.get_month_number(end_month)
marray = np.arange(smonths, emonths+1, 1)
months = tu.num2str_list(marray)
print(months)
days = np.arange(start_day, end_day+1, 1)
sdays = tu.num2str_list(days)
times = [
'00:00', '01:00', '02:00',
'03:00', '04:00', '05:00',
'06:00', '07:00', '08:00',
'09:00', '10:00', '11:00',
'12:00', '13:00', '14:00',
'15:00', '16:00', '17:00',
'18:00', '19:00', '20:00',
'21:00', '22:00', '23:00',
]
cdo = Cdo() # Parameters

for plevel in plevels:
inFiles = []
for year in years:
gut.myprint(
f"Year {year}, Pressure Level {plevel}, Variable {variable}")

if spl is None:
folder = f'./multi_pressure_level/{variable}/{plevel}/'
fname_daymean = folder + \
f'{variable}_{year}_{plevel}_daymean.nc'
filename = folder + f'{variable}_{year}_{plevel}.nc'
else:
folder = f'./single_pressure_level/{variable}/'
fname_daymean = folder + f'{variable}_{year}_daymean.nc'
filename = folder + f'{variable}_{year}.nc'
if filename is None:
if not spl:
folder += f'/multi_pressure_level/{variable}/{plevel}/'
fname_daymean = f'{variable}_{year}_{plevel}_daymean.nc'
filename = f'{variable}_{year}_{plevel}.nc'
else:
folder += f'/single_pressure_level/{variable}/'
fname_daymean = f'{variable}_{year}_daymean.nc'
filename = f'{variable}_{year}.nc'

if not os.path.exists(folder):
gut.myprint(f"Make Dir: {folder}")
os.makedirs(folder)

filename = folder + filename
fname_daymean = folder + fname_daymean
if os.path.exists(filename):
gut.myprint(f"File {filename} already exists!")
else:
Expand All @@ -117,16 +137,7 @@ def download_era5(variable, plevels=None,
'pressure_level': plevel,
'month': months,
'day': sdays,
'time': [
'00:00', '01:00', '02:00',
'03:00', '04:00', '05:00',
'06:00', '07:00', '08:00',
'09:00', '10:00', '11:00',
'12:00', '13:00', '14:00',
'15:00', '16:00', '17:00',
'18:00', '19:00', '20:00',
'21:00', '22:00', '23:00',
],
'time': times,
},
filename)
else:
Expand All @@ -137,35 +148,9 @@ def download_era5(variable, plevels=None,
'format': 'netcdf',
'variable': [variable],
'year': [str(year)],
'month': [
'01', '02', '03',
'04', '05', '06',
'07', '08', '09',
'10', '11', '12',
],
'day': [
'01', '02', '03',
'04', '05', '06',
'07', '08', '09',
'10', '11', '12',
'13', '14', '15',
'16', '17', '18',
'19', '20', '21',
'22', '23', '24',
'25', '26', '27',
'28', '29', '30',
'31',
],
'time': [
'00:00', '01:00', '02:00',
'03:00', '04:00', '05:00',
'06:00', '07:00', '08:00',
'09:00', '10:00', '11:00',
'12:00', '13:00', '14:00',
'15:00', '16:00', '17:00',
'18:00', '19:00', '20:00',
'21:00', '22:00', '23:00',
],
'month': months,
'day': sdays,
'time': times,
},
filename)
del c
Expand All @@ -177,8 +162,9 @@ def download_era5(variable, plevels=None,

inFiles.append(fname_daymean)
if run:
fname_daymean_yrange = folder + \
f'{variable}_{plevel}_{starty}_{endy}.nc'
cdo.mergetime(options='-b F32 -f nc',
input=inFiles,
output=fname_daymean_yrange)
if daymean:
fname_daymean_yrange = folder + \
f'{variable}_{plevel}_{starty}_{endy}.nc'
cdo.mergetime(options='-b F32 -f nc',
input=inFiles,
output=fname_daymean_yrange)
13 changes: 10 additions & 3 deletions geoutils/geodata/downloader/downloader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,13 @@
# %%
reload(d5)

d5.download_era5(variable='t2m',
starty=2023, endy=2023,
start_month='Jun', end_month='Sep')
d5.download_era5(variable='pr',
starty=2023,
endy=2023,
start_month='Jun',
end_month='Jun',
folder='/home/strnad/data/era5/', # full path
run=True,
)

# %%
10 changes: 8 additions & 2 deletions geoutils/utils/spatial_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -844,7 +844,8 @@ def compute_correlation(data_array, ts,
(data_array.lon.size, data_array.lat.size, len(lag_arr))),
dims=("lon", "lat", "lag"),
name='p')
gut.myprint(f"Computing {correlation_type} correlation for {len(data_array.lon)} locations ...")
gut.myprint(
f"Computing {correlation_type} correlation for {len(data_array.lon)} locations ...")
for i, lon in enumerate(tqdm(data_array.lon)):
for j, lat in enumerate(data_array.lat):
for l, lag in enumerate(lag_arr):
Expand Down Expand Up @@ -1042,7 +1043,12 @@ def remove_useless_variables(ds):
vars = gut.get_vars(ds=ds)
for var in vars:
this_dims = gut.get_dims(ds[var])
if(
if 'expver' in this_dims:
gut.myprint(f'Remove expver from {var}!')
ds = ds.sel(expver=1).combine_first(ds.sel(expver=5))
ds.load()
gut.myprint(f'Combined expver 1 and 5 for {var}!')
if (
(gut.compare_lists(this_dims, ['lat', 'lon', 'time'])) or
(gut.compare_lists(this_dims, ['lat', 'lon'])) or
(gut.compare_lists(this_dims, ['lat', 'lon', 'time', 'plevel'])) or
Expand Down
23 changes: 21 additions & 2 deletions geoutils/utils/time_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,15 +157,34 @@ def get_month_number(*month_list):
return idx_lst


def num2str_list(num_list):
str_list = []
for num in num_list:
str_list.append(num2str(num))
return str_list


def num2str(mi):
if isinstance(mi, int) or isinstance(mi, np.int64):
mstr = f'{mi}' if mi > 9 else f'0{mi}'
else:
print(type(mi))
raise ValueError(
f'Month number should be integer but is of type {type(mi)}!')

return mstr


def month2str(month):
mi = get_month_number(month)
mstr = f'{mi}' if mi > 9 else f'0{mi}'
mstr = num2str(mi)
return mstr


def get_month_name(month_number):
if not isinstance(month_number, int):
raise ValueError(f'Month should be integer but is {type(int)}!')
raise ValueError(
f'Month should be integer but is {type(month_number)}!')
if month_number < 1 or month_number > 12:
raise ValueError(
f'Month should be in range 1-12 but is {month_number}!')
Expand Down

0 comments on commit 07a2387

Please sign in to comment.